mirror of
https://github.com/AmruthPillai/Reactive-Resume.git
synced 2026-10-02 09:44:26 +10:00
188 lines
8.3 KiB
TypeScript
188 lines
8.3 KiB
TypeScript
// @vitest-environment happy-dom
|
|
|
|
import type { SectionTitleResolver } from "@reactive-resume/pdf/section-title";
|
|
import type { ExportVariant } from "./fixture";
|
|
import type { ExportMetrics } from "./metrics";
|
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
import { join, resolve } from "node:path";
|
|
import { describe, expect, it } from "vitest";
|
|
import { buildDocx } from "@reactive-resume/docx";
|
|
import { createResumePdfFile } from "@reactive-resume/pdf/server";
|
|
import { extractDocx, extractPdf } from "./extract";
|
|
import { createSyntheticCorpus } from "./fixture";
|
|
import { evaluateExport, tokenize } from "./metrics";
|
|
|
|
const outputDirectory = resolve(process.cwd(), "ats-export-evaluation/test-results");
|
|
|
|
const pdfTitleResolver: SectionTitleResolver = ({ defaultEnglishTitle, sectionId }) => defaultEnglishTitle ?? sectionId;
|
|
const docxTitleResolver = (sectionId: string) => sectionId;
|
|
|
|
type FormatResult = {
|
|
format: "pdf" | "docx";
|
|
metrics: ExportMetrics;
|
|
links: readonly string[];
|
|
paragraphs: number;
|
|
pageCount?: number;
|
|
fontCount?: number;
|
|
numberingDefinitions?: number;
|
|
numberedParagraphs?: number;
|
|
hiddenLeaks: readonly string[];
|
|
};
|
|
|
|
type VariantResult = {
|
|
variant: ExportVariant;
|
|
formats: FormatResult[];
|
|
};
|
|
|
|
function hiddenLeaks(
|
|
paragraphs: readonly string[],
|
|
hiddenTokens: readonly string[],
|
|
links: readonly string[] = [],
|
|
): string[] {
|
|
const observed = [...paragraphs, ...links].flatMap(tokenize);
|
|
return hiddenTokens.filter((value) => {
|
|
const expected = tokenize(value);
|
|
return observed.some((_, index) => expected.every((token, offset) => observed[index + offset] === token));
|
|
});
|
|
}
|
|
|
|
async function measureVariant(variant: ExportVariant): Promise<VariantResult> {
|
|
const corpus = createSyntheticCorpus(variant);
|
|
const before = JSON.stringify(corpus.data);
|
|
const pdfFile = await createResumePdfFile({
|
|
data: corpus.data,
|
|
filename: `${corpus.name}.pdf`,
|
|
template: corpus.data.metadata.template,
|
|
resolveSectionTitle: pdfTitleResolver,
|
|
});
|
|
const pdfBytes = new Uint8Array(await pdfFile.arrayBuffer());
|
|
const pdf = await extractPdf(pdfBytes);
|
|
await writeFile(join(outputDirectory, `${corpus.name}.pdf`), pdfBytes);
|
|
|
|
const docxBlob = await buildDocx(corpus.data, docxTitleResolver);
|
|
const docxBytes = new Uint8Array(await docxBlob.arrayBuffer());
|
|
const docx = await extractDocx(docxBytes);
|
|
await writeFile(join(outputDirectory, `${corpus.name}.docx`), docxBytes);
|
|
|
|
expect(JSON.stringify(corpus.data)).toBe(before);
|
|
|
|
return {
|
|
variant,
|
|
formats: [
|
|
{
|
|
format: "pdf",
|
|
metrics: evaluateExport(corpus, { paragraphs: pdf.paragraphs, links: pdf.links }),
|
|
links: pdf.links,
|
|
paragraphs: pdf.paragraphs.length,
|
|
pageCount: pdf.raw.pageCount,
|
|
fontCount: pdf.raw.fonts.length,
|
|
hiddenLeaks: hiddenLeaks(pdf.paragraphs, corpus.hiddenTokens, pdf.links),
|
|
},
|
|
{
|
|
format: "docx",
|
|
metrics: evaluateExport(corpus, {
|
|
paragraphs: docx.paragraphs.map((paragraph) => paragraph.text),
|
|
links: docx.links,
|
|
}),
|
|
links: docx.links,
|
|
paragraphs: docx.paragraphs.length,
|
|
numberingDefinitions: docx.numberingDefinitions,
|
|
numberedParagraphs: docx.numberedParagraphs,
|
|
hiddenLeaks: hiddenLeaks(
|
|
docx.paragraphs.map((paragraph) => paragraph.text),
|
|
corpus.hiddenTokens,
|
|
docx.links,
|
|
),
|
|
},
|
|
],
|
|
};
|
|
}
|
|
|
|
function percentage(value: number): string {
|
|
return `${(value * 100).toFixed(1)}%`;
|
|
}
|
|
|
|
function reportMarkdown(results: readonly VariantResult[]): string {
|
|
const lines = [
|
|
"# ATS export evaluation",
|
|
"",
|
|
"Synthetic extraction measurements only. These are not vendor parsing accuracy claims.",
|
|
"",
|
|
"Token rules: NFC normalization, case-folding with `en-US`, and Unicode letter/number runs; punctuation separates tokens. Recall numerator is distinct expected tokens recovered; duplicate counts report matching occurrences separately. Order uses adjacent distinct expected-token pairs; grouping uses same-group expected occurrence pairs and recovered in the same extracted paragraph/line. Link targets are normalized before expected/observed set comparison.",
|
|
"",
|
|
"Corpus: one deterministic resume fixture per layout variant, covering header/contact, two roles, free-text dates, education, skills, project, custom section, long lines, links, hidden item, and CJK text. Hidden item tokens are intentionally excluded from expected recall and checked for leakage.",
|
|
"",
|
|
"| Variant | Format | Recall raw | Order raw | Duplicate raw | Grouping raw | Pages/paragraphs | Links | Numbering defs/paragraphs | Hidden leaks |",
|
|
"| --- | --- | --- | --- | --- | --- | ---: | ---: | ---: | --- |",
|
|
];
|
|
for (const result of results) {
|
|
for (const format of result.formats) {
|
|
const m = format.metrics;
|
|
lines.push(
|
|
`| ${result.variant} | ${format.format} | ${m.recall.numerator}/${m.recall.denominator} (${percentage(m.recall.value)}) | ${m.order.numerator}/${m.order.denominator} (${percentage(m.order.value)}) | expected ${m.duplicates.expected}, observed ${m.duplicates.observed}, extra ${m.duplicates.extra} | ${m.grouping.numerator}/${m.grouping.denominator} (${percentage(m.grouping.value)}) | ${format.pageCount ?? "—"}/${format.paragraphs} | expected ${m.links.expected.length}, observed ${m.links.observed.length}, dropped ${m.links.missing.length}, changed/extra ${m.links.unexpected.length} | ${format.numberingDefinitions ?? "—"}/${format.numberedParagraphs ?? "—"} | ${format.hiddenLeaks.length === 0 ? "none" : format.hiddenLeaks.join(", ")} |`,
|
|
);
|
|
}
|
|
}
|
|
lines.push(
|
|
"",
|
|
"PDF font objects and raw link targets are recorded in adjacent JSON output.",
|
|
"",
|
|
"## Concrete loss/order evidence",
|
|
"",
|
|
);
|
|
for (const result of results) {
|
|
for (const format of result.formats) {
|
|
const m = format.metrics;
|
|
lines.push(
|
|
`- ${result.variant} ${format.format}: missing tokens = ${m.missingTokens.length === 0 ? "none" : m.missingTokens.join(", ")}; out-of-order adjacent pairs = ${m.outOfOrderPairs.length === 0 ? "none" : m.outOfOrderPairs.map(([left, right]) => `${left} → ${right}`).join(", ")}; dropped links = ${m.links.missing.length === 0 ? "none" : m.links.missing.join(", ")}; changed/extra links = ${m.links.unexpected.length === 0 ? "none" : m.links.unexpected.join(", ")}.`,
|
|
);
|
|
}
|
|
}
|
|
lines.push("");
|
|
return lines.join("\n");
|
|
}
|
|
|
|
describe("current unchanged PDF and DOCX exports", () => {
|
|
it("detects hidden content emitted through link targets", () => {
|
|
expect(hiddenLeaks(["Visible"], ["https://hidden.example"], ["https://hidden.example"])).toEqual([
|
|
"https://hidden.example",
|
|
]);
|
|
});
|
|
|
|
it("measures two-column and full-width synthetic corpus without mutating input", { timeout: 120_000 }, async () => {
|
|
await mkdir(outputDirectory, { recursive: true });
|
|
const results = [await measureVariant("two-column"), await measureVariant("full-width")];
|
|
const report = {
|
|
claimBoundary: "Local extraction metrics only; no vendor parsing accuracy claim.",
|
|
tokenRules: "NFC, en-US case-folding, Unicode letter/number runs; punctuation separates tokens.",
|
|
corpus: {
|
|
variants: results.map((result) => result.variant),
|
|
expectedDistinctTokens: createSyntheticCorpus("full-width")
|
|
.tokens.flatMap((entry) => tokenize(entry.value))
|
|
.filter((value, index, values) => values.indexOf(value) === index).length,
|
|
},
|
|
results,
|
|
};
|
|
await writeFile(join(outputDirectory, "ats-export-report.json"), `${JSON.stringify(report, null, 2)}\n`);
|
|
await writeFile(join(outputDirectory, "ats-export-report.md"), reportMarkdown(results));
|
|
|
|
for (const result of results) {
|
|
const pdf = result.formats.find((format) => format.format === "pdf");
|
|
const docx = result.formats.find((format) => format.format === "docx");
|
|
if (!pdf || !docx) throw new Error(`Missing measured format for ${result.variant}`);
|
|
expect(pdf.metrics.recall.denominator).toBeGreaterThan(20);
|
|
expect(docx.metrics.recall.denominator).toBe(pdf.metrics.recall.denominator);
|
|
expect(pdf.metrics.links.missing).toEqual([]);
|
|
expect(pdf.metrics.links.unexpected).toEqual([]);
|
|
// DOCX currently emits email but not telephone hyperlinks; retain this known loss in the measurement.
|
|
expect(docx.metrics.links.missing).toEqual(["tel:+49 30 555 0142"]);
|
|
expect(docx.metrics.links.unexpected).toEqual([]);
|
|
expect(pdf.hiddenLeaks).toEqual([]);
|
|
expect(docx.hiddenLeaks).toEqual([]);
|
|
expect(pdf.fontCount).toBeGreaterThan(0);
|
|
expect(docx.numberingDefinitions).toBeGreaterThan(0);
|
|
expect(docx.numberedParagraphs).toBe(0);
|
|
}
|
|
});
|
|
});
|