新增样例验收报告工具
This commit is contained in:
@@ -0,0 +1,130 @@
|
||||
import { mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
|
||||
import { join, relative } from "node:path";
|
||||
|
||||
import type { InvalidArticleSample, RunSummary, SampleResult } from "./types";
|
||||
|
||||
interface AggregateInput {
|
||||
startedAt: string;
|
||||
finishedAt: string;
|
||||
provider: string;
|
||||
model: string;
|
||||
baseURL: string;
|
||||
reportDir: string;
|
||||
samples: SampleResult[];
|
||||
invalidSamples: InvalidArticleSample[];
|
||||
}
|
||||
|
||||
export function writeSampleResult(reportDir: string, result: SampleResult) {
|
||||
const sampleDir = join(reportDir, "samples", result.slug);
|
||||
mkdirSync(sampleDir, { recursive: true });
|
||||
writeFileSync(
|
||||
join(sampleDir, "result.json"),
|
||||
`${JSON.stringify(result, null, 2)}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
export function readSampleResults(reportDir: string): SampleResult[] {
|
||||
const samplesDir = join(reportDir, "samples");
|
||||
try {
|
||||
return readdirSync(samplesDir, { withFileTypes: true })
|
||||
.filter((entry) => entry.isDirectory())
|
||||
.map((entry) => join(samplesDir, entry.name, "result.json"))
|
||||
.map(
|
||||
(filePath) => JSON.parse(readFileSync(filePath, "utf8")) as SampleResult,
|
||||
)
|
||||
.sort((left, right) => left.file.localeCompare(right.file));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export function aggregateSummary(input: AggregateInput): RunSummary {
|
||||
const passed = input.samples.filter((sample) => sample.status === "passed").length;
|
||||
const failed = input.samples.filter((sample) => sample.status === "failed").length;
|
||||
const skipped =
|
||||
input.samples.filter((sample) => sample.status === "skipped").length +
|
||||
input.invalidSamples.length;
|
||||
|
||||
return {
|
||||
started_at: input.startedAt,
|
||||
finished_at: input.finishedAt,
|
||||
mode: "live",
|
||||
provider: input.provider,
|
||||
model: input.model,
|
||||
base_url: input.baseURL,
|
||||
report_dir: input.reportDir,
|
||||
totals: { passed, failed, skipped },
|
||||
samples: input.samples,
|
||||
invalid_samples: input.invalidSamples,
|
||||
};
|
||||
}
|
||||
|
||||
export function writeSummaryFiles(reportDir: string, summary: RunSummary) {
|
||||
mkdirSync(reportDir, { recursive: true });
|
||||
writeFileSync(
|
||||
join(reportDir, "summary.json"),
|
||||
`${JSON.stringify(summary, null, 2)}\n`,
|
||||
);
|
||||
writeFileSync(join(reportDir, "summary.md"), renderSummaryMarkdown(summary));
|
||||
}
|
||||
|
||||
export function redactSensitiveText(
|
||||
text: string,
|
||||
secrets: Array<string | undefined>,
|
||||
) {
|
||||
let redacted = text
|
||||
.replace(/(x-api-key:\s*)([^\s]+)/gi, "$1[REDACTED]")
|
||||
.replace(/(authorization:\s*bearer\s+)([^\s]+)/gi, "$1[REDACTED]")
|
||||
.replace(
|
||||
/((?:DEEPSEEK|OPENAI|API)_?[A-Z_]*KEY=)"?([^"\s]+)"?/g,
|
||||
'$1"[REDACTED]"',
|
||||
);
|
||||
|
||||
for (const secret of secrets) {
|
||||
if (!secret) continue;
|
||||
redacted = redacted.split(secret).join("[REDACTED]");
|
||||
}
|
||||
|
||||
return redacted;
|
||||
}
|
||||
|
||||
function renderSummaryMarkdown(summary: RunSummary) {
|
||||
const lines = [
|
||||
"# GEO 样例文章 E2E 测试报告",
|
||||
"",
|
||||
`- 模式：${summary.mode}`,
|
||||
`- Provider：${summary.provider}`,
|
||||
`- Model：${summary.model}`,
|
||||
`- Base URL：${summary.base_url}`,
|
||||
`- 报告目录：${summary.report_dir}`,
|
||||
`- 结果：通过 ${summary.totals.passed}，失败 ${summary.totals.failed}，跳过 ${summary.totals.skipped}`,
|
||||
"",
|
||||
"| 样例 | 状态 | QA | 耗时 | 失败分类 |",
|
||||
"| --- | --- | --- | ---: | --- |",
|
||||
...summary.samples
|
||||
.map((sample) =>
|
||||
[
|
||||
sample.name,
|
||||
sample.status,
|
||||
sample.qa_status ?? "",
|
||||
`${Math.round(sample.duration_ms / 1000)}s`,
|
||||
sample.failure_category ?? "",
|
||||
].join(" | "),
|
||||
)
|
||||
.map((row) => `| ${row} |`),
|
||||
];
|
||||
|
||||
if (summary.invalid_samples.length > 0) {
|
||||
lines.push("", "## 无效样例", "");
|
||||
for (const sample of summary.invalid_samples) {
|
||||
lines.push(`- ${sample.fileName}: ${sample.reason}`);
|
||||
}
|
||||
}
|
||||
|
||||
lines.push("");
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
export function relativeArtifactPath(reportDir: string, artifactPath: string) {
|
||||
return relative(reportDir, artifactPath);
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
aggregateSummary,
|
||||
redactSensitiveText,
|
||||
writeSampleResult,
|
||||
writeSummaryFiles,
|
||||
} from "../e2e/sample-flow/reporting";
|
||||
import type { InvalidArticleSample, SampleResult } from "../e2e/sample-flow/types";
|
||||
|
||||
describe("reporting helpers", () => {
|
||||
let reportDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
reportDir = mkdtempSync(join(tmpdir(), "geo-report-"));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(reportDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("redacts API keys and authorization tokens", () => {
|
||||
const redacted = redactSensitiveText(
|
||||
'x-api-key: local-dev-key Authorization: Bearer secret DEEPSEEK_API_KEY="abc"',
|
||||
["local-dev-key", "abc"],
|
||||
);
|
||||
|
||||
expect(redacted).toBe(
|
||||
'x-api-key: [REDACTED] Authorization: Bearer [REDACTED] DEEPSEEK_API_KEY="[REDACTED]"',
|
||||
);
|
||||
});
|
||||
|
||||
it("writes per-sample results and aggregate summary files", () => {
|
||||
const sample: SampleResult = {
|
||||
file: "samples/articles/title-quality.json",
|
||||
name: "Title quality",
|
||||
slug: "title-quality",
|
||||
status: "passed",
|
||||
duration_ms: 1234,
|
||||
job_id: "job_123",
|
||||
qa_status: "warn",
|
||||
qa_fail_rules: [],
|
||||
qa_warn_rules: ["title_quality"],
|
||||
expected_hard_failures: [],
|
||||
expected_warnings: ["title_quality"],
|
||||
exports: {
|
||||
"optimized.md": "passed",
|
||||
"optimized.docx": "passed",
|
||||
"qa_report.json": "passed",
|
||||
},
|
||||
llm_tasks: ["fact_extractor", "article_optimizer", "quality_inspector"],
|
||||
artifacts: {
|
||||
final_screenshot: "samples/title-quality/final.png",
|
||||
},
|
||||
};
|
||||
const invalid: InvalidArticleSample[] = [
|
||||
{
|
||||
filePath: "/repo/samples/articles/bad.json",
|
||||
fileName: "bad.json",
|
||||
reason: "input.body must be a non-empty string",
|
||||
},
|
||||
];
|
||||
|
||||
writeSampleResult(reportDir, sample);
|
||||
const summary = aggregateSummary({
|
||||
startedAt: "2026-07-01T00:00:00.000Z",
|
||||
finishedAt: "2026-07-01T00:01:00.000Z",
|
||||
provider: "deepseek",
|
||||
model: "deepseek-v4-pro",
|
||||
baseURL: "http://127.0.0.1:3000",
|
||||
reportDir,
|
||||
samples: [sample],
|
||||
invalidSamples: invalid,
|
||||
});
|
||||
writeSummaryFiles(reportDir, summary);
|
||||
|
||||
expect(
|
||||
existsSync(join(reportDir, "samples", "title-quality", "result.json")),
|
||||
).toBe(true);
|
||||
expect(
|
||||
JSON.parse(readFileSync(join(reportDir, "summary.json"), "utf8")),
|
||||
).toEqual(
|
||||
expect.objectContaining({
|
||||
provider: "deepseek",
|
||||
totals: { passed: 1, failed: 0, skipped: 1 },
|
||||
}),
|
||||
);
|
||||
expect(readFileSync(join(reportDir, "summary.md"), "utf8")).toContain(
|
||||
"| Title quality | passed | warn |",
|
||||
);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user