import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { expect, type APIRequestContext, type Locator, type Page, } from "@playwright/test"; import type { ArticleSample, ExportValidationResult, SampleResult, } from "./types"; const exportFileNames = [ "optimized.md", "optimized.docx", "qa_report.json", ] as const; type ExportFileName = (typeof exportFileNames)[number]; interface RunSamplePageFlowOptions { page: Page; request: APIRequestContext; sample: ArticleSample; baseURL: string; apiAccessKey: string; reportDir: string; timeoutMs: number; } interface QaReportJson { overall_status?: "pass" | "warn" | "fail"; checks?: Array<{ rule_id?: string; status?: "pass" | "warn" | "fail" }>; } interface TraceManifestJson { run?: { job_id?: string; status?: string; trace_completeness?: string }; calls?: Array<{ task?: string; status?: string; request_available?: boolean; response_available?: boolean; }>; } export async function runSamplePageFlow({ page, request, sample, baseURL, apiAccessKey, reportDir, timeoutMs, }: RunSamplePageFlowOptions): Promise { const startedAt = Date.now(); const sampleDir = join(reportDir, "samples", sample.slug); const exportsDir = join(sampleDir, "exports"); mkdirSync(exportsDir, { recursive: true }); await page.goto(baseURL); await fillIfVisible(page, "访问密钥", apiAccessKey); const bodyField = await fillFirstAvailable( page, ["文章内容", "粘贴文章", "正文"], sample.input.body, ); await fillIfVisible(page, "图片描述或图片链接", sample.input.image_lines); await page.getByLabel("目标平台").selectOption(sample.input.platform); await fillIfVisible(page, "用户要求", sample.input.user_instructions); await ensureSubmitEnabled(page, bodyField, sample.input.body); await page.getByRole("button", { name: "开始优化" }).click(); const factCardPanel = page.locator("section").filter({ hasText: "事实卡" }); await expect(factCardPanel.getByText(/可用|待复核/)).toBeVisible({ timeout: timeoutMs, }); await expect(page.getByText("优化结果")).toBeVisible({ timeout: 30_000 }); await expect(page.getByText(/优化完成|质检发现需要复核的问题/)).toBeVisible({ timeout: timeoutMs, }); await expect(page.getByText("质量报告")).toBeVisible(); for (const fileName of exportFileNames) { await expect(page.getByRole("link", { name: fileName })).toBeVisible(); } const jobId = await extractJobIdFromExportLink(page); const exportResults = await validateExports({ request, baseURL, apiAccessKey, jobId, exportsDir, }); const traceResult = await validateLlmTrace({ request, baseURL, apiAccessKey, jobId, }); const qa = readQaReport(exportsDir); const finalScreenshot = join(sampleDir, "final.png"); await page.screenshot({ path: finalScreenshot, fullPage: true }); const failedExports = exportResults.filter((result) => result.status === "failed"); const failureMessage = [ ...failedExports.map( (result) => `${result.fileName}: ${result.error ?? result.statusCode}`, ), ...(traceResult.error ? [`LLM trace: ${traceResult.error}`] : []), ].join("; "); return { file: sample.filePath, name: sample.name, slug: sample.slug, status: failedExports.length === 0 && !traceResult.error ? "passed" : "failed", duration_ms: Date.now() - startedAt, job_id: jobId, qa_status: qa.overall_status, qa_fail_rules: rulesWithStatus(qa, "fail"), qa_warn_rules: rulesWithStatus(qa, "warn"), expected_hard_failures: sample.expectedHardFailures, expected_warnings: sample.expectedWarnings, exports: Object.fromEntries( exportResults.map((result) => [result.fileName, result.status]), ), llm_tasks: traceResult.tasks, failure_category: failedExports.length > 0 ? "export_failed" : traceResult.error ? "llm_failed" : undefined, failure_message: failureMessage || undefined, artifacts: { final_screenshot: finalScreenshot, }, }; } async function validateLlmTrace({ request, baseURL, apiAccessKey, jobId, }: { request: APIRequestContext; baseURL: string; apiAccessKey: string; jobId: string; }) { const headers: Record | undefined = apiAccessKey ? { "x-api-key": apiAccessKey } : undefined; const response = await request.get( `${baseURL}/api/jobs/${jobId}/llm-trace`, { headers }, ); if (!response.ok()) { return { tasks: [], error: `metadata endpoint returned ${response.status()}` }; } const manifest = await response.json() as TraceManifestJson; const calls = manifest.calls ?? []; const tasks = calls .map((call) => call.task) .filter((task): task is string => Boolean(task)); if (manifest.run?.job_id !== jobId) { return { tasks, error: "run job_id does not match completed job" }; } if (manifest.run.status !== "completed") { return { tasks, error: `run status is ${manifest.run.status ?? "missing"}` }; } if (calls.length === 0) { return { tasks, error: "no LLM calls were recorded" }; } if (calls.some((call) => !call.status || call.request_available !== true)) { return { tasks, error: "call metadata is incomplete" }; } return { tasks, error: undefined }; } async function fillFirstAvailable(page: Page, labels: string[], value: string) { for (const label of labels) { const locator = page.getByLabel(label); if ((await locator.count()) > 0) { await locator.fill(value); return locator; } } throw new Error(`none of these labels were found: ${labels.join(", ")}`); } async function ensureSubmitEnabled( page: Page, bodyField: Locator, body: string, ) { const submitButton = page.getByRole("button", { name: "开始优化" }); for (let attempt = 0; attempt < 4; attempt += 1) { await bodyField.fill(""); await bodyField.fill(body); if (await submitButton.isEnabled()) { return; } await page.waitForTimeout(250); } await expect(bodyField).toHaveValue(body); await expect(submitButton).toBeEnabled(); } async function fillIfVisible(page: Page, label: string, value: string) { const locator = page.getByLabel(label); if ((await locator.count()) > 0) { await locator.fill(value); } } async function extractJobIdFromExportLink(page: Page) { const href = await page .getByRole("link", { name: "optimized.md" }) .getAttribute("href"); return extractJobIdFromExportHref(href ?? ""); } export function extractJobIdFromExportHref(href: string) { const match = href.match(/\/api\/jobs\/([^/]+)\/exports\//); if (!match?.[1]) { throw new Error(`could not extract job id from export link: ${href}`); } return match[1]; } async function validateExports({ request, baseURL, apiAccessKey, jobId, exportsDir, }: { request: APIRequestContext; baseURL: string; apiAccessKey: string; jobId: string; exportsDir: string; }): Promise { const results: ExportValidationResult[] = []; const headers: Record | undefined = apiAccessKey ? { "x-api-key": apiAccessKey } : undefined; for (const fileName of exportFileNames) { const response = await request.get( `${baseURL}/api/jobs/${jobId}/exports/${fileName}`, { headers }, ); const buffer = await response.body(); const outputPath = join(exportsDir, fileName); writeFileSync(outputPath, buffer); if (!response.ok()) { results.push({ fileName, status: "failed", statusCode: response.status(), error: response.statusText(), }); continue; } const error = validateExportBody(fileName, buffer); results.push({ fileName, status: error ? "failed" : "passed", statusCode: response.status(), error, }); } return results; } export function validateExportBody(fileName: ExportFileName, buffer: Buffer) { if (buffer.length === 0) return "empty export body"; if (fileName === "optimized.md") { return buffer.toString("utf8").trim().length > 0 ? undefined : "optimized.md is blank"; } if (fileName === "optimized.docx") { return buffer.subarray(0, 2).toString("utf8") === "PK" ? undefined : "optimized.docx does not look like a zip-based docx"; } try { const parsed = JSON.parse(buffer.toString("utf8")) as QaReportJson; if (!parsed.overall_status || !Array.isArray(parsed.checks)) { return "qa_report.json must contain overall_status and checks"; } return undefined; } catch { return "qa_report.json is not valid JSON"; } } function readQaReport(exportsDir: string): QaReportJson { const text = readFileSync(join(exportsDir, "qa_report.json"), "utf8"); return JSON.parse(text) as QaReportJson; } function rulesWithStatus(report: QaReportJson, status: "warn" | "fail") { return (report.checks ?? []) .filter((check) => check.status === status && check.rule_id) .map((check) => check.rule_id as string); }