import { mkdirSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { test } from "@playwright/test"; import { loadArticleSamples } from "./sample-flow/sample-loader"; import { runSamplePageFlow } from "./sample-flow/page-flow"; import { writeSampleResult } from "./sample-flow/reporting"; import type { ArticleSample, FailureCategory, SampleResult, } from "./sample-flow/types"; const repoRoot = process.cwd(); const samplesDir = process.env.GEO_SAMPLE_DIR ?? join(repoRoot, "samples", "articles"); const reportDir = process.env.GEO_SAMPLE_REPORT_DIR ?? join(repoRoot, "test-results", "geo-sample-flow", "manual"); const baseURL = process.env.GEO_SAMPLE_BASE_URL ?? "http://127.0.0.1:3000"; const apiAccessKey = process.env.API_ACCESS_KEY ?? "local-dev-key"; const timeoutMs = Number(process.env.GEO_SAMPLE_TIMEOUT_MS ?? "600000"); const filter = process.env.GEO_SAMPLE_FILTER; const limit = Number(process.env.GEO_SAMPLE_LIMIT ?? "0"); const loaded = loadArticleSamples(samplesDir); const selectedSamples = selectSamples(loaded.valid, filter, limit); test.describe("GEO sample article live E2E flow", () => { test.beforeAll(() => { mkdirSync(reportDir, { recursive: true }); writeFileSync( join(reportDir, "invalid-samples.json"), `${JSON.stringify(loaded.invalid, null, 2)}\n`, ); }); for (const sample of selectedSamples) { test(sample.name, async ({ page, request }) => { test.setTimeout(timeoutMs + 60_000); const startedAt = Date.now(); const consoleLines: string[] = []; const pageErrors: string[] = []; page.on("console", (message) => { if (message.type() === "error") { consoleLines.push(`${message.type()}: ${message.text()}`); } }); page.on("pageerror", (error) => { pageErrors.push(error.message); }); let result: SampleResult; try { result = await runSamplePageFlow({ page, request, sample, baseURL, apiAccessKey, reportDir, timeoutMs, }); } catch (error) { const failureScreenshot = join( reportDir, "samples", sample.slug, "failure.png", ); await page .screenshot({ path: failureScreenshot, fullPage: true }) .catch(() => {}); const failedResult = buildFailureResult({ sample, error, startedAt, failureScreenshot, consoleLines, pageErrors, }); writeSampleResult(reportDir, failedResult); throw error; } const enriched = enrichWithBrowserErrors(result, consoleLines, pageErrors); writeSampleResult(reportDir, enriched); if (enriched.status === "failed") { throw new Error(enriched.failure_message ?? enriched.failure_category); } }); } }); function selectSamples( samples: ArticleSample[], sampleFilter: string | undefined, sampleLimit: number, ) { const filtered = sampleFilter ? samples.filter( (sample) => sample.fileName.includes(sampleFilter) || sample.name.includes(sampleFilter) || sample.slug.includes(sampleFilter), ) : samples; return sampleLimit > 0 ? filtered.slice(0, sampleLimit) : filtered; } function enrichWithBrowserErrors( result: SampleResult, consoleLines: string[], pageErrors: string[], ): SampleResult { if (consoleLines.length === 0 && pageErrors.length === 0) return result; return { ...result, status: "failed", failure_category: "console_error", failure_message: [...consoleLines, ...pageErrors].join("\n"), }; } function buildFailureResult({ sample, error, startedAt, failureScreenshot, consoleLines, pageErrors, }: { sample: ArticleSample; error: unknown; startedAt: number; failureScreenshot: string; consoleLines: string[]; pageErrors: string[]; }): SampleResult { const message = error instanceof Error ? error.message : String(error); return { file: sample.filePath, name: sample.name, slug: sample.slug, status: "failed", duration_ms: Date.now() - startedAt, qa_fail_rules: [], qa_warn_rules: [], expected_hard_failures: sample.expectedHardFailures, expected_warnings: sample.expectedWarnings, exports: { "optimized.md": "failed", "optimized.docx": "failed", "qa_report.json": "failed", }, llm_tasks: [], failure_category: categorizeFailure(message), failure_message: [...consoleLines, ...pageErrors, message] .filter(Boolean) .join("\n"), artifacts: { failure_screenshot: failureScreenshot, }, }; } function categorizeFailure(message: string): FailureCategory { if (/timeout/i.test(message)) return "stream_timeout"; if (/LLM|provider|schema|quota|rate/i.test(message)) return "llm_failed"; if (/export|optimized\.md|optimized\.docx|qa_report\.json/i.test(message)) { return "export_failed"; } if (/console|pageerror/i.test(message)) return "console_error"; if (/failed|优化失败/i.test(message)) return "stream_failed"; return "page_flow_failed"; }