diff --git a/playwright.samples.config.ts b/playwright.samples.config.ts new file mode 100644 index 0000000..0582a44 --- /dev/null +++ b/playwright.samples.config.ts @@ -0,0 +1,28 @@ +import { defineConfig, devices } from "@playwright/test"; + +const baseURL = process.env.GEO_SAMPLE_BASE_URL ?? "http://127.0.0.1:3000"; +const timeout = Number(process.env.GEO_SAMPLE_TIMEOUT_MS ?? "600000") + 60_000; + +export default defineConfig({ + testDir: "./tests/e2e", + testMatch: "sample-flow.spec.ts", + fullyParallel: false, + workers: 1, + timeout, + retries: 0, + reporter: [["list"]], + use: { + baseURL, + trace: "retain-on-failure", + screenshot: "only-on-failure", + video: "retain-on-failure", + actionTimeout: 30_000, + navigationTimeout: 60_000, + }, + projects: [ + { + name: "chromium", + use: { ...devices["Desktop Chrome"] }, + }, + ], +}); diff --git a/tests/e2e/sample-flow.spec.ts b/tests/e2e/sample-flow.spec.ts new file mode 100644 index 0000000..fa4a098 --- /dev/null +++ b/tests/e2e/sample-flow.spec.ts @@ -0,0 +1,179 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +import { test } from "@playwright/test"; + +import { loadArticleSamples } from "./sample-flow/sample-loader"; +import { runSamplePageFlow } from "./sample-flow/page-flow"; +import { writeSampleResult } from "./sample-flow/reporting"; +import type { + ArticleSample, + FailureCategory, + SampleResult, +} from "./sample-flow/types"; + +const repoRoot = process.cwd(); +const samplesDir = + process.env.GEO_SAMPLE_DIR ?? join(repoRoot, "samples", "articles"); +const reportDir = + process.env.GEO_SAMPLE_REPORT_DIR ?? + join(repoRoot, "test-results", "geo-sample-flow", "manual"); +const baseURL = process.env.GEO_SAMPLE_BASE_URL ?? "http://127.0.0.1:3000"; +const apiAccessKey = process.env.API_ACCESS_KEY ?? ""; +const timeoutMs = Number(process.env.GEO_SAMPLE_TIMEOUT_MS ?? "600000"); +const filter = process.env.GEO_SAMPLE_FILTER; +const limit = Number(process.env.GEO_SAMPLE_LIMIT ?? "0"); + +const loaded = loadArticleSamples(samplesDir); +const selectedSamples = selectSamples(loaded.valid, filter, limit); + +test.describe("GEO sample article live E2E flow", () => { + test.beforeAll(() => { + mkdirSync(reportDir, { recursive: true }); + writeFileSync( + join(reportDir, "invalid-samples.json"), + `${JSON.stringify(loaded.invalid, null, 2)}\n`, + ); + }); + + for (const sample of selectedSamples) { + test(sample.name, async ({ page, request }) => { + test.setTimeout(timeoutMs + 60_000); + const startedAt = Date.now(); + const consoleLines: string[] = []; + const pageErrors: string[] = []; + + page.on("console", (message) => { + if (message.type() === "error") { + consoleLines.push(`${message.type()}: ${message.text()}`); + } + }); + page.on("pageerror", (error) => { + pageErrors.push(error.message); + }); + + let result: SampleResult; + try { + result = await runSamplePageFlow({ + page, + request, + sample, + baseURL, + apiAccessKey, + reportDir, + timeoutMs, + }); + } catch (error) { + const failureScreenshot = join( + reportDir, + "samples", + sample.slug, + "failure.png", + ); + await page + .screenshot({ path: failureScreenshot, fullPage: true }) + .catch(() => {}); + const failedResult = buildFailureResult({ + sample, + error, + startedAt, + failureScreenshot, + consoleLines, + pageErrors, + }); + writeSampleResult(reportDir, failedResult); + throw error; + } + + const enriched = enrichWithBrowserErrors(result, consoleLines, pageErrors); + writeSampleResult(reportDir, enriched); + if (enriched.status === "failed") { + throw new Error(enriched.failure_message ?? enriched.failure_category); + } + }); + } +}); + +function selectSamples( + samples: ArticleSample[], + sampleFilter: string | undefined, + sampleLimit: number, +) { + const filtered = sampleFilter + ? samples.filter( + (sample) => + sample.fileName.includes(sampleFilter) || + sample.name.includes(sampleFilter) || + sample.slug.includes(sampleFilter), + ) + : samples; + + return sampleLimit > 0 ? filtered.slice(0, sampleLimit) : filtered; +} + +function enrichWithBrowserErrors( + result: SampleResult, + consoleLines: string[], + pageErrors: string[], +): SampleResult { + if (consoleLines.length === 0 && pageErrors.length === 0) return result; + return { + ...result, + status: "failed", + failure_category: "console_error", + failure_message: [...consoleLines, ...pageErrors].join("\n"), + }; +} + +function buildFailureResult({ + sample, + error, + startedAt, + failureScreenshot, + consoleLines, + pageErrors, +}: { + sample: ArticleSample; + error: unknown; + startedAt: number; + failureScreenshot: string; + consoleLines: string[]; + pageErrors: string[]; +}): SampleResult { + const message = error instanceof Error ? error.message : String(error); + return { + file: sample.filePath, + name: sample.name, + slug: sample.slug, + status: "failed", + duration_ms: Date.now() - startedAt, + qa_fail_rules: [], + qa_warn_rules: [], + expected_hard_failures: sample.expectedHardFailures, + expected_warnings: sample.expectedWarnings, + exports: { + "optimized.md": "failed", + "optimized.docx": "failed", + "qa_report.json": "failed", + }, + llm_tasks: [], + failure_category: categorizeFailure(message), + failure_message: [...consoleLines, ...pageErrors, message] + .filter(Boolean) + .join("\n"), + artifacts: { + failure_screenshot: failureScreenshot, + }, + }; +} + +function categorizeFailure(message: string): FailureCategory { + if (/timeout/i.test(message)) return "stream_timeout"; + if (/LLM|provider|schema|quota|rate/i.test(message)) return "llm_failed"; + if (/export|optimized\.md|optimized\.docx|qa_report\.json/i.test(message)) { + return "export_failed"; + } + if (/console|pageerror/i.test(message)) return "console_error"; + if (/failed|优化失败/i.test(message)) return "stream_failed"; + return "page_flow_failed"; +}