#!/usr/bin/env node import { spawn } from "node:child_process"; import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync, } from "node:fs"; import { createServer } from "node:net"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; const scriptPath = resolveScriptPath(import.meta.url); const repoRoot = resolve(dirname(scriptPath), ".."); if (process.argv[1] && resolve(process.argv[1]) === scriptPath) { await main().catch((error) => { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; }); } export async function main(rawArgs = process.argv.slice(2)) { const startedAt = new Date().toISOString(); const args = parseArgs(rawArgs); const envFile = join(repoRoot, ".env.local"); const fileEnv = existsSync(envFile) ? parseEnvFile(readFileSync(envFile, "utf8")) : {}; const mergedEnv = { ...fileEnv, ...process.env }; const provider = (mergedEnv.LLM_PROVIDER || "deepseek").toLowerCase(); const model = provider === "openai" ? mergedEnv.OPENAI_MODEL || "gpt-4.1-mini" : mergedEnv.DEEPSEEK_MODEL || "deepseek-v4-pro"; const timestamp = startedAt.replace(/[:.]/g, "-"); const reportDir = resolve( repoRoot, "test-results", "geo-sample-flow", timestamp, ); preflight(mergedEnv, provider); mkdirSync(reportDir, { recursive: true }); const port = args.port ? Number(args.port) : await findAvailablePort(3000); const baseURL = args.baseURL || `http://127.0.0.1:${port}`; const appDataDir = join(reportDir, "app-data"); mkdirSync(appDataDir, { recursive: true }); const runEnv = { ...mergedEnv, APP_DATA_DIR: appDataDir, GEO_SAMPLE_BASE_URL: baseURL, GEO_SAMPLE_REPORT_DIR: reportDir, GEO_SAMPLE_TIMEOUT_MS: String(args.timeoutMs ?? 600000), GEO_SAMPLE_FILTER: args.sample ?? "", GEO_SAMPLE_LIMIT: args.limit ? String(args.limit) : "", }; let serverProcess = null; let status = 1; try { if (!args.reuseServer) { serverProcess = spawn("npm", ["run", "dev", "--", "--port", String(port)], { cwd: repoRoot, env: runEnv, stdio: ["ignore", "pipe", "pipe"], }); captureServerLogs(serverProcess, reportDir, runEnv); await waitForServer(baseURL, 120000); } status = await runPlaywright(runEnv, Boolean(args.headed)); aggregateReport({ reportDir, startedAt, provider, model, baseURL, invalidSamplesPath: join(reportDir, "invalid-samples.json"), }); console.log(`GEO sample E2E report: ${join(reportDir, "summary.md")}`); } finally { if (serverProcess) { serverProcess.kill("SIGTERM"); } } process.exitCode = status; return status; } export function parseArgs(rawArgs) { const parsed = {}; for (let index = 0; index < rawArgs.length; index += 1) { const arg = rawArgs[index]; if (arg === "--sample") parsed.sample = rawArgs[++index]; else if (arg === "--limit") parsed.limit = Number(rawArgs[++index]); else if (arg === "--timeout-ms") parsed.timeoutMs = Number(rawArgs[++index]); else if (arg === "--port") parsed.port = Number(rawArgs[++index]); else if (arg === "--base-url") parsed.baseURL = rawArgs[++index]; else if (arg === "--reuse-server") parsed.reuseServer = true; else if (arg === "--headed") parsed.headed = true; } return parsed; } export function parseEnvFile(content) { const values = {}; for (const line of content.split(/\r?\n/)) { const trimmed = line.trim(); if (!trimmed || trimmed.startsWith("#")) continue; const match = trimmed.match(/^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/); if (!match) continue; values[match[1]] = match[2].replace(/^['"]|['"]$/g, ""); } return values; } export function preflight(env, providerName) { if (env.API_AUTH_DISABLED !== "true" && !env.API_ACCESS_KEY) { throw new Error("API_ACCESS_KEY is required unless API_AUTH_DISABLED=true"); } if (providerName === "openai" && !env.OPENAI_API_KEY) { throw new Error("OPENAI_API_KEY is required for live sample E2E"); } if (providerName !== "openai" && !env.DEEPSEEK_API_KEY) { throw new Error("DEEPSEEK_API_KEY is required for live sample E2E"); } } async function findAvailablePort(startPort) { for (let port = startPort; port < startPort + 50; port += 1) { if (await canListen(port)) return port; } throw new Error(`no available port found from ${startPort}`); } function canListen(port) { return new Promise((resolveCanListen) => { const server = createServer(); server.once("error", () => resolveCanListen(false)); server.once("listening", () => { server.close(() => resolveCanListen(true)); }); server.listen(port, "127.0.0.1"); }); } function captureServerLogs(child, outputDir, env) { const logPath = join(outputDir, "server.log"); const secrets = [ env.API_ACCESS_KEY, env.DEEPSEEK_API_KEY, env.OPENAI_API_KEY, ].filter(Boolean); const append = (chunk) => { const text = redact(String(chunk), secrets); writeFileSync(logPath, text, { flag: "a" }); }; child.stdout.on("data", append); child.stderr.on("data", append); } export function redact(text, secrets) { let next = text .replace(/(x-api-key:\s*)([^\s]+)/gi, "$1[REDACTED]") .replace(/(authorization:\s*bearer\s+)([^\s]+)/gi, "$1[REDACTED]"); for (const secret of secrets) { next = next.split(secret).join("[REDACTED]"); } return next; } async function waitForServer(baseURL, timeoutMs) { const started = Date.now(); while (Date.now() - started < timeoutMs) { try { const response = await fetch(baseURL, { method: "HEAD" }); if (response.ok) return; } catch { await delay(1000); } await delay(1000); } throw new Error(`server did not become ready at ${baseURL}`); } function delay(ms) { return new Promise((resolveDelay) => setTimeout(resolveDelay, ms)); } function runPlaywright(env, headed) { return new Promise((resolveRun) => { const argsForPlaywright = [ "playwright", "test", "--config", "playwright.samples.config.ts", ]; if (headed) argsForPlaywright.push("--headed"); const child = spawn("npx", argsForPlaywright, { cwd: repoRoot, env, stdio: "inherit", }); child.on("close", (code) => resolveRun(code ?? 1)); }); } function aggregateReport({ reportDir, startedAt, provider, model, baseURL, invalidSamplesPath, }) { const invalidSamples = existsSync(invalidSamplesPath) ? JSON.parse(readFileSync(invalidSamplesPath, "utf8")) : []; const samples = readSampleResults(reportDir); const summary = { started_at: startedAt, finished_at: new Date().toISOString(), mode: "live", provider, model, base_url: baseURL, report_dir: reportDir, totals: { passed: samples.filter((sample) => sample.status === "passed").length, failed: samples.filter((sample) => sample.status === "failed").length, skipped: samples.filter((sample) => sample.status === "skipped").length + invalidSamples.length, }, samples, invalid_samples: invalidSamples, }; writeFileSync( join(reportDir, "summary.json"), `${JSON.stringify(summary, null, 2)}\n`, ); writeFileSync(join(reportDir, "summary.md"), renderSummaryMarkdown(summary)); } function readSampleResults(reportDir) { const samplesDir = join(reportDir, "samples"); if (!existsSync(samplesDir)) return []; return readdirSync(samplesDir, { withFileTypes: true }) .filter((entry) => entry.isDirectory()) .map((entry) => join(samplesDir, entry.name, "result.json")) .filter((filePath) => existsSync(filePath)) .map((filePath) => JSON.parse(readFileSync(filePath, "utf8"))) .sort((left, right) => left.file.localeCompare(right.file)); } function renderSummaryMarkdown(summary) { const rows = summary.samples.map( (sample) => `| ${sample.name} | ${sample.status} | ${sample.qa_status ?? ""} | ${Math.round(sample.duration_ms / 1000)}s | ${sample.failure_category ?? ""} |`, ); const lines = [ "# GEO 样例文章 E2E 测试报告", "", `- 模式:${summary.mode}`, `- Provider:${summary.provider}`, `- Model:${summary.model}`, `- Base URL:${summary.base_url}`, `- 报告目录:${summary.report_dir}`, `- 结果:通过 ${summary.totals.passed},失败 ${summary.totals.failed},跳过 ${summary.totals.skipped}`, "", "| 样例 | 状态 | QA | 耗时 | 失败分类 |", "| --- | --- | --- | ---: | --- |", ...rows, ]; if (summary.invalid_samples.length > 0) { lines.push("", "## 无效样例", ""); for (const sample of summary.invalid_samples) { lines.push(`- ${sample.fileName}: ${sample.reason}`); } } lines.push(""); return lines.join("\n"); } function resolveScriptPath(metaUrl) { try { return fileURLToPath(metaUrl); } catch { return resolve(metaUrl); } }