diff --git a/package.json b/package.json index fd70baf..5c61700 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,7 @@ "version": "0.1.0", "scripts": { "test": "vitest run", + "test:e2e:samples": "node scripts/run-geo-sample-e2e.mjs", "dev": "next dev", "build": "next build", "start": "next start", diff --git a/scripts/run-geo-sample-e2e.mjs b/scripts/run-geo-sample-e2e.mjs new file mode 100644 index 0000000..27209e9 --- /dev/null +++ b/scripts/run-geo-sample-e2e.mjs @@ -0,0 +1,301 @@ +#!/usr/bin/env node +import { spawn } from "node:child_process"; +import { + existsSync, + mkdirSync, + readdirSync, + readFileSync, + writeFileSync, +} from "node:fs"; +import { createServer } from "node:net"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptPath = resolveScriptPath(import.meta.url); +const repoRoot = resolve(dirname(scriptPath), ".."); + +if (process.argv[1] && resolve(process.argv[1]) === scriptPath) { + await main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} + +export async function main(rawArgs = process.argv.slice(2)) { + const startedAt = new Date().toISOString(); + const args = parseArgs(rawArgs); + const envFile = join(repoRoot, ".env.local"); + const fileEnv = existsSync(envFile) + ? parseEnvFile(readFileSync(envFile, "utf8")) + : {}; + const mergedEnv = { ...fileEnv, ...process.env }; + const provider = (mergedEnv.LLM_PROVIDER || "deepseek").toLowerCase(); + const model = + provider === "openai" + ? mergedEnv.OPENAI_MODEL || "gpt-4.1-mini" + : mergedEnv.DEEPSEEK_MODEL || "deepseek-v4-pro"; + const timestamp = startedAt.replace(/[:.]/g, "-"); + const reportDir = resolve( + repoRoot, + "test-results", + "geo-sample-flow", + timestamp, + ); + + preflight(mergedEnv, provider); + mkdirSync(reportDir, { recursive: true }); + + const port = args.port ? Number(args.port) : await findAvailablePort(3000); + const baseURL = args.baseURL || `http://127.0.0.1:${port}`; + const appDataDir = join(reportDir, "app-data"); + mkdirSync(appDataDir, { recursive: true }); + + const runEnv = { + ...mergedEnv, + APP_DATA_DIR: appDataDir, + GEO_SAMPLE_BASE_URL: baseURL, + GEO_SAMPLE_REPORT_DIR: reportDir, + GEO_SAMPLE_TIMEOUT_MS: String(args.timeoutMs ?? 600000), + GEO_SAMPLE_FILTER: args.sample ?? "", + GEO_SAMPLE_LIMIT: args.limit ? String(args.limit) : "", + }; + + let serverProcess = null; + let status = 1; + try { + if (!args.reuseServer) { + serverProcess = spawn("npm", ["run", "dev", "--", "--port", String(port)], { + cwd: repoRoot, + env: runEnv, + stdio: ["ignore", "pipe", "pipe"], + }); + captureServerLogs(serverProcess, reportDir, runEnv); + await waitForServer(baseURL, 120000); + } + + status = await runPlaywright(runEnv, Boolean(args.headed)); + aggregateReport({ + reportDir, + startedAt, + provider, + model, + baseURL, + invalidSamplesPath: join(reportDir, "invalid-samples.json"), + }); + console.log(`GEO sample E2E report: ${join(reportDir, "summary.md")}`); + } finally { + if (serverProcess) { + serverProcess.kill("SIGTERM"); + } + } + + process.exitCode = status; + return status; +} + +export function parseArgs(rawArgs) { + const parsed = {}; + for (let index = 0; index < rawArgs.length; index += 1) { + const arg = rawArgs[index]; + if (arg === "--sample") parsed.sample = rawArgs[++index]; + else if (arg === "--limit") parsed.limit = Number(rawArgs[++index]); + else if (arg === "--timeout-ms") parsed.timeoutMs = Number(rawArgs[++index]); + else if (arg === "--port") parsed.port = Number(rawArgs[++index]); + else if (arg === "--base-url") parsed.baseURL = rawArgs[++index]; + else if (arg === "--reuse-server") parsed.reuseServer = true; + else if (arg === "--headed") parsed.headed = true; + } + return parsed; +} + +export function parseEnvFile(content) { + const values = {}; + for (const line of content.split(/\r?\n/)) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) continue; + const match = trimmed.match(/^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/); + if (!match) continue; + values[match[1]] = match[2].replace(/^['"]|['"]$/g, ""); + } + return values; +} + +export function preflight(env, providerName) { + if (env.API_AUTH_DISABLED !== "true" && !env.API_ACCESS_KEY) { + throw new Error("API_ACCESS_KEY is required unless API_AUTH_DISABLED=true"); + } + if (providerName === "openai" && !env.OPENAI_API_KEY) { + throw new Error("OPENAI_API_KEY is required for live sample E2E"); + } + if (providerName !== "openai" && !env.DEEPSEEK_API_KEY) { + throw new Error("DEEPSEEK_API_KEY is required for live sample E2E"); + } +} + +async function findAvailablePort(startPort) { + for (let port = startPort; port < startPort + 50; port += 1) { + if (await canListen(port)) return port; + } + throw new Error(`no available port found from ${startPort}`); +} + +function canListen(port) { + return new Promise((resolveCanListen) => { + const server = createServer(); + server.once("error", () => resolveCanListen(false)); + server.once("listening", () => { + server.close(() => resolveCanListen(true)); + }); + server.listen(port, "127.0.0.1"); + }); +} + +function captureServerLogs(child, outputDir, env) { + const logPath = join(outputDir, "server.log"); + const secrets = [ + env.API_ACCESS_KEY, + env.DEEPSEEK_API_KEY, + env.OPENAI_API_KEY, + ].filter(Boolean); + + const append = (chunk) => { + const text = redact(String(chunk), secrets); + writeFileSync(logPath, text, { flag: "a" }); + }; + + child.stdout.on("data", append); + child.stderr.on("data", append); +} + +export function redact(text, secrets) { + let next = text + .replace(/(x-api-key:\s*)([^\s]+)/gi, "$1[REDACTED]") + .replace(/(authorization:\s*bearer\s+)([^\s]+)/gi, "$1[REDACTED]"); + for (const secret of secrets) { + next = next.split(secret).join("[REDACTED]"); + } + return next; +} + +async function waitForServer(baseURL, timeoutMs) { + const started = Date.now(); + while (Date.now() - started < timeoutMs) { + try { + const response = await fetch(baseURL, { method: "HEAD" }); + if (response.ok) return; + } catch { + await delay(1000); + } + await delay(1000); + } + throw new Error(`server did not become ready at ${baseURL}`); +} + +function delay(ms) { + return new Promise((resolveDelay) => setTimeout(resolveDelay, ms)); +} + +function runPlaywright(env, headed) { + return new Promise((resolveRun) => { + const argsForPlaywright = [ + "playwright", + "test", + "--config", + "playwright.samples.config.ts", + ]; + if (headed) argsForPlaywright.push("--headed"); + const child = spawn("npx", argsForPlaywright, { + cwd: repoRoot, + env, + stdio: "inherit", + }); + child.on("close", (code) => resolveRun(code ?? 1)); + }); +} + +function aggregateReport({ + reportDir, + startedAt, + provider, + model, + baseURL, + invalidSamplesPath, +}) { + const invalidSamples = existsSync(invalidSamplesPath) + ? JSON.parse(readFileSync(invalidSamplesPath, "utf8")) + : []; + const samples = readSampleResults(reportDir); + const summary = { + started_at: startedAt, + finished_at: new Date().toISOString(), + mode: "live", + provider, + model, + base_url: baseURL, + report_dir: reportDir, + totals: { + passed: samples.filter((sample) => sample.status === "passed").length, + failed: samples.filter((sample) => sample.status === "failed").length, + skipped: + samples.filter((sample) => sample.status === "skipped").length + + invalidSamples.length, + }, + samples, + invalid_samples: invalidSamples, + }; + writeFileSync( + join(reportDir, "summary.json"), + `${JSON.stringify(summary, null, 2)}\n`, + ); + writeFileSync(join(reportDir, "summary.md"), renderSummaryMarkdown(summary)); +} + +function readSampleResults(reportDir) { + const samplesDir = join(reportDir, "samples"); + if (!existsSync(samplesDir)) return []; + return readdirSync(samplesDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => join(samplesDir, entry.name, "result.json")) + .filter((filePath) => existsSync(filePath)) + .map((filePath) => JSON.parse(readFileSync(filePath, "utf8"))) + .sort((left, right) => left.file.localeCompare(right.file)); +} + +function renderSummaryMarkdown(summary) { + const rows = summary.samples.map( + (sample) => + `| ${sample.name} | ${sample.status} | ${sample.qa_status ?? ""} | ${Math.round(sample.duration_ms / 1000)}s | ${sample.failure_category ?? ""} |`, + ); + const lines = [ + "# GEO 样例文章 E2E 测试报告", + "", + `- 模式:${summary.mode}`, + `- Provider:${summary.provider}`, + `- Model:${summary.model}`, + `- Base URL:${summary.base_url}`, + `- 报告目录:${summary.report_dir}`, + `- 结果:通过 ${summary.totals.passed},失败 ${summary.totals.failed},跳过 ${summary.totals.skipped}`, + "", + "| 样例 | 状态 | QA | 耗时 | 失败分类 |", + "| --- | --- | --- | ---: | --- |", + ...rows, + ]; + + if (summary.invalid_samples.length > 0) { + lines.push("", "## 无效样例", ""); + for (const sample of summary.invalid_samples) { + lines.push(`- ${sample.fileName}: ${sample.reason}`); + } + } + + lines.push(""); + return lines.join("\n"); +} + +function resolveScriptPath(metaUrl) { + try { + return fileURLToPath(metaUrl); + } catch { + return resolve(metaUrl); + } +} diff --git a/tests/sample-flow/runner.test.ts b/tests/sample-flow/runner.test.ts new file mode 100644 index 0000000..4947943 --- /dev/null +++ b/tests/sample-flow/runner.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "vitest"; + +import { + parseEnvFile, + preflight, + redact, +} from "../../scripts/run-geo-sample-e2e.mjs"; + +describe("run-geo-sample-e2e helpers", () => { + it("parses simple dotenv files while ignoring comments", () => { + expect( + parseEnvFile(` +# comment +API_ACCESS_KEY=local-dev-key +DEEPSEEK_API_KEY="secret" +EMPTY= +`), + ).toEqual({ + API_ACCESS_KEY: "local-dev-key", + DEEPSEEK_API_KEY: "secret", + EMPTY: "", + }); + }); + + it("requires a provider key in live mode", () => { + expect(() => + preflight( + { + API_ACCESS_KEY: "local-dev-key", + LLM_PROVIDER: "deepseek", + }, + "deepseek", + ), + ).toThrow("DEEPSEEK_API_KEY is required for live sample E2E"); + }); + + it("redacts configured secrets from logs", () => { + expect( + redact("x-api-key: local-dev-key provider secret", [ + "local-dev-key", + "provider secret", + ]), + ).toBe("x-api-key: [REDACTED] [REDACTED]"); + }); +});