新增样例文章验收运行脚本

2026-07-01 15:05:07 +08:00
parent 7f6557cd4e
commit 762380c937
3 changed files with 347 additions and 0 deletions
@@ -3,6 +3,7 @@
  "version": "0.1.0",
  "scripts": {
    "test": "vitest run",
+    "test:e2e:samples": "node scripts/run-geo-sample-e2e.mjs",
    "dev": "next dev",
    "build": "next build",
    "start": "next start",
@@ -0,0 +1,301 @@
+#!/usr/bin/env node
+import { spawn } from "node:child_process";
+import {
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  writeFileSync,
+} from "node:fs";
+import { createServer } from "node:net";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const scriptPath = resolveScriptPath(import.meta.url);
+const repoRoot = resolve(dirname(scriptPath), "..");
+
+if (process.argv[1] && resolve(process.argv[1]) === scriptPath) {
+  await main().catch((error) => {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exitCode = 1;
+  });
+}
+
+export async function main(rawArgs = process.argv.slice(2)) {
+  const startedAt = new Date().toISOString();
+  const args = parseArgs(rawArgs);
+  const envFile = join(repoRoot, ".env.local");
+  const fileEnv = existsSync(envFile)
+    ? parseEnvFile(readFileSync(envFile, "utf8"))
+    : {};
+  const mergedEnv = { ...fileEnv, ...process.env };
+  const provider = (mergedEnv.LLM_PROVIDER || "deepseek").toLowerCase();
+  const model =
+    provider === "openai"
+      ? mergedEnv.OPENAI_MODEL || "gpt-4.1-mini"
+      : mergedEnv.DEEPSEEK_MODEL || "deepseek-v4-pro";
+  const timestamp = startedAt.replace(/[:.]/g, "-");
+  const reportDir = resolve(
+    repoRoot,
+    "test-results",
+    "geo-sample-flow",
+    timestamp,
+  );
+
+  preflight(mergedEnv, provider);
+  mkdirSync(reportDir, { recursive: true });
+
+  const port = args.port ? Number(args.port) : await findAvailablePort(3000);
+  const baseURL = args.baseURL || `http://127.0.0.1:${port}`;
+  const appDataDir = join(reportDir, "app-data");
+  mkdirSync(appDataDir, { recursive: true });
+
+  const runEnv = {
+    ...mergedEnv,
+    APP_DATA_DIR: appDataDir,
+    GEO_SAMPLE_BASE_URL: baseURL,
+    GEO_SAMPLE_REPORT_DIR: reportDir,
+    GEO_SAMPLE_TIMEOUT_MS: String(args.timeoutMs ?? 600000),
+    GEO_SAMPLE_FILTER: args.sample ?? "",
+    GEO_SAMPLE_LIMIT: args.limit ? String(args.limit) : "",
+  };
+
+  let serverProcess = null;
+  let status = 1;
+  try {
+    if (!args.reuseServer) {
+      serverProcess = spawn("npm", ["run", "dev", "--", "--port", String(port)], {
+        cwd: repoRoot,
+        env: runEnv,
+        stdio: ["ignore", "pipe", "pipe"],
+      });
+      captureServerLogs(serverProcess, reportDir, runEnv);
+      await waitForServer(baseURL, 120000);
+    }
+
+    status = await runPlaywright(runEnv, Boolean(args.headed));
+    aggregateReport({
+      reportDir,
+      startedAt,
+      provider,
+      model,
+      baseURL,
+      invalidSamplesPath: join(reportDir, "invalid-samples.json"),
+    });
+    console.log(`GEO sample E2E report: ${join(reportDir, "summary.md")}`);
+  } finally {
+    if (serverProcess) {
+      serverProcess.kill("SIGTERM");
+    }
+  }
+
+  process.exitCode = status;
+  return status;
+}
+
+export function parseArgs(rawArgs) {
+  const parsed = {};
+  for (let index = 0; index < rawArgs.length; index += 1) {
+    const arg = rawArgs[index];
+    if (arg === "--sample") parsed.sample = rawArgs[++index];
+    else if (arg === "--limit") parsed.limit = Number(rawArgs[++index]);
+    else if (arg === "--timeout-ms") parsed.timeoutMs = Number(rawArgs[++index]);
+    else if (arg === "--port") parsed.port = Number(rawArgs[++index]);
+    else if (arg === "--base-url") parsed.baseURL = rawArgs[++index];
+    else if (arg === "--reuse-server") parsed.reuseServer = true;
+    else if (arg === "--headed") parsed.headed = true;
+  }
+  return parsed;
+}
+
+export function parseEnvFile(content) {
+  const values = {};
+  for (const line of content.split(/\r?\n/)) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith("#")) continue;
+    const match = trimmed.match(/^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/);
+    if (!match) continue;
+    values[match[1]] = match[2].replace(/^['"]|['"]$/g, "");
+  }
+  return values;
+}
+
+export function preflight(env, providerName) {
+  if (env.API_AUTH_DISABLED !== "true" && !env.API_ACCESS_KEY) {
+    throw new Error("API_ACCESS_KEY is required unless API_AUTH_DISABLED=true");
+  }
+  if (providerName === "openai" && !env.OPENAI_API_KEY) {
+    throw new Error("OPENAI_API_KEY is required for live sample E2E");
+  }
+  if (providerName !== "openai" && !env.DEEPSEEK_API_KEY) {
+    throw new Error("DEEPSEEK_API_KEY is required for live sample E2E");
+  }
+}
+
+async function findAvailablePort(startPort) {
+  for (let port = startPort; port < startPort + 50; port += 1) {
+    if (await canListen(port)) return port;
+  }
+  throw new Error(`no available port found from ${startPort}`);
+}
+
+function canListen(port) {
+  return new Promise((resolveCanListen) => {
+    const server = createServer();
+    server.once("error", () => resolveCanListen(false));
+    server.once("listening", () => {
+      server.close(() => resolveCanListen(true));
+    });
+    server.listen(port, "127.0.0.1");
+  });
+}
+
+function captureServerLogs(child, outputDir, env) {
+  const logPath = join(outputDir, "server.log");
+  const secrets = [
+    env.API_ACCESS_KEY,
+    env.DEEPSEEK_API_KEY,
+    env.OPENAI_API_KEY,
+  ].filter(Boolean);
+
+  const append = (chunk) => {
+    const text = redact(String(chunk), secrets);
+    writeFileSync(logPath, text, { flag: "a" });
+  };
+
+  child.stdout.on("data", append);
+  child.stderr.on("data", append);
+}
+
+export function redact(text, secrets) {
+  let next = text
+    .replace(/(x-api-key:\s*)([^\s]+)/gi, "$1[REDACTED]")
+    .replace(/(authorization:\s*bearer\s+)([^\s]+)/gi, "$1[REDACTED]");
+  for (const secret of secrets) {
+    next = next.split(secret).join("[REDACTED]");
+  }
+  return next;
+}
+
+async function waitForServer(baseURL, timeoutMs) {
+  const started = Date.now();
+  while (Date.now() - started < timeoutMs) {
+    try {
+      const response = await fetch(baseURL, { method: "HEAD" });
+      if (response.ok) return;
+    } catch {
+      await delay(1000);
+    }
+    await delay(1000);
+  }
+  throw new Error(`server did not become ready at ${baseURL}`);
+}
+
+function delay(ms) {
+  return new Promise((resolveDelay) => setTimeout(resolveDelay, ms));
+}
+
+function runPlaywright(env, headed) {
+  return new Promise((resolveRun) => {
+    const argsForPlaywright = [
+      "playwright",
+      "test",
+      "--config",
+      "playwright.samples.config.ts",
+    ];
+    if (headed) argsForPlaywright.push("--headed");
+    const child = spawn("npx", argsForPlaywright, {
+      cwd: repoRoot,
+      env,
+      stdio: "inherit",
+    });
+    child.on("close", (code) => resolveRun(code ?? 1));
+  });
+}
+
+function aggregateReport({
+  reportDir,
+  startedAt,
+  provider,
+  model,
+  baseURL,
+  invalidSamplesPath,
+}) {
+  const invalidSamples = existsSync(invalidSamplesPath)
+    ? JSON.parse(readFileSync(invalidSamplesPath, "utf8"))
+    : [];
+  const samples = readSampleResults(reportDir);
+  const summary = {
+    started_at: startedAt,
+    finished_at: new Date().toISOString(),
+    mode: "live",
+    provider,
+    model,
+    base_url: baseURL,
+    report_dir: reportDir,
+    totals: {
+      passed: samples.filter((sample) => sample.status === "passed").length,
+      failed: samples.filter((sample) => sample.status === "failed").length,
+      skipped:
+        samples.filter((sample) => sample.status === "skipped").length +
+        invalidSamples.length,
+    },
+    samples,
+    invalid_samples: invalidSamples,
+  };
+  writeFileSync(
+    join(reportDir, "summary.json"),
+    `${JSON.stringify(summary, null, 2)}\n`,
+  );
+  writeFileSync(join(reportDir, "summary.md"), renderSummaryMarkdown(summary));
+}
+
+function readSampleResults(reportDir) {
+  const samplesDir = join(reportDir, "samples");
+  if (!existsSync(samplesDir)) return [];
+  return readdirSync(samplesDir, { withFileTypes: true })
+    .filter((entry) => entry.isDirectory())
+    .map((entry) => join(samplesDir, entry.name, "result.json"))
+    .filter((filePath) => existsSync(filePath))
+    .map((filePath) => JSON.parse(readFileSync(filePath, "utf8")))
+    .sort((left, right) => left.file.localeCompare(right.file));
+}
+
+function renderSummaryMarkdown(summary) {
+  const rows = summary.samples.map(
+    (sample) =>
+      `| ${sample.name} | ${sample.status} | ${sample.qa_status ?? ""} | ${Math.round(sample.duration_ms / 1000)}s | ${sample.failure_category ?? ""} |`,
+  );
+  const lines = [
+    "# GEO 样例文章 E2E 测试报告",
+    "",
+    `- 模式：${summary.mode}`,
+    `- Provider：${summary.provider}`,
+    `- Model：${summary.model}`,
+    `- Base URL：${summary.base_url}`,
+    `- 报告目录：${summary.report_dir}`,
+    `- 结果：通过 ${summary.totals.passed}，失败 ${summary.totals.failed}，跳过 ${summary.totals.skipped}`,
+    "",
+    "| 样例 | 状态 | QA | 耗时 | 失败分类 |",
+    "| --- | --- | --- | ---: | --- |",
+    ...rows,
+  ];
+
+  if (summary.invalid_samples.length > 0) {
+    lines.push("", "## 无效样例", "");
+    for (const sample of summary.invalid_samples) {
+      lines.push(`- ${sample.fileName}: ${sample.reason}`);
+    }
+  }
+
+  lines.push("");
+  return lines.join("\n");
+}
+
+function resolveScriptPath(metaUrl) {
+  try {
+    return fileURLToPath(metaUrl);
+  } catch {
+    return resolve(metaUrl);
+  }
+}
@@ -0,0 +1,45 @@
+import { describe, expect, it } from "vitest";
+
+import {
+  parseEnvFile,
+  preflight,
+  redact,
+} from "../../scripts/run-geo-sample-e2e.mjs";
+
+describe("run-geo-sample-e2e helpers", () => {
+  it("parses simple dotenv files while ignoring comments", () => {
+    expect(
+      parseEnvFile(`
+# comment
+API_ACCESS_KEY=local-dev-key
+DEEPSEEK_API_KEY="secret"
+EMPTY=
+`),
+    ).toEqual({
+      API_ACCESS_KEY: "local-dev-key",
+      DEEPSEEK_API_KEY: "secret",
+      EMPTY: "",
+    });
+  });
+
+  it("requires a provider key in live mode", () => {
+    expect(() =>
+      preflight(
+        {
+          API_ACCESS_KEY: "local-dev-key",
+          LLM_PROVIDER: "deepseek",
+        },
+        "deepseek",
+      ),
+    ).toThrow("DEEPSEEK_API_KEY is required for live sample E2E");
+  });
+
+  it("redacts configured secrets from logs", () => {
+    expect(
+      redact("x-api-key: local-dev-key provider secret", [
+        "local-dev-key",
+        "provider secret",
+      ]),
+    ).toBe("x-api-key: [REDACTED] [REDACTED]");
+  });
+});