302 lines
8.9 KiB
JavaScript
302 lines
8.9 KiB
JavaScript
#!/usr/bin/env node
|
||
import { spawn } from "node:child_process";
|
||
import {
|
||
existsSync,
|
||
mkdirSync,
|
||
readdirSync,
|
||
readFileSync,
|
||
writeFileSync,
|
||
} from "node:fs";
|
||
import { createServer } from "node:net";
|
||
import { dirname, join, resolve } from "node:path";
|
||
import { fileURLToPath } from "node:url";
|
||
|
||
const scriptPath = resolveScriptPath(import.meta.url);
|
||
const repoRoot = resolve(dirname(scriptPath), "..");
|
||
|
||
if (process.argv[1] && resolve(process.argv[1]) === scriptPath) {
|
||
await main().catch((error) => {
|
||
console.error(error instanceof Error ? error.message : String(error));
|
||
process.exitCode = 1;
|
||
});
|
||
}
|
||
|
||
export async function main(rawArgs = process.argv.slice(2)) {
|
||
const startedAt = new Date().toISOString();
|
||
const args = parseArgs(rawArgs);
|
||
const envFile = join(repoRoot, ".env.local");
|
||
const fileEnv = existsSync(envFile)
|
||
? parseEnvFile(readFileSync(envFile, "utf8"))
|
||
: {};
|
||
const mergedEnv = { ...fileEnv, ...process.env };
|
||
const provider = (mergedEnv.LLM_PROVIDER || "deepseek").toLowerCase();
|
||
const model =
|
||
provider === "openai"
|
||
? mergedEnv.OPENAI_MODEL || "gpt-4.1-mini"
|
||
: mergedEnv.DEEPSEEK_MODEL || "deepseek-v4-pro";
|
||
const timestamp = startedAt.replace(/[:.]/g, "-");
|
||
const reportDir = resolve(
|
||
repoRoot,
|
||
"test-results",
|
||
"geo-sample-flow",
|
||
timestamp,
|
||
);
|
||
|
||
preflight(mergedEnv, provider);
|
||
mkdirSync(reportDir, { recursive: true });
|
||
|
||
const port = args.port ? Number(args.port) : await findAvailablePort(3000);
|
||
const baseURL = args.baseURL || `http://127.0.0.1:${port}`;
|
||
const appDataDir = join(reportDir, "app-data");
|
||
mkdirSync(appDataDir, { recursive: true });
|
||
|
||
const runEnv = {
|
||
...mergedEnv,
|
||
APP_DATA_DIR: appDataDir,
|
||
GEO_SAMPLE_BASE_URL: baseURL,
|
||
GEO_SAMPLE_REPORT_DIR: reportDir,
|
||
GEO_SAMPLE_TIMEOUT_MS: String(args.timeoutMs ?? 600000),
|
||
GEO_SAMPLE_FILTER: args.sample ?? "",
|
||
GEO_SAMPLE_LIMIT: args.limit ? String(args.limit) : "",
|
||
};
|
||
|
||
let serverProcess = null;
|
||
let status = 1;
|
||
try {
|
||
if (!args.reuseServer) {
|
||
serverProcess = spawn("npm", ["run", "dev", "--", "--port", String(port)], {
|
||
cwd: repoRoot,
|
||
env: runEnv,
|
||
stdio: ["ignore", "pipe", "pipe"],
|
||
});
|
||
captureServerLogs(serverProcess, reportDir, runEnv);
|
||
await waitForServer(baseURL, 120000);
|
||
}
|
||
|
||
status = await runPlaywright(runEnv, Boolean(args.headed));
|
||
aggregateReport({
|
||
reportDir,
|
||
startedAt,
|
||
provider,
|
||
model,
|
||
baseURL,
|
||
invalidSamplesPath: join(reportDir, "invalid-samples.json"),
|
||
});
|
||
console.log(`GEO sample E2E report: ${join(reportDir, "summary.md")}`);
|
||
} finally {
|
||
if (serverProcess) {
|
||
serverProcess.kill("SIGTERM");
|
||
}
|
||
}
|
||
|
||
process.exitCode = status;
|
||
return status;
|
||
}
|
||
|
||
export function parseArgs(rawArgs) {
|
||
const parsed = {};
|
||
for (let index = 0; index < rawArgs.length; index += 1) {
|
||
const arg = rawArgs[index];
|
||
if (arg === "--sample") parsed.sample = rawArgs[++index];
|
||
else if (arg === "--limit") parsed.limit = Number(rawArgs[++index]);
|
||
else if (arg === "--timeout-ms") parsed.timeoutMs = Number(rawArgs[++index]);
|
||
else if (arg === "--port") parsed.port = Number(rawArgs[++index]);
|
||
else if (arg === "--base-url") parsed.baseURL = rawArgs[++index];
|
||
else if (arg === "--reuse-server") parsed.reuseServer = true;
|
||
else if (arg === "--headed") parsed.headed = true;
|
||
}
|
||
return parsed;
|
||
}
|
||
|
||
export function parseEnvFile(content) {
|
||
const values = {};
|
||
for (const line of content.split(/\r?\n/)) {
|
||
const trimmed = line.trim();
|
||
if (!trimmed || trimmed.startsWith("#")) continue;
|
||
const match = trimmed.match(/^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/);
|
||
if (!match) continue;
|
||
values[match[1]] = match[2].replace(/^['"]|['"]$/g, "");
|
||
}
|
||
return values;
|
||
}
|
||
|
||
export function preflight(env, providerName) {
|
||
if (env.API_AUTH_DISABLED !== "true" && !env.API_ACCESS_KEY) {
|
||
throw new Error("API_ACCESS_KEY is required unless API_AUTH_DISABLED=true");
|
||
}
|
||
if (providerName === "openai" && !env.OPENAI_API_KEY) {
|
||
throw new Error("OPENAI_API_KEY is required for live sample E2E");
|
||
}
|
||
if (providerName !== "openai" && !env.DEEPSEEK_API_KEY) {
|
||
throw new Error("DEEPSEEK_API_KEY is required for live sample E2E");
|
||
}
|
||
}
|
||
|
||
async function findAvailablePort(startPort) {
|
||
for (let port = startPort; port < startPort + 50; port += 1) {
|
||
if (await canListen(port)) return port;
|
||
}
|
||
throw new Error(`no available port found from ${startPort}`);
|
||
}
|
||
|
||
function canListen(port) {
|
||
return new Promise((resolveCanListen) => {
|
||
const server = createServer();
|
||
server.once("error", () => resolveCanListen(false));
|
||
server.once("listening", () => {
|
||
server.close(() => resolveCanListen(true));
|
||
});
|
||
server.listen(port, "127.0.0.1");
|
||
});
|
||
}
|
||
|
||
function captureServerLogs(child, outputDir, env) {
|
||
const logPath = join(outputDir, "server.log");
|
||
const secrets = [
|
||
env.API_ACCESS_KEY,
|
||
env.DEEPSEEK_API_KEY,
|
||
env.OPENAI_API_KEY,
|
||
].filter(Boolean);
|
||
|
||
const append = (chunk) => {
|
||
const text = redact(String(chunk), secrets);
|
||
writeFileSync(logPath, text, { flag: "a" });
|
||
};
|
||
|
||
child.stdout.on("data", append);
|
||
child.stderr.on("data", append);
|
||
}
|
||
|
||
export function redact(text, secrets) {
|
||
let next = text
|
||
.replace(/(x-api-key:\s*)([^\s]+)/gi, "$1[REDACTED]")
|
||
.replace(/(authorization:\s*bearer\s+)([^\s]+)/gi, "$1[REDACTED]");
|
||
for (const secret of secrets) {
|
||
next = next.split(secret).join("[REDACTED]");
|
||
}
|
||
return next;
|
||
}
|
||
|
||
async function waitForServer(baseURL, timeoutMs) {
|
||
const started = Date.now();
|
||
while (Date.now() - started < timeoutMs) {
|
||
try {
|
||
const response = await fetch(baseURL, { method: "HEAD" });
|
||
if (response.ok) return;
|
||
} catch {
|
||
await delay(1000);
|
||
}
|
||
await delay(1000);
|
||
}
|
||
throw new Error(`server did not become ready at ${baseURL}`);
|
||
}
|
||
|
||
function delay(ms) {
|
||
return new Promise((resolveDelay) => setTimeout(resolveDelay, ms));
|
||
}
|
||
|
||
function runPlaywright(env, headed) {
|
||
return new Promise((resolveRun) => {
|
||
const argsForPlaywright = [
|
||
"playwright",
|
||
"test",
|
||
"--config",
|
||
"playwright.samples.config.ts",
|
||
];
|
||
if (headed) argsForPlaywright.push("--headed");
|
||
const child = spawn("npx", argsForPlaywright, {
|
||
cwd: repoRoot,
|
||
env,
|
||
stdio: "inherit",
|
||
});
|
||
child.on("close", (code) => resolveRun(code ?? 1));
|
||
});
|
||
}
|
||
|
||
function aggregateReport({
|
||
reportDir,
|
||
startedAt,
|
||
provider,
|
||
model,
|
||
baseURL,
|
||
invalidSamplesPath,
|
||
}) {
|
||
const invalidSamples = existsSync(invalidSamplesPath)
|
||
? JSON.parse(readFileSync(invalidSamplesPath, "utf8"))
|
||
: [];
|
||
const samples = readSampleResults(reportDir);
|
||
const summary = {
|
||
started_at: startedAt,
|
||
finished_at: new Date().toISOString(),
|
||
mode: "live",
|
||
provider,
|
||
model,
|
||
base_url: baseURL,
|
||
report_dir: reportDir,
|
||
totals: {
|
||
passed: samples.filter((sample) => sample.status === "passed").length,
|
||
failed: samples.filter((sample) => sample.status === "failed").length,
|
||
skipped:
|
||
samples.filter((sample) => sample.status === "skipped").length +
|
||
invalidSamples.length,
|
||
},
|
||
samples,
|
||
invalid_samples: invalidSamples,
|
||
};
|
||
writeFileSync(
|
||
join(reportDir, "summary.json"),
|
||
`${JSON.stringify(summary, null, 2)}\n`,
|
||
);
|
||
writeFileSync(join(reportDir, "summary.md"), renderSummaryMarkdown(summary));
|
||
}
|
||
|
||
function readSampleResults(reportDir) {
|
||
const samplesDir = join(reportDir, "samples");
|
||
if (!existsSync(samplesDir)) return [];
|
||
return readdirSync(samplesDir, { withFileTypes: true })
|
||
.filter((entry) => entry.isDirectory())
|
||
.map((entry) => join(samplesDir, entry.name, "result.json"))
|
||
.filter((filePath) => existsSync(filePath))
|
||
.map((filePath) => JSON.parse(readFileSync(filePath, "utf8")))
|
||
.sort((left, right) => left.file.localeCompare(right.file));
|
||
}
|
||
|
||
function renderSummaryMarkdown(summary) {
|
||
const rows = summary.samples.map(
|
||
(sample) =>
|
||
`| ${sample.name} | ${sample.status} | ${sample.qa_status ?? ""} | ${Math.round(sample.duration_ms / 1000)}s | ${sample.failure_category ?? ""} |`,
|
||
);
|
||
const lines = [
|
||
"# GEO 样例文章 E2E 测试报告",
|
||
"",
|
||
`- 模式：${summary.mode}`,
|
||
`- Provider：${summary.provider}`,
|
||
`- Model：${summary.model}`,
|
||
`- Base URL：${summary.base_url}`,
|
||
`- 报告目录：${summary.report_dir}`,
|
||
`- 结果：通过 ${summary.totals.passed}，失败 ${summary.totals.failed}，跳过 ${summary.totals.skipped}`,
|
||
"",
|
||
"| 样例 | 状态 | QA | 耗时 | 失败分类 |",
|
||
"| --- | --- | --- | ---: | --- |",
|
||
...rows,
|
||
];
|
||
|
||
if (summary.invalid_samples.length > 0) {
|
||
lines.push("", "## 无效样例", "");
|
||
for (const sample of summary.invalid_samples) {
|
||
lines.push(`- ${sample.fileName}: ${sample.reason}`);
|
||
}
|
||
}
|
||
|
||
lines.push("");
|
||
return lines.join("\n");
|
||
}
|
||
|
||
function resolveScriptPath(metaUrl) {
|
||
try {
|
||
return fileURLToPath(metaUrl);
|
||
} catch {
|
||
return resolve(metaUrl);
|
||
}
|
||
}
|