接入后台架构标签与端到端验证

tests/e2e/sample-flow/page-flow.ts

@@ -37,6 +37,16 @@ interface QaReportJson {
   checks?: Array<{ rule_id?: string; status?: "pass" | "warn" | "fail" }>;
 }
 
+interface TraceManifestJson {
+  run?: { job_id?: string; status?: string; trace_completeness?: string };
+  calls?: Array<{
+    task?: string;
+    status?: string;
+    request_available?: boolean;
+    response_available?: boolean;
+  }>;
+}
+
 export async function runSamplePageFlow({
   page,
   request,
@@ -87,17 +97,29 @@ export async function runSamplePageFlow({
     jobId,
     exportsDir,
   });
+  const traceResult = await validateLlmTrace({
+    request,
+    baseURL,
+    apiAccessKey,
+    jobId,
+  });
   const qa = readQaReport(exportsDir);
   const finalScreenshot = join(sampleDir, "final.png");
   await page.screenshot({ path: finalScreenshot, fullPage: true });
 
   const failedExports = exportResults.filter((result) => result.status === "failed");
+  const failureMessage = [
+    ...failedExports.map(
+      (result) => `${result.fileName}: ${result.error ?? result.statusCode}`,
+    ),
+    ...(traceResult.error ? [`LLM trace: ${traceResult.error}`] : []),
+  ].join("; ");
 
   return {
     file: sample.filePath,
     name: sample.name,
     slug: sample.slug,
-    status: failedExports.length === 0 ? "passed" : "failed",
+    status: failedExports.length === 0 && !traceResult.error ? "passed" : "failed",
     duration_ms: Date.now() - startedAt,
     job_id: jobId,
     qa_status: qa.overall_status,
@@ -108,18 +130,61 @@ export async function runSamplePageFlow({
     exports: Object.fromEntries(
       exportResults.map((result) => [result.fileName, result.status]),
     ),
-    llm_tasks: [],
-    failure_category: failedExports.length > 0 ? "export_failed" : undefined,
-    failure_message:
-      failedExports
-        .map((result) => `${result.fileName}: ${result.error ?? result.statusCode}`)
-        .join("; ") || undefined,
+    llm_tasks: traceResult.tasks,
+    failure_category: failedExports.length > 0
+      ? "export_failed"
+      : traceResult.error
+        ? "llm_failed"
+        : undefined,
+    failure_message: failureMessage || undefined,
     artifacts: {
       final_screenshot: finalScreenshot,
     },
   };
 }
 
+async function validateLlmTrace({
+  request,
+  baseURL,
+  apiAccessKey,
+  jobId,
+}: {
+  request: APIRequestContext;
+  baseURL: string;
+  apiAccessKey: string;
+  jobId: string;
+}) {
+  const headers: Record<string, string> | undefined = apiAccessKey
+    ? { "x-api-key": apiAccessKey }
+    : undefined;
+  const response = await request.get(
+    `${baseURL}/api/jobs/${jobId}/llm-trace`,
+    { headers },
+  );
+  if (!response.ok()) {
+    return { tasks: [], error: `metadata endpoint returned ${response.status()}` };
+  }
+
+  const manifest = await response.json() as TraceManifestJson;
+  const calls = manifest.calls ?? [];
+  const tasks = calls
+    .map((call) => call.task)
+    .filter((task): task is string => Boolean(task));
+  if (manifest.run?.job_id !== jobId) {
+    return { tasks, error: "run job_id does not match completed job" };
+  }
+  if (manifest.run.status !== "completed") {
+    return { tasks, error: `run status is ${manifest.run.status ?? "missing"}` };
+  }
+  if (calls.length === 0) {
+    return { tasks, error: "no LLM calls were recorded" };
+  }
+  if (calls.some((call) => !call.status || call.request_available !== true)) {
+    return { tasks, error: "call metadata is incomplete" };
+  }
+  return { tasks, error: undefined };
+}
+
 async function fillFirstAvailable(page: Page, labels: string[], value: string) {
   for (const label of labels) {
     const locator = page.getByLabel(label);