GEOAgentArticleOptimizer/tests/e2e/sample-flow/page-flow.ts

import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";

import {
  expect,
  type APIRequestContext,
  type Locator,
  type Page,
} from "@playwright/test";

import type {
  ArticleSample,
  ExportValidationResult,
  SampleResult,
} from "./types";

const exportFileNames = [
  "optimized.md",
  "optimized.docx",
  "qa_report.json",
] as const;

type ExportFileName = (typeof exportFileNames)[number];

interface RunSamplePageFlowOptions {
  page: Page;
  request: APIRequestContext;
  sample: ArticleSample;
  baseURL: string;
  apiAccessKey: string;
  reportDir: string;
  timeoutMs: number;
}

interface QaReportJson {
  overall_status?: "pass" | "warn" | "fail";
  checks?: Array<{ rule_id?: string; status?: "pass" | "warn" | "fail" }>;
}

interface TraceManifestJson {
  run?: { job_id?: string; status?: string; trace_completeness?: string };
  calls?: Array<{
    task?: string;
    status?: string;
    request_available?: boolean;
    response_available?: boolean;
  }>;
}

export async function runSamplePageFlow({
  page,
  request,
  sample,
  baseURL,
  apiAccessKey,
  reportDir,
  timeoutMs,
}: RunSamplePageFlowOptions): Promise<SampleResult> {
  const startedAt = Date.now();
  const sampleDir = join(reportDir, "samples", sample.slug);
  const exportsDir = join(sampleDir, "exports");
  mkdirSync(exportsDir, { recursive: true });

  await page.goto(baseURL);
  await fillIfVisible(page, "访问密钥", apiAccessKey);
  const bodyField = await fillFirstAvailable(
    page,
    ["文章内容", "粘贴文章", "正文"],
    sample.input.body,
  );
  await fillIfVisible(page, "图片描述或图片链接", sample.input.image_lines);
  await page.getByLabel("目标平台").selectOption(sample.input.platform);
  await fillIfVisible(page, "用户要求", sample.input.user_instructions);
  await ensureSubmitEnabled(page, bodyField, sample.input.body);

  await page.getByRole("button", { name: "开始优化" }).click();

  const factCardPanel = page.locator("section").filter({ hasText: "事实卡" });
  await expect(factCardPanel.getByText(/可用|待复核/)).toBeVisible({
    timeout: timeoutMs,
  });
  await expect(page.getByText("优化结果")).toBeVisible({ timeout: 30_000 });
  await expect(page.getByText(/优化完成|质检发现需要复核的问题/)).toBeVisible({
    timeout: timeoutMs,
  });
  await expect(page.getByText("质量报告")).toBeVisible();

  for (const fileName of exportFileNames) {
    await expect(page.getByRole("link", { name: fileName })).toBeVisible();
  }

  const jobId = await extractJobIdFromExportLink(page);
  const exportResults = await validateExports({
    request,
    baseURL,
    apiAccessKey,
    jobId,
    exportsDir,
  });
  const traceResult = await validateLlmTrace({
    request,
    baseURL,
    apiAccessKey,
    jobId,
  });
  const qa = readQaReport(exportsDir);
  const finalScreenshot = join(sampleDir, "final.png");
  await page.screenshot({ path: finalScreenshot, fullPage: true });

  const failedExports = exportResults.filter((result) => result.status === "failed");
  const failureMessage = [
    ...failedExports.map(
      (result) => `${result.fileName}: ${result.error ?? result.statusCode}`,
    ),
    ...(traceResult.error ? [`LLM trace: ${traceResult.error}`] : []),
  ].join("; ");

  return {
    file: sample.filePath,
    name: sample.name,
    slug: sample.slug,
    status: failedExports.length === 0 && !traceResult.error ? "passed" : "failed",
    duration_ms: Date.now() - startedAt,
    job_id: jobId,
    qa_status: qa.overall_status,
    qa_fail_rules: rulesWithStatus(qa, "fail"),
    qa_warn_rules: rulesWithStatus(qa, "warn"),
    expected_hard_failures: sample.expectedHardFailures,
    expected_warnings: sample.expectedWarnings,
    exports: Object.fromEntries(
      exportResults.map((result) => [result.fileName, result.status]),
    ),
    llm_tasks: traceResult.tasks,
    failure_category: failedExports.length > 0
      ? "export_failed"
      : traceResult.error
        ? "llm_failed"
        : undefined,
    failure_message: failureMessage || undefined,
    artifacts: {
      final_screenshot: finalScreenshot,
    },
  };
}

async function validateLlmTrace({
  request,
  baseURL,
  apiAccessKey,
  jobId,
}: {
  request: APIRequestContext;
  baseURL: string;
  apiAccessKey: string;
  jobId: string;
}) {
  const headers: Record<string, string> | undefined = apiAccessKey
    ? { "x-api-key": apiAccessKey }
    : undefined;
  const response = await request.get(
    `${baseURL}/api/jobs/${jobId}/llm-trace`,
    { headers },
  );
  if (!response.ok()) {
    return { tasks: [], error: `metadata endpoint returned ${response.status()}` };
  }

  const manifest = await response.json() as TraceManifestJson;
  const calls = manifest.calls ?? [];
  const tasks = calls
    .map((call) => call.task)
    .filter((task): task is string => Boolean(task));
  if (manifest.run?.job_id !== jobId) {
    return { tasks, error: "run job_id does not match completed job" };
  }
  if (manifest.run.status !== "completed") {
    return { tasks, error: `run status is ${manifest.run.status ?? "missing"}` };
  }
  if (calls.length === 0) {
    return { tasks, error: "no LLM calls were recorded" };
  }
  if (calls.some((call) => !call.status || call.request_available !== true)) {
    return { tasks, error: "call metadata is incomplete" };
  }
  return { tasks, error: undefined };
}

async function fillFirstAvailable(page: Page, labels: string[], value: string) {
  for (const label of labels) {
    const locator = page.getByLabel(label);
    if ((await locator.count()) > 0) {
      await locator.fill(value);
      return locator;
    }
  }

  throw new Error(`none of these labels were found: ${labels.join(", ")}`);
}

async function ensureSubmitEnabled(
  page: Page,
  bodyField: Locator,
  body: string,
) {
  const submitButton = page.getByRole("button", { name: "开始优化" });
  for (let attempt = 0; attempt < 4; attempt += 1) {
    await bodyField.fill("");
    await bodyField.fill(body);
    if (await submitButton.isEnabled()) {
      return;
    }
    await page.waitForTimeout(250);
  }
  await expect(bodyField).toHaveValue(body);
  await expect(submitButton).toBeEnabled();
}

async function fillIfVisible(page: Page, label: string, value: string) {
  const locator = page.getByLabel(label);
  if ((await locator.count()) > 0) {
    await locator.fill(value);
  }
}

async function extractJobIdFromExportLink(page: Page) {
  const href = await page
    .getByRole("link", { name: "optimized.md" })
    .getAttribute("href");
  return extractJobIdFromExportHref(href ?? "");
}

export function extractJobIdFromExportHref(href: string) {
  const match = href.match(/\/api\/jobs\/([^/]+)\/exports\//);
  if (!match?.[1]) {
    throw new Error(`could not extract job id from export link: ${href}`);
  }
  return match[1];
}

async function validateExports({
  request,
  baseURL,
  apiAccessKey,
  jobId,
  exportsDir,
}: {
  request: APIRequestContext;
  baseURL: string;
  apiAccessKey: string;
  jobId: string;
  exportsDir: string;
}): Promise<ExportValidationResult[]> {
  const results: ExportValidationResult[] = [];
  const headers: Record<string, string> | undefined = apiAccessKey
    ? { "x-api-key": apiAccessKey }
    : undefined;

  for (const fileName of exportFileNames) {
    const response = await request.get(
      `${baseURL}/api/jobs/${jobId}/exports/${fileName}`,
      { headers },
    );
    const buffer = await response.body();
    const outputPath = join(exportsDir, fileName);
    writeFileSync(outputPath, buffer);

    if (!response.ok()) {
      results.push({
        fileName,
        status: "failed",
        statusCode: response.status(),
        error: response.statusText(),
      });
      continue;
    }

    const error = validateExportBody(fileName, buffer);
    results.push({
      fileName,
      status: error ? "failed" : "passed",
      statusCode: response.status(),
      error,
    });
  }

  return results;
}

export function validateExportBody(fileName: ExportFileName, buffer: Buffer) {
  if (buffer.length === 0) return "empty export body";

  if (fileName === "optimized.md") {
    return buffer.toString("utf8").trim().length > 0
      ? undefined
      : "optimized.md is blank";
  }

  if (fileName === "optimized.docx") {
    return buffer.subarray(0, 2).toString("utf8") === "PK"
      ? undefined
      : "optimized.docx does not look like a zip-based docx";
  }

  try {
    const parsed = JSON.parse(buffer.toString("utf8")) as QaReportJson;
    if (!parsed.overall_status || !Array.isArray(parsed.checks)) {
      return "qa_report.json must contain overall_status and checks";
    }
    return undefined;
  } catch {
    return "qa_report.json is not valid JSON";
  }
}

function readQaReport(exportsDir: string): QaReportJson {
  const text = readFileSync(join(exportsDir, "qa_report.json"), "utf8");
  return JSON.parse(text) as QaReportJson;
}

function rulesWithStatus(report: QaReportJson, status: "warn" | "fail") {
  return (report.checks ?? [])
    .filter((check) => check.status === status && check.rule_id)
    .map((check) => check.rule_id as string);
}