GEOAgentArticleOptimizer/tests/e2e/sample-flow.spec.ts

import { mkdirSync, writeFileSync } from "node:fs";
import { join } from "node:path";

import { test } from "@playwright/test";

import { loadArticleSamples } from "./sample-flow/sample-loader";
import { runSamplePageFlow } from "./sample-flow/page-flow";
import { writeSampleResult } from "./sample-flow/reporting";
import type {
  ArticleSample,
  FailureCategory,
  SampleResult,
} from "./sample-flow/types";

const repoRoot = process.cwd();
const samplesDir =
  process.env.GEO_SAMPLE_DIR ?? join(repoRoot, "samples", "articles");
const reportDir =
  process.env.GEO_SAMPLE_REPORT_DIR ??
  join(repoRoot, "test-results", "geo-sample-flow", "manual");
const baseURL = process.env.GEO_SAMPLE_BASE_URL ?? "http://localhost:3000";
const apiAccessKey = process.env.API_ACCESS_KEY ?? "local-dev-key";
const timeoutMs = Number(process.env.GEO_SAMPLE_TIMEOUT_MS ?? "600000");
const filter = process.env.GEO_SAMPLE_FILTER;
const limit = Number(process.env.GEO_SAMPLE_LIMIT ?? "0");

const loaded = loadArticleSamples(samplesDir);
const selectedSamples = selectSamples(loaded.valid, filter, limit);
const provider = process.env.LLM_PROVIDER ?? "deepseek";
const hasLiveLlmCredentials = provider === "openai"
  ? Boolean(process.env.OPENAI_API_KEY)
  : Boolean(process.env.DEEPSEEK_API_KEY);

test.describe("GEO sample article live E2E flow", () => {
  test.skip(!hasLiveLlmCredentials, "需要真实 LLM 提供商密钥");
  test.beforeAll(() => {
    mkdirSync(reportDir, { recursive: true });
    writeFileSync(
      join(reportDir, "invalid-samples.json"),
      `${JSON.stringify(loaded.invalid, null, 2)}\n`,
    );
  });

  for (const sample of selectedSamples) {
    test(sample.name, async ({ page, request }) => {
      test.setTimeout(timeoutMs + 60_000);
      const startedAt = Date.now();
      const consoleLines: string[] = [];
      const pageErrors: string[] = [];

      page.on("console", (message) => {
        if (message.type() === "error") {
          consoleLines.push(`${message.type()}: ${message.text()}`);
        }
      });
      page.on("pageerror", (error) => {
        pageErrors.push(error.message);
      });

      let result: SampleResult;
      try {
        result = await runSamplePageFlow({
          page,
          request,
          sample,
          baseURL,
          apiAccessKey,
          reportDir,
          timeoutMs,
        });
      } catch (error) {
        const failureScreenshot = join(
          reportDir,
          "samples",
          sample.slug,
          "failure.png",
        );
        await page
          .screenshot({ path: failureScreenshot, fullPage: true })
          .catch(() => {});
        const failedResult = buildFailureResult({
          sample,
          error,
          startedAt,
          failureScreenshot,
          consoleLines,
          pageErrors,
        });
        writeSampleResult(reportDir, failedResult);
        throw error;
      }

      const enriched = enrichWithBrowserErrors(result, consoleLines, pageErrors);
      writeSampleResult(reportDir, enriched);
      if (enriched.status === "failed") {
        throw new Error(enriched.failure_message ?? enriched.failure_category);
      }
    });
  }
});

function selectSamples(
  samples: ArticleSample[],
  sampleFilter: string | undefined,
  sampleLimit: number,
) {
  const filtered = sampleFilter
    ? samples.filter(
        (sample) =>
          sample.fileName.includes(sampleFilter) ||
          sample.name.includes(sampleFilter) ||
          sample.slug.includes(sampleFilter),
      )
    : samples;

  return sampleLimit > 0 ? filtered.slice(0, sampleLimit) : filtered;
}

function enrichWithBrowserErrors(
  result: SampleResult,
  consoleLines: string[],
  pageErrors: string[],
): SampleResult {
  if (consoleLines.length === 0 && pageErrors.length === 0) return result;
  return {
    ...result,
    status: "failed",
    failure_category: "console_error",
    failure_message: [...consoleLines, ...pageErrors].join("\n"),
  };
}

function buildFailureResult({
  sample,
  error,
  startedAt,
  failureScreenshot,
  consoleLines,
  pageErrors,
}: {
  sample: ArticleSample;
  error: unknown;
  startedAt: number;
  failureScreenshot: string;
  consoleLines: string[];
  pageErrors: string[];
}): SampleResult {
  const message = error instanceof Error ? error.message : String(error);
  return {
    file: sample.filePath,
    name: sample.name,
    slug: sample.slug,
    status: "failed",
    duration_ms: Date.now() - startedAt,
    qa_fail_rules: [],
    qa_warn_rules: [],
    expected_hard_failures: sample.expectedHardFailures,
    expected_warnings: sample.expectedWarnings,
    exports: {
      "optimized.md": "failed",
      "optimized.docx": "failed",
      "qa_report.json": "failed",
    },
    llm_tasks: [],
    failure_category: categorizeFailure(message),
    failure_message: [...consoleLines, ...pageErrors, message]
      .filter(Boolean)
      .join("\n"),
    artifacts: {
      failure_screenshot: failureScreenshot,
    },
  };
}

function categorizeFailure(message: string): FailureCategory {
  if (/timeout/i.test(message)) return "stream_timeout";
  if (/LLM|provider|schema|quota|rate/i.test(message)) return "llm_failed";
  if (/export|optimized\.md|optimized\.docx|qa_report\.json/i.test(message)) {
    return "export_failed";
  }
  if (/console|pageerror/i.test(message)) return "console_error";
  if (/failed|优化失败/i.test(message)) return "stream_failed";
  return "page_flow_failed";
}