fix: surface llm workflow errors

2026-06-21 23:43:03 +08:00
parent 4bb5ca5eb0
commit f1dac15400
12 changed files with 286 additions and 500 deletions
@@ -39,6 +39,21 @@ const validFactCard = {
  confirmed_by_user: true,
 };

+const validCandidateFactCard = {
+  company_full_name: validFactCard.company_full_name,
+  company_short_names: validFactCard.company_short_names,
+  brand_names: validFactCard.brand_names,
+  product_names: validFactCard.product_names,
+  target_industry: validFactCard.target_industry,
+  target_audience: validFactCard.target_audience,
+  experience_years: validFactCard.experience_years,
+  core_claims: validFactCard.core_claims,
+  forbidden_claims: validFactCard.forbidden_claims,
+  image_topics: validFactCard.image_topics,
+  uncertain_items: validFactCard.uncertain_items,
+  is_ready_for_optimization: true,
+};
+
 interface CreateJobResponse {
  job: { id: string };
  candidateFactCard: { company_full_name: string };
@@ -86,6 +101,8 @@ describe("job API routes", () => {
  });

  it("validates input, creates a job, and returns a candidate fact card", async () => {
+    llmMocks.generateValidatedJson.mockResolvedValueOnce(validCandidateFactCard);
+
    const response = await createJob(
      request({
        title: "Example Technology Co., Ltd. GEO guide",
@@ -104,6 +121,38 @@ describe("job API routes", () => {
    );
  });

+  it("returns a clear error when LLM fact extraction fails", async () => {
+    llmMocks.generateValidatedJson.mockRejectedValueOnce(
+      new Error("LLM response failed schema validation: target_audience"),
+    );
+
+    const response = await createJob(
+      request({
+        title: "Example Technology Co., Ltd. GEO guide",
+        body: "Example Technology Co., Ltd. has 8 years of GEO optimization experience.",
+        platform: "official_site",
+      }),
+    );
+    const body = (await response.json()) as { error: string };
+
+    expect(response.status).toBe(502);
+    expect(body.error).toBe(
+      "LLM response failed schema validation: target_audience",
+    );
+  });
+
+  it("still returns 400 for invalid article input", async () => {
+    const response = await createJob(
+      request({
+        title: "",
+        body: "",
+        platform: "official_site",
+      }),
+    );
+
+    expect(response.status).toBe(400);
+  });
+
  it("rejects unresolved uncertain items when confirming a fact card", async () => {
    const { job } = await createJobFixture();
    const response = await confirmFactCard(
@@ -135,6 +184,19 @@ describe("job API routes", () => {
      params<{ jobId: string }>({ jobId: job.id }),
    );

+    llmMocks.generateValidatedJson
+      .mockResolvedValueOnce({
+        title: "API LLM Optimized GEO Article",
+        summary:
+          "A official site article for Marketing teams about GEO optimization.",
+        body_markdown:
+          "Example Technology Co., Ltd. has 8 years of GEO optimization experience.",
+        image_suggestions: [],
+        changed_sections: ["title", "body"],
+        requires_user_confirmation: [],
+      })
+      .mockResolvedValueOnce({ checks: [] });
+
    const response = await optimizeJob(
      request({}),
      params<{ jobId: string }>({ jobId: job.id }),
@@ -142,7 +204,7 @@ describe("job API routes", () => {
    const body = (await response.json()) as OptimizeJobResponse;

    expect(response.status).toBe(200);
-    expect(body.optimizedArticle.title).toContain("GEO optimization");
+    expect(body.optimizedArticle.title).toBe("API LLM Optimized GEO Article");
    expect(body.qaReport.checks).toHaveLength(10);
  });

@@ -166,7 +228,7 @@ describe("job API routes", () => {
        changed_sections: ["title", "body"],
        requires_user_confirmation: [],
      })
-      .mockResolvedValue(null);
+      .mockResolvedValue({ checks: [] });

    const response = await optimizeJob(
      request({}),
@@ -179,6 +241,27 @@ describe("job API routes", () => {
    expect(llmMocks.generateValidatedJson).toHaveBeenCalled();
  });

+  it("returns a clear error when LLM optimization fails", async () => {
+    const { job } = await createJobFixture();
+    await confirmFactCard(
+      request(validFactCard),
+      params<{ jobId: string }>({ jobId: job.id }),
+    );
+
+    llmMocks.generateValidatedJson.mockRejectedValueOnce(
+      new Error("LLM response failed schema validation: body_markdown"),
+    );
+
+    const response = await optimizeJob(
+      request({}),
+      params<{ jobId: string }>({ jobId: job.id }),
+    );
+    const body = (await response.json()) as { error: string };
+
+    expect(response.status).toBe(502);
+    expect(body.error).toBe("LLM response failed schema validation: body_markdown");
+  });
+
  it("rejects unknown export filenames", async () => {
    const { job } = await createJobFixture();
    const exportDir = join(tempDir, "exports", job.id);
@@ -198,6 +281,8 @@ describe("job API routes", () => {
 });

 async function createJobFixture() {
+  llmMocks.generateValidatedJson.mockResolvedValueOnce(validCandidateFactCard);
+
  const response = await createJob(
    request({
      title: "Example Technology Co., Ltd. GEO guide",
@@ -2,6 +2,7 @@ import { NextResponse } from "next/server";

 import { requireApiAccess } from "../../../../../lib/api/auth";
 import { getRepositoryFromRuntime } from "../../../../../lib/db/repository";
+import { LlmValidationError } from "../../../../../lib/llm/client";
 import { getExportStoreFromRuntime } from "../../../../../lib/workflow/export-store";
 import { runOptimizationWorkflow } from "../../../../../lib/workflow/orchestrator";

@@ -30,41 +31,55 @@ export async function POST(request: Request, context: RouteContext) {
    );
  }

-  const result = await runOptimizationWorkflow({
-    input: {
-      title: job.source_title,
-      body: job.source_body,
-      images: job.image_inputs,
-      platform: job.publish_platform,
-      user_instructions: job.user_instructions,
-    },
-    factCard: factCardRecord,
-  });
-  const optimizedArticle = await repository.saveOptimizedArticle(jobId, result.article);
-  const qaReport = await repository.saveQaReport(
-    jobId,
-    optimizedArticle.revision ?? 1,
-    result.qaReport,
-  );
-  const exportStore = getExportStoreFromRuntime();
-  const exportPaths =
-    qaReport.overall_status === "fail"
-      ? {}
-      : await exportStore.writeJobExports({
-          jobId,
-          article: optimizedArticle,
-          qaReport,
-        });
-  await repository.updateArticleJob(jobId, {
-    status: qaReport.overall_status === "fail" ? "qa_failed" : "optimized",
-    export_paths: exportPaths,
-  });
+  try {
+    const result = await runOptimizationWorkflow({
+      input: {
+        title: job.source_title,
+        body: job.source_body,
+        images: job.image_inputs,
+        platform: job.publish_platform,
+        user_instructions: job.user_instructions,
+      },
+      factCard: factCardRecord,
+    });
+    const optimizedArticle = await repository.saveOptimizedArticle(
+      jobId,
+      result.article,
+    );
+    const qaReport = await repository.saveQaReport(
+      jobId,
+      optimizedArticle.revision ?? 1,
+      result.qaReport,
+    );
+    const exportStore = getExportStoreFromRuntime();
+    const exportPaths =
+      qaReport.overall_status === "fail"
+        ? {}
+        : await exportStore.writeJobExports({
+            jobId,
+            article: optimizedArticle,
+            qaReport,
+          });
+    await repository.updateArticleJob(jobId, {
+      status: qaReport.overall_status === "fail" ? "qa_failed" : "optimized",
+      export_paths: exportPaths,
+    });

-  return NextResponse.json({
-    optimizedArticle,
-    qaReport,
-    exportPaths,
-    rewriteRounds: result.rewrite_rounds,
-    stoppedAfterMaxRewrites: result.stopped_after_max_rewrites,
-  });
+    return NextResponse.json({
+      optimizedArticle,
+      qaReport,
+      exportPaths,
+      rewriteRounds: result.rewrite_rounds,
+      stoppedAfterMaxRewrites: result.stopped_after_max_rewrites,
+    });
+  } catch (error) {
+    const message = error instanceof Error ? error.message : "LLM optimization failed";
+    return NextResponse.json({ error: message }, { status: getErrorStatus(error) });
+  }
+}
+
+function getErrorStatus(error: unknown) {
+  if (error instanceof LlmValidationError) return 502;
+  if (error instanceof Error && /^LLM\b|provider/i.test(error.message)) return 502;
+  return 500;
 }
@@ -2,6 +2,7 @@ import { NextResponse } from "next/server";

 import { requireApiAccess } from "../../../lib/api/auth";
 import { getRepositoryFromRuntime } from "../../../lib/db/repository";
+import { LlmValidationError } from "../../../lib/llm/client";
 import { extractCandidateFactCard } from "../../../lib/workflow/fact-extractor";
 import { normalizeInput, type RawArticleInput } from "../../../lib/workflow/input-normalizer";

@@ -26,7 +27,7 @@ export async function POST(request: Request) {

    return NextResponse.json({ job, candidateFactCard }, { status: 201 });
  } catch (error) {
-    return jsonError(error, 400);
+    return jsonError(error, getErrorStatus(error));
  }
 }

@@ -34,3 +35,9 @@ function jsonError(error: unknown, status: number) {
  const message = error instanceof Error ? error.message : "Request failed";
  return NextResponse.json({ error: message }, { status });
 }
+
+function getErrorStatus(error: unknown) {
+  if (error instanceof LlmValidationError) return 502;
+  if (error instanceof Error && /^LLM\b|provider/i.test(error.message)) return 502;
+  return 400;
+}