fix: surface llm workflow errors

2026-06-21 23:43:03 +08:00
parent 4bb5ca5eb0
commit f1dac15400
12 changed files with 286 additions and 500 deletions
@@ -16,16 +16,16 @@ describe("generateValidatedJson", () => {
    vi.restoreAllMocks();
  });

-  it("returns null when no provider key is configured", async () => {
+  it("throws clearly when no provider key is configured", async () => {
    process.env.LLM_PROVIDER = "deepseek";
    delete process.env.DEEPSEEK_API_KEY;

-    const result = await client.generateValidatedJson({
-      schema: z.object({ value: z.string() }),
-      prompt: "Return JSON.",
-    });
-
-    expect(result).toBeNull();
+    await expect(
+      client.generateValidatedJson({
+        schema: z.object({ value: z.string() }),
+        prompt: "Return JSON.",
+      }),
+    ).rejects.toThrow("DEEPSEEK_API_KEY is missing");
  });

  it("returns parsed data when the model response matches the schema", async () => {
@@ -41,32 +41,32 @@ describe("generateValidatedJson", () => {
    expect(result).toEqual({ value: "from-llm" });
  });

-  it("returns null when the model response fails schema validation", async () => {
+  it("throws clearly when the model response fails schema validation", async () => {
    process.env.LLM_PROVIDER = "deepseek";
    process.env.DEEPSEEK_API_KEY = "test-key";
    client.setGenerateJsonForValidation(async () => ({ value: 42 }));

-    const result = await client.generateValidatedJson({
-      schema: z.object({ value: z.string() }),
-      prompt: "Return JSON.",
-    });
-
-    expect(result).toBeNull();
+    await expect(
+      client.generateValidatedJson({
+        schema: z.object({ value: z.string() }),
+        prompt: "Return JSON.",
+      }),
+    ).rejects.toThrow("LLM response failed schema validation");
  });

-  it("returns null when the provider call rejects", async () => {
+  it("throws clearly when the provider call rejects", async () => {
    process.env.LLM_PROVIDER = "deepseek";
    process.env.DEEPSEEK_API_KEY = "test-key";
    client.setGenerateJsonForValidation(async () => {
      throw new Error("provider down");
    });

-    const result = await client.generateValidatedJson({
-      schema: z.object({ value: z.string() }),
-      prompt: "Return JSON.",
-    });
-
-    expect(result).toBeNull();
+    await expect(
+      client.generateValidatedJson({
+        schema: z.object({ value: z.string() }),
+        prompt: "Return JSON.",
+      }),
+    ).rejects.toThrow("provider down");
  });

  it("logs validation success with the supplied task label", async () => {
@@ -93,13 +93,13 @@ describe("generateValidatedJson", () => {
    process.env.DEEPSEEK_API_KEY = "test-key";
    client.setGenerateJsonForValidation(async () => ({ value: 42 }));

-    const result = await client.generateValidatedJson({
-      schema: z.object({ value: z.string() }),
-      prompt: "Return JSON.",
-      task: "fact_extractor",
-    });
-
-    expect(result).toBeNull();
+    await expect(
+      client.generateValidatedJson({
+        schema: z.object({ value: z.string() }),
+        prompt: "Return JSON.",
+        task: "fact_extractor",
+      }),
+    ).rejects.toThrow("LLM response failed schema validation");
    expect(warnSpy).toHaveBeenCalledWith(
      expect.stringContaining("[llm:validated] task=fact_extractor ok=false"),
    );
@@ -150,16 +150,16 @@ describe("generateValidatedJson", () => {
    const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
    client.setGenerateJsonForValidation(async () => ({ value: 42 }));

-    const result = await client.generateValidatedJson({
-      schema: z.object({ value: z.string() }),
-      task: "fact_extractor",
-      prompt: "Return JSON.",
-    });
-
+    await expect(
+      client.generateValidatedJson({
+        schema: z.object({ value: z.string() }),
+        task: "fact_extractor",
+        prompt: "Return JSON.",
+      }),
+    ).rejects.toThrow("LLM response failed schema validation");
    const allLogs = [...infoSpy.mock.calls, ...warnSpy.mock.calls]
      .flat()
      .join("\n");
-    expect(result).toBeNull();
    expect(allLogs).toContain("[llm:validated] task=fact_extractor ok=false");
    expect(allLogs).toContain("zod_error=");
    expect(allLogs).not.toContain("super-secret-key");
@@ -173,13 +173,13 @@ describe("generateValidatedJson", () => {
      throw new Error("provider unavailable");
    });

-    const result = await client.generateValidatedJson({
-      schema: z.object({ value: z.string() }),
-      task: "quality_inspector",
-      prompt: "Return JSON.",
-    });
-
-    expect(result).toBeNull();
+    await expect(
+      client.generateValidatedJson({
+        schema: z.object({ value: z.string() }),
+        task: "quality_inspector",
+        prompt: "Return JSON.",
+      }),
+    ).rejects.toThrow("provider unavailable");
    expect(warnSpy).toHaveBeenCalledWith(
      expect.stringContaining("[llm:error] task=quality_inspector"),
    );
@@ -28,6 +28,16 @@ export interface LlmProviderStatus {
  reason?: string;
 }

+export class LlmValidationError extends Error {
+  constructor(
+    message: string,
+    public readonly task: LlmTaskName,
+  ) {
+    super(message);
+    this.name = "LlmValidationError";
+  }
+}
+
 function getProvider() {
  return (process.env.LLM_PROVIDER || "deepseek").toLowerCase();
 }
@@ -80,8 +90,8 @@ export function setChatCompletionForTesting(
  chatCompletionForTesting = handler;
 }

-function getTask(input: GenerateInput) {
-  return input.task?.trim() || "unknown";
+function getTask(input: GenerateInput): LlmTaskName {
+  return input.task || "unknown";
 }

 function getRawLogLimit() {
@@ -204,11 +214,14 @@ export function setGenerateJsonForValidation(
 export async function generateValidatedJson<T>({
  schema,
  ...input
-}: GenerateValidatedJsonInput<T>): Promise<T | null> {
+}: GenerateValidatedJsonInput<T>): Promise<T> {
  const task = getTask(input);
  if (!isLlmConfigured()) {
    console.info(`[llm:validated] task=${task} ok=false reason=not_configured`);
-    return null;
+    throw new LlmValidationError(
+      getLlmProviderStatus().reason ?? "LLM provider is not configured",
+      task,
+    );
  }

  const status = getLlmProviderStatus();
@@ -230,14 +243,20 @@ export async function generateValidatedJson<T>({
    console.warn(
      `[llm:validated] task=${task} ok=false zod_error=${quoteLogValue(summarizeZodError(parsed.error))}`,
    );
-    return null;
+    throw new LlmValidationError(
+      `LLM response failed schema validation: ${summarizeZodError(parsed.error)}`,
+      task,
+    );
  } catch (error) {
+    if (error instanceof LlmValidationError) {
+      throw error;
+    }
    console.info(`[llm:validated] task=${task} ok=false reason=provider_error`);
    const message = error instanceof Error ? error.message : String(error);
    console.warn(
      `[llm:error] task=${task} duration_ms=${Date.now() - startedAt} message=${quoteLogValue(message)}`,
    );
-    return null;
+    throw error instanceof Error ? error : new Error(message);
  }
 }

@@ -10,7 +10,7 @@ export const JSON_ONLY_PROMPT =
  "Return valid JSON only. Do not include markdown fences or commentary.";

 const CUSTOMER_RISK_GUIDANCE = [
-  "客户最担心的内容风险：行业漂移、公司全称/简称/品牌名不一致、图片主题与正文描述不匹配、官网文章出现第三方口吻、平台语气和文章类型不匹配、标题或正文语义不顺、虚构资质/年限/案例/能力、产品/服务/年限前后冲突。",
+  "客户最担心的内容风险：行业漂移、公司全称/简称/品牌名不一致、官网文章出现第三方口吻、平台语气和文章类型不匹配、标题或正文语义不顺、虚构资质/年限/案例/能力、产品/服务/年限前后冲突。",
  "任何客户案例、资质荣誉、经验年限、服务能力、出海/多语种/合规能力、效果承诺和排名，都必须能从原文或已确认事实卡中找到明确依据。",
 ].join(" ");

@@ -99,7 +99,7 @@ export function buildArticleOptimizerPrompt(
    "- 必须保留事实卡确认的公司全称、目标行业、目标受众和核心事实。",
    "- 必须删除或弱化 factCard.forbidden_claims 中的主张。",
    "- 不得新增客户案例、数字、资质、排名、奖项、服务能力、效果承诺。",
-    "- image_suggestions 必须基于 factCard.image_topics 或原始 images；没有图片主题时返回空数组。",
+    "- 当前版本只优化文本，不生成图片建议；image_suggestions 必须返回空数组 []。",
    "",
    "Confirmed fact card:",
    JSON.stringify(factCard, null, 2),
@@ -124,8 +124,9 @@ export function buildQualityInspectorPrompt(input: {
    formatPlatformGuidance(input.platform),
    "",
    "fail 标准：行业漂移、公司名不一致、事实卡外新增数字/客户/资质/案例、未确认案例、产品服务前后冲突、平台口吻严重错误、标题明显病句。",
-    "warn 标准：图片证据不足、句子过长、表达可读性一般、平台适配轻微不足。",
-    "target_agent 只能使用 title、body、image、fact_card 或 null。",
+    "warn 标准：句子过长、表达可读性一般、平台适配轻微不足。",
+    "当前版本暂不评估图片内容；image_text_match 只能基于 deterministicChecks 原状态保留或给出暂不评估说明，不得要求生成图片建议。",
+    "target_agent 只能使用 title、body、fact_card 或 null。",
    "",
    "Confirmed fact card:",
    JSON.stringify(input.factCard, null, 2),
@@ -155,7 +156,7 @@ export function buildTargetedRewritePrompt(input: {
    "- title_quality：生成自然中文标题，禁止英文模板词。",
    "- body_quality：拆分长句，修复病句和断裂表达。",
    "- voice_consistency / platform_fit：改成目标平台对应口吻。",
-    "- image_text_match：只补充图片建议或人工确认项，不虚构图片内容。",
+    "- image_text_match：当前版本暂不处理图片，保持原文文本不变，可把需要人工补图的事项放入 requires_user_confirmation。",
    "不得新增事实。无法修复的内容放入 requires_user_confirmation。",
    "",
    "Confirmed fact card:",