fix: surface llm workflow errors

This commit is contained in:
Codex
2026-06-21 23:43:03 +08:00
parent 4bb5ca5eb0
commit f1dac15400
12 changed files with 286 additions and 500 deletions
@@ -72,19 +72,20 @@ describe("LLM workflow integration", () => {
);
});
it("falls back to deterministic candidate extraction when LLM returns null", async () => {
llmMocks.generateValidatedJson.mockResolvedValueOnce(null);
it("surfaces candidate fact extraction LLM failures instead of falling back", async () => {
llmMocks.generateValidatedJson.mockRejectedValueOnce(
new Error("LLM response failed schema validation: target_audience"),
);
const card = await extractCandidateFactCard({
title: "Fallback Technology Co., Ltd. GEO guide",
body: "Fallback Technology Co., Ltd. has 8 years of GEO optimization experience.",
images: [{ type: "description", content: "dashboard" }],
platform: "official_site",
user_instructions: "",
});
expect(card.company_full_name).toBe("Fallback Technology Co., Ltd.");
expect(card.experience_years).toBe(8);
await expect(
extractCandidateFactCard({
title: "Fallback Technology Co., Ltd. GEO guide",
body: "Fallback Technology Co., Ltd. has 8 years of GEO optimization experience.",
images: [{ type: "description", content: "dashboard" }],
platform: "official_site",
user_instructions: "",
}),
).rejects.toThrow("LLM response failed schema validation: target_audience");
});
it("uses LLM output for article optimization when valid", async () => {
@@ -92,7 +93,7 @@ describe("LLM workflow integration", () => {
title: "LLM Optimized GEO Article",
summary: "LLM summary constrained by the fact card.",
body_markdown: "## LLM Body\nExample Technology Co., Ltd. keeps claims factual.",
image_suggestions: [{ source: "image_1", suggestion: "Use dashboard." }],
image_suggestions: [],
changed_sections: ["title", "body"],
requires_user_confirmation: [],
});
@@ -110,29 +111,31 @@ describe("LLM workflow integration", () => {
expect(article.title).toBe("LLM Optimized GEO Article");
expect(article.body_markdown).toContain("LLM Body");
expect(article.image_suggestions).toEqual([]);
expect(llmMocks.generateValidatedJson).toHaveBeenCalledOnce();
expect(llmMocks.generateValidatedJson).toHaveBeenCalledWith(
expect.objectContaining({ task: "article_optimizer" }),
);
});
it("falls back to deterministic article optimization when LLM returns null", async () => {
llmMocks.generateValidatedJson.mockResolvedValueOnce(null);
it("surfaces article optimization LLM failures instead of falling back", async () => {
llmMocks.generateValidatedJson.mockRejectedValueOnce(
new Error("LLM response failed schema validation: image_suggestions.0.source"),
);
const article = await optimizeArticle({
input: {
title: "Original",
body: "Example Technology Co., Ltd. has 8 years of GEO optimization experience.",
images: [{ type: "description", content: "dashboard" }],
platform: "official_site",
user_instructions: "Say we have 99 patents.",
},
factCard: confirmedFactCard,
});
expect(article.title).toContain("GEO optimization Guide");
expect(article.requires_user_confirmation).toContain(
"Unsupported requested claim: 99 patents",
await expect(
optimizeArticle({
input: {
title: "Original",
body: "Example Technology Co., Ltd. has 8 years of GEO optimization experience.",
images: [{ type: "description", content: "dashboard" }],
platform: "official_site",
user_instructions: "Say we have 99 patents.",
},
factCard: confirmedFactCard,
}),
).rejects.toThrow(
"LLM response failed schema validation: image_suggestions.0.source",
);
});
@@ -176,33 +179,32 @@ describe("LLM workflow integration", () => {
);
});
it("falls back to deterministic targeted rewrite when LLM returns null", async () => {
llmMocks.generateValidatedJson.mockResolvedValueOnce(null);
it("surfaces targeted rewrite LLM failures instead of falling back", async () => {
llmMocks.generateValidatedJson.mockRejectedValueOnce(new Error("provider unavailable"));
const rewritten = await rewriteFailedSections({
article: {
title: "Bad title!!!",
summary: "Original summary",
body_markdown: "## Body\nOriginal body",
image_suggestions: [],
changed_sections: [],
requires_user_confirmation: [],
},
factCard: confirmedFactCard,
failedChecks: [
{
rule_id: "title_quality",
status: "fail",
evidence: "Bad title!!!",
reason: "Title has punctuation stuffing.",
suggested_fix: "Rewrite title.",
target_agent: "title",
await expect(
rewriteFailedSections({
article: {
title: "Bad title!!!",
summary: "Original summary",
body_markdown: "## Body\nOriginal body",
image_suggestions: [],
changed_sections: [],
requires_user_confirmation: [],
},
],
});
expect(rewritten.title).toContain("GEO optimization Guide");
expect(rewritten.summary).toBe("Original summary");
factCard: confirmedFactCard,
failedChecks: [
{
rule_id: "title_quality",
status: "fail",
evidence: "Bad title!!!",
reason: "Title has punctuation stuffing.",
suggested_fix: "Rewrite title.",
target_agent: "title",
},
],
}),
).rejects.toThrow("provider unavailable");
});
it("uses LLM quality checks to enrich non-failing deterministic checks", async () => {