fix: surface llm workflow errors
This commit is contained in:
@@ -72,19 +72,20 @@ describe("LLM workflow integration", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("falls back to deterministic candidate extraction when LLM returns null", async () => {
|
||||
llmMocks.generateValidatedJson.mockResolvedValueOnce(null);
|
||||
it("surfaces candidate fact extraction LLM failures instead of falling back", async () => {
|
||||
llmMocks.generateValidatedJson.mockRejectedValueOnce(
|
||||
new Error("LLM response failed schema validation: target_audience"),
|
||||
);
|
||||
|
||||
const card = await extractCandidateFactCard({
|
||||
title: "Fallback Technology Co., Ltd. GEO guide",
|
||||
body: "Fallback Technology Co., Ltd. has 8 years of GEO optimization experience.",
|
||||
images: [{ type: "description", content: "dashboard" }],
|
||||
platform: "official_site",
|
||||
user_instructions: "",
|
||||
});
|
||||
|
||||
expect(card.company_full_name).toBe("Fallback Technology Co., Ltd.");
|
||||
expect(card.experience_years).toBe(8);
|
||||
await expect(
|
||||
extractCandidateFactCard({
|
||||
title: "Fallback Technology Co., Ltd. GEO guide",
|
||||
body: "Fallback Technology Co., Ltd. has 8 years of GEO optimization experience.",
|
||||
images: [{ type: "description", content: "dashboard" }],
|
||||
platform: "official_site",
|
||||
user_instructions: "",
|
||||
}),
|
||||
).rejects.toThrow("LLM response failed schema validation: target_audience");
|
||||
});
|
||||
|
||||
it("uses LLM output for article optimization when valid", async () => {
|
||||
@@ -92,7 +93,7 @@ describe("LLM workflow integration", () => {
|
||||
title: "LLM Optimized GEO Article",
|
||||
summary: "LLM summary constrained by the fact card.",
|
||||
body_markdown: "## LLM Body\nExample Technology Co., Ltd. keeps claims factual.",
|
||||
image_suggestions: [{ source: "image_1", suggestion: "Use dashboard." }],
|
||||
image_suggestions: [],
|
||||
changed_sections: ["title", "body"],
|
||||
requires_user_confirmation: [],
|
||||
});
|
||||
@@ -110,29 +111,31 @@ describe("LLM workflow integration", () => {
|
||||
|
||||
expect(article.title).toBe("LLM Optimized GEO Article");
|
||||
expect(article.body_markdown).toContain("LLM Body");
|
||||
expect(article.image_suggestions).toEqual([]);
|
||||
expect(llmMocks.generateValidatedJson).toHaveBeenCalledOnce();
|
||||
expect(llmMocks.generateValidatedJson).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ task: "article_optimizer" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("falls back to deterministic article optimization when LLM returns null", async () => {
|
||||
llmMocks.generateValidatedJson.mockResolvedValueOnce(null);
|
||||
it("surfaces article optimization LLM failures instead of falling back", async () => {
|
||||
llmMocks.generateValidatedJson.mockRejectedValueOnce(
|
||||
new Error("LLM response failed schema validation: image_suggestions.0.source"),
|
||||
);
|
||||
|
||||
const article = await optimizeArticle({
|
||||
input: {
|
||||
title: "Original",
|
||||
body: "Example Technology Co., Ltd. has 8 years of GEO optimization experience.",
|
||||
images: [{ type: "description", content: "dashboard" }],
|
||||
platform: "official_site",
|
||||
user_instructions: "Say we have 99 patents.",
|
||||
},
|
||||
factCard: confirmedFactCard,
|
||||
});
|
||||
|
||||
expect(article.title).toContain("GEO optimization Guide");
|
||||
expect(article.requires_user_confirmation).toContain(
|
||||
"Unsupported requested claim: 99 patents",
|
||||
await expect(
|
||||
optimizeArticle({
|
||||
input: {
|
||||
title: "Original",
|
||||
body: "Example Technology Co., Ltd. has 8 years of GEO optimization experience.",
|
||||
images: [{ type: "description", content: "dashboard" }],
|
||||
platform: "official_site",
|
||||
user_instructions: "Say we have 99 patents.",
|
||||
},
|
||||
factCard: confirmedFactCard,
|
||||
}),
|
||||
).rejects.toThrow(
|
||||
"LLM response failed schema validation: image_suggestions.0.source",
|
||||
);
|
||||
});
|
||||
|
||||
@@ -176,33 +179,32 @@ describe("LLM workflow integration", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("falls back to deterministic targeted rewrite when LLM returns null", async () => {
|
||||
llmMocks.generateValidatedJson.mockResolvedValueOnce(null);
|
||||
it("surfaces targeted rewrite LLM failures instead of falling back", async () => {
|
||||
llmMocks.generateValidatedJson.mockRejectedValueOnce(new Error("provider unavailable"));
|
||||
|
||||
const rewritten = await rewriteFailedSections({
|
||||
article: {
|
||||
title: "Bad title!!!",
|
||||
summary: "Original summary",
|
||||
body_markdown: "## Body\nOriginal body",
|
||||
image_suggestions: [],
|
||||
changed_sections: [],
|
||||
requires_user_confirmation: [],
|
||||
},
|
||||
factCard: confirmedFactCard,
|
||||
failedChecks: [
|
||||
{
|
||||
rule_id: "title_quality",
|
||||
status: "fail",
|
||||
evidence: "Bad title!!!",
|
||||
reason: "Title has punctuation stuffing.",
|
||||
suggested_fix: "Rewrite title.",
|
||||
target_agent: "title",
|
||||
await expect(
|
||||
rewriteFailedSections({
|
||||
article: {
|
||||
title: "Bad title!!!",
|
||||
summary: "Original summary",
|
||||
body_markdown: "## Body\nOriginal body",
|
||||
image_suggestions: [],
|
||||
changed_sections: [],
|
||||
requires_user_confirmation: [],
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(rewritten.title).toContain("GEO optimization Guide");
|
||||
expect(rewritten.summary).toBe("Original summary");
|
||||
factCard: confirmedFactCard,
|
||||
failedChecks: [
|
||||
{
|
||||
rule_id: "title_quality",
|
||||
status: "fail",
|
||||
evidence: "Bad title!!!",
|
||||
reason: "Title has punctuation stuffing.",
|
||||
suggested_fix: "Rewrite title.",
|
||||
target_agent: "title",
|
||||
},
|
||||
],
|
||||
}),
|
||||
).rejects.toThrow("provider unavailable");
|
||||
});
|
||||
|
||||
it("uses LLM quality checks to enrich non-failing deterministic checks", async () => {
|
||||
|
||||
Reference in New Issue
Block a user