import { afterEach, describe, expect, it, vi } from "vitest"; const llmMocks = vi.hoisted(() => ({ generateValidatedJson: vi.fn(), })); vi.mock("../../llm/client", async () => { const actual = await vi.importActual( "../../llm/client", ); return { ...actual, generateValidatedJson: llmMocks.generateValidatedJson, }; }); import { extractCandidateFactCard } from "../fact-extractor"; import { optimizeArticle } from "../article-optimizer"; import { inspectQualityWithLlm } from "../quality-inspector"; import { rewriteFailedSections } from "../targeted-rewriter"; describe("LLM workflow integration", () => { const confirmedFactCard = { company_full_name: "Example Technology Co., Ltd.", company_short_names: ["Example Tech"], brand_names: ["Example"], product_names: ["Example GEO"], target_industry: "GEO optimization", target_audience: "Marketing teams", experience_years: 8, core_claims: ["8 years of GEO optimization experience"], forbidden_claims: ["industry first"], image_topics: ["dashboard"], uncertain_items: [], is_ready_for_optimization: true, confirmed_by_user: true, } as const; afterEach(() => { llmMocks.generateValidatedJson.mockReset(); }); it("uses LLM output for candidate fact extraction when valid", async () => { llmMocks.generateValidatedJson.mockResolvedValueOnce({ company_full_name: "DeepSeek Example Co., Ltd.", company_short_names: ["DeepSeek Example"], brand_names: ["DSExample"], product_names: ["DS GEO"], target_industry: "GEO optimization", target_audience: "Marketing teams", experience_years: 9, core_claims: ["9 years of GEO optimization experience"], forbidden_claims: ["industry first"], image_topics: ["dashboard"], uncertain_items: [], is_ready_for_optimization: true, }); const card = await extractCandidateFactCard({ title: "Example source", body: "Fallback Technology Co., Ltd. has 8 years of GEO optimization experience.", images: [{ type: "description", content: "dashboard" }], platform: "official_site", user_instructions: "", }); expect(card.company_full_name).toBe("DeepSeek Example Co., Ltd."); expect(card.experience_years).toBe(9); expect(llmMocks.generateValidatedJson).toHaveBeenCalledOnce(); expect(llmMocks.generateValidatedJson).toHaveBeenCalledWith( expect.objectContaining({ task: "fact_extractor" }), ); }); it("surfaces candidate fact extraction LLM failures instead of falling back", async () => { llmMocks.generateValidatedJson.mockRejectedValueOnce( new Error("LLM response failed schema validation: target_audience"), ); await expect( extractCandidateFactCard({ title: "Fallback Technology Co., Ltd. GEO guide", body: "Fallback Technology Co., Ltd. has 8 years of GEO optimization experience.", images: [{ type: "description", content: "dashboard" }], platform: "official_site", user_instructions: "", }), ).rejects.toThrow("LLM response failed schema validation: target_audience"); }); it("uses LLM output for article optimization when valid", async () => { llmMocks.generateValidatedJson.mockResolvedValueOnce({ title: "LLM Optimized GEO Article", summary: "LLM summary constrained by the fact card.", body_markdown: "## LLM Body\nExample Technology Co., Ltd. keeps claims factual.", image_suggestions: [], changed_sections: ["title", "body"], requires_user_confirmation: [], }); const article = await optimizeArticle({ input: { title: "Original", body: "Example Technology Co., Ltd. has 8 years of GEO optimization experience.", images: [{ type: "description", content: "dashboard" }], platform: "official_site", user_instructions: "", }, factCard: confirmedFactCard, }); expect(article.title).toBe("LLM Optimized GEO Article"); expect(article.body_markdown).toContain("LLM Body"); expect(article.image_suggestions).toEqual([]); expect(llmMocks.generateValidatedJson).toHaveBeenCalledOnce(); expect(llmMocks.generateValidatedJson).toHaveBeenCalledWith( expect.objectContaining({ task: "article_optimizer" }), ); }); it("surfaces article optimization LLM failures instead of falling back", async () => { llmMocks.generateValidatedJson.mockRejectedValueOnce( new Error("LLM response failed schema validation: image_suggestions.0.source"), ); await expect( optimizeArticle({ input: { title: "Original", body: "Example Technology Co., Ltd. has 8 years of GEO optimization experience.", images: [{ type: "description", content: "dashboard" }], platform: "official_site", user_instructions: "Say we have 99 patents.", }, factCard: confirmedFactCard, }), ).rejects.toThrow( "LLM response failed schema validation: image_suggestions.0.source", ); }); it("uses LLM output for targeted rewrite when valid", async () => { llmMocks.generateValidatedJson.mockResolvedValueOnce({ title: "Rewritten By LLM", summary: "Original summary", body_markdown: "## Body\nExample Technology Co., Ltd. focuses on GEO optimization.", image_suggestions: [], changed_sections: ["title"], requires_user_confirmation: [], }); const rewritten = await rewriteFailedSections({ article: { title: "Bad title!!!", summary: "Original summary", body_markdown: "## Body\nOriginal body", image_suggestions: [], changed_sections: [], requires_user_confirmation: [], }, factCard: confirmedFactCard, failedChecks: [ { rule_id: "title_quality", status: "fail", evidence: "Bad title!!!", reason: "Title has punctuation stuffing.", suggested_fix: "Rewrite title.", target_agent: "title", }, ], }); expect(rewritten.title).toBe("Rewritten By LLM"); expect(llmMocks.generateValidatedJson).toHaveBeenCalledOnce(); expect(llmMocks.generateValidatedJson).toHaveBeenCalledWith( expect.objectContaining({ task: "targeted_rewriter" }), ); }); it("surfaces targeted rewrite LLM failures instead of falling back", async () => { llmMocks.generateValidatedJson.mockRejectedValueOnce(new Error("provider unavailable")); await expect( rewriteFailedSections({ article: { title: "Bad title!!!", summary: "Original summary", body_markdown: "## Body\nOriginal body", image_suggestions: [], changed_sections: [], requires_user_confirmation: [], }, factCard: confirmedFactCard, failedChecks: [ { rule_id: "title_quality", status: "fail", evidence: "Bad title!!!", reason: "Title has punctuation stuffing.", suggested_fix: "Rewrite title.", target_agent: "title", }, ], }), ).rejects.toThrow("provider unavailable"); }); it("uses LLM quality checks to enrich non-failing deterministic checks", async () => { llmMocks.generateValidatedJson.mockResolvedValueOnce({ checks: [ { rule_id: "platform_fit", status: "warn", evidence: "LLM noticed the article reads like a generic blog post.", reason: "The structure is not specific enough for an official site.", suggested_fix: "Add a clearer brand-owned introduction.", target_agent: "body", }, ], }); const report = await inspectQualityWithLlm({ article: { title: "Example GEO Optimization Guide", summary: "A official site article for Marketing teams about GEO optimization.", body_markdown: "Example Technology Co., Ltd. has 8 years of GEO optimization experience.", image_suggestions: [{ source: "image_1", suggestion: "Use dashboard." }], changed_sections: [], requires_user_confirmation: [], }, factCard: confirmedFactCard, platform: "official_site", sourceImages: [{ type: "description", content: "dashboard" }], }); const platformCheck = report.checks.find( (check) => check.rule_id === "platform_fit", ); expect(platformCheck?.status).toBe("warn"); expect(platformCheck?.evidence).toContain("LLM noticed"); expect(llmMocks.generateValidatedJson).toHaveBeenCalledWith( expect.objectContaining({ task: "quality_inspector" }), ); }); it("does not let LLM downgrade deterministic hard failures", async () => { llmMocks.generateValidatedJson.mockResolvedValueOnce({ checks: [ { rule_id: "company_name_integrity", status: "pass", evidence: "LLM says it is fine.", reason: "LLM attempted to downgrade a failure.", suggested_fix: "", target_agent: null, }, ], }); const report = await inspectQualityWithLlm({ article: { title: "Example GEO Optimization Guide", summary: "A official site article for Marketing teams about GEO optimization.", body_markdown: "Example has 8 years of GEO optimization experience.", image_suggestions: [{ source: "image_1", suggestion: "Use dashboard." }], changed_sections: [], requires_user_confirmation: [], }, factCard: confirmedFactCard, platform: "official_site", sourceImages: [{ type: "description", content: "dashboard" }], }); const companyCheck = report.checks.find( (check) => check.rule_id === "company_name_integrity", ); expect(companyCheck?.status).toBe("fail"); expect(companyCheck?.reason).toContain("公司"); }); it("does not let LLM escalate soft quality checks to hard failures", async () => { llmMocks.generateValidatedJson.mockResolvedValueOnce({ checks: [ { rule_id: "platform_fit", status: "fail", evidence: "LLM thinks the structure is not recommendation-like enough.", reason: "This is a soft platform-fit concern.", suggested_fix: "Adjust structure if needed.", target_agent: "body", }, ], }); const report = await inspectQualityWithLlm({ article: { title: "Example GEO Optimization Guide", summary: "A official site article for Marketing teams about GEO optimization.", body_markdown: "Example Technology Co., Ltd. has 8 years of GEO optimization experience.", image_suggestions: [], changed_sections: [], requires_user_confirmation: [], }, factCard: confirmedFactCard, platform: "official_site", sourceImages: [], }); const platformCheck = report.checks.find( (check) => check.rule_id === "platform_fit", ); expect(platformCheck?.status).toBe("warn"); expect(report.overall_status).not.toBe("fail"); }); });