323 lines
11 KiB
TypeScript
323 lines
11 KiB
TypeScript
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
|
|
const llmMocks = vi.hoisted(() => ({
|
|
generateValidatedJson: vi.fn(),
|
|
}));
|
|
|
|
vi.mock("../../llm/client", async () => {
|
|
const actual = await vi.importActual<typeof import("../../llm/client")>(
|
|
"../../llm/client",
|
|
);
|
|
return {
|
|
...actual,
|
|
generateValidatedJson: llmMocks.generateValidatedJson,
|
|
};
|
|
});
|
|
|
|
import { extractCandidateFactCard } from "../fact-extractor";
|
|
import { optimizeArticle } from "../article-optimizer";
|
|
import { inspectQualityWithLlm } from "../quality-inspector";
|
|
import { rewriteFailedSections } from "../targeted-rewriter";
|
|
|
|
describe("LLM workflow integration", () => {
|
|
const confirmedFactCard = {
|
|
company_full_name: "Example Technology Co., Ltd.",
|
|
company_short_names: ["Example Tech"],
|
|
brand_names: ["Example"],
|
|
product_names: ["Example GEO"],
|
|
target_industry: "GEO optimization",
|
|
target_audience: "Marketing teams",
|
|
experience_years: 8,
|
|
core_claims: ["8 years of GEO optimization experience"],
|
|
forbidden_claims: ["industry first"],
|
|
image_topics: ["dashboard"],
|
|
uncertain_items: [],
|
|
is_ready_for_optimization: true,
|
|
confirmed_by_user: true,
|
|
} as const;
|
|
|
|
afterEach(() => {
|
|
llmMocks.generateValidatedJson.mockReset();
|
|
});
|
|
|
|
it("uses LLM output for candidate fact extraction when valid", async () => {
|
|
llmMocks.generateValidatedJson.mockResolvedValueOnce({
|
|
company_full_name: "DeepSeek Example Co., Ltd.",
|
|
company_short_names: ["DeepSeek Example"],
|
|
brand_names: ["DSExample"],
|
|
product_names: ["DS GEO"],
|
|
target_industry: "GEO optimization",
|
|
target_audience: "Marketing teams",
|
|
experience_years: 9,
|
|
core_claims: ["9 years of GEO optimization experience"],
|
|
forbidden_claims: ["industry first"],
|
|
image_topics: ["dashboard"],
|
|
uncertain_items: [],
|
|
is_ready_for_optimization: true,
|
|
});
|
|
|
|
const card = await extractCandidateFactCard({
|
|
title: "Example source",
|
|
body: "Fallback Technology Co., Ltd. has 8 years of GEO optimization experience.",
|
|
images: [{ type: "description", content: "dashboard" }],
|
|
platform: "official_site",
|
|
user_instructions: "",
|
|
});
|
|
|
|
expect(card.company_full_name).toBe("DeepSeek Example Co., Ltd.");
|
|
expect(card.experience_years).toBe(9);
|
|
expect(llmMocks.generateValidatedJson).toHaveBeenCalledOnce();
|
|
expect(llmMocks.generateValidatedJson).toHaveBeenCalledWith(
|
|
expect.objectContaining({ task: "fact_extractor" }),
|
|
);
|
|
});
|
|
|
|
it("surfaces candidate fact extraction LLM failures instead of falling back", async () => {
|
|
llmMocks.generateValidatedJson.mockRejectedValueOnce(
|
|
new Error("LLM response failed schema validation: target_audience"),
|
|
);
|
|
|
|
await expect(
|
|
extractCandidateFactCard({
|
|
title: "Fallback Technology Co., Ltd. GEO guide",
|
|
body: "Fallback Technology Co., Ltd. has 8 years of GEO optimization experience.",
|
|
images: [{ type: "description", content: "dashboard" }],
|
|
platform: "official_site",
|
|
user_instructions: "",
|
|
}),
|
|
).rejects.toThrow("LLM response failed schema validation: target_audience");
|
|
});
|
|
|
|
it("uses LLM output for article optimization when valid", async () => {
|
|
llmMocks.generateValidatedJson.mockResolvedValueOnce({
|
|
title: "LLM Optimized GEO Article",
|
|
summary: "LLM summary constrained by the fact card.",
|
|
body_markdown: "## LLM Body\nExample Technology Co., Ltd. keeps claims factual.",
|
|
image_suggestions: [],
|
|
changed_sections: ["title", "body"],
|
|
requires_user_confirmation: [],
|
|
});
|
|
|
|
const article = await optimizeArticle({
|
|
input: {
|
|
title: "Original",
|
|
body: "Example Technology Co., Ltd. has 8 years of GEO optimization experience.",
|
|
images: [{ type: "description", content: "dashboard" }],
|
|
platform: "official_site",
|
|
user_instructions: "",
|
|
},
|
|
factCard: confirmedFactCard,
|
|
});
|
|
|
|
expect(article.title).toBe("LLM Optimized GEO Article");
|
|
expect(article.body_markdown).toContain("LLM Body");
|
|
expect(article.image_suggestions).toEqual([]);
|
|
expect(llmMocks.generateValidatedJson).toHaveBeenCalledOnce();
|
|
expect(llmMocks.generateValidatedJson).toHaveBeenCalledWith(
|
|
expect.objectContaining({ task: "article_optimizer" }),
|
|
);
|
|
});
|
|
|
|
it("surfaces article optimization LLM failures instead of falling back", async () => {
|
|
llmMocks.generateValidatedJson.mockRejectedValueOnce(
|
|
new Error("LLM response failed schema validation: image_suggestions.0.source"),
|
|
);
|
|
|
|
await expect(
|
|
optimizeArticle({
|
|
input: {
|
|
title: "Original",
|
|
body: "Example Technology Co., Ltd. has 8 years of GEO optimization experience.",
|
|
images: [{ type: "description", content: "dashboard" }],
|
|
platform: "official_site",
|
|
user_instructions: "Say we have 99 patents.",
|
|
},
|
|
factCard: confirmedFactCard,
|
|
}),
|
|
).rejects.toThrow(
|
|
"LLM response failed schema validation: image_suggestions.0.source",
|
|
);
|
|
});
|
|
|
|
it("uses LLM output for targeted rewrite when valid", async () => {
|
|
llmMocks.generateValidatedJson.mockResolvedValueOnce({
|
|
title: "Rewritten By LLM",
|
|
summary: "Original summary",
|
|
body_markdown:
|
|
"## Body\nExample Technology Co., Ltd. focuses on GEO optimization.",
|
|
image_suggestions: [],
|
|
changed_sections: ["title"],
|
|
requires_user_confirmation: [],
|
|
});
|
|
|
|
const rewritten = await rewriteFailedSections({
|
|
article: {
|
|
title: "Bad title!!!",
|
|
summary: "Original summary",
|
|
body_markdown: "## Body\nOriginal body",
|
|
image_suggestions: [],
|
|
changed_sections: [],
|
|
requires_user_confirmation: [],
|
|
},
|
|
factCard: confirmedFactCard,
|
|
failedChecks: [
|
|
{
|
|
rule_id: "title_quality",
|
|
status: "fail",
|
|
evidence: "Bad title!!!",
|
|
reason: "Title has punctuation stuffing.",
|
|
suggested_fix: "Rewrite title.",
|
|
target_agent: "title",
|
|
},
|
|
],
|
|
});
|
|
|
|
expect(rewritten.title).toBe("Rewritten By LLM");
|
|
expect(llmMocks.generateValidatedJson).toHaveBeenCalledOnce();
|
|
expect(llmMocks.generateValidatedJson).toHaveBeenCalledWith(
|
|
expect.objectContaining({ task: "targeted_rewriter" }),
|
|
);
|
|
});
|
|
|
|
it("surfaces targeted rewrite LLM failures instead of falling back", async () => {
|
|
llmMocks.generateValidatedJson.mockRejectedValueOnce(new Error("provider unavailable"));
|
|
|
|
await expect(
|
|
rewriteFailedSections({
|
|
article: {
|
|
title: "Bad title!!!",
|
|
summary: "Original summary",
|
|
body_markdown: "## Body\nOriginal body",
|
|
image_suggestions: [],
|
|
changed_sections: [],
|
|
requires_user_confirmation: [],
|
|
},
|
|
factCard: confirmedFactCard,
|
|
failedChecks: [
|
|
{
|
|
rule_id: "title_quality",
|
|
status: "fail",
|
|
evidence: "Bad title!!!",
|
|
reason: "Title has punctuation stuffing.",
|
|
suggested_fix: "Rewrite title.",
|
|
target_agent: "title",
|
|
},
|
|
],
|
|
}),
|
|
).rejects.toThrow("provider unavailable");
|
|
});
|
|
|
|
it("uses LLM quality checks to enrich non-failing deterministic checks", async () => {
|
|
llmMocks.generateValidatedJson.mockResolvedValueOnce({
|
|
checks: [
|
|
{
|
|
rule_id: "platform_fit",
|
|
status: "warn",
|
|
evidence: "LLM noticed the article reads like a generic blog post.",
|
|
reason: "The structure is not specific enough for an official site.",
|
|
suggested_fix: "Add a clearer brand-owned introduction.",
|
|
target_agent: "body",
|
|
},
|
|
],
|
|
});
|
|
|
|
const report = await inspectQualityWithLlm({
|
|
article: {
|
|
title: "Example GEO Optimization Guide",
|
|
summary:
|
|
"A official site article for Marketing teams about GEO optimization.",
|
|
body_markdown:
|
|
"Example Technology Co., Ltd. has 8 years of GEO optimization experience.",
|
|
image_suggestions: [{ source: "image_1", suggestion: "Use dashboard." }],
|
|
changed_sections: [],
|
|
requires_user_confirmation: [],
|
|
},
|
|
factCard: confirmedFactCard,
|
|
platform: "official_site",
|
|
sourceImages: [{ type: "description", content: "dashboard" }],
|
|
});
|
|
|
|
const platformCheck = report.checks.find(
|
|
(check) => check.rule_id === "platform_fit",
|
|
);
|
|
expect(platformCheck?.status).toBe("warn");
|
|
expect(platformCheck?.evidence).toContain("LLM noticed");
|
|
expect(llmMocks.generateValidatedJson).toHaveBeenCalledWith(
|
|
expect.objectContaining({ task: "quality_inspector" }),
|
|
);
|
|
});
|
|
|
|
it("does not let LLM downgrade deterministic hard failures", async () => {
|
|
llmMocks.generateValidatedJson.mockResolvedValueOnce({
|
|
checks: [
|
|
{
|
|
rule_id: "company_name_integrity",
|
|
status: "pass",
|
|
evidence: "LLM says it is fine.",
|
|
reason: "LLM attempted to downgrade a failure.",
|
|
suggested_fix: "",
|
|
target_agent: null,
|
|
},
|
|
],
|
|
});
|
|
|
|
const report = await inspectQualityWithLlm({
|
|
article: {
|
|
title: "Example GEO Optimization Guide",
|
|
summary:
|
|
"A official site article for Marketing teams about GEO optimization.",
|
|
body_markdown: "Example has 8 years of GEO optimization experience.",
|
|
image_suggestions: [{ source: "image_1", suggestion: "Use dashboard." }],
|
|
changed_sections: [],
|
|
requires_user_confirmation: [],
|
|
},
|
|
factCard: confirmedFactCard,
|
|
platform: "official_site",
|
|
sourceImages: [{ type: "description", content: "dashboard" }],
|
|
});
|
|
|
|
const companyCheck = report.checks.find(
|
|
(check) => check.rule_id === "company_name_integrity",
|
|
);
|
|
expect(companyCheck?.status).toBe("fail");
|
|
expect(companyCheck?.reason).toContain("公司");
|
|
});
|
|
|
|
it("does not let LLM escalate soft quality checks to hard failures", async () => {
|
|
llmMocks.generateValidatedJson.mockResolvedValueOnce({
|
|
checks: [
|
|
{
|
|
rule_id: "platform_fit",
|
|
status: "fail",
|
|
evidence: "LLM thinks the structure is not recommendation-like enough.",
|
|
reason: "This is a soft platform-fit concern.",
|
|
suggested_fix: "Adjust structure if needed.",
|
|
target_agent: "body",
|
|
},
|
|
],
|
|
});
|
|
|
|
const report = await inspectQualityWithLlm({
|
|
article: {
|
|
title: "Example GEO Optimization Guide",
|
|
summary:
|
|
"A official site article for Marketing teams about GEO optimization.",
|
|
body_markdown:
|
|
"Example Technology Co., Ltd. has 8 years of GEO optimization experience.",
|
|
image_suggestions: [],
|
|
changed_sections: [],
|
|
requires_user_confirmation: [],
|
|
},
|
|
factCard: confirmedFactCard,
|
|
platform: "official_site",
|
|
sourceImages: [],
|
|
});
|
|
|
|
const platformCheck = report.checks.find(
|
|
(check) => check.rule_id === "platform_fit",
|
|
);
|
|
expect(platformCheck?.status).toBe("warn");
|
|
expect(report.overall_status).not.toBe("fail");
|
|
});
|
|
});
|