diff --git a/src/lib/workflow/__tests__/llm-integration.test.ts b/src/lib/workflow/__tests__/llm-integration.test.ts index 4c9ad21..7a0cc57 100644 --- a/src/lib/workflow/__tests__/llm-integration.test.ts +++ b/src/lib/workflow/__tests__/llm-integration.test.ts @@ -16,6 +16,7 @@ vi.mock("../../llm/client", async () => { import { extractCandidateFactCard } from "../fact-extractor"; import { optimizeArticle } from "../article-optimizer"; +import { inspectQualityWithLlm } from "../quality-inspector"; import { rewriteFailedSections } from "../targeted-rewriter"; describe("LLM workflow integration", () => { @@ -194,4 +195,77 @@ describe("LLM workflow integration", () => { expect(rewritten.title).toContain("GEO optimization Guide"); expect(rewritten.summary).toBe("Original summary"); }); + + it("uses LLM quality checks to enrich non-failing deterministic checks", async () => { + llmMocks.generateValidatedJson.mockResolvedValueOnce({ + checks: [ + { + rule_id: "platform_fit", + status: "warn", + evidence: "LLM noticed the article reads like a generic blog post.", + reason: "The structure is not specific enough for an official site.", + suggested_fix: "Add a clearer brand-owned introduction.", + target_agent: "body", + }, + ], + }); + + const report = await inspectQualityWithLlm({ + article: { + title: "Example GEO Optimization Guide", + summary: + "A official site article for Marketing teams about GEO optimization.", + body_markdown: + "Example Technology Co., Ltd. has 8 years of GEO optimization experience.", + image_suggestions: [{ source: "image_1", suggestion: "Use dashboard." }], + changed_sections: [], + requires_user_confirmation: [], + }, + factCard: confirmedFactCard, + platform: "official_site", + sourceImages: [{ type: "description", content: "dashboard" }], + }); + + const platformCheck = report.checks.find( + (check) => check.rule_id === "platform_fit", + ); + expect(platformCheck?.status).toBe("warn"); + expect(platformCheck?.evidence).toContain("LLM noticed"); + }); + + it("does not let LLM downgrade deterministic hard failures", async () => { + llmMocks.generateValidatedJson.mockResolvedValueOnce({ + checks: [ + { + rule_id: "company_name_integrity", + status: "pass", + evidence: "LLM says it is fine.", + reason: "LLM attempted to downgrade a failure.", + suggested_fix: "", + target_agent: null, + }, + ], + }); + + const report = await inspectQualityWithLlm({ + article: { + title: "Example GEO Optimization Guide", + summary: + "A official site article for Marketing teams about GEO optimization.", + body_markdown: "Example has 8 years of GEO optimization experience.", + image_suggestions: [{ source: "image_1", suggestion: "Use dashboard." }], + changed_sections: [], + requires_user_confirmation: [], + }, + factCard: confirmedFactCard, + platform: "official_site", + sourceImages: [{ type: "description", content: "dashboard" }], + }); + + const companyCheck = report.checks.find( + (check) => check.rule_id === "company_name_integrity", + ); + expect(companyCheck?.status).toBe("fail"); + expect(companyCheck?.reason).toContain("公司"); + }); }); diff --git a/src/lib/workflow/orchestrator.ts b/src/lib/workflow/orchestrator.ts index da584a1..4508b29 100644 --- a/src/lib/workflow/orchestrator.ts +++ b/src/lib/workflow/orchestrator.ts @@ -1,7 +1,7 @@ import type { ArticleInput, ConfirmedFactCard } from "../domain/types"; import { optimizeArticle } from "./article-optimizer"; -import { inspectQuality } from "./quality-inspector"; +import { inspectQualityWithLlm } from "./quality-inspector"; import { rewriteFailedSections } from "./targeted-rewriter"; export interface RunOptimizationWorkflowInput { @@ -14,7 +14,7 @@ export async function runOptimizationWorkflow({ factCard, }: RunOptimizationWorkflowInput) { let article = await optimizeArticle({ input, factCard }); - let qaReport = inspectQuality({ + let qaReport = await inspectQualityWithLlm({ article, factCard, platform: input.platform, @@ -26,7 +26,7 @@ export async function runOptimizationWorkflow({ const failedChecks = qaReport.checks.filter((check) => check.status === "fail"); article = await rewriteFailedSections({ article, factCard, failedChecks }); rewriteRounds += 1; - qaReport = inspectQuality({ + qaReport = await inspectQualityWithLlm({ article, factCard, platform: input.platform, diff --git a/src/lib/workflow/quality-inspector.ts b/src/lib/workflow/quality-inspector.ts index 5ad715a..66ed7fd 100644 --- a/src/lib/workflow/quality-inspector.ts +++ b/src/lib/workflow/quality-inspector.ts @@ -8,7 +8,13 @@ import type { QaReport, QualityRuleId, } from "../domain/types"; -import { qaReportSchema } from "../domain/validation"; +import { qaCheckSchema, qaReportSchema } from "../domain/validation"; +import { generateValidatedJson } from "../llm/client"; +import { + QUALITY_INSPECTOR_SYSTEM_PROMPT, + buildQualityInspectorPrompt, +} from "../llm/prompts"; +import { z } from "zod"; const REQUIRED_RULES: QualityRuleId[] = [ "industry_alignment", @@ -23,6 +29,10 @@ const REQUIRED_RULES: QualityRuleId[] = [ "context_sensitive_terms", ]; +const llmQaPatchSchema = z.object({ + checks: z.array(qaCheckSchema).default([]), +}); + export interface InspectQualityInput { article: OptimizedArticle; factCard: ConfirmedFactCard; @@ -41,6 +51,57 @@ export function inspectQuality(input: InspectQualityInput): QaReport { return qaReportSchema.parse({ overall_status, checks }); } +export async function inspectQualityWithLlm( + input: InspectQualityInput, +): Promise { + const deterministicReport = inspectQuality(input); + const llmPatch = await generateValidatedJson({ + schema: llmQaPatchSchema, + system: QUALITY_INSPECTOR_SYSTEM_PROMPT, + prompt: buildQualityInspectorPrompt({ + article: input.article, + factCard: input.factCard, + platform: input.platform, + deterministicChecks: deterministicReport.checks, + }), + temperature: 0.1, + }); + + if (!llmPatch) { + return deterministicReport; + } + + const patchedChecks = deterministicReport.checks.map((deterministicCheck) => { + const llmCheck = llmPatch.checks.find( + (check) => check.rule_id === deterministicCheck.rule_id, + ); + if (!llmCheck) { + return deterministicCheck; + } + if (deterministicCheck.status === "fail") { + return deterministicCheck; + } + return { + ...deterministicCheck, + status: llmCheck.status, + evidence: llmCheck.evidence, + reason: llmCheck.reason, + suggested_fix: llmCheck.suggested_fix, + target_agent: llmCheck.target_agent, + }; + }); + + const overall_status: CheckStatus = patchedChecks.some( + (check) => check.status === "fail", + ) + ? "fail" + : patchedChecks.some((check) => check.status === "warn") + ? "warn" + : "pass"; + + return qaReportSchema.parse({ overall_status, checks: patchedChecks }); +} + function inspectRule( ruleId: QualityRuleId, { article, factCard, platform, sourceImages }: InspectQualityInput,