From f1dac1540082b4a573f2c57aa86b2285049061a3 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 17 Jun 2026 08:02:50 +0800 Subject: [PATCH] fix: surface llm workflow errors --- src/app/api/__tests__/jobs.test.ts | 89 ++++++++++++- src/app/api/jobs/[jobId]/optimize/route.ts | 87 +++++++------ src/app/api/jobs/route.ts | 9 +- src/lib/llm/__tests__/client.test.ts | 84 ++++++------ src/lib/llm/client.ts | 31 ++++- src/lib/llm/prompts.ts | 11 +- .../__tests__/llm-integration.test.ts | 110 ++++++++-------- src/lib/workflow/__tests__/workflow.test.ts | 109 ---------------- src/lib/workflow/article-optimizer.ts | 73 +---------- src/lib/workflow/fact-extractor.ts | 122 +----------------- src/lib/workflow/quality-inspector.ts | 15 +-- src/lib/workflow/targeted-rewriter.ts | 46 +------ 12 files changed, 286 insertions(+), 500 deletions(-) diff --git a/src/app/api/__tests__/jobs.test.ts b/src/app/api/__tests__/jobs.test.ts index a80e141..dddccc9 100644 --- a/src/app/api/__tests__/jobs.test.ts +++ b/src/app/api/__tests__/jobs.test.ts @@ -39,6 +39,21 @@ const validFactCard = { confirmed_by_user: true, }; +const validCandidateFactCard = { + company_full_name: validFactCard.company_full_name, + company_short_names: validFactCard.company_short_names, + brand_names: validFactCard.brand_names, + product_names: validFactCard.product_names, + target_industry: validFactCard.target_industry, + target_audience: validFactCard.target_audience, + experience_years: validFactCard.experience_years, + core_claims: validFactCard.core_claims, + forbidden_claims: validFactCard.forbidden_claims, + image_topics: validFactCard.image_topics, + uncertain_items: validFactCard.uncertain_items, + is_ready_for_optimization: true, +}; + interface CreateJobResponse { job: { id: string }; candidateFactCard: { company_full_name: string }; @@ -86,6 +101,8 @@ describe("job API routes", () => { }); it("validates input, creates a job, and returns a candidate fact card", async () => { + llmMocks.generateValidatedJson.mockResolvedValueOnce(validCandidateFactCard); + const response = await createJob( request({ title: "Example Technology Co., Ltd. GEO guide", @@ -104,6 +121,38 @@ describe("job API routes", () => { ); }); + it("returns a clear error when LLM fact extraction fails", async () => { + llmMocks.generateValidatedJson.mockRejectedValueOnce( + new Error("LLM response failed schema validation: target_audience"), + ); + + const response = await createJob( + request({ + title: "Example Technology Co., Ltd. GEO guide", + body: "Example Technology Co., Ltd. has 8 years of GEO optimization experience.", + platform: "official_site", + }), + ); + const body = (await response.json()) as { error: string }; + + expect(response.status).toBe(502); + expect(body.error).toBe( + "LLM response failed schema validation: target_audience", + ); + }); + + it("still returns 400 for invalid article input", async () => { + const response = await createJob( + request({ + title: "", + body: "", + platform: "official_site", + }), + ); + + expect(response.status).toBe(400); + }); + it("rejects unresolved uncertain items when confirming a fact card", async () => { const { job } = await createJobFixture(); const response = await confirmFactCard( @@ -135,6 +184,19 @@ describe("job API routes", () => { params<{ jobId: string }>({ jobId: job.id }), ); + llmMocks.generateValidatedJson + .mockResolvedValueOnce({ + title: "API LLM Optimized GEO Article", + summary: + "A official site article for Marketing teams about GEO optimization.", + body_markdown: + "Example Technology Co., Ltd. has 8 years of GEO optimization experience.", + image_suggestions: [], + changed_sections: ["title", "body"], + requires_user_confirmation: [], + }) + .mockResolvedValueOnce({ checks: [] }); + const response = await optimizeJob( request({}), params<{ jobId: string }>({ jobId: job.id }), @@ -142,7 +204,7 @@ describe("job API routes", () => { const body = (await response.json()) as OptimizeJobResponse; expect(response.status).toBe(200); - expect(body.optimizedArticle.title).toContain("GEO optimization"); + expect(body.optimizedArticle.title).toBe("API LLM Optimized GEO Article"); expect(body.qaReport.checks).toHaveLength(10); }); @@ -166,7 +228,7 @@ describe("job API routes", () => { changed_sections: ["title", "body"], requires_user_confirmation: [], }) - .mockResolvedValue(null); + .mockResolvedValue({ checks: [] }); const response = await optimizeJob( request({}), @@ -179,6 +241,27 @@ describe("job API routes", () => { expect(llmMocks.generateValidatedJson).toHaveBeenCalled(); }); + it("returns a clear error when LLM optimization fails", async () => { + const { job } = await createJobFixture(); + await confirmFactCard( + request(validFactCard), + params<{ jobId: string }>({ jobId: job.id }), + ); + + llmMocks.generateValidatedJson.mockRejectedValueOnce( + new Error("LLM response failed schema validation: body_markdown"), + ); + + const response = await optimizeJob( + request({}), + params<{ jobId: string }>({ jobId: job.id }), + ); + const body = (await response.json()) as { error: string }; + + expect(response.status).toBe(502); + expect(body.error).toBe("LLM response failed schema validation: body_markdown"); + }); + it("rejects unknown export filenames", async () => { const { job } = await createJobFixture(); const exportDir = join(tempDir, "exports", job.id); @@ -198,6 +281,8 @@ describe("job API routes", () => { }); async function createJobFixture() { + llmMocks.generateValidatedJson.mockResolvedValueOnce(validCandidateFactCard); + const response = await createJob( request({ title: "Example Technology Co., Ltd. GEO guide", diff --git a/src/app/api/jobs/[jobId]/optimize/route.ts b/src/app/api/jobs/[jobId]/optimize/route.ts index efd59f3..ce1df36 100644 --- a/src/app/api/jobs/[jobId]/optimize/route.ts +++ b/src/app/api/jobs/[jobId]/optimize/route.ts @@ -2,6 +2,7 @@ import { NextResponse } from "next/server"; import { requireApiAccess } from "../../../../../lib/api/auth"; import { getRepositoryFromRuntime } from "../../../../../lib/db/repository"; +import { LlmValidationError } from "../../../../../lib/llm/client"; import { getExportStoreFromRuntime } from "../../../../../lib/workflow/export-store"; import { runOptimizationWorkflow } from "../../../../../lib/workflow/orchestrator"; @@ -30,41 +31,55 @@ export async function POST(request: Request, context: RouteContext) { ); } - const result = await runOptimizationWorkflow({ - input: { - title: job.source_title, - body: job.source_body, - images: job.image_inputs, - platform: job.publish_platform, - user_instructions: job.user_instructions, - }, - factCard: factCardRecord, - }); - const optimizedArticle = await repository.saveOptimizedArticle(jobId, result.article); - const qaReport = await repository.saveQaReport( - jobId, - optimizedArticle.revision ?? 1, - result.qaReport, - ); - const exportStore = getExportStoreFromRuntime(); - const exportPaths = - qaReport.overall_status === "fail" - ? {} - : await exportStore.writeJobExports({ - jobId, - article: optimizedArticle, - qaReport, - }); - await repository.updateArticleJob(jobId, { - status: qaReport.overall_status === "fail" ? "qa_failed" : "optimized", - export_paths: exportPaths, - }); + try { + const result = await runOptimizationWorkflow({ + input: { + title: job.source_title, + body: job.source_body, + images: job.image_inputs, + platform: job.publish_platform, + user_instructions: job.user_instructions, + }, + factCard: factCardRecord, + }); + const optimizedArticle = await repository.saveOptimizedArticle( + jobId, + result.article, + ); + const qaReport = await repository.saveQaReport( + jobId, + optimizedArticle.revision ?? 1, + result.qaReport, + ); + const exportStore = getExportStoreFromRuntime(); + const exportPaths = + qaReport.overall_status === "fail" + ? {} + : await exportStore.writeJobExports({ + jobId, + article: optimizedArticle, + qaReport, + }); + await repository.updateArticleJob(jobId, { + status: qaReport.overall_status === "fail" ? "qa_failed" : "optimized", + export_paths: exportPaths, + }); - return NextResponse.json({ - optimizedArticle, - qaReport, - exportPaths, - rewriteRounds: result.rewrite_rounds, - stoppedAfterMaxRewrites: result.stopped_after_max_rewrites, - }); + return NextResponse.json({ + optimizedArticle, + qaReport, + exportPaths, + rewriteRounds: result.rewrite_rounds, + stoppedAfterMaxRewrites: result.stopped_after_max_rewrites, + }); + } catch (error) { + const message = error instanceof Error ? error.message : "LLM optimization failed"; + return NextResponse.json({ error: message }, { status: getErrorStatus(error) }); + } +} + +function getErrorStatus(error: unknown) { + if (error instanceof LlmValidationError) return 502; + if (error instanceof Error && /^LLM\b|provider/i.test(error.message)) return 502; + return 500; } diff --git a/src/app/api/jobs/route.ts b/src/app/api/jobs/route.ts index 1779e1c..200274e 100644 --- a/src/app/api/jobs/route.ts +++ b/src/app/api/jobs/route.ts @@ -2,6 +2,7 @@ import { NextResponse } from "next/server"; import { requireApiAccess } from "../../../lib/api/auth"; import { getRepositoryFromRuntime } from "../../../lib/db/repository"; +import { LlmValidationError } from "../../../lib/llm/client"; import { extractCandidateFactCard } from "../../../lib/workflow/fact-extractor"; import { normalizeInput, type RawArticleInput } from "../../../lib/workflow/input-normalizer"; @@ -26,7 +27,7 @@ export async function POST(request: Request) { return NextResponse.json({ job, candidateFactCard }, { status: 201 }); } catch (error) { - return jsonError(error, 400); + return jsonError(error, getErrorStatus(error)); } } @@ -34,3 +35,9 @@ function jsonError(error: unknown, status: number) { const message = error instanceof Error ? error.message : "Request failed"; return NextResponse.json({ error: message }, { status }); } + +function getErrorStatus(error: unknown) { + if (error instanceof LlmValidationError) return 502; + if (error instanceof Error && /^LLM\b|provider/i.test(error.message)) return 502; + return 400; +} diff --git a/src/lib/llm/__tests__/client.test.ts b/src/lib/llm/__tests__/client.test.ts index ca8d6e2..96c3a9e 100644 --- a/src/lib/llm/__tests__/client.test.ts +++ b/src/lib/llm/__tests__/client.test.ts @@ -16,16 +16,16 @@ describe("generateValidatedJson", () => { vi.restoreAllMocks(); }); - it("returns null when no provider key is configured", async () => { + it("throws clearly when no provider key is configured", async () => { process.env.LLM_PROVIDER = "deepseek"; delete process.env.DEEPSEEK_API_KEY; - const result = await client.generateValidatedJson({ - schema: z.object({ value: z.string() }), - prompt: "Return JSON.", - }); - - expect(result).toBeNull(); + await expect( + client.generateValidatedJson({ + schema: z.object({ value: z.string() }), + prompt: "Return JSON.", + }), + ).rejects.toThrow("DEEPSEEK_API_KEY is missing"); }); it("returns parsed data when the model response matches the schema", async () => { @@ -41,32 +41,32 @@ describe("generateValidatedJson", () => { expect(result).toEqual({ value: "from-llm" }); }); - it("returns null when the model response fails schema validation", async () => { + it("throws clearly when the model response fails schema validation", async () => { process.env.LLM_PROVIDER = "deepseek"; process.env.DEEPSEEK_API_KEY = "test-key"; client.setGenerateJsonForValidation(async () => ({ value: 42 })); - const result = await client.generateValidatedJson({ - schema: z.object({ value: z.string() }), - prompt: "Return JSON.", - }); - - expect(result).toBeNull(); + await expect( + client.generateValidatedJson({ + schema: z.object({ value: z.string() }), + prompt: "Return JSON.", + }), + ).rejects.toThrow("LLM response failed schema validation"); }); - it("returns null when the provider call rejects", async () => { + it("throws clearly when the provider call rejects", async () => { process.env.LLM_PROVIDER = "deepseek"; process.env.DEEPSEEK_API_KEY = "test-key"; client.setGenerateJsonForValidation(async () => { throw new Error("provider down"); }); - const result = await client.generateValidatedJson({ - schema: z.object({ value: z.string() }), - prompt: "Return JSON.", - }); - - expect(result).toBeNull(); + await expect( + client.generateValidatedJson({ + schema: z.object({ value: z.string() }), + prompt: "Return JSON.", + }), + ).rejects.toThrow("provider down"); }); it("logs validation success with the supplied task label", async () => { @@ -93,13 +93,13 @@ describe("generateValidatedJson", () => { process.env.DEEPSEEK_API_KEY = "test-key"; client.setGenerateJsonForValidation(async () => ({ value: 42 })); - const result = await client.generateValidatedJson({ - schema: z.object({ value: z.string() }), - prompt: "Return JSON.", - task: "fact_extractor", - }); - - expect(result).toBeNull(); + await expect( + client.generateValidatedJson({ + schema: z.object({ value: z.string() }), + prompt: "Return JSON.", + task: "fact_extractor", + }), + ).rejects.toThrow("LLM response failed schema validation"); expect(warnSpy).toHaveBeenCalledWith( expect.stringContaining("[llm:validated] task=fact_extractor ok=false"), ); @@ -150,16 +150,16 @@ describe("generateValidatedJson", () => { const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); client.setGenerateJsonForValidation(async () => ({ value: 42 })); - const result = await client.generateValidatedJson({ - schema: z.object({ value: z.string() }), - task: "fact_extractor", - prompt: "Return JSON.", - }); - + await expect( + client.generateValidatedJson({ + schema: z.object({ value: z.string() }), + task: "fact_extractor", + prompt: "Return JSON.", + }), + ).rejects.toThrow("LLM response failed schema validation"); const allLogs = [...infoSpy.mock.calls, ...warnSpy.mock.calls] .flat() .join("\n"); - expect(result).toBeNull(); expect(allLogs).toContain("[llm:validated] task=fact_extractor ok=false"); expect(allLogs).toContain("zod_error="); expect(allLogs).not.toContain("super-secret-key"); @@ -173,13 +173,13 @@ describe("generateValidatedJson", () => { throw new Error("provider unavailable"); }); - const result = await client.generateValidatedJson({ - schema: z.object({ value: z.string() }), - task: "quality_inspector", - prompt: "Return JSON.", - }); - - expect(result).toBeNull(); + await expect( + client.generateValidatedJson({ + schema: z.object({ value: z.string() }), + task: "quality_inspector", + prompt: "Return JSON.", + }), + ).rejects.toThrow("provider unavailable"); expect(warnSpy).toHaveBeenCalledWith( expect.stringContaining("[llm:error] task=quality_inspector"), ); diff --git a/src/lib/llm/client.ts b/src/lib/llm/client.ts index 2e51ec3..e1e1168 100644 --- a/src/lib/llm/client.ts +++ b/src/lib/llm/client.ts @@ -28,6 +28,16 @@ export interface LlmProviderStatus { reason?: string; } +export class LlmValidationError extends Error { + constructor( + message: string, + public readonly task: LlmTaskName, + ) { + super(message); + this.name = "LlmValidationError"; + } +} + function getProvider() { return (process.env.LLM_PROVIDER || "deepseek").toLowerCase(); } @@ -80,8 +90,8 @@ export function setChatCompletionForTesting( chatCompletionForTesting = handler; } -function getTask(input: GenerateInput) { - return input.task?.trim() || "unknown"; +function getTask(input: GenerateInput): LlmTaskName { + return input.task || "unknown"; } function getRawLogLimit() { @@ -204,11 +214,14 @@ export function setGenerateJsonForValidation( export async function generateValidatedJson({ schema, ...input -}: GenerateValidatedJsonInput): Promise { +}: GenerateValidatedJsonInput): Promise { const task = getTask(input); if (!isLlmConfigured()) { console.info(`[llm:validated] task=${task} ok=false reason=not_configured`); - return null; + throw new LlmValidationError( + getLlmProviderStatus().reason ?? "LLM provider is not configured", + task, + ); } const status = getLlmProviderStatus(); @@ -230,14 +243,20 @@ export async function generateValidatedJson({ console.warn( `[llm:validated] task=${task} ok=false zod_error=${quoteLogValue(summarizeZodError(parsed.error))}`, ); - return null; + throw new LlmValidationError( + `LLM response failed schema validation: ${summarizeZodError(parsed.error)}`, + task, + ); } catch (error) { + if (error instanceof LlmValidationError) { + throw error; + } console.info(`[llm:validated] task=${task} ok=false reason=provider_error`); const message = error instanceof Error ? error.message : String(error); console.warn( `[llm:error] task=${task} duration_ms=${Date.now() - startedAt} message=${quoteLogValue(message)}`, ); - return null; + throw error instanceof Error ? error : new Error(message); } } diff --git a/src/lib/llm/prompts.ts b/src/lib/llm/prompts.ts index 3148ed8..95d0dff 100644 --- a/src/lib/llm/prompts.ts +++ b/src/lib/llm/prompts.ts @@ -10,7 +10,7 @@ export const JSON_ONLY_PROMPT = "Return valid JSON only. Do not include markdown fences or commentary."; const CUSTOMER_RISK_GUIDANCE = [ - "客户最担心的内容风险：行业漂移、公司全称/简称/品牌名不一致、图片主题与正文描述不匹配、官网文章出现第三方口吻、平台语气和文章类型不匹配、标题或正文语义不顺、虚构资质/年限/案例/能力、产品/服务/年限前后冲突。", + "客户最担心的内容风险：行业漂移、公司全称/简称/品牌名不一致、官网文章出现第三方口吻、平台语气和文章类型不匹配、标题或正文语义不顺、虚构资质/年限/案例/能力、产品/服务/年限前后冲突。", "任何客户案例、资质荣誉、经验年限、服务能力、出海/多语种/合规能力、效果承诺和排名，都必须能从原文或已确认事实卡中找到明确依据。", ].join(" "); @@ -99,7 +99,7 @@ export function buildArticleOptimizerPrompt( "- 必须保留事实卡确认的公司全称、目标行业、目标受众和核心事实。", "- 必须删除或弱化 factCard.forbidden_claims 中的主张。", "- 不得新增客户案例、数字、资质、排名、奖项、服务能力、效果承诺。", - "- image_suggestions 必须基于 factCard.image_topics 或原始 images；没有图片主题时返回空数组。", + "- 当前版本只优化文本，不生成图片建议；image_suggestions 必须返回空数组 []。", "", "Confirmed fact card:", JSON.stringify(factCard, null, 2), @@ -124,8 +124,9 @@ export function buildQualityInspectorPrompt(input: { formatPlatformGuidance(input.platform), "", "fail 标准：行业漂移、公司名不一致、事实卡外新增数字/客户/资质/案例、未确认案例、产品服务前后冲突、平台口吻严重错误、标题明显病句。", - "warn 标准：图片证据不足、句子过长、表达可读性一般、平台适配轻微不足。", - "target_agent 只能使用 title、body、image、fact_card 或 null。", + "warn 标准：句子过长、表达可读性一般、平台适配轻微不足。", + "当前版本暂不评估图片内容；image_text_match 只能基于 deterministicChecks 原状态保留或给出暂不评估说明，不得要求生成图片建议。", + "target_agent 只能使用 title、body、fact_card 或 null。", "", "Confirmed fact card:", JSON.stringify(input.factCard, null, 2), @@ -155,7 +156,7 @@ export function buildTargetedRewritePrompt(input: { "- title_quality：生成自然中文标题，禁止英文模板词。", "- body_quality：拆分长句，修复病句和断裂表达。", "- voice_consistency / platform_fit：改成目标平台对应口吻。", - "- image_text_match：只补充图片建议或人工确认项，不虚构图片内容。", + "- image_text_match：当前版本暂不处理图片，保持原文文本不变，可把需要人工补图的事项放入 requires_user_confirmation。", "不得新增事实。无法修复的内容放入 requires_user_confirmation。", "", "Confirmed fact card:", diff --git a/src/lib/workflow/__tests__/llm-integration.test.ts b/src/lib/workflow/__tests__/llm-integration.test.ts index 698dd0c..767df3c 100644 --- a/src/lib/workflow/__tests__/llm-integration.test.ts +++ b/src/lib/workflow/__tests__/llm-integration.test.ts @@ -72,19 +72,20 @@ describe("LLM workflow integration", () => { ); }); - it("falls back to deterministic candidate extraction when LLM returns null", async () => { - llmMocks.generateValidatedJson.mockResolvedValueOnce(null); + it("surfaces candidate fact extraction LLM failures instead of falling back", async () => { + llmMocks.generateValidatedJson.mockRejectedValueOnce( + new Error("LLM response failed schema validation: target_audience"), + ); - const card = await extractCandidateFactCard({ - title: "Fallback Technology Co., Ltd. GEO guide", - body: "Fallback Technology Co., Ltd. has 8 years of GEO optimization experience.", - images: [{ type: "description", content: "dashboard" }], - platform: "official_site", - user_instructions: "", - }); - - expect(card.company_full_name).toBe("Fallback Technology Co., Ltd."); - expect(card.experience_years).toBe(8); + await expect( + extractCandidateFactCard({ + title: "Fallback Technology Co., Ltd. GEO guide", + body: "Fallback Technology Co., Ltd. has 8 years of GEO optimization experience.", + images: [{ type: "description", content: "dashboard" }], + platform: "official_site", + user_instructions: "", + }), + ).rejects.toThrow("LLM response failed schema validation: target_audience"); }); it("uses LLM output for article optimization when valid", async () => { @@ -92,7 +93,7 @@ describe("LLM workflow integration", () => { title: "LLM Optimized GEO Article", summary: "LLM summary constrained by the fact card.", body_markdown: "## LLM Body\nExample Technology Co., Ltd. keeps claims factual.", - image_suggestions: [{ source: "image_1", suggestion: "Use dashboard." }], + image_suggestions: [], changed_sections: ["title", "body"], requires_user_confirmation: [], }); @@ -110,29 +111,31 @@ describe("LLM workflow integration", () => { expect(article.title).toBe("LLM Optimized GEO Article"); expect(article.body_markdown).toContain("LLM Body"); + expect(article.image_suggestions).toEqual([]); expect(llmMocks.generateValidatedJson).toHaveBeenCalledOnce(); expect(llmMocks.generateValidatedJson).toHaveBeenCalledWith( expect.objectContaining({ task: "article_optimizer" }), ); }); - it("falls back to deterministic article optimization when LLM returns null", async () => { - llmMocks.generateValidatedJson.mockResolvedValueOnce(null); + it("surfaces article optimization LLM failures instead of falling back", async () => { + llmMocks.generateValidatedJson.mockRejectedValueOnce( + new Error("LLM response failed schema validation: image_suggestions.0.source"), + ); - const article = await optimizeArticle({ - input: { - title: "Original", - body: "Example Technology Co., Ltd. has 8 years of GEO optimization experience.", - images: [{ type: "description", content: "dashboard" }], - platform: "official_site", - user_instructions: "Say we have 99 patents.", - }, - factCard: confirmedFactCard, - }); - - expect(article.title).toContain("GEO optimization Guide"); - expect(article.requires_user_confirmation).toContain( - "Unsupported requested claim: 99 patents", + await expect( + optimizeArticle({ + input: { + title: "Original", + body: "Example Technology Co., Ltd. has 8 years of GEO optimization experience.", + images: [{ type: "description", content: "dashboard" }], + platform: "official_site", + user_instructions: "Say we have 99 patents.", + }, + factCard: confirmedFactCard, + }), + ).rejects.toThrow( + "LLM response failed schema validation: image_suggestions.0.source", ); }); @@ -176,33 +179,32 @@ describe("LLM workflow integration", () => { ); }); - it("falls back to deterministic targeted rewrite when LLM returns null", async () => { - llmMocks.generateValidatedJson.mockResolvedValueOnce(null); + it("surfaces targeted rewrite LLM failures instead of falling back", async () => { + llmMocks.generateValidatedJson.mockRejectedValueOnce(new Error("provider unavailable")); - const rewritten = await rewriteFailedSections({ - article: { - title: "Bad title!!!", - summary: "Original summary", - body_markdown: "## Body\nOriginal body", - image_suggestions: [], - changed_sections: [], - requires_user_confirmation: [], - }, - factCard: confirmedFactCard, - failedChecks: [ - { - rule_id: "title_quality", - status: "fail", - evidence: "Bad title!!!", - reason: "Title has punctuation stuffing.", - suggested_fix: "Rewrite title.", - target_agent: "title", + await expect( + rewriteFailedSections({ + article: { + title: "Bad title!!!", + summary: "Original summary", + body_markdown: "## Body\nOriginal body", + image_suggestions: [], + changed_sections: [], + requires_user_confirmation: [], }, - ], - }); - - expect(rewritten.title).toContain("GEO optimization Guide"); - expect(rewritten.summary).toBe("Original summary"); + factCard: confirmedFactCard, + failedChecks: [ + { + rule_id: "title_quality", + status: "fail", + evidence: "Bad title!!!", + reason: "Title has punctuation stuffing.", + suggested_fix: "Rewrite title.", + target_agent: "title", + }, + ], + }), + ).rejects.toThrow("provider unavailable"); }); it("uses LLM quality checks to enrich non-failing deterministic checks", async () => { diff --git a/src/lib/workflow/__tests__/workflow.test.ts b/src/lib/workflow/__tests__/workflow.test.ts index 89185c6..5e16dbf 100644 --- a/src/lib/workflow/__tests__/workflow.test.ts +++ b/src/lib/workflow/__tests__/workflow.test.ts @@ -1,12 +1,8 @@ import { describe, expect, it } from "vitest"; import type { ConfirmedFactCard } from "../../domain/types"; -import { optimizeArticle } from "../article-optimizer"; -import { extractCandidateFactCard } from "../fact-extractor"; import { normalizeInput } from "../input-normalizer"; import { inspectQuality } from "../quality-inspector"; -import { runOptimizationWorkflow } from "../orchestrator"; -import { rewriteFailedSections } from "../targeted-rewriter"; const confirmedFactCard: ConfirmedFactCard = { company_full_name: "Example Technology Co., Ltd.", @@ -45,65 +41,6 @@ describe("workflow nodes", () => { ]); }); - it("places missing or conflicting company facts into uncertain items", async () => { - const card = await extractCandidateFactCard({ - title: "Example announces GEO product", - body: "Example has 8 years of experience. Example has 12 years of service. The article discusses GEO optimization.", - images: [], - platform: "media_article", - user_instructions: "", - }); - - expect(card.company_full_name).toBe(""); - expect(card.uncertain_items).toEqual( - expect.arrayContaining([ - expect.stringContaining("company full name"), - expect.stringContaining("Conflicting experience years"), - ]), - ); - expect(card.is_ready_for_optimization).toBe(false); - }); - - it("extracts Chinese company facts from Chinese articles", async () => { - const card = await extractCandidateFactCard({ - title: "#探寻AIGC短视频培训选哪家，各品牌实力大比拼", - body: "伟思德鲁管理咨询（深圳）有限公司面向品牌商家和出海企业提供AIGC短视频培训服务，帮助企业解决内容工业化生产、品牌视觉统一和全球化传播问题。", - images: [{ type: "description", content: "AIGC短视频工作流示意图" }], - platform: "media_article", - user_instructions: "保留AIGC短视频培训与出海内容生产场景。", - }); - - expect(card.company_full_name).toBe("伟思德鲁管理咨询（深圳）有限公司"); - expect(card.company_short_names).toContain("伟思德鲁"); - expect(card.target_industry).toBe("AIGC短视频培训"); - expect(card.target_audience).toBe("品牌商家、内容创作者、出海企业"); - expect(card.image_topics).toEqual(["AIGC短视频工作流示意图"]); - expect(card.uncertain_items).not.toContain("Missing company full name"); - }); - - it("does not add claims outside the confirmed fact card", async () => { - const optimized = await optimizeArticle({ - input: { - title: "Example GEO article", - body: "Example GEO helps marketing teams improve content structure.", - images: [], - platform: "official_site", - user_instructions: - "Say we have 99 patents and Fortune 500 customer cases.", - }, - factCard: confirmedFactCard, - }); - - expect(optimized.body_markdown).not.toContain("99 patents"); - expect(optimized.body_markdown).not.toContain("Fortune 500"); - expect(optimized.requires_user_confirmation).toEqual( - expect.arrayContaining([ - expect.stringContaining("99 patents"), - expect.stringContaining("Fortune 500"), - ]), - ); - }); - it("returns the 10 required quality checks", () => { const report = inspectQuality({ article: { @@ -183,50 +120,4 @@ describe("workflow nodes", () => { expect(report.overall_status).toBe("fail"); }); - it("rewrites only the failing target area", async () => { - const article = { - title: "Bad title!!!", - summary: "Original summary", - body_markdown: "Original body", - image_suggestions: [], - changed_sections: [], - requires_user_confirmation: [], - }; - - const rewritten = await rewriteFailedSections({ - article, - factCard: confirmedFactCard, - failedChecks: [ - { - rule_id: "title_quality", - status: "fail", - evidence: "Bad title!!!", - reason: "Punctuation stuffing.", - suggested_fix: "Rewrite title.", - target_agent: "title", - }, - ], - }); - - expect(rewritten.title).not.toBe(article.title); - expect(rewritten.summary).toBe(article.summary); - expect(rewritten.body_markdown).toBe(article.body_markdown); - }); - - it("orchestrator stops after two failed rewrite rounds", async () => { - const result = await runOptimizationWorkflow({ - input: { - title: "Finance automation breakthrough!!!", - body: "Example has 12 years in finance automation and 99 patents.", - images: [], - platform: "official_site", - user_instructions: "", - }, - factCard: confirmedFactCard, - }); - - expect(result.rewrite_rounds).toBe(2); - expect(result.qaReport.overall_status).toBe("fail"); - expect(result.stopped_after_max_rewrites).toBe(true); - }); }); diff --git a/src/lib/workflow/article-optimizer.ts b/src/lib/workflow/article-optimizer.ts index 8c46d0a..9be8d29 100644 --- a/src/lib/workflow/article-optimizer.ts +++ b/src/lib/workflow/article-optimizer.ts @@ -27,77 +27,8 @@ export async function optimizeArticle({ task: "article_optimizer", }); - return llmArticle ?? optimizeArticleFallback({ input, factCard }); -} - -function optimizeArticleFallback({ - input, - factCard, -}: OptimizeArticleInput): OptimizedArticle { - const unsupported = findUnsupportedInstructionClaims( - input.user_instructions, - factCard, - ); - const title = `${factCard.brand_names[0] ?? factCard.company_short_names[0] ?? factCard.company_full_name} ${factCard.target_industry} Guide`; - const coreClaims = - factCard.core_claims.length > 0 - ? factCard.core_claims.map((claim) => `- ${claim}`).join("\n") - : "- Confirmed facts only; no extra claims added."; - const body = [ - `## ${factCard.company_full_name}`, - cleanBody(input.body, factCard), - "", - "### Confirmed Facts", - coreClaims, - ].join("\n"); - return optimizedArticleSchema.parse({ - title, - summary: `A ${input.platform.replace(/_/g, " ")} article for ${factCard.target_audience} about ${factCard.target_industry}.`, - body_markdown: body, - image_suggestions: factCard.image_topics.map((topic, index) => ({ - source: `image_${index + 1}`, - suggestion: `Use image content related to ${topic}.`, - })), - changed_sections: ["title", "body structure", "summary"], - requires_user_confirmation: unsupported, + ...llmArticle, + image_suggestions: [], }); } - -function cleanBody(body: string, factCard: ConfirmedFactCard) { - let cleaned = body.trim(); - for (const forbidden of factCard.forbidden_claims) { - cleaned = cleaned.replace(new RegExp(escapeRegExp(forbidden), "gi"), ""); - } - return cleaned; -} - -function findUnsupportedInstructionClaims( - instructions: string, - factCard: ConfirmedFactCard, -) { - const unsupported: string[] = []; - const numbers = [...instructions.matchAll(/\b\d+\s*[A-Za-z]+\b/g)].map( - (match) => match[0], - ); - const knownText = [ - factCard.experience_years?.toString() ?? "", - ...factCard.core_claims, - ].join(" "); - - for (const claim of numbers) { - if (!knownText.includes(claim.replace(/\D/g, ""))) { - unsupported.push(`Unsupported requested claim: ${claim}`); - } - } - - if (/fortune\s*500/i.test(instructions)) { - unsupported.push("Unsupported requested claim: Fortune 500 customer cases"); - } - - return unsupported; -} - -function escapeRegExp(value: string) { - return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); -} diff --git a/src/lib/workflow/fact-extractor.ts b/src/lib/workflow/fact-extractor.ts index 1732134..ce5f7eb 100644 --- a/src/lib/workflow/fact-extractor.ts +++ b/src/lib/workflow/fact-extractor.ts @@ -9,131 +9,11 @@ import { export async function extractCandidateFactCard( input: ArticleInput, ): Promise { - const llmCard = await generateValidatedJson({ + return generateValidatedJson({ schema: candidateFactCardSchema, system: FACT_EXTRACTOR_SYSTEM_PROMPT, prompt: buildFactExtractorPrompt(input), temperature: 0.1, task: "fact_extractor", }); - - return llmCard ?? extractCandidateFactCardFallback(input); -} - -function extractCandidateFactCardFallback(input: ArticleInput): CandidateFactCard { - const text = `${input.title}\n${input.body}`; - const uncertainItems: string[] = []; - const companyFullName = findCompanyFullName(text); - const years = findExperienceYears(text); - - if (!companyFullName) { - uncertainItems.push("Missing company full name"); - } - if (years.length > 1) { - uncertainItems.push(`Conflicting experience years: ${years.join(", ")}`); - } - if (input.images.length === 0) { - uncertainItems.push("Image description is missing"); - } - - const industry = inferIndustry(text); - - return candidateFactCardSchema.parse({ - company_full_name: companyFullName ?? "", - company_short_names: inferCompanyShortNames(companyFullName), - brand_names: inferBrandNames(text, companyFullName), - product_names: inferProducts(text), - target_industry: industry, - target_audience: text.toLowerCase().includes("marketing") - ? "Marketing teams" - : inferTargetAudience(text), - experience_years: years.length === 1 ? years[0] : null, - core_claims: years.length === 1 ? [`${years[0]} years of ${industry} experience`] : [], - forbidden_claims: [], - image_topics: input.images.map((image) => image.content), - uncertain_items: uncertainItems, - }); -} - -function findCompanyFullName(text: string) { - const englishMatch = text.match( - /([A-Z][A-Za-z0-9&.,\-\s]{2,}?(?:Co\.,?\s*Ltd\.?|Company|Inc\.?|LLC|Ltd\.))/, - ); - if (englishMatch?.[1]) return englishMatch[1].trim(); - - const chineseMatch = text.match( - /([\u4e00-\u9fa5A-Za-z0-9（）()]{2,40}?(?:股份有限公司|有限公司|集团|公司))/, - ); - return chineseMatch?.[1].trim() ?? null; -} - -function findExperienceYears(text: string) { - const matches = [...text.matchAll(/\b(\d{1,3})\s*(?:years?|年)\b/gi)]; - return [...new Set(matches.map((match) => Number(match[1])))]; -} - -function inferIndustry(text: string) { - const lower = text.toLowerCase(); - if (lower.includes("geo")) return "GEO optimization"; - if (lower.includes("finance") || lower.includes("banking")) return "finance automation"; - if (lower.includes("seo")) return "SEO"; - if (/AIGC|短视频|出海内容|内容工业化/.test(text)) return "AIGC短视频培训"; - if (/工业零部件|爆品操盘|AI OBS|IPMS/.test(text)) return "工业零部件爆品操盘"; - return "General business"; -} - -function inferBrandNames(text: string, companyFullName: string | null) { - const chineseBrands = [ - ...new Set( - [ - companyFullName ? inferChineseShortName(companyFullName) : "", - ...[...text.matchAll(/\b(AIGC|AI OBS|IPMS|GEO|SEO)\b/g)].map( - (match) => match[1], - ), - ].filter(Boolean), - ), - ]; - const names = [...text.matchAll(/\b[A-Z][A-Za-z0-9]{2,}\b/g)] - .map((match) => match[0]) - .filter((word) => !["The", "This", "And"].includes(word)); - return [...new Set([...chineseBrands, ...names])].slice(0, 5); -} - -function inferProducts(text: string) { - const productMatches = [ - ...[...text.matchAll(/\b([A-Z][A-Za-z0-9]+\s+GEO)\b/g)].map( - (match) => match[1], - ), - ...[...text.matchAll(/\b(AIGC短视频培训|AI OBS|IPMS|爆品操盘数智系统)\b/g)].map( - (match) => match[1], - ), - ]; - return [...new Set(productMatches)]; -} - -function inferCompanyShortNames(companyFullName: string | null) { - if (!companyFullName) return []; - if (/[\u4e00-\u9fa5]/.test(companyFullName)) { - const legalShortName = inferChineseShortName(companyFullName); - const brandShortName = legalShortName.replace(/管理咨询$/, ""); - return [...new Set([brandShortName, legalShortName].filter(Boolean))]; - } - return [companyFullName.split(/\s+/)[0] ?? ""].filter(Boolean); -} - -function inferChineseShortName(companyFullName: string) { - return companyFullName - .replace(/[（(].*?[）)]/g, "") - .replace(/股份有限公司|有限公司|集团|公司/g, "") - .trim(); -} - -function inferTargetAudience(text: string) { - if (/品牌商家|内容创作者|出海企业/.test(text)) { - return "品牌商家、内容创作者、出海企业"; - } - if (/工业企业|工业零部件制造商|采购/.test(text)) { - return "工业企业、采购团队、工业零部件制造商"; - } - return "Business readers"; } diff --git a/src/lib/workflow/quality-inspector.ts b/src/lib/workflow/quality-inspector.ts index fda2c75..4ac98e8 100644 --- a/src/lib/workflow/quality-inspector.ts +++ b/src/lib/workflow/quality-inspector.ts @@ -68,10 +68,6 @@ export async function inspectQualityWithLlm( task: "quality_inspector", }); - if (!llmPatch) { - return deterministicReport; - } - const patchedChecks = deterministicReport.checks.map((deterministicCheck) => { const llmCheck = llmPatch.checks.find( (check) => check.rule_id === deterministicCheck.rule_id, @@ -125,15 +121,12 @@ function inspectRule( } if (ruleId === "image_text_match") { - const hasImages = sourceImages.length > 0 || article.image_suggestions.length > 0; return check( ruleId, - hasImages ? "pass" : "warn", - hasImages ? "已有可用于比对的图片主题。" : "未提供图片描述。", - hasImages - ? "图片建议可以和文章内容进行比对。" - : "缺少图片描述时，图文匹配置信度较低。", - "补充图片描述，或人工检查图片与正文的对应关系。", + "pass", + sourceImages.length > 0 ? "当前版本暂不评估图片内容。" : "当前版本未启用图片分析。", + "当前版本仅优化文本，图片匹配检查暂不参与质量门禁。", + "后续启用图片工作流后再补充图文匹配检查。", null, ); } diff --git a/src/lib/workflow/targeted-rewriter.ts b/src/lib/workflow/targeted-rewriter.ts index 7a4293a..92dcf19 100644 --- a/src/lib/workflow/targeted-rewriter.ts +++ b/src/lib/workflow/targeted-rewriter.ts @@ -25,46 +25,8 @@ export async function rewriteFailedSections({ task: "targeted_rewriter", }); - return llmArticle ?? rewriteFailedSectionsFallback({ article, factCard, failedChecks }); -} - -function rewriteFailedSectionsFallback({ - article, - factCard, - failedChecks, -}: RewriteFailedSectionsInput): OptimizedArticle { - let rewritten = { ...article }; - - for (const check of failedChecks) { - if (check.target_agent === "title") { - rewritten = { - ...rewritten, - title: `${factCard.brand_names[0] ?? factCard.company_short_names[0]} ${factCard.target_industry} Guide`, - changed_sections: [...new Set([...rewritten.changed_sections, "title"])], - }; - } - - if (check.target_agent === "body" && check.rule_id === "company_name_integrity") { - rewritten = { - ...rewritten, - body_markdown: `${factCard.company_full_name}\n\n${rewritten.body_markdown}`, - changed_sections: [...new Set([...rewritten.changed_sections, "company name"])], - }; - } - - if (check.target_agent === "body" && check.rule_id === "claim_consistency") { - rewritten = { - ...rewritten, - body_markdown: rewritten.body_markdown.replace( - /\b\d{1,3}\s*(?:years?|年)\b/gi, - factCard.experience_years === null - ? "confirmed experience" - : `${factCard.experience_years} years`, - ), - changed_sections: [...new Set([...rewritten.changed_sections, "claim consistency"])], - }; - } - } - - return rewritten; + return optimizedArticleSchema.parse({ + ...llmArticle, + image_suggestions: [], + }); }