fix: surface llm workflow errors

This commit is contained in:
Codex
2026-06-21 23:43:03 +08:00
parent 4bb5ca5eb0
commit f1dac15400
12 changed files with 286 additions and 500 deletions
+87 -2
View File
@@ -39,6 +39,21 @@ const validFactCard = {
confirmed_by_user: true, confirmed_by_user: true,
}; };
const validCandidateFactCard = {
company_full_name: validFactCard.company_full_name,
company_short_names: validFactCard.company_short_names,
brand_names: validFactCard.brand_names,
product_names: validFactCard.product_names,
target_industry: validFactCard.target_industry,
target_audience: validFactCard.target_audience,
experience_years: validFactCard.experience_years,
core_claims: validFactCard.core_claims,
forbidden_claims: validFactCard.forbidden_claims,
image_topics: validFactCard.image_topics,
uncertain_items: validFactCard.uncertain_items,
is_ready_for_optimization: true,
};
interface CreateJobResponse { interface CreateJobResponse {
job: { id: string }; job: { id: string };
candidateFactCard: { company_full_name: string }; candidateFactCard: { company_full_name: string };
@@ -86,6 +101,8 @@ describe("job API routes", () => {
}); });
it("validates input, creates a job, and returns a candidate fact card", async () => { it("validates input, creates a job, and returns a candidate fact card", async () => {
llmMocks.generateValidatedJson.mockResolvedValueOnce(validCandidateFactCard);
const response = await createJob( const response = await createJob(
request({ request({
title: "Example Technology Co., Ltd. GEO guide", title: "Example Technology Co., Ltd. GEO guide",
@@ -104,6 +121,38 @@ describe("job API routes", () => {
); );
}); });
it("returns a clear error when LLM fact extraction fails", async () => {
llmMocks.generateValidatedJson.mockRejectedValueOnce(
new Error("LLM response failed schema validation: target_audience"),
);
const response = await createJob(
request({
title: "Example Technology Co., Ltd. GEO guide",
body: "Example Technology Co., Ltd. has 8 years of GEO optimization experience.",
platform: "official_site",
}),
);
const body = (await response.json()) as { error: string };
expect(response.status).toBe(502);
expect(body.error).toBe(
"LLM response failed schema validation: target_audience",
);
});
it("still returns 400 for invalid article input", async () => {
const response = await createJob(
request({
title: "",
body: "",
platform: "official_site",
}),
);
expect(response.status).toBe(400);
});
it("rejects unresolved uncertain items when confirming a fact card", async () => { it("rejects unresolved uncertain items when confirming a fact card", async () => {
const { job } = await createJobFixture(); const { job } = await createJobFixture();
const response = await confirmFactCard( const response = await confirmFactCard(
@@ -135,6 +184,19 @@ describe("job API routes", () => {
params<{ jobId: string }>({ jobId: job.id }), params<{ jobId: string }>({ jobId: job.id }),
); );
llmMocks.generateValidatedJson
.mockResolvedValueOnce({
title: "API LLM Optimized GEO Article",
summary:
"A official site article for Marketing teams about GEO optimization.",
body_markdown:
"Example Technology Co., Ltd. has 8 years of GEO optimization experience.",
image_suggestions: [],
changed_sections: ["title", "body"],
requires_user_confirmation: [],
})
.mockResolvedValueOnce({ checks: [] });
const response = await optimizeJob( const response = await optimizeJob(
request({}), request({}),
params<{ jobId: string }>({ jobId: job.id }), params<{ jobId: string }>({ jobId: job.id }),
@@ -142,7 +204,7 @@ describe("job API routes", () => {
const body = (await response.json()) as OptimizeJobResponse; const body = (await response.json()) as OptimizeJobResponse;
expect(response.status).toBe(200); expect(response.status).toBe(200);
expect(body.optimizedArticle.title).toContain("GEO optimization"); expect(body.optimizedArticle.title).toBe("API LLM Optimized GEO Article");
expect(body.qaReport.checks).toHaveLength(10); expect(body.qaReport.checks).toHaveLength(10);
}); });
@@ -166,7 +228,7 @@ describe("job API routes", () => {
changed_sections: ["title", "body"], changed_sections: ["title", "body"],
requires_user_confirmation: [], requires_user_confirmation: [],
}) })
.mockResolvedValue(null); .mockResolvedValue({ checks: [] });
const response = await optimizeJob( const response = await optimizeJob(
request({}), request({}),
@@ -179,6 +241,27 @@ describe("job API routes", () => {
expect(llmMocks.generateValidatedJson).toHaveBeenCalled(); expect(llmMocks.generateValidatedJson).toHaveBeenCalled();
}); });
it("returns a clear error when LLM optimization fails", async () => {
const { job } = await createJobFixture();
await confirmFactCard(
request(validFactCard),
params<{ jobId: string }>({ jobId: job.id }),
);
llmMocks.generateValidatedJson.mockRejectedValueOnce(
new Error("LLM response failed schema validation: body_markdown"),
);
const response = await optimizeJob(
request({}),
params<{ jobId: string }>({ jobId: job.id }),
);
const body = (await response.json()) as { error: string };
expect(response.status).toBe(502);
expect(body.error).toBe("LLM response failed schema validation: body_markdown");
});
it("rejects unknown export filenames", async () => { it("rejects unknown export filenames", async () => {
const { job } = await createJobFixture(); const { job } = await createJobFixture();
const exportDir = join(tempDir, "exports", job.id); const exportDir = join(tempDir, "exports", job.id);
@@ -198,6 +281,8 @@ describe("job API routes", () => {
}); });
async function createJobFixture() { async function createJobFixture() {
llmMocks.generateValidatedJson.mockResolvedValueOnce(validCandidateFactCard);
const response = await createJob( const response = await createJob(
request({ request({
title: "Example Technology Co., Ltd. GEO guide", title: "Example Technology Co., Ltd. GEO guide",
+16 -1
View File
@@ -2,6 +2,7 @@ import { NextResponse } from "next/server";
import { requireApiAccess } from "../../../../../lib/api/auth"; import { requireApiAccess } from "../../../../../lib/api/auth";
import { getRepositoryFromRuntime } from "../../../../../lib/db/repository"; import { getRepositoryFromRuntime } from "../../../../../lib/db/repository";
import { LlmValidationError } from "../../../../../lib/llm/client";
import { getExportStoreFromRuntime } from "../../../../../lib/workflow/export-store"; import { getExportStoreFromRuntime } from "../../../../../lib/workflow/export-store";
import { runOptimizationWorkflow } from "../../../../../lib/workflow/orchestrator"; import { runOptimizationWorkflow } from "../../../../../lib/workflow/orchestrator";
@@ -30,6 +31,7 @@ export async function POST(request: Request, context: RouteContext) {
); );
} }
try {
const result = await runOptimizationWorkflow({ const result = await runOptimizationWorkflow({
input: { input: {
title: job.source_title, title: job.source_title,
@@ -40,7 +42,10 @@ export async function POST(request: Request, context: RouteContext) {
}, },
factCard: factCardRecord, factCard: factCardRecord,
}); });
const optimizedArticle = await repository.saveOptimizedArticle(jobId, result.article); const optimizedArticle = await repository.saveOptimizedArticle(
jobId,
result.article,
);
const qaReport = await repository.saveQaReport( const qaReport = await repository.saveQaReport(
jobId, jobId,
optimizedArticle.revision ?? 1, optimizedArticle.revision ?? 1,
@@ -67,4 +72,14 @@ export async function POST(request: Request, context: RouteContext) {
rewriteRounds: result.rewrite_rounds, rewriteRounds: result.rewrite_rounds,
stoppedAfterMaxRewrites: result.stopped_after_max_rewrites, stoppedAfterMaxRewrites: result.stopped_after_max_rewrites,
}); });
} catch (error) {
const message = error instanceof Error ? error.message : "LLM optimization failed";
return NextResponse.json({ error: message }, { status: getErrorStatus(error) });
}
}
function getErrorStatus(error: unknown) {
if (error instanceof LlmValidationError) return 502;
if (error instanceof Error && /^LLM\b|provider/i.test(error.message)) return 502;
return 500;
} }
+8 -1
View File
@@ -2,6 +2,7 @@ import { NextResponse } from "next/server";
import { requireApiAccess } from "../../../lib/api/auth"; import { requireApiAccess } from "../../../lib/api/auth";
import { getRepositoryFromRuntime } from "../../../lib/db/repository"; import { getRepositoryFromRuntime } from "../../../lib/db/repository";
import { LlmValidationError } from "../../../lib/llm/client";
import { extractCandidateFactCard } from "../../../lib/workflow/fact-extractor"; import { extractCandidateFactCard } from "../../../lib/workflow/fact-extractor";
import { normalizeInput, type RawArticleInput } from "../../../lib/workflow/input-normalizer"; import { normalizeInput, type RawArticleInput } from "../../../lib/workflow/input-normalizer";
@@ -26,7 +27,7 @@ export async function POST(request: Request) {
return NextResponse.json({ job, candidateFactCard }, { status: 201 }); return NextResponse.json({ job, candidateFactCard }, { status: 201 });
} catch (error) { } catch (error) {
return jsonError(error, 400); return jsonError(error, getErrorStatus(error));
} }
} }
@@ -34,3 +35,9 @@ function jsonError(error: unknown, status: number) {
const message = error instanceof Error ? error.message : "Request failed"; const message = error instanceof Error ? error.message : "Request failed";
return NextResponse.json({ error: message }, { status }); return NextResponse.json({ error: message }, { status });
} }
function getErrorStatus(error: unknown) {
if (error instanceof LlmValidationError) return 502;
if (error instanceof Error && /^LLM\b|provider/i.test(error.message)) return 502;
return 400;
}
+27 -27
View File
@@ -16,16 +16,16 @@ describe("generateValidatedJson", () => {
vi.restoreAllMocks(); vi.restoreAllMocks();
}); });
it("returns null when no provider key is configured", async () => { it("throws clearly when no provider key is configured", async () => {
process.env.LLM_PROVIDER = "deepseek"; process.env.LLM_PROVIDER = "deepseek";
delete process.env.DEEPSEEK_API_KEY; delete process.env.DEEPSEEK_API_KEY;
const result = await client.generateValidatedJson({ await expect(
client.generateValidatedJson({
schema: z.object({ value: z.string() }), schema: z.object({ value: z.string() }),
prompt: "Return JSON.", prompt: "Return JSON.",
}); }),
).rejects.toThrow("DEEPSEEK_API_KEY is missing");
expect(result).toBeNull();
}); });
it("returns parsed data when the model response matches the schema", async () => { it("returns parsed data when the model response matches the schema", async () => {
@@ -41,32 +41,32 @@ describe("generateValidatedJson", () => {
expect(result).toEqual({ value: "from-llm" }); expect(result).toEqual({ value: "from-llm" });
}); });
it("returns null when the model response fails schema validation", async () => { it("throws clearly when the model response fails schema validation", async () => {
process.env.LLM_PROVIDER = "deepseek"; process.env.LLM_PROVIDER = "deepseek";
process.env.DEEPSEEK_API_KEY = "test-key"; process.env.DEEPSEEK_API_KEY = "test-key";
client.setGenerateJsonForValidation(async () => ({ value: 42 })); client.setGenerateJsonForValidation(async () => ({ value: 42 }));
const result = await client.generateValidatedJson({ await expect(
client.generateValidatedJson({
schema: z.object({ value: z.string() }), schema: z.object({ value: z.string() }),
prompt: "Return JSON.", prompt: "Return JSON.",
}),
).rejects.toThrow("LLM response failed schema validation");
}); });
expect(result).toBeNull(); it("throws clearly when the provider call rejects", async () => {
});
it("returns null when the provider call rejects", async () => {
process.env.LLM_PROVIDER = "deepseek"; process.env.LLM_PROVIDER = "deepseek";
process.env.DEEPSEEK_API_KEY = "test-key"; process.env.DEEPSEEK_API_KEY = "test-key";
client.setGenerateJsonForValidation(async () => { client.setGenerateJsonForValidation(async () => {
throw new Error("provider down"); throw new Error("provider down");
}); });
const result = await client.generateValidatedJson({ await expect(
client.generateValidatedJson({
schema: z.object({ value: z.string() }), schema: z.object({ value: z.string() }),
prompt: "Return JSON.", prompt: "Return JSON.",
}); }),
).rejects.toThrow("provider down");
expect(result).toBeNull();
}); });
it("logs validation success with the supplied task label", async () => { it("logs validation success with the supplied task label", async () => {
@@ -93,13 +93,13 @@ describe("generateValidatedJson", () => {
process.env.DEEPSEEK_API_KEY = "test-key"; process.env.DEEPSEEK_API_KEY = "test-key";
client.setGenerateJsonForValidation(async () => ({ value: 42 })); client.setGenerateJsonForValidation(async () => ({ value: 42 }));
const result = await client.generateValidatedJson({ await expect(
client.generateValidatedJson({
schema: z.object({ value: z.string() }), schema: z.object({ value: z.string() }),
prompt: "Return JSON.", prompt: "Return JSON.",
task: "fact_extractor", task: "fact_extractor",
}); }),
).rejects.toThrow("LLM response failed schema validation");
expect(result).toBeNull();
expect(warnSpy).toHaveBeenCalledWith( expect(warnSpy).toHaveBeenCalledWith(
expect.stringContaining("[llm:validated] task=fact_extractor ok=false"), expect.stringContaining("[llm:validated] task=fact_extractor ok=false"),
); );
@@ -150,16 +150,16 @@ describe("generateValidatedJson", () => {
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
client.setGenerateJsonForValidation(async () => ({ value: 42 })); client.setGenerateJsonForValidation(async () => ({ value: 42 }));
const result = await client.generateValidatedJson({ await expect(
client.generateValidatedJson({
schema: z.object({ value: z.string() }), schema: z.object({ value: z.string() }),
task: "fact_extractor", task: "fact_extractor",
prompt: "Return JSON.", prompt: "Return JSON.",
}); }),
).rejects.toThrow("LLM response failed schema validation");
const allLogs = [...infoSpy.mock.calls, ...warnSpy.mock.calls] const allLogs = [...infoSpy.mock.calls, ...warnSpy.mock.calls]
.flat() .flat()
.join("\n"); .join("\n");
expect(result).toBeNull();
expect(allLogs).toContain("[llm:validated] task=fact_extractor ok=false"); expect(allLogs).toContain("[llm:validated] task=fact_extractor ok=false");
expect(allLogs).toContain("zod_error="); expect(allLogs).toContain("zod_error=");
expect(allLogs).not.toContain("super-secret-key"); expect(allLogs).not.toContain("super-secret-key");
@@ -173,13 +173,13 @@ describe("generateValidatedJson", () => {
throw new Error("provider unavailable"); throw new Error("provider unavailable");
}); });
const result = await client.generateValidatedJson({ await expect(
client.generateValidatedJson({
schema: z.object({ value: z.string() }), schema: z.object({ value: z.string() }),
task: "quality_inspector", task: "quality_inspector",
prompt: "Return JSON.", prompt: "Return JSON.",
}); }),
).rejects.toThrow("provider unavailable");
expect(result).toBeNull();
expect(warnSpy).toHaveBeenCalledWith( expect(warnSpy).toHaveBeenCalledWith(
expect.stringContaining("[llm:error] task=quality_inspector"), expect.stringContaining("[llm:error] task=quality_inspector"),
); );
+25 -6
View File
@@ -28,6 +28,16 @@ export interface LlmProviderStatus {
reason?: string; reason?: string;
} }
export class LlmValidationError extends Error {
constructor(
message: string,
public readonly task: LlmTaskName,
) {
super(message);
this.name = "LlmValidationError";
}
}
function getProvider() { function getProvider() {
return (process.env.LLM_PROVIDER || "deepseek").toLowerCase(); return (process.env.LLM_PROVIDER || "deepseek").toLowerCase();
} }
@@ -80,8 +90,8 @@ export function setChatCompletionForTesting(
chatCompletionForTesting = handler; chatCompletionForTesting = handler;
} }
function getTask(input: GenerateInput) { function getTask(input: GenerateInput): LlmTaskName {
return input.task?.trim() || "unknown"; return input.task || "unknown";
} }
function getRawLogLimit() { function getRawLogLimit() {
@@ -204,11 +214,14 @@ export function setGenerateJsonForValidation(
export async function generateValidatedJson<T>({ export async function generateValidatedJson<T>({
schema, schema,
...input ...input
}: GenerateValidatedJsonInput<T>): Promise<T | null> { }: GenerateValidatedJsonInput<T>): Promise<T> {
const task = getTask(input); const task = getTask(input);
if (!isLlmConfigured()) { if (!isLlmConfigured()) {
console.info(`[llm:validated] task=${task} ok=false reason=not_configured`); console.info(`[llm:validated] task=${task} ok=false reason=not_configured`);
return null; throw new LlmValidationError(
getLlmProviderStatus().reason ?? "LLM provider is not configured",
task,
);
} }
const status = getLlmProviderStatus(); const status = getLlmProviderStatus();
@@ -230,14 +243,20 @@ export async function generateValidatedJson<T>({
console.warn( console.warn(
`[llm:validated] task=${task} ok=false zod_error=${quoteLogValue(summarizeZodError(parsed.error))}`, `[llm:validated] task=${task} ok=false zod_error=${quoteLogValue(summarizeZodError(parsed.error))}`,
); );
return null; throw new LlmValidationError(
`LLM response failed schema validation: ${summarizeZodError(parsed.error)}`,
task,
);
} catch (error) { } catch (error) {
if (error instanceof LlmValidationError) {
throw error;
}
console.info(`[llm:validated] task=${task} ok=false reason=provider_error`); console.info(`[llm:validated] task=${task} ok=false reason=provider_error`);
const message = error instanceof Error ? error.message : String(error); const message = error instanceof Error ? error.message : String(error);
console.warn( console.warn(
`[llm:error] task=${task} duration_ms=${Date.now() - startedAt} message=${quoteLogValue(message)}`, `[llm:error] task=${task} duration_ms=${Date.now() - startedAt} message=${quoteLogValue(message)}`,
); );
return null; throw error instanceof Error ? error : new Error(message);
} }
} }
+6 -5
View File
@@ -10,7 +10,7 @@ export const JSON_ONLY_PROMPT =
"Return valid JSON only. Do not include markdown fences or commentary."; "Return valid JSON only. Do not include markdown fences or commentary.";
const CUSTOMER_RISK_GUIDANCE = [ const CUSTOMER_RISK_GUIDANCE = [
"客户最担心的内容风险:行业漂移、公司全称/简称/品牌名不一致、图片主题与正文描述不匹配、官网文章出现第三方口吻、平台语气和文章类型不匹配、标题或正文语义不顺、虚构资质/年限/案例/能力、产品/服务/年限前后冲突。", "客户最担心的内容风险:行业漂移、公司全称/简称/品牌名不一致、官网文章出现第三方口吻、平台语气和文章类型不匹配、标题或正文语义不顺、虚构资质/年限/案例/能力、产品/服务/年限前后冲突。",
"任何客户案例、资质荣誉、经验年限、服务能力、出海/多语种/合规能力、效果承诺和排名,都必须能从原文或已确认事实卡中找到明确依据。", "任何客户案例、资质荣誉、经验年限、服务能力、出海/多语种/合规能力、效果承诺和排名,都必须能从原文或已确认事实卡中找到明确依据。",
].join(" "); ].join(" ");
@@ -99,7 +99,7 @@ export function buildArticleOptimizerPrompt(
"- 必须保留事实卡确认的公司全称、目标行业、目标受众和核心事实。", "- 必须保留事实卡确认的公司全称、目标行业、目标受众和核心事实。",
"- 必须删除或弱化 factCard.forbidden_claims 中的主张。", "- 必须删除或弱化 factCard.forbidden_claims 中的主张。",
"- 不得新增客户案例、数字、资质、排名、奖项、服务能力、效果承诺。", "- 不得新增客户案例、数字、资质、排名、奖项、服务能力、效果承诺。",
"- image_suggestions 必须基于 factCard.image_topics 或原始 images;没有图片主题时返回空数组。", "- 当前版本只优化文本,不生成图片建议;image_suggestions 必须返回空数组 []。",
"", "",
"Confirmed fact card:", "Confirmed fact card:",
JSON.stringify(factCard, null, 2), JSON.stringify(factCard, null, 2),
@@ -124,8 +124,9 @@ export function buildQualityInspectorPrompt(input: {
formatPlatformGuidance(input.platform), formatPlatformGuidance(input.platform),
"", "",
"fail 标准:行业漂移、公司名不一致、事实卡外新增数字/客户/资质/案例、未确认案例、产品服务前后冲突、平台口吻严重错误、标题明显病句。", "fail 标准:行业漂移、公司名不一致、事实卡外新增数字/客户/资质/案例、未确认案例、产品服务前后冲突、平台口吻严重错误、标题明显病句。",
"warn 标准:图片证据不足、句子过长、表达可读性一般、平台适配轻微不足。", "warn 标准:句子过长、表达可读性一般、平台适配轻微不足。",
"target_agent 只能使用 title、body、image、fact_card 或 null。", "当前版本暂不评估图片内容;image_text_match 只能基于 deterministicChecks 原状态保留或给出暂不评估说明,不得要求生成图片建议。",
"target_agent 只能使用 title、body、fact_card 或 null。",
"", "",
"Confirmed fact card:", "Confirmed fact card:",
JSON.stringify(input.factCard, null, 2), JSON.stringify(input.factCard, null, 2),
@@ -155,7 +156,7 @@ export function buildTargetedRewritePrompt(input: {
"- title_quality:生成自然中文标题,禁止英文模板词。", "- title_quality:生成自然中文标题,禁止英文模板词。",
"- body_quality:拆分长句,修复病句和断裂表达。", "- body_quality:拆分长句,修复病句和断裂表达。",
"- voice_consistency / platform_fit:改成目标平台对应口吻。", "- voice_consistency / platform_fit:改成目标平台对应口吻。",
"- image_text_match只补充图片建议或人工确认项,不虚构图片内容。", "- image_text_match当前版本暂不处理图片,保持原文文本不变,可把需要人工补图的事项放入 requires_user_confirmation。",
"不得新增事实。无法修复的内容放入 requires_user_confirmation。", "不得新增事实。无法修复的内容放入 requires_user_confirmation。",
"", "",
"Confirmed fact card:", "Confirmed fact card:",
@@ -72,19 +72,20 @@ describe("LLM workflow integration", () => {
); );
}); });
it("falls back to deterministic candidate extraction when LLM returns null", async () => { it("surfaces candidate fact extraction LLM failures instead of falling back", async () => {
llmMocks.generateValidatedJson.mockResolvedValueOnce(null); llmMocks.generateValidatedJson.mockRejectedValueOnce(
new Error("LLM response failed schema validation: target_audience"),
);
const card = await extractCandidateFactCard({ await expect(
extractCandidateFactCard({
title: "Fallback Technology Co., Ltd. GEO guide", title: "Fallback Technology Co., Ltd. GEO guide",
body: "Fallback Technology Co., Ltd. has 8 years of GEO optimization experience.", body: "Fallback Technology Co., Ltd. has 8 years of GEO optimization experience.",
images: [{ type: "description", content: "dashboard" }], images: [{ type: "description", content: "dashboard" }],
platform: "official_site", platform: "official_site",
user_instructions: "", user_instructions: "",
}); }),
).rejects.toThrow("LLM response failed schema validation: target_audience");
expect(card.company_full_name).toBe("Fallback Technology Co., Ltd.");
expect(card.experience_years).toBe(8);
}); });
it("uses LLM output for article optimization when valid", async () => { it("uses LLM output for article optimization when valid", async () => {
@@ -92,7 +93,7 @@ describe("LLM workflow integration", () => {
title: "LLM Optimized GEO Article", title: "LLM Optimized GEO Article",
summary: "LLM summary constrained by the fact card.", summary: "LLM summary constrained by the fact card.",
body_markdown: "## LLM Body\nExample Technology Co., Ltd. keeps claims factual.", body_markdown: "## LLM Body\nExample Technology Co., Ltd. keeps claims factual.",
image_suggestions: [{ source: "image_1", suggestion: "Use dashboard." }], image_suggestions: [],
changed_sections: ["title", "body"], changed_sections: ["title", "body"],
requires_user_confirmation: [], requires_user_confirmation: [],
}); });
@@ -110,16 +111,20 @@ describe("LLM workflow integration", () => {
expect(article.title).toBe("LLM Optimized GEO Article"); expect(article.title).toBe("LLM Optimized GEO Article");
expect(article.body_markdown).toContain("LLM Body"); expect(article.body_markdown).toContain("LLM Body");
expect(article.image_suggestions).toEqual([]);
expect(llmMocks.generateValidatedJson).toHaveBeenCalledOnce(); expect(llmMocks.generateValidatedJson).toHaveBeenCalledOnce();
expect(llmMocks.generateValidatedJson).toHaveBeenCalledWith( expect(llmMocks.generateValidatedJson).toHaveBeenCalledWith(
expect.objectContaining({ task: "article_optimizer" }), expect.objectContaining({ task: "article_optimizer" }),
); );
}); });
it("falls back to deterministic article optimization when LLM returns null", async () => { it("surfaces article optimization LLM failures instead of falling back", async () => {
llmMocks.generateValidatedJson.mockResolvedValueOnce(null); llmMocks.generateValidatedJson.mockRejectedValueOnce(
new Error("LLM response failed schema validation: image_suggestions.0.source"),
);
const article = await optimizeArticle({ await expect(
optimizeArticle({
input: { input: {
title: "Original", title: "Original",
body: "Example Technology Co., Ltd. has 8 years of GEO optimization experience.", body: "Example Technology Co., Ltd. has 8 years of GEO optimization experience.",
@@ -128,11 +133,9 @@ describe("LLM workflow integration", () => {
user_instructions: "Say we have 99 patents.", user_instructions: "Say we have 99 patents.",
}, },
factCard: confirmedFactCard, factCard: confirmedFactCard,
}); }),
).rejects.toThrow(
expect(article.title).toContain("GEO optimization Guide"); "LLM response failed schema validation: image_suggestions.0.source",
expect(article.requires_user_confirmation).toContain(
"Unsupported requested claim: 99 patents",
); );
}); });
@@ -176,10 +179,11 @@ describe("LLM workflow integration", () => {
); );
}); });
it("falls back to deterministic targeted rewrite when LLM returns null", async () => { it("surfaces targeted rewrite LLM failures instead of falling back", async () => {
llmMocks.generateValidatedJson.mockResolvedValueOnce(null); llmMocks.generateValidatedJson.mockRejectedValueOnce(new Error("provider unavailable"));
const rewritten = await rewriteFailedSections({ await expect(
rewriteFailedSections({
article: { article: {
title: "Bad title!!!", title: "Bad title!!!",
summary: "Original summary", summary: "Original summary",
@@ -199,10 +203,8 @@ describe("LLM workflow integration", () => {
target_agent: "title", target_agent: "title",
}, },
], ],
}); }),
).rejects.toThrow("provider unavailable");
expect(rewritten.title).toContain("GEO optimization Guide");
expect(rewritten.summary).toBe("Original summary");
}); });
it("uses LLM quality checks to enrich non-failing deterministic checks", async () => { it("uses LLM quality checks to enrich non-failing deterministic checks", async () => {
-109
View File
@@ -1,12 +1,8 @@
import { describe, expect, it } from "vitest"; import { describe, expect, it } from "vitest";
import type { ConfirmedFactCard } from "../../domain/types"; import type { ConfirmedFactCard } from "../../domain/types";
import { optimizeArticle } from "../article-optimizer";
import { extractCandidateFactCard } from "../fact-extractor";
import { normalizeInput } from "../input-normalizer"; import { normalizeInput } from "../input-normalizer";
import { inspectQuality } from "../quality-inspector"; import { inspectQuality } from "../quality-inspector";
import { runOptimizationWorkflow } from "../orchestrator";
import { rewriteFailedSections } from "../targeted-rewriter";
const confirmedFactCard: ConfirmedFactCard = { const confirmedFactCard: ConfirmedFactCard = {
company_full_name: "Example Technology Co., Ltd.", company_full_name: "Example Technology Co., Ltd.",
@@ -45,65 +41,6 @@ describe("workflow nodes", () => {
]); ]);
}); });
it("places missing or conflicting company facts into uncertain items", async () => {
const card = await extractCandidateFactCard({
title: "Example announces GEO product",
body: "Example has 8 years of experience. Example has 12 years of service. The article discusses GEO optimization.",
images: [],
platform: "media_article",
user_instructions: "",
});
expect(card.company_full_name).toBe("");
expect(card.uncertain_items).toEqual(
expect.arrayContaining([
expect.stringContaining("company full name"),
expect.stringContaining("Conflicting experience years"),
]),
);
expect(card.is_ready_for_optimization).toBe(false);
});
it("extracts Chinese company facts from Chinese articles", async () => {
const card = await extractCandidateFactCard({
title: "#探寻AIGC短视频培训选哪家,各品牌实力大比拼",
body: "伟思德鲁管理咨询(深圳)有限公司面向品牌商家和出海企业提供AIGC短视频培训服务,帮助企业解决内容工业化生产、品牌视觉统一和全球化传播问题。",
images: [{ type: "description", content: "AIGC短视频工作流示意图" }],
platform: "media_article",
user_instructions: "保留AIGC短视频培训与出海内容生产场景。",
});
expect(card.company_full_name).toBe("伟思德鲁管理咨询(深圳)有限公司");
expect(card.company_short_names).toContain("伟思德鲁");
expect(card.target_industry).toBe("AIGC短视频培训");
expect(card.target_audience).toBe("品牌商家、内容创作者、出海企业");
expect(card.image_topics).toEqual(["AIGC短视频工作流示意图"]);
expect(card.uncertain_items).not.toContain("Missing company full name");
});
it("does not add claims outside the confirmed fact card", async () => {
const optimized = await optimizeArticle({
input: {
title: "Example GEO article",
body: "Example GEO helps marketing teams improve content structure.",
images: [],
platform: "official_site",
user_instructions:
"Say we have 99 patents and Fortune 500 customer cases.",
},
factCard: confirmedFactCard,
});
expect(optimized.body_markdown).not.toContain("99 patents");
expect(optimized.body_markdown).not.toContain("Fortune 500");
expect(optimized.requires_user_confirmation).toEqual(
expect.arrayContaining([
expect.stringContaining("99 patents"),
expect.stringContaining("Fortune 500"),
]),
);
});
it("returns the 10 required quality checks", () => { it("returns the 10 required quality checks", () => {
const report = inspectQuality({ const report = inspectQuality({
article: { article: {
@@ -183,50 +120,4 @@ describe("workflow nodes", () => {
expect(report.overall_status).toBe("fail"); expect(report.overall_status).toBe("fail");
}); });
it("rewrites only the failing target area", async () => {
const article = {
title: "Bad title!!!",
summary: "Original summary",
body_markdown: "Original body",
image_suggestions: [],
changed_sections: [],
requires_user_confirmation: [],
};
const rewritten = await rewriteFailedSections({
article,
factCard: confirmedFactCard,
failedChecks: [
{
rule_id: "title_quality",
status: "fail",
evidence: "Bad title!!!",
reason: "Punctuation stuffing.",
suggested_fix: "Rewrite title.",
target_agent: "title",
},
],
});
expect(rewritten.title).not.toBe(article.title);
expect(rewritten.summary).toBe(article.summary);
expect(rewritten.body_markdown).toBe(article.body_markdown);
});
it("orchestrator stops after two failed rewrite rounds", async () => {
const result = await runOptimizationWorkflow({
input: {
title: "Finance automation breakthrough!!!",
body: "Example has 12 years in finance automation and 99 patents.",
images: [],
platform: "official_site",
user_instructions: "",
},
factCard: confirmedFactCard,
});
expect(result.rewrite_rounds).toBe(2);
expect(result.qaReport.overall_status).toBe("fail");
expect(result.stopped_after_max_rewrites).toBe(true);
});
}); });
+2 -71
View File
@@ -27,77 +27,8 @@ export async function optimizeArticle({
task: "article_optimizer", task: "article_optimizer",
}); });
return llmArticle ?? optimizeArticleFallback({ input, factCard });
}
function optimizeArticleFallback({
input,
factCard,
}: OptimizeArticleInput): OptimizedArticle {
const unsupported = findUnsupportedInstructionClaims(
input.user_instructions,
factCard,
);
const title = `${factCard.brand_names[0] ?? factCard.company_short_names[0] ?? factCard.company_full_name} ${factCard.target_industry} Guide`;
const coreClaims =
factCard.core_claims.length > 0
? factCard.core_claims.map((claim) => `- ${claim}`).join("\n")
: "- Confirmed facts only; no extra claims added.";
const body = [
`## ${factCard.company_full_name}`,
cleanBody(input.body, factCard),
"",
"### Confirmed Facts",
coreClaims,
].join("\n");
return optimizedArticleSchema.parse({ return optimizedArticleSchema.parse({
title, ...llmArticle,
summary: `A ${input.platform.replace(/_/g, " ")} article for ${factCard.target_audience} about ${factCard.target_industry}.`, image_suggestions: [],
body_markdown: body,
image_suggestions: factCard.image_topics.map((topic, index) => ({
source: `image_${index + 1}`,
suggestion: `Use image content related to ${topic}.`,
})),
changed_sections: ["title", "body structure", "summary"],
requires_user_confirmation: unsupported,
}); });
} }
function cleanBody(body: string, factCard: ConfirmedFactCard) {
let cleaned = body.trim();
for (const forbidden of factCard.forbidden_claims) {
cleaned = cleaned.replace(new RegExp(escapeRegExp(forbidden), "gi"), "");
}
return cleaned;
}
function findUnsupportedInstructionClaims(
instructions: string,
factCard: ConfirmedFactCard,
) {
const unsupported: string[] = [];
const numbers = [...instructions.matchAll(/\b\d+\s*[A-Za-z]+\b/g)].map(
(match) => match[0],
);
const knownText = [
factCard.experience_years?.toString() ?? "",
...factCard.core_claims,
].join(" ");
for (const claim of numbers) {
if (!knownText.includes(claim.replace(/\D/g, ""))) {
unsupported.push(`Unsupported requested claim: ${claim}`);
}
}
if (/fortune\s*500/i.test(instructions)) {
unsupported.push("Unsupported requested claim: Fortune 500 customer cases");
}
return unsupported;
}
function escapeRegExp(value: string) {
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
+1 -121
View File
@@ -9,131 +9,11 @@ import {
export async function extractCandidateFactCard( export async function extractCandidateFactCard(
input: ArticleInput, input: ArticleInput,
): Promise<CandidateFactCard> { ): Promise<CandidateFactCard> {
const llmCard = await generateValidatedJson({ return generateValidatedJson({
schema: candidateFactCardSchema, schema: candidateFactCardSchema,
system: FACT_EXTRACTOR_SYSTEM_PROMPT, system: FACT_EXTRACTOR_SYSTEM_PROMPT,
prompt: buildFactExtractorPrompt(input), prompt: buildFactExtractorPrompt(input),
temperature: 0.1, temperature: 0.1,
task: "fact_extractor", task: "fact_extractor",
}); });
return llmCard ?? extractCandidateFactCardFallback(input);
}
function extractCandidateFactCardFallback(input: ArticleInput): CandidateFactCard {
const text = `${input.title}\n${input.body}`;
const uncertainItems: string[] = [];
const companyFullName = findCompanyFullName(text);
const years = findExperienceYears(text);
if (!companyFullName) {
uncertainItems.push("Missing company full name");
}
if (years.length > 1) {
uncertainItems.push(`Conflicting experience years: ${years.join(", ")}`);
}
if (input.images.length === 0) {
uncertainItems.push("Image description is missing");
}
const industry = inferIndustry(text);
return candidateFactCardSchema.parse({
company_full_name: companyFullName ?? "",
company_short_names: inferCompanyShortNames(companyFullName),
brand_names: inferBrandNames(text, companyFullName),
product_names: inferProducts(text),
target_industry: industry,
target_audience: text.toLowerCase().includes("marketing")
? "Marketing teams"
: inferTargetAudience(text),
experience_years: years.length === 1 ? years[0] : null,
core_claims: years.length === 1 ? [`${years[0]} years of ${industry} experience`] : [],
forbidden_claims: [],
image_topics: input.images.map((image) => image.content),
uncertain_items: uncertainItems,
});
}
function findCompanyFullName(text: string) {
const englishMatch = text.match(
/([A-Z][A-Za-z0-9&.,\-\s]{2,}?(?:Co\.,?\s*Ltd\.?|Company|Inc\.?|LLC|Ltd\.))/,
);
if (englishMatch?.[1]) return englishMatch[1].trim();
const chineseMatch = text.match(
/([\u4e00-\u9fa5A-Za-z0-9()()]{2,40}?(?:股份有限公司|有限公司|集团|公司))/,
);
return chineseMatch?.[1].trim() ?? null;
}
function findExperienceYears(text: string) {
const matches = [...text.matchAll(/\b(\d{1,3})\s*(?:years?|年)\b/gi)];
return [...new Set(matches.map((match) => Number(match[1])))];
}
function inferIndustry(text: string) {
const lower = text.toLowerCase();
if (lower.includes("geo")) return "GEO optimization";
if (lower.includes("finance") || lower.includes("banking")) return "finance automation";
if (lower.includes("seo")) return "SEO";
if (/AIGC|短视频|出海内容|内容工业化/.test(text)) return "AIGC短视频培训";
if (/工业零部件|爆品操盘|AI OBS|IPMS/.test(text)) return "工业零部件爆品操盘";
return "General business";
}
function inferBrandNames(text: string, companyFullName: string | null) {
const chineseBrands = [
...new Set(
[
companyFullName ? inferChineseShortName(companyFullName) : "",
...[...text.matchAll(/\b(AIGC|AI OBS|IPMS|GEO|SEO)\b/g)].map(
(match) => match[1],
),
].filter(Boolean),
),
];
const names = [...text.matchAll(/\b[A-Z][A-Za-z0-9]{2,}\b/g)]
.map((match) => match[0])
.filter((word) => !["The", "This", "And"].includes(word));
return [...new Set([...chineseBrands, ...names])].slice(0, 5);
}
function inferProducts(text: string) {
const productMatches = [
...[...text.matchAll(/\b([A-Z][A-Za-z0-9]+\s+GEO)\b/g)].map(
(match) => match[1],
),
...[...text.matchAll(/\b(AIGC短视频培训|AI OBS|IPMS|爆品操盘数智系统)\b/g)].map(
(match) => match[1],
),
];
return [...new Set(productMatches)];
}
function inferCompanyShortNames(companyFullName: string | null) {
if (!companyFullName) return [];
if (/[\u4e00-\u9fa5]/.test(companyFullName)) {
const legalShortName = inferChineseShortName(companyFullName);
const brandShortName = legalShortName.replace(/管理咨询$/, "");
return [...new Set([brandShortName, legalShortName].filter(Boolean))];
}
return [companyFullName.split(/\s+/)[0] ?? ""].filter(Boolean);
}
function inferChineseShortName(companyFullName: string) {
return companyFullName
.replace(/[(].*?[)]/g, "")
.replace(/股份有限公司|有限公司|集团|公司/g, "")
.trim();
}
function inferTargetAudience(text: string) {
if (/品牌商家|内容创作者|出海企业/.test(text)) {
return "品牌商家、内容创作者、出海企业";
}
if (/工业企业|工业零部件制造商|采购/.test(text)) {
return "工业企业、采购团队、工业零部件制造商";
}
return "Business readers";
} }
+4 -11
View File
@@ -68,10 +68,6 @@ export async function inspectQualityWithLlm(
task: "quality_inspector", task: "quality_inspector",
}); });
if (!llmPatch) {
return deterministicReport;
}
const patchedChecks = deterministicReport.checks.map((deterministicCheck) => { const patchedChecks = deterministicReport.checks.map((deterministicCheck) => {
const llmCheck = llmPatch.checks.find( const llmCheck = llmPatch.checks.find(
(check) => check.rule_id === deterministicCheck.rule_id, (check) => check.rule_id === deterministicCheck.rule_id,
@@ -125,15 +121,12 @@ function inspectRule(
} }
if (ruleId === "image_text_match") { if (ruleId === "image_text_match") {
const hasImages = sourceImages.length > 0 || article.image_suggestions.length > 0;
return check( return check(
ruleId, ruleId,
hasImages ? "pass" : "warn", "pass",
hasImages ? "已有可用于比对的图片主题。" : "未提供图片描述。", sourceImages.length > 0 ? "当前版本暂不评估图片内容。" : "当前版本未启用图片分析。",
hasImages "当前版本仅优化文本,图片匹配检查暂不参与质量门禁。",
? "图片建议可以和文章内容进行比对。" "后续启用图片工作流后再补充图文匹配检查。",
: "缺少图片描述时,图文匹配置信度较低。",
"补充图片描述,或人工检查图片与正文的对应关系。",
null, null,
); );
} }
+4 -42
View File
@@ -25,46 +25,8 @@ export async function rewriteFailedSections({
task: "targeted_rewriter", task: "targeted_rewriter",
}); });
return llmArticle ?? rewriteFailedSectionsFallback({ article, factCard, failedChecks }); return optimizedArticleSchema.parse({
} ...llmArticle,
image_suggestions: [],
function rewriteFailedSectionsFallback({ });
article,
factCard,
failedChecks,
}: RewriteFailedSectionsInput): OptimizedArticle {
let rewritten = { ...article };
for (const check of failedChecks) {
if (check.target_agent === "title") {
rewritten = {
...rewritten,
title: `${factCard.brand_names[0] ?? factCard.company_short_names[0]} ${factCard.target_industry} Guide`,
changed_sections: [...new Set([...rewritten.changed_sections, "title"])],
};
}
if (check.target_agent === "body" && check.rule_id === "company_name_integrity") {
rewritten = {
...rewritten,
body_markdown: `${factCard.company_full_name}\n\n${rewritten.body_markdown}`,
changed_sections: [...new Set([...rewritten.changed_sections, "company name"])],
};
}
if (check.target_agent === "body" && check.rule_id === "claim_consistency") {
rewritten = {
...rewritten,
body_markdown: rewritten.body_markdown.replace(
/\b\d{1,3}\s*(?:years?|年)\b/gi,
factCard.experience_years === null
? "confirmed experience"
: `${factCard.experience_years} years`,
),
changed_sections: [...new Set([...rewritten.changed_sections, "claim consistency"])],
};
}
}
return rewritten;
} }