fix: surface llm workflow errors
This commit is contained in:
@@ -39,6 +39,21 @@ const validFactCard = {
|
||||
confirmed_by_user: true,
|
||||
};
|
||||
|
||||
const validCandidateFactCard = {
|
||||
company_full_name: validFactCard.company_full_name,
|
||||
company_short_names: validFactCard.company_short_names,
|
||||
brand_names: validFactCard.brand_names,
|
||||
product_names: validFactCard.product_names,
|
||||
target_industry: validFactCard.target_industry,
|
||||
target_audience: validFactCard.target_audience,
|
||||
experience_years: validFactCard.experience_years,
|
||||
core_claims: validFactCard.core_claims,
|
||||
forbidden_claims: validFactCard.forbidden_claims,
|
||||
image_topics: validFactCard.image_topics,
|
||||
uncertain_items: validFactCard.uncertain_items,
|
||||
is_ready_for_optimization: true,
|
||||
};
|
||||
|
||||
interface CreateJobResponse {
|
||||
job: { id: string };
|
||||
candidateFactCard: { company_full_name: string };
|
||||
@@ -86,6 +101,8 @@ describe("job API routes", () => {
|
||||
});
|
||||
|
||||
it("validates input, creates a job, and returns a candidate fact card", async () => {
|
||||
llmMocks.generateValidatedJson.mockResolvedValueOnce(validCandidateFactCard);
|
||||
|
||||
const response = await createJob(
|
||||
request({
|
||||
title: "Example Technology Co., Ltd. GEO guide",
|
||||
@@ -104,6 +121,38 @@ describe("job API routes", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("returns a clear error when LLM fact extraction fails", async () => {
|
||||
llmMocks.generateValidatedJson.mockRejectedValueOnce(
|
||||
new Error("LLM response failed schema validation: target_audience"),
|
||||
);
|
||||
|
||||
const response = await createJob(
|
||||
request({
|
||||
title: "Example Technology Co., Ltd. GEO guide",
|
||||
body: "Example Technology Co., Ltd. has 8 years of GEO optimization experience.",
|
||||
platform: "official_site",
|
||||
}),
|
||||
);
|
||||
const body = (await response.json()) as { error: string };
|
||||
|
||||
expect(response.status).toBe(502);
|
||||
expect(body.error).toBe(
|
||||
"LLM response failed schema validation: target_audience",
|
||||
);
|
||||
});
|
||||
|
||||
it("still returns 400 for invalid article input", async () => {
|
||||
const response = await createJob(
|
||||
request({
|
||||
title: "",
|
||||
body: "",
|
||||
platform: "official_site",
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
});
|
||||
|
||||
it("rejects unresolved uncertain items when confirming a fact card", async () => {
|
||||
const { job } = await createJobFixture();
|
||||
const response = await confirmFactCard(
|
||||
@@ -135,6 +184,19 @@ describe("job API routes", () => {
|
||||
params<{ jobId: string }>({ jobId: job.id }),
|
||||
);
|
||||
|
||||
llmMocks.generateValidatedJson
|
||||
.mockResolvedValueOnce({
|
||||
title: "API LLM Optimized GEO Article",
|
||||
summary:
|
||||
"A official site article for Marketing teams about GEO optimization.",
|
||||
body_markdown:
|
||||
"Example Technology Co., Ltd. has 8 years of GEO optimization experience.",
|
||||
image_suggestions: [],
|
||||
changed_sections: ["title", "body"],
|
||||
requires_user_confirmation: [],
|
||||
})
|
||||
.mockResolvedValueOnce({ checks: [] });
|
||||
|
||||
const response = await optimizeJob(
|
||||
request({}),
|
||||
params<{ jobId: string }>({ jobId: job.id }),
|
||||
@@ -142,7 +204,7 @@ describe("job API routes", () => {
|
||||
const body = (await response.json()) as OptimizeJobResponse;
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(body.optimizedArticle.title).toContain("GEO optimization");
|
||||
expect(body.optimizedArticle.title).toBe("API LLM Optimized GEO Article");
|
||||
expect(body.qaReport.checks).toHaveLength(10);
|
||||
});
|
||||
|
||||
@@ -166,7 +228,7 @@ describe("job API routes", () => {
|
||||
changed_sections: ["title", "body"],
|
||||
requires_user_confirmation: [],
|
||||
})
|
||||
.mockResolvedValue(null);
|
||||
.mockResolvedValue({ checks: [] });
|
||||
|
||||
const response = await optimizeJob(
|
||||
request({}),
|
||||
@@ -179,6 +241,27 @@ describe("job API routes", () => {
|
||||
expect(llmMocks.generateValidatedJson).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns a clear error when LLM optimization fails", async () => {
|
||||
const { job } = await createJobFixture();
|
||||
await confirmFactCard(
|
||||
request(validFactCard),
|
||||
params<{ jobId: string }>({ jobId: job.id }),
|
||||
);
|
||||
|
||||
llmMocks.generateValidatedJson.mockRejectedValueOnce(
|
||||
new Error("LLM response failed schema validation: body_markdown"),
|
||||
);
|
||||
|
||||
const response = await optimizeJob(
|
||||
request({}),
|
||||
params<{ jobId: string }>({ jobId: job.id }),
|
||||
);
|
||||
const body = (await response.json()) as { error: string };
|
||||
|
||||
expect(response.status).toBe(502);
|
||||
expect(body.error).toBe("LLM response failed schema validation: body_markdown");
|
||||
});
|
||||
|
||||
it("rejects unknown export filenames", async () => {
|
||||
const { job } = await createJobFixture();
|
||||
const exportDir = join(tempDir, "exports", job.id);
|
||||
@@ -198,6 +281,8 @@ describe("job API routes", () => {
|
||||
});
|
||||
|
||||
async function createJobFixture() {
|
||||
llmMocks.generateValidatedJson.mockResolvedValueOnce(validCandidateFactCard);
|
||||
|
||||
const response = await createJob(
|
||||
request({
|
||||
title: "Example Technology Co., Ltd. GEO guide",
|
||||
|
||||
@@ -2,6 +2,7 @@ import { NextResponse } from "next/server";
|
||||
|
||||
import { requireApiAccess } from "../../../../../lib/api/auth";
|
||||
import { getRepositoryFromRuntime } from "../../../../../lib/db/repository";
|
||||
import { LlmValidationError } from "../../../../../lib/llm/client";
|
||||
import { getExportStoreFromRuntime } from "../../../../../lib/workflow/export-store";
|
||||
import { runOptimizationWorkflow } from "../../../../../lib/workflow/orchestrator";
|
||||
|
||||
@@ -30,6 +31,7 @@ export async function POST(request: Request, context: RouteContext) {
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await runOptimizationWorkflow({
|
||||
input: {
|
||||
title: job.source_title,
|
||||
@@ -40,7 +42,10 @@ export async function POST(request: Request, context: RouteContext) {
|
||||
},
|
||||
factCard: factCardRecord,
|
||||
});
|
||||
const optimizedArticle = await repository.saveOptimizedArticle(jobId, result.article);
|
||||
const optimizedArticle = await repository.saveOptimizedArticle(
|
||||
jobId,
|
||||
result.article,
|
||||
);
|
||||
const qaReport = await repository.saveQaReport(
|
||||
jobId,
|
||||
optimizedArticle.revision ?? 1,
|
||||
@@ -67,4 +72,14 @@ export async function POST(request: Request, context: RouteContext) {
|
||||
rewriteRounds: result.rewrite_rounds,
|
||||
stoppedAfterMaxRewrites: result.stopped_after_max_rewrites,
|
||||
});
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "LLM optimization failed";
|
||||
return NextResponse.json({ error: message }, { status: getErrorStatus(error) });
|
||||
}
|
||||
}
|
||||
|
||||
function getErrorStatus(error: unknown) {
|
||||
if (error instanceof LlmValidationError) return 502;
|
||||
if (error instanceof Error && /^LLM\b|provider/i.test(error.message)) return 502;
|
||||
return 500;
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ import { NextResponse } from "next/server";
|
||||
|
||||
import { requireApiAccess } from "../../../lib/api/auth";
|
||||
import { getRepositoryFromRuntime } from "../../../lib/db/repository";
|
||||
import { LlmValidationError } from "../../../lib/llm/client";
|
||||
import { extractCandidateFactCard } from "../../../lib/workflow/fact-extractor";
|
||||
import { normalizeInput, type RawArticleInput } from "../../../lib/workflow/input-normalizer";
|
||||
|
||||
@@ -26,7 +27,7 @@ export async function POST(request: Request) {
|
||||
|
||||
return NextResponse.json({ job, candidateFactCard }, { status: 201 });
|
||||
} catch (error) {
|
||||
return jsonError(error, 400);
|
||||
return jsonError(error, getErrorStatus(error));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,3 +35,9 @@ function jsonError(error: unknown, status: number) {
|
||||
const message = error instanceof Error ? error.message : "Request failed";
|
||||
return NextResponse.json({ error: message }, { status });
|
||||
}
|
||||
|
||||
function getErrorStatus(error: unknown) {
|
||||
if (error instanceof LlmValidationError) return 502;
|
||||
if (error instanceof Error && /^LLM\b|provider/i.test(error.message)) return 502;
|
||||
return 400;
|
||||
}
|
||||
|
||||
@@ -16,16 +16,16 @@ describe("generateValidatedJson", () => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("returns null when no provider key is configured", async () => {
|
||||
it("throws clearly when no provider key is configured", async () => {
|
||||
process.env.LLM_PROVIDER = "deepseek";
|
||||
delete process.env.DEEPSEEK_API_KEY;
|
||||
|
||||
const result = await client.generateValidatedJson({
|
||||
await expect(
|
||||
client.generateValidatedJson({
|
||||
schema: z.object({ value: z.string() }),
|
||||
prompt: "Return JSON.",
|
||||
});
|
||||
|
||||
expect(result).toBeNull();
|
||||
}),
|
||||
).rejects.toThrow("DEEPSEEK_API_KEY is missing");
|
||||
});
|
||||
|
||||
it("returns parsed data when the model response matches the schema", async () => {
|
||||
@@ -41,32 +41,32 @@ describe("generateValidatedJson", () => {
|
||||
expect(result).toEqual({ value: "from-llm" });
|
||||
});
|
||||
|
||||
it("returns null when the model response fails schema validation", async () => {
|
||||
it("throws clearly when the model response fails schema validation", async () => {
|
||||
process.env.LLM_PROVIDER = "deepseek";
|
||||
process.env.DEEPSEEK_API_KEY = "test-key";
|
||||
client.setGenerateJsonForValidation(async () => ({ value: 42 }));
|
||||
|
||||
const result = await client.generateValidatedJson({
|
||||
await expect(
|
||||
client.generateValidatedJson({
|
||||
schema: z.object({ value: z.string() }),
|
||||
prompt: "Return JSON.",
|
||||
}),
|
||||
).rejects.toThrow("LLM response failed schema validation");
|
||||
});
|
||||
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null when the provider call rejects", async () => {
|
||||
it("throws clearly when the provider call rejects", async () => {
|
||||
process.env.LLM_PROVIDER = "deepseek";
|
||||
process.env.DEEPSEEK_API_KEY = "test-key";
|
||||
client.setGenerateJsonForValidation(async () => {
|
||||
throw new Error("provider down");
|
||||
});
|
||||
|
||||
const result = await client.generateValidatedJson({
|
||||
await expect(
|
||||
client.generateValidatedJson({
|
||||
schema: z.object({ value: z.string() }),
|
||||
prompt: "Return JSON.",
|
||||
});
|
||||
|
||||
expect(result).toBeNull();
|
||||
}),
|
||||
).rejects.toThrow("provider down");
|
||||
});
|
||||
|
||||
it("logs validation success with the supplied task label", async () => {
|
||||
@@ -93,13 +93,13 @@ describe("generateValidatedJson", () => {
|
||||
process.env.DEEPSEEK_API_KEY = "test-key";
|
||||
client.setGenerateJsonForValidation(async () => ({ value: 42 }));
|
||||
|
||||
const result = await client.generateValidatedJson({
|
||||
await expect(
|
||||
client.generateValidatedJson({
|
||||
schema: z.object({ value: z.string() }),
|
||||
prompt: "Return JSON.",
|
||||
task: "fact_extractor",
|
||||
});
|
||||
|
||||
expect(result).toBeNull();
|
||||
}),
|
||||
).rejects.toThrow("LLM response failed schema validation");
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining("[llm:validated] task=fact_extractor ok=false"),
|
||||
);
|
||||
@@ -150,16 +150,16 @@ describe("generateValidatedJson", () => {
|
||||
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
client.setGenerateJsonForValidation(async () => ({ value: 42 }));
|
||||
|
||||
const result = await client.generateValidatedJson({
|
||||
await expect(
|
||||
client.generateValidatedJson({
|
||||
schema: z.object({ value: z.string() }),
|
||||
task: "fact_extractor",
|
||||
prompt: "Return JSON.",
|
||||
});
|
||||
|
||||
}),
|
||||
).rejects.toThrow("LLM response failed schema validation");
|
||||
const allLogs = [...infoSpy.mock.calls, ...warnSpy.mock.calls]
|
||||
.flat()
|
||||
.join("\n");
|
||||
expect(result).toBeNull();
|
||||
expect(allLogs).toContain("[llm:validated] task=fact_extractor ok=false");
|
||||
expect(allLogs).toContain("zod_error=");
|
||||
expect(allLogs).not.toContain("super-secret-key");
|
||||
@@ -173,13 +173,13 @@ describe("generateValidatedJson", () => {
|
||||
throw new Error("provider unavailable");
|
||||
});
|
||||
|
||||
const result = await client.generateValidatedJson({
|
||||
await expect(
|
||||
client.generateValidatedJson({
|
||||
schema: z.object({ value: z.string() }),
|
||||
task: "quality_inspector",
|
||||
prompt: "Return JSON.",
|
||||
});
|
||||
|
||||
expect(result).toBeNull();
|
||||
}),
|
||||
).rejects.toThrow("provider unavailable");
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining("[llm:error] task=quality_inspector"),
|
||||
);
|
||||
|
||||
+25
-6
@@ -28,6 +28,16 @@ export interface LlmProviderStatus {
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
export class LlmValidationError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public readonly task: LlmTaskName,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "LlmValidationError";
|
||||
}
|
||||
}
|
||||
|
||||
function getProvider() {
|
||||
return (process.env.LLM_PROVIDER || "deepseek").toLowerCase();
|
||||
}
|
||||
@@ -80,8 +90,8 @@ export function setChatCompletionForTesting(
|
||||
chatCompletionForTesting = handler;
|
||||
}
|
||||
|
||||
function getTask(input: GenerateInput) {
|
||||
return input.task?.trim() || "unknown";
|
||||
function getTask(input: GenerateInput): LlmTaskName {
|
||||
return input.task || "unknown";
|
||||
}
|
||||
|
||||
function getRawLogLimit() {
|
||||
@@ -204,11 +214,14 @@ export function setGenerateJsonForValidation(
|
||||
export async function generateValidatedJson<T>({
|
||||
schema,
|
||||
...input
|
||||
}: GenerateValidatedJsonInput<T>): Promise<T | null> {
|
||||
}: GenerateValidatedJsonInput<T>): Promise<T> {
|
||||
const task = getTask(input);
|
||||
if (!isLlmConfigured()) {
|
||||
console.info(`[llm:validated] task=${task} ok=false reason=not_configured`);
|
||||
return null;
|
||||
throw new LlmValidationError(
|
||||
getLlmProviderStatus().reason ?? "LLM provider is not configured",
|
||||
task,
|
||||
);
|
||||
}
|
||||
|
||||
const status = getLlmProviderStatus();
|
||||
@@ -230,14 +243,20 @@ export async function generateValidatedJson<T>({
|
||||
console.warn(
|
||||
`[llm:validated] task=${task} ok=false zod_error=${quoteLogValue(summarizeZodError(parsed.error))}`,
|
||||
);
|
||||
return null;
|
||||
throw new LlmValidationError(
|
||||
`LLM response failed schema validation: ${summarizeZodError(parsed.error)}`,
|
||||
task,
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof LlmValidationError) {
|
||||
throw error;
|
||||
}
|
||||
console.info(`[llm:validated] task=${task} ok=false reason=provider_error`);
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.warn(
|
||||
`[llm:error] task=${task} duration_ms=${Date.now() - startedAt} message=${quoteLogValue(message)}`,
|
||||
);
|
||||
return null;
|
||||
throw error instanceof Error ? error : new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ export const JSON_ONLY_PROMPT =
|
||||
"Return valid JSON only. Do not include markdown fences or commentary.";
|
||||
|
||||
const CUSTOMER_RISK_GUIDANCE = [
|
||||
"客户最担心的内容风险:行业漂移、公司全称/简称/品牌名不一致、图片主题与正文描述不匹配、官网文章出现第三方口吻、平台语气和文章类型不匹配、标题或正文语义不顺、虚构资质/年限/案例/能力、产品/服务/年限前后冲突。",
|
||||
"客户最担心的内容风险:行业漂移、公司全称/简称/品牌名不一致、官网文章出现第三方口吻、平台语气和文章类型不匹配、标题或正文语义不顺、虚构资质/年限/案例/能力、产品/服务/年限前后冲突。",
|
||||
"任何客户案例、资质荣誉、经验年限、服务能力、出海/多语种/合规能力、效果承诺和排名,都必须能从原文或已确认事实卡中找到明确依据。",
|
||||
].join(" ");
|
||||
|
||||
@@ -99,7 +99,7 @@ export function buildArticleOptimizerPrompt(
|
||||
"- 必须保留事实卡确认的公司全称、目标行业、目标受众和核心事实。",
|
||||
"- 必须删除或弱化 factCard.forbidden_claims 中的主张。",
|
||||
"- 不得新增客户案例、数字、资质、排名、奖项、服务能力、效果承诺。",
|
||||
"- image_suggestions 必须基于 factCard.image_topics 或原始 images;没有图片主题时返回空数组。",
|
||||
"- 当前版本只优化文本,不生成图片建议;image_suggestions 必须返回空数组 []。",
|
||||
"",
|
||||
"Confirmed fact card:",
|
||||
JSON.stringify(factCard, null, 2),
|
||||
@@ -124,8 +124,9 @@ export function buildQualityInspectorPrompt(input: {
|
||||
formatPlatformGuidance(input.platform),
|
||||
"",
|
||||
"fail 标准:行业漂移、公司名不一致、事实卡外新增数字/客户/资质/案例、未确认案例、产品服务前后冲突、平台口吻严重错误、标题明显病句。",
|
||||
"warn 标准:图片证据不足、句子过长、表达可读性一般、平台适配轻微不足。",
|
||||
"target_agent 只能使用 title、body、image、fact_card 或 null。",
|
||||
"warn 标准:句子过长、表达可读性一般、平台适配轻微不足。",
|
||||
"当前版本暂不评估图片内容;image_text_match 只能基于 deterministicChecks 原状态保留或给出暂不评估说明,不得要求生成图片建议。",
|
||||
"target_agent 只能使用 title、body、fact_card 或 null。",
|
||||
"",
|
||||
"Confirmed fact card:",
|
||||
JSON.stringify(input.factCard, null, 2),
|
||||
@@ -155,7 +156,7 @@ export function buildTargetedRewritePrompt(input: {
|
||||
"- title_quality:生成自然中文标题,禁止英文模板词。",
|
||||
"- body_quality:拆分长句,修复病句和断裂表达。",
|
||||
"- voice_consistency / platform_fit:改成目标平台对应口吻。",
|
||||
"- image_text_match:只补充图片建议或人工确认项,不虚构图片内容。",
|
||||
"- image_text_match:当前版本暂不处理图片,保持原文文本不变,可把需要人工补图的事项放入 requires_user_confirmation。",
|
||||
"不得新增事实。无法修复的内容放入 requires_user_confirmation。",
|
||||
"",
|
||||
"Confirmed fact card:",
|
||||
|
||||
@@ -72,19 +72,20 @@ describe("LLM workflow integration", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("falls back to deterministic candidate extraction when LLM returns null", async () => {
|
||||
llmMocks.generateValidatedJson.mockResolvedValueOnce(null);
|
||||
it("surfaces candidate fact extraction LLM failures instead of falling back", async () => {
|
||||
llmMocks.generateValidatedJson.mockRejectedValueOnce(
|
||||
new Error("LLM response failed schema validation: target_audience"),
|
||||
);
|
||||
|
||||
const card = await extractCandidateFactCard({
|
||||
await expect(
|
||||
extractCandidateFactCard({
|
||||
title: "Fallback Technology Co., Ltd. GEO guide",
|
||||
body: "Fallback Technology Co., Ltd. has 8 years of GEO optimization experience.",
|
||||
images: [{ type: "description", content: "dashboard" }],
|
||||
platform: "official_site",
|
||||
user_instructions: "",
|
||||
});
|
||||
|
||||
expect(card.company_full_name).toBe("Fallback Technology Co., Ltd.");
|
||||
expect(card.experience_years).toBe(8);
|
||||
}),
|
||||
).rejects.toThrow("LLM response failed schema validation: target_audience");
|
||||
});
|
||||
|
||||
it("uses LLM output for article optimization when valid", async () => {
|
||||
@@ -92,7 +93,7 @@ describe("LLM workflow integration", () => {
|
||||
title: "LLM Optimized GEO Article",
|
||||
summary: "LLM summary constrained by the fact card.",
|
||||
body_markdown: "## LLM Body\nExample Technology Co., Ltd. keeps claims factual.",
|
||||
image_suggestions: [{ source: "image_1", suggestion: "Use dashboard." }],
|
||||
image_suggestions: [],
|
||||
changed_sections: ["title", "body"],
|
||||
requires_user_confirmation: [],
|
||||
});
|
||||
@@ -110,16 +111,20 @@ describe("LLM workflow integration", () => {
|
||||
|
||||
expect(article.title).toBe("LLM Optimized GEO Article");
|
||||
expect(article.body_markdown).toContain("LLM Body");
|
||||
expect(article.image_suggestions).toEqual([]);
|
||||
expect(llmMocks.generateValidatedJson).toHaveBeenCalledOnce();
|
||||
expect(llmMocks.generateValidatedJson).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ task: "article_optimizer" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("falls back to deterministic article optimization when LLM returns null", async () => {
|
||||
llmMocks.generateValidatedJson.mockResolvedValueOnce(null);
|
||||
it("surfaces article optimization LLM failures instead of falling back", async () => {
|
||||
llmMocks.generateValidatedJson.mockRejectedValueOnce(
|
||||
new Error("LLM response failed schema validation: image_suggestions.0.source"),
|
||||
);
|
||||
|
||||
const article = await optimizeArticle({
|
||||
await expect(
|
||||
optimizeArticle({
|
||||
input: {
|
||||
title: "Original",
|
||||
body: "Example Technology Co., Ltd. has 8 years of GEO optimization experience.",
|
||||
@@ -128,11 +133,9 @@ describe("LLM workflow integration", () => {
|
||||
user_instructions: "Say we have 99 patents.",
|
||||
},
|
||||
factCard: confirmedFactCard,
|
||||
});
|
||||
|
||||
expect(article.title).toContain("GEO optimization Guide");
|
||||
expect(article.requires_user_confirmation).toContain(
|
||||
"Unsupported requested claim: 99 patents",
|
||||
}),
|
||||
).rejects.toThrow(
|
||||
"LLM response failed schema validation: image_suggestions.0.source",
|
||||
);
|
||||
});
|
||||
|
||||
@@ -176,10 +179,11 @@ describe("LLM workflow integration", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("falls back to deterministic targeted rewrite when LLM returns null", async () => {
|
||||
llmMocks.generateValidatedJson.mockResolvedValueOnce(null);
|
||||
it("surfaces targeted rewrite LLM failures instead of falling back", async () => {
|
||||
llmMocks.generateValidatedJson.mockRejectedValueOnce(new Error("provider unavailable"));
|
||||
|
||||
const rewritten = await rewriteFailedSections({
|
||||
await expect(
|
||||
rewriteFailedSections({
|
||||
article: {
|
||||
title: "Bad title!!!",
|
||||
summary: "Original summary",
|
||||
@@ -199,10 +203,8 @@ describe("LLM workflow integration", () => {
|
||||
target_agent: "title",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(rewritten.title).toContain("GEO optimization Guide");
|
||||
expect(rewritten.summary).toBe("Original summary");
|
||||
}),
|
||||
).rejects.toThrow("provider unavailable");
|
||||
});
|
||||
|
||||
it("uses LLM quality checks to enrich non-failing deterministic checks", async () => {
|
||||
|
||||
@@ -1,12 +1,8 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { ConfirmedFactCard } from "../../domain/types";
|
||||
import { optimizeArticle } from "../article-optimizer";
|
||||
import { extractCandidateFactCard } from "../fact-extractor";
|
||||
import { normalizeInput } from "../input-normalizer";
|
||||
import { inspectQuality } from "../quality-inspector";
|
||||
import { runOptimizationWorkflow } from "../orchestrator";
|
||||
import { rewriteFailedSections } from "../targeted-rewriter";
|
||||
|
||||
const confirmedFactCard: ConfirmedFactCard = {
|
||||
company_full_name: "Example Technology Co., Ltd.",
|
||||
@@ -45,65 +41,6 @@ describe("workflow nodes", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("places missing or conflicting company facts into uncertain items", async () => {
|
||||
const card = await extractCandidateFactCard({
|
||||
title: "Example announces GEO product",
|
||||
body: "Example has 8 years of experience. Example has 12 years of service. The article discusses GEO optimization.",
|
||||
images: [],
|
||||
platform: "media_article",
|
||||
user_instructions: "",
|
||||
});
|
||||
|
||||
expect(card.company_full_name).toBe("");
|
||||
expect(card.uncertain_items).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.stringContaining("company full name"),
|
||||
expect.stringContaining("Conflicting experience years"),
|
||||
]),
|
||||
);
|
||||
expect(card.is_ready_for_optimization).toBe(false);
|
||||
});
|
||||
|
||||
it("extracts Chinese company facts from Chinese articles", async () => {
|
||||
const card = await extractCandidateFactCard({
|
||||
title: "#探寻AIGC短视频培训选哪家,各品牌实力大比拼",
|
||||
body: "伟思德鲁管理咨询(深圳)有限公司面向品牌商家和出海企业提供AIGC短视频培训服务,帮助企业解决内容工业化生产、品牌视觉统一和全球化传播问题。",
|
||||
images: [{ type: "description", content: "AIGC短视频工作流示意图" }],
|
||||
platform: "media_article",
|
||||
user_instructions: "保留AIGC短视频培训与出海内容生产场景。",
|
||||
});
|
||||
|
||||
expect(card.company_full_name).toBe("伟思德鲁管理咨询(深圳)有限公司");
|
||||
expect(card.company_short_names).toContain("伟思德鲁");
|
||||
expect(card.target_industry).toBe("AIGC短视频培训");
|
||||
expect(card.target_audience).toBe("品牌商家、内容创作者、出海企业");
|
||||
expect(card.image_topics).toEqual(["AIGC短视频工作流示意图"]);
|
||||
expect(card.uncertain_items).not.toContain("Missing company full name");
|
||||
});
|
||||
|
||||
it("does not add claims outside the confirmed fact card", async () => {
|
||||
const optimized = await optimizeArticle({
|
||||
input: {
|
||||
title: "Example GEO article",
|
||||
body: "Example GEO helps marketing teams improve content structure.",
|
||||
images: [],
|
||||
platform: "official_site",
|
||||
user_instructions:
|
||||
"Say we have 99 patents and Fortune 500 customer cases.",
|
||||
},
|
||||
factCard: confirmedFactCard,
|
||||
});
|
||||
|
||||
expect(optimized.body_markdown).not.toContain("99 patents");
|
||||
expect(optimized.body_markdown).not.toContain("Fortune 500");
|
||||
expect(optimized.requires_user_confirmation).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.stringContaining("99 patents"),
|
||||
expect.stringContaining("Fortune 500"),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("returns the 10 required quality checks", () => {
|
||||
const report = inspectQuality({
|
||||
article: {
|
||||
@@ -183,50 +120,4 @@ describe("workflow nodes", () => {
|
||||
expect(report.overall_status).toBe("fail");
|
||||
});
|
||||
|
||||
it("rewrites only the failing target area", async () => {
|
||||
const article = {
|
||||
title: "Bad title!!!",
|
||||
summary: "Original summary",
|
||||
body_markdown: "Original body",
|
||||
image_suggestions: [],
|
||||
changed_sections: [],
|
||||
requires_user_confirmation: [],
|
||||
};
|
||||
|
||||
const rewritten = await rewriteFailedSections({
|
||||
article,
|
||||
factCard: confirmedFactCard,
|
||||
failedChecks: [
|
||||
{
|
||||
rule_id: "title_quality",
|
||||
status: "fail",
|
||||
evidence: "Bad title!!!",
|
||||
reason: "Punctuation stuffing.",
|
||||
suggested_fix: "Rewrite title.",
|
||||
target_agent: "title",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(rewritten.title).not.toBe(article.title);
|
||||
expect(rewritten.summary).toBe(article.summary);
|
||||
expect(rewritten.body_markdown).toBe(article.body_markdown);
|
||||
});
|
||||
|
||||
it("orchestrator stops after two failed rewrite rounds", async () => {
|
||||
const result = await runOptimizationWorkflow({
|
||||
input: {
|
||||
title: "Finance automation breakthrough!!!",
|
||||
body: "Example has 12 years in finance automation and 99 patents.",
|
||||
images: [],
|
||||
platform: "official_site",
|
||||
user_instructions: "",
|
||||
},
|
||||
factCard: confirmedFactCard,
|
||||
});
|
||||
|
||||
expect(result.rewrite_rounds).toBe(2);
|
||||
expect(result.qaReport.overall_status).toBe("fail");
|
||||
expect(result.stopped_after_max_rewrites).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -27,77 +27,8 @@ export async function optimizeArticle({
|
||||
task: "article_optimizer",
|
||||
});
|
||||
|
||||
return llmArticle ?? optimizeArticleFallback({ input, factCard });
|
||||
}
|
||||
|
||||
function optimizeArticleFallback({
|
||||
input,
|
||||
factCard,
|
||||
}: OptimizeArticleInput): OptimizedArticle {
|
||||
const unsupported = findUnsupportedInstructionClaims(
|
||||
input.user_instructions,
|
||||
factCard,
|
||||
);
|
||||
const title = `${factCard.brand_names[0] ?? factCard.company_short_names[0] ?? factCard.company_full_name} ${factCard.target_industry} Guide`;
|
||||
const coreClaims =
|
||||
factCard.core_claims.length > 0
|
||||
? factCard.core_claims.map((claim) => `- ${claim}`).join("\n")
|
||||
: "- Confirmed facts only; no extra claims added.";
|
||||
const body = [
|
||||
`## ${factCard.company_full_name}`,
|
||||
cleanBody(input.body, factCard),
|
||||
"",
|
||||
"### Confirmed Facts",
|
||||
coreClaims,
|
||||
].join("\n");
|
||||
|
||||
return optimizedArticleSchema.parse({
|
||||
title,
|
||||
summary: `A ${input.platform.replace(/_/g, " ")} article for ${factCard.target_audience} about ${factCard.target_industry}.`,
|
||||
body_markdown: body,
|
||||
image_suggestions: factCard.image_topics.map((topic, index) => ({
|
||||
source: `image_${index + 1}`,
|
||||
suggestion: `Use image content related to ${topic}.`,
|
||||
})),
|
||||
changed_sections: ["title", "body structure", "summary"],
|
||||
requires_user_confirmation: unsupported,
|
||||
...llmArticle,
|
||||
image_suggestions: [],
|
||||
});
|
||||
}
|
||||
|
||||
function cleanBody(body: string, factCard: ConfirmedFactCard) {
|
||||
let cleaned = body.trim();
|
||||
for (const forbidden of factCard.forbidden_claims) {
|
||||
cleaned = cleaned.replace(new RegExp(escapeRegExp(forbidden), "gi"), "");
|
||||
}
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
function findUnsupportedInstructionClaims(
|
||||
instructions: string,
|
||||
factCard: ConfirmedFactCard,
|
||||
) {
|
||||
const unsupported: string[] = [];
|
||||
const numbers = [...instructions.matchAll(/\b\d+\s*[A-Za-z]+\b/g)].map(
|
||||
(match) => match[0],
|
||||
);
|
||||
const knownText = [
|
||||
factCard.experience_years?.toString() ?? "",
|
||||
...factCard.core_claims,
|
||||
].join(" ");
|
||||
|
||||
for (const claim of numbers) {
|
||||
if (!knownText.includes(claim.replace(/\D/g, ""))) {
|
||||
unsupported.push(`Unsupported requested claim: ${claim}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (/fortune\s*500/i.test(instructions)) {
|
||||
unsupported.push("Unsupported requested claim: Fortune 500 customer cases");
|
||||
}
|
||||
|
||||
return unsupported;
|
||||
}
|
||||
|
||||
function escapeRegExp(value: string) {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
|
||||
@@ -9,131 +9,11 @@ import {
|
||||
export async function extractCandidateFactCard(
|
||||
input: ArticleInput,
|
||||
): Promise<CandidateFactCard> {
|
||||
const llmCard = await generateValidatedJson({
|
||||
return generateValidatedJson({
|
||||
schema: candidateFactCardSchema,
|
||||
system: FACT_EXTRACTOR_SYSTEM_PROMPT,
|
||||
prompt: buildFactExtractorPrompt(input),
|
||||
temperature: 0.1,
|
||||
task: "fact_extractor",
|
||||
});
|
||||
|
||||
return llmCard ?? extractCandidateFactCardFallback(input);
|
||||
}
|
||||
|
||||
function extractCandidateFactCardFallback(input: ArticleInput): CandidateFactCard {
|
||||
const text = `${input.title}\n${input.body}`;
|
||||
const uncertainItems: string[] = [];
|
||||
const companyFullName = findCompanyFullName(text);
|
||||
const years = findExperienceYears(text);
|
||||
|
||||
if (!companyFullName) {
|
||||
uncertainItems.push("Missing company full name");
|
||||
}
|
||||
if (years.length > 1) {
|
||||
uncertainItems.push(`Conflicting experience years: ${years.join(", ")}`);
|
||||
}
|
||||
if (input.images.length === 0) {
|
||||
uncertainItems.push("Image description is missing");
|
||||
}
|
||||
|
||||
const industry = inferIndustry(text);
|
||||
|
||||
return candidateFactCardSchema.parse({
|
||||
company_full_name: companyFullName ?? "",
|
||||
company_short_names: inferCompanyShortNames(companyFullName),
|
||||
brand_names: inferBrandNames(text, companyFullName),
|
||||
product_names: inferProducts(text),
|
||||
target_industry: industry,
|
||||
target_audience: text.toLowerCase().includes("marketing")
|
||||
? "Marketing teams"
|
||||
: inferTargetAudience(text),
|
||||
experience_years: years.length === 1 ? years[0] : null,
|
||||
core_claims: years.length === 1 ? [`${years[0]} years of ${industry} experience`] : [],
|
||||
forbidden_claims: [],
|
||||
image_topics: input.images.map((image) => image.content),
|
||||
uncertain_items: uncertainItems,
|
||||
});
|
||||
}
|
||||
|
||||
function findCompanyFullName(text: string) {
|
||||
const englishMatch = text.match(
|
||||
/([A-Z][A-Za-z0-9&.,\-\s]{2,}?(?:Co\.,?\s*Ltd\.?|Company|Inc\.?|LLC|Ltd\.))/,
|
||||
);
|
||||
if (englishMatch?.[1]) return englishMatch[1].trim();
|
||||
|
||||
const chineseMatch = text.match(
|
||||
/([\u4e00-\u9fa5A-Za-z0-9()()]{2,40}?(?:股份有限公司|有限公司|集团|公司))/,
|
||||
);
|
||||
return chineseMatch?.[1].trim() ?? null;
|
||||
}
|
||||
|
||||
function findExperienceYears(text: string) {
|
||||
const matches = [...text.matchAll(/\b(\d{1,3})\s*(?:years?|年)\b/gi)];
|
||||
return [...new Set(matches.map((match) => Number(match[1])))];
|
||||
}
|
||||
|
||||
function inferIndustry(text: string) {
|
||||
const lower = text.toLowerCase();
|
||||
if (lower.includes("geo")) return "GEO optimization";
|
||||
if (lower.includes("finance") || lower.includes("banking")) return "finance automation";
|
||||
if (lower.includes("seo")) return "SEO";
|
||||
if (/AIGC|短视频|出海内容|内容工业化/.test(text)) return "AIGC短视频培训";
|
||||
if (/工业零部件|爆品操盘|AI OBS|IPMS/.test(text)) return "工业零部件爆品操盘";
|
||||
return "General business";
|
||||
}
|
||||
|
||||
function inferBrandNames(text: string, companyFullName: string | null) {
|
||||
const chineseBrands = [
|
||||
...new Set(
|
||||
[
|
||||
companyFullName ? inferChineseShortName(companyFullName) : "",
|
||||
...[...text.matchAll(/\b(AIGC|AI OBS|IPMS|GEO|SEO)\b/g)].map(
|
||||
(match) => match[1],
|
||||
),
|
||||
].filter(Boolean),
|
||||
),
|
||||
];
|
||||
const names = [...text.matchAll(/\b[A-Z][A-Za-z0-9]{2,}\b/g)]
|
||||
.map((match) => match[0])
|
||||
.filter((word) => !["The", "This", "And"].includes(word));
|
||||
return [...new Set([...chineseBrands, ...names])].slice(0, 5);
|
||||
}
|
||||
|
||||
function inferProducts(text: string) {
|
||||
const productMatches = [
|
||||
...[...text.matchAll(/\b([A-Z][A-Za-z0-9]+\s+GEO)\b/g)].map(
|
||||
(match) => match[1],
|
||||
),
|
||||
...[...text.matchAll(/\b(AIGC短视频培训|AI OBS|IPMS|爆品操盘数智系统)\b/g)].map(
|
||||
(match) => match[1],
|
||||
),
|
||||
];
|
||||
return [...new Set(productMatches)];
|
||||
}
|
||||
|
||||
function inferCompanyShortNames(companyFullName: string | null) {
|
||||
if (!companyFullName) return [];
|
||||
if (/[\u4e00-\u9fa5]/.test(companyFullName)) {
|
||||
const legalShortName = inferChineseShortName(companyFullName);
|
||||
const brandShortName = legalShortName.replace(/管理咨询$/, "");
|
||||
return [...new Set([brandShortName, legalShortName].filter(Boolean))];
|
||||
}
|
||||
return [companyFullName.split(/\s+/)[0] ?? ""].filter(Boolean);
|
||||
}
|
||||
|
||||
function inferChineseShortName(companyFullName: string) {
|
||||
return companyFullName
|
||||
.replace(/[((].*?[))]/g, "")
|
||||
.replace(/股份有限公司|有限公司|集团|公司/g, "")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function inferTargetAudience(text: string) {
|
||||
if (/品牌商家|内容创作者|出海企业/.test(text)) {
|
||||
return "品牌商家、内容创作者、出海企业";
|
||||
}
|
||||
if (/工业企业|工业零部件制造商|采购/.test(text)) {
|
||||
return "工业企业、采购团队、工业零部件制造商";
|
||||
}
|
||||
return "Business readers";
|
||||
}
|
||||
|
||||
@@ -68,10 +68,6 @@ export async function inspectQualityWithLlm(
|
||||
task: "quality_inspector",
|
||||
});
|
||||
|
||||
if (!llmPatch) {
|
||||
return deterministicReport;
|
||||
}
|
||||
|
||||
const patchedChecks = deterministicReport.checks.map((deterministicCheck) => {
|
||||
const llmCheck = llmPatch.checks.find(
|
||||
(check) => check.rule_id === deterministicCheck.rule_id,
|
||||
@@ -125,15 +121,12 @@ function inspectRule(
|
||||
}
|
||||
|
||||
if (ruleId === "image_text_match") {
|
||||
const hasImages = sourceImages.length > 0 || article.image_suggestions.length > 0;
|
||||
return check(
|
||||
ruleId,
|
||||
hasImages ? "pass" : "warn",
|
||||
hasImages ? "已有可用于比对的图片主题。" : "未提供图片描述。",
|
||||
hasImages
|
||||
? "图片建议可以和文章内容进行比对。"
|
||||
: "缺少图片描述时,图文匹配置信度较低。",
|
||||
"补充图片描述,或人工检查图片与正文的对应关系。",
|
||||
"pass",
|
||||
sourceImages.length > 0 ? "当前版本暂不评估图片内容。" : "当前版本未启用图片分析。",
|
||||
"当前版本仅优化文本,图片匹配检查暂不参与质量门禁。",
|
||||
"后续启用图片工作流后再补充图文匹配检查。",
|
||||
null,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -25,46 +25,8 @@ export async function rewriteFailedSections({
|
||||
task: "targeted_rewriter",
|
||||
});
|
||||
|
||||
return llmArticle ?? rewriteFailedSectionsFallback({ article, factCard, failedChecks });
|
||||
}
|
||||
|
||||
function rewriteFailedSectionsFallback({
|
||||
article,
|
||||
factCard,
|
||||
failedChecks,
|
||||
}: RewriteFailedSectionsInput): OptimizedArticle {
|
||||
let rewritten = { ...article };
|
||||
|
||||
for (const check of failedChecks) {
|
||||
if (check.target_agent === "title") {
|
||||
rewritten = {
|
||||
...rewritten,
|
||||
title: `${factCard.brand_names[0] ?? factCard.company_short_names[0]} ${factCard.target_industry} Guide`,
|
||||
changed_sections: [...new Set([...rewritten.changed_sections, "title"])],
|
||||
};
|
||||
}
|
||||
|
||||
if (check.target_agent === "body" && check.rule_id === "company_name_integrity") {
|
||||
rewritten = {
|
||||
...rewritten,
|
||||
body_markdown: `${factCard.company_full_name}\n\n${rewritten.body_markdown}`,
|
||||
changed_sections: [...new Set([...rewritten.changed_sections, "company name"])],
|
||||
};
|
||||
}
|
||||
|
||||
if (check.target_agent === "body" && check.rule_id === "claim_consistency") {
|
||||
rewritten = {
|
||||
...rewritten,
|
||||
body_markdown: rewritten.body_markdown.replace(
|
||||
/\b\d{1,3}\s*(?:years?|年)\b/gi,
|
||||
factCard.experience_years === null
|
||||
? "confirmed experience"
|
||||
: `${factCard.experience_years} years`,
|
||||
),
|
||||
changed_sections: [...new Set([...rewritten.changed_sections, "claim consistency"])],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return rewritten;
|
||||
return optimizedArticleSchema.parse({
|
||||
...llmArticle,
|
||||
image_suggestions: [],
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user