feat: show backend optimization progress

This commit is contained in:
Codex
2026-06-21 23:43:03 +08:00
parent 5a627bcaa8
commit 21aa75d240
15 changed files with 549 additions and 43 deletions
+25
View File
@@ -142,6 +142,31 @@ describe("generateValidatedJson", () => {
expect(warnSpy).not.toHaveBeenCalled();
});
it("does not duplicate start and response logs when using the default JSON generator", async () => {
process.env.LLM_PROVIDER = "deepseek";
process.env.DEEPSEEK_API_KEY = "test-key";
const infoSpy = vi.spyOn(console, "info").mockImplementation(() => {});
client.setChatCompletionForTesting(async () => ({
choices: [{ message: { content: '{"value":"from-llm"}' } }],
}));
const result = await client.generateValidatedJson({
schema: z.object({ value: z.string() }),
prompt: "Return JSON.",
task: "article_optimizer",
});
expect(result).toEqual({ value: "from-llm" });
const startLogs = infoSpy.mock.calls
.map((call) => call[0])
.filter((line) => line.startsWith("[llm:start]"));
const responseLogs = infoSpy.mock.calls
.map((call) => call[0])
.filter((line) => line.startsWith("[llm:response]"));
expect(startLogs).toHaveLength(1);
expect(responseLogs).toHaveLength(1);
});
it("logs validation failure without leaking provider secrets", async () => {
process.env.LLM_PROVIDER = "deepseek";
process.env.DEEPSEEK_API_KEY = "super-secret-key";
+14 -7
View File
@@ -224,17 +224,22 @@ export async function generateValidatedJson<T>({
);
}
const usesDefaultGenerator = generateJsonForValidation === generateJson;
const status = getLlmProviderStatus();
const startedAt = Date.now();
console.info(
`[llm:start] provider=${status.provider} model=${input.model ?? status.model} task=${task}`,
);
if (!usesDefaultGenerator) {
console.info(
`[llm:start] provider=${status.provider} model=${input.model ?? status.model} task=${task}`,
);
}
try {
const generated = await generateJsonForValidation<unknown>(input);
console.info(
`[llm:response] task=${task} duration_ms=${Date.now() - startedAt} raw=${truncateRaw(stringifyForLog(generated))}`,
);
if (!usesDefaultGenerator) {
console.info(
`[llm:response] task=${task} duration_ms=${Date.now() - startedAt} raw=${truncateRaw(stringifyForLog(generated))}`,
);
}
const parsed = schema.safeParse(generated);
if (parsed.success) {
console.info(`[llm:validated] task=${task} ok=true`);
@@ -254,7 +259,9 @@ export async function generateValidatedJson<T>({
console.info(`[llm:validated] task=${task} ok=false reason=provider_error`);
const message = error instanceof Error ? error.message : String(error);
console.warn(
`[llm:error] task=${task} duration_ms=${Date.now() - startedAt} message=${quoteLogValue(message)}`,
usesDefaultGenerator
? `[llm:error] task=${task} message=${quoteLogValue(message)}`
: `[llm:error] task=${task} duration_ms=${Date.now() - startedAt} message=${quoteLogValue(message)}`,
);
throw error instanceof Error ? error : new Error(message);
}
+1 -1
View File
@@ -29,7 +29,7 @@ describe("progress helpers", () => {
expect(getProgressStages("optimize").map((stage) => stage.label)).toEqual([
"生成优化稿",
"质量检查",
"必要时定向修复",
"等待后端步骤更新",
"整理结果",
]);
});
+1 -1
View File
@@ -17,7 +17,7 @@ const progressStages: Record<ProgressAction, ProgressStage[]> = {
optimize: [
{ label: "生成优化稿" },
{ label: "质量检查" },
{ label: "必要时定向修复" },
{ label: "等待后端步骤更新" },
{ label: "整理结果" },
],
};
@@ -282,4 +282,41 @@ describe("LLM workflow integration", () => {
expect(companyCheck?.status).toBe("fail");
expect(companyCheck?.reason).toContain("公司");
});
it("does not let LLM escalate soft quality checks to hard failures", async () => {
llmMocks.generateValidatedJson.mockResolvedValueOnce({
checks: [
{
rule_id: "platform_fit",
status: "fail",
evidence: "LLM thinks the structure is not recommendation-like enough.",
reason: "This is a soft platform-fit concern.",
suggested_fix: "Adjust structure if needed.",
target_agent: "body",
},
],
});
const report = await inspectQualityWithLlm({
article: {
title: "Example GEO Optimization Guide",
summary:
"A official site article for Marketing teams about GEO optimization.",
body_markdown:
"Example Technology Co., Ltd. has 8 years of GEO optimization experience.",
image_suggestions: [],
changed_sections: [],
requires_user_confirmation: [],
},
factCard: confirmedFactCard,
platform: "official_site",
sourceImages: [],
});
const platformCheck = report.checks.find(
(check) => check.rule_id === "platform_fit",
);
expect(platformCheck?.status).toBe("warn");
expect(report.overall_status).not.toBe("fail");
});
});
@@ -0,0 +1,101 @@
import { describe, expect, it, vi } from "vitest";
import type { ConfirmedFactCard } from "../../domain/types";
import { runOptimizationWorkflow } from "../orchestrator";
const workflowMocks = vi.hoisted(() => ({
optimizeArticle: vi.fn(),
inspectQualityWithLlm: vi.fn(),
rewriteFailedSections: vi.fn(),
}));
vi.mock("../article-optimizer", () => ({
optimizeArticle: workflowMocks.optimizeArticle,
}));
vi.mock("../quality-inspector", () => ({
inspectQualityWithLlm: workflowMocks.inspectQualityWithLlm,
}));
vi.mock("../targeted-rewriter", () => ({
rewriteFailedSections: workflowMocks.rewriteFailedSections,
}));
const factCard: ConfirmedFactCard = {
company_full_name: "Example Technology Co., Ltd.",
company_short_names: ["Example Tech"],
brand_names: ["Example"],
product_names: ["Example GEO"],
target_industry: "GEO optimization",
target_audience: "Marketing teams",
experience_years: 8,
core_claims: ["Eight years of GEO optimization experience"],
forbidden_claims: [],
image_topics: [],
uncertain_items: [],
is_ready_for_optimization: true,
confirmed_by_user: true,
};
const article = {
title: "Optimized",
summary: "Summary",
body_markdown: "Body",
image_suggestions: [],
changed_sections: [],
requires_user_confirmation: [],
};
const failCheck = {
rule_id: "body_quality" as const,
status: "fail" as const,
evidence: "Bad body.",
reason: "Needs rewrite.",
suggested_fix: "Rewrite body.",
target_agent: "body",
};
describe("runOptimizationWorkflow", () => {
it("reports live progress for initial generation, QA, rewrite rounds, and finalization", async () => {
workflowMocks.optimizeArticle.mockResolvedValueOnce(article);
workflowMocks.inspectQualityWithLlm
.mockResolvedValueOnce({
overall_status: "fail",
checks: [failCheck],
})
.mockResolvedValueOnce({
overall_status: "pass",
checks: [],
});
workflowMocks.rewriteFailedSections.mockResolvedValueOnce({
...article,
body_markdown: "Rewritten body",
});
const events: Array<{ label: string; status: string }> = [];
await runOptimizationWorkflow({
input: {
title: "Original",
body: "Original body",
images: [],
platform: "official_site",
user_instructions: "",
},
factCard,
onProgress: (event) => events.push({ label: event.label, status: event.status }),
});
expect(events).toEqual([
{ label: "生成优化稿", status: "running" },
{ label: "生成优化稿", status: "completed" },
{ label: "质量检查", status: "running" },
{ label: "质量检查", status: "completed" },
{ label: "定向修复第 1 轮", status: "running" },
{ label: "定向修复第 1 轮", status: "completed" },
{ label: "质量复检第 1 轮", status: "running" },
{ label: "质量复检第 1 轮", status: "completed" },
{ label: "整理结果", status: "running" },
{ label: "整理结果", status: "completed" },
]);
});
});
+27 -2
View File
@@ -92,7 +92,7 @@ describe("workflow nodes", () => {
expect(report.checks[0]?.evidence).not.toContain("Article");
});
it("hard-fails incomplete company names, hallucinated numbers, industry drift, and conflicting years", () => {
it("hard-fails incomplete company names, hallucinated numbers, and conflicting years while warning on industry drift", () => {
const report = inspectQuality({
article: {
title: "Example Wins Finance Automation Market!!!",
@@ -113,11 +113,36 @@ describe("workflow nodes", () => {
expect.arrayContaining([
"company_name_integrity",
"hallucination_risk",
"industry_alignment",
"claim_consistency",
]),
);
const industryCheck = report.checks.find(
(check) => check.rule_id === "industry_alignment",
);
expect(industryCheck?.status).toBe("warn");
expect(report.overall_status).toBe("fail");
});
it("does not flag numbers already present in confirmed fact card claims", () => {
const report = inspectQuality({
article: {
title: "Example GEO Optimization Guide",
summary: "Example Technology Co., Ltd. serves teams with 8 years of experience.",
body_markdown:
"Example Technology Co., Ltd. has 8 years of GEO optimization experience.",
image_suggestions: [],
changed_sections: [],
requires_user_confirmation: [],
},
factCard: confirmedFactCard,
platform: "official_site",
sourceImages: [],
});
const hallucinationCheck = report.checks.find(
(check) => check.rule_id === "hallucination_risk",
);
expect(hallucinationCheck?.status).toBe("pass");
});
});
+69 -23
View File
@@ -7,6 +7,7 @@ import { rewriteFailedSections } from "./targeted-rewriter";
export interface RunOptimizationWorkflowInput {
input: ArticleInput;
factCard: ConfirmedFactCard;
onProgress?: (event: WorkflowProgressEvent) => void | Promise<void>;
}
export interface WorkflowTimingStep {
@@ -14,6 +15,14 @@ export interface WorkflowTimingStep {
duration_ms: number;
}
export type WorkflowProgressStatus = "running" | "completed" | "failed";
export interface WorkflowProgressEvent {
label: string;
status: WorkflowProgressStatus;
duration_ms?: number;
}
export interface WorkflowTimingSummary {
total_ms: number;
steps: WorkflowTimingStep[];
@@ -23,54 +32,91 @@ async function timedStep<T>(
label: string,
steps: WorkflowTimingStep[],
action: () => Promise<T>,
onProgress?: (event: WorkflowProgressEvent) => void | Promise<void>,
): Promise<T> {
const startedAt = Date.now();
await onProgress?.({ label, status: "running" });
try {
return await action();
} finally {
const result = await action();
const duration = Date.now() - startedAt;
steps.push({
label,
duration_ms: Date.now() - startedAt,
duration_ms: duration,
});
await onProgress?.({ label, status: "completed", duration_ms: duration });
return result;
} catch (error) {
const duration = Date.now() - startedAt;
steps.push({
label,
duration_ms: duration,
});
await onProgress?.({ label, status: "failed", duration_ms: duration });
throw error;
}
}
async function completeStep(
label: string,
steps: WorkflowTimingStep[],
onProgress?: (event: WorkflowProgressEvent) => void | Promise<void>,
) {
await timedStep(label, steps, async () => undefined, onProgress);
}
export async function runOptimizationWorkflow({
input,
factCard,
onProgress,
}: RunOptimizationWorkflowInput) {
const startedAt = Date.now();
const timingSteps: WorkflowTimingStep[] = [];
let article = await timedStep("生成优化稿", timingSteps, () =>
optimizeArticle({ input, factCard }),
let article = await timedStep(
"生成优化稿",
timingSteps,
() => optimizeArticle({ input, factCard }),
onProgress,
);
let qaReport = await timedStep("质量检查", timingSteps, () =>
inspectQualityWithLlm({
article,
factCard,
platform: input.platform,
sourceImages: input.images,
}),
);
let rewriteRounds = 0;
while (qaReport.overall_status === "fail" && rewriteRounds < 2) {
const nextRound = rewriteRounds + 1;
const failedChecks = qaReport.checks.filter((check) => check.status === "fail");
article = await timedStep(`定向修复第 ${nextRound}`, timingSteps, () =>
rewriteFailedSections({ article, factCard, failedChecks }),
);
rewriteRounds = nextRound;
qaReport = await timedStep(`质量复检第 ${nextRound}`, timingSteps, () =>
let qaReport = await timedStep(
"质量检查",
timingSteps,
() =>
inspectQualityWithLlm({
article,
factCard,
platform: input.platform,
sourceImages: input.images,
}),
onProgress,
);
let rewriteRounds = 0;
while (qaReport.overall_status === "fail" && rewriteRounds < 2) {
const nextRound = rewriteRounds + 1;
const failedChecks = qaReport.checks.filter((check) => check.status === "fail");
article = await timedStep(
`定向修复第 ${nextRound}`,
timingSteps,
() => rewriteFailedSections({ article, factCard, failedChecks }),
onProgress,
);
rewriteRounds = nextRound;
qaReport = await timedStep(
`质量复检第 ${nextRound}`,
timingSteps,
() =>
inspectQualityWithLlm({
article,
factCard,
platform: input.platform,
sourceImages: input.images,
}),
onProgress,
);
}
await completeStep("整理结果", timingSteps, onProgress);
return {
article,
qaReport,
+88
View File
@@ -0,0 +1,88 @@
import type {
WorkflowProgressEvent,
WorkflowProgressStatus,
} from "./orchestrator";
export interface WorkflowProgressStep {
label: string;
status: WorkflowProgressStatus;
started_at: string;
completed_at?: string;
duration_ms?: number;
}
export interface WorkflowProgressSnapshot {
job_id: string;
current_step: string | null;
status: "idle" | "running" | "completed" | "failed";
started_at: string | null;
updated_at: string | null;
steps: WorkflowProgressStep[];
}
const progressByJob = new Map<string, WorkflowProgressSnapshot>();
export function startWorkflowProgress(jobId: string) {
const now = new Date().toISOString();
const snapshot: WorkflowProgressSnapshot = {
job_id: jobId,
current_step: null,
status: "running",
started_at: now,
updated_at: now,
steps: [],
};
progressByJob.set(jobId, snapshot);
return snapshot;
}
export function recordWorkflowProgress(
jobId: string,
event: WorkflowProgressEvent,
) {
const now = new Date().toISOString();
const snapshot = progressByJob.get(jobId) ?? startWorkflowProgress(jobId);
const existing = snapshot.steps.find((step) => step.label === event.label);
if (existing) {
existing.status = event.status;
existing.duration_ms = event.duration_ms ?? existing.duration_ms;
if (event.status !== "running") {
existing.completed_at = now;
}
} else {
snapshot.steps.push({
label: event.label,
status: event.status,
started_at: now,
completed_at: event.status === "running" ? undefined : now,
duration_ms: event.duration_ms,
});
}
snapshot.current_step =
event.status === "running" ? event.label : snapshot.current_step;
if (event.status === "failed") {
snapshot.status = "failed";
} else if (event.label === "整理结果" && event.status === "completed") {
snapshot.status = "completed";
snapshot.current_step = null;
} else {
snapshot.status = "running";
}
snapshot.updated_at = now;
return snapshot;
}
export function getWorkflowProgress(jobId: string) {
return (
progressByJob.get(jobId) ?? {
job_id: jobId,
current_step: null,
status: "idle",
started_at: null,
updated_at: null,
steps: [],
}
);
}
+36 -6
View File
@@ -29,6 +29,12 @@ const REQUIRED_RULES: QualityRuleId[] = [
"context_sensitive_terms",
];
const HARD_FAILURE_RULES = new Set<QualityRuleId>([
"company_name_integrity",
"hallucination_risk",
"claim_consistency",
]);
const llmQaPatchSchema = z.object({
checks: z.array(qaCheckSchema).default([]),
});
@@ -78,9 +84,13 @@ export async function inspectQualityWithLlm(
if (deterministicCheck.status === "fail") {
return deterministicCheck;
}
const status =
llmCheck.status === "fail" && !HARD_FAILURE_RULES.has(deterministicCheck.rule_id)
? "warn"
: llmCheck.status;
return {
...deterministicCheck,
status: llmCheck.status,
status,
evidence: llmCheck.evidence,
reason: llmCheck.reason,
suggested_fix: llmCheck.suggested_fix,
@@ -107,14 +117,15 @@ function inspectRule(
const lower = combined.toLowerCase();
if (ruleId === "industry_alignment") {
const aligned = lower.includes(factCard.target_industry.toLowerCase());
const targetIndustry = factCard.target_industry.trim().toLowerCase();
const aligned = targetIndustry.length === 0 || lower.includes(targetIndustry);
return check(
ruleId,
aligned ? "pass" : "fail",
aligned ? "pass" : "warn",
aligned ? factCard.target_industry : article.summary,
aligned
? "文章内容与事实卡确认的目标行业一致。"
: "文章内容偏离事实卡确认的目标行业。",
: "文章可能没有充分体现事实卡确认的目标行业。",
"围绕事实卡确认的目标行业重写相关段落。",
aligned ? null : "body",
);
@@ -254,11 +265,30 @@ function check(
function findUnsupportedNumbers(text: string, factCard: ConfirmedFactCard) {
const allowed = new Set(
[factCard.experience_years]
.filter((value): value is number => typeof value === "number")
[
factCard.experience_years,
...extractNumbersFromFactCard(factCard),
]
.filter((value): value is number | string =>
typeof value === "number" || typeof value === "string",
)
.map(String),
);
return [...text.matchAll(/\b\d{1,4}\b/g)]
.map((match) => match[0])
.filter((number) => !allowed.has(number));
}
function extractNumbersFromFactCard(factCard: ConfirmedFactCard) {
return [
factCard.company_full_name,
...factCard.company_short_names,
...factCard.brand_names,
...factCard.product_names,
factCard.target_industry,
factCard.target_audience,
...factCard.core_claims,
...factCard.forbidden_claims,
...factCard.image_topics,
].flatMap((value) => [...value.matchAll(/\b\d{1,4}\b/g)].map((match) => match[0]));
}