feat: show backend optimization progress
This commit is contained in:
@@ -21,6 +21,7 @@ vi.mock("../../../lib/llm/client", async () => {
|
|||||||
import { POST as confirmFactCard } from "../jobs/[jobId]/confirm-fact-card/route";
|
import { POST as confirmFactCard } from "../jobs/[jobId]/confirm-fact-card/route";
|
||||||
import { GET as downloadExport } from "../jobs/[jobId]/exports/[fileName]/route";
|
import { GET as downloadExport } from "../jobs/[jobId]/exports/[fileName]/route";
|
||||||
import { POST as optimizeJob } from "../jobs/[jobId]/optimize/route";
|
import { POST as optimizeJob } from "../jobs/[jobId]/optimize/route";
|
||||||
|
import { GET as getJobProgress } from "../jobs/[jobId]/progress/route";
|
||||||
import { POST as createJob } from "../jobs/route";
|
import { POST as createJob } from "../jobs/route";
|
||||||
|
|
||||||
const validFactCard = {
|
const validFactCard = {
|
||||||
@@ -236,10 +237,50 @@ describe("job API routes", () => {
|
|||||||
expect(body.timing.steps.map((step) => step.label)).toEqual([
|
expect(body.timing.steps.map((step) => step.label)).toEqual([
|
||||||
"生成优化稿",
|
"生成优化稿",
|
||||||
"质量检查",
|
"质量检查",
|
||||||
|
"整理结果",
|
||||||
]);
|
]);
|
||||||
expect(body.timing.steps.every((step) => step.duration_ms >= 0)).toBe(true);
|
expect(body.timing.steps.every((step) => step.duration_ms >= 0)).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("returns backend-visible optimization progress after a job runs", async () => {
|
||||||
|
const { job } = await createJobFixture();
|
||||||
|
await confirmFactCard(
|
||||||
|
request(validFactCard),
|
||||||
|
params<{ jobId: string }>({ jobId: job.id }),
|
||||||
|
);
|
||||||
|
|
||||||
|
llmMocks.generateValidatedJson
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
title: "API LLM Optimized GEO Article",
|
||||||
|
summary:
|
||||||
|
"A official site article for Marketing teams about GEO optimization.",
|
||||||
|
body_markdown:
|
||||||
|
"Example Technology Co., Ltd. has 8 years of GEO optimization experience.",
|
||||||
|
image_suggestions: [],
|
||||||
|
changed_sections: ["title", "body"],
|
||||||
|
requires_user_confirmation: [],
|
||||||
|
})
|
||||||
|
.mockResolvedValueOnce({ checks: [] });
|
||||||
|
|
||||||
|
await optimizeJob(request({}), params<{ jobId: string }>({ jobId: job.id }));
|
||||||
|
|
||||||
|
const response = await getJobProgress(
|
||||||
|
request({}),
|
||||||
|
params<{ jobId: string }>({ jobId: job.id }),
|
||||||
|
);
|
||||||
|
const body = (await response.json()) as {
|
||||||
|
progress: { status: string; steps: Array<{ label: string; status: string }> };
|
||||||
|
};
|
||||||
|
|
||||||
|
expect(response.status).toBe(200);
|
||||||
|
expect(body.progress.status).toBe("completed");
|
||||||
|
expect(body.progress.steps.map((step) => step.label)).toEqual([
|
||||||
|
"生成优化稿",
|
||||||
|
"质量检查",
|
||||||
|
"整理结果",
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
it("uses mocked LLM article output during optimize route", async () => {
|
it("uses mocked LLM article output during optimize route", async () => {
|
||||||
const { job } = await createJobFixture();
|
const { job } = await createJobFixture();
|
||||||
await confirmFactCard(
|
await confirmFactCard(
|
||||||
|
|||||||
@@ -5,6 +5,10 @@ import { getRepositoryFromRuntime } from "../../../../../lib/db/repository";
|
|||||||
import { LlmValidationError } from "../../../../../lib/llm/client";
|
import { LlmValidationError } from "../../../../../lib/llm/client";
|
||||||
import { getExportStoreFromRuntime } from "../../../../../lib/workflow/export-store";
|
import { getExportStoreFromRuntime } from "../../../../../lib/workflow/export-store";
|
||||||
import { runOptimizationWorkflow } from "../../../../../lib/workflow/orchestrator";
|
import { runOptimizationWorkflow } from "../../../../../lib/workflow/orchestrator";
|
||||||
|
import {
|
||||||
|
recordWorkflowProgress,
|
||||||
|
startWorkflowProgress,
|
||||||
|
} from "../../../../../lib/workflow/progress-store";
|
||||||
|
|
||||||
interface RouteContext {
|
interface RouteContext {
|
||||||
params: Promise<{ jobId: string }>;
|
params: Promise<{ jobId: string }>;
|
||||||
@@ -32,6 +36,7 @@ export async function POST(request: Request, context: RouteContext) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const requestStartedAt = Date.now();
|
const requestStartedAt = Date.now();
|
||||||
|
startWorkflowProgress(jobId);
|
||||||
try {
|
try {
|
||||||
const result = await runOptimizationWorkflow({
|
const result = await runOptimizationWorkflow({
|
||||||
input: {
|
input: {
|
||||||
@@ -42,6 +47,9 @@ export async function POST(request: Request, context: RouteContext) {
|
|||||||
user_instructions: job.user_instructions,
|
user_instructions: job.user_instructions,
|
||||||
},
|
},
|
||||||
factCard: factCardRecord,
|
factCard: factCardRecord,
|
||||||
|
onProgress: (event) => {
|
||||||
|
recordWorkflowProgress(jobId, event);
|
||||||
|
},
|
||||||
});
|
});
|
||||||
const optimizedArticle = await repository.saveOptimizedArticle(
|
const optimizedArticle = await repository.saveOptimizedArticle(
|
||||||
jobId,
|
jobId,
|
||||||
|
|||||||
@@ -0,0 +1,18 @@
|
|||||||
|
import { NextResponse } from "next/server";
|
||||||
|
|
||||||
|
import { requireApiAccess } from "../../../../../lib/api/auth";
|
||||||
|
import { getWorkflowProgress } from "../../../../../lib/workflow/progress-store";
|
||||||
|
|
||||||
|
interface RouteContext {
|
||||||
|
params: Promise<{ jobId: string }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function GET(request: Request, context: RouteContext) {
|
||||||
|
const access = requireApiAccess(request);
|
||||||
|
if (!access.ok) {
|
||||||
|
return access.response;
|
||||||
|
}
|
||||||
|
|
||||||
|
const { jobId } = await context.params;
|
||||||
|
return NextResponse.json({ progress: getWorkflowProgress(jobId) });
|
||||||
|
}
|
||||||
@@ -43,6 +43,22 @@ interface TimingSummary {
|
|||||||
steps: TimingStep[];
|
steps: TimingStep[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface ProgressStep {
|
||||||
|
label: string;
|
||||||
|
status: "running" | "completed" | "failed";
|
||||||
|
duration_ms?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface WorkflowProgress {
|
||||||
|
current_step: string | null;
|
||||||
|
status: "idle" | "running" | "completed" | "failed";
|
||||||
|
steps: ProgressStep[];
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ProgressResponse extends ApiErrorResponse {
|
||||||
|
progress: WorkflowProgress;
|
||||||
|
}
|
||||||
|
|
||||||
interface CreateJobResponse extends ApiErrorResponse {
|
interface CreateJobResponse extends ApiErrorResponse {
|
||||||
job: { id: string };
|
job: { id: string };
|
||||||
candidateFactCard: CandidateFactCard;
|
candidateFactCard: CandidateFactCard;
|
||||||
@@ -71,6 +87,7 @@ export default function Home() {
|
|||||||
const [apiAccessKey, setApiAccessKey] = useState("");
|
const [apiAccessKey, setApiAccessKey] = useState("");
|
||||||
const [elapsedSeconds, setElapsedSeconds] = useState(0);
|
const [elapsedSeconds, setElapsedSeconds] = useState(0);
|
||||||
const [lastTiming, setLastTiming] = useState<TimingSummary | null>(null);
|
const [lastTiming, setLastTiming] = useState<TimingSummary | null>(null);
|
||||||
|
const [liveProgress, setLiveProgress] = useState<WorkflowProgress | null>(null);
|
||||||
|
|
||||||
const exportBlocked = useMemo(
|
const exportBlocked = useMemo(
|
||||||
() => qaReport?.overall_status === "fail",
|
() => qaReport?.overall_status === "fail",
|
||||||
@@ -89,10 +106,37 @@ export default function Home() {
|
|||||||
return () => window.clearInterval(timer);
|
return () => window.clearInterval(timer);
|
||||||
}, [busyAction]);
|
}, [busyAction]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (busyAction !== "optimize" || !jobId) return;
|
||||||
|
let cancelled = false;
|
||||||
|
|
||||||
|
async function pollProgress() {
|
||||||
|
try {
|
||||||
|
const response = await fetch(`/api/jobs/${jobId}/progress`, {
|
||||||
|
headers: apiHeaders(apiAccessKey),
|
||||||
|
});
|
||||||
|
const body = (await response.json()) as ProgressResponse;
|
||||||
|
if (!cancelled && response.ok) {
|
||||||
|
setLiveProgress(body.progress);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// The main optimization request still owns user-facing errors.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void pollProgress();
|
||||||
|
const timer = window.setInterval(pollProgress, 2000);
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
window.clearInterval(timer);
|
||||||
|
};
|
||||||
|
}, [apiAccessKey, busyAction, jobId]);
|
||||||
|
|
||||||
async function analyze() {
|
async function analyze() {
|
||||||
setBusyAction("analyze");
|
setBusyAction("analyze");
|
||||||
setMessage("");
|
setMessage("");
|
||||||
setLastTiming(null);
|
setLastTiming(null);
|
||||||
|
setLiveProgress(null);
|
||||||
setOptimizedArticle(null);
|
setOptimizedArticle(null);
|
||||||
setQaReport(null);
|
setQaReport(null);
|
||||||
try {
|
try {
|
||||||
@@ -148,6 +192,7 @@ export default function Home() {
|
|||||||
setBusyAction("optimize");
|
setBusyAction("optimize");
|
||||||
setMessage("");
|
setMessage("");
|
||||||
setLastTiming(null);
|
setLastTiming(null);
|
||||||
|
setLiveProgress(null);
|
||||||
try {
|
try {
|
||||||
const response = await fetch(`/api/jobs/${jobId}/optimize`, {
|
const response = await fetch(`/api/jobs/${jobId}/optimize`, {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
@@ -158,6 +203,7 @@ export default function Home() {
|
|||||||
setOptimizedArticle(body.optimizedArticle);
|
setOptimizedArticle(body.optimizedArticle);
|
||||||
setQaReport(body.qaReport);
|
setQaReport(body.qaReport);
|
||||||
setLastTiming(body.timing ?? null);
|
setLastTiming(body.timing ?? null);
|
||||||
|
setLiveProgress(null);
|
||||||
const timingText = body.timing
|
const timingText = body.timing
|
||||||
? `用时 ${formatTiming(body.timing.total_ms)}。`
|
? `用时 ${formatTiming(body.timing.total_ms)}。`
|
||||||
: "";
|
: "";
|
||||||
@@ -204,6 +250,7 @@ export default function Home() {
|
|||||||
action={busyAction as ProgressAction | null}
|
action={busyAction as ProgressAction | null}
|
||||||
elapsedSeconds={elapsedSeconds}
|
elapsedSeconds={elapsedSeconds}
|
||||||
lastTiming={lastTiming}
|
lastTiming={lastTiming}
|
||||||
|
liveProgress={liveProgress}
|
||||||
/>
|
/>
|
||||||
<div className="workflow-grid">
|
<div className="workflow-grid">
|
||||||
<ArticleInputForm
|
<ArticleInputForm
|
||||||
|
|||||||
@@ -17,19 +17,37 @@ interface TimingSummary {
|
|||||||
steps: TimingStep[];
|
steps: TimingStep[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface ProgressStep {
|
||||||
|
label: string;
|
||||||
|
status: "running" | "completed" | "failed";
|
||||||
|
duration_ms?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface WorkflowProgress {
|
||||||
|
current_step: string | null;
|
||||||
|
status: "idle" | "running" | "completed" | "failed";
|
||||||
|
steps: ProgressStep[];
|
||||||
|
}
|
||||||
|
|
||||||
interface ProgressPanelProps {
|
interface ProgressPanelProps {
|
||||||
action: ProgressAction | null;
|
action: ProgressAction | null;
|
||||||
elapsedSeconds: number;
|
elapsedSeconds: number;
|
||||||
lastTiming: TimingSummary | null;
|
lastTiming: TimingSummary | null;
|
||||||
|
liveProgress: WorkflowProgress | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function ProgressPanel({
|
export function ProgressPanel({
|
||||||
action,
|
action,
|
||||||
elapsedSeconds,
|
elapsedSeconds,
|
||||||
lastTiming,
|
lastTiming,
|
||||||
|
liveProgress,
|
||||||
}: ProgressPanelProps) {
|
}: ProgressPanelProps) {
|
||||||
if (!action && !lastTiming) return null;
|
if (!action && !lastTiming) return null;
|
||||||
const completedTiming = lastTiming;
|
const completedTiming = lastTiming;
|
||||||
|
const liveSteps =
|
||||||
|
action === "optimize" && liveProgress?.steps.length
|
||||||
|
? liveProgress.steps
|
||||||
|
: null;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<section className="progress-panel" aria-live="polite">
|
<section className="progress-panel" aria-live="polite">
|
||||||
@@ -41,7 +59,16 @@ export function ProgressPanel({
|
|||||||
</div>
|
</div>
|
||||||
<p>{getElapsedNotice(elapsedSeconds)}</p>
|
<p>{getElapsedNotice(elapsedSeconds)}</p>
|
||||||
<ol className="progress-steps">
|
<ol className="progress-steps">
|
||||||
{getProgressStages(action).map((stage) => (
|
{liveSteps
|
||||||
|
? liveSteps.map((step) => (
|
||||||
|
<li key={step.label}>
|
||||||
|
{formatStepStatus(step.status)} {step.label}
|
||||||
|
{typeof step.duration_ms === "number"
|
||||||
|
? `: ${formatMilliseconds(step.duration_ms)}`
|
||||||
|
: ""}
|
||||||
|
</li>
|
||||||
|
))
|
||||||
|
: getProgressStages(action).map((stage) => (
|
||||||
<li key={stage.label}>{stage.label}</li>
|
<li key={stage.label}>{stage.label}</li>
|
||||||
))}
|
))}
|
||||||
</ol>
|
</ol>
|
||||||
@@ -73,6 +100,12 @@ function getActionLabel(action: ProgressAction) {
|
|||||||
return "正在优化文章";
|
return "正在优化文章";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function formatStepStatus(status: ProgressStep["status"]) {
|
||||||
|
if (status === "running") return "进行中";
|
||||||
|
if (status === "failed") return "失败";
|
||||||
|
return "完成";
|
||||||
|
}
|
||||||
|
|
||||||
function formatMilliseconds(milliseconds: number) {
|
function formatMilliseconds(milliseconds: number) {
|
||||||
const seconds = Math.round(milliseconds / 1000);
|
const seconds = Math.round(milliseconds / 1000);
|
||||||
if (seconds < 60) return `${seconds} 秒`;
|
if (seconds < 60) return `${seconds} 秒`;
|
||||||
|
|||||||
@@ -142,6 +142,31 @@ describe("generateValidatedJson", () => {
|
|||||||
expect(warnSpy).not.toHaveBeenCalled();
|
expect(warnSpy).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("does not duplicate start and response logs when using the default JSON generator", async () => {
|
||||||
|
process.env.LLM_PROVIDER = "deepseek";
|
||||||
|
process.env.DEEPSEEK_API_KEY = "test-key";
|
||||||
|
const infoSpy = vi.spyOn(console, "info").mockImplementation(() => {});
|
||||||
|
client.setChatCompletionForTesting(async () => ({
|
||||||
|
choices: [{ message: { content: '{"value":"from-llm"}' } }],
|
||||||
|
}));
|
||||||
|
|
||||||
|
const result = await client.generateValidatedJson({
|
||||||
|
schema: z.object({ value: z.string() }),
|
||||||
|
prompt: "Return JSON.",
|
||||||
|
task: "article_optimizer",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toEqual({ value: "from-llm" });
|
||||||
|
const startLogs = infoSpy.mock.calls
|
||||||
|
.map((call) => call[0])
|
||||||
|
.filter((line) => line.startsWith("[llm:start]"));
|
||||||
|
const responseLogs = infoSpy.mock.calls
|
||||||
|
.map((call) => call[0])
|
||||||
|
.filter((line) => line.startsWith("[llm:response]"));
|
||||||
|
expect(startLogs).toHaveLength(1);
|
||||||
|
expect(responseLogs).toHaveLength(1);
|
||||||
|
});
|
||||||
|
|
||||||
it("logs validation failure without leaking provider secrets", async () => {
|
it("logs validation failure without leaking provider secrets", async () => {
|
||||||
process.env.LLM_PROVIDER = "deepseek";
|
process.env.LLM_PROVIDER = "deepseek";
|
||||||
process.env.DEEPSEEK_API_KEY = "super-secret-key";
|
process.env.DEEPSEEK_API_KEY = "super-secret-key";
|
||||||
|
|||||||
@@ -224,17 +224,22 @@ export async function generateValidatedJson<T>({
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const usesDefaultGenerator = generateJsonForValidation === generateJson;
|
||||||
const status = getLlmProviderStatus();
|
const status = getLlmProviderStatus();
|
||||||
const startedAt = Date.now();
|
const startedAt = Date.now();
|
||||||
|
if (!usesDefaultGenerator) {
|
||||||
console.info(
|
console.info(
|
||||||
`[llm:start] provider=${status.provider} model=${input.model ?? status.model} task=${task}`,
|
`[llm:start] provider=${status.provider} model=${input.model ?? status.model} task=${task}`,
|
||||||
);
|
);
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const generated = await generateJsonForValidation<unknown>(input);
|
const generated = await generateJsonForValidation<unknown>(input);
|
||||||
|
if (!usesDefaultGenerator) {
|
||||||
console.info(
|
console.info(
|
||||||
`[llm:response] task=${task} duration_ms=${Date.now() - startedAt} raw=${truncateRaw(stringifyForLog(generated))}`,
|
`[llm:response] task=${task} duration_ms=${Date.now() - startedAt} raw=${truncateRaw(stringifyForLog(generated))}`,
|
||||||
);
|
);
|
||||||
|
}
|
||||||
const parsed = schema.safeParse(generated);
|
const parsed = schema.safeParse(generated);
|
||||||
if (parsed.success) {
|
if (parsed.success) {
|
||||||
console.info(`[llm:validated] task=${task} ok=true`);
|
console.info(`[llm:validated] task=${task} ok=true`);
|
||||||
@@ -254,7 +259,9 @@ export async function generateValidatedJson<T>({
|
|||||||
console.info(`[llm:validated] task=${task} ok=false reason=provider_error`);
|
console.info(`[llm:validated] task=${task} ok=false reason=provider_error`);
|
||||||
const message = error instanceof Error ? error.message : String(error);
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
console.warn(
|
console.warn(
|
||||||
`[llm:error] task=${task} duration_ms=${Date.now() - startedAt} message=${quoteLogValue(message)}`,
|
usesDefaultGenerator
|
||||||
|
? `[llm:error] task=${task} message=${quoteLogValue(message)}`
|
||||||
|
: `[llm:error] task=${task} duration_ms=${Date.now() - startedAt} message=${quoteLogValue(message)}`,
|
||||||
);
|
);
|
||||||
throw error instanceof Error ? error : new Error(message);
|
throw error instanceof Error ? error : new Error(message);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ describe("progress helpers", () => {
|
|||||||
expect(getProgressStages("optimize").map((stage) => stage.label)).toEqual([
|
expect(getProgressStages("optimize").map((stage) => stage.label)).toEqual([
|
||||||
"生成优化稿",
|
"生成优化稿",
|
||||||
"质量检查",
|
"质量检查",
|
||||||
"必要时定向修复",
|
"等待后端步骤更新",
|
||||||
"整理结果",
|
"整理结果",
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ const progressStages: Record<ProgressAction, ProgressStage[]> = {
|
|||||||
optimize: [
|
optimize: [
|
||||||
{ label: "生成优化稿" },
|
{ label: "生成优化稿" },
|
||||||
{ label: "质量检查" },
|
{ label: "质量检查" },
|
||||||
{ label: "必要时定向修复" },
|
{ label: "等待后端步骤更新" },
|
||||||
{ label: "整理结果" },
|
{ label: "整理结果" },
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -282,4 +282,41 @@ describe("LLM workflow integration", () => {
|
|||||||
expect(companyCheck?.status).toBe("fail");
|
expect(companyCheck?.status).toBe("fail");
|
||||||
expect(companyCheck?.reason).toContain("公司");
|
expect(companyCheck?.reason).toContain("公司");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("does not let LLM escalate soft quality checks to hard failures", async () => {
|
||||||
|
llmMocks.generateValidatedJson.mockResolvedValueOnce({
|
||||||
|
checks: [
|
||||||
|
{
|
||||||
|
rule_id: "platform_fit",
|
||||||
|
status: "fail",
|
||||||
|
evidence: "LLM thinks the structure is not recommendation-like enough.",
|
||||||
|
reason: "This is a soft platform-fit concern.",
|
||||||
|
suggested_fix: "Adjust structure if needed.",
|
||||||
|
target_agent: "body",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
const report = await inspectQualityWithLlm({
|
||||||
|
article: {
|
||||||
|
title: "Example GEO Optimization Guide",
|
||||||
|
summary:
|
||||||
|
"A official site article for Marketing teams about GEO optimization.",
|
||||||
|
body_markdown:
|
||||||
|
"Example Technology Co., Ltd. has 8 years of GEO optimization experience.",
|
||||||
|
image_suggestions: [],
|
||||||
|
changed_sections: [],
|
||||||
|
requires_user_confirmation: [],
|
||||||
|
},
|
||||||
|
factCard: confirmedFactCard,
|
||||||
|
platform: "official_site",
|
||||||
|
sourceImages: [],
|
||||||
|
});
|
||||||
|
|
||||||
|
const platformCheck = report.checks.find(
|
||||||
|
(check) => check.rule_id === "platform_fit",
|
||||||
|
);
|
||||||
|
expect(platformCheck?.status).toBe("warn");
|
||||||
|
expect(report.overall_status).not.toBe("fail");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -0,0 +1,101 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
|
||||||
|
import type { ConfirmedFactCard } from "../../domain/types";
|
||||||
|
import { runOptimizationWorkflow } from "../orchestrator";
|
||||||
|
|
||||||
|
const workflowMocks = vi.hoisted(() => ({
|
||||||
|
optimizeArticle: vi.fn(),
|
||||||
|
inspectQualityWithLlm: vi.fn(),
|
||||||
|
rewriteFailedSections: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock("../article-optimizer", () => ({
|
||||||
|
optimizeArticle: workflowMocks.optimizeArticle,
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock("../quality-inspector", () => ({
|
||||||
|
inspectQualityWithLlm: workflowMocks.inspectQualityWithLlm,
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock("../targeted-rewriter", () => ({
|
||||||
|
rewriteFailedSections: workflowMocks.rewriteFailedSections,
|
||||||
|
}));
|
||||||
|
|
||||||
|
const factCard: ConfirmedFactCard = {
|
||||||
|
company_full_name: "Example Technology Co., Ltd.",
|
||||||
|
company_short_names: ["Example Tech"],
|
||||||
|
brand_names: ["Example"],
|
||||||
|
product_names: ["Example GEO"],
|
||||||
|
target_industry: "GEO optimization",
|
||||||
|
target_audience: "Marketing teams",
|
||||||
|
experience_years: 8,
|
||||||
|
core_claims: ["Eight years of GEO optimization experience"],
|
||||||
|
forbidden_claims: [],
|
||||||
|
image_topics: [],
|
||||||
|
uncertain_items: [],
|
||||||
|
is_ready_for_optimization: true,
|
||||||
|
confirmed_by_user: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
const article = {
|
||||||
|
title: "Optimized",
|
||||||
|
summary: "Summary",
|
||||||
|
body_markdown: "Body",
|
||||||
|
image_suggestions: [],
|
||||||
|
changed_sections: [],
|
||||||
|
requires_user_confirmation: [],
|
||||||
|
};
|
||||||
|
|
||||||
|
const failCheck = {
|
||||||
|
rule_id: "body_quality" as const,
|
||||||
|
status: "fail" as const,
|
||||||
|
evidence: "Bad body.",
|
||||||
|
reason: "Needs rewrite.",
|
||||||
|
suggested_fix: "Rewrite body.",
|
||||||
|
target_agent: "body",
|
||||||
|
};
|
||||||
|
|
||||||
|
describe("runOptimizationWorkflow", () => {
|
||||||
|
it("reports live progress for initial generation, QA, rewrite rounds, and finalization", async () => {
|
||||||
|
workflowMocks.optimizeArticle.mockResolvedValueOnce(article);
|
||||||
|
workflowMocks.inspectQualityWithLlm
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
overall_status: "fail",
|
||||||
|
checks: [failCheck],
|
||||||
|
})
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
overall_status: "pass",
|
||||||
|
checks: [],
|
||||||
|
});
|
||||||
|
workflowMocks.rewriteFailedSections.mockResolvedValueOnce({
|
||||||
|
...article,
|
||||||
|
body_markdown: "Rewritten body",
|
||||||
|
});
|
||||||
|
const events: Array<{ label: string; status: string }> = [];
|
||||||
|
|
||||||
|
await runOptimizationWorkflow({
|
||||||
|
input: {
|
||||||
|
title: "Original",
|
||||||
|
body: "Original body",
|
||||||
|
images: [],
|
||||||
|
platform: "official_site",
|
||||||
|
user_instructions: "",
|
||||||
|
},
|
||||||
|
factCard,
|
||||||
|
onProgress: (event) => events.push({ label: event.label, status: event.status }),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(events).toEqual([
|
||||||
|
{ label: "生成优化稿", status: "running" },
|
||||||
|
{ label: "生成优化稿", status: "completed" },
|
||||||
|
{ label: "质量检查", status: "running" },
|
||||||
|
{ label: "质量检查", status: "completed" },
|
||||||
|
{ label: "定向修复第 1 轮", status: "running" },
|
||||||
|
{ label: "定向修复第 1 轮", status: "completed" },
|
||||||
|
{ label: "质量复检第 1 轮", status: "running" },
|
||||||
|
{ label: "质量复检第 1 轮", status: "completed" },
|
||||||
|
{ label: "整理结果", status: "running" },
|
||||||
|
{ label: "整理结果", status: "completed" },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -92,7 +92,7 @@ describe("workflow nodes", () => {
|
|||||||
expect(report.checks[0]?.evidence).not.toContain("Article");
|
expect(report.checks[0]?.evidence).not.toContain("Article");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("hard-fails incomplete company names, hallucinated numbers, industry drift, and conflicting years", () => {
|
it("hard-fails incomplete company names, hallucinated numbers, and conflicting years while warning on industry drift", () => {
|
||||||
const report = inspectQuality({
|
const report = inspectQuality({
|
||||||
article: {
|
article: {
|
||||||
title: "Example Wins Finance Automation Market!!!",
|
title: "Example Wins Finance Automation Market!!!",
|
||||||
@@ -113,11 +113,36 @@ describe("workflow nodes", () => {
|
|||||||
expect.arrayContaining([
|
expect.arrayContaining([
|
||||||
"company_name_integrity",
|
"company_name_integrity",
|
||||||
"hallucination_risk",
|
"hallucination_risk",
|
||||||
"industry_alignment",
|
|
||||||
"claim_consistency",
|
"claim_consistency",
|
||||||
]),
|
]),
|
||||||
);
|
);
|
||||||
|
const industryCheck = report.checks.find(
|
||||||
|
(check) => check.rule_id === "industry_alignment",
|
||||||
|
);
|
||||||
|
expect(industryCheck?.status).toBe("warn");
|
||||||
expect(report.overall_status).toBe("fail");
|
expect(report.overall_status).toBe("fail");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("does not flag numbers already present in confirmed fact card claims", () => {
|
||||||
|
const report = inspectQuality({
|
||||||
|
article: {
|
||||||
|
title: "Example GEO Optimization Guide",
|
||||||
|
summary: "Example Technology Co., Ltd. serves teams with 8 years of experience.",
|
||||||
|
body_markdown:
|
||||||
|
"Example Technology Co., Ltd. has 8 years of GEO optimization experience.",
|
||||||
|
image_suggestions: [],
|
||||||
|
changed_sections: [],
|
||||||
|
requires_user_confirmation: [],
|
||||||
|
},
|
||||||
|
factCard: confirmedFactCard,
|
||||||
|
platform: "official_site",
|
||||||
|
sourceImages: [],
|
||||||
|
});
|
||||||
|
|
||||||
|
const hallucinationCheck = report.checks.find(
|
||||||
|
(check) => check.rule_id === "hallucination_risk",
|
||||||
|
);
|
||||||
|
expect(hallucinationCheck?.status).toBe("pass");
|
||||||
|
});
|
||||||
|
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import { rewriteFailedSections } from "./targeted-rewriter";
|
|||||||
export interface RunOptimizationWorkflowInput {
|
export interface RunOptimizationWorkflowInput {
|
||||||
input: ArticleInput;
|
input: ArticleInput;
|
||||||
factCard: ConfirmedFactCard;
|
factCard: ConfirmedFactCard;
|
||||||
|
onProgress?: (event: WorkflowProgressEvent) => void | Promise<void>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface WorkflowTimingStep {
|
export interface WorkflowTimingStep {
|
||||||
@@ -14,6 +15,14 @@ export interface WorkflowTimingStep {
|
|||||||
duration_ms: number;
|
duration_ms: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type WorkflowProgressStatus = "running" | "completed" | "failed";
|
||||||
|
|
||||||
|
export interface WorkflowProgressEvent {
|
||||||
|
label: string;
|
||||||
|
status: WorkflowProgressStatus;
|
||||||
|
duration_ms?: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface WorkflowTimingSummary {
|
export interface WorkflowTimingSummary {
|
||||||
total_ms: number;
|
total_ms: number;
|
||||||
steps: WorkflowTimingStep[];
|
steps: WorkflowTimingStep[];
|
||||||
@@ -23,54 +32,91 @@ async function timedStep<T>(
|
|||||||
label: string,
|
label: string,
|
||||||
steps: WorkflowTimingStep[],
|
steps: WorkflowTimingStep[],
|
||||||
action: () => Promise<T>,
|
action: () => Promise<T>,
|
||||||
|
onProgress?: (event: WorkflowProgressEvent) => void | Promise<void>,
|
||||||
): Promise<T> {
|
): Promise<T> {
|
||||||
const startedAt = Date.now();
|
const startedAt = Date.now();
|
||||||
|
await onProgress?.({ label, status: "running" });
|
||||||
try {
|
try {
|
||||||
return await action();
|
const result = await action();
|
||||||
} finally {
|
const duration = Date.now() - startedAt;
|
||||||
steps.push({
|
steps.push({
|
||||||
label,
|
label,
|
||||||
duration_ms: Date.now() - startedAt,
|
duration_ms: duration,
|
||||||
});
|
});
|
||||||
|
await onProgress?.({ label, status: "completed", duration_ms: duration });
|
||||||
|
return result;
|
||||||
|
} catch (error) {
|
||||||
|
const duration = Date.now() - startedAt;
|
||||||
|
steps.push({
|
||||||
|
label,
|
||||||
|
duration_ms: duration,
|
||||||
|
});
|
||||||
|
await onProgress?.({ label, status: "failed", duration_ms: duration });
|
||||||
|
throw error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function completeStep(
|
||||||
|
label: string,
|
||||||
|
steps: WorkflowTimingStep[],
|
||||||
|
onProgress?: (event: WorkflowProgressEvent) => void | Promise<void>,
|
||||||
|
) {
|
||||||
|
await timedStep(label, steps, async () => undefined, onProgress);
|
||||||
|
}
|
||||||
|
|
||||||
export async function runOptimizationWorkflow({
|
export async function runOptimizationWorkflow({
|
||||||
input,
|
input,
|
||||||
factCard,
|
factCard,
|
||||||
|
onProgress,
|
||||||
}: RunOptimizationWorkflowInput) {
|
}: RunOptimizationWorkflowInput) {
|
||||||
const startedAt = Date.now();
|
const startedAt = Date.now();
|
||||||
const timingSteps: WorkflowTimingStep[] = [];
|
const timingSteps: WorkflowTimingStep[] = [];
|
||||||
let article = await timedStep("生成优化稿", timingSteps, () =>
|
let article = await timedStep(
|
||||||
optimizeArticle({ input, factCard }),
|
"生成优化稿",
|
||||||
|
timingSteps,
|
||||||
|
() => optimizeArticle({ input, factCard }),
|
||||||
|
onProgress,
|
||||||
);
|
);
|
||||||
let qaReport = await timedStep("质量检查", timingSteps, () =>
|
let qaReport = await timedStep(
|
||||||
|
"质量检查",
|
||||||
|
timingSteps,
|
||||||
|
() =>
|
||||||
inspectQualityWithLlm({
|
inspectQualityWithLlm({
|
||||||
article,
|
article,
|
||||||
factCard,
|
factCard,
|
||||||
platform: input.platform,
|
platform: input.platform,
|
||||||
sourceImages: input.images,
|
sourceImages: input.images,
|
||||||
}),
|
}),
|
||||||
|
onProgress,
|
||||||
);
|
);
|
||||||
let rewriteRounds = 0;
|
let rewriteRounds = 0;
|
||||||
|
|
||||||
while (qaReport.overall_status === "fail" && rewriteRounds < 2) {
|
while (qaReport.overall_status === "fail" && rewriteRounds < 2) {
|
||||||
const nextRound = rewriteRounds + 1;
|
const nextRound = rewriteRounds + 1;
|
||||||
const failedChecks = qaReport.checks.filter((check) => check.status === "fail");
|
const failedChecks = qaReport.checks.filter((check) => check.status === "fail");
|
||||||
article = await timedStep(`定向修复第 ${nextRound} 轮`, timingSteps, () =>
|
article = await timedStep(
|
||||||
rewriteFailedSections({ article, factCard, failedChecks }),
|
`定向修复第 ${nextRound} 轮`,
|
||||||
|
timingSteps,
|
||||||
|
() => rewriteFailedSections({ article, factCard, failedChecks }),
|
||||||
|
onProgress,
|
||||||
);
|
);
|
||||||
rewriteRounds = nextRound;
|
rewriteRounds = nextRound;
|
||||||
qaReport = await timedStep(`质量复检第 ${nextRound} 轮`, timingSteps, () =>
|
qaReport = await timedStep(
|
||||||
|
`质量复检第 ${nextRound} 轮`,
|
||||||
|
timingSteps,
|
||||||
|
() =>
|
||||||
inspectQualityWithLlm({
|
inspectQualityWithLlm({
|
||||||
article,
|
article,
|
||||||
factCard,
|
factCard,
|
||||||
platform: input.platform,
|
platform: input.platform,
|
||||||
sourceImages: input.images,
|
sourceImages: input.images,
|
||||||
}),
|
}),
|
||||||
|
onProgress,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
await completeStep("整理结果", timingSteps, onProgress);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
article,
|
article,
|
||||||
qaReport,
|
qaReport,
|
||||||
|
|||||||
@@ -0,0 +1,88 @@
|
|||||||
|
import type {
|
||||||
|
WorkflowProgressEvent,
|
||||||
|
WorkflowProgressStatus,
|
||||||
|
} from "./orchestrator";
|
||||||
|
|
||||||
|
export interface WorkflowProgressStep {
|
||||||
|
label: string;
|
||||||
|
status: WorkflowProgressStatus;
|
||||||
|
started_at: string;
|
||||||
|
completed_at?: string;
|
||||||
|
duration_ms?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface WorkflowProgressSnapshot {
|
||||||
|
job_id: string;
|
||||||
|
current_step: string | null;
|
||||||
|
status: "idle" | "running" | "completed" | "failed";
|
||||||
|
started_at: string | null;
|
||||||
|
updated_at: string | null;
|
||||||
|
steps: WorkflowProgressStep[];
|
||||||
|
}
|
||||||
|
|
||||||
|
const progressByJob = new Map<string, WorkflowProgressSnapshot>();
|
||||||
|
|
||||||
|
export function startWorkflowProgress(jobId: string) {
|
||||||
|
const now = new Date().toISOString();
|
||||||
|
const snapshot: WorkflowProgressSnapshot = {
|
||||||
|
job_id: jobId,
|
||||||
|
current_step: null,
|
||||||
|
status: "running",
|
||||||
|
started_at: now,
|
||||||
|
updated_at: now,
|
||||||
|
steps: [],
|
||||||
|
};
|
||||||
|
progressByJob.set(jobId, snapshot);
|
||||||
|
return snapshot;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function recordWorkflowProgress(
|
||||||
|
jobId: string,
|
||||||
|
event: WorkflowProgressEvent,
|
||||||
|
) {
|
||||||
|
const now = new Date().toISOString();
|
||||||
|
const snapshot = progressByJob.get(jobId) ?? startWorkflowProgress(jobId);
|
||||||
|
const existing = snapshot.steps.find((step) => step.label === event.label);
|
||||||
|
|
||||||
|
if (existing) {
|
||||||
|
existing.status = event.status;
|
||||||
|
existing.duration_ms = event.duration_ms ?? existing.duration_ms;
|
||||||
|
if (event.status !== "running") {
|
||||||
|
existing.completed_at = now;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
snapshot.steps.push({
|
||||||
|
label: event.label,
|
||||||
|
status: event.status,
|
||||||
|
started_at: now,
|
||||||
|
completed_at: event.status === "running" ? undefined : now,
|
||||||
|
duration_ms: event.duration_ms,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshot.current_step =
|
||||||
|
event.status === "running" ? event.label : snapshot.current_step;
|
||||||
|
if (event.status === "failed") {
|
||||||
|
snapshot.status = "failed";
|
||||||
|
} else if (event.label === "整理结果" && event.status === "completed") {
|
||||||
|
snapshot.status = "completed";
|
||||||
|
snapshot.current_step = null;
|
||||||
|
} else {
|
||||||
|
snapshot.status = "running";
|
||||||
|
}
|
||||||
|
snapshot.updated_at = now;
|
||||||
|
return snapshot;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getWorkflowProgress(jobId: string) {
|
||||||
|
return (
|
||||||
|
progressByJob.get(jobId) ?? {
|
||||||
|
job_id: jobId,
|
||||||
|
current_step: null,
|
||||||
|
status: "idle",
|
||||||
|
started_at: null,
|
||||||
|
updated_at: null,
|
||||||
|
steps: [],
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -29,6 +29,12 @@ const REQUIRED_RULES: QualityRuleId[] = [
|
|||||||
"context_sensitive_terms",
|
"context_sensitive_terms",
|
||||||
];
|
];
|
||||||
|
|
||||||
|
const HARD_FAILURE_RULES = new Set<QualityRuleId>([
|
||||||
|
"company_name_integrity",
|
||||||
|
"hallucination_risk",
|
||||||
|
"claim_consistency",
|
||||||
|
]);
|
||||||
|
|
||||||
const llmQaPatchSchema = z.object({
|
const llmQaPatchSchema = z.object({
|
||||||
checks: z.array(qaCheckSchema).default([]),
|
checks: z.array(qaCheckSchema).default([]),
|
||||||
});
|
});
|
||||||
@@ -78,9 +84,13 @@ export async function inspectQualityWithLlm(
|
|||||||
if (deterministicCheck.status === "fail") {
|
if (deterministicCheck.status === "fail") {
|
||||||
return deterministicCheck;
|
return deterministicCheck;
|
||||||
}
|
}
|
||||||
|
const status =
|
||||||
|
llmCheck.status === "fail" && !HARD_FAILURE_RULES.has(deterministicCheck.rule_id)
|
||||||
|
? "warn"
|
||||||
|
: llmCheck.status;
|
||||||
return {
|
return {
|
||||||
...deterministicCheck,
|
...deterministicCheck,
|
||||||
status: llmCheck.status,
|
status,
|
||||||
evidence: llmCheck.evidence,
|
evidence: llmCheck.evidence,
|
||||||
reason: llmCheck.reason,
|
reason: llmCheck.reason,
|
||||||
suggested_fix: llmCheck.suggested_fix,
|
suggested_fix: llmCheck.suggested_fix,
|
||||||
@@ -107,14 +117,15 @@ function inspectRule(
|
|||||||
const lower = combined.toLowerCase();
|
const lower = combined.toLowerCase();
|
||||||
|
|
||||||
if (ruleId === "industry_alignment") {
|
if (ruleId === "industry_alignment") {
|
||||||
const aligned = lower.includes(factCard.target_industry.toLowerCase());
|
const targetIndustry = factCard.target_industry.trim().toLowerCase();
|
||||||
|
const aligned = targetIndustry.length === 0 || lower.includes(targetIndustry);
|
||||||
return check(
|
return check(
|
||||||
ruleId,
|
ruleId,
|
||||||
aligned ? "pass" : "fail",
|
aligned ? "pass" : "warn",
|
||||||
aligned ? factCard.target_industry : article.summary,
|
aligned ? factCard.target_industry : article.summary,
|
||||||
aligned
|
aligned
|
||||||
? "文章内容与事实卡确认的目标行业一致。"
|
? "文章内容与事实卡确认的目标行业一致。"
|
||||||
: "文章内容偏离事实卡确认的目标行业。",
|
: "文章可能没有充分体现事实卡确认的目标行业。",
|
||||||
"围绕事实卡确认的目标行业重写相关段落。",
|
"围绕事实卡确认的目标行业重写相关段落。",
|
||||||
aligned ? null : "body",
|
aligned ? null : "body",
|
||||||
);
|
);
|
||||||
@@ -254,11 +265,30 @@ function check(
|
|||||||
|
|
||||||
function findUnsupportedNumbers(text: string, factCard: ConfirmedFactCard) {
|
function findUnsupportedNumbers(text: string, factCard: ConfirmedFactCard) {
|
||||||
const allowed = new Set(
|
const allowed = new Set(
|
||||||
[factCard.experience_years]
|
[
|
||||||
.filter((value): value is number => typeof value === "number")
|
factCard.experience_years,
|
||||||
|
...extractNumbersFromFactCard(factCard),
|
||||||
|
]
|
||||||
|
.filter((value): value is number | string =>
|
||||||
|
typeof value === "number" || typeof value === "string",
|
||||||
|
)
|
||||||
.map(String),
|
.map(String),
|
||||||
);
|
);
|
||||||
return [...text.matchAll(/\b\d{1,4}\b/g)]
|
return [...text.matchAll(/\b\d{1,4}\b/g)]
|
||||||
.map((match) => match[0])
|
.map((match) => match[0])
|
||||||
.filter((number) => !allowed.has(number));
|
.filter((number) => !allowed.has(number));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function extractNumbersFromFactCard(factCard: ConfirmedFactCard) {
|
||||||
|
return [
|
||||||
|
factCard.company_full_name,
|
||||||
|
...factCard.company_short_names,
|
||||||
|
...factCard.brand_names,
|
||||||
|
...factCard.product_names,
|
||||||
|
factCard.target_industry,
|
||||||
|
factCard.target_audience,
|
||||||
|
...factCard.core_claims,
|
||||||
|
...factCard.forbidden_claims,
|
||||||
|
...factCard.image_topics,
|
||||||
|
].flatMap((value) => [...value.matchAll(/\b\d{1,4}\b/g)].map((match) => match[0]));
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user