fix: allow exports with QA warnings
This commit is contained in:
@@ -73,7 +73,8 @@ interface CreateJobResponse {
|
|||||||
|
|
||||||
interface OptimizeJobResponse {
|
interface OptimizeJobResponse {
|
||||||
optimizedArticle: { title: string };
|
optimizedArticle: { title: string };
|
||||||
qaReport: { checks: unknown[] };
|
qaReport: { checks: unknown[]; overall_status?: string };
|
||||||
|
exportPaths?: Record<string, string>;
|
||||||
timing: TimingResponse;
|
timing: TimingResponse;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -314,6 +315,65 @@ describe("job API routes", () => {
|
|||||||
expect(llmMocks.generateValidatedJson).toHaveBeenCalled();
|
expect(llmMocks.generateValidatedJson).toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("still writes export files when the QA report fails", async () => {
|
||||||
|
const { job } = await createJobFixture();
|
||||||
|
await confirmFactCard(
|
||||||
|
request(validFactCard),
|
||||||
|
params<{ jobId: string }>({ jobId: job.id }),
|
||||||
|
);
|
||||||
|
|
||||||
|
llmMocks.generateValidatedJson
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
title: "API LLM Optimized GEO Article!!!",
|
||||||
|
summary:
|
||||||
|
"A official site article for Marketing teams about GEO optimization.",
|
||||||
|
body_markdown:
|
||||||
|
"Example has 12 years of GEO optimization experience and 99 patents.",
|
||||||
|
image_suggestions: [],
|
||||||
|
changed_sections: ["title", "body"],
|
||||||
|
requires_user_confirmation: [],
|
||||||
|
})
|
||||||
|
.mockResolvedValueOnce({ checks: [] })
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
title: "API LLM Optimized GEO Article",
|
||||||
|
summary:
|
||||||
|
"A official site article for Marketing teams about GEO optimization.",
|
||||||
|
body_markdown:
|
||||||
|
"Example has 12 years of GEO optimization experience and 99 patents.",
|
||||||
|
image_suggestions: [],
|
||||||
|
changed_sections: ["body"],
|
||||||
|
requires_user_confirmation: [],
|
||||||
|
})
|
||||||
|
.mockResolvedValueOnce({ checks: [] })
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
title: "API LLM Optimized GEO Article",
|
||||||
|
summary:
|
||||||
|
"A official site article for Marketing teams about GEO optimization.",
|
||||||
|
body_markdown:
|
||||||
|
"Example has 12 years of GEO optimization experience and 99 patents.",
|
||||||
|
image_suggestions: [],
|
||||||
|
changed_sections: ["body"],
|
||||||
|
requires_user_confirmation: [],
|
||||||
|
})
|
||||||
|
.mockResolvedValueOnce({ checks: [] });
|
||||||
|
|
||||||
|
const response = await optimizeJob(
|
||||||
|
request({}),
|
||||||
|
params<{ jobId: string }>({ jobId: job.id }),
|
||||||
|
);
|
||||||
|
const body = (await response.json()) as OptimizeJobResponse;
|
||||||
|
|
||||||
|
expect(response.status).toBe(200);
|
||||||
|
expect(body.qaReport.overall_status).toBe("fail");
|
||||||
|
expect(body.exportPaths).toEqual(
|
||||||
|
expect.objectContaining({
|
||||||
|
markdown: expect.stringContaining("optimized.md"),
|
||||||
|
docx: expect.stringContaining("optimized.docx"),
|
||||||
|
qaJson: expect.stringContaining("qa_report.json"),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it("returns a clear error when LLM optimization fails", async () => {
|
it("returns a clear error when LLM optimization fails", async () => {
|
||||||
const { job } = await createJobFixture();
|
const { job } = await createJobFixture();
|
||||||
await confirmFactCard(
|
await confirmFactCard(
|
||||||
|
|||||||
@@ -61,16 +61,13 @@ export async function POST(request: Request, context: RouteContext) {
|
|||||||
result.qaReport,
|
result.qaReport,
|
||||||
);
|
);
|
||||||
const exportStore = getExportStoreFromRuntime();
|
const exportStore = getExportStoreFromRuntime();
|
||||||
const exportPaths =
|
const exportPaths = await exportStore.writeJobExports({
|
||||||
qaReport.overall_status === "fail"
|
jobId,
|
||||||
? {}
|
article: optimizedArticle,
|
||||||
: await exportStore.writeJobExports({
|
qaReport,
|
||||||
jobId,
|
});
|
||||||
article: optimizedArticle,
|
|
||||||
qaReport,
|
|
||||||
});
|
|
||||||
await repository.updateArticleJob(jobId, {
|
await repository.updateArticleJob(jobId, {
|
||||||
status: qaReport.overall_status === "fail" ? "qa_failed" : "optimized",
|
status: "optimized",
|
||||||
export_paths: exportPaths,
|
export_paths: exportPaths,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
+2
-7
@@ -1,6 +1,6 @@
|
|||||||
"use client";
|
"use client";
|
||||||
|
|
||||||
import { useEffect, useMemo, useState } from "react";
|
import { useEffect, useState } from "react";
|
||||||
|
|
||||||
import {
|
import {
|
||||||
ArticleInputForm,
|
ArticleInputForm,
|
||||||
@@ -89,10 +89,6 @@ export default function Home() {
|
|||||||
const [lastTiming, setLastTiming] = useState<TimingSummary | null>(null);
|
const [lastTiming, setLastTiming] = useState<TimingSummary | null>(null);
|
||||||
const [liveProgress, setLiveProgress] = useState<WorkflowProgress | null>(null);
|
const [liveProgress, setLiveProgress] = useState<WorkflowProgress | null>(null);
|
||||||
|
|
||||||
const exportBlocked = useMemo(
|
|
||||||
() => qaReport?.overall_status === "fail",
|
|
||||||
[qaReport],
|
|
||||||
);
|
|
||||||
const canOptimize =
|
const canOptimize =
|
||||||
Boolean(jobId) && Boolean(factCard) && factCard?.uncertain_items.length === 0;
|
Boolean(jobId) && Boolean(factCard) && factCard?.uncertain_items.length === 0;
|
||||||
|
|
||||||
@@ -209,7 +205,7 @@ export default function Home() {
|
|||||||
: "";
|
: "";
|
||||||
setMessage(
|
setMessage(
|
||||||
body.qaReport.overall_status === "fail"
|
body.qaReport.overall_status === "fail"
|
||||||
? `质检发现硬性失败,已阻止导出。${timingText}`
|
? `质检发现需要复核的问题,已保留导出文件。${timingText}`
|
||||||
: `优化完成。${timingText}`,
|
: `优化完成。${timingText}`,
|
||||||
);
|
);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -267,7 +263,6 @@ export default function Home() {
|
|||||||
/>
|
/>
|
||||||
<OptimizedPreview
|
<OptimizedPreview
|
||||||
article={optimizedArticle}
|
article={optimizedArticle}
|
||||||
exportBlocked={exportBlocked}
|
|
||||||
jobId={jobId}
|
jobId={jobId}
|
||||||
/>
|
/>
|
||||||
<QaReportPanel report={qaReport} />
|
<QaReportPanel report={qaReport} />
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import type { OptimizedArticle } from "../lib/domain/types";
|
|||||||
interface OptimizedPreviewProps {
|
interface OptimizedPreviewProps {
|
||||||
article: OptimizedArticle | null;
|
article: OptimizedArticle | null;
|
||||||
jobId: string | null;
|
jobId: string | null;
|
||||||
exportBlocked: boolean;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const exportFiles = ["optimized.md", "optimized.docx", "qa_report.json"];
|
const exportFiles = ["optimized.md", "optimized.docx", "qa_report.json"];
|
||||||
@@ -13,7 +12,6 @@ const exportFiles = ["optimized.md", "optimized.docx", "qa_report.json"];
|
|||||||
export function OptimizedPreview({
|
export function OptimizedPreview({
|
||||||
article,
|
article,
|
||||||
jobId,
|
jobId,
|
||||||
exportBlocked,
|
|
||||||
}: OptimizedPreviewProps) {
|
}: OptimizedPreviewProps) {
|
||||||
if (!article) {
|
if (!article) {
|
||||||
return (
|
return (
|
||||||
@@ -63,7 +61,7 @@ export function OptimizedPreview({
|
|||||||
)}
|
)}
|
||||||
<div className="export-row">
|
<div className="export-row">
|
||||||
{exportFiles.map((fileName) =>
|
{exportFiles.map((fileName) =>
|
||||||
jobId && !exportBlocked ? (
|
jobId ? (
|
||||||
<a
|
<a
|
||||||
className="download-link"
|
className="download-link"
|
||||||
href={`/api/jobs/${jobId}/exports/${fileName}`}
|
href={`/api/jobs/${jobId}/exports/${fileName}`}
|
||||||
|
|||||||
@@ -319,4 +319,41 @@ describe("LLM workflow integration", () => {
|
|||||||
expect(platformCheck?.status).toBe("warn");
|
expect(platformCheck?.status).toBe("warn");
|
||||||
expect(report.overall_status).not.toBe("fail");
|
expect(report.overall_status).not.toBe("fail");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("does not let LLM escalate hallucination risk to a hard failure", async () => {
|
||||||
|
llmMocks.generateValidatedJson.mockResolvedValueOnce({
|
||||||
|
checks: [
|
||||||
|
{
|
||||||
|
rule_id: "hallucination_risk",
|
||||||
|
status: "fail",
|
||||||
|
evidence: "LLM wants strict fact confirmation.",
|
||||||
|
reason: "This should remain a warning for user review.",
|
||||||
|
suggested_fix: "Review manually.",
|
||||||
|
target_agent: "body",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
const report = await inspectQualityWithLlm({
|
||||||
|
article: {
|
||||||
|
title: "Example GEO Optimization Guide",
|
||||||
|
summary:
|
||||||
|
"Example Technology Co., Ltd. mentions a 99-day rollout plan for Marketing teams.",
|
||||||
|
body_markdown:
|
||||||
|
"Example Technology Co., Ltd. has 8 years of GEO optimization experience.",
|
||||||
|
image_suggestions: [],
|
||||||
|
changed_sections: [],
|
||||||
|
requires_user_confirmation: [],
|
||||||
|
},
|
||||||
|
factCard: confirmedFactCard,
|
||||||
|
platform: "official_site",
|
||||||
|
sourceImages: [],
|
||||||
|
});
|
||||||
|
|
||||||
|
const hallucinationCheck = report.checks.find(
|
||||||
|
(check) => check.rule_id === "hallucination_risk",
|
||||||
|
);
|
||||||
|
expect(hallucinationCheck?.status).toBe("warn");
|
||||||
|
expect(report.overall_status).toBe("warn");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -92,7 +92,7 @@ describe("workflow nodes", () => {
|
|||||||
expect(report.checks[0]?.evidence).not.toContain("Article");
|
expect(report.checks[0]?.evidence).not.toContain("Article");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("hard-fails incomplete company names, hallucinated numbers, and conflicting years while warning on industry drift", () => {
|
it("hard-fails incomplete company names and conflicting years while warning on industry drift and hallucination risk", () => {
|
||||||
const report = inspectQuality({
|
const report = inspectQuality({
|
||||||
article: {
|
article: {
|
||||||
title: "Example Wins Finance Automation Market!!!",
|
title: "Example Wins Finance Automation Market!!!",
|
||||||
@@ -112,7 +112,6 @@ describe("workflow nodes", () => {
|
|||||||
expect(failures.map((check) => check.rule_id)).toEqual(
|
expect(failures.map((check) => check.rule_id)).toEqual(
|
||||||
expect.arrayContaining([
|
expect.arrayContaining([
|
||||||
"company_name_integrity",
|
"company_name_integrity",
|
||||||
"hallucination_risk",
|
|
||||||
"claim_consistency",
|
"claim_consistency",
|
||||||
]),
|
]),
|
||||||
);
|
);
|
||||||
@@ -120,6 +119,10 @@ describe("workflow nodes", () => {
|
|||||||
(check) => check.rule_id === "industry_alignment",
|
(check) => check.rule_id === "industry_alignment",
|
||||||
);
|
);
|
||||||
expect(industryCheck?.status).toBe("warn");
|
expect(industryCheck?.status).toBe("warn");
|
||||||
|
const hallucinationCheck = report.checks.find(
|
||||||
|
(check) => check.rule_id === "hallucination_risk",
|
||||||
|
);
|
||||||
|
expect(hallucinationCheck?.status).toBe("warn");
|
||||||
expect(report.overall_status).toBe("fail");
|
expect(report.overall_status).toBe("fail");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -31,7 +31,6 @@ const REQUIRED_RULES: QualityRuleId[] = [
|
|||||||
|
|
||||||
const HARD_FAILURE_RULES = new Set<QualityRuleId>([
|
const HARD_FAILURE_RULES = new Set<QualityRuleId>([
|
||||||
"company_name_integrity",
|
"company_name_integrity",
|
||||||
"hallucination_risk",
|
|
||||||
"claim_consistency",
|
"claim_consistency",
|
||||||
]);
|
]);
|
||||||
|
|
||||||
@@ -213,14 +212,14 @@ function inspectRule(
|
|||||||
const unsupportedNumber = findUnsupportedNumbers(combined, factCard).length > 0;
|
const unsupportedNumber = findUnsupportedNumbers(combined, factCard).length > 0;
|
||||||
return check(
|
return check(
|
||||||
ruleId,
|
ruleId,
|
||||||
unsupportedNumber || article.requires_user_confirmation.length > 0 ? "fail" : "pass",
|
unsupportedNumber || article.requires_user_confirmation.length > 0 ? "warn" : "pass",
|
||||||
unsupportedNumber
|
unsupportedNumber
|
||||||
? findUnsupportedNumbers(combined, factCard).join(", ")
|
? findUnsupportedNumbers(combined, factCard).join(", ")
|
||||||
: "未发现未确认的数字类事实主张。",
|
: "未发现未确认的数字类事实主张。",
|
||||||
unsupportedNumber
|
unsupportedNumber
|
||||||
? "数字类主张无法追溯到已确认事实卡。"
|
? "发现需要人工复核的数字类事实主张。"
|
||||||
: "事实主张可以追溯到已确认事实卡。",
|
: "事实主张可以追溯到已确认事实卡。",
|
||||||
"删除未确认主张,或先补充到事实卡并确认。",
|
"建议人工复核,必要时删除未确认主张,或补充到事实卡后再确认。",
|
||||||
unsupportedNumber ? "body" : null,
|
unsupportedNumber ? "body" : null,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user