新增发布表现评分服务
This commit is contained in:
Codex
@@ -0,0 +1,106 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { OptimizedArticle, QaReport } from "../../domain/types";
|
||||
import { createManualPerformanceAdapter } from "../manual-adapter";
|
||||
import {
|
||||
createCalibrationEvent,
|
||||
GEO_RUBRIC_V1,
|
||||
scoreOptimizedArticle,
|
||||
} from "../scoring";
|
||||
|
||||
const article: OptimizedArticle = {
|
||||
job_id: "job_123",
|
||||
revision: 2,
|
||||
title: "示例科技 GEO 内容优化方案",
|
||||
summary: "示例科技有限公司面向市场团队提供GEO内容优化服务。",
|
||||
body_markdown:
|
||||
"## 服务能力\n示例科技有限公司提供GEO内容优化服务，帮助市场团队提升AI搜索可见性。\n## 可信依据\n文章保留事实卡中的8年经验描述。",
|
||||
image_suggestions: [],
|
||||
changed_sections: ["title", "body"],
|
||||
requires_user_confirmation: [],
|
||||
};
|
||||
|
||||
const qaReport: QaReport = {
|
||||
job_id: "job_123",
|
||||
revision: 2,
|
||||
overall_status: "warn",
|
||||
checks: [
|
||||
{
|
||||
rule_id: "hallucination_risk",
|
||||
status: "warn",
|
||||
evidence: "8年经验",
|
||||
reason: "需要人工复核经验年限依据。",
|
||||
suggested_fix: "确认事实卡。",
|
||||
target_agent: "body",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
describe("calibration scoring", () => {
|
||||
it("scores an optimized article with the active GEO rubric", () => {
|
||||
const run = scoreOptimizedArticle({
|
||||
jobId: "job_123",
|
||||
article,
|
||||
qaReport,
|
||||
});
|
||||
|
||||
expect(run.rubric_version_id).toBe(GEO_RUBRIC_V1.id);
|
||||
expect(run.dimension_scores.fact_integrity).toBe(4);
|
||||
expect(run.dimension_scores.readability).toBeGreaterThanOrEqual(4);
|
||||
expect(run.composite_score).toBeGreaterThan(6);
|
||||
expect(run.composite_score).toBeLessThanOrEqual(10);
|
||||
});
|
||||
|
||||
it("normalizes manual performance input through the adapter boundary", async () => {
|
||||
const adapter = createManualPerformanceAdapter();
|
||||
const snapshot = await adapter.fetch({
|
||||
publication: {
|
||||
id: "pub_123",
|
||||
job_id: "job_123",
|
||||
revision: 2,
|
||||
platform: "official_site",
|
||||
url: "https://example.com/article",
|
||||
published_at: "2026-06-24T12:00:00.000Z",
|
||||
status: "published",
|
||||
notes: "",
|
||||
created_at: "2026-06-24T12:00:00.000Z",
|
||||
updated_at: "2026-06-24T12:00:00.000Z",
|
||||
},
|
||||
window_label: "T+7d",
|
||||
manualInput: {
|
||||
window_label: "T+7d",
|
||||
views: "1200",
|
||||
inquiries: "7",
|
||||
feedback_summary: "用户追问案例依据",
|
||||
},
|
||||
});
|
||||
|
||||
expect(snapshot.source).toBe("manual");
|
||||
expect(snapshot.metrics).toEqual({ views: 1200, inquiries: 7 });
|
||||
});
|
||||
|
||||
it("creates a reviewable calibration event without changing the article", () => {
|
||||
const scoringRun = scoreOptimizedArticle({
|
||||
jobId: "job_123",
|
||||
article,
|
||||
qaReport,
|
||||
});
|
||||
const event = createCalibrationEvent({
|
||||
scoringRun,
|
||||
qaReport,
|
||||
snapshot: {
|
||||
id: "perf_123",
|
||||
publication_id: "pub_123",
|
||||
source: "manual",
|
||||
window_label: "T+7d",
|
||||
metrics: { views: 1200, inquiries: 7 },
|
||||
feedback_summary: "用户追问案例依据",
|
||||
snapshot_at: "2026-07-01T12:00:00.000Z",
|
||||
},
|
||||
});
|
||||
|
||||
expect(event.direction).toBe("better_than_expected");
|
||||
expect(event.observations.join(" ")).toContain("询盘");
|
||||
expect(event.recommended_action).toContain("积累");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,33 @@
|
||||
import { nanoid } from "nanoid";
|
||||
|
||||
import type {
|
||||
AdapterFetchInput,
|
||||
PerformanceAdapter,
|
||||
PerformanceSnapshot,
|
||||
} from "./types";
|
||||
import { manualPerformanceInputSchema, performanceSnapshotSchema } from "./validation";
|
||||
|
||||
interface ManualAdapterFetchInput extends AdapterFetchInput {
|
||||
manualInput: unknown;
|
||||
}
|
||||
|
||||
export function createManualPerformanceAdapter(): PerformanceAdapter & {
|
||||
fetch(input: ManualAdapterFetchInput): Promise<PerformanceSnapshot>;
|
||||
} {
|
||||
return {
|
||||
source: "manual",
|
||||
async fetch(input) {
|
||||
const parsed = manualPerformanceInputSchema.parse(input.manualInput);
|
||||
return performanceSnapshotSchema.parse({
|
||||
id: `perf_${nanoid(10)}`,
|
||||
publication_id: input.publication.id,
|
||||
source: "manual",
|
||||
window_label: parsed.window_label,
|
||||
metrics: parsed.metrics,
|
||||
feedback_summary: parsed.feedback_summary,
|
||||
raw_reference: parsed.raw_reference,
|
||||
snapshot_at: new Date().toISOString(),
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
import { nanoid } from "nanoid";
|
||||
|
||||
import type { OptimizedArticle, QaReport } from "../domain/types";
|
||||
import type {
|
||||
CalibrationContext,
|
||||
CalibrationDirection,
|
||||
CalibrationEvent,
|
||||
RubricVersion,
|
||||
ScoringRun,
|
||||
} from "./types";
|
||||
|
||||
export const GEO_RUBRIC_V1: RubricVersion = {
|
||||
id: "rubric_geo_v1",
|
||||
version: "v1",
|
||||
name: "GEO article performance rubric",
|
||||
formula: "weighted_average_0_to_10",
|
||||
is_active: true,
|
||||
created_at: "2026-06-24T00:00:00.000Z",
|
||||
dimensions: [
|
||||
{
|
||||
id: "fact_integrity",
|
||||
label: "事实一致性",
|
||||
weight: 2,
|
||||
description: "事实、公司名、产品名和经验年限是否遵守事实卡。",
|
||||
},
|
||||
{
|
||||
id: "platform_fit",
|
||||
label: "平台适配",
|
||||
weight: 1.5,
|
||||
description: "表达是否匹配目标发布平台。",
|
||||
},
|
||||
{
|
||||
id: "search_intent_fit",
|
||||
label: "搜索意图匹配",
|
||||
weight: 1.5,
|
||||
description: "是否回答GEO/Search背后的用户问题。",
|
||||
},
|
||||
{
|
||||
id: "answer_density",
|
||||
label: "答案密度",
|
||||
weight: 1.5,
|
||||
description: "是否提供具体信息而不是泛泛宣传。",
|
||||
},
|
||||
{
|
||||
id: "trust_signal_quality",
|
||||
label: "信任信号质量",
|
||||
weight: 1.5,
|
||||
description: "可信依据是否具体、克制且可复核。",
|
||||
},
|
||||
{
|
||||
id: "readability",
|
||||
label: "可读性",
|
||||
weight: 1,
|
||||
description: "标题、摘要、正文是否清晰易读。",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
interface ScoreOptimizedArticleInput {
|
||||
jobId: string;
|
||||
article: OptimizedArticle;
|
||||
qaReport: QaReport;
|
||||
}
|
||||
|
||||
export function scoreOptimizedArticle({
|
||||
jobId,
|
||||
article,
|
||||
qaReport,
|
||||
}: ScoreOptimizedArticleInput): ScoringRun {
|
||||
const combined = `${article.title}\n${article.summary}\n${article.body_markdown}`;
|
||||
const dimensionScores = {
|
||||
fact_integrity: scoreFactIntegrity(qaReport),
|
||||
platform_fit: scoreRuleGroup(qaReport, ["platform_fit", "voice_consistency"]),
|
||||
search_intent_fit: hasGeoIntent(combined) ? 4 : 2,
|
||||
answer_density: scoreAnswerDensity(combined),
|
||||
trust_signal_quality: scoreTrustSignals(combined, qaReport),
|
||||
readability: scoreReadability(article),
|
||||
};
|
||||
|
||||
return {
|
||||
id: `score_${nanoid(10)}`,
|
||||
job_id: jobId,
|
||||
revision: article.revision ?? 1,
|
||||
rubric_version_id: GEO_RUBRIC_V1.id,
|
||||
dimension_scores: dimensionScores,
|
||||
composite_score: weightedComposite(dimensionScores),
|
||||
rationale: buildRationale(dimensionScores),
|
||||
created_at: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
function scoreFactIntegrity(report: QaReport) {
|
||||
const hardRules = ["company_name_integrity", "claim_consistency", "hallucination_risk"];
|
||||
const statuses = report.checks
|
||||
.filter((check) => hardRules.includes(check.rule_id))
|
||||
.map((check) => check.status);
|
||||
if (statuses.includes("fail")) return 1;
|
||||
if (statuses.includes("warn")) return 4;
|
||||
return 5;
|
||||
}
|
||||
|
||||
function scoreRuleGroup(report: QaReport, ruleIds: string[]) {
|
||||
const statuses = report.checks
|
||||
.filter((check) => ruleIds.includes(check.rule_id))
|
||||
.map((check) => check.status);
|
||||
if (statuses.includes("fail")) return 2;
|
||||
if (statuses.includes("warn")) return 3;
|
||||
return 4;
|
||||
}
|
||||
|
||||
function hasGeoIntent(text: string) {
|
||||
return /GEO|AI搜索|生成式引擎|搜索|可见性|问答|推荐/i.test(text);
|
||||
}
|
||||
|
||||
function scoreAnswerDensity(text: string) {
|
||||
const headings = (text.match(/^##\s+/gm) ?? []).length;
|
||||
const concreteSignals = (
|
||||
text.match(/服务|流程|方案|能力|团队|行业|客户|案例/g) ?? []
|
||||
).length;
|
||||
if (headings >= 2 && concreteSignals >= 8) return 5;
|
||||
if (headings >= 1 && concreteSignals >= 4) return 4;
|
||||
if (concreteSignals >= 2) return 3;
|
||||
return 2;
|
||||
}
|
||||
|
||||
function scoreTrustSignals(text: string, report: QaReport) {
|
||||
const hallucination = report.checks.find(
|
||||
(check) => check.rule_id === "hallucination_risk",
|
||||
);
|
||||
if (hallucination?.status === "fail") return 1;
|
||||
const trustSignals = (text.match(/依据|经验|资质|案例|事实卡|复核|客户/g) ?? [])
|
||||
.length;
|
||||
if (hallucination?.status === "warn") return trustSignals >= 2 ? 3 : 2;
|
||||
return trustSignals >= 2 ? 4 : 3;
|
||||
}
|
||||
|
||||
function scoreReadability(article: OptimizedArticle) {
|
||||
const hasLongSentence = `${article.summary}\n${article.body_markdown}`
|
||||
.split(/[。！？.!?]/)
|
||||
.some((sentence) => sentence.length > 180);
|
||||
if (article.title.length > 42 || hasLongSentence) return 3;
|
||||
return 4;
|
||||
}
|
||||
|
||||
function weightedComposite(scores: Record<string, number>) {
|
||||
const totalWeight = GEO_RUBRIC_V1.dimensions.reduce(
|
||||
(sum, dimension) => sum + dimension.weight,
|
||||
0,
|
||||
);
|
||||
const weighted = GEO_RUBRIC_V1.dimensions.reduce(
|
||||
(sum, dimension) => sum + (scores[dimension.id] ?? 0) * dimension.weight,
|
||||
0,
|
||||
);
|
||||
return Math.round((weighted / totalWeight) * 2 * 10) / 10;
|
||||
}
|
||||
|
||||
function buildRationale(scores: Record<string, number>) {
|
||||
return `事实一致性 ${scores.fact_integrity}/5，平台适配 ${scores.platform_fit}/5，答案密度 ${scores.answer_density}/5，信任信号 ${scores.trust_signal_quality}/5。`;
|
||||
}
|
||||
|
||||
export function createCalibrationEvent({
|
||||
scoringRun,
|
||||
qaReport,
|
||||
snapshot,
|
||||
}: CalibrationContext): CalibrationEvent {
|
||||
const direction = inferDirection(scoringRun.composite_score, snapshot.metrics);
|
||||
const observations = buildObservations(
|
||||
direction,
|
||||
scoringRun,
|
||||
qaReport,
|
||||
snapshot.feedback_summary,
|
||||
);
|
||||
|
||||
return {
|
||||
id: `cal_${nanoid(10)}`,
|
||||
publication_id: snapshot.publication_id,
|
||||
scoring_run_id: scoringRun.id,
|
||||
performance_snapshot_id: snapshot.id,
|
||||
direction,
|
||||
observations,
|
||||
recommended_action:
|
||||
"先积累至少 5 篇同类样本，再评估是否调整 GEO rubric 权重。",
|
||||
created_at: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
function inferDirection(
|
||||
composite: number,
|
||||
metrics: { views?: number; clicks?: number; inquiries?: number },
|
||||
): CalibrationDirection {
|
||||
if (!metrics.views && !metrics.clicks && !metrics.inquiries) {
|
||||
return "needs_more_data";
|
||||
}
|
||||
if ((metrics.inquiries ?? 0) >= 3 || (metrics.clicks ?? 0) >= 50) {
|
||||
return "better_than_expected";
|
||||
}
|
||||
if ((metrics.views ?? 0) < 100 && composite >= 7) {
|
||||
return "worse_than_expected";
|
||||
}
|
||||
return "as_expected";
|
||||
}
|
||||
|
||||
function buildObservations(
|
||||
direction: CalibrationDirection,
|
||||
scoringRun: ScoringRun,
|
||||
qaReport: QaReport,
|
||||
feedbackSummary: string,
|
||||
) {
|
||||
const observations = [
|
||||
`综合评分 ${scoringRun.composite_score}/10，真实表现方向为 ${direction}。`,
|
||||
];
|
||||
if (qaReport.overall_status !== "pass") {
|
||||
observations.push(`QA 状态为 ${qaReport.overall_status}，需要和表现数据一起复盘。`);
|
||||
}
|
||||
if (feedbackSummary) {
|
||||
observations.push(`反馈摘要：${feedbackSummary}`);
|
||||
}
|
||||
if ((scoringRun.dimension_scores.trust_signal_quality ?? 0) <= 3) {
|
||||
observations.push("信任信号质量偏低，后续观察是否影响询盘。");
|
||||
}
|
||||
return observations;
|
||||
}
|
||||
Reference in New Issue
Block a user