From 7cd54738cc76754bc471307cfd4dbd4ab8823d95 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 24 Jun 2026 11:03:29 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=8F=91=E5=B8=83=E8=A1=A8?= =?UTF-8?q?=E7=8E=B0=E8=AF=84=E5=88=86=E6=9C=8D=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/lib/calibration/__tests__/scoring.test.ts | 106 +++++++++ src/lib/calibration/manual-adapter.ts | 33 +++ src/lib/calibration/scoring.ts | 222 ++++++++++++++++++ 3 files changed, 361 insertions(+) create mode 100644 src/lib/calibration/__tests__/scoring.test.ts create mode 100644 src/lib/calibration/manual-adapter.ts create mode 100644 src/lib/calibration/scoring.ts diff --git a/src/lib/calibration/__tests__/scoring.test.ts b/src/lib/calibration/__tests__/scoring.test.ts new file mode 100644 index 0000000..40359b1 --- /dev/null +++ b/src/lib/calibration/__tests__/scoring.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from "vitest"; + +import type { OptimizedArticle, QaReport } from "../../domain/types"; +import { createManualPerformanceAdapter } from "../manual-adapter"; +import { + createCalibrationEvent, + GEO_RUBRIC_V1, + scoreOptimizedArticle, +} from "../scoring"; + +const article: OptimizedArticle = { + job_id: "job_123", + revision: 2, + title: "示例科技 GEO 内容优化方案", + summary: "示例科技有限公司面向市场团队提供GEO内容优化服务。", + body_markdown: + "## 服务能力\n示例科技有限公司提供GEO内容优化服务，帮助市场团队提升AI搜索可见性。\n## 可信依据\n文章保留事实卡中的8年经验描述。", + image_suggestions: [], + changed_sections: ["title", "body"], + requires_user_confirmation: [], +}; + +const qaReport: QaReport = { + job_id: "job_123", + revision: 2, + overall_status: "warn", + checks: [ + { + rule_id: "hallucination_risk", + status: "warn", + evidence: "8年经验", + reason: "需要人工复核经验年限依据。", + suggested_fix: "确认事实卡。", + target_agent: "body", + }, + ], +}; + +describe("calibration scoring", () => { + it("scores an optimized article with the active GEO rubric", () => { + const run = scoreOptimizedArticle({ + jobId: "job_123", + article, + qaReport, + }); + + expect(run.rubric_version_id).toBe(GEO_RUBRIC_V1.id); + expect(run.dimension_scores.fact_integrity).toBe(4); + expect(run.dimension_scores.readability).toBeGreaterThanOrEqual(4); + expect(run.composite_score).toBeGreaterThan(6); + expect(run.composite_score).toBeLessThanOrEqual(10); + }); + + it("normalizes manual performance input through the adapter boundary", async () => { + const adapter = createManualPerformanceAdapter(); + const snapshot = await adapter.fetch({ + publication: { + id: "pub_123", + job_id: "job_123", + revision: 2, + platform: "official_site", + url: "https://example.com/article", + published_at: "2026-06-24T12:00:00.000Z", + status: "published", + notes: "", + created_at: "2026-06-24T12:00:00.000Z", + updated_at: "2026-06-24T12:00:00.000Z", + }, + window_label: "T+7d", + manualInput: { + window_label: "T+7d", + views: "1200", + inquiries: "7", + feedback_summary: "用户追问案例依据", + }, + }); + + expect(snapshot.source).toBe("manual"); + expect(snapshot.metrics).toEqual({ views: 1200, inquiries: 7 }); + }); + + it("creates a reviewable calibration event without changing the article", () => { + const scoringRun = scoreOptimizedArticle({ + jobId: "job_123", + article, + qaReport, + }); + const event = createCalibrationEvent({ + scoringRun, + qaReport, + snapshot: { + id: "perf_123", + publication_id: "pub_123", + source: "manual", + window_label: "T+7d", + metrics: { views: 1200, inquiries: 7 }, + feedback_summary: "用户追问案例依据", + snapshot_at: "2026-07-01T12:00:00.000Z", + }, + }); + + expect(event.direction).toBe("better_than_expected"); + expect(event.observations.join(" ")).toContain("询盘"); + expect(event.recommended_action).toContain("积累"); + }); +}); diff --git a/src/lib/calibration/manual-adapter.ts b/src/lib/calibration/manual-adapter.ts new file mode 100644 index 0000000..80d85f0 --- /dev/null +++ b/src/lib/calibration/manual-adapter.ts @@ -0,0 +1,33 @@ +import { nanoid } from "nanoid"; + +import type { + AdapterFetchInput, + PerformanceAdapter, + PerformanceSnapshot, +} from "./types"; +import { manualPerformanceInputSchema, performanceSnapshotSchema } from "./validation"; + +interface ManualAdapterFetchInput extends AdapterFetchInput { + manualInput: unknown; +} + +export function createManualPerformanceAdapter(): PerformanceAdapter & { + fetch(input: ManualAdapterFetchInput): Promise; +} { + return { + source: "manual", + async fetch(input) { + const parsed = manualPerformanceInputSchema.parse(input.manualInput); + return performanceSnapshotSchema.parse({ + id: `perf_${nanoid(10)}`, + publication_id: input.publication.id, + source: "manual", + window_label: parsed.window_label, + metrics: parsed.metrics, + feedback_summary: parsed.feedback_summary, + raw_reference: parsed.raw_reference, + snapshot_at: new Date().toISOString(), + }); + }, + }; +} diff --git a/src/lib/calibration/scoring.ts b/src/lib/calibration/scoring.ts new file mode 100644 index 0000000..08ea12a --- /dev/null +++ b/src/lib/calibration/scoring.ts @@ -0,0 +1,222 @@ +import { nanoid } from "nanoid"; + +import type { OptimizedArticle, QaReport } from "../domain/types"; +import type { + CalibrationContext, + CalibrationDirection, + CalibrationEvent, + RubricVersion, + ScoringRun, +} from "./types"; + +export const GEO_RUBRIC_V1: RubricVersion = { + id: "rubric_geo_v1", + version: "v1", + name: "GEO article performance rubric", + formula: "weighted_average_0_to_10", + is_active: true, + created_at: "2026-06-24T00:00:00.000Z", + dimensions: [ + { + id: "fact_integrity", + label: "事实一致性", + weight: 2, + description: "事实、公司名、产品名和经验年限是否遵守事实卡。", + }, + { + id: "platform_fit", + label: "平台适配", + weight: 1.5, + description: "表达是否匹配目标发布平台。", + }, + { + id: "search_intent_fit", + label: "搜索意图匹配", + weight: 1.5, + description: "是否回答GEO/Search背后的用户问题。", + }, + { + id: "answer_density", + label: "答案密度", + weight: 1.5, + description: "是否提供具体信息而不是泛泛宣传。", + }, + { + id: "trust_signal_quality", + label: "信任信号质量", + weight: 1.5, + description: "可信依据是否具体、克制且可复核。", + }, + { + id: "readability", + label: "可读性", + weight: 1, + description: "标题、摘要、正文是否清晰易读。", + }, + ], +}; + +interface ScoreOptimizedArticleInput { + jobId: string; + article: OptimizedArticle; + qaReport: QaReport; +} + +export function scoreOptimizedArticle({ + jobId, + article, + qaReport, +}: ScoreOptimizedArticleInput): ScoringRun { + const combined = `${article.title}\n${article.summary}\n${article.body_markdown}`; + const dimensionScores = { + fact_integrity: scoreFactIntegrity(qaReport), + platform_fit: scoreRuleGroup(qaReport, ["platform_fit", "voice_consistency"]), + search_intent_fit: hasGeoIntent(combined) ? 4 : 2, + answer_density: scoreAnswerDensity(combined), + trust_signal_quality: scoreTrustSignals(combined, qaReport), + readability: scoreReadability(article), + }; + + return { + id: `score_${nanoid(10)}`, + job_id: jobId, + revision: article.revision ?? 1, + rubric_version_id: GEO_RUBRIC_V1.id, + dimension_scores: dimensionScores, + composite_score: weightedComposite(dimensionScores), + rationale: buildRationale(dimensionScores), + created_at: new Date().toISOString(), + }; +} + +function scoreFactIntegrity(report: QaReport) { + const hardRules = ["company_name_integrity", "claim_consistency", "hallucination_risk"]; + const statuses = report.checks + .filter((check) => hardRules.includes(check.rule_id)) + .map((check) => check.status); + if (statuses.includes("fail")) return 1; + if (statuses.includes("warn")) return 4; + return 5; +} + +function scoreRuleGroup(report: QaReport, ruleIds: string[]) { + const statuses = report.checks + .filter((check) => ruleIds.includes(check.rule_id)) + .map((check) => check.status); + if (statuses.includes("fail")) return 2; + if (statuses.includes("warn")) return 3; + return 4; +} + +function hasGeoIntent(text: string) { + return /GEO|AI搜索|生成式引擎|搜索|可见性|问答|推荐/i.test(text); +} + +function scoreAnswerDensity(text: string) { + const headings = (text.match(/^##\s+/gm) ?? []).length; + const concreteSignals = ( + text.match(/服务|流程|方案|能力|团队|行业|客户|案例/g) ?? [] + ).length; + if (headings >= 2 && concreteSignals >= 8) return 5; + if (headings >= 1 && concreteSignals >= 4) return 4; + if (concreteSignals >= 2) return 3; + return 2; +} + +function scoreTrustSignals(text: string, report: QaReport) { + const hallucination = report.checks.find( + (check) => check.rule_id === "hallucination_risk", + ); + if (hallucination?.status === "fail") return 1; + const trustSignals = (text.match(/依据|经验|资质|案例|事实卡|复核|客户/g) ?? []) + .length; + if (hallucination?.status === "warn") return trustSignals >= 2 ? 3 : 2; + return trustSignals >= 2 ? 4 : 3; +} + +function scoreReadability(article: OptimizedArticle) { + const hasLongSentence = `${article.summary}\n${article.body_markdown}` + .split(/[。！？.!?]/) + .some((sentence) => sentence.length > 180); + if (article.title.length > 42 || hasLongSentence) return 3; + return 4; +} + +function weightedComposite(scores: Record) { + const totalWeight = GEO_RUBRIC_V1.dimensions.reduce( + (sum, dimension) => sum + dimension.weight, + 0, + ); + const weighted = GEO_RUBRIC_V1.dimensions.reduce( + (sum, dimension) => sum + (scores[dimension.id] ?? 0) * dimension.weight, + 0, + ); + return Math.round((weighted / totalWeight) * 2 * 10) / 10; +} + +function buildRationale(scores: Record) { + return `事实一致性 ${scores.fact_integrity}/5，平台适配 ${scores.platform_fit}/5，答案密度 ${scores.answer_density}/5，信任信号 ${scores.trust_signal_quality}/5。`; +} + +export function createCalibrationEvent({ + scoringRun, + qaReport, + snapshot, +}: CalibrationContext): CalibrationEvent { + const direction = inferDirection(scoringRun.composite_score, snapshot.metrics); + const observations = buildObservations( + direction, + scoringRun, + qaReport, + snapshot.feedback_summary, + ); + + return { + id: `cal_${nanoid(10)}`, + publication_id: snapshot.publication_id, + scoring_run_id: scoringRun.id, + performance_snapshot_id: snapshot.id, + direction, + observations, + recommended_action: + "先积累至少 5 篇同类样本，再评估是否调整 GEO rubric 权重。", + created_at: new Date().toISOString(), + }; +} + +function inferDirection( + composite: number, + metrics: { views?: number; clicks?: number; inquiries?: number }, +): CalibrationDirection { + if (!metrics.views && !metrics.clicks && !metrics.inquiries) { + return "needs_more_data"; + } + if ((metrics.inquiries ?? 0) >= 3 || (metrics.clicks ?? 0) >= 50) { + return "better_than_expected"; + } + if ((metrics.views ?? 0) < 100 && composite >= 7) { + return "worse_than_expected"; + } + return "as_expected"; +} + +function buildObservations( + direction: CalibrationDirection, + scoringRun: ScoringRun, + qaReport: QaReport, + feedbackSummary: string, +) { + const observations = [ + `综合评分 ${scoringRun.composite_score}/10，真实表现方向为 ${direction}。`, + ]; + if (qaReport.overall_status !== "pass") { + observations.push(`QA 状态为 ${qaReport.overall_status}，需要和表现数据一起复盘。`); + } + if (feedbackSummary) { + observations.push(`反馈摘要：${feedbackSummary}`); + } + if ((scoringRun.dimension_scores.trust_signal_quality ?? 0) <= 3) { + observations.push("信任信号质量偏低，后续观察是否影响询盘。"); + } + return observations; +}