czj/GEOAgentArticleOptimizer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

新增人味文案评分和效果学习

Browse Source
This commit is contained in:
czj
2026-07-08 14:05:40 +08:00
parent c9ca93faa0
commit cf74ad920c
9 changed files with 338 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/calibration/__tests__/scoring.test.ts
+38
View File
@@ -5,6 +5,7 @@ import { createManualPerformanceAdapter } from "../manual-adapter";
import {
createCalibrationEvent,
GEO_RUBRIC_V1,
scoreHumanCopyResult,
scoreOptimizedArticle,
} from "../scoring";
@@ -56,8 +57,10 @@ describe("calibration scoring", () => {
const snapshot = await adapter.fetch({
publication: {
id: "pub_123",
result_version_id: null,
job_id: "job_123",
revision: 2,
publish_target: "official_site",
platform: "official_site",
url: "https://example.com/article",
published_at: "2026-06-24T12:00:00.000Z",
@@ -103,4 +106,39 @@ describe("calibration scoring", () => {
expect(event.observations.join(" ")).toContain("询盘");
expect(event.recommended_action).toContain("积累");
});
it("scores human-copy result with a separate rubric", () => {
const scoringRun = scoreHumanCopyResult({
resultVersionId: "ver_1",
result: {
optimized_text: "我把这段文案顺了一下。",
change_notes: [
{
original: "我把这段文案顺顺。",
revised: "我把这段文案顺了一下。",
reason: "修正重复表达。",
confidence: "confident",
revertible: false,
},
],
ai_taste_checks: [
{
rule_id: "promotion_tone",
status: "pass",
evidence: "没有新增宣传腔。",
suggestion: "",
},
],
warnings: [],
},
});
expect(scoringRun).toMatchObject({
result_version_id: "ver_1",
case_type: "human_copy",
rubric_version_id: "rubric_human_copy_v1",
composite_score: expect.any(Number),
});
expect(scoringRun.composite_score).toBeGreaterThan(0);
});
});
src/lib/calibration/scoring.ts
+93 -3
View File
@@ -1,6 +1,10 @@
import { nanoid } from "nanoid";
import type { OptimizedArticle, QaReport } from "../domain/types";
import type {
CopyOptimizationResult,
OptimizedArticle,
QaReport,
} from "../domain/types";
import type {
CalibrationContext,
CalibrationDirection,
@@ -56,16 +60,59 @@ export const GEO_RUBRIC_V1: RubricVersion = {
],
};
export const HUMAN_COPY_RUBRIC_V1: RubricVersion = {
id: "rubric_human_copy_v1",
version: "v1",
name: "人味文案优化评分口径",
dimensions: [
{
id: "restraint",
label: "改动克制",
weight: 0.2,
description: "减少机械润色，不把短文案扩写成宣传稿。",
},
{
id: "meaning_fidelity",
label: "原意保真",
weight: 0.25,
description: "保留原文意图、事实和表达边界。",
},
{
id: "natural_tone",
label: "语气自然度",
weight: 0.25,
description: "读起来像真人表达，少套路句和格式痕迹。",
},
{
id: "goal_fit",
label: "目标匹配",
weight: 0.15,
description: "符合优化目标和发布场景。",
},
{
id: "ai_taste_risk",
label: "AI味风险",
weight: 0.15,
description: "宣传腔、套话、聊天痕迹和填充词风险低。",
},
],
formula: "weighted_average_0_to_10",
is_active: true,
created_at: "2026-07-08T00:00:00.000Z",
};
interface ScoreOptimizedArticleInput {
jobId: string;
article: OptimizedArticle;
qaReport: QaReport;
resultVersionId?: string | null;
}
export function scoreOptimizedArticle({
jobId,
article,
qaReport,
resultVersionId = null,
}: ScoreOptimizedArticleInput): ScoringRun {
const combined = `${article.title}\n${article.summary}\n${article.body_markdown}`;
const dimensionScores = {
@@ -79,6 +126,8 @@ export function scoreOptimizedArticle({
return {
id: `score_${nanoid(10)}`,
result_version_id: resultVersionId,
case_type: "article",
job_id: jobId,
revision: article.revision ?? 1,
rubric_version_id: GEO_RUBRIC_V1.id,
@@ -89,6 +138,43 @@ export function scoreOptimizedArticle({
};
}
export function scoreHumanCopyResult({
resultVersionId,
result,
}: {
resultVersionId: string;
result: CopyOptimizationResult;
}): ScoringRun {
const warningsPenalty = Math.min(result.warnings.length, 3) * 0.5;
const aiTasteWarnings = result.ai_taste_checks.filter(
(check) => check.status === "warn",
).length;
const dimensionScores = {
restraint: result.optimized_text.length > 280 ? 3 : 5,
meaning_fidelity: result.change_notes.some(
(note) => note.confidence === "uncertain",
)
? 3.5
: 5,
natural_tone: Math.max(2, 5 - aiTasteWarnings * 0.75),
goal_fit: 4.5,
ai_taste_risk: Math.max(1, 5 - aiTasteWarnings - warningsPenalty),
};
return {
id: `score_${nanoid(10)}`,
result_version_id: resultVersionId,
case_type: "human_copy",
job_id: null,
revision: null,
rubric_version_id: HUMAN_COPY_RUBRIC_V1.id,
dimension_scores: dimensionScores,
composite_score: weightedAverage0To10(HUMAN_COPY_RUBRIC_V1, dimensionScores),
rationale: "基于改动克制、原意保真、自然度、目标匹配和AI味风险生成评分。",
created_at: new Date().toISOString(),
};
}
function scoreFactIntegrity(report: QaReport) {
const hardRules = ["company_name_integrity", "claim_consistency", "hallucination_risk"];
const statuses = report.checks
@@ -143,11 +229,15 @@ function scoreReadability(article: OptimizedArticle) {
}
function weightedComposite(scores: Record<string, number>) {
const totalWeight = GEO_RUBRIC_V1.dimensions.reduce(
return weightedAverage0To10(GEO_RUBRIC_V1, scores);
}
function weightedAverage0To10(rubric: RubricVersion, scores: Record<string, number>) {
const totalWeight = rubric.dimensions.reduce(
(sum, dimension) => sum + dimension.weight,
0,
);
const weighted = GEO_RUBRIC_V1.dimensions.reduce(
const weighted = rubric.dimensions.reduce(
(sum, dimension) => sum + (scores[dimension.id] ?? 0) * dimension.weight,
0,
);
src/lib/db/schema.ts
+64 -3
View File
@@ -207,8 +207,7 @@ export function initializeSchema(db: Database.Database) {
`);
ensureColumn(db, "article_jobs", "case_id", "text");
ensureColumn(db, "scoring_runs", "result_version_id", "text");
ensureColumn(db, "scoring_runs", "case_type", "text not null default 'article'");
migrateScoringRunsForCases(db);
ensureColumn(db, "publication_records", "result_version_id", "text");
ensureColumn(
db,
@@ -234,10 +233,72 @@ export function initializeSchema(db: Database.Database) {
}
function getTableColumns(db: Database.Database, tableName: string) {
return getTableColumnInfo(db, tableName).map((row) => row.name);
}
function getTableColumnInfo(db: Database.Database, tableName: string) {
return db
.prepare(`pragma table_info(${tableName})`)
.all()
.map((row) => (row as { name: string }).name);
.map((row) => row as { name: string; notnull: number });
}
function migrateScoringRunsForCases(db: Database.Database) {
const columns = getTableColumnInfo(db, "scoring_runs");
const columnNames = columns.map((column) => column.name);
const jobIdColumn = columns.find((column) => column.name === "job_id");
const revisionColumn = columns.find((column) => column.name === "revision");
const needsRebuild =
!columnNames.includes("result_version_id") ||
!columnNames.includes("case_type") ||
jobIdColumn?.notnull === 1 ||
revisionColumn?.notnull === 1;
if (!needsRebuild) return;
const resultVersionSelect = columnNames.includes("result_version_id")
? "result_version_id"
: "NULL";
const caseTypeSelect = columnNames.includes("case_type")
? "case_type"
: "'article'";
db.pragma("foreign_keys = OFF");
try {
db.exec(`
drop table if exists scoring_runs_next;
create table scoring_runs_next (
id text primary key,
result_version_id text,
case_type text not null default 'article',
job_id text,
revision integer,
rubric_version_id text not null,
dimension_scores text not null,
composite_score real not null,
rationale text not null,
created_at text not null,
foreign key (result_version_id)
references optimization_result_versions(id) on delete cascade,
foreign key (rubric_version_id) references rubric_versions(id)
);
insert into scoring_runs_next (
id, result_version_id, case_type, job_id, revision, rubric_version_id,
dimension_scores, composite_score, rationale, created_at
)
select
id, ${resultVersionSelect}, ${caseTypeSelect}, job_id, revision,
rubric_version_id, dimension_scores, composite_score, rationale, created_at
from scoring_runs;
drop table scoring_runs;
alter table scoring_runs_next rename to scoring_runs;
`);
} finally {
db.pragma("foreign_keys = ON");
}
}
function ensureColumn(