From 01c02049d4ed1ded2892ae4083c594a3a0de4e4a Mon Sep 17 00:00:00 2001 From: czj <13261895355@163.com> Date: Wed, 8 Jul 2026 13:10:36 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9ELLM=E5=AE=A1=E8=AE=A1?= =?UTF-8?q?=E6=91=98=E8=A6=81=E8=BE=B9=E7=95=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/lib/cases/__tests__/summaries.test.ts | 65 ++++++++++++++++++++++ src/lib/cases/summaries.ts | 67 +++++++++++++++++++++++ src/lib/cases/types.ts | 3 +- src/lib/llm/__tests__/audit.test.ts | 37 +++++++++++++ src/lib/llm/__tests__/client.test.ts | 26 +++++++++ src/lib/llm/audit.ts | 51 +++++++++++++++++ src/lib/llm/client.ts | 49 ++++++++++++++++- 7 files changed, 294 insertions(+), 4 deletions(-) create mode 100644 src/lib/cases/__tests__/summaries.test.ts create mode 100644 src/lib/cases/summaries.ts create mode 100644 src/lib/llm/__tests__/audit.test.ts create mode 100644 src/lib/llm/audit.ts diff --git a/src/lib/cases/__tests__/summaries.test.ts b/src/lib/cases/__tests__/summaries.test.ts new file mode 100644 index 0000000..394d62f --- /dev/null +++ b/src/lib/cases/__tests__/summaries.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from "vitest"; + +import { + buildArticleCaseSummary, + buildHumanCopyCaseSummary, + createProcessStep, + excerpt, +} from "../summaries"; + +describe("case summaries", () => { + it("creates compact source excerpts", () => { + expect(excerpt("第一段。\n\n第二段内容很长".repeat(20), 20)).toHaveLength(21); + expect(excerpt(" 一段文案 ", 20)).toBe("一段文案"); + }); + + it("builds article case title, summary, and publish target", () => { + expect( + buildArticleCaseSummary({ + source_title: "IPMS 推荐机构文章", + source_body: "正文内容", + publish_platform: "media_article", + }), + ).toEqual({ + title: "IPMS 推荐机构文章", + summary: "正文内容", + publish_target: "media_article", + source_excerpt: "正文内容", + }); + }); + + it("builds human-copy case summary from source and publish target", () => { + expect( + buildHumanCopyCaseSummary({ + source_text: "帮客户解释智能体授课的价值。", + goal: "自然一点", + intensity: "light", + user_instructions: "", + publish_target: "朋友圈", + }), + ).toMatchObject({ + title: "人味文案优化：朋友圈", + publish_target: "朋友圈", + source_excerpt: "帮客户解释智能体授课的价值。", + }); + }); + + it("creates a process summary step without draft text", () => { + expect( + createProcessStep({ + stage: "draft", + startedAt: 100, + endedAt: 250, + status: "success", + producedResultVersion: false, + }), + ).toEqual({ + stage: "draft", + started_at: expect.any(String), + ended_at: expect.any(String), + duration_ms: 150, + status: "success", + produced_result_version: false, + }); + }); +}); diff --git a/src/lib/cases/summaries.ts b/src/lib/cases/summaries.ts new file mode 100644 index 0000000..40f030b --- /dev/null +++ b/src/lib/cases/summaries.ts @@ -0,0 +1,67 @@ +import type { + ArticleCaseInputPayload, + HumanCopyCaseInputPayload, + ProcessSummaryStep, +} from "./types"; + +export function excerpt(value: string, maxLength = 120) { + const compact = value.replace(/\s+/g, " ").trim(); + return compact.length > maxLength + ? `${compact.slice(0, maxLength)}…` + : compact; +} + +export function buildArticleCaseSummary( + input: Pick< + ArticleCaseInputPayload, + "source_title" | "source_body" | "publish_platform" + >, +) { + const sourceExcerpt = excerpt(input.source_body); + return { + title: input.source_title.trim() || excerpt(input.source_body, 32), + summary: sourceExcerpt, + publish_target: input.publish_platform, + source_excerpt: sourceExcerpt, + }; +} + +export function buildHumanCopyCaseSummary(input: HumanCopyCaseInputPayload) { + const publishTarget = input.publish_target.trim() || "未指定"; + const sourceExcerpt = excerpt(input.source_text); + return { + title: `人味文案优化：${publishTarget}`, + summary: sourceExcerpt, + publish_target: publishTarget, + source_excerpt: sourceExcerpt, + }; +} + +export function createProcessStep({ + stage, + startedAt, + endedAt, + status, + errorSummary, + rewriteRound, + producedResultVersion, +}: { + stage: string; + startedAt: number; + endedAt: number; + status: ProcessSummaryStep["status"]; + errorSummary?: string; + rewriteRound?: number; + producedResultVersion: boolean; +}): ProcessSummaryStep { + return { + stage, + started_at: new Date(startedAt).toISOString(), + ended_at: new Date(endedAt).toISOString(), + duration_ms: Math.max(0, endedAt - startedAt), + status, + ...(errorSummary ? { error_summary: errorSummary } : {}), + ...(rewriteRound ? { rewrite_round: rewriteRound } : {}), + produced_result_version: producedResultVersion, + }; +} diff --git a/src/lib/cases/types.ts b/src/lib/cases/types.ts index a28860c..423fc52 100644 --- a/src/lib/cases/types.ts +++ b/src/lib/cases/types.ts @@ -4,6 +4,7 @@ import type { OptimizedArticle, QaReport, } from "../domain/types"; +import type { LlmAuditSummary } from "../llm/audit"; export type OptimizationCaseType = "article" | "human_copy"; export type OptimizationCaseStatus = @@ -85,7 +86,7 @@ export interface OptimizationResultVersion { result_summary: string; payload: ArticleResultVersionPayload | HumanCopyResultVersionPayload | null; process_summary: ProcessSummaryStep[]; - llm_audit_summary: unknown[]; + llm_audit_summary: LlmAuditSummary[]; error_stage: string | null; error_summary: string | null; created_at: string; diff --git a/src/lib/llm/__tests__/audit.test.ts b/src/lib/llm/__tests__/audit.test.ts new file mode 100644 index 0000000..bccee85 --- /dev/null +++ b/src/lib/llm/__tests__/audit.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; + +import { createLlmAuditSummary, hashContent } from "../audit"; + +describe("LLM audit summary", () => { + it("hashes content deterministically", async () => { + await expect(hashContent("abc")).resolves.toBe( + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + ); + }); + + it("does not retain raw prompt or response", async () => { + const summary = await createLlmAuditSummary({ + provider: "deepseek", + model: "deepseek-v4-pro", + task: "renwei_copy_optimizer", + duration_ms: 12, + schema_valid: true, + prompt: "完整 prompt 不应长期保存", + output: "完整 response 不应长期保存", + error_summary: null, + }); + + expect(summary).toMatchObject({ + provider: "deepseek", + model: "deepseek-v4-pro", + task: "renwei_copy_optimizer", + duration_ms: 12, + schema_valid: true, + error_summary: null, + }); + expect(JSON.stringify(summary)).not.toContain("完整 prompt"); + expect(JSON.stringify(summary)).not.toContain("完整 response"); + expect(summary.input_hash).toHaveLength(64); + expect(summary.output_hash).toHaveLength(64); + }); +}); diff --git a/src/lib/llm/__tests__/client.test.ts b/src/lib/llm/__tests__/client.test.ts index 276fbf5..9b4e326 100644 --- a/src/lib/llm/__tests__/client.test.ts +++ b/src/lib/llm/__tests__/client.test.ts @@ -41,6 +41,32 @@ describe("generateValidatedJson", () => { expect(result).toEqual({ value: "from-llm" }); }); + it("emits an audit summary without raw prompt or response", async () => { + const audits: unknown[] = []; + process.env.LLM_PROVIDER = "deepseek"; + process.env.DEEPSEEK_API_KEY = "test-key"; + client.setGenerateJsonForValidation(async () => ({ value: "ok" })); + + const result = await client.generateValidatedJson({ + schema: z.object({ value: z.string() }), + prompt: "raw prompt", + task: "unknown", + onAuditSummary: (summary) => audits.push(summary), + }); + + expect(result).toEqual({ value: "ok" }); + expect(JSON.stringify(audits)).not.toContain("raw prompt"); + expect(JSON.stringify(audits)).not.toContain("ok"); + expect(audits).toEqual([ + expect.objectContaining({ + task: "unknown", + schema_valid: true, + input_hash: expect.any(String), + output_hash: expect.any(String), + }), + ]); + }); + it("throws clearly when the model response fails schema validation", async () => { process.env.LLM_PROVIDER = "deepseek"; process.env.DEEPSEEK_API_KEY = "test-key"; diff --git a/src/lib/llm/audit.ts b/src/lib/llm/audit.ts new file mode 100644 index 0000000..e0c7e7f --- /dev/null +++ b/src/lib/llm/audit.ts @@ -0,0 +1,51 @@ +import type { LlmProviderStatus, LlmTaskName } from "./client"; + +export interface LlmAuditSummary { + provider: LlmProviderStatus["provider"]; + model: string; + task: LlmTaskName; + duration_ms: number; + schema_valid: boolean; + error_summary: string | null; + input_hash: string; + output_hash: string | null; +} + +export async function hashContent(value: string) { + const data = new TextEncoder().encode(value); + const digest = await crypto.subtle.digest("SHA-256", data); + return Array.from(new Uint8Array(digest)) + .map((byte) => byte.toString(16).padStart(2, "0")) + .join(""); +} + +export async function createLlmAuditSummary({ + provider, + model, + task, + duration_ms, + schema_valid, + prompt, + output, + error_summary, +}: { + provider: LlmProviderStatus["provider"]; + model: string; + task: LlmTaskName; + duration_ms: number; + schema_valid: boolean; + prompt: string; + output: string | null; + error_summary: string | null; +}): Promise { + return { + provider, + model, + task, + duration_ms, + schema_valid, + error_summary, + input_hash: await hashContent(prompt), + output_hash: output == null ? null : await hashContent(output), + }; +} diff --git a/src/lib/llm/client.ts b/src/lib/llm/client.ts index 07bd9fb..3195400 100644 --- a/src/lib/llm/client.ts +++ b/src/lib/llm/client.ts @@ -1,6 +1,8 @@ import OpenAI from "openai"; import type { z } from "zod"; +import { createLlmAuditSummary, type LlmAuditSummary } from "./audit"; + export type LlmTaskName = | "unknown" | "fact_extractor" @@ -15,6 +17,7 @@ export interface GenerateInput { model?: string; temperature?: number; task?: LlmTaskName; + onAuditSummary?: (summary: LlmAuditSummary) => void | Promise; } export interface GenerateValidatedJsonInput extends GenerateInput { @@ -228,9 +231,33 @@ export async function generateValidatedJson({ const usesDefaultGenerator = generateJsonForValidation === generateJson; const status = getLlmProviderStatus(); const startedAt = Date.now(); + const effectiveModel = input.model ?? status.model; + const emitAudit = async ({ + schemaValid, + output, + errorSummary, + }: { + schemaValid: boolean; + output: unknown | null; + errorSummary: string | null; + }) => { + if (!input.onAuditSummary) return; + await input.onAuditSummary( + await createLlmAuditSummary({ + provider: status.provider, + model: effectiveModel, + task, + duration_ms: Date.now() - startedAt, + schema_valid: schemaValid, + prompt: input.prompt, + output: output == null ? null : stringifyForLog(output), + error_summary: errorSummary, + }), + ); + }; if (!usesDefaultGenerator) { console.info( - `[llm:start] provider=${status.provider} model=${input.model ?? status.model} task=${task}`, + `[llm:start] provider=${status.provider} model=${effectiveModel} task=${task}`, ); } @@ -243,14 +270,25 @@ export async function generateValidatedJson({ } const parsed = schema.safeParse(generated); if (parsed.success) { + await emitAudit({ + schemaValid: true, + output: parsed.data, + errorSummary: null, + }); console.info(`[llm:validated] task=${task} ok=true`); return parsed.data; } + const zodSummary = summarizeZodError(parsed.error); console.warn( - `[llm:validated] task=${task} ok=false zod_error=${quoteLogValue(summarizeZodError(parsed.error))}`, + `[llm:validated] task=${task} ok=false zod_error=${quoteLogValue(zodSummary)}`, ); + await emitAudit({ + schemaValid: false, + output: generated, + errorSummary: zodSummary, + }); throw new LlmValidationError( - `LLM response failed schema validation: ${summarizeZodError(parsed.error)}`, + `LLM response failed schema validation: ${zodSummary}`, task, ); } catch (error) { @@ -259,6 +297,11 @@ export async function generateValidatedJson({ } console.info(`[llm:validated] task=${task} ok=false reason=provider_error`); const message = error instanceof Error ? error.message : String(error); + await emitAudit({ + schemaValid: false, + output: null, + errorSummary: message, + }); console.warn( usesDefaultGenerator ? `[llm:error] task=${task} message=${quoteLogValue(message)}`