新增LLM审计摘要边界

src/lib/llm/__tests__/audit.test.ts

@@ -0,0 +1,37 @@
+import { describe, expect, it } from "vitest";
+
+import { createLlmAuditSummary, hashContent } from "../audit";
+
+describe("LLM audit summary", () => {
+  it("hashes content deterministically", async () => {
+    await expect(hashContent("abc")).resolves.toBe(
+      "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
+    );
+  });
+
+  it("does not retain raw prompt or response", async () => {
+    const summary = await createLlmAuditSummary({
+      provider: "deepseek",
+      model: "deepseek-v4-pro",
+      task: "renwei_copy_optimizer",
+      duration_ms: 12,
+      schema_valid: true,
+      prompt: "完整 prompt 不应长期保存",
+      output: "完整 response 不应长期保存",
+      error_summary: null,
+    });
+
+    expect(summary).toMatchObject({
+      provider: "deepseek",
+      model: "deepseek-v4-pro",
+      task: "renwei_copy_optimizer",
+      duration_ms: 12,
+      schema_valid: true,
+      error_summary: null,
+    });
+    expect(JSON.stringify(summary)).not.toContain("完整 prompt");
+    expect(JSON.stringify(summary)).not.toContain("完整 response");
+    expect(summary.input_hash).toHaveLength(64);
+    expect(summary.output_hash).toHaveLength(64);
+  });
+});

src/lib/llm/__tests__/client.test.ts

@@ -41,6 +41,32 @@ describe("generateValidatedJson", () => {
     expect(result).toEqual({ value: "from-llm" });
   });
 
+  it("emits an audit summary without raw prompt or response", async () => {
+    const audits: unknown[] = [];
+    process.env.LLM_PROVIDER = "deepseek";
+    process.env.DEEPSEEK_API_KEY = "test-key";
+    client.setGenerateJsonForValidation(async () => ({ value: "ok" }));
+
+    const result = await client.generateValidatedJson({
+      schema: z.object({ value: z.string() }),
+      prompt: "raw prompt",
+      task: "unknown",
+      onAuditSummary: (summary) => audits.push(summary),
+    });
+
+    expect(result).toEqual({ value: "ok" });
+    expect(JSON.stringify(audits)).not.toContain("raw prompt");
+    expect(JSON.stringify(audits)).not.toContain("ok");
+    expect(audits).toEqual([
+      expect.objectContaining({
+        task: "unknown",
+        schema_valid: true,
+        input_hash: expect.any(String),
+        output_hash: expect.any(String),
+      }),
+    ]);
+  });
+
   it("throws clearly when the model response fails schema validation", async () => {
     process.env.LLM_PROVIDER = "deepseek";
     process.env.DEEPSEEK_API_KEY = "test-key";

src/lib/llm/audit.ts

@@ -0,0 +1,51 @@
+import type { LlmProviderStatus, LlmTaskName } from "./client";
+
+export interface LlmAuditSummary {
+  provider: LlmProviderStatus["provider"];
+  model: string;
+  task: LlmTaskName;
+  duration_ms: number;
+  schema_valid: boolean;
+  error_summary: string | null;
+  input_hash: string;
+  output_hash: string | null;
+}
+
+export async function hashContent(value: string) {
+  const data = new TextEncoder().encode(value);
+  const digest = await crypto.subtle.digest("SHA-256", data);
+  return Array.from(new Uint8Array(digest))
+    .map((byte) => byte.toString(16).padStart(2, "0"))
+    .join("");
+}
+
+export async function createLlmAuditSummary({
+  provider,
+  model,
+  task,
+  duration_ms,
+  schema_valid,
+  prompt,
+  output,
+  error_summary,
+}: {
+  provider: LlmProviderStatus["provider"];
+  model: string;
+  task: LlmTaskName;
+  duration_ms: number;
+  schema_valid: boolean;
+  prompt: string;
+  output: string | null;
+  error_summary: string | null;
+}): Promise<LlmAuditSummary> {
+  return {
+    provider,
+    model,
+    task,
+    duration_ms,
+    schema_valid,
+    error_summary,
+    input_hash: await hashContent(prompt),
+    output_hash: output == null ? null : await hashContent(output),
+  };
+}

src/lib/llm/client.ts

@@ -1,6 +1,8 @@
 import OpenAI from "openai";
 import type { z } from "zod";
 
+import { createLlmAuditSummary, type LlmAuditSummary } from "./audit";
+
 export type LlmTaskName =
   | "unknown"
   | "fact_extractor"
@@ -15,6 +17,7 @@ export interface GenerateInput {
   model?: string;
   temperature?: number;
   task?: LlmTaskName;
+  onAuditSummary?: (summary: LlmAuditSummary) => void | Promise<void>;
 }
 
 export interface GenerateValidatedJsonInput<T> extends GenerateInput {
@@ -228,9 +231,33 @@ export async function generateValidatedJson<T>({
   const usesDefaultGenerator = generateJsonForValidation === generateJson;
   const status = getLlmProviderStatus();
   const startedAt = Date.now();
+  const effectiveModel = input.model ?? status.model;
+  const emitAudit = async ({
+    schemaValid,
+    output,
+    errorSummary,
+  }: {
+    schemaValid: boolean;
+    output: unknown | null;
+    errorSummary: string | null;
+  }) => {
+    if (!input.onAuditSummary) return;
+    await input.onAuditSummary(
+      await createLlmAuditSummary({
+        provider: status.provider,
+        model: effectiveModel,
+        task,
+        duration_ms: Date.now() - startedAt,
+        schema_valid: schemaValid,
+        prompt: input.prompt,
+        output: output == null ? null : stringifyForLog(output),
+        error_summary: errorSummary,
+      }),
+    );
+  };
   if (!usesDefaultGenerator) {
     console.info(
-      `[llm:start] provider=${status.provider} model=${input.model ?? status.model} task=${task}`,
+      `[llm:start] provider=${status.provider} model=${effectiveModel} task=${task}`,
     );
   }
 
@@ -243,14 +270,25 @@ export async function generateValidatedJson<T>({
     }
     const parsed = schema.safeParse(generated);
     if (parsed.success) {
+      await emitAudit({
+        schemaValid: true,
+        output: parsed.data,
+        errorSummary: null,
+      });
       console.info(`[llm:validated] task=${task} ok=true`);
       return parsed.data;
     }
+    const zodSummary = summarizeZodError(parsed.error);
     console.warn(
-      `[llm:validated] task=${task} ok=false zod_error=${quoteLogValue(summarizeZodError(parsed.error))}`,
+      `[llm:validated] task=${task} ok=false zod_error=${quoteLogValue(zodSummary)}`,
     );
+    await emitAudit({
+      schemaValid: false,
+      output: generated,
+      errorSummary: zodSummary,
+    });
     throw new LlmValidationError(
-      `LLM response failed schema validation: ${summarizeZodError(parsed.error)}`,
+      `LLM response failed schema validation: ${zodSummary}`,
       task,
     );
   } catch (error) {
@@ -259,6 +297,11 @@ export async function generateValidatedJson<T>({
     }
     console.info(`[llm:validated] task=${task} ok=false reason=provider_error`);
     const message = error instanceof Error ? error.message : String(error);
+    await emitAudit({
+      schemaValid: false,
+      output: null,
+      errorSummary: message,
+    });
     console.warn(
       usesDefaultGenerator
         ? `[llm:error] task=${task} message=${quoteLogValue(message)}`