From a22844d2dd16e8fb9fb2fc7e7f0d9ad6310ac8d0 Mon Sep 17 00:00:00 2001 From: czj <13261895355@163.com> Date: Thu, 16 Jul 2026 11:58:11 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=9A=E4=B9=89LLM=E8=BF=BD=E8=B8=AA?= =?UTF-8?q?=E4=BA=8B=E4=BB=B6=E5=A5=91=E7=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/lib/llm/audit.ts | 3 +- src/lib/llm/client.ts | 9 +- src/lib/llm/trace-types.ts | 181 ++++++++++++++++++ .../workflow/__tests__/stream-events.test.ts | 22 +++ src/lib/workflow/stream-events.ts | 7 +- 5 files changed, 213 insertions(+), 9 deletions(-) create mode 100644 src/lib/llm/trace-types.ts diff --git a/src/lib/llm/audit.ts b/src/lib/llm/audit.ts index e0c7e7f..96cd04b 100644 --- a/src/lib/llm/audit.ts +++ b/src/lib/llm/audit.ts @@ -1,4 +1,5 @@ -import type { LlmProviderStatus, LlmTaskName } from "./client"; +import type { LlmProviderStatus } from "./client"; +import type { LlmTaskName } from "./trace-types"; export interface LlmAuditSummary { provider: LlmProviderStatus["provider"]; diff --git a/src/lib/llm/client.ts b/src/lib/llm/client.ts index 3195400..54a8499 100644 --- a/src/lib/llm/client.ts +++ b/src/lib/llm/client.ts @@ -2,14 +2,9 @@ import OpenAI from "openai"; import type { z } from "zod"; import { createLlmAuditSummary, type LlmAuditSummary } from "./audit"; +import type { LlmTaskName } from "./trace-types"; -export type LlmTaskName = - | "unknown" - | "fact_extractor" - | "article_optimizer" - | "quality_inspector" - | "targeted_rewriter" - | "renwei_copy_optimizer"; +export type { LlmTaskName } from "./trace-types"; export interface GenerateInput { system?: string; diff --git a/src/lib/llm/trace-types.ts b/src/lib/llm/trace-types.ts new file mode 100644 index 0000000..2781e22 --- /dev/null +++ b/src/lib/llm/trace-types.ts @@ -0,0 +1,181 @@ +export type LlmProviderName = "deepseek" | "openai"; + +export type LlmTaskName = + | "unknown" + | "fact_extractor" + | "article_optimizer" + | "quality_inspector" + | "targeted_rewriter" + | "renwei_copy_optimizer"; + +export type LlmTraceWorkflowStage = + | "unknown" + | "input" + | "fact_card" + | "draft" + | "qa" + | "rewrite" + | "final"; + +export type LlmTraceRunStatus = + | "running" + | "completed" + | "failed" + | "interrupted"; + +export type LlmTraceCallStatus = + | "started" + | "responded" + | "validated" + | "failed"; + +export type LlmTraceCompleteness = "complete" | "incomplete"; +export type LlmBusinessStatus = "pass" | "warn" | "fail"; +export type LlmTraceErrorType = + | "provider" + | "json_parse" + | "schema_validation"; + +export interface LlmTraceContext { + workflow_stage: LlmTraceWorkflowStage; + rewrite_round?: number; + schema_name?: string; +} + +export interface LlmTraceRun { + job_id: string; + case_id: string | null; + status: LlmTraceRunStatus; + current_stage: LlmTraceWorkflowStage; + trace_completeness: LlmTraceCompleteness; + error_stage: string | null; + error_summary: string | null; + started_at: string; + finished_at: string | null; + updated_at: string; +} + +export interface LlmTraceCall { + call_id: string; + job_id: string; + sequence: number; + task: LlmTaskName; + workflow_stage: LlmTraceWorkflowStage; + rewrite_round: number | null; + provider: LlmProviderName; + model: string; + status: LlmTraceCallStatus; + request_object_key: string | null; + response_object_key: string | null; + token_usage: Record | null; + schema_name: string | null; + schema_valid: boolean | null; + validation_issues: string[]; + business_status: LlmBusinessStatus | null; + duration_ms: number | null; + started_at: string; + responded_at: string | null; + validated_at: string | null; + failed_at: string | null; + error_type: LlmTraceErrorType | null; + error_summary: string | null; +} + +export type LlmTraceCallPublic = Omit< + LlmTraceCall, + "request_object_key" | "response_object_key" +> & { + request_available: boolean; + response_available: boolean; +}; + +export interface LlmTraceManifest { + run: LlmTraceRun; + calls: LlmTraceCallPublic[]; +} + +export type LlmClientTraceEvent = + | { + type: "started"; + call_id: string; + task: LlmTaskName; + context: LlmTraceContext; + provider: LlmProviderName; + model: string; + request: unknown; + started_at: string; + } + | { + type: "responded"; + call_id: string; + response: unknown; + duration_ms: number; + responded_at: string; + } + | { + type: "validated"; + call_id: string; + schema_name: string; + schema_valid: boolean; + validation_issues: string[]; + validated_at: string; + } + | { + type: "failed"; + call_id: string; + error_type: LlmTraceErrorType; + error_summary: string; + duration_ms: number; + failed_at: string; + }; + +export type LlmClientTraceHandler = ( + event: LlmClientTraceEvent, +) => void | Promise; + +export type LlmTraceStreamEvent = + | { + type: "llm_call_started"; + job_id: string; + call_id: string; + sequence: number; + task: LlmTaskName; + workflow_stage: LlmTraceWorkflowStage; + rewrite_round: number | null; + provider: LlmProviderName; + model: string; + started_at: string; + request_available: boolean; + } + | { + type: "llm_call_responded"; + job_id: string; + call_id: string; + duration_ms: number; + token_usage: Record | null; + responded_at: string; + response_available: boolean; + } + | { + type: "llm_call_validated"; + job_id: string; + call_id: string; + schema_name: string; + schema_valid: boolean; + validation_issues: string[]; + validated_at: string; + } + | { + type: "llm_call_failed"; + job_id: string; + call_id: string; + error_type: LlmTraceErrorType; + error_summary: string; + failed_at: string; + } + | { + type: "trace_warning"; + job_id: string; + trace_completeness: "incomplete"; + error_summary: string; + }; diff --git a/src/lib/workflow/__tests__/stream-events.test.ts b/src/lib/workflow/__tests__/stream-events.test.ts index 4074ddd..7b43f38 100644 --- a/src/lib/workflow/__tests__/stream-events.test.ts +++ b/src/lib/workflow/__tests__/stream-events.test.ts @@ -19,6 +19,28 @@ describe("optimization stream events", () => { ); }); + it("encodes LLM call metadata without raw request or response bodies", () => { + const event: OptimizationStreamEvent = { + type: "llm_call_started", + job_id: "job_123", + call_id: "llmcall_1", + sequence: 1, + task: "fact_extractor", + workflow_stage: "fact_card", + rewrite_round: null, + provider: "deepseek", + model: "deepseek-v4-pro", + started_at: "2026-07-16T00:00:00.000Z", + request_available: true, + }; + + const encoded = encodeOptimizationStreamEvent(event); + + expect(JSON.parse(encoded)).toEqual(event); + expect(encoded).not.toContain("messages"); + expect(encoded).not.toContain("Authorization"); + }); + it("parses chunked NDJSON while preserving incomplete lines", () => { const first = parseOptimizationStreamChunk( "", diff --git a/src/lib/workflow/stream-events.ts b/src/lib/workflow/stream-events.ts index 63ab59c..5527062 100644 --- a/src/lib/workflow/stream-events.ts +++ b/src/lib/workflow/stream-events.ts @@ -3,6 +3,7 @@ import type { OptimizedArticle, QaReport, } from "../domain/types"; +import type { LlmTraceStreamEvent } from "../llm/trace-types"; export type OptimizationStreamStage = | "input" @@ -13,7 +14,7 @@ export type OptimizationStreamStage = | "rewrite" | "final"; -export type OptimizationStreamEvent = +type ExistingOptimizationStreamEvents = | { type: "job_created"; job: { id: string }; @@ -53,6 +54,10 @@ export type OptimizationStreamEvent = error: string; }; +export type OptimizationStreamEvent = + | LlmTraceStreamEvent + | ExistingOptimizationStreamEvents; + export function encodeOptimizationStreamEvent( event: OptimizationStreamEvent, ) {