定义LLM追踪事件契约

This commit is contained in:
czj
2026-07-16 11:58:11 +08:00
parent 001f0402b2
commit a22844d2dd
5 changed files with 213 additions and 9 deletions
+2 -1
View File
@@ -1,4 +1,5 @@
import type { LlmProviderStatus, LlmTaskName } from "./client";
import type { LlmProviderStatus } from "./client";
import type { LlmTaskName } from "./trace-types";
export interface LlmAuditSummary {
provider: LlmProviderStatus["provider"];
+2 -7
View File
@@ -2,14 +2,9 @@ import OpenAI from "openai";
import type { z } from "zod";
import { createLlmAuditSummary, type LlmAuditSummary } from "./audit";
import type { LlmTaskName } from "./trace-types";
export type LlmTaskName =
| "unknown"
| "fact_extractor"
| "article_optimizer"
| "quality_inspector"
| "targeted_rewriter"
| "renwei_copy_optimizer";
export type { LlmTaskName } from "./trace-types";
export interface GenerateInput {
system?: string;
+181
View File
@@ -0,0 +1,181 @@
export type LlmProviderName = "deepseek" | "openai";
export type LlmTaskName =
| "unknown"
| "fact_extractor"
| "article_optimizer"
| "quality_inspector"
| "targeted_rewriter"
| "renwei_copy_optimizer";
export type LlmTraceWorkflowStage =
| "unknown"
| "input"
| "fact_card"
| "draft"
| "qa"
| "rewrite"
| "final";
export type LlmTraceRunStatus =
| "running"
| "completed"
| "failed"
| "interrupted";
export type LlmTraceCallStatus =
| "started"
| "responded"
| "validated"
| "failed";
export type LlmTraceCompleteness = "complete" | "incomplete";
export type LlmBusinessStatus = "pass" | "warn" | "fail";
export type LlmTraceErrorType =
| "provider"
| "json_parse"
| "schema_validation";
export interface LlmTraceContext {
workflow_stage: LlmTraceWorkflowStage;
rewrite_round?: number;
schema_name?: string;
}
export interface LlmTraceRun {
job_id: string;
case_id: string | null;
status: LlmTraceRunStatus;
current_stage: LlmTraceWorkflowStage;
trace_completeness: LlmTraceCompleteness;
error_stage: string | null;
error_summary: string | null;
started_at: string;
finished_at: string | null;
updated_at: string;
}
export interface LlmTraceCall {
call_id: string;
job_id: string;
sequence: number;
task: LlmTaskName;
workflow_stage: LlmTraceWorkflowStage;
rewrite_round: number | null;
provider: LlmProviderName;
model: string;
status: LlmTraceCallStatus;
request_object_key: string | null;
response_object_key: string | null;
token_usage: Record<string, number> | null;
schema_name: string | null;
schema_valid: boolean | null;
validation_issues: string[];
business_status: LlmBusinessStatus | null;
duration_ms: number | null;
started_at: string;
responded_at: string | null;
validated_at: string | null;
failed_at: string | null;
error_type: LlmTraceErrorType | null;
error_summary: string | null;
}
export type LlmTraceCallPublic = Omit<
LlmTraceCall,
"request_object_key" | "response_object_key"
> & {
request_available: boolean;
response_available: boolean;
};
export interface LlmTraceManifest {
run: LlmTraceRun;
calls: LlmTraceCallPublic[];
}
export type LlmClientTraceEvent =
| {
type: "started";
call_id: string;
task: LlmTaskName;
context: LlmTraceContext;
provider: LlmProviderName;
model: string;
request: unknown;
started_at: string;
}
| {
type: "responded";
call_id: string;
response: unknown;
duration_ms: number;
responded_at: string;
}
| {
type: "validated";
call_id: string;
schema_name: string;
schema_valid: boolean;
validation_issues: string[];
validated_at: string;
}
| {
type: "failed";
call_id: string;
error_type: LlmTraceErrorType;
error_summary: string;
duration_ms: number;
failed_at: string;
};
export type LlmClientTraceHandler = (
event: LlmClientTraceEvent,
) => void | Promise<void>;
export type LlmTraceStreamEvent =
| {
type: "llm_call_started";
job_id: string;
call_id: string;
sequence: number;
task: LlmTaskName;
workflow_stage: LlmTraceWorkflowStage;
rewrite_round: number | null;
provider: LlmProviderName;
model: string;
started_at: string;
request_available: boolean;
}
| {
type: "llm_call_responded";
job_id: string;
call_id: string;
duration_ms: number;
token_usage: Record<string, number> | null;
responded_at: string;
response_available: boolean;
}
| {
type: "llm_call_validated";
job_id: string;
call_id: string;
schema_name: string;
schema_valid: boolean;
validation_issues: string[];
validated_at: string;
}
| {
type: "llm_call_failed";
job_id: string;
call_id: string;
error_type: LlmTraceErrorType;
error_summary: string;
failed_at: string;
}
| {
type: "trace_warning";
job_id: string;
trace_completeness: "incomplete";
error_summary: string;
};
@@ -19,6 +19,28 @@ describe("optimization stream events", () => {
);
});
it("encodes LLM call metadata without raw request or response bodies", () => {
const event: OptimizationStreamEvent = {
type: "llm_call_started",
job_id: "job_123",
call_id: "llmcall_1",
sequence: 1,
task: "fact_extractor",
workflow_stage: "fact_card",
rewrite_round: null,
provider: "deepseek",
model: "deepseek-v4-pro",
started_at: "2026-07-16T00:00:00.000Z",
request_available: true,
};
const encoded = encodeOptimizationStreamEvent(event);
expect(JSON.parse(encoded)).toEqual(event);
expect(encoded).not.toContain("messages");
expect(encoded).not.toContain("Authorization");
});
it("parses chunked NDJSON while preserving incomplete lines", () => {
const first = parseOptimizationStreamChunk(
"",
+6 -1
View File
@@ -3,6 +3,7 @@ import type {
OptimizedArticle,
QaReport,
} from "../domain/types";
import type { LlmTraceStreamEvent } from "../llm/trace-types";
export type OptimizationStreamStage =
| "input"
@@ -13,7 +14,7 @@ export type OptimizationStreamStage =
| "rewrite"
| "final";
export type OptimizationStreamEvent =
type ExistingOptimizationStreamEvents =
| {
type: "job_created";
job: { id: string };
@@ -53,6 +54,10 @@ export type OptimizationStreamEvent =
error: string;
};
export type OptimizationStreamEvent =
| LlmTraceStreamEvent
| ExistingOptimizationStreamEvents;
export function encodeOptimizationStreamEvent(
event: OptimizationStreamEvent,
) {