定义LLM追踪事件契约
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
import type { LlmProviderStatus, LlmTaskName } from "./client";
|
||||
import type { LlmProviderStatus } from "./client";
|
||||
import type { LlmTaskName } from "./trace-types";
|
||||
|
||||
export interface LlmAuditSummary {
|
||||
provider: LlmProviderStatus["provider"];
|
||||
|
||||
@@ -2,14 +2,9 @@ import OpenAI from "openai";
|
||||
import type { z } from "zod";
|
||||
|
||||
import { createLlmAuditSummary, type LlmAuditSummary } from "./audit";
|
||||
import type { LlmTaskName } from "./trace-types";
|
||||
|
||||
export type LlmTaskName =
|
||||
| "unknown"
|
||||
| "fact_extractor"
|
||||
| "article_optimizer"
|
||||
| "quality_inspector"
|
||||
| "targeted_rewriter"
|
||||
| "renwei_copy_optimizer";
|
||||
export type { LlmTaskName } from "./trace-types";
|
||||
|
||||
export interface GenerateInput {
|
||||
system?: string;
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
export type LlmProviderName = "deepseek" | "openai";
|
||||
|
||||
export type LlmTaskName =
|
||||
| "unknown"
|
||||
| "fact_extractor"
|
||||
| "article_optimizer"
|
||||
| "quality_inspector"
|
||||
| "targeted_rewriter"
|
||||
| "renwei_copy_optimizer";
|
||||
|
||||
export type LlmTraceWorkflowStage =
|
||||
| "unknown"
|
||||
| "input"
|
||||
| "fact_card"
|
||||
| "draft"
|
||||
| "qa"
|
||||
| "rewrite"
|
||||
| "final";
|
||||
|
||||
export type LlmTraceRunStatus =
|
||||
| "running"
|
||||
| "completed"
|
||||
| "failed"
|
||||
| "interrupted";
|
||||
|
||||
export type LlmTraceCallStatus =
|
||||
| "started"
|
||||
| "responded"
|
||||
| "validated"
|
||||
| "failed";
|
||||
|
||||
export type LlmTraceCompleteness = "complete" | "incomplete";
|
||||
export type LlmBusinessStatus = "pass" | "warn" | "fail";
|
||||
export type LlmTraceErrorType =
|
||||
| "provider"
|
||||
| "json_parse"
|
||||
| "schema_validation";
|
||||
|
||||
export interface LlmTraceContext {
|
||||
workflow_stage: LlmTraceWorkflowStage;
|
||||
rewrite_round?: number;
|
||||
schema_name?: string;
|
||||
}
|
||||
|
||||
export interface LlmTraceRun {
|
||||
job_id: string;
|
||||
case_id: string | null;
|
||||
status: LlmTraceRunStatus;
|
||||
current_stage: LlmTraceWorkflowStage;
|
||||
trace_completeness: LlmTraceCompleteness;
|
||||
error_stage: string | null;
|
||||
error_summary: string | null;
|
||||
started_at: string;
|
||||
finished_at: string | null;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
export interface LlmTraceCall {
|
||||
call_id: string;
|
||||
job_id: string;
|
||||
sequence: number;
|
||||
task: LlmTaskName;
|
||||
workflow_stage: LlmTraceWorkflowStage;
|
||||
rewrite_round: number | null;
|
||||
provider: LlmProviderName;
|
||||
model: string;
|
||||
status: LlmTraceCallStatus;
|
||||
request_object_key: string | null;
|
||||
response_object_key: string | null;
|
||||
token_usage: Record<string, number> | null;
|
||||
schema_name: string | null;
|
||||
schema_valid: boolean | null;
|
||||
validation_issues: string[];
|
||||
business_status: LlmBusinessStatus | null;
|
||||
duration_ms: number | null;
|
||||
started_at: string;
|
||||
responded_at: string | null;
|
||||
validated_at: string | null;
|
||||
failed_at: string | null;
|
||||
error_type: LlmTraceErrorType | null;
|
||||
error_summary: string | null;
|
||||
}
|
||||
|
||||
export type LlmTraceCallPublic = Omit<
|
||||
LlmTraceCall,
|
||||
"request_object_key" | "response_object_key"
|
||||
> & {
|
||||
request_available: boolean;
|
||||
response_available: boolean;
|
||||
};
|
||||
|
||||
export interface LlmTraceManifest {
|
||||
run: LlmTraceRun;
|
||||
calls: LlmTraceCallPublic[];
|
||||
}
|
||||
|
||||
export type LlmClientTraceEvent =
|
||||
| {
|
||||
type: "started";
|
||||
call_id: string;
|
||||
task: LlmTaskName;
|
||||
context: LlmTraceContext;
|
||||
provider: LlmProviderName;
|
||||
model: string;
|
||||
request: unknown;
|
||||
started_at: string;
|
||||
}
|
||||
| {
|
||||
type: "responded";
|
||||
call_id: string;
|
||||
response: unknown;
|
||||
duration_ms: number;
|
||||
responded_at: string;
|
||||
}
|
||||
| {
|
||||
type: "validated";
|
||||
call_id: string;
|
||||
schema_name: string;
|
||||
schema_valid: boolean;
|
||||
validation_issues: string[];
|
||||
validated_at: string;
|
||||
}
|
||||
| {
|
||||
type: "failed";
|
||||
call_id: string;
|
||||
error_type: LlmTraceErrorType;
|
||||
error_summary: string;
|
||||
duration_ms: number;
|
||||
failed_at: string;
|
||||
};
|
||||
|
||||
export type LlmClientTraceHandler = (
|
||||
event: LlmClientTraceEvent,
|
||||
) => void | Promise<void>;
|
||||
|
||||
export type LlmTraceStreamEvent =
|
||||
| {
|
||||
type: "llm_call_started";
|
||||
job_id: string;
|
||||
call_id: string;
|
||||
sequence: number;
|
||||
task: LlmTaskName;
|
||||
workflow_stage: LlmTraceWorkflowStage;
|
||||
rewrite_round: number | null;
|
||||
provider: LlmProviderName;
|
||||
model: string;
|
||||
started_at: string;
|
||||
request_available: boolean;
|
||||
}
|
||||
| {
|
||||
type: "llm_call_responded";
|
||||
job_id: string;
|
||||
call_id: string;
|
||||
duration_ms: number;
|
||||
token_usage: Record<string, number> | null;
|
||||
responded_at: string;
|
||||
response_available: boolean;
|
||||
}
|
||||
| {
|
||||
type: "llm_call_validated";
|
||||
job_id: string;
|
||||
call_id: string;
|
||||
schema_name: string;
|
||||
schema_valid: boolean;
|
||||
validation_issues: string[];
|
||||
validated_at: string;
|
||||
}
|
||||
| {
|
||||
type: "llm_call_failed";
|
||||
job_id: string;
|
||||
call_id: string;
|
||||
error_type: LlmTraceErrorType;
|
||||
error_summary: string;
|
||||
failed_at: string;
|
||||
}
|
||||
| {
|
||||
type: "trace_warning";
|
||||
job_id: string;
|
||||
trace_completeness: "incomplete";
|
||||
error_summary: string;
|
||||
};
|
||||
@@ -19,6 +19,28 @@ describe("optimization stream events", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("encodes LLM call metadata without raw request or response bodies", () => {
|
||||
const event: OptimizationStreamEvent = {
|
||||
type: "llm_call_started",
|
||||
job_id: "job_123",
|
||||
call_id: "llmcall_1",
|
||||
sequence: 1,
|
||||
task: "fact_extractor",
|
||||
workflow_stage: "fact_card",
|
||||
rewrite_round: null,
|
||||
provider: "deepseek",
|
||||
model: "deepseek-v4-pro",
|
||||
started_at: "2026-07-16T00:00:00.000Z",
|
||||
request_available: true,
|
||||
};
|
||||
|
||||
const encoded = encodeOptimizationStreamEvent(event);
|
||||
|
||||
expect(JSON.parse(encoded)).toEqual(event);
|
||||
expect(encoded).not.toContain("messages");
|
||||
expect(encoded).not.toContain("Authorization");
|
||||
});
|
||||
|
||||
it("parses chunked NDJSON while preserving incomplete lines", () => {
|
||||
const first = parseOptimizationStreamChunk(
|
||||
"",
|
||||
|
||||
@@ -3,6 +3,7 @@ import type {
|
||||
OptimizedArticle,
|
||||
QaReport,
|
||||
} from "../domain/types";
|
||||
import type { LlmTraceStreamEvent } from "../llm/trace-types";
|
||||
|
||||
export type OptimizationStreamStage =
|
||||
| "input"
|
||||
@@ -13,7 +14,7 @@ export type OptimizationStreamStage =
|
||||
| "rewrite"
|
||||
| "final";
|
||||
|
||||
export type OptimizationStreamEvent =
|
||||
type ExistingOptimizationStreamEvents =
|
||||
| {
|
||||
type: "job_created";
|
||||
job: { id: string };
|
||||
@@ -53,6 +54,10 @@ export type OptimizationStreamEvent =
|
||||
error: string;
|
||||
};
|
||||
|
||||
export type OptimizationStreamEvent =
|
||||
| LlmTraceStreamEvent
|
||||
| ExistingOptimizationStreamEvents;
|
||||
|
||||
export function encodeOptimizationStreamEvent(
|
||||
event: OptimizationStreamEvent,
|
||||
) {
|
||||
|
||||
Reference in New Issue
Block a user