定义LLM追踪事件契约
This commit is contained in:
@@ -1,4 +1,5 @@
|
|||||||
import type { LlmProviderStatus, LlmTaskName } from "./client";
|
import type { LlmProviderStatus } from "./client";
|
||||||
|
import type { LlmTaskName } from "./trace-types";
|
||||||
|
|
||||||
export interface LlmAuditSummary {
|
export interface LlmAuditSummary {
|
||||||
provider: LlmProviderStatus["provider"];
|
provider: LlmProviderStatus["provider"];
|
||||||
|
|||||||
@@ -2,14 +2,9 @@ import OpenAI from "openai";
|
|||||||
import type { z } from "zod";
|
import type { z } from "zod";
|
||||||
|
|
||||||
import { createLlmAuditSummary, type LlmAuditSummary } from "./audit";
|
import { createLlmAuditSummary, type LlmAuditSummary } from "./audit";
|
||||||
|
import type { LlmTaskName } from "./trace-types";
|
||||||
|
|
||||||
export type LlmTaskName =
|
export type { LlmTaskName } from "./trace-types";
|
||||||
| "unknown"
|
|
||||||
| "fact_extractor"
|
|
||||||
| "article_optimizer"
|
|
||||||
| "quality_inspector"
|
|
||||||
| "targeted_rewriter"
|
|
||||||
| "renwei_copy_optimizer";
|
|
||||||
|
|
||||||
export interface GenerateInput {
|
export interface GenerateInput {
|
||||||
system?: string;
|
system?: string;
|
||||||
|
|||||||
@@ -0,0 +1,181 @@
|
|||||||
|
export type LlmProviderName = "deepseek" | "openai";
|
||||||
|
|
||||||
|
export type LlmTaskName =
|
||||||
|
| "unknown"
|
||||||
|
| "fact_extractor"
|
||||||
|
| "article_optimizer"
|
||||||
|
| "quality_inspector"
|
||||||
|
| "targeted_rewriter"
|
||||||
|
| "renwei_copy_optimizer";
|
||||||
|
|
||||||
|
export type LlmTraceWorkflowStage =
|
||||||
|
| "unknown"
|
||||||
|
| "input"
|
||||||
|
| "fact_card"
|
||||||
|
| "draft"
|
||||||
|
| "qa"
|
||||||
|
| "rewrite"
|
||||||
|
| "final";
|
||||||
|
|
||||||
|
export type LlmTraceRunStatus =
|
||||||
|
| "running"
|
||||||
|
| "completed"
|
||||||
|
| "failed"
|
||||||
|
| "interrupted";
|
||||||
|
|
||||||
|
export type LlmTraceCallStatus =
|
||||||
|
| "started"
|
||||||
|
| "responded"
|
||||||
|
| "validated"
|
||||||
|
| "failed";
|
||||||
|
|
||||||
|
export type LlmTraceCompleteness = "complete" | "incomplete";
|
||||||
|
export type LlmBusinessStatus = "pass" | "warn" | "fail";
|
||||||
|
export type LlmTraceErrorType =
|
||||||
|
| "provider"
|
||||||
|
| "json_parse"
|
||||||
|
| "schema_validation";
|
||||||
|
|
||||||
|
export interface LlmTraceContext {
|
||||||
|
workflow_stage: LlmTraceWorkflowStage;
|
||||||
|
rewrite_round?: number;
|
||||||
|
schema_name?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface LlmTraceRun {
|
||||||
|
job_id: string;
|
||||||
|
case_id: string | null;
|
||||||
|
status: LlmTraceRunStatus;
|
||||||
|
current_stage: LlmTraceWorkflowStage;
|
||||||
|
trace_completeness: LlmTraceCompleteness;
|
||||||
|
error_stage: string | null;
|
||||||
|
error_summary: string | null;
|
||||||
|
started_at: string;
|
||||||
|
finished_at: string | null;
|
||||||
|
updated_at: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface LlmTraceCall {
|
||||||
|
call_id: string;
|
||||||
|
job_id: string;
|
||||||
|
sequence: number;
|
||||||
|
task: LlmTaskName;
|
||||||
|
workflow_stage: LlmTraceWorkflowStage;
|
||||||
|
rewrite_round: number | null;
|
||||||
|
provider: LlmProviderName;
|
||||||
|
model: string;
|
||||||
|
status: LlmTraceCallStatus;
|
||||||
|
request_object_key: string | null;
|
||||||
|
response_object_key: string | null;
|
||||||
|
token_usage: Record<string, number> | null;
|
||||||
|
schema_name: string | null;
|
||||||
|
schema_valid: boolean | null;
|
||||||
|
validation_issues: string[];
|
||||||
|
business_status: LlmBusinessStatus | null;
|
||||||
|
duration_ms: number | null;
|
||||||
|
started_at: string;
|
||||||
|
responded_at: string | null;
|
||||||
|
validated_at: string | null;
|
||||||
|
failed_at: string | null;
|
||||||
|
error_type: LlmTraceErrorType | null;
|
||||||
|
error_summary: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type LlmTraceCallPublic = Omit<
|
||||||
|
LlmTraceCall,
|
||||||
|
"request_object_key" | "response_object_key"
|
||||||
|
> & {
|
||||||
|
request_available: boolean;
|
||||||
|
response_available: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
export interface LlmTraceManifest {
|
||||||
|
run: LlmTraceRun;
|
||||||
|
calls: LlmTraceCallPublic[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export type LlmClientTraceEvent =
|
||||||
|
| {
|
||||||
|
type: "started";
|
||||||
|
call_id: string;
|
||||||
|
task: LlmTaskName;
|
||||||
|
context: LlmTraceContext;
|
||||||
|
provider: LlmProviderName;
|
||||||
|
model: string;
|
||||||
|
request: unknown;
|
||||||
|
started_at: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
type: "responded";
|
||||||
|
call_id: string;
|
||||||
|
response: unknown;
|
||||||
|
duration_ms: number;
|
||||||
|
responded_at: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
type: "validated";
|
||||||
|
call_id: string;
|
||||||
|
schema_name: string;
|
||||||
|
schema_valid: boolean;
|
||||||
|
validation_issues: string[];
|
||||||
|
validated_at: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
type: "failed";
|
||||||
|
call_id: string;
|
||||||
|
error_type: LlmTraceErrorType;
|
||||||
|
error_summary: string;
|
||||||
|
duration_ms: number;
|
||||||
|
failed_at: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type LlmClientTraceHandler = (
|
||||||
|
event: LlmClientTraceEvent,
|
||||||
|
) => void | Promise<void>;
|
||||||
|
|
||||||
|
export type LlmTraceStreamEvent =
|
||||||
|
| {
|
||||||
|
type: "llm_call_started";
|
||||||
|
job_id: string;
|
||||||
|
call_id: string;
|
||||||
|
sequence: number;
|
||||||
|
task: LlmTaskName;
|
||||||
|
workflow_stage: LlmTraceWorkflowStage;
|
||||||
|
rewrite_round: number | null;
|
||||||
|
provider: LlmProviderName;
|
||||||
|
model: string;
|
||||||
|
started_at: string;
|
||||||
|
request_available: boolean;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
type: "llm_call_responded";
|
||||||
|
job_id: string;
|
||||||
|
call_id: string;
|
||||||
|
duration_ms: number;
|
||||||
|
token_usage: Record<string, number> | null;
|
||||||
|
responded_at: string;
|
||||||
|
response_available: boolean;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
type: "llm_call_validated";
|
||||||
|
job_id: string;
|
||||||
|
call_id: string;
|
||||||
|
schema_name: string;
|
||||||
|
schema_valid: boolean;
|
||||||
|
validation_issues: string[];
|
||||||
|
validated_at: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
type: "llm_call_failed";
|
||||||
|
job_id: string;
|
||||||
|
call_id: string;
|
||||||
|
error_type: LlmTraceErrorType;
|
||||||
|
error_summary: string;
|
||||||
|
failed_at: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
type: "trace_warning";
|
||||||
|
job_id: string;
|
||||||
|
trace_completeness: "incomplete";
|
||||||
|
error_summary: string;
|
||||||
|
};
|
||||||
@@ -19,6 +19,28 @@ describe("optimization stream events", () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("encodes LLM call metadata without raw request or response bodies", () => {
|
||||||
|
const event: OptimizationStreamEvent = {
|
||||||
|
type: "llm_call_started",
|
||||||
|
job_id: "job_123",
|
||||||
|
call_id: "llmcall_1",
|
||||||
|
sequence: 1,
|
||||||
|
task: "fact_extractor",
|
||||||
|
workflow_stage: "fact_card",
|
||||||
|
rewrite_round: null,
|
||||||
|
provider: "deepseek",
|
||||||
|
model: "deepseek-v4-pro",
|
||||||
|
started_at: "2026-07-16T00:00:00.000Z",
|
||||||
|
request_available: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
const encoded = encodeOptimizationStreamEvent(event);
|
||||||
|
|
||||||
|
expect(JSON.parse(encoded)).toEqual(event);
|
||||||
|
expect(encoded).not.toContain("messages");
|
||||||
|
expect(encoded).not.toContain("Authorization");
|
||||||
|
});
|
||||||
|
|
||||||
it("parses chunked NDJSON while preserving incomplete lines", () => {
|
it("parses chunked NDJSON while preserving incomplete lines", () => {
|
||||||
const first = parseOptimizationStreamChunk(
|
const first = parseOptimizationStreamChunk(
|
||||||
"",
|
"",
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import type {
|
|||||||
OptimizedArticle,
|
OptimizedArticle,
|
||||||
QaReport,
|
QaReport,
|
||||||
} from "../domain/types";
|
} from "../domain/types";
|
||||||
|
import type { LlmTraceStreamEvent } from "../llm/trace-types";
|
||||||
|
|
||||||
export type OptimizationStreamStage =
|
export type OptimizationStreamStage =
|
||||||
| "input"
|
| "input"
|
||||||
@@ -13,7 +14,7 @@ export type OptimizationStreamStage =
|
|||||||
| "rewrite"
|
| "rewrite"
|
||||||
| "final";
|
| "final";
|
||||||
|
|
||||||
export type OptimizationStreamEvent =
|
type ExistingOptimizationStreamEvents =
|
||||||
| {
|
| {
|
||||||
type: "job_created";
|
type: "job_created";
|
||||||
job: { id: string };
|
job: { id: string };
|
||||||
@@ -53,6 +54,10 @@ export type OptimizationStreamEvent =
|
|||||||
error: string;
|
error: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type OptimizationStreamEvent =
|
||||||
|
| LlmTraceStreamEvent
|
||||||
|
| ExistingOptimizationStreamEvents;
|
||||||
|
|
||||||
export function encodeOptimizationStreamEvent(
|
export function encodeOptimizationStreamEvent(
|
||||||
event: OptimizationStreamEvent,
|
event: OptimizationStreamEvent,
|
||||||
) {
|
) {
|
||||||
|
|||||||
Reference in New Issue
Block a user