feat: log llm runtime responses
This commit is contained in:
@@ -10,6 +10,8 @@ describe("generateValidatedJson", () => {
|
||||
afterEach(() => {
|
||||
process.env.LLM_PROVIDER = originalProvider;
|
||||
process.env.DEEPSEEK_API_KEY = originalDeepSeekKey;
|
||||
delete process.env.LLM_LOG_RAW_LIMIT;
|
||||
delete process.env.DEEPSEEK_MODEL;
|
||||
client.setGenerateJsonForValidation(client.generateJson);
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
@@ -103,6 +105,88 @@ describe("generateValidatedJson", () => {
|
||||
);
|
||||
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining("value"));
|
||||
});
|
||||
|
||||
it("logs provider, model, task, raw response snippet, and validation success", async () => {
|
||||
process.env.LLM_PROVIDER = "deepseek";
|
||||
process.env.DEEPSEEK_API_KEY = "test-key";
|
||||
process.env.DEEPSEEK_MODEL = "deepseek-test";
|
||||
process.env.LLM_LOG_RAW_LIMIT = "80";
|
||||
const infoSpy = vi.spyOn(console, "info").mockImplementation(() => {});
|
||||
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
client.setGenerateJsonForValidation(async () => ({
|
||||
value: "from-llm",
|
||||
longField: "x".repeat(200),
|
||||
}));
|
||||
|
||||
const result = await client.generateValidatedJson({
|
||||
schema: z.object({ value: z.string(), longField: z.string() }),
|
||||
task: "article_optimizer",
|
||||
prompt: "Return JSON.",
|
||||
});
|
||||
|
||||
expect(result?.value).toBe("from-llm");
|
||||
expect(infoSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining(
|
||||
"[llm:start] provider=deepseek model=deepseek-test task=article_optimizer",
|
||||
),
|
||||
);
|
||||
expect(infoSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining("[llm:response] task=article_optimizer"),
|
||||
);
|
||||
expect(infoSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining('"value":"from-llm"'),
|
||||
);
|
||||
expect(infoSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining("[llm:validated] task=article_optimizer ok=true"),
|
||||
);
|
||||
expect(warnSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("logs validation failure without leaking provider secrets", async () => {
|
||||
process.env.LLM_PROVIDER = "deepseek";
|
||||
process.env.DEEPSEEK_API_KEY = "super-secret-key";
|
||||
process.env.DEEPSEEK_MODEL = "deepseek-test";
|
||||
const infoSpy = vi.spyOn(console, "info").mockImplementation(() => {});
|
||||
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
client.setGenerateJsonForValidation(async () => ({ value: 42 }));
|
||||
|
||||
const result = await client.generateValidatedJson({
|
||||
schema: z.object({ value: z.string() }),
|
||||
task: "fact_extractor",
|
||||
prompt: "Return JSON.",
|
||||
});
|
||||
|
||||
const allLogs = [...infoSpy.mock.calls, ...warnSpy.mock.calls]
|
||||
.flat()
|
||||
.join("\n");
|
||||
expect(result).toBeNull();
|
||||
expect(allLogs).toContain("[llm:validated] task=fact_extractor ok=false");
|
||||
expect(allLogs).toContain("zod_error=");
|
||||
expect(allLogs).not.toContain("super-secret-key");
|
||||
});
|
||||
|
||||
it("logs provider call failure for validated JSON calls", async () => {
|
||||
process.env.LLM_PROVIDER = "deepseek";
|
||||
process.env.DEEPSEEK_API_KEY = "test-key";
|
||||
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
client.setGenerateJsonForValidation(async () => {
|
||||
throw new Error("provider unavailable");
|
||||
});
|
||||
|
||||
const result = await client.generateValidatedJson({
|
||||
schema: z.object({ value: z.string() }),
|
||||
task: "quality_inspector",
|
||||
prompt: "Return JSON.",
|
||||
});
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining("[llm:error] task=quality_inspector"),
|
||||
);
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining("provider unavailable"),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("generateJson logging", () => {
|
||||
|
||||
Reference in New Issue
Block a user