fix: surface llm workflow errors

This commit is contained in:
Codex
2026-06-21 23:43:03 +08:00
parent 4bb5ca5eb0
commit f1dac15400
12 changed files with 286 additions and 500 deletions
+42 -42
View File
@@ -16,16 +16,16 @@ describe("generateValidatedJson", () => {
vi.restoreAllMocks();
});
it("returns null when no provider key is configured", async () => {
it("throws clearly when no provider key is configured", async () => {
process.env.LLM_PROVIDER = "deepseek";
delete process.env.DEEPSEEK_API_KEY;
const result = await client.generateValidatedJson({
schema: z.object({ value: z.string() }),
prompt: "Return JSON.",
});
expect(result).toBeNull();
await expect(
client.generateValidatedJson({
schema: z.object({ value: z.string() }),
prompt: "Return JSON.",
}),
).rejects.toThrow("DEEPSEEK_API_KEY is missing");
});
it("returns parsed data when the model response matches the schema", async () => {
@@ -41,32 +41,32 @@ describe("generateValidatedJson", () => {
expect(result).toEqual({ value: "from-llm" });
});
it("returns null when the model response fails schema validation", async () => {
it("throws clearly when the model response fails schema validation", async () => {
process.env.LLM_PROVIDER = "deepseek";
process.env.DEEPSEEK_API_KEY = "test-key";
client.setGenerateJsonForValidation(async () => ({ value: 42 }));
const result = await client.generateValidatedJson({
schema: z.object({ value: z.string() }),
prompt: "Return JSON.",
});
expect(result).toBeNull();
await expect(
client.generateValidatedJson({
schema: z.object({ value: z.string() }),
prompt: "Return JSON.",
}),
).rejects.toThrow("LLM response failed schema validation");
});
it("returns null when the provider call rejects", async () => {
it("throws clearly when the provider call rejects", async () => {
process.env.LLM_PROVIDER = "deepseek";
process.env.DEEPSEEK_API_KEY = "test-key";
client.setGenerateJsonForValidation(async () => {
throw new Error("provider down");
});
const result = await client.generateValidatedJson({
schema: z.object({ value: z.string() }),
prompt: "Return JSON.",
});
expect(result).toBeNull();
await expect(
client.generateValidatedJson({
schema: z.object({ value: z.string() }),
prompt: "Return JSON.",
}),
).rejects.toThrow("provider down");
});
it("logs validation success with the supplied task label", async () => {
@@ -93,13 +93,13 @@ describe("generateValidatedJson", () => {
process.env.DEEPSEEK_API_KEY = "test-key";
client.setGenerateJsonForValidation(async () => ({ value: 42 }));
const result = await client.generateValidatedJson({
schema: z.object({ value: z.string() }),
prompt: "Return JSON.",
task: "fact_extractor",
});
expect(result).toBeNull();
await expect(
client.generateValidatedJson({
schema: z.object({ value: z.string() }),
prompt: "Return JSON.",
task: "fact_extractor",
}),
).rejects.toThrow("LLM response failed schema validation");
expect(warnSpy).toHaveBeenCalledWith(
expect.stringContaining("[llm:validated] task=fact_extractor ok=false"),
);
@@ -150,16 +150,16 @@ describe("generateValidatedJson", () => {
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
client.setGenerateJsonForValidation(async () => ({ value: 42 }));
const result = await client.generateValidatedJson({
schema: z.object({ value: z.string() }),
task: "fact_extractor",
prompt: "Return JSON.",
});
await expect(
client.generateValidatedJson({
schema: z.object({ value: z.string() }),
task: "fact_extractor",
prompt: "Return JSON.",
}),
).rejects.toThrow("LLM response failed schema validation");
const allLogs = [...infoSpy.mock.calls, ...warnSpy.mock.calls]
.flat()
.join("\n");
expect(result).toBeNull();
expect(allLogs).toContain("[llm:validated] task=fact_extractor ok=false");
expect(allLogs).toContain("zod_error=");
expect(allLogs).not.toContain("super-secret-key");
@@ -173,13 +173,13 @@ describe("generateValidatedJson", () => {
throw new Error("provider unavailable");
});
const result = await client.generateValidatedJson({
schema: z.object({ value: z.string() }),
task: "quality_inspector",
prompt: "Return JSON.",
});
expect(result).toBeNull();
await expect(
client.generateValidatedJson({
schema: z.object({ value: z.string() }),
task: "quality_inspector",
prompt: "Return JSON.",
}),
).rejects.toThrow("provider unavailable");
expect(warnSpy).toHaveBeenCalledWith(
expect.stringContaining("[llm:error] task=quality_inspector"),
);