fix: surface llm workflow errors
This commit is contained in:
@@ -16,16 +16,16 @@ describe("generateValidatedJson", () => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("returns null when no provider key is configured", async () => {
|
||||
it("throws clearly when no provider key is configured", async () => {
|
||||
process.env.LLM_PROVIDER = "deepseek";
|
||||
delete process.env.DEEPSEEK_API_KEY;
|
||||
|
||||
const result = await client.generateValidatedJson({
|
||||
schema: z.object({ value: z.string() }),
|
||||
prompt: "Return JSON.",
|
||||
});
|
||||
|
||||
expect(result).toBeNull();
|
||||
await expect(
|
||||
client.generateValidatedJson({
|
||||
schema: z.object({ value: z.string() }),
|
||||
prompt: "Return JSON.",
|
||||
}),
|
||||
).rejects.toThrow("DEEPSEEK_API_KEY is missing");
|
||||
});
|
||||
|
||||
it("returns parsed data when the model response matches the schema", async () => {
|
||||
@@ -41,32 +41,32 @@ describe("generateValidatedJson", () => {
|
||||
expect(result).toEqual({ value: "from-llm" });
|
||||
});
|
||||
|
||||
it("returns null when the model response fails schema validation", async () => {
|
||||
it("throws clearly when the model response fails schema validation", async () => {
|
||||
process.env.LLM_PROVIDER = "deepseek";
|
||||
process.env.DEEPSEEK_API_KEY = "test-key";
|
||||
client.setGenerateJsonForValidation(async () => ({ value: 42 }));
|
||||
|
||||
const result = await client.generateValidatedJson({
|
||||
schema: z.object({ value: z.string() }),
|
||||
prompt: "Return JSON.",
|
||||
});
|
||||
|
||||
expect(result).toBeNull();
|
||||
await expect(
|
||||
client.generateValidatedJson({
|
||||
schema: z.object({ value: z.string() }),
|
||||
prompt: "Return JSON.",
|
||||
}),
|
||||
).rejects.toThrow("LLM response failed schema validation");
|
||||
});
|
||||
|
||||
it("returns null when the provider call rejects", async () => {
|
||||
it("throws clearly when the provider call rejects", async () => {
|
||||
process.env.LLM_PROVIDER = "deepseek";
|
||||
process.env.DEEPSEEK_API_KEY = "test-key";
|
||||
client.setGenerateJsonForValidation(async () => {
|
||||
throw new Error("provider down");
|
||||
});
|
||||
|
||||
const result = await client.generateValidatedJson({
|
||||
schema: z.object({ value: z.string() }),
|
||||
prompt: "Return JSON.",
|
||||
});
|
||||
|
||||
expect(result).toBeNull();
|
||||
await expect(
|
||||
client.generateValidatedJson({
|
||||
schema: z.object({ value: z.string() }),
|
||||
prompt: "Return JSON.",
|
||||
}),
|
||||
).rejects.toThrow("provider down");
|
||||
});
|
||||
|
||||
it("logs validation success with the supplied task label", async () => {
|
||||
@@ -93,13 +93,13 @@ describe("generateValidatedJson", () => {
|
||||
process.env.DEEPSEEK_API_KEY = "test-key";
|
||||
client.setGenerateJsonForValidation(async () => ({ value: 42 }));
|
||||
|
||||
const result = await client.generateValidatedJson({
|
||||
schema: z.object({ value: z.string() }),
|
||||
prompt: "Return JSON.",
|
||||
task: "fact_extractor",
|
||||
});
|
||||
|
||||
expect(result).toBeNull();
|
||||
await expect(
|
||||
client.generateValidatedJson({
|
||||
schema: z.object({ value: z.string() }),
|
||||
prompt: "Return JSON.",
|
||||
task: "fact_extractor",
|
||||
}),
|
||||
).rejects.toThrow("LLM response failed schema validation");
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining("[llm:validated] task=fact_extractor ok=false"),
|
||||
);
|
||||
@@ -150,16 +150,16 @@ describe("generateValidatedJson", () => {
|
||||
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
client.setGenerateJsonForValidation(async () => ({ value: 42 }));
|
||||
|
||||
const result = await client.generateValidatedJson({
|
||||
schema: z.object({ value: z.string() }),
|
||||
task: "fact_extractor",
|
||||
prompt: "Return JSON.",
|
||||
});
|
||||
|
||||
await expect(
|
||||
client.generateValidatedJson({
|
||||
schema: z.object({ value: z.string() }),
|
||||
task: "fact_extractor",
|
||||
prompt: "Return JSON.",
|
||||
}),
|
||||
).rejects.toThrow("LLM response failed schema validation");
|
||||
const allLogs = [...infoSpy.mock.calls, ...warnSpy.mock.calls]
|
||||
.flat()
|
||||
.join("\n");
|
||||
expect(result).toBeNull();
|
||||
expect(allLogs).toContain("[llm:validated] task=fact_extractor ok=false");
|
||||
expect(allLogs).toContain("zod_error=");
|
||||
expect(allLogs).not.toContain("super-secret-key");
|
||||
@@ -173,13 +173,13 @@ describe("generateValidatedJson", () => {
|
||||
throw new Error("provider unavailable");
|
||||
});
|
||||
|
||||
const result = await client.generateValidatedJson({
|
||||
schema: z.object({ value: z.string() }),
|
||||
task: "quality_inspector",
|
||||
prompt: "Return JSON.",
|
||||
});
|
||||
|
||||
expect(result).toBeNull();
|
||||
await expect(
|
||||
client.generateValidatedJson({
|
||||
schema: z.object({ value: z.string() }),
|
||||
task: "quality_inspector",
|
||||
prompt: "Return JSON.",
|
||||
}),
|
||||
).rejects.toThrow("provider unavailable");
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining("[llm:error] task=quality_inspector"),
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user