diff --git a/.gitignore b/.gitignore index 19bc52c..6ca5b93 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ build/ .env.* !.env.example data/app.db +data/app.db-* data/exports/ *.log test-results/ @@ -14,5 +15,6 @@ playwright-report/ .open-next .wrangler .dev.vars +.secrets/ cloudflare-env.d.ts tsconfig.tsbuildinfo diff --git a/docs/superpowers/plans/2026-06-16-strengthen-llm-prompts-and-logging.md b/docs/superpowers/plans/2026-06-16-strengthen-llm-prompts-and-logging.md new file mode 100644 index 0000000..448acf0 --- /dev/null +++ b/docs/superpowers/plans/2026-06-16-strengthen-llm-prompts-and-logging.md @@ -0,0 +1,883 @@ +# Strengthen LLM Prompts And Logging Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make every LLM call observable in normal runtime logs and strengthen prompts so the workflow better addresses platform templates, confirmed fact-card constraints, and the content risks highlighted in `docs/project-architecture-showcase.html`. + +**Architecture:** Add a small logging contract in `src/lib/llm/client.ts` so all current and future `generateValidatedJson` calls emit provider/model/task/duration/raw-response/schema-result logs without exposing secrets. Extend prompt builders in `src/lib/llm/prompts.ts` and pass explicit `task` values from workflow nodes so fact extraction, article optimization, QA, and targeted rewrite each receive stronger instructions tied to platform mode, fact-card fields, and the eight customer-risk categories. + +**Tech Stack:** TypeScript, Next.js route handlers, OpenAI-compatible SDK, DeepSeek-compatible chat completions, Zod validation, Vitest. + +--- + +## Scope + +Implement only prompt/logging behavior and tests. Do not add a UI log viewer, database log persistence, streaming, new workflow states, or a new provider. Logs go to server stdout/stderr through `console.info` / `console.warn`, which is visible in local `npm run dev`, platform logs, and test spies. + +## File Structure + +- Modify: `src/lib/llm/client.ts` + - Add `task?: LlmTaskName` to `GenerateInput`. + - Add runtime logging around `generateJson` and `generateValidatedJson`. + - Keep raw response logging truncated by default and configurable through `LLM_LOG_RAW_LIMIT`. +- Modify: `src/lib/llm/__tests__/client.test.ts` + - Verify runtime logging includes provider/model/task, raw response snippets, schema pass/fail, and no secret values. +- Modify: `src/lib/llm/prompts.ts` + - Add platform labels/guidance. + - Strengthen FactExtractor, ArticleOptimizer, QualityInspector, and TargetedRewriter prompts. +- Create: `src/lib/llm/__tests__/prompts.test.ts` + - Verify prompts include platform guidance, customer risk categories, fact-card field rules, Chinese output rules, and targeted rewrite actions. +- Modify: `src/lib/workflow/fact-extractor.ts` + - Pass `task: "fact_extractor"` to `generateValidatedJson`. +- Modify: `src/lib/workflow/article-optimizer.ts` + - Pass `task: "article_optimizer"` to `generateValidatedJson`. +- Modify: `src/lib/workflow/quality-inspector.ts` + - Pass `task: "quality_inspector"` to `generateValidatedJson`. +- Modify: `src/lib/workflow/targeted-rewriter.ts` + - Pass `task: "targeted_rewriter"` to `generateValidatedJson`. +- Modify: `src/lib/workflow/__tests__/llm-integration.test.ts` + - Assert task names are passed by each workflow node. +- Modify: `README.md` + - Document normal LLM runtime logs and the `LLM_LOG_RAW_LIMIT` setting. + +## Task 1: Add Runtime LLM Logging Contract + +**Files:** + +- Modify: `src/lib/llm/client.ts` +- Modify: `src/lib/llm/__tests__/client.test.ts` + +- [ ] **Step 1: Add failing logging tests** + + Append these tests to `src/lib/llm/__tests__/client.test.ts` inside `describe("generateValidatedJson", () => { ... })`: + + ```ts + it("logs provider, model, task, raw response snippet, and validation success", async () => { + process.env.LLM_PROVIDER = "deepseek"; + process.env.DEEPSEEK_API_KEY = "test-key"; + process.env.DEEPSEEK_MODEL = "deepseek-test"; + process.env.LLM_LOG_RAW_LIMIT = "80"; + const infoSpy = vi.spyOn(console, "info").mockImplementation(() => undefined); + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined); + client.setGenerateJsonForValidation(async () => ({ + value: "from-llm", + longField: "x".repeat(200), + })); + + const result = await client.generateValidatedJson({ + schema: z.object({ value: z.string(), longField: z.string() }), + task: "article_optimizer", + prompt: "Return JSON.", + }); + + expect(result?.value).toBe("from-llm"); + expect(infoSpy).toHaveBeenCalledWith( + expect.stringContaining("[llm:start] provider=deepseek model=deepseek-test task=article_optimizer"), + ); + expect(infoSpy).toHaveBeenCalledWith( + expect.stringContaining("[llm:response] task=article_optimizer"), + ); + expect(infoSpy).toHaveBeenCalledWith( + expect.stringContaining('"value":"from-llm"'), + ); + expect(infoSpy).toHaveBeenCalledWith( + expect.stringContaining("[llm:validated] task=article_optimizer ok=true"), + ); + expect(warnSpy).not.toHaveBeenCalled(); + }); + + it("logs validation failure without leaking provider secrets", async () => { + process.env.LLM_PROVIDER = "deepseek"; + process.env.DEEPSEEK_API_KEY = "super-secret-key"; + process.env.DEEPSEEK_MODEL = "deepseek-test"; + const infoSpy = vi.spyOn(console, "info").mockImplementation(() => undefined); + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined); + client.setGenerateJsonForValidation(async () => ({ value: 42 })); + + const result = await client.generateValidatedJson({ + schema: z.object({ value: z.string() }), + task: "fact_extractor", + prompt: "Return JSON.", + }); + + const allLogs = [...infoSpy.mock.calls, ...warnSpy.mock.calls] + .flat() + .join("\n"); + expect(result).toBeNull(); + expect(allLogs).toContain("[llm:validated] task=fact_extractor ok=false"); + expect(allLogs).toContain("[llm:validation-error] task=fact_extractor"); + expect(allLogs).not.toContain("super-secret-key"); + }); + + it("logs provider call failure for validated JSON calls", async () => { + process.env.LLM_PROVIDER = "deepseek"; + process.env.DEEPSEEK_API_KEY = "test-key"; + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined); + client.setGenerateJsonForValidation(async () => { + throw new Error("provider unavailable"); + }); + + const result = await client.generateValidatedJson({ + schema: z.object({ value: z.string() }), + task: "quality_inspector", + prompt: "Return JSON.", + }); + + expect(result).toBeNull(); + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining("[llm:error] task=quality_inspector"), + ); + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining("provider unavailable"), + ); + }); + ``` + + Extend the existing `afterEach` in that file with: + + ```ts + delete process.env.LLM_LOG_RAW_LIMIT; + delete process.env.DEEPSEEK_MODEL; + ``` + +- [ ] **Step 2: Run the test to verify it fails** + + Run: + + ```bash + npm test -- src/lib/llm/__tests__/client.test.ts + ``` + + Expected: FAIL because `GenerateInput` does not accept `task` and no LLM logs are emitted. + +- [ ] **Step 3: Implement log fields and helpers** + + Modify `src/lib/llm/client.ts`. + + Replace the `GenerateInput` interface with: + + ```ts + export type LlmTaskName = + | "unknown" + | "fact_extractor" + | "article_optimizer" + | "quality_inspector" + | "targeted_rewriter"; + + export interface GenerateInput { + system?: string; + prompt: string; + model?: string; + temperature?: number; + task?: LlmTaskName; + } + ``` + + Add these helpers before `generateText`: + + ```ts + function getRawLogLimit() { + const parsed = Number(process.env.LLM_LOG_RAW_LIMIT ?? "4000"); + return Number.isFinite(parsed) && parsed >= 0 ? parsed : 4000; + } + + function stringifyForLog(value: unknown) { + const raw = typeof value === "string" ? value : JSON.stringify(value); + const limit = getRawLogLimit(); + if (limit === 0) return ""; + return raw.length > limit ? `${raw.slice(0, limit)}...` : raw; + } + + function getTask(input: GenerateInput) { + return input.task ?? "unknown"; + } + + function logLlmStart(input: GenerateInput) { + const status = getLlmProviderStatus(); + console.info( + `[llm:start] provider=${status.provider} model=${input.model ?? status.model} task=${getTask(input)}`, + ); + } + + function logLlmResponse(input: GenerateInput, startedAt: number, content: unknown) { + console.info( + `[llm:response] task=${getTask(input)} duration_ms=${Date.now() - startedAt} raw=${stringifyForLog(content)}`, + ); + } + + function logLlmValidation(input: GenerateInput, ok: boolean) { + console.info(`[llm:validated] task=${getTask(input)} ok=${ok}`); + } + + function logLlmWarning(prefix: string, input: GenerateInput, error: unknown) { + const message = error instanceof Error ? error.message : String(error); + console.warn(`${prefix} task=${getTask(input)} error=${message}`); + } + ``` + +- [ ] **Step 4: Log raw content from `generateJson`** + + Replace `generateJson` with: + + ```ts + export async function generateJson(input: GenerateInput): Promise { + const startedAt = Date.now(); + logLlmStart(input); + try { + const { client, model } = createClient(); + const response = await client.chat.completions.create({ + model: input.model ?? model, + temperature: input.temperature ?? 0.1, + response_format: { type: "json_object" }, + messages: [ + ...(input.system ? [{ role: "system" as const, content: input.system }] : []), + { role: "user" as const, content: input.prompt }, + ], + }); + const content = response.choices[0]?.message.content ?? "{}"; + logLlmResponse(input, startedAt, content); + return JSON.parse(content) as T; + } catch (error) { + logLlmWarning("[llm:error]", input, error); + throw normalizeLlmError(error); + } + } + ``` + + Keep `generateText` unchanged in this task. It is not used by the workflow today. + +- [ ] **Step 5: Log validation results from `generateValidatedJson`** + + Replace `generateValidatedJson` with: + + ```ts + export async function generateValidatedJson({ + schema, + ...input + }: GenerateValidatedJsonInput): Promise { + if (!isLlmConfigured()) { + console.info(`[llm:skip] task=${getTask(input)} reason=provider_not_configured`); + return null; + } + + const startedAt = Date.now(); + logLlmStart(input); + try { + const generated = await generateJsonForValidation(input); + logLlmResponse(input, startedAt, generated); + const parsed = schema.safeParse(generated); + logLlmValidation(input, parsed.success); + if (!parsed.success) { + console.warn( + `[llm:validation-error] task=${getTask(input)} error=${parsed.error.message}`, + ); + return null; + } + return parsed.data; + } catch (error) { + logLlmValidation(input, false); + logLlmWarning("[llm:error]", input, error); + return null; + } + } + ``` + + This means real `generateJson` calls log start/response once inside `generateJson`, while tests that replace `generateJsonForValidation` still get start/response logs from `generateValidatedJson`. + +- [ ] **Step 6: Run the client tests** + + Run: + + ```bash + npm test -- src/lib/llm/__tests__/client.test.ts + ``` + + Expected: PASS. + +- [ ] **Step 7: Commit** + + ```bash + git add src/lib/llm/client.ts src/lib/llm/__tests__/client.test.ts + git commit -m "feat: log llm runtime responses" + ``` + +## Task 2: Strengthen Prompt Builders + +**Files:** + +- Modify: `src/lib/llm/prompts.ts` +- Create: `src/lib/llm/__tests__/prompts.test.ts` + +- [ ] **Step 1: Add failing prompt tests** + + Create `src/lib/llm/__tests__/prompts.test.ts`: + + ```ts + import { describe, expect, it } from "vitest"; + + import { + ARTICLE_OPTIMIZER_SYSTEM_PROMPT, + FACT_EXTRACTOR_SYSTEM_PROMPT, + QUALITY_INSPECTOR_SYSTEM_PROMPT, + TARGETED_REWRITER_SYSTEM_PROMPT, + buildArticleOptimizerPrompt, + buildFactExtractorPrompt, + buildQualityInspectorPrompt, + buildTargetedRewritePrompt, + } from "../prompts"; + + const articleInput = { + title: "探寻AIGC短视频培训选哪家，各品牌实力大比拼", + body: "伟思德鲁管理咨询（深圳）有限公司提供AIGC短视频培训，文章提到客户案例和出海内容生产。", + images: [{ type: "description" as const, content: "AIGC短视频工作流示意图" }], + platform: "official_site" as const, + user_instructions: "改成官网文章，避免虚构客户案例。", + }; + + const factCard = { + company_full_name: "伟思德鲁管理咨询（深圳）有限公司", + company_short_names: ["伟思德鲁", "伟思德鲁管理咨询"], + brand_names: ["伟思德鲁管理咨询"], + product_names: ["AIGC短视频培训"], + target_industry: "AIGC短视频培训", + target_audience: "品牌商家、内容创作者、出海企业", + experience_years: null, + core_claims: ["提供AIGC短视频培训"], + forbidden_claims: ["行业第一", "服务过500强客户"], + image_topics: ["AIGC短视频工作流示意图"], + uncertain_items: [], + is_ready_for_optimization: true, + confirmed_by_user: true, + } as const; + + describe("LLM prompt builders", () => { + it("fact extraction prompt names customer risk categories and uncertainty rules", () => { + const prompt = `${FACT_EXTRACTOR_SYSTEM_PROMPT}\n${buildFactExtractorPrompt(articleInput)}`; + + expect(prompt).toContain("客户案例"); + expect(prompt).toContain("资质荣誉"); + expect(prompt).toContain("经验年限"); + expect(prompt).toContain("uncertain_items"); + expect(prompt).toContain("不要把营销夸张词当事实"); + expect(prompt).toContain("图片主题"); + }); + + it("article optimizer prompt includes platform templates and Chinese output rules", () => { + const prompt = `${ARTICLE_OPTIMIZER_SYSTEM_PROMPT}\n${buildArticleOptimizerPrompt(articleInput, factCard)}`; + + expect(prompt).toContain("official_site"); + expect(prompt).toContain("官网文章"); + expect(prompt).toContain("第一方品牌口吻"); + expect(prompt).toContain("输出中文标题"); + expect(prompt).toContain("禁止使用英文模板词"); + expect(prompt).toContain("实质性重写"); + expect(prompt).toContain("不得新增客户案例"); + expect(prompt).toContain("forbidden_claims"); + }); + + it("quality inspector prompt encodes fail and warn standards for all risk gates", () => { + const prompt = `${QUALITY_INSPECTOR_SYSTEM_PROMPT}\n${buildQualityInspectorPrompt({ + article: { + title: "伟思德鲁 AIGC短视频培训", + summary: "官网文章摘要", + body_markdown: "正文", + image_suggestions: [], + changed_sections: [], + requires_user_confirmation: [], + }, + factCard, + platform: "official_site", + deterministicChecks: [ + { + rule_id: "hallucination_risk", + status: "pass", + evidence: "未发现", + reason: "本地规则通过", + suggested_fix: "", + target_agent: null, + }, + ], + })}`; + + expect(prompt).toContain("不得把 deterministic fail 降级"); + expect(prompt).toContain("行业漂移"); + expect(prompt).toContain("公司名不一致"); + expect(prompt).toContain("未确认案例"); + expect(prompt).toContain("平台口吻严重错误"); + expect(prompt).toContain("warn 标准"); + }); + + it("targeted rewrite prompt gives rule-specific repair actions", () => { + const prompt = `${TARGETED_REWRITER_SYSTEM_PROMPT}\n${buildTargetedRewritePrompt({ + article: { + title: "Bad title!!!", + summary: "摘要", + body_markdown: "正文", + image_suggestions: [], + changed_sections: [], + requires_user_confirmation: [], + }, + factCard, + failedChecks: [ + { + rule_id: "title_quality", + status: "fail", + evidence: "Bad title!!!", + reason: "标题问题", + suggested_fix: "重写标题", + target_agent: "title", + }, + ], + })}`; + + expect(prompt).toContain("industry_alignment"); + expect(prompt).toContain("company_name_integrity"); + expect(prompt).toContain("hallucination_risk"); + expect(prompt).toContain("claim_consistency"); + expect(prompt).toContain("title_quality"); + expect(prompt).toContain("body_quality"); + expect(prompt).toContain("不得新增事实"); + }); + }); + ``` + +- [ ] **Step 2: Run the prompt tests to verify they fail** + + Run: + + ```bash + npm test -- src/lib/llm/__tests__/prompts.test.ts + ``` + + Expected: FAIL because the current prompts do not include the required platform templates and risk language. + +- [ ] **Step 3: Replace `prompts.ts` with strengthened prompt builders** + + Replace `src/lib/llm/prompts.ts` with: + + ```ts + import type { + ArticleInput, + ConfirmedFactCard, + OptimizedArticle, + PublishPlatform, + QaCheck, + } from "../domain/types"; + + export const JSON_ONLY_PROMPT = + "Return valid JSON only. Do not include markdown fences or commentary."; + + const CUSTOMER_RISK_GUIDANCE = [ + "客户最担心的内容风险：行业漂移、公司全称/简称/品牌名不一致、图片主题与正文描述不匹配、官网文章出现第三方口吻、平台语气和文章类型不匹配、标题或正文语义不顺、虚构资质/年限/案例/能力、产品/服务/年限前后冲突。", + "任何客户案例、资质荣誉、经验年限、服务能力、出海/多语种/合规能力、效果承诺和排名，都必须能从原文或已确认事实卡中找到明确依据。", + ].join(" "); + + const PLATFORM_GUIDANCE: Record = { + official_site: + "official_site / 官网文章：第一方品牌口吻，表达克制可信，可使用“我们/公司/品牌”视角；避免第三方评测腔、夸大宣传和无依据背书。", + media_article: + "media_article / 媒体稿：第三方客观报道口吻，突出事实、背景和行业价值；不要替品牌自夸，不制造新闻来源。", + comparison_review: + "comparison_review / 对比评测：对比分析口吻，只比较原文或事实卡中明确出现的信息；不得虚构竞品、排名、参数或测试结论。", + recommendation_list: + "recommendation_list / 推荐榜单：推荐榜单口吻，必须解释推荐依据；不得制造榜单排名、奖项或第三方认证。", + }; + + function formatPlatformGuidance(selected: PublishPlatform) { + return [ + "平台模板选项：", + ...Object.values(PLATFORM_GUIDANCE).map((item) => `- ${item}`), + `当前目标平台：${selected}。必须优先遵守该平台模板。`, + ].join("\n"); + } + + export const ARTICLE_OPTIMIZER_SYSTEM_PROMPT = [ + "你是中文 GEO 内容优化编辑。目标不是复述原文，而是在事实卡约束下进行实质性重写。", + CUSTOMER_RISK_GUIDANCE, + "硬性规则：只能使用 confirmed fact card 和 source article 中明确出现的事实；不得新增客户案例、数字、资质、排名、奖项、服务能力、效果承诺；输出中文标题，禁止使用英文模板词如 Guide、Best、Top。", + "必须修复原文中的病句、断裂句、错别字、逻辑跳跃和段落结构问题。", + JSON_ONLY_PROMPT, + ].join(" "); + + export const QUALITY_INSPECTOR_SYSTEM_PROMPT = [ + "你是质量门禁审核器。逐项检查 deterministicChecks 中的每个 rule_id。", + CUSTOMER_RISK_GUIDANCE, + "不得把 deterministic fail 降级。每项必须给出 status、evidence、reason、suggested_fix、target_agent。", + JSON_ONLY_PROMPT, + ].join(" "); + + export const FACT_EXTRACTOR_SYSTEM_PROMPT = [ + "你是事实卡提取器。只从原文和图片描述中抽取事实，不推测、不补全。", + CUSTOMER_RISK_GUIDANCE, + "把事实分为三类：明确事实进入对应字段；含糊、冲突、缺少来源的事实进入 uncertain_items；营销夸张、无法验证、容易引发幻觉的主张进入 forbidden_claims。", + "特别检查：公司全称、简称、品牌名、产品/服务名、目标行业、目标受众、经验年限、客户案例、资质荣誉、服务能力、地域/出海能力、图片主题。不要把营销夸张词当事实。", + JSON_ONLY_PROMPT, + ].join(" "); + + export const TARGETED_REWRITER_SYSTEM_PROMPT = [ + "你是定向修复器。只修复 failedChecks 指出的失败项，不整篇重写。", + CUSTOMER_RISK_GUIDANCE, + "不得新增事实。无法修复或需要客户确认的内容放入 requires_user_confirmation。", + JSON_ONLY_PROMPT, + ].join(" "); + + export function buildFactExtractorPrompt(input: ArticleInput) { + return [ + "Return a CandidateFactCard JSON object with these exact keys:", + "company_full_name, company_short_names, brand_names, product_names, target_industry, target_audience, experience_years, core_claims, forbidden_claims, image_topics, uncertain_items.", + "Do not include confirmed_by_user.", + "", + "字段要求：", + "- core_claims 只放原文明示、可作为后续优化硬约束的事实。", + "- forbidden_claims 放行业第一、最强、知名客户、显著提升、保证转化等无法验证或高风险主张。", + "- uncertain_items 放客户案例、资质荣誉、经验年限、产品能力、出海能力等证据不足或前后冲突的内容。", + "- image_topics 必须来自图片描述或图片链接文本；没有图片就返回空数组。", + "- target_industry 必须是文章主行业，避免被局部案例带偏。", + "", + "Article input:", + JSON.stringify(input, null, 2), + ].join("\n"); + } + + export function buildArticleOptimizerPrompt( + input: ArticleInput, + factCard: ConfirmedFactCard, + ) { + return [ + "Return an OptimizedArticle JSON object with these exact keys:", + "title, summary, body_markdown, image_suggestions, changed_sections, requires_user_confirmation.", + "", + formatPlatformGuidance(input.platform), + "", + "输出要求：", + "- 输出中文标题，禁止使用英文模板词如 Guide、Best、Top。", + "- 进行实质性重写：重组标题、摘要、段落顺序、长句、病句和逻辑连接，而不是只在原文前后拼接内容。", + "- body_markdown 使用清晰的 Markdown 层级；标题层级要规范，中文标题后保留空格，例如 `## 标题`。", + "- 如果用户要求包含未确认事实，正文不写入，放入 requires_user_confirmation。", + "- 必须保留事实卡确认的公司全称、目标行业、目标受众和核心事实。", + "- 必须删除或弱化 factCard.forbidden_claims 中的主张。", + "- 不得新增客户案例、数字、资质、排名、奖项、服务能力、效果承诺。", + "- image_suggestions 必须基于 factCard.image_topics 或原始 images；没有图片主题时返回空数组。", + "", + "Confirmed fact card:", + JSON.stringify(factCard, null, 2), + "", + "Article input:", + JSON.stringify(input, null, 2), + ].join("\n"); + } + + export function buildQualityInspectorPrompt(input: { + article: OptimizedArticle; + factCard: ConfirmedFactCard; + platform: PublishPlatform; + deterministicChecks: QaCheck[]; + }) { + return [ + "Return a JSON object with a checks array.", + "Each check must include rule_id, status, evidence, reason, suggested_fix, and target_agent.", + "Only use rule_id values already present in deterministicChecks.", + "不得把 deterministic fail 降级。", + "", + formatPlatformGuidance(input.platform), + "", + "fail 标准：行业漂移、公司名不一致、事实卡外新增数字/客户/资质/案例、产品服务前后冲突、平台口吻严重错误、标题明显病句。", + "warn 标准：图片证据不足、句子过长、表达可读性一般、平台适配轻微不足。", + "target_agent 只能使用 title、body、image、fact_card 或 null。", + "", + "Confirmed fact card:", + JSON.stringify(input.factCard, null, 2), + "", + "Optimized article:", + JSON.stringify(input.article, null, 2), + "", + "Deterministic checks:", + JSON.stringify(input.deterministicChecks, null, 2), + ].join("\n"); + } + + export function buildTargetedRewritePrompt(input: { + article: OptimizedArticle; + factCard: ConfirmedFactCard; + failedChecks: QaCheck[]; + }) { + return [ + "Return an OptimizedArticle JSON object.", + "Rewrite only the fields needed for failedChecks.", + "", + "按 rule_id 执行修复动作：", + "- industry_alignment：删除偏离行业的段落，改回 factCard.target_industry。", + "- company_name_integrity：统一公司全称和简称。", + "- hallucination_risk：删除所有未确认数字、客户案例、资质、奖项、排名、效果承诺。", + "- claim_consistency：统一年限、产品名、服务能力和核心主张。", + "- title_quality：生成自然中文标题，禁止英文模板词。", + "- body_quality：拆分长句，修复病句和断裂表达。", + "- voice_consistency / platform_fit：改成目标平台对应口吻。", + "- image_text_match：只补充图片建议或人工确认项，不虚构图片内容。", + "不得新增事实。无法修复的内容放入 requires_user_confirmation。", + "", + "Confirmed fact card:", + JSON.stringify(input.factCard, null, 2), + "", + "Current optimized article:", + JSON.stringify(input.article, null, 2), + "", + "Failed checks:", + JSON.stringify(input.failedChecks, null, 2), + ].join("\n"); + } + ``` + +- [ ] **Step 4: Run prompt tests** + + Run: + + ```bash + npm test -- src/lib/llm/__tests__/prompts.test.ts + ``` + + Expected: PASS. + +- [ ] **Step 5: Commit** + + ```bash + git add src/lib/llm/prompts.ts src/lib/llm/__tests__/prompts.test.ts + git commit -m "feat: strengthen llm workflow prompts" + ``` + +## Task 3: Pass Explicit Task Names From Workflow Nodes + +**Files:** + +- Modify: `src/lib/workflow/fact-extractor.ts` +- Modify: `src/lib/workflow/article-optimizer.ts` +- Modify: `src/lib/workflow/quality-inspector.ts` +- Modify: `src/lib/workflow/targeted-rewriter.ts` +- Modify: `src/lib/workflow/__tests__/llm-integration.test.ts` + +- [ ] **Step 1: Add failing task-name assertions** + + In `src/lib/workflow/__tests__/llm-integration.test.ts`, after each existing `expect(llmMocks.generateValidatedJson).toHaveBeenCalledOnce();`, add the matching assertion: + + ```ts + expect(llmMocks.generateValidatedJson).toHaveBeenCalledWith( + expect.objectContaining({ task: "fact_extractor" }), + ); + ``` + + For the article optimizer test, use: + + ```ts + expect(llmMocks.generateValidatedJson).toHaveBeenCalledWith( + expect.objectContaining({ task: "article_optimizer" }), + ); + ``` + + For the targeted rewrite test, use: + + ```ts + expect(llmMocks.generateValidatedJson).toHaveBeenCalledWith( + expect.objectContaining({ task: "targeted_rewriter" }), + ); + ``` + + At the end of the `"uses LLM quality checks to enrich non-failing deterministic checks"` test, add: + + ```ts + expect(llmMocks.generateValidatedJson).toHaveBeenCalledWith( + expect.objectContaining({ task: "quality_inspector" }), + ); + ``` + +- [ ] **Step 2: Run the integration test to verify it fails** + + Run: + + ```bash + npm test -- src/lib/workflow/__tests__/llm-integration.test.ts + ``` + + Expected: FAIL because workflow nodes do not pass task names yet. + +- [ ] **Step 3: Add task names to workflow LLM calls** + + In `src/lib/workflow/fact-extractor.ts`, change: + + ```ts + const llmCard = await generateValidatedJson({ + schema: candidateFactCardSchema, + ``` + + to: + + ```ts + const llmCard = await generateValidatedJson({ + schema: candidateFactCardSchema, + task: "fact_extractor", + ``` + + In `src/lib/workflow/article-optimizer.ts`, change: + + ```ts + const llmArticle = await generateValidatedJson({ + schema: optimizedArticleSchema, + ``` + + to: + + ```ts + const llmArticle = await generateValidatedJson({ + schema: optimizedArticleSchema, + task: "article_optimizer", + ``` + + In `src/lib/workflow/quality-inspector.ts`, change: + + ```ts + const llmPatch = await generateValidatedJson({ + schema: llmQaPatchSchema, + ``` + + to: + + ```ts + const llmPatch = await generateValidatedJson({ + schema: llmQaPatchSchema, + task: "quality_inspector", + ``` + + In `src/lib/workflow/targeted-rewriter.ts`, change: + + ```ts + const llmArticle = await generateValidatedJson({ + schema: optimizedArticleSchema, + ``` + + to: + + ```ts + const llmArticle = await generateValidatedJson({ + schema: optimizedArticleSchema, + task: "targeted_rewriter", + ``` + +- [ ] **Step 4: Run integration tests** + + Run: + + ```bash + npm test -- src/lib/workflow/__tests__/llm-integration.test.ts + ``` + + Expected: PASS. + +- [ ] **Step 5: Commit** + + ```bash + git add src/lib/workflow/fact-extractor.ts src/lib/workflow/article-optimizer.ts src/lib/workflow/quality-inspector.ts src/lib/workflow/targeted-rewriter.ts src/lib/workflow/__tests__/llm-integration.test.ts + git commit -m "feat: tag llm workflow calls" + ``` + +## Task 4: Document LLM Logs And Prompt Scope + +**Files:** + +- Modify: `README.md` + +- [ ] **Step 1: Update README with LLM logging behavior** + + Add this section after the existing Environment section: + + ```md + ## LLM Runtime Logs + + Every workflow LLM call logs to server stdout/stderr: + + - `[llm:start]` with provider, model, and task name. + - `[llm:response]` with task name, duration, and a raw response snippet. + - `[llm:validated]` with schema validation result. + - `[llm:validation-error]` when Zod rejects the response. + - `[llm:error]` when the provider call or JSON parsing fails. + + Raw response logging is intentionally enabled for operational visibility. It is + truncated to 4000 characters by default. Set `LLM_LOG_RAW_LIMIT=0` to suppress + raw response snippets, or set a larger number when diagnosing model output. + + Workflow task names are: + + - `fact_extractor` + - `article_optimizer` + - `quality_inspector` + - `targeted_rewriter` + ``` + +- [ ] **Step 2: Run full verification** + + Run: + + ```bash + npm test + npm run build + ``` + + Expected: both commands PASS. + +- [ ] **Step 3: Commit** + + ```bash + git add README.md + git commit -m "docs: document llm runtime logs" + ``` + +## Task 5: Manual Smoke Test With Logs + +**Files:** + +- Verify: `src/lib/llm/client.ts` +- Verify: `src/lib/llm/prompts.ts` +- Verify: running dev server output + +- [ ] **Step 1: Start dev server** + + Run: + + ```bash + npm run dev + ``` + + Expected: server prints `Local: http://localhost:3000`. + +- [ ] **Step 2: Trigger analysis with curl** + + Run: + + ```bash + curl -s -X POST http://localhost:3000/api/jobs \ + -H 'content-type: application/json' \ + -H "x-api-key: ${API_ACCESS_KEY:-local-dev-key}" \ + --data '{ + "title":"伟思德鲁管理咨询（深圳）有限公司 AIGC短视频培训官网文章", + "body":"伟思德鲁管理咨询（深圳）有限公司面向品牌商家和出海企业提供AIGC短视频培训服务，帮助企业解决内容工业化生产、品牌视觉统一和全球化传播问题。", + "image_lines":"AIGC短视频工作流示意图", + "platform":"official_site", + "user_instructions":"改成官网文章，保持事实准确，不新增客户案例。" + }' | jq '.candidateFactCard.company_full_name' + ``` + + Expected: command prints `"伟思德鲁管理咨询（深圳）有限公司"` or a non-empty company name. Server logs include: + + ```text + [llm:start] provider=deepseek ... task=fact_extractor + [llm:response] task=fact_extractor ... + [llm:validated] task=fact_extractor ok=true + ``` + +- [ ] **Step 3: Stop dev server** + + Press `Ctrl-C`. + +## Self-Review + +- Spec coverage: The plan covers permanent LLM response logging, prompt strengthening for all four LLM workflow nodes, task labels for observability, documentation, and manual smoke testing. +- Placeholder scan: The plan includes exact file paths, concrete test code, concrete implementation snippets, commands, and expected outcomes. It contains no unresolved placeholders. +- Type consistency: `LlmTaskName`, `task`, prompt builder names, existing schema names, and workflow function names match the current codebase. diff --git a/wrangler.jsonc b/wrangler.jsonc index f86dc4d..fad9e40 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -40,5 +40,68 @@ "bucket_name": "geo-agent-article-optimizer-local" } ], - "env": {} + "env": { + "staging": { + "name": "geo-agent-article-optimizer-staging", + "services": [ + { + "binding": "WORKER_SELF_REFERENCE", + "service": "geo-agent-article-optimizer-staging" + } + ], + "vars": { + "APP_RUNTIME": "cloudflare", + "LLM_PROVIDER": "deepseek", + "DEEPSEEK_BASE_URL": "https://api.deepseek.com", + "DEEPSEEK_MODEL": "deepseek-v4-pro", + "DEEPSEEK_THINKING": "disabled", + "OPENAI_MODEL": "gpt-4.1-mini" + }, + "d1_databases": [ + { + "binding": "DB", + "database_name": "geo-agent-article-optimizer-staging", + "database_id": "0162e94c-9f71-472c-ae78-13786578a2ac", + "migrations_dir": "migrations" + } + ], + "r2_buckets": [ + { + "binding": "EXPORT_BUCKET", + "bucket_name": "geo-agent-article-optimizer-staging" + } + ] + }, + "production": { + "name": "geo-agent-article-optimizer-production", + "services": [ + { + "binding": "WORKER_SELF_REFERENCE", + "service": "geo-agent-article-optimizer-production" + } + ], + "vars": { + "APP_RUNTIME": "cloudflare", + "LLM_PROVIDER": "deepseek", + "DEEPSEEK_BASE_URL": "https://api.deepseek.com", + "DEEPSEEK_MODEL": "deepseek-v4-pro", + "DEEPSEEK_THINKING": "disabled", + "OPENAI_MODEL": "gpt-4.1-mini" + }, + "d1_databases": [ + { + "binding": "DB", + "database_name": "geo-agent-article-optimizer-production", + "database_id": "8161ceeb-76c6-40fd-9f38-32ec7ab8f2b1", + "migrations_dir": "migrations" + } + ], + "r2_buckets": [ + { + "binding": "EXPORT_BUCKET", + "bucket_name": "geo-agent-article-optimizer-production" + } + ] + } + } }