chore: add cloudflare env config and prompt plan
This commit is contained in:
@@ -7,6 +7,7 @@ build/
|
||||
.env.*
|
||||
!.env.example
|
||||
data/app.db
|
||||
data/app.db-*
|
||||
data/exports/
|
||||
*.log
|
||||
test-results/
|
||||
@@ -14,5 +15,6 @@ playwright-report/
|
||||
.open-next
|
||||
.wrangler
|
||||
.dev.vars
|
||||
.secrets/
|
||||
cloudflare-env.d.ts
|
||||
tsconfig.tsbuildinfo
|
||||
|
||||
@@ -0,0 +1,883 @@
|
||||
# Strengthen LLM Prompts And Logging Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Make every LLM call observable in normal runtime logs and strengthen prompts so the workflow better addresses platform templates, confirmed fact-card constraints, and the content risks highlighted in `docs/project-architecture-showcase.html`.
|
||||
|
||||
**Architecture:** Add a small logging contract in `src/lib/llm/client.ts` so all current and future `generateValidatedJson` calls emit provider/model/task/duration/raw-response/schema-result logs without exposing secrets. Extend prompt builders in `src/lib/llm/prompts.ts` and pass explicit `task` values from workflow nodes so fact extraction, article optimization, QA, and targeted rewrite each receive stronger instructions tied to platform mode, fact-card fields, and the eight customer-risk categories.
|
||||
|
||||
**Tech Stack:** TypeScript, Next.js route handlers, OpenAI-compatible SDK, DeepSeek-compatible chat completions, Zod validation, Vitest.
|
||||
|
||||
---
|
||||
|
||||
## Scope
|
||||
|
||||
Implement only prompt/logging behavior and tests. Do not add a UI log viewer, database log persistence, streaming, new workflow states, or a new provider. Logs go to server stdout/stderr through `console.info` / `console.warn`, which is visible in local `npm run dev`, platform logs, and test spies.
|
||||
|
||||
## File Structure
|
||||
|
||||
- Modify: `src/lib/llm/client.ts`
|
||||
- Add `task?: LlmTaskName` to `GenerateInput`.
|
||||
- Add runtime logging around `generateJson` and `generateValidatedJson`.
|
||||
- Keep raw response logging truncated by default and configurable through `LLM_LOG_RAW_LIMIT`.
|
||||
- Modify: `src/lib/llm/__tests__/client.test.ts`
|
||||
- Verify runtime logging includes provider/model/task, raw response snippets, schema pass/fail, and no secret values.
|
||||
- Modify: `src/lib/llm/prompts.ts`
|
||||
- Add platform labels/guidance.
|
||||
- Strengthen FactExtractor, ArticleOptimizer, QualityInspector, and TargetedRewriter prompts.
|
||||
- Create: `src/lib/llm/__tests__/prompts.test.ts`
|
||||
- Verify prompts include platform guidance, customer risk categories, fact-card field rules, Chinese output rules, and targeted rewrite actions.
|
||||
- Modify: `src/lib/workflow/fact-extractor.ts`
|
||||
- Pass `task: "fact_extractor"` to `generateValidatedJson`.
|
||||
- Modify: `src/lib/workflow/article-optimizer.ts`
|
||||
- Pass `task: "article_optimizer"` to `generateValidatedJson`.
|
||||
- Modify: `src/lib/workflow/quality-inspector.ts`
|
||||
- Pass `task: "quality_inspector"` to `generateValidatedJson`.
|
||||
- Modify: `src/lib/workflow/targeted-rewriter.ts`
|
||||
- Pass `task: "targeted_rewriter"` to `generateValidatedJson`.
|
||||
- Modify: `src/lib/workflow/__tests__/llm-integration.test.ts`
|
||||
- Assert task names are passed by each workflow node.
|
||||
- Modify: `README.md`
|
||||
- Document normal LLM runtime logs and the `LLM_LOG_RAW_LIMIT` setting.
|
||||
|
||||
## Task 1: Add Runtime LLM Logging Contract
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `src/lib/llm/client.ts`
|
||||
- Modify: `src/lib/llm/__tests__/client.test.ts`
|
||||
|
||||
- [ ] **Step 1: Add failing logging tests**
|
||||
|
||||
Append these tests to `src/lib/llm/__tests__/client.test.ts` inside `describe("generateValidatedJson", () => { ... })`:
|
||||
|
||||
```ts
|
||||
it("logs provider, model, task, raw response snippet, and validation success", async () => {
|
||||
process.env.LLM_PROVIDER = "deepseek";
|
||||
process.env.DEEPSEEK_API_KEY = "test-key";
|
||||
process.env.DEEPSEEK_MODEL = "deepseek-test";
|
||||
process.env.LLM_LOG_RAW_LIMIT = "80";
|
||||
const infoSpy = vi.spyOn(console, "info").mockImplementation(() => undefined);
|
||||
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined);
|
||||
client.setGenerateJsonForValidation(async () => ({
|
||||
value: "from-llm",
|
||||
longField: "x".repeat(200),
|
||||
}));
|
||||
|
||||
const result = await client.generateValidatedJson({
|
||||
schema: z.object({ value: z.string(), longField: z.string() }),
|
||||
task: "article_optimizer",
|
||||
prompt: "Return JSON.",
|
||||
});
|
||||
|
||||
expect(result?.value).toBe("from-llm");
|
||||
expect(infoSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining("[llm:start] provider=deepseek model=deepseek-test task=article_optimizer"),
|
||||
);
|
||||
expect(infoSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining("[llm:response] task=article_optimizer"),
|
||||
);
|
||||
expect(infoSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining('"value":"from-llm"'),
|
||||
);
|
||||
expect(infoSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining("[llm:validated] task=article_optimizer ok=true"),
|
||||
);
|
||||
expect(warnSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("logs validation failure without leaking provider secrets", async () => {
|
||||
process.env.LLM_PROVIDER = "deepseek";
|
||||
process.env.DEEPSEEK_API_KEY = "super-secret-key";
|
||||
process.env.DEEPSEEK_MODEL = "deepseek-test";
|
||||
const infoSpy = vi.spyOn(console, "info").mockImplementation(() => undefined);
|
||||
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined);
|
||||
client.setGenerateJsonForValidation(async () => ({ value: 42 }));
|
||||
|
||||
const result = await client.generateValidatedJson({
|
||||
schema: z.object({ value: z.string() }),
|
||||
task: "fact_extractor",
|
||||
prompt: "Return JSON.",
|
||||
});
|
||||
|
||||
const allLogs = [...infoSpy.mock.calls, ...warnSpy.mock.calls]
|
||||
.flat()
|
||||
.join("\n");
|
||||
expect(result).toBeNull();
|
||||
expect(allLogs).toContain("[llm:validated] task=fact_extractor ok=false");
|
||||
expect(allLogs).toContain("[llm:validation-error] task=fact_extractor");
|
||||
expect(allLogs).not.toContain("super-secret-key");
|
||||
});
|
||||
|
||||
it("logs provider call failure for validated JSON calls", async () => {
|
||||
process.env.LLM_PROVIDER = "deepseek";
|
||||
process.env.DEEPSEEK_API_KEY = "test-key";
|
||||
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined);
|
||||
client.setGenerateJsonForValidation(async () => {
|
||||
throw new Error("provider unavailable");
|
||||
});
|
||||
|
||||
const result = await client.generateValidatedJson({
|
||||
schema: z.object({ value: z.string() }),
|
||||
task: "quality_inspector",
|
||||
prompt: "Return JSON.",
|
||||
});
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining("[llm:error] task=quality_inspector"),
|
||||
);
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining("provider unavailable"),
|
||||
);
|
||||
});
|
||||
```
|
||||
|
||||
Extend the existing `afterEach` in that file with:
|
||||
|
||||
```ts
|
||||
delete process.env.LLM_LOG_RAW_LIMIT;
|
||||
delete process.env.DEEPSEEK_MODEL;
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run the test to verify it fails**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
npm test -- src/lib/llm/__tests__/client.test.ts
|
||||
```
|
||||
|
||||
Expected: FAIL because `GenerateInput` does not accept `task` and no LLM logs are emitted.
|
||||
|
||||
- [ ] **Step 3: Implement log fields and helpers**
|
||||
|
||||
Modify `src/lib/llm/client.ts`.
|
||||
|
||||
Replace the `GenerateInput` interface with:
|
||||
|
||||
```ts
|
||||
export type LlmTaskName =
|
||||
| "unknown"
|
||||
| "fact_extractor"
|
||||
| "article_optimizer"
|
||||
| "quality_inspector"
|
||||
| "targeted_rewriter";
|
||||
|
||||
export interface GenerateInput {
|
||||
system?: string;
|
||||
prompt: string;
|
||||
model?: string;
|
||||
temperature?: number;
|
||||
task?: LlmTaskName;
|
||||
}
|
||||
```
|
||||
|
||||
Add these helpers before `generateText`:
|
||||
|
||||
```ts
|
||||
function getRawLogLimit() {
|
||||
const parsed = Number(process.env.LLM_LOG_RAW_LIMIT ?? "4000");
|
||||
return Number.isFinite(parsed) && parsed >= 0 ? parsed : 4000;
|
||||
}
|
||||
|
||||
function stringifyForLog(value: unknown) {
|
||||
const raw = typeof value === "string" ? value : JSON.stringify(value);
|
||||
const limit = getRawLogLimit();
|
||||
if (limit === 0) return "";
|
||||
return raw.length > limit ? `${raw.slice(0, limit)}...<truncated>` : raw;
|
||||
}
|
||||
|
||||
function getTask(input: GenerateInput) {
|
||||
return input.task ?? "unknown";
|
||||
}
|
||||
|
||||
function logLlmStart(input: GenerateInput) {
|
||||
const status = getLlmProviderStatus();
|
||||
console.info(
|
||||
`[llm:start] provider=${status.provider} model=${input.model ?? status.model} task=${getTask(input)}`,
|
||||
);
|
||||
}
|
||||
|
||||
function logLlmResponse(input: GenerateInput, startedAt: number, content: unknown) {
|
||||
console.info(
|
||||
`[llm:response] task=${getTask(input)} duration_ms=${Date.now() - startedAt} raw=${stringifyForLog(content)}`,
|
||||
);
|
||||
}
|
||||
|
||||
function logLlmValidation(input: GenerateInput, ok: boolean) {
|
||||
console.info(`[llm:validated] task=${getTask(input)} ok=${ok}`);
|
||||
}
|
||||
|
||||
function logLlmWarning(prefix: string, input: GenerateInput, error: unknown) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.warn(`${prefix} task=${getTask(input)} error=${message}`);
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Log raw content from `generateJson`**
|
||||
|
||||
Replace `generateJson` with:
|
||||
|
||||
```ts
|
||||
export async function generateJson<T>(input: GenerateInput): Promise<T> {
|
||||
const startedAt = Date.now();
|
||||
logLlmStart(input);
|
||||
try {
|
||||
const { client, model } = createClient();
|
||||
const response = await client.chat.completions.create({
|
||||
model: input.model ?? model,
|
||||
temperature: input.temperature ?? 0.1,
|
||||
response_format: { type: "json_object" },
|
||||
messages: [
|
||||
...(input.system ? [{ role: "system" as const, content: input.system }] : []),
|
||||
{ role: "user" as const, content: input.prompt },
|
||||
],
|
||||
});
|
||||
const content = response.choices[0]?.message.content ?? "{}";
|
||||
logLlmResponse(input, startedAt, content);
|
||||
return JSON.parse(content) as T;
|
||||
} catch (error) {
|
||||
logLlmWarning("[llm:error]", input, error);
|
||||
throw normalizeLlmError(error);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Keep `generateText` unchanged in this task. It is not used by the workflow today.
|
||||
|
||||
- [ ] **Step 5: Log validation results from `generateValidatedJson`**
|
||||
|
||||
Replace `generateValidatedJson` with:
|
||||
|
||||
```ts
|
||||
export async function generateValidatedJson<T>({
|
||||
schema,
|
||||
...input
|
||||
}: GenerateValidatedJsonInput<T>): Promise<T | null> {
|
||||
if (!isLlmConfigured()) {
|
||||
console.info(`[llm:skip] task=${getTask(input)} reason=provider_not_configured`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const startedAt = Date.now();
|
||||
logLlmStart(input);
|
||||
try {
|
||||
const generated = await generateJsonForValidation<unknown>(input);
|
||||
logLlmResponse(input, startedAt, generated);
|
||||
const parsed = schema.safeParse(generated);
|
||||
logLlmValidation(input, parsed.success);
|
||||
if (!parsed.success) {
|
||||
console.warn(
|
||||
`[llm:validation-error] task=${getTask(input)} error=${parsed.error.message}`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
return parsed.data;
|
||||
} catch (error) {
|
||||
logLlmValidation(input, false);
|
||||
logLlmWarning("[llm:error]", input, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
This means real `generateJson` calls log start/response once inside `generateJson`, while tests that replace `generateJsonForValidation` still get start/response logs from `generateValidatedJson`.
|
||||
|
||||
- [ ] **Step 6: Run the client tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
npm test -- src/lib/llm/__tests__/client.test.ts
|
||||
```
|
||||
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git add src/lib/llm/client.ts src/lib/llm/__tests__/client.test.ts
|
||||
git commit -m "feat: log llm runtime responses"
|
||||
```
|
||||
|
||||
## Task 2: Strengthen Prompt Builders
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `src/lib/llm/prompts.ts`
|
||||
- Create: `src/lib/llm/__tests__/prompts.test.ts`
|
||||
|
||||
- [ ] **Step 1: Add failing prompt tests**
|
||||
|
||||
Create `src/lib/llm/__tests__/prompts.test.ts`:
|
||||
|
||||
```ts
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
ARTICLE_OPTIMIZER_SYSTEM_PROMPT,
|
||||
FACT_EXTRACTOR_SYSTEM_PROMPT,
|
||||
QUALITY_INSPECTOR_SYSTEM_PROMPT,
|
||||
TARGETED_REWRITER_SYSTEM_PROMPT,
|
||||
buildArticleOptimizerPrompt,
|
||||
buildFactExtractorPrompt,
|
||||
buildQualityInspectorPrompt,
|
||||
buildTargetedRewritePrompt,
|
||||
} from "../prompts";
|
||||
|
||||
const articleInput = {
|
||||
title: "探寻AIGC短视频培训选哪家,各品牌实力大比拼",
|
||||
body: "伟思德鲁管理咨询(深圳)有限公司提供AIGC短视频培训,文章提到客户案例和出海内容生产。",
|
||||
images: [{ type: "description" as const, content: "AIGC短视频工作流示意图" }],
|
||||
platform: "official_site" as const,
|
||||
user_instructions: "改成官网文章,避免虚构客户案例。",
|
||||
};
|
||||
|
||||
const factCard = {
|
||||
company_full_name: "伟思德鲁管理咨询(深圳)有限公司",
|
||||
company_short_names: ["伟思德鲁", "伟思德鲁管理咨询"],
|
||||
brand_names: ["伟思德鲁管理咨询"],
|
||||
product_names: ["AIGC短视频培训"],
|
||||
target_industry: "AIGC短视频培训",
|
||||
target_audience: "品牌商家、内容创作者、出海企业",
|
||||
experience_years: null,
|
||||
core_claims: ["提供AIGC短视频培训"],
|
||||
forbidden_claims: ["行业第一", "服务过500强客户"],
|
||||
image_topics: ["AIGC短视频工作流示意图"],
|
||||
uncertain_items: [],
|
||||
is_ready_for_optimization: true,
|
||||
confirmed_by_user: true,
|
||||
} as const;
|
||||
|
||||
describe("LLM prompt builders", () => {
|
||||
it("fact extraction prompt names customer risk categories and uncertainty rules", () => {
|
||||
const prompt = `${FACT_EXTRACTOR_SYSTEM_PROMPT}\n${buildFactExtractorPrompt(articleInput)}`;
|
||||
|
||||
expect(prompt).toContain("客户案例");
|
||||
expect(prompt).toContain("资质荣誉");
|
||||
expect(prompt).toContain("经验年限");
|
||||
expect(prompt).toContain("uncertain_items");
|
||||
expect(prompt).toContain("不要把营销夸张词当事实");
|
||||
expect(prompt).toContain("图片主题");
|
||||
});
|
||||
|
||||
it("article optimizer prompt includes platform templates and Chinese output rules", () => {
|
||||
const prompt = `${ARTICLE_OPTIMIZER_SYSTEM_PROMPT}\n${buildArticleOptimizerPrompt(articleInput, factCard)}`;
|
||||
|
||||
expect(prompt).toContain("official_site");
|
||||
expect(prompt).toContain("官网文章");
|
||||
expect(prompt).toContain("第一方品牌口吻");
|
||||
expect(prompt).toContain("输出中文标题");
|
||||
expect(prompt).toContain("禁止使用英文模板词");
|
||||
expect(prompt).toContain("实质性重写");
|
||||
expect(prompt).toContain("不得新增客户案例");
|
||||
expect(prompt).toContain("forbidden_claims");
|
||||
});
|
||||
|
||||
it("quality inspector prompt encodes fail and warn standards for all risk gates", () => {
|
||||
const prompt = `${QUALITY_INSPECTOR_SYSTEM_PROMPT}\n${buildQualityInspectorPrompt({
|
||||
article: {
|
||||
title: "伟思德鲁 AIGC短视频培训",
|
||||
summary: "官网文章摘要",
|
||||
body_markdown: "正文",
|
||||
image_suggestions: [],
|
||||
changed_sections: [],
|
||||
requires_user_confirmation: [],
|
||||
},
|
||||
factCard,
|
||||
platform: "official_site",
|
||||
deterministicChecks: [
|
||||
{
|
||||
rule_id: "hallucination_risk",
|
||||
status: "pass",
|
||||
evidence: "未发现",
|
||||
reason: "本地规则通过",
|
||||
suggested_fix: "",
|
||||
target_agent: null,
|
||||
},
|
||||
],
|
||||
})}`;
|
||||
|
||||
expect(prompt).toContain("不得把 deterministic fail 降级");
|
||||
expect(prompt).toContain("行业漂移");
|
||||
expect(prompt).toContain("公司名不一致");
|
||||
expect(prompt).toContain("未确认案例");
|
||||
expect(prompt).toContain("平台口吻严重错误");
|
||||
expect(prompt).toContain("warn 标准");
|
||||
});
|
||||
|
||||
it("targeted rewrite prompt gives rule-specific repair actions", () => {
|
||||
const prompt = `${TARGETED_REWRITER_SYSTEM_PROMPT}\n${buildTargetedRewritePrompt({
|
||||
article: {
|
||||
title: "Bad title!!!",
|
||||
summary: "摘要",
|
||||
body_markdown: "正文",
|
||||
image_suggestions: [],
|
||||
changed_sections: [],
|
||||
requires_user_confirmation: [],
|
||||
},
|
||||
factCard,
|
||||
failedChecks: [
|
||||
{
|
||||
rule_id: "title_quality",
|
||||
status: "fail",
|
||||
evidence: "Bad title!!!",
|
||||
reason: "标题问题",
|
||||
suggested_fix: "重写标题",
|
||||
target_agent: "title",
|
||||
},
|
||||
],
|
||||
})}`;
|
||||
|
||||
expect(prompt).toContain("industry_alignment");
|
||||
expect(prompt).toContain("company_name_integrity");
|
||||
expect(prompt).toContain("hallucination_risk");
|
||||
expect(prompt).toContain("claim_consistency");
|
||||
expect(prompt).toContain("title_quality");
|
||||
expect(prompt).toContain("body_quality");
|
||||
expect(prompt).toContain("不得新增事实");
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run the prompt tests to verify they fail**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
npm test -- src/lib/llm/__tests__/prompts.test.ts
|
||||
```
|
||||
|
||||
Expected: FAIL because the current prompts do not include the required platform templates and risk language.
|
||||
|
||||
- [ ] **Step 3: Replace `prompts.ts` with strengthened prompt builders**
|
||||
|
||||
Replace `src/lib/llm/prompts.ts` with:
|
||||
|
||||
```ts
|
||||
import type {
|
||||
ArticleInput,
|
||||
ConfirmedFactCard,
|
||||
OptimizedArticle,
|
||||
PublishPlatform,
|
||||
QaCheck,
|
||||
} from "../domain/types";
|
||||
|
||||
export const JSON_ONLY_PROMPT =
|
||||
"Return valid JSON only. Do not include markdown fences or commentary.";
|
||||
|
||||
const CUSTOMER_RISK_GUIDANCE = [
|
||||
"客户最担心的内容风险:行业漂移、公司全称/简称/品牌名不一致、图片主题与正文描述不匹配、官网文章出现第三方口吻、平台语气和文章类型不匹配、标题或正文语义不顺、虚构资质/年限/案例/能力、产品/服务/年限前后冲突。",
|
||||
"任何客户案例、资质荣誉、经验年限、服务能力、出海/多语种/合规能力、效果承诺和排名,都必须能从原文或已确认事实卡中找到明确依据。",
|
||||
].join(" ");
|
||||
|
||||
const PLATFORM_GUIDANCE: Record<PublishPlatform, string> = {
|
||||
official_site:
|
||||
"official_site / 官网文章:第一方品牌口吻,表达克制可信,可使用“我们/公司/品牌”视角;避免第三方评测腔、夸大宣传和无依据背书。",
|
||||
media_article:
|
||||
"media_article / 媒体稿:第三方客观报道口吻,突出事实、背景和行业价值;不要替品牌自夸,不制造新闻来源。",
|
||||
comparison_review:
|
||||
"comparison_review / 对比评测:对比分析口吻,只比较原文或事实卡中明确出现的信息;不得虚构竞品、排名、参数或测试结论。",
|
||||
recommendation_list:
|
||||
"recommendation_list / 推荐榜单:推荐榜单口吻,必须解释推荐依据;不得制造榜单排名、奖项或第三方认证。",
|
||||
};
|
||||
|
||||
function formatPlatformGuidance(selected: PublishPlatform) {
|
||||
return [
|
||||
"平台模板选项:",
|
||||
...Object.values(PLATFORM_GUIDANCE).map((item) => `- ${item}`),
|
||||
`当前目标平台:${selected}。必须优先遵守该平台模板。`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export const ARTICLE_OPTIMIZER_SYSTEM_PROMPT = [
|
||||
"你是中文 GEO 内容优化编辑。目标不是复述原文,而是在事实卡约束下进行实质性重写。",
|
||||
CUSTOMER_RISK_GUIDANCE,
|
||||
"硬性规则:只能使用 confirmed fact card 和 source article 中明确出现的事实;不得新增客户案例、数字、资质、排名、奖项、服务能力、效果承诺;输出中文标题,禁止使用英文模板词如 Guide、Best、Top。",
|
||||
"必须修复原文中的病句、断裂句、错别字、逻辑跳跃和段落结构问题。",
|
||||
JSON_ONLY_PROMPT,
|
||||
].join(" ");
|
||||
|
||||
export const QUALITY_INSPECTOR_SYSTEM_PROMPT = [
|
||||
"你是质量门禁审核器。逐项检查 deterministicChecks 中的每个 rule_id。",
|
||||
CUSTOMER_RISK_GUIDANCE,
|
||||
"不得把 deterministic fail 降级。每项必须给出 status、evidence、reason、suggested_fix、target_agent。",
|
||||
JSON_ONLY_PROMPT,
|
||||
].join(" ");
|
||||
|
||||
export const FACT_EXTRACTOR_SYSTEM_PROMPT = [
|
||||
"你是事实卡提取器。只从原文和图片描述中抽取事实,不推测、不补全。",
|
||||
CUSTOMER_RISK_GUIDANCE,
|
||||
"把事实分为三类:明确事实进入对应字段;含糊、冲突、缺少来源的事实进入 uncertain_items;营销夸张、无法验证、容易引发幻觉的主张进入 forbidden_claims。",
|
||||
"特别检查:公司全称、简称、品牌名、产品/服务名、目标行业、目标受众、经验年限、客户案例、资质荣誉、服务能力、地域/出海能力、图片主题。不要把营销夸张词当事实。",
|
||||
JSON_ONLY_PROMPT,
|
||||
].join(" ");
|
||||
|
||||
export const TARGETED_REWRITER_SYSTEM_PROMPT = [
|
||||
"你是定向修复器。只修复 failedChecks 指出的失败项,不整篇重写。",
|
||||
CUSTOMER_RISK_GUIDANCE,
|
||||
"不得新增事实。无法修复或需要客户确认的内容放入 requires_user_confirmation。",
|
||||
JSON_ONLY_PROMPT,
|
||||
].join(" ");
|
||||
|
||||
export function buildFactExtractorPrompt(input: ArticleInput) {
|
||||
return [
|
||||
"Return a CandidateFactCard JSON object with these exact keys:",
|
||||
"company_full_name, company_short_names, brand_names, product_names, target_industry, target_audience, experience_years, core_claims, forbidden_claims, image_topics, uncertain_items.",
|
||||
"Do not include confirmed_by_user.",
|
||||
"",
|
||||
"字段要求:",
|
||||
"- core_claims 只放原文明示、可作为后续优化硬约束的事实。",
|
||||
"- forbidden_claims 放行业第一、最强、知名客户、显著提升、保证转化等无法验证或高风险主张。",
|
||||
"- uncertain_items 放客户案例、资质荣誉、经验年限、产品能力、出海能力等证据不足或前后冲突的内容。",
|
||||
"- image_topics 必须来自图片描述或图片链接文本;没有图片就返回空数组。",
|
||||
"- target_industry 必须是文章主行业,避免被局部案例带偏。",
|
||||
"",
|
||||
"Article input:",
|
||||
JSON.stringify(input, null, 2),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function buildArticleOptimizerPrompt(
|
||||
input: ArticleInput,
|
||||
factCard: ConfirmedFactCard,
|
||||
) {
|
||||
return [
|
||||
"Return an OptimizedArticle JSON object with these exact keys:",
|
||||
"title, summary, body_markdown, image_suggestions, changed_sections, requires_user_confirmation.",
|
||||
"",
|
||||
formatPlatformGuidance(input.platform),
|
||||
"",
|
||||
"输出要求:",
|
||||
"- 输出中文标题,禁止使用英文模板词如 Guide、Best、Top。",
|
||||
"- 进行实质性重写:重组标题、摘要、段落顺序、长句、病句和逻辑连接,而不是只在原文前后拼接内容。",
|
||||
"- body_markdown 使用清晰的 Markdown 层级;标题层级要规范,中文标题后保留空格,例如 `## 标题`。",
|
||||
"- 如果用户要求包含未确认事实,正文不写入,放入 requires_user_confirmation。",
|
||||
"- 必须保留事实卡确认的公司全称、目标行业、目标受众和核心事实。",
|
||||
"- 必须删除或弱化 factCard.forbidden_claims 中的主张。",
|
||||
"- 不得新增客户案例、数字、资质、排名、奖项、服务能力、效果承诺。",
|
||||
"- image_suggestions 必须基于 factCard.image_topics 或原始 images;没有图片主题时返回空数组。",
|
||||
"",
|
||||
"Confirmed fact card:",
|
||||
JSON.stringify(factCard, null, 2),
|
||||
"",
|
||||
"Article input:",
|
||||
JSON.stringify(input, null, 2),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function buildQualityInspectorPrompt(input: {
|
||||
article: OptimizedArticle;
|
||||
factCard: ConfirmedFactCard;
|
||||
platform: PublishPlatform;
|
||||
deterministicChecks: QaCheck[];
|
||||
}) {
|
||||
return [
|
||||
"Return a JSON object with a checks array.",
|
||||
"Each check must include rule_id, status, evidence, reason, suggested_fix, and target_agent.",
|
||||
"Only use rule_id values already present in deterministicChecks.",
|
||||
"不得把 deterministic fail 降级。",
|
||||
"",
|
||||
formatPlatformGuidance(input.platform),
|
||||
"",
|
||||
"fail 标准:行业漂移、公司名不一致、事实卡外新增数字/客户/资质/案例、产品服务前后冲突、平台口吻严重错误、标题明显病句。",
|
||||
"warn 标准:图片证据不足、句子过长、表达可读性一般、平台适配轻微不足。",
|
||||
"target_agent 只能使用 title、body、image、fact_card 或 null。",
|
||||
"",
|
||||
"Confirmed fact card:",
|
||||
JSON.stringify(input.factCard, null, 2),
|
||||
"",
|
||||
"Optimized article:",
|
||||
JSON.stringify(input.article, null, 2),
|
||||
"",
|
||||
"Deterministic checks:",
|
||||
JSON.stringify(input.deterministicChecks, null, 2),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function buildTargetedRewritePrompt(input: {
|
||||
article: OptimizedArticle;
|
||||
factCard: ConfirmedFactCard;
|
||||
failedChecks: QaCheck[];
|
||||
}) {
|
||||
return [
|
||||
"Return an OptimizedArticle JSON object.",
|
||||
"Rewrite only the fields needed for failedChecks.",
|
||||
"",
|
||||
"按 rule_id 执行修复动作:",
|
||||
"- industry_alignment:删除偏离行业的段落,改回 factCard.target_industry。",
|
||||
"- company_name_integrity:统一公司全称和简称。",
|
||||
"- hallucination_risk:删除所有未确认数字、客户案例、资质、奖项、排名、效果承诺。",
|
||||
"- claim_consistency:统一年限、产品名、服务能力和核心主张。",
|
||||
"- title_quality:生成自然中文标题,禁止英文模板词。",
|
||||
"- body_quality:拆分长句,修复病句和断裂表达。",
|
||||
"- voice_consistency / platform_fit:改成目标平台对应口吻。",
|
||||
"- image_text_match:只补充图片建议或人工确认项,不虚构图片内容。",
|
||||
"不得新增事实。无法修复的内容放入 requires_user_confirmation。",
|
||||
"",
|
||||
"Confirmed fact card:",
|
||||
JSON.stringify(input.factCard, null, 2),
|
||||
"",
|
||||
"Current optimized article:",
|
||||
JSON.stringify(input.article, null, 2),
|
||||
"",
|
||||
"Failed checks:",
|
||||
JSON.stringify(input.failedChecks, null, 2),
|
||||
].join("\n");
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run prompt tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
npm test -- src/lib/llm/__tests__/prompts.test.ts
|
||||
```
|
||||
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/lib/llm/prompts.ts src/lib/llm/__tests__/prompts.test.ts
|
||||
git commit -m "feat: strengthen llm workflow prompts"
|
||||
```
|
||||
|
||||
## Task 3: Pass Explicit Task Names From Workflow Nodes
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `src/lib/workflow/fact-extractor.ts`
|
||||
- Modify: `src/lib/workflow/article-optimizer.ts`
|
||||
- Modify: `src/lib/workflow/quality-inspector.ts`
|
||||
- Modify: `src/lib/workflow/targeted-rewriter.ts`
|
||||
- Modify: `src/lib/workflow/__tests__/llm-integration.test.ts`
|
||||
|
||||
- [ ] **Step 1: Add failing task-name assertions**
|
||||
|
||||
In `src/lib/workflow/__tests__/llm-integration.test.ts`, after each existing `expect(llmMocks.generateValidatedJson).toHaveBeenCalledOnce();`, add the matching assertion:
|
||||
|
||||
```ts
|
||||
expect(llmMocks.generateValidatedJson).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ task: "fact_extractor" }),
|
||||
);
|
||||
```
|
||||
|
||||
For the article optimizer test, use:
|
||||
|
||||
```ts
|
||||
expect(llmMocks.generateValidatedJson).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ task: "article_optimizer" }),
|
||||
);
|
||||
```
|
||||
|
||||
For the targeted rewrite test, use:
|
||||
|
||||
```ts
|
||||
expect(llmMocks.generateValidatedJson).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ task: "targeted_rewriter" }),
|
||||
);
|
||||
```
|
||||
|
||||
At the end of the `"uses LLM quality checks to enrich non-failing deterministic checks"` test, add:
|
||||
|
||||
```ts
|
||||
expect(llmMocks.generateValidatedJson).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ task: "quality_inspector" }),
|
||||
);
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run the integration test to verify it fails**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
npm test -- src/lib/workflow/__tests__/llm-integration.test.ts
|
||||
```
|
||||
|
||||
Expected: FAIL because workflow nodes do not pass task names yet.
|
||||
|
||||
- [ ] **Step 3: Add task names to workflow LLM calls**
|
||||
|
||||
In `src/lib/workflow/fact-extractor.ts`, change:
|
||||
|
||||
```ts
|
||||
const llmCard = await generateValidatedJson({
|
||||
schema: candidateFactCardSchema,
|
||||
```
|
||||
|
||||
to:
|
||||
|
||||
```ts
|
||||
const llmCard = await generateValidatedJson({
|
||||
schema: candidateFactCardSchema,
|
||||
task: "fact_extractor",
|
||||
```
|
||||
|
||||
In `src/lib/workflow/article-optimizer.ts`, change:
|
||||
|
||||
```ts
|
||||
const llmArticle = await generateValidatedJson({
|
||||
schema: optimizedArticleSchema,
|
||||
```
|
||||
|
||||
to:
|
||||
|
||||
```ts
|
||||
const llmArticle = await generateValidatedJson({
|
||||
schema: optimizedArticleSchema,
|
||||
task: "article_optimizer",
|
||||
```
|
||||
|
||||
In `src/lib/workflow/quality-inspector.ts`, change:
|
||||
|
||||
```ts
|
||||
const llmPatch = await generateValidatedJson({
|
||||
schema: llmQaPatchSchema,
|
||||
```
|
||||
|
||||
to:
|
||||
|
||||
```ts
|
||||
const llmPatch = await generateValidatedJson({
|
||||
schema: llmQaPatchSchema,
|
||||
task: "quality_inspector",
|
||||
```
|
||||
|
||||
In `src/lib/workflow/targeted-rewriter.ts`, change:
|
||||
|
||||
```ts
|
||||
const llmArticle = await generateValidatedJson({
|
||||
schema: optimizedArticleSchema,
|
||||
```
|
||||
|
||||
to:
|
||||
|
||||
```ts
|
||||
const llmArticle = await generateValidatedJson({
|
||||
schema: optimizedArticleSchema,
|
||||
task: "targeted_rewriter",
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run integration tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
npm test -- src/lib/workflow/__tests__/llm-integration.test.ts
|
||||
```
|
||||
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/lib/workflow/fact-extractor.ts src/lib/workflow/article-optimizer.ts src/lib/workflow/quality-inspector.ts src/lib/workflow/targeted-rewriter.ts src/lib/workflow/__tests__/llm-integration.test.ts
|
||||
git commit -m "feat: tag llm workflow calls"
|
||||
```
|
||||
|
||||
## Task 4: Document LLM Logs And Prompt Scope
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `README.md`
|
||||
|
||||
- [ ] **Step 1: Update README with LLM logging behavior**
|
||||
|
||||
Add this section after the existing Environment section:
|
||||
|
||||
```md
|
||||
## LLM Runtime Logs
|
||||
|
||||
Every workflow LLM call logs to server stdout/stderr:
|
||||
|
||||
- `[llm:start]` with provider, model, and task name.
|
||||
- `[llm:response]` with task name, duration, and a raw response snippet.
|
||||
- `[llm:validated]` with schema validation result.
|
||||
- `[llm:validation-error]` when Zod rejects the response.
|
||||
- `[llm:error]` when the provider call or JSON parsing fails.
|
||||
|
||||
Raw response logging is intentionally enabled for operational visibility. It is
|
||||
truncated to 4000 characters by default. Set `LLM_LOG_RAW_LIMIT=0` to suppress
|
||||
raw response snippets, or set a larger number when diagnosing model output.
|
||||
|
||||
Workflow task names are:
|
||||
|
||||
- `fact_extractor`
|
||||
- `article_optimizer`
|
||||
- `quality_inspector`
|
||||
- `targeted_rewriter`
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run full verification**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
npm test
|
||||
npm run build
|
||||
```
|
||||
|
||||
Expected: both commands PASS.
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add README.md
|
||||
git commit -m "docs: document llm runtime logs"
|
||||
```
|
||||
|
||||
## Task 5: Manual Smoke Test With Logs
|
||||
|
||||
**Files:**
|
||||
|
||||
- Verify: `src/lib/llm/client.ts`
|
||||
- Verify: `src/lib/llm/prompts.ts`
|
||||
- Verify: running dev server output
|
||||
|
||||
- [ ] **Step 1: Start dev server**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
npm run dev
|
||||
```
|
||||
|
||||
Expected: server prints `Local: http://localhost:3000`.
|
||||
|
||||
- [ ] **Step 2: Trigger analysis with curl**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
curl -s -X POST http://localhost:3000/api/jobs \
|
||||
-H 'content-type: application/json' \
|
||||
-H "x-api-key: ${API_ACCESS_KEY:-local-dev-key}" \
|
||||
--data '{
|
||||
"title":"伟思德鲁管理咨询(深圳)有限公司 AIGC短视频培训官网文章",
|
||||
"body":"伟思德鲁管理咨询(深圳)有限公司面向品牌商家和出海企业提供AIGC短视频培训服务,帮助企业解决内容工业化生产、品牌视觉统一和全球化传播问题。",
|
||||
"image_lines":"AIGC短视频工作流示意图",
|
||||
"platform":"official_site",
|
||||
"user_instructions":"改成官网文章,保持事实准确,不新增客户案例。"
|
||||
}' | jq '.candidateFactCard.company_full_name'
|
||||
```
|
||||
|
||||
Expected: command prints `"伟思德鲁管理咨询(深圳)有限公司"` or a non-empty company name. Server logs include:
|
||||
|
||||
```text
|
||||
[llm:start] provider=deepseek ... task=fact_extractor
|
||||
[llm:response] task=fact_extractor ...
|
||||
[llm:validated] task=fact_extractor ok=true
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Stop dev server**
|
||||
|
||||
Press `Ctrl-C`.
|
||||
|
||||
## Self-Review
|
||||
|
||||
- Spec coverage: The plan covers permanent LLM response logging, prompt strengthening for all four LLM workflow nodes, task labels for observability, documentation, and manual smoke testing.
|
||||
- Placeholder scan: The plan includes exact file paths, concrete test code, concrete implementation snippets, commands, and expected outcomes. It contains no unresolved placeholders.
|
||||
- Type consistency: `LlmTaskName`, `task`, prompt builder names, existing schema names, and workflow function names match the current codebase.
|
||||
+64
-1
@@ -40,5 +40,68 @@
|
||||
"bucket_name": "geo-agent-article-optimizer-local"
|
||||
}
|
||||
],
|
||||
"env": {}
|
||||
"env": {
|
||||
"staging": {
|
||||
"name": "geo-agent-article-optimizer-staging",
|
||||
"services": [
|
||||
{
|
||||
"binding": "WORKER_SELF_REFERENCE",
|
||||
"service": "geo-agent-article-optimizer-staging"
|
||||
}
|
||||
],
|
||||
"vars": {
|
||||
"APP_RUNTIME": "cloudflare",
|
||||
"LLM_PROVIDER": "deepseek",
|
||||
"DEEPSEEK_BASE_URL": "https://api.deepseek.com",
|
||||
"DEEPSEEK_MODEL": "deepseek-v4-pro",
|
||||
"DEEPSEEK_THINKING": "disabled",
|
||||
"OPENAI_MODEL": "gpt-4.1-mini"
|
||||
},
|
||||
"d1_databases": [
|
||||
{
|
||||
"binding": "DB",
|
||||
"database_name": "geo-agent-article-optimizer-staging",
|
||||
"database_id": "0162e94c-9f71-472c-ae78-13786578a2ac",
|
||||
"migrations_dir": "migrations"
|
||||
}
|
||||
],
|
||||
"r2_buckets": [
|
||||
{
|
||||
"binding": "EXPORT_BUCKET",
|
||||
"bucket_name": "geo-agent-article-optimizer-staging"
|
||||
}
|
||||
]
|
||||
},
|
||||
"production": {
|
||||
"name": "geo-agent-article-optimizer-production",
|
||||
"services": [
|
||||
{
|
||||
"binding": "WORKER_SELF_REFERENCE",
|
||||
"service": "geo-agent-article-optimizer-production"
|
||||
}
|
||||
],
|
||||
"vars": {
|
||||
"APP_RUNTIME": "cloudflare",
|
||||
"LLM_PROVIDER": "deepseek",
|
||||
"DEEPSEEK_BASE_URL": "https://api.deepseek.com",
|
||||
"DEEPSEEK_MODEL": "deepseek-v4-pro",
|
||||
"DEEPSEEK_THINKING": "disabled",
|
||||
"OPENAI_MODEL": "gpt-4.1-mini"
|
||||
},
|
||||
"d1_databases": [
|
||||
{
|
||||
"binding": "DB",
|
||||
"database_name": "geo-agent-article-optimizer-production",
|
||||
"database_id": "8161ceeb-76c6-40fd-9f38-32ec7ab8f2b1",
|
||||
"migrations_dir": "migrations"
|
||||
}
|
||||
],
|
||||
"r2_buckets": [
|
||||
{
|
||||
"binding": "EXPORT_BUCKET",
|
||||
"bucket_name": "geo-agent-article-optimizer-production"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user