fix(feishu-sync): harden AI timeout recovery

Noveris-Teams · Noveris-Teams · commit 4e6d5e71c5cc · 2026-03-09T09:29:27.000+08:00
diff --git a/.github/workflows/feishu-sync.yml b/.github/workflows/feishu-sync.yml
@@ -32,6 +32,9 @@ jobs:
       FEISHU_AI_TRANSLATE_ZH_TO_EN: ${{ vars.FEISHU_AI_TRANSLATE_ZH_TO_EN || 'true' }}
       FEISHU_AI_STRICT: ${{ vars.FEISHU_AI_STRICT || 'true' }}
       FEISHU_AI_TEMPERATURE: ${{ vars.FEISHU_AI_TEMPERATURE }}
+      FEISHU_AI_TIMEOUT_MS: ${{ vars.FEISHU_AI_TIMEOUT_MS || '180000' }}
+      FEISHU_AI_RETRY_BACKOFF_MS: ${{ vars.FEISHU_AI_RETRY_BACKOFF_MS || '1500' }}
+      FEISHU_AI_FAIL_ON_TRANSLATION_ERROR: ${{ vars.FEISHU_AI_FAIL_ON_TRANSLATION_ERROR || 'false' }}
       FEISHU_AI_MAX_TOKENS: ${{ vars.FEISHU_AI_MAX_TOKENS }}
       FEISHU_AI_TAG_MAX: ${{ vars.FEISHU_AI_TAG_MAX }}
       FEISHU_AI_KEYWORD_MAX: ${{ vars.FEISHU_AI_KEYWORD_MAX }}
diff --git a/README.md b/README.md
@@ -225,6 +225,8 @@ Build output is deployed to GitHub Pages via `actions/deploy-pages`.
 - Feishu sync exits early:
   - verify `FEISHU_APP_ID`, `FEISHU_APP_SECRET`, and folder token/name
   - verify `TRANSLATE_API_KEY` when AI translation is enabled
+  - tune `FEISHU_AI_TIMEOUT_MS` if the provider is slow for long documents
+  - keep `FEISHU_AI_FAIL_ON_TRANSLATION_ERROR=false` to let nightly sync continue with zh content and preserved stale en output on retryable AI failures
 
 ## License
 
diff --git a/README.zh-CN.md b/README.zh-CN.md
@@ -225,6 +225,8 @@ npm run feishu:sync
 - 飞书同步中断：
   - 检查 `FEISHU_APP_ID`、`FEISHU_APP_SECRET`、目录 token/name
   - 开启 AI 翻译时检查 `TRANSLATE_API_KEY`
+  - 长文档遇到模型侧慢响应时，可调大 `FEISHU_AI_TIMEOUT_MS`
+  - 保持 `FEISHU_AI_FAIL_ON_TRANSLATION_ERROR=false`，可在 AI 可重试异常时继续同步中文并保留旧英文稿，不再让夜间任务整批失败
 
 ## License
 
diff --git a/scripts/feishu-cms-bridge/README.md b/scripts/feishu-cms-bridge/README.md
@@ -20,6 +20,9 @@ Optional:
 - `FEISHU_AI_ENABLED` (default: `true`)
 - `FEISHU_AI_TRANSLATE_ZH_TO_EN` (default: `true`)
 - `FEISHU_AI_STRICT` (default: `true`)
+- `FEISHU_AI_TIMEOUT_MS` (default: `180000`)
+- `FEISHU_AI_RETRY_BACKOFF_MS` (default: `1500`)
+- `FEISHU_AI_FAIL_ON_TRANSLATION_ERROR` (default: `false`)
 - `TRANSLATE_API_KEY` (or `FEISHU_AI_API_KEY`)
 - `TRANSLATE_MODEL` / `TRANSLATE_BASE_URL`
 - taxonomy config file: `config/blog-taxonomy.json`
@@ -53,6 +56,8 @@ Each category's default tags and zh/en display names are defined in
 - AI zh->en pipeline is constrained to allowed categories
 
 When `FEISHU_AI_TRANSLATE_ZH_TO_EN=true`, `zh` docs will auto-generate `en` posts via AI.
+If the AI provider hits a retryable timeout or transient error, the bridge now syncs the `zh`
+post first and preserves the previous managed `en` post when one already exists.
 
 ## Run
 
diff --git a/scripts/feishu-cms-bridge/ai/content-pipeline.mjs b/scripts/feishu-cms-bridge/ai/content-pipeline.mjs
@@ -1,3 +1,4 @@
+import { setTimeout as sleep } from 'node:timers/promises';
 import { slugify } from '../transform/slug.mjs';
 
 function normalizeModelResponse(raw) {
@@ -74,6 +75,46 @@ function getFinishReason(payload) {
   return compactText(payload?.choices?.[0]?.finish_reason || '');
 }
 
+function createAiError(message, details = {}) {
+  const error = new Error(message);
+  Object.assign(error, details);
+  return error;
+}
+
+function isRetryableStatus(status) {
+  return status === 408 || status === 409 || status === 425 || status === 429 || status >= 500;
+}
+
+function buildAttemptPlan(options) {
+  const baseTimeoutMs = Number(options.aiTimeoutMs) > 0 ? Number(options.aiTimeoutMs) : 180000;
+  const baseMaxTokens = Number(options.aiMaxTokens) > 0 ? Number(options.aiMaxTokens) : 8192;
+  const expandedMaxTokens = Math.max(baseMaxTokens, 12288);
+  const timeoutCandidates = [...new Set([baseTimeoutMs, Math.round(baseTimeoutMs * 1.5), Math.round(baseTimeoutMs * 2)])];
+
+  return [
+    {
+      forceJsonObject: true,
+      maxTokens: baseMaxTokens,
+      timeoutMs: timeoutCandidates[0]
+    },
+    {
+      forceJsonObject: false,
+      maxTokens: baseMaxTokens,
+      timeoutMs: timeoutCandidates[Math.min(1, timeoutCandidates.length - 1)]
+    },
+    {
+      forceJsonObject: false,
+      maxTokens: expandedMaxTokens,
+      timeoutMs: timeoutCandidates[timeoutCandidates.length - 1]
+    }
+  ];
+}
+
+function getRetryDelayMs(options, attemptIndex) {
+  const baseDelayMs = Number(options.aiRetryBackoffMs) > 0 ? Number(options.aiRetryBackoffMs) : 1500;
+  return Math.min(baseDelayMs * 2 ** attemptIndex, 10000);
+}
+
 function buildCategoryGuide(options) {
   if (!options?.taxonomy || typeof options.taxonomy.buildPromptCategoryGuide !== 'function') {
     return [];
@@ -227,20 +268,14 @@ export async function runAiContentPipeline(input, options) {
 
   const endpoint = `${String(options.aiBaseUrl || '').replace(/\/$/, '')}/chat/completions`;
   const prompt = buildPrompt(input, options);
-
-  const requestTimeoutMs = Number(options.aiTimeoutMs) > 0 ? Number(options.aiTimeoutMs) : 120000;
-  const baseMaxTokens = Number(options.aiMaxTokens) > 0 ? Number(options.aiMaxTokens) : 8192;
-  const tokenCandidates = [...new Set([baseMaxTokens, Math.max(baseMaxTokens, 12288)])];
-  const attempts = tokenCandidates.flatMap((maxTokens) => [
-    { maxTokens, forceJsonObject: true },
-    { maxTokens, forceJsonObject: false }
-  ]);
+  const title = compactText(input?.title, `document ${String(input?.token || '').slice(0, 8)}`);
+  const attempts = buildAttemptPlan(options);
 
   let lastError = null;
   let lastFinishReason = '';
   let lastPreview = '';
 
-  for (const attempt of attempts) {
+  for (const [attemptIndex, attempt] of attempts.entries()) {
     const requestBody = {
       model: options.aiModel,
       temperature: options.aiTemperature,
@@ -264,7 +299,7 @@ export async function runAiContentPipeline(input, options) {
 
     try {
       const controller = new AbortController();
-      const timeout = setTimeout(() => controller.abort(), requestTimeoutMs);
+      const timeout = setTimeout(() => controller.abort(), attempt.timeoutMs);
       let response;
       try {
         response = await fetch(endpoint, {
@@ -282,7 +317,11 @@ export async function runAiContentPipeline(input, options) {
 
       if (!response.ok) {
         const errorText = await response.text();
-        throw new Error(`AI pipeline request failed: ${response.status} ${errorText.slice(0, 400)}`);
+        throw createAiError(`AI pipeline request failed: ${response.status} ${errorText.slice(0, 400)}`, {
+          code: 'AI_HTTP_ERROR',
+          retryable: isRetryableStatus(response.status),
+          status: response.status
+        });
       }
 
       const payload = await response.json();
@@ -305,15 +344,35 @@ export async function runAiContentPipeline(input, options) {
       return normalized;
     } catch (error) {
       if (error?.name === 'AbortError') {
-        lastError = new Error(`AI pipeline request timed out after ${requestTimeoutMs}ms`);
-        continue;
+        lastError = createAiError(`AI pipeline request timed out after ${attempt.timeoutMs}ms`, {
+          code: 'AI_TIMEOUT',
+          retryable: true,
+          timeoutMs: attempt.timeoutMs
+        });
+      } else {
+        lastError = error;
+      }
+
+      const canRetry = attemptIndex < attempts.length - 1 && lastError?.retryable !== false;
+      if (!canRetry) {
+        break;
       }
-      lastError = error;
+
+      const delayMs = getRetryDelayMs(options, attemptIndex);
+      console.warn(
+        `[feishu-sync] AI retry ${attemptIndex + 2}/${attempts.length} for "${title}" after ${lastError.message} (wait ${delayMs}ms)`
+      );
+      await sleep(delayMs);
     }
   }
 
   const finishReasonPart = lastFinishReason ? ` finish_reason=${lastFinishReason}.` : '';
   const previewPart = lastPreview ? ` preview=${JSON.stringify(lastPreview)}.` : '';
   const errorMessage = lastError?.message || 'AI pipeline failed after retries';
-  throw new Error(`${errorMessage}.${finishReasonPart}${previewPart}`);
+  throw createAiError(`${errorMessage}.${finishReasonPart}${previewPart}`, {
+    code: lastError?.code || 'AI_PIPELINE_FAILED',
+    retryable: lastError?.retryable !== false,
+    status: lastError?.status,
+    timeoutMs: lastError?.timeoutMs
+  });
 }
diff --git a/scripts/feishu-cms-bridge/config.mjs b/scripts/feishu-cms-bridge/config.mjs
@@ -71,7 +71,9 @@ export const config = {
     'FEISHU_AI_TEMPERATURE',
     readNumber('TRANSLATE_TEMPERATURE', Number(translationDefaults.temperature || 0.2))
   ),
-  aiTimeoutMs: readNumber('FEISHU_AI_TIMEOUT_MS', 120000),
+  aiTimeoutMs: readNumber('FEISHU_AI_TIMEOUT_MS', 180000),
+  aiRetryBackoffMs: readNumber('FEISHU_AI_RETRY_BACKOFF_MS', 1500),
+  aiFailOnTranslationError: readBool('FEISHU_AI_FAIL_ON_TRANSLATION_ERROR', false),
   aiMaxTokens: readNumber(
     'FEISHU_AI_MAX_TOKENS',
     readNumber('TRANSLATE_MAX_TOKENS', Number(translationDefaults.maxTokens || 8192))
diff --git a/scripts/feishu-cms-bridge/index.mjs b/scripts/feishu-cms-bridge/index.mjs
@@ -52,9 +52,11 @@ async function main() {
     skipped: 0,
     changed: 0,
     unchanged: 0,
+    degraded: 0,
     failed: 0,
     removed: 0,
-    errors: []
+    errors: [],
+    warnings: []
   };
 
   for (const sourceDoc of docs) {
@@ -79,12 +81,18 @@ async function main() {
         aiModel: config.aiModel,
         aiTemperature: config.aiTemperature,
         aiTimeoutMs: config.aiTimeoutMs,
+        aiRetryBackoffMs: config.aiRetryBackoffMs,
+        aiFailOnTranslationError: config.aiFailOnTranslationError,
         aiMaxTokens: config.aiMaxTokens,
         aiTagMax: config.aiTagMax,
         aiKeywordMax: config.aiKeywordMax,
         aiProtectedTerms: config.aiProtectedTerms,
         aiTermStyleRules: config.aiTermStyleRules,
-        taxonomy: config.taxonomy
+        taxonomy: config.taxonomy,
+        existingOutputs: {
+          zh: previousState.docs[sourceDoc.token] || null,
+          en: previousState.docs[`${sourceDoc.token}::en`] || null
+        }
       });
 
       const outputs = Array.isArray(result.outputs) && result.outputs.length > 0 ? result.outputs : [result];
@@ -109,6 +117,11 @@ async function main() {
       } else {
         summary.unchanged += 1;
       }
+
+      if (Array.isArray(result.warnings) && result.warnings.length > 0) {
+        summary.degraded += 1;
+        summary.warnings.push(...result.warnings);
+      }
     } catch (error) {
       summary.failed += 1;
       summary.errors.push({ token: sourceDoc.token, title: sourceDoc.title, message: error.message });
@@ -127,13 +140,17 @@ async function main() {
   }
 
   console.log(
-    `[feishu-sync] done: total=${summary.total}, skipped=${summary.skipped}, changed=${summary.changed}, unchanged=${summary.unchanged}, removed=${summary.removed}, failed=${summary.failed}`
+    `[feishu-sync] done: total=${summary.total}, skipped=${summary.skipped}, changed=${summary.changed}, unchanged=${summary.unchanged}, degraded=${summary.degraded}, removed=${summary.removed}, failed=${summary.failed}`
   );
 
   if (summary.failed > 0) {
     throw new Error(`[feishu-sync] ${summary.failed} document(s) failed`);
   }
 
+  if (summary.degraded > 0) {
+    console.warn(`[feishu-sync] completed with ${summary.degraded} degraded document(s); zh content was synced and stale en content was preserved when available.`);
+  }
+
   if (config.autoPush) {
     const pushed = commitAndPushIfNeeded(config, summary);
     if (pushed) {
diff --git a/scripts/feishu-cms-bridge/sync/doc-syncer.mjs b/scripts/feishu-cms-bridge/sync/doc-syncer.mjs
@@ -40,6 +40,16 @@ function compactText(value, fallback = '') {
   return normalized || fallback;
 }
 
+async function pathExists(targetPath) {
+  if (!targetPath) return false;
+  try {
+    await fs.access(targetPath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 function normalizeList(input, max = 12) {
   const values = Array.isArray(input)
     ? input
@@ -220,6 +230,29 @@ async function resolveManagedOutputPath({ blogContentDir, lang, preferredSlug, t
   };
 }
 
+async function buildPreservedOutput(existingEntry, stateKey) {
+  if (!existingEntry?.contentPath) {
+    return null;
+  }
+
+  const contentExists = await pathExists(existingEntry.contentPath);
+  if (!contentExists) {
+    return null;
+  }
+
+  return {
+    stateKey,
+    token: existingEntry.token,
+    lang: existingEntry.lang,
+    slug: existingEntry.slug,
+    title: existingEntry.title,
+    changed: false,
+    contentPath: existingEntry.contentPath,
+    assetDir: existingEntry.assetDir || null,
+    preserved: true
+  };
+}
+
 export async function syncSingleDocument(client, sourceDoc, context) {
   const blocks = await fetchDocBlocks(client, sourceDoc.token);
   if (!Array.isArray(blocks) || blocks.length === 0) {
@@ -233,7 +266,9 @@ export async function syncSingleDocument(client, sourceDoc, context) {
     blocks
   };
 
-  const renderer = new MarkdownRenderer(docPayload);
+  const MarkdownRendererImpl = context.markdownRenderer || MarkdownRenderer;
+  const aiRunner = context.aiRunner || runAiContentPipeline;
+  const renderer = new MarkdownRendererImpl(docPayload);
   const rawMarkdown = renderer.parse();
   const docMeta = renderer.meta && typeof renderer.meta === 'object' ? renderer.meta : {};
   const fileTokens = renderer.fileTokens || {};
@@ -284,6 +319,7 @@ export async function syncSingleDocument(client, sourceDoc, context) {
 
   const outputs = [];
   let aiResult = null;
+  const warnings = [];
 
   const shouldTranslateToEn = Boolean(context.aiEnabled && context.aiTranslateZhToEn && lang === 'zh');
   if (shouldTranslateToEn) {
@@ -293,18 +329,29 @@ export async function syncSingleDocument(client, sourceDoc, context) {
       }
       console.warn(`[feishu-sync] AI disabled for "${title}" because API key is empty.`);
     } else {
-      aiResult = await runAiContentPipeline(
-        {
-          token: sourceDoc.token,
-          title,
-          description,
-          category,
-          tags,
-          slug,
-          markdownBody
-        },
-        context
-      );
+      try {
+        aiResult = await aiRunner(
+          {
+            token: sourceDoc.token,
+            title,
+            description,
+            category,
+            tags,
+            slug,
+            markdownBody
+          },
+          context
+        );
+      } catch (error) {
+        const canDowngrade = error?.retryable !== false && !context.aiFailOnTranslationError;
+        if (!canDowngrade) {
+          throw error;
+        }
+
+        const fallbackMessage = `[feishu-sync] AI translation degraded for "${title}": ${error.message}`;
+        warnings.push(fallbackMessage);
+        console.warn(fallbackMessage);
+      }
     }
   }
 
@@ -403,6 +450,18 @@ export async function syncSingleDocument(client, sourceDoc, context) {
       contentPath: enOutputPath,
       assetDir: null
     });
+  } else if (shouldTranslateToEn) {
+    const preservedEn = await buildPreservedOutput(context?.existingOutputs?.en, `${sourceDoc.token}::en`);
+    if (preservedEn) {
+      outputs.push(preservedEn);
+      const preserveMessage = `[feishu-sync] preserved existing en/${preservedEn.slug} for "${title}" until AI translation recovers.`;
+      warnings.push(preserveMessage);
+      console.warn(preserveMessage);
+    } else if (warnings.length > 0) {
+      const zhOnlyMessage = `[feishu-sync] no existing English post to preserve for "${title}", synced zh only.`;
+      warnings.push(zhOnlyMessage);
+      console.warn(zhOnlyMessage);
+    }
   }
 
   const changed = outputs.some((entry) => entry.changed);
@@ -415,6 +474,7 @@ export async function syncSingleDocument(client, sourceDoc, context) {
     changed,
     contentPath: sourceOutputPath,
     assetDir,
-    outputs
+    outputs,
+    warnings
   };
 }
