Parcourir la source

fix(provider): claude-code resume 卡死不再杀 run — 重试 + 缩短 resume 超时

工作区对话大上下文 resume 首字节卡死 → run 直接失败。两个根因+两处修:

① ConversationLam(conversation.py)把 ProviderError 吞成 [X_ERROR] 字符串、
   不抛出 → 任何引擎层重试都见不到异常。修:在 apply/apply_typed 抛错那一层加
   _provider_call_with_retry —— retryable ProviderError(claude-code 卡死/瞬时
   API 抖动)重试 3 次再退化成错误串。provider 在 stall 时已清 session_id →
   重试退化成 fresh first-turn(小 prompt 通常很快)→ 恢复而非死。

② claude_code_provider:resume 首字节超时按冷启动 120s 算,但 warm resume 健康
   只要 5-8s → 卡死要白等 120s×2=240s。加 _DEFAULT_RESUME_FIRST_BYTE_TIMEOUT_S
   =45s(可配 claude_resume_first_byte_timeout),_call_resume 用它。

回归测试 TestReactProviderRetry:provider 首调抛 retryable、二调成功 → react
agent 恢复完成而非崩。lambdagent 567 全绿。

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
kenny67nju il y a 2 mois
Parent
commit
bd39cf981a

+ 37 - 7
lambdagent/src/lambdagent/conversation.py

@@ -22,12 +22,40 @@ Usage:
 """
 from __future__ import annotations
 
+import logging
 import time
 from typing import Any, Callable, List, Optional
 
 from lambdagent.core import Term, Context
 from lambdagent.providers.base import LLMProvider, ProviderError, ChatMessage, ChatResponse
 
+logger = logging.getLogger("lambdagent.conversation")
+
+# 可重试 provider 错误(如 claude-code resume 首字节卡死 / 瞬时 API 抖动)的总尝试次数。
+# 不重试则一次卡死就把这步退化成 [X_ERROR] 字符串、run 失败。provider 在 stall 时已
+# 清 session_id → 重试时退化成 fresh first-turn(小 prompt,通常很快)→ 恢复而非死。
+_PROVIDER_RETRY_ATTEMPTS = 3
+
+
+def _provider_call_with_retry(fn, what: str):
+    """调用 provider,对 retryable ProviderError 重试。非 retryable 立即上抛;
+    重试耗尽后把最后一次错误上抛,由调用方退化成 [X_ERROR] 字符串。"""
+    last: Optional[ProviderError] = None
+    for attempt in range(1, _PROVIDER_RETRY_ATTEMPTS + 1):
+        try:
+            return fn()
+        except ProviderError as e:
+            last = e
+            if not getattr(e, "retryable", False) or attempt >= _PROVIDER_RETRY_ATTEMPTS:
+                raise
+            logger.warning(
+                "provider retryable error on %s (attempt %d/%d) — 重试: %s",
+                what, attempt, _PROVIDER_RETRY_ATTEMPTS, str(e)[:200],
+            )
+            time.sleep(1.0)
+    assert last is not None
+    raise last
+
 
 class ConversationLam(Term):
     """
@@ -109,7 +137,8 @@ class ConversationLam(Term):
         # Call provider
         tokens_used = 0
         try:
-            response = self.provider.chat(managed)
+            response = _provider_call_with_retry(
+                lambda: self.provider.chat(managed), "chat")
         except ProviderError as e:
             response = f"[{e.provider.upper()}_ERROR] {e}"
             ctx.record_provider_error(e.provider, str(e))  # AUDIT ②: 结构化上报
@@ -144,12 +173,13 @@ class ConversationLam(Term):
 
         # Call provider via typed interface
         try:
-            response = self.provider.chat_typed(
-                messages=messages,
-                model=self._model,
-                temperature=self._temperature,
-                max_tokens=self._max_tokens,
-            )
+            response = _provider_call_with_retry(
+                lambda: self.provider.chat_typed(
+                    messages=messages,
+                    model=self._model,
+                    temperature=self._temperature,
+                    max_tokens=self._max_tokens,
+                ), "chat_typed")
         except ProviderError as e:
             response = ChatResponse(
                 text=f"[{e.provider.upper()}_ERROR] {e}",

+ 3 - 0
lambdagent/src/lambdagent/fromconfig/compiler.py

@@ -930,6 +930,9 @@ def _compile_react(cfg: Dict, overrides: Dict) -> Term:
         if not _has_session and hasattr(think, 'reset'):
             think.reset()
 
+        # provider 调用容错(claude-code resume 卡死 / 瞬时 API 抖动的 retryable
+        # ProviderError 重试)在 ConversationLam 层做 —— think.apply 在那里已把
+        # ProviderError 吞成 [X_ERROR] 字符串,retry 必须放在抛错那一层。
         thought = think.apply(llm_input, ctx)
         think_ms = (time.time() - t0) * 1000
 

+ 11 - 2
lambdagent/src/lambdagent/providers/claude_code_provider.py

@@ -40,6 +40,12 @@ _DEFAULT_IDLE_TIMEOUT_S = 60.0
 # Initial idle window is more generous since cold start of claude-code's
 # session can take 30-60s before the first byte (model load + auth).
 _DEFAULT_FIRST_BYTE_TIMEOUT_S = 120.0
+# Resume calls reuse a WARM session — healthy first-byte is 5-8s (observed).
+# 120s here means a wedged large-context resume burns 120s×2=240s before
+# failing; warm resume never legitimately needs that long. Use a tighter
+# window so a stalled resume is detected fast and the engine can recover.
+# Override via config.extra["claude_resume_first_byte_timeout"].
+_DEFAULT_RESUME_FIRST_BYTE_TIMEOUT_S = 45.0
 
 
 class _StallError(Exception):
@@ -522,7 +528,8 @@ class ClaudeCodeProvider(LLMProvider):
 
         extra_top = self.config.extra or {}
         _it = float(extra_top.get("claude_idle_timeout", _DEFAULT_IDLE_TIMEOUT_S))
-        _fbt = float(extra_top.get("claude_first_byte_timeout", _DEFAULT_FIRST_BYTE_TIMEOUT_S))
+        _fbt = float(extra_top.get(
+            "claude_resume_first_byte_timeout", _DEFAULT_RESUME_FIRST_BYTE_TIMEOUT_S))
         logger.info(
             "claude-code resume spawn: prompt=%d session=%s hard=%ds idle=%.0fs first-byte=%.0fs",
             len(prompt_arg), self._session_id[:8] if self._session_id else "?",
@@ -549,7 +556,9 @@ class ClaudeCodeProvider(LLMProvider):
 
         extra = self.config.extra or {}
         idle_timeout = float(extra.get("claude_idle_timeout", _DEFAULT_IDLE_TIMEOUT_S))
-        first_byte_timeout = float(extra.get("claude_first_byte_timeout", _DEFAULT_FIRST_BYTE_TIMEOUT_S))
+        # Resume is warm → tighter first-byte window than cold first-turn.
+        first_byte_timeout = float(extra.get(
+            "claude_resume_first_byte_timeout", _DEFAULT_RESUME_FIRST_BYTE_TIMEOUT_S))
 
         def _run_once():
             t0 = time.time()

+ 51 - 0
lambdagent/tests/test_providers.py

@@ -324,3 +324,54 @@ class TestLLMProviderDefaults(unittest.TestCase):
 
 if __name__ == "__main__":
     unittest.main()
+
+
+class TestReactProviderRetry(unittest.TestCase):
+    """回归:provider 抛 retryable ProviderError(如 claude-code resume 卡死)时,
+    react 引擎应重试恢复,而不是整个 run 崩掉。"""
+
+    def test_react_retries_retryable_provider_error(self):
+        from lambdagent.providers.base import ProviderError, ProviderConfig, ChatResponse, LLMProvider
+        from lambdagent.fromconfig import compiler as _compiler
+
+        class FlakeProvider(LLMProvider):
+            """首次 chat 抛 retryable ProviderError,之后返回可终止的纯文本。"""
+            def __init__(self):
+                super().__init__(ProviderConfig(model="claude-code/sonnet"))
+                self.calls = 0
+            def _resp(self):
+                self.calls += 1
+                if self.calls == 1:
+                    raise ProviderError("stalled (first-byte) on resume",
+                                        "claude-code", retryable=True)
+                return "任务已完成:测试通过。"
+            def chat(self, messages):
+                return self._resp()
+            def chat_typed(self, messages, model="", temperature=0.0, max_tokens=4096):
+                txt = self._resp()
+                return ChatResponse(text=txt, input_tokens=5, output_tokens=5,
+                                    model="claude-code/sonnet", finish_reason="end_turn")
+            @property
+            def provider_name(self):
+                return "claude-code"
+
+        flake = FlakeProvider()
+        cfg = {
+            "agentId": "t", "name": "t", "type": "react",
+            "model": {"provider": "claude-code", "name": "sonnet"},
+            "systemPrompt": "你是助手。",
+            "react": {"maxSteps": 4},
+            "mcp": {"localTools": ["terminate"]},
+        }
+        # 注入 flaky provider(session 模式,模拟 claude-code)
+        with patch.object(_compiler, "_create_provider", return_value=(flake, True)):
+            term = _compiler.build_agent(cfg)
+            result = term.apply("做个小任务", Context())
+
+        # 没有抛异常 = 引擎吞下首次 retryable 错误并重试恢复
+        self.assertGreaterEqual(flake.calls, 2, "应至少重试一次(首调抛错、二调成功)")
+        self.assertIn("完成", str(result))
+
+
+if __name__ == "__main__":
+    unittest.main()