Jelajahi Sumber

fix(provider): claude-code resume 卡死不再杀 run — 重试 + 缩短 resume 超时

工作区对话大上下文 resume 首字节卡死 → run 直接失败。两个根因+两处修:

① ConversationLam(conversation.py)把 ProviderError 吞成 [X_ERROR] 字符串、
   不抛出 → 任何引擎层重试都见不到异常。修:在 apply/apply_typed 抛错那一层加
   _provider_call_with_retry —— retryable ProviderError(claude-code 卡死/瞬时
   API 抖动)重试 3 次再退化成错误串。provider 在 stall 时已清 session_id →
   重试退化成 fresh first-turn(小 prompt 通常很快)→ 恢复而非死。

② claude_code_provider:resume 首字节超时按冷启动 120s 算,但 warm resume 健康
   只要 5-8s → 卡死要白等 120s×2=240s。加 _DEFAULT_RESUME_FIRST_BYTE_TIMEOUT_S
   =45s(可配 claude_resume_first_byte_timeout),_call_resume 用它。

回归测试 TestReactProviderRetry:provider 首调抛 retryable、二调成功 → react
agent 恢复完成而非崩。lambdagent 567 全绿。

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
kenny67nju 3 bulan lalu
induk
melakukan
bd39cf981a

+ 37 - 7
lambdagent/src/lambdagent/conversation.py

@@ -22,12 +22,40 @@ Usage:
 """
 """
 from __future__ import annotations
 from __future__ import annotations
 
 
+import logging
 import time
 import time
 from typing import Any, Callable, List, Optional
 from typing import Any, Callable, List, Optional
 
 
 from lambdagent.core import Term, Context
 from lambdagent.core import Term, Context
 from lambdagent.providers.base import LLMProvider, ProviderError, ChatMessage, ChatResponse
 from lambdagent.providers.base import LLMProvider, ProviderError, ChatMessage, ChatResponse
 
 
+logger = logging.getLogger("lambdagent.conversation")
+
+# 可重试 provider 错误(如 claude-code resume 首字节卡死 / 瞬时 API 抖动)的总尝试次数。
+# 不重试则一次卡死就把这步退化成 [X_ERROR] 字符串、run 失败。provider 在 stall 时已
+# 清 session_id → 重试时退化成 fresh first-turn(小 prompt,通常很快)→ 恢复而非死。
+_PROVIDER_RETRY_ATTEMPTS = 3
+
+
+def _provider_call_with_retry(fn, what: str):
+    """调用 provider,对 retryable ProviderError 重试。非 retryable 立即上抛;
+    重试耗尽后把最后一次错误上抛,由调用方退化成 [X_ERROR] 字符串。"""
+    last: Optional[ProviderError] = None
+    for attempt in range(1, _PROVIDER_RETRY_ATTEMPTS + 1):
+        try:
+            return fn()
+        except ProviderError as e:
+            last = e
+            if not getattr(e, "retryable", False) or attempt >= _PROVIDER_RETRY_ATTEMPTS:
+                raise
+            logger.warning(
+                "provider retryable error on %s (attempt %d/%d) — 重试: %s",
+                what, attempt, _PROVIDER_RETRY_ATTEMPTS, str(e)[:200],
+            )
+            time.sleep(1.0)
+    assert last is not None
+    raise last
+
 
 
 class ConversationLam(Term):
 class ConversationLam(Term):
     """
     """
@@ -109,7 +137,8 @@ class ConversationLam(Term):
         # Call provider
         # Call provider
         tokens_used = 0
         tokens_used = 0
         try:
         try:
-            response = self.provider.chat(managed)
+            response = _provider_call_with_retry(
+                lambda: self.provider.chat(managed), "chat")
         except ProviderError as e:
         except ProviderError as e:
             response = f"[{e.provider.upper()}_ERROR] {e}"
             response = f"[{e.provider.upper()}_ERROR] {e}"
             ctx.record_provider_error(e.provider, str(e))  # AUDIT ②: 结构化上报
             ctx.record_provider_error(e.provider, str(e))  # AUDIT ②: 结构化上报
@@ -144,12 +173,13 @@ class ConversationLam(Term):
 
 
         # Call provider via typed interface
         # Call provider via typed interface
         try:
         try:
-            response = self.provider.chat_typed(
-                messages=messages,
-                model=self._model,
-                temperature=self._temperature,
-                max_tokens=self._max_tokens,
-            )
+            response = _provider_call_with_retry(
+                lambda: self.provider.chat_typed(
+                    messages=messages,
+                    model=self._model,
+                    temperature=self._temperature,
+                    max_tokens=self._max_tokens,
+                ), "chat_typed")
         except ProviderError as e:
         except ProviderError as e:
             response = ChatResponse(
             response = ChatResponse(
                 text=f"[{e.provider.upper()}_ERROR] {e}",
                 text=f"[{e.provider.upper()}_ERROR] {e}",

+ 3 - 0
lambdagent/src/lambdagent/fromconfig/compiler.py

@@ -930,6 +930,9 @@ def _compile_react(cfg: Dict, overrides: Dict) -> Term:
         if not _has_session and hasattr(think, 'reset'):
         if not _has_session and hasattr(think, 'reset'):
             think.reset()
             think.reset()
 
 
+        # provider 调用容错(claude-code resume 卡死 / 瞬时 API 抖动的 retryable
+        # ProviderError 重试)在 ConversationLam 层做 —— think.apply 在那里已把
+        # ProviderError 吞成 [X_ERROR] 字符串,retry 必须放在抛错那一层。
         thought = think.apply(llm_input, ctx)
         thought = think.apply(llm_input, ctx)
         think_ms = (time.time() - t0) * 1000
         think_ms = (time.time() - t0) * 1000
 
 

+ 11 - 2
lambdagent/src/lambdagent/providers/claude_code_provider.py

@@ -40,6 +40,12 @@ _DEFAULT_IDLE_TIMEOUT_S = 60.0
 # Initial idle window is more generous since cold start of claude-code's
 # Initial idle window is more generous since cold start of claude-code's
 # session can take 30-60s before the first byte (model load + auth).
 # session can take 30-60s before the first byte (model load + auth).
 _DEFAULT_FIRST_BYTE_TIMEOUT_S = 120.0
 _DEFAULT_FIRST_BYTE_TIMEOUT_S = 120.0
+# Resume calls reuse a WARM session — healthy first-byte is 5-8s (observed).
+# 120s here means a wedged large-context resume burns 120s×2=240s before
+# failing; warm resume never legitimately needs that long. Use a tighter
+# window so a stalled resume is detected fast and the engine can recover.
+# Override via config.extra["claude_resume_first_byte_timeout"].
+_DEFAULT_RESUME_FIRST_BYTE_TIMEOUT_S = 45.0
 
 
 
 
 class _StallError(Exception):
 class _StallError(Exception):
@@ -522,7 +528,8 @@ class ClaudeCodeProvider(LLMProvider):
 
 
         extra_top = self.config.extra or {}
         extra_top = self.config.extra or {}
         _it = float(extra_top.get("claude_idle_timeout", _DEFAULT_IDLE_TIMEOUT_S))
         _it = float(extra_top.get("claude_idle_timeout", _DEFAULT_IDLE_TIMEOUT_S))
-        _fbt = float(extra_top.get("claude_first_byte_timeout", _DEFAULT_FIRST_BYTE_TIMEOUT_S))
+        _fbt = float(extra_top.get(
+            "claude_resume_first_byte_timeout", _DEFAULT_RESUME_FIRST_BYTE_TIMEOUT_S))
         logger.info(
         logger.info(
             "claude-code resume spawn: prompt=%d session=%s hard=%ds idle=%.0fs first-byte=%.0fs",
             "claude-code resume spawn: prompt=%d session=%s hard=%ds idle=%.0fs first-byte=%.0fs",
             len(prompt_arg), self._session_id[:8] if self._session_id else "?",
             len(prompt_arg), self._session_id[:8] if self._session_id else "?",
@@ -549,7 +556,9 @@ class ClaudeCodeProvider(LLMProvider):
 
 
         extra = self.config.extra or {}
         extra = self.config.extra or {}
         idle_timeout = float(extra.get("claude_idle_timeout", _DEFAULT_IDLE_TIMEOUT_S))
         idle_timeout = float(extra.get("claude_idle_timeout", _DEFAULT_IDLE_TIMEOUT_S))
-        first_byte_timeout = float(extra.get("claude_first_byte_timeout", _DEFAULT_FIRST_BYTE_TIMEOUT_S))
+        # Resume is warm → tighter first-byte window than cold first-turn.
+        first_byte_timeout = float(extra.get(
+            "claude_resume_first_byte_timeout", _DEFAULT_RESUME_FIRST_BYTE_TIMEOUT_S))
 
 
         def _run_once():
         def _run_once():
             t0 = time.time()
             t0 = time.time()

+ 51 - 0
lambdagent/tests/test_providers.py

@@ -324,3 +324,54 @@ class TestLLMProviderDefaults(unittest.TestCase):
 
 
 if __name__ == "__main__":
 if __name__ == "__main__":
     unittest.main()
     unittest.main()
+
+
+class TestReactProviderRetry(unittest.TestCase):
+    """回归:provider 抛 retryable ProviderError(如 claude-code resume 卡死)时,
+    react 引擎应重试恢复,而不是整个 run 崩掉。"""
+
+    def test_react_retries_retryable_provider_error(self):
+        from lambdagent.providers.base import ProviderError, ProviderConfig, ChatResponse, LLMProvider
+        from lambdagent.fromconfig import compiler as _compiler
+
+        class FlakeProvider(LLMProvider):
+            """首次 chat 抛 retryable ProviderError,之后返回可终止的纯文本。"""
+            def __init__(self):
+                super().__init__(ProviderConfig(model="claude-code/sonnet"))
+                self.calls = 0
+            def _resp(self):
+                self.calls += 1
+                if self.calls == 1:
+                    raise ProviderError("stalled (first-byte) on resume",
+                                        "claude-code", retryable=True)
+                return "任务已完成:测试通过。"
+            def chat(self, messages):
+                return self._resp()
+            def chat_typed(self, messages, model="", temperature=0.0, max_tokens=4096):
+                txt = self._resp()
+                return ChatResponse(text=txt, input_tokens=5, output_tokens=5,
+                                    model="claude-code/sonnet", finish_reason="end_turn")
+            @property
+            def provider_name(self):
+                return "claude-code"
+
+        flake = FlakeProvider()
+        cfg = {
+            "agentId": "t", "name": "t", "type": "react",
+            "model": {"provider": "claude-code", "name": "sonnet"},
+            "systemPrompt": "你是助手。",
+            "react": {"maxSteps": 4},
+            "mcp": {"localTools": ["terminate"]},
+        }
+        # 注入 flaky provider(session 模式,模拟 claude-code)
+        with patch.object(_compiler, "_create_provider", return_value=(flake, True)):
+            term = _compiler.build_agent(cfg)
+            result = term.apply("做个小任务", Context())
+
+        # 没有抛异常 = 引擎吞下首次 retryable 错误并重试恢复
+        self.assertGreaterEqual(flake.calls, 2, "应至少重试一次(首调抛错、二调成功)")
+        self.assertIn("完成", str(result))
+
+
+if __name__ == "__main__":
+    unittest.main()