Browse Source

fix(provider): 连续 resume 卡死熔断 — 堵上轮换的真缺口

铁证:一条失败 run 全程只用 1 个 session(d02c1395),却 STALL 8 次打转到死。
根因 = 上一版轮换计数 `_resume_count` 只在 resume **成功**时 +1;而这条 run 是
resume **每次都首字节卡死(失败)**,永不成功 → 永不轮换 → 大 session 钉死。
叠加"首字节卡死保留会话"(兜瞬时抖动)→ 正好把它焊死在那个大 session 上。

补熔断:`_resume_stall_streak` 跨 _call_resume 累计连续整体首字节卡死次数,
连卡 ≥K(默认 2,可配 claude_resume_stall_giveup)即判 session 太大/已废 →
清 session_id 换新,下次走 fresh first-turn(快)。成功一次即清零。
保留(瞬时)+熔断(连卡)+轮换(成功侧控大小)三层闭合。

回归 TestResumeStallGiveup + lambdagent 572 全绿。

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
kenny67nju 4 months ago
parent
commit
7052cc2fcb

+ 24 - 0
lambdagent/src/lambdagent/providers/claude_code_provider.py

@@ -51,6 +51,9 @@ _DEFAULT_RESUME_FIRST_BYTE_TIMEOUT_S = 45.0
 # resume 后主动开新会话,让 session 永远停在"小而快"的区间。0=不轮换。
 # 可配 config.extra["claude_session_rotate_after"]。
 _DEFAULT_SESSION_ROTATE_AFTER = 8
+# 连续多少次 _call_resume 整体首字节卡死后,判定该 session 本身太大/废了,
+# 强制清掉换新会话。保留会话只兜瞬时抖动;连卡说明是 session 大小问题,必须换。
+_DEFAULT_RESUME_STALL_GIVEUP = 2
 
 
 class _StallError(Exception):
@@ -328,6 +331,7 @@ class ClaudeCodeProvider(LLMProvider):
         self.claude_bin = _find_working_claude(preferred) or preferred
         self._session_id = None
         self._resume_count = 0  # 会话轮换计数(每 N 次 resume 换新会话,防大会话 resume 卡死)
+        self._resume_stall_streak = 0  # 连续首字节卡死计数(熔断:连卡 K 次判 session 废了→清掉换新)
 
         if not shutil.which(self.claude_bin) and not os.path.isfile(self.claude_bin):
             raise ProviderError(
@@ -611,6 +615,25 @@ class ClaudeCodeProvider(LLMProvider):
                 #   → 清掉,下次退化成 fresh first-turn 重启。
                 if stall_exc.kind != "first-byte":
                     self._session_id = None
+                    self._resume_stall_streak = 0
+                else:
+                    # 首字节卡死本应保留会话(兜瞬时抖动)。但若**连续**多次
+                    # _call_resume 都整体卡死,说明不是抖动而是这个 session 已
+                    # 大到 resume 必卡 → 熔断:清掉会话,下次退化成 fresh
+                    # first-turn(快)。否则会像那条 run 一样在同一大 session 上
+                    # 连卡 8 次打转到死。
+                    self._resume_stall_streak += 1
+                    _giveup = int((self.config.extra or {}).get(
+                        "claude_resume_stall_giveup", _DEFAULT_RESUME_STALL_GIVEUP))
+                    if _giveup > 0 and self._resume_stall_streak >= _giveup:
+                        logger.warning(
+                            "claude-code resume 连续首字节卡死 %d 次 — 判定 session "
+                            "太大/已废,清会话换新(下次走 fresh first-turn)",
+                            self._resume_stall_streak,
+                        )
+                        self._session_id = None
+                        self._resume_count = 0
+                        self._resume_stall_streak = 0
                 raise ProviderError(
                     f"Claude Code stalled ({stall_exc.kind}-timeout after "
                     f"{stall_exc.elapsed:.0f}s, no output) on resume",
@@ -622,6 +645,7 @@ class ClaudeCodeProvider(LLMProvider):
                     "claude-code resume ok (attempt %d): elapsed=%.1fs out_len=%d",
                     attempt, elapsed, len(result.stdout or ""),
                 )
+                self._resume_stall_streak = 0  # 成功一次即清零连卡计数
                 break
 
             # Non-zero exit. Inspect for transport-error marker.

+ 35 - 0
lambdagent/tests/test_providers.py

@@ -388,6 +388,8 @@ class TestResumeSessionPreserved(unittest.TestCase):
         p.config = ProviderConfig(model="sonnet", timeout=600, extra={})
         p.claude_bin = "claude"
         p._session_id = "sess-123"
+        p._resume_count = 0
+        p._resume_stall_streak = 0
         return p, ccp
 
     def test_first_byte_stall_keeps_session(self):
@@ -439,3 +441,36 @@ class TestSessionRotation(unittest.TestCase):
             p._call_resume([{"role": "user", "content": "x"}])
             self.assertIsNone(p._session_id, "第3次 resume 后应轮换(清会话)")
             self.assertEqual(p._resume_count, 0, "轮换后计数归零")
+
+
+class TestResumeStallGiveup(unittest.TestCase):
+    """连续首字节卡死必须熔断:连卡 K 次后清掉大 session 换新,
+    否则会像那条 run 一样在同一大 session 上连卡 8 次打转到死。"""
+
+    def test_consecutive_first_byte_stalls_clear_session(self):
+        from lambdagent.providers import claude_code_provider as ccp
+        from lambdagent.providers.base import ProviderConfig, ProviderError
+
+        p = ccp.ClaudeCodeProvider.__new__(ccp.ClaudeCodeProvider)
+        p.config = ProviderConfig(model="sonnet", timeout=600,
+                                  extra={"claude_resume_stall_giveup": 2})
+        p.claude_bin = "claude"
+        p._session_id = "sess-BIG"
+        p._resume_count = 0
+        p._resume_stall_streak = 0
+
+        def always_stall(*a, **k):
+            raise ccp._StallError("first-byte", 45.0, "", "")
+
+        with patch.object(ccp, "_run_with_idle_timeout", side_effect=always_stall), \
+             patch.object(ccp.time, "sleep", lambda *_: None):
+            # 第 1 次 _call_resume 全卡:保留会话(兜瞬时抖动)
+            with self.assertRaises(ProviderError):
+                p._call_resume([{"role": "user", "content": "x"}])
+            self.assertEqual(p._session_id, "sess-BIG", "第1次连卡应保留会话")
+            self.assertEqual(p._resume_stall_streak, 1)
+            # 第 2 次 _call_resume 又全卡:达阈值 → 熔断清会话
+            with self.assertRaises(ProviderError):
+                p._call_resume([{"role": "user", "content": "x"}])
+            self.assertIsNone(p._session_id, "连卡2次后应清会话换新")
+            self.assertEqual(p._resume_stall_streak, 0, "熔断后计数归零")