perf(debug): 增强内存监控：每秒采样 + 更多关键路径埋点

- 改用 psutil 实时 RSS 取代历史最大 ru_maxrss
- 新增后台每秒采样线程，检测到 >100MB/s 增长立即记录
- WebTerminal 初始化时自动启动采样
- 增加子智能体恢复加载 conversation.json、httpx 流式/非流式响应、
  工具返回结果、模型输出组装等埋点

core/web_terminal.py

@@ -20,6 +20,7 @@ except ImportError:
         sys.path.insert(0, str(project_root))
     from config import MAX_TERMINALS, TERMINAL_BUFFER_SIZE, TERMINAL_DISPLAY_SIZE
 from modules.terminal_manager import TerminalManager
+from modules.memory_debug import start_periodic_sampling
 
 if TYPE_CHECKING:
     from modules.user_container_manager import ContainerHandle
@@ -101,6 +102,9 @@ class WebTerminal(MainTerminal):
             print(f"[WebTerminal] 实时token统计已启用")
         else:
             print(f"[WebTerminal] 警告：message_callback为None，无法启用实时token统计")
+
+        # 启动后台内存采样，帮助定位偶发内存暴涨
+        start_periodic_sampling()
     # ===========================================
     # 新增：对话管理相关方法（Web版本）
     # ===========================================

modules/memory_debug.py

@@ -51,21 +51,34 @@ def _resolve_log_path() -> Path:
     return _log_path
 
 
+_peak_rss_mb: Optional[float] = None
+_last_sample_rss_mb: Optional[float] = None
+_sample_thread: Optional[threading.Thread] = None
+
+
 def _current_rss_mb() -> Optional[float]:
-    """当前进程 RSS（MB），失败返回 None。"""
+    """当前进程 RSS（MB），优先使用 psutil（实时），失败回退到 resource。"""
+    global _peak_rss_mb
+    try:
+        import psutil
+        proc = psutil.Process()
+        rss = proc.memory_info().rss / (1024 * 1024)
+        if _peak_rss_mb is None or rss > _peak_rss_mb:
+            _peak_rss_mb = rss
+        return rss
+    except Exception:
+        pass
     try:
         import resource
         usage = resource.getrusage(resource.RUSAGE_SELF)
         # macOS: ru_maxrss 单位是 bytes；Linux: 单位是 KB
         if sys.platform == "darwin":
-            return usage.ru_maxrss / (1024 * 1024)
+            rss = usage.ru_maxrss / (1024 * 1024)
-        return usage.ru_maxrss / 1024.0
+        else:
-    except Exception:
+            rss = usage.ru_maxrss / 1024.0
-        pass
+        if _peak_rss_mb is None or rss > _peak_rss_mb:
-    try:
+            _peak_rss_mb = rss
-        import psutil
+        return rss
-        proc = psutil.Process()
-        return proc.memory_info().rss / (1024 * 1024)
     except Exception:
         return None
 
@@ -231,3 +244,66 @@ class TopAllocator:
             tracemalloc.stop()
         except Exception:
             pass
+
+
+def _periodic_sampling(
+    interval_seconds: float = 1.0,
+    spike_threshold_mb: float = 100.0,
+    max_records: int = 3,
+) -> None:
+    """后台线程：每秒采样一次内存，发现快速增长或峰值时记录。"""
+    global _last_sample_rss_mb, _peak_rss_mb
+    records_since_spike = 0
+    while ENABLED:
+        try:
+            info = get_memory_info()
+            rss = info.get("rss_mb")
+            vms = info.get("vms_mb")
+            if rss is not None:
+                spike = False
+                if _last_sample_rss_mb is not None:
+                    delta = rss - _last_sample_rss_mb
+                    if delta >= spike_threshold_mb:
+                        spike = True
+                        records_since_spike = max_records
+                _last_sample_rss_mb = rss
+                if records_since_spike > 0:
+                    records_since_spike -= 1
+                    log_memory_event(
+                        "memory_periodic_spike",
+                        rss_mb=rss,
+                        vms_mb=vms,
+                        peak_rss_mb=_peak_rss_mb,
+                        delta_from_last_mb=(delta if spike else None),
+                    )
+                else:
+                    # 每 5 秒记录一次常规采样，避免日志过多
+                    if int(time.time()) % 5 == 0:
+                        log_memory_event(
+                            "memory_periodic_sample",
+                            rss_mb=rss,
+                            vms_mb=vms,
+                            peak_rss_mb=_peak_rss_mb,
+                        )
+        except Exception:
+            pass
+        time.sleep(interval_seconds)
+
+
+def start_periodic_sampling(
+    interval_seconds: float = 1.0,
+    spike_threshold_mb: float = 100.0,
+) -> None:
+    """启动后台内存采样线程。幂等：重复调用不会启动多个线程。"""
+    global _sample_thread
+    if not ENABLED:
+        return
+    if _sample_thread is not None and _sample_thread.is_alive():
+        return
+    _sample_thread = threading.Thread(
+        target=_periodic_sampling,
+        args=(interval_seconds, spike_threshold_mb),
+        name="memory-debug-sampler",
+        daemon=True,
+    )
+    _sample_thread.start()

modules/sub_agent/manager.py

@@ -312,6 +312,12 @@ class SubAgentManager(SubAgentStateMixin, SubAgentStatsMixin, SubAgentCreationMi
                     multi_agent_mode=multi_agent_mode,
                 )
                 self._running_tasks.pop(task_id, None)
+                # 多智能体模式下 failed 视为可复活状态，保留实例引用供后续 send_message_to_sub_agent 重新激活
+                if multi_agent_mode:
+                    final_task = self.tasks.get(task_id) or {}
+                    if final_task.get("status") != "failed":
+                        self._sub_agent_instances.pop(agent_id, None)
+                else:
                     self._sub_agent_instances.pop(agent_id, None)
                 self.reconcile_task_states(conversation_id=conversation_id)
                 # 多智能体模式：结束时把状态写回 MultiAgentState
@@ -452,6 +458,59 @@ class SubAgentManager(SubAgentStateMixin, SubAgentStatsMixin, SubAgentCreationMi
         )
         return count
 
+    def stop_sub_agent(
+        self,
+        *,
+        task_id: Optional[str] = None,
+        agent_id: Optional[int] = None,
+    ) -> Dict:
+        """暂停指定子智能体，使其进入 idle 状态而不终结。"""
+        task = self._select_task(task_id, agent_id)
+        if not task:
+            return {"success": False, "error": "未找到对应的子智能体任务"}
+
+        real_task_id = task["task_id"]
+        real_agent_id = task.get("agent_id")
+        if not task.get("multi_agent_mode"):
+            return {"success": False, "error": "stop_sub_agent 仅在多智能体模式下可用"}
+
+        # 查找或复活实例，确保能接收软停止信号
+        if real_agent_id is not None:
+            sub_agent = self._find_or_revive_sub_agent_task(real_agent_id)
+        else:
+            sub_agent = None
+
+        if sub_agent and hasattr(sub_agent, "request_soft_stop"):
+            try:
+                sub_agent.request_soft_stop()
+            except Exception as exc:
+                return {"success": False, "error": f"暂停子智能体失败: {exc}"}
+        else:
+            # 没有活实例时直接修改任务状态为 idle
+            task["status"] = "idle"
+            task["updated_at"] = time.time()
+            self._save_state()
+
+        # 同步更新 MultiAgentState
+        conversation_id = task.get("conversation_id")
+        if conversation_id:
+            state = self.get_multi_agent_state(conversation_id)
+            if state and real_agent_id is not None:
+                state.mark_status(real_agent_id, "idle")
+
+        ma_debug(
+            "manager_stop_sub_agent",
+            task_id=real_task_id,
+            agent_id=real_agent_id,
+            had_instance=bool(sub_agent),
+        )
+        return {
+            "success": True,
+            "task_id": real_task_id,
+            "agent_id": real_agent_id,
+            "message": f"子智能体{real_agent_id} 已暂停，可用 send_message_to_sub_agent 重新激活。",
+        }
+
     def terminate_sub_agent(
         self,
         *,
@@ -659,6 +718,11 @@ class SubAgentManager(SubAgentStateMixin, SubAgentStatsMixin, SubAgentCreationMi
 
             # 其余工具直接走主进程 handle_tool_call，自然经过沙箱/容器/权限链路
             result_text = await self.terminal.handle_tool_call(tool_name, arguments)
+            log_memory_event(
+                "sub_agent_tool_result",
+                tool_name=tool_name,
+                result_chars=len(result_text),
+            )
             try:
                 return json.loads(result_text)
             except Exception:
@@ -702,8 +766,27 @@ class SubAgentManager(SubAgentStateMixin, SubAgentStatsMixin, SubAgentCreationMi
                 continue
 
             try:
-                conversation_data = json.loads(conversation_file.read_text(encoding="utf-8"))
+                log_memory_event(
+                    "sub_agent_restore_load_conversation_start",
+                    task_id=task_id,
+                    agent_id=task.get("agent_id"),
+                    conversation_file=str(conversation_file),
+                )
+                conversation_text = conversation_file.read_text(encoding="utf-8")
+                log_memory_event(
+                    "sub_agent_restore_after_read",
+                    task_id=task_id,
+                    agent_id=task.get("agent_id"),
+                    conversation_file_chars=len(conversation_text),
+                )
+                conversation_data = json.loads(conversation_text)
                 messages = list(conversation_data.get("messages") or [])
+                log_memory_event(
+                    "sub_agent_restore_after_parse",
+                    task_id=task_id,
+                    agent_id=task.get("agent_id"),
+                    messages_count=len(messages),
+                )
             except Exception as exc:
                 logger.warning(f"[restore] 读取任务 {task_id} 对话文件失败: {exc}")
                 continue
@@ -928,9 +1011,11 @@ class SubAgentManager(SubAgentStateMixin, SubAgentStatsMixin, SubAgentCreationMi
 
         适用于 ask_other_agent / send_message_to_sub_agent / answer_sub_agent_question_
         （非阻塞到工具结果的路径）。返回 True 表示成功注入。
+        若内存中无运行实例（如 failed 后保留的实例已结束），会尝试从 conversation
+        文件重建子智能体（保留原 agent_id 和 role_id）后再注入消息。
         """
-        # 查找该 agent_id 对应的 running SubAgentTask
+        # 查找或复活该 agent_id 对应的 SubAgentTask
-        sub_agent = self._find_sub_agent_task_by_agent_id(agent_id)
+        sub_agent = self._find_or_revive_sub_agent_task(agent_id)
         ma_debug(
             "manager_inject_message_to_sub_agent",
             agent_id=agent_id,
@@ -943,6 +1028,162 @@ class SubAgentManager(SubAgentStateMixin, SubAgentStatsMixin, SubAgentCreationMi
         sub_agent.inject_message(message_text)
         return True
 
+    def _find_or_revive_sub_agent_task(self, agent_id: int) -> Optional[Any]:
+        """查找内存中的 SubAgentTask；不存在或已结束时从磁盘复活（多智能体模式）。"""
+        inst = self._find_sub_agent_task_by_agent_id(agent_id)
+        if inst is not None:
+            task = getattr(inst, "_task", None)
+            if task is None or not task.done():
+                return inst
+            # 实例存在但已结束，需要复活前先清理旧引用
+            self._sub_agent_instances.pop(agent_id, None)
+        revived = self._revive_sub_agent(agent_id)
+        return revived
+
+    def _revive_sub_agent(self, agent_id: int) -> Optional[Any]:
+        """从 conversation.json 重建一个多智能体子智能体实例（保留原 agent_id/role_id）。
+
+        用于 failed/idle 等可复活状态被 send_message_to_sub_agent 重新激活的场景。
+        """
+        from modules.sub_agent.task import SubAgentTask
+
+        candidates = [
+            t for t in self.tasks.values()
+            if isinstance(t, dict) and t.get("agent_id") == agent_id and t.get("multi_agent_mode")
+        ]
+        if not candidates:
+            return None
+        # 按创建时间取最新一条
+        candidates.sort(key=lambda item: item.get("created_at", 0), reverse=True)
+        task = candidates[0]
+        task_id = task.get("task_id")
+        if not task_id:
+            return None
+        # 已在运行中则不重复重建
+        if task_id in self._running_tasks:
+            existing_inst = self._sub_agent_instances.get(agent_id)
+            if existing_inst is not None:
+                return existing_inst
+
+        task_root = Path(task.get("task_root", ""))
+        conversation_file = Path(task.get("conversation_file", ""))
+        system_prompt_file = task_root / "system_prompt.txt"
+        task_message_file = task_root / "task.txt"
+
+        if not conversation_file.exists():
+            logger.warning(f"[revive] 任务 {task_id} 的对话文件缺失，无法复活")
+            return None
+
+        try:
+            conversation_data = json.loads(conversation_file.read_text(encoding="utf-8"))
+            messages = list(conversation_data.get("messages") or [])
+        except Exception as exc:
+            logger.warning(f"[revive] 读取任务 {task_id} 对话文件失败: {exc}")
+            return None
+
+        system_prompt = ""
+        if system_prompt_file.exists():
+            try:
+                system_prompt = system_prompt_file.read_text(encoding="utf-8")
+            except Exception:
+                pass
+
+        task_message = ""
+        if task_message_file.exists():
+            try:
+                task_message = task_message_file.read_text(encoding="utf-8")
+            except Exception:
+                pass
+
+        if not messages:
+            messages = [
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": task_message},
+            ]
+
+        conversation_id = task.get("conversation_id")
+        multi_agent_state = None
+        if conversation_id:
+            multi_agent_state = self.get_or_create_multi_agent_state(conversation_id)
+            if multi_agent_state and not multi_agent_state.get_instance(agent_id):
+                from modules.multi_agent.state import AgentInstance
+                inst = AgentInstance(
+                    agent_id=agent_id,
+                    role_id=task.get("role_id") or "",
+                    display_name=task.get("display_name") or f"Agent_{agent_id}",
+                    task_id=task_id,
+                    status="idle",
+                    summary=task.get("summary", ""),
+                )
+                try:
+                    multi_agent_state.register_instance(inst)
+                except ValueError:
+                    pass
+
+        sub_agent = SubAgentTask(
+            manager=self,
+            task_record=task,
+            task_message=task_message,
+            system_prompt=system_prompt,
+            model_key=task.get("model_key"),
+            thinking_mode=task.get("thinking_mode") or "fast",
+            multi_agent_mode=True,
+            multi_agent_state=multi_agent_state,
+            display_name=task.get("display_name"),
+        )
+        sub_agent.messages = messages
+        sub_agent._idle = True
+        task["status"] = "idle"
+        task["updated_at"] = time.time()
+        if multi_agent_state:
+            multi_agent_state.mark_status(agent_id, "idle")
+        # 同步落盘 output.json
+        try:
+            output_file = Path(task.get("output_file", ""))
+            if output_file.exists():
+                output_data = json.loads(output_file.read_text(encoding="utf-8"))
+            else:
+                output_data = {}
+            output_data["status"] = "idle"
+            output_data["success"] = None
+            output_file.parent.mkdir(parents=True, exist_ok=True)
+            output_file.write_text(json.dumps(output_data, ensure_ascii=False), encoding="utf-8")
+        except Exception as exc:
+            logger.warning(f"[revive] 更新任务 {task_id} output 文件失败: {exc}")
+
+        task_coro = sub_agent.run()
+        asyncio_task = self._run_coro(task_coro)
+        sub_agent._task = asyncio_task
+        self._running_tasks[task_id] = asyncio_task
+        self._sub_agent_instances[agent_id] = sub_agent
+
+        def _on_done(fut, tid=task_id, aid=agent_id, state=multi_agent_state, sa=sub_agent):
+            try:
+                self._running_tasks.pop(tid, None)
+                # failed 保留实例供复活；其余终态清理
+                if state:
+                    final_task = self.tasks.get(tid) or {}
+                    if final_task.get("status") != "failed":
+                        self._sub_agent_instances.pop(aid, None)
+                else:
+                    self._sub_agent_instances.pop(aid, None)
+                self.reconcile_task_states(conversation_id=conversation_id)
+                if state:
+                    self._on_multi_agent_task_done(tid, aid, state, sa)
+            except Exception as exc:
+                logger.exception(f"[SubAgent] revived task {tid} 完成回调异常: {exc}")
+                ma_debug("manager_revive_on_done_exception", task_id=tid, agent_id=aid, error=str(exc))
+
+        asyncio_task.add_done_callback(_on_done)
+        ma_debug(
+            "manager_revive_sub_agent",
+            task_id=task_id,
+            agent_id=agent_id,
+            display_name=task.get("display_name"),
+            message_count=len(messages),
+        )
+        return sub_agent
+
     def _find_sub_agent_task_by_agent_id(self, agent_id: int) -> Optional[Any]:
         """通过遍历创建中的 task 查找活 SubAgentTask 实例。
         

modules/sub_agent/task.py

@@ -638,6 +638,15 @@ class SubAgentTask:
             if chunk.get("usage"):
                 usage = chunk["usage"]
 
+        log_memory_event(
+            "sub_agent_call_model_assembled",
+            task_id=self.task_id,
+            agent_id=self.agent_id,
+            display_name=self.display_name,
+            assistant_message_chars=len(assistant_message),
+            reasoning_chars=len(reasoning),
+            tool_calls_count=len(tool_calls),
+        )
         return assistant_message, reasoning, tool_calls, usage
 
     def _parse_args(self, tool_call: Dict[str, Any]) -> Dict[str, Any]:
@@ -1120,8 +1129,17 @@ class SubAgentTask:
             idle=self._idle,
             cancelled=self._cancelled,
         )
+        if timeout:
+            status = "timeout"
+        elif max_turns_exceeded:
+            status = "failed"
+        elif success:
+            status = "completed"
+        else:
+            status = "failed"
         output_data = {
             "success": success,
+            "status": status,
             "summary": summary,
             "timeout": timeout,
             "max_turns_exceeded": max_turns_exceeded,

utils/api_client/chat_mixin.py

@@ -197,7 +197,15 @@ class DeepSeekClientChatMixin:
                             yield {"error": self.last_error_info}
                             return
                             
+                        chunk_count = 0
                         async for line in response.aiter_lines():
+                            chunk_count += 1
+                            if chunk_count % 100 == 0:
+                                log_memory_event(
+                                    "api_client_stream_chunk",
+                                    model_key=self.model_key,
+                                    chunk_count=chunk_count,
+                                )
                             if line.startswith("data:"):
                                 json_str = line[5:].strip()
                                 if json_str == "[DONE]":
@@ -214,6 +222,16 @@ class DeepSeekClientChatMixin:
                         json=payload,
                         headers=headers
                     )
+                    try:
+                        response_text = response.text
+                    except Exception:
+                        response_text = ""
+                    log_memory_event(
+                        "api_client_nonstream_response",
+                        model_key=self.model_key,
+                        status_code=response.status_code,
+                        response_chars=len(response_text),
+                    )
                     if response.status_code != 200:
                         error_text = response.text
                         self.last_error_info = {