fix(windows): 超时终止链弃用 CTRL_BREAK_EVENT,修复后端自杀式崩溃

根因(日志实证):沙箱命令超时后 _interrupt_subprocess 发送的
CTRL_BREAK_EVENT 会投递到本进程自身(即使子进程带
CREATE_NEW_PROCESS_GROUP),无保护的后端收到后直接终止,
终端仅显示 ^C、无任何报错。沙箱 git 病态缓慢导致频繁超时,
是该崩溃的触发器(已由上一个提交的 git 配置修复)。

- terminal_ops/run.py: Windows 超时中断直接 taskkill /F /T 杀整棵树
  (抽出共用 _taskkill_tree),不再尝试 CTRL_BREAK 优雅中断
- background_command_manager.py: _terminate_pid 同步弃用 CTRL_BREAK
- 拆除全部 [SBX-DEBUG] 诊断埋点,删除 modules/sbx_debug.py
  (探针会吞掉 Ctrl+C,诊断任务完成,归还正常信号行为)
This commit is contained in:
JOJO 2026-07-31 19:33:43 +08:00
parent 5ef0a04ed0
commit 7aab7da8aa
3 changed files with 34 additions and 208 deletions

View File

@ -18,11 +18,6 @@ from modules.host_sandbox_runner import (
build_host_sandbox_plan, build_host_sandbox_plan,
host_sandbox_enabled, host_sandbox_enabled,
) )
from modules import sbx_debug # [SBX-DEBUG]
try: # [SBX-DEBUG]
sbx_debug.install_ctrl_event_probes("bg_manager") # [SBX-DEBUG]
except Exception: # [SBX-DEBUG]
pass # [SBX-DEBUG]
TERMINAL_STATUSES = {"completed", "failed", "timeout", "cancelled"} TERMINAL_STATUSES = {"completed", "failed", "timeout", "cancelled"}
@ -294,7 +289,6 @@ class BackgroundCommandManager:
rec["pid"] = process.pid rec["pid"] = process.pid
rec["updated_at"] = time.time() rec["updated_at"] = time.time()
self._processes[command_id] = process self._processes[command_id] = process
sbx_debug.sbx_log("BG-SPAWNED", pid=getattr(process, "pid", None), command=str(command)[:200], timeout=timeout, console_pids=sbx_debug.console_pids()) # [SBX-DEBUG]
def _reader(stream, collector, rec_key: str): def _reader(stream, collector, rec_key: str):
try: try:
@ -327,8 +321,7 @@ class BackgroundCommandManager:
except subprocess.TimeoutExpired: except subprocess.TimeoutExpired:
status = "timeout" status = "timeout"
message = f"命令执行超时 ({timeout}秒)" message = f"命令执行超时 ({timeout}秒)"
# 跨平台终止POSIX killpg(SIGINT→SIGKILL)Windows CTRL_BREAK→taskkill # 跨平台终止POSIX killpg(SIGINT→SIGKILL)Windows taskkill /F /T
sbx_debug.sbx_log_stack("BG-TIMEOUT-TERMINATE", pid=getattr(process, "pid", None), timeout=timeout) # [SBX-DEBUG]
self._terminate_pid(process.pid) self._terminate_pid(process.pid)
try: try:
process.wait(timeout=2) process.wait(timeout=2)
@ -436,23 +429,13 @@ class BackgroundCommandManager:
return True return True
def _terminate_pid(self, pid: Any) -> bool: def _terminate_pid(self, pid: Any) -> bool:
sbx_debug.sbx_log_stack("TERMINATE-PID", pid=pid) # [SBX-DEBUG]
normalized = self._coerce_pid(pid) normalized = self._coerce_pid(pid)
if not normalized: if not normalized:
return False return False
if os.name == "nt": if os.name == "nt":
# Windows无 killpg/SIGKILL。先尝试 CTRL_BREAK_EVENT 优雅中断 # Windows无 killpg/SIGKILL。弃用 CTRL_BREAK_EVENT——实测该事件会
# (子进程以 start_new_session=True 启动,即独立进程组), # 投递到本进程自身2026-07 WSL 沙箱排查,见 terminal_ops/run.py 注释),
# 超时后用 taskkill /F /T 强制终止整棵进程树,避免孙进程残留。 # 直接 taskkill /F /T 强制终止整棵进程树,避免孙进程残留。
try:
os.kill(normalized, signal.CTRL_BREAK_EVENT)
except Exception:
pass
deadline = time.time() + 2.0
while time.time() < deadline:
if not self._is_pid_alive(normalized):
return True
time.sleep(0.05)
try: try:
subprocess.run( subprocess.run(
["taskkill", "/F", "/T", "/PID", str(normalized)], ["taskkill", "/F", "/T", "/PID", str(normalized)],

View File

@ -1,147 +0,0 @@
# modules/sbx_debug.py - 沙箱崩溃排查的临时调试日志模块(诊断完成后移除)
#
# 双通道日志:
# 1) UDP 发送到独立监控进程(后端死亡前消息已送达,监控进程独立存活)
# 2) 文件追加 .wsl-exp/sbx_debug.log每行立即落盘保底冗余
# 环境变量:
# SBX_DEBUG=0 关闭全部调试日志
# SBX_DEBUG_PORT=9956 UDP 监控端口
# SBX_DEBUG_SWALLOW_CTRL=0 控制台事件不吞掉(记录后恢复默认行为=真实死亡)
from __future__ import annotations
import datetime
import os
import signal
import socket
import threading
import traceback
_ENABLED = os.environ.get("SBX_DEBUG", "1").strip().lower() not in {"0", "false", "no", "off"}
_PORT = int(os.environ.get("SBX_DEBUG_PORT", "8092") or "8092") # UDP与后端 TCP 端口不冲突
_SWALLOW = os.environ.get("SBX_DEBUG_SWALLOW_CTRL", "1").strip().lower() not in {"0", "false", "no", "off"}
_LOCK = threading.Lock()
_SOCK = None
_PROBES_INSTALLED = False
def _log_path() -> str:
try:
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
except Exception:
root = os.getcwd()
d = os.path.join(root, ".wsl-exp")
try:
os.makedirs(d, exist_ok=True)
except Exception:
pass
return os.path.join(d, "sbx_debug.log")
def _get_sock():
global _SOCK
if _SOCK is None:
try:
_SOCK = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
except Exception:
_SOCK = False
return _SOCK or None
def sbx_log(event: str, **fields) -> None:
"""写一条调试日志(任何异常都吞掉,绝不影响主流程)。"""
if not _ENABLED:
return
try:
ts = datetime.datetime.now().strftime("%H:%M:%S.%f")[:-3]
parts = [f"{k}={fields[k]!r}" for k in sorted(fields)]
line = f"[{ts}] [pid={os.getpid()}] [th={threading.current_thread().name}] {event}"
if parts:
line += " | " + " ".join(parts)
data = (line + "\n").encode("utf-8", "replace")
with _LOCK:
sock = _get_sock()
if sock is not None:
try:
sock.sendto(data, ("127.0.0.1", _PORT))
except Exception:
pass
try:
with open(_log_path(), "a", encoding="utf-8") as f:
f.write(line + "\n")
except Exception:
pass
except Exception:
pass
def sbx_log_stack(event: str, **fields) -> None:
"""写一条带调用堆栈的调试日志。"""
try:
stack = "".join(traceback.format_stack(limit=14)[:-2])
except Exception:
stack = "<no stack>"
fields["stack"] = stack
sbx_log(event, **fields)
def console_pids():
"""返回当前控制台附加的进程 pid 列表Windows用于判断子进程是否共享后端控制台。"""
if os.name != "nt":
return []
try:
import ctypes
arr = (ctypes.c_ulong * 64)()
got = ctypes.windll.kernel32.GetConsoleProcessList(arr, 64)
if not got:
return []
return [int(arr[i]) for i in range(min(got, 64))]
except Exception:
return ["?"]
def install_ctrl_event_probes(tag: str = "backend") -> None:
"""安装 SIGINT/SIGBREAK 探针:控制台事件到达时先记录日志。
SBX_DEBUG_SWALLOW_CTRL=1默认记录后吞掉事件进程存活可继续观察后续行为
=0记录后恢复原处理器语义SIGINTKeyboardInterruptSIGBREAK默认终止
只能在主线程安装失败静默
"""
global _PROBES_INSTALLED
if not _ENABLED or os.name != "nt" or _PROBES_INSTALLED:
return
if threading.current_thread() is not threading.main_thread():
sbx_log("CTRL-PROBES-SKIPPED", reason="not main thread", tag=tag)
return
def _make_handler(sig, name):
prev = signal.getsignal(sig)
def _handler(signum, frame):
sbx_log_stack("CTRL-EVENT-RECEIVED", signal=name, tag=tag, swallowed=_SWALLOW)
if _SWALLOW:
return
if callable(prev):
prev(signum, frame)
return
try:
signal.signal(sig, signal.SIG_DFL)
os.kill(os.getpid(), sig)
except Exception:
os._exit(128 + int(signum))
return _handler
try:
signal.signal(signal.SIGINT, _make_handler(signal.SIGINT, "SIGINT(CTRL_C)"))
except Exception:
pass
try:
sigbreak = getattr(signal, "SIGBREAK", None)
if sigbreak is not None:
signal.signal(sigbreak, _make_handler(sigbreak, "SIGBREAK(CTRL_BREAK)"))
except Exception:
pass
_PROBES_INSTALLED = True
sbx_log("CTRL-PROBES-INSTALLED", tag=tag, swallow=_SWALLOW, port=_PORT)

View File

@ -40,11 +40,6 @@ from modules.host_sandbox_runner import (
build_host_sandbox_readonly_plan, build_host_sandbox_readonly_plan,
host_sandbox_enabled, host_sandbox_enabled,
) )
from modules import sbx_debug # [SBX-DEBUG]
try: # [SBX-DEBUG]
sbx_debug.install_ctrl_event_probes("terminal_ops") # [SBX-DEBUG]
except Exception: # [SBX-DEBUG]
pass # [SBX-DEBUG]
if TYPE_CHECKING: if TYPE_CHECKING:
from modules.user_container_manager import ContainerHandle from modules.user_container_manager import ContainerHandle
@ -54,22 +49,38 @@ if TYPE_CHECKING:
class RunMixin: class RunMixin:
"""TerminalOperator run 能力 mixin。""" """TerminalOperator run 能力 mixin。"""
async def _interrupt_subprocess(self, process) -> None: @staticmethod
"""超时后的优雅中断。 async def _taskkill_tree(process) -> None:
"""Windowstaskkill /F /T 强制终止整棵进程树,失败退化为 process.kill()。
POSIX killpg(SIGINT) 中断整个进程组Windows 没有 killpg/SIGINT 语义 避免 shell 子进程被杀后孙进程残留为孤儿
子进程以 start_new_session=True 启动映射为 CREATE_NEW_PROCESS_GROUP
可用 CTRL_BREAK_EVENT 发送到该进程组失败则直接强杀
""" """
sbx_debug.sbx_log_stack("INTERRUPT-SUBPROCESS", pid=getattr(process, "pid", None), nt=(os.name == "nt")) # [SBX-DEBUG]
if os.name == "nt":
try: try:
process.send_signal(signal.CTRL_BREAK_EVENT) await asyncio.to_thread(
subprocess.run,
["taskkill", "/F", "/T", "/PID", str(process.pid)],
capture_output=True,
timeout=5,
)
return
except Exception: except Exception:
pass
try: try:
process.kill() process.kill()
except Exception: except Exception:
pass pass
async def _interrupt_subprocess(self, process) -> None:
"""超时后的中断。
POSIX killpg(SIGINT) 中断整个进程组
Windows 弃用 CTRL_BREAK_EVENT实测2026-07 WSL 沙箱排查即使子进程以
start_new_session=TrueCREATE_NEW_PROCESS_GROUP启动控制台事件仍会
投递到本进程自身把后端一并杀死终端仅显示 ^C无任何报错
Windows 控制台事件没有安全的定向语义超时场景直接 taskkill 杀整棵树
"""
if os.name == "nt":
await self._taskkill_tree(process)
return return
try: try:
os.killpg(process.pid, signal.SIGINT) os.killpg(process.pid, signal.SIGINT)
@ -85,22 +96,8 @@ class RunMixin:
POSIX killpg(SIGKILL)Windows 没有 SIGKILL taskkill /F /T 终止整棵进程树 POSIX killpg(SIGKILL)Windows 没有 SIGKILL taskkill /F /T 终止整棵进程树
避免 shell 子进程被杀后孙进程残留为孤儿 避免 shell 子进程被杀后孙进程残留为孤儿
""" """
sbx_debug.sbx_log_stack("KILL-SUBPROCESS", pid=getattr(process, "pid", None), nt=(os.name == "nt")) # [SBX-DEBUG]
if os.name == "nt": if os.name == "nt":
try: await self._taskkill_tree(process)
await asyncio.to_thread(
subprocess.run,
["taskkill", "/F", "/T", "/PID", str(process.pid)],
capture_output=True,
timeout=5,
)
return
except Exception:
pass
try:
process.kill()
except Exception:
pass
return return
try: try:
os.killpg(process.pid, signal.SIGKILL) os.killpg(process.pid, signal.SIGKILL)
@ -168,7 +165,6 @@ class RunMixin:
network_permission: Optional[str] = None, network_permission: Optional[str] = None,
) -> Dict: ) -> Dict:
start_ts = time.time() start_ts = time.time()
sbx_debug.sbx_log("RUN-CMD-ENTER", command=str(command)[:200], timeout=timeout, mode=getattr(self, "host_execution_mode", None), sandbox_write=sandbox_write_access) # [SBX-DEBUG]
try: try:
process = None process = None
exec_cmd = None exec_cmd = None
@ -242,7 +238,6 @@ class RunMixin:
os.close(seccomp_fd) os.close(seccomp_fd)
except OSError: except OSError:
pass pass
sbx_debug.sbx_log("SANDBOX-SPAWNED", pid=getattr(process, "pid", None), argv0=(cmd_args[0] if cmd_args else None), argv_tail=str(cmd_args[-3:])[:160], console_pids=sbx_debug.console_pids()) # [SBX-DEBUG]
elif use_host_sandbox: elif use_host_sandbox:
return { return {
"success": False, "success": False,
@ -307,13 +302,11 @@ class RunMixin:
pass pass
timed_out = False timed_out = False
sbx_debug.sbx_log("WAIT-BEGIN", pid=getattr(process, "pid", None), timeout=timeout) # [SBX-DEBUG]
try: try:
try: try:
await asyncio.wait_for(process.wait(), timeout=timeout) await asyncio.wait_for(process.wait(), timeout=timeout)
except asyncio.TimeoutError: except asyncio.TimeoutError:
timed_out = True timed_out = True
sbx_debug.sbx_log_stack("TIMEOUT-INTERRUPT", pid=getattr(process, "pid", None), timeout=timeout) # [SBX-DEBUG]
await self._interrupt_subprocess(process) await self._interrupt_subprocess(process)
try: try:
await asyncio.wait_for(process.wait(), timeout=2) await asyncio.wait_for(process.wait(), timeout=2)
@ -322,7 +315,6 @@ class RunMixin:
await process.wait() await process.wait()
except asyncio.CancelledError: except asyncio.CancelledError:
# 用户主动停止任务或会话断开,立即终止子进程 # 用户主动停止任务或会话断开,立即终止子进程
sbx_debug.sbx_log_stack("CANCELLED-KILL", pid=getattr(process, "pid", None)) # [SBX-DEBUG]
await self._kill_subprocess(process) await self._kill_subprocess(process)
raise raise
finally: finally:
@ -331,7 +323,6 @@ class RunMixin:
# Task was destroyed but it is pending / unclosed transport 告警) # Task was destroyed but it is pending / unclosed transport 告警)
await _finish_reader_tasks(force=timed_out) await _finish_reader_tasks(force=timed_out)
sbx_debug.sbx_log("WAIT-END", pid=getattr(process, "pid", None), returncode=getattr(process, "returncode", None), timed_out=timed_out, stdout_len=sum(len(c) for c in stdout_buf), stderr_len=sum(len(c) for c in stderr_buf)) # [SBX-DEBUG]
# 非超时场景下兜底再读一次,防止剩余缓冲未被读取 # 非超时场景下兜底再读一次,防止剩余缓冲未被读取
if not timed_out: if not timed_out:
try: try:
@ -398,7 +389,6 @@ class RunMixin:
"elapsed_ms": int((time.time() - start_ts) * 1000) "elapsed_ms": int((time.time() - start_ts) * 1000)
} }
except Exception as exc: except Exception as exc:
sbx_debug.sbx_log_stack("RUN-EXCEPTION", error=str(exc)[:200]) # [SBX-DEBUG]
return { return {
"success": False, "success": False,
"status": "error", "status": "error",