fix(windows): 超时终止链弃用 CTRL_BREAK_EVENT，修复后端自杀式崩溃
根因（日志实证）：沙箱命令超时后 _interrupt_subprocess 发送的 CTRL_BREAK_EVENT 会投递到本进程自身（即使子进程带 CREATE_NEW_PROCESS_GROUP），无保护的后端收到后直接终止， 终端仅显示 ^C、无任何报错。沙箱 git 病态缓慢导致频繁超时， 是该崩溃的触发器（已由上一个提交的 git 配置修复）。 - terminal_ops/run.py: Windows 超时中断直接 taskkill /F /T 杀整棵树 （抽出共用 _taskkill_tree），不再尝试 CTRL_BREAK 优雅中断 - background_command_manager.py: _terminate_pid 同步弃用 CTRL_BREAK - 拆除全部 [SBX-DEBUG] 诊断埋点，删除 modules/sbx_debug.py （探针会吞掉 Ctrl+C，诊断任务完成，归还正常信号行为）
This commit is contained in:
parent
5ef0a04ed0
commit
7aab7da8aa
@ -18,11 +18,6 @@ from modules.host_sandbox_runner import (
|
||||
build_host_sandbox_plan,
|
||||
host_sandbox_enabled,
|
||||
)
|
||||
from modules import sbx_debug # [SBX-DEBUG]
|
||||
try: # [SBX-DEBUG]
|
||||
sbx_debug.install_ctrl_event_probes("bg_manager") # [SBX-DEBUG]
|
||||
except Exception: # [SBX-DEBUG]
|
||||
pass # [SBX-DEBUG]
|
||||
|
||||
|
||||
TERMINAL_STATUSES = {"completed", "failed", "timeout", "cancelled"}
|
||||
@ -294,7 +289,6 @@ class BackgroundCommandManager:
|
||||
rec["pid"] = process.pid
|
||||
rec["updated_at"] = time.time()
|
||||
self._processes[command_id] = process
|
||||
sbx_debug.sbx_log("BG-SPAWNED", pid=getattr(process, "pid", None), command=str(command)[:200], timeout=timeout, console_pids=sbx_debug.console_pids()) # [SBX-DEBUG]
|
||||
|
||||
def _reader(stream, collector, rec_key: str):
|
||||
try:
|
||||
@ -327,8 +321,7 @@ class BackgroundCommandManager:
|
||||
except subprocess.TimeoutExpired:
|
||||
status = "timeout"
|
||||
message = f"命令执行超时 ({timeout}秒)"
|
||||
# 跨平台终止：POSIX killpg(SIGINT→SIGKILL)，Windows CTRL_BREAK→taskkill
|
||||
sbx_debug.sbx_log_stack("BG-TIMEOUT-TERMINATE", pid=getattr(process, "pid", None), timeout=timeout) # [SBX-DEBUG]
|
||||
# 跨平台终止：POSIX killpg(SIGINT→SIGKILL)，Windows taskkill /F /T
|
||||
self._terminate_pid(process.pid)
|
||||
try:
|
||||
process.wait(timeout=2)
|
||||
@ -436,23 +429,13 @@ class BackgroundCommandManager:
|
||||
return True
|
||||
|
||||
def _terminate_pid(self, pid: Any) -> bool:
|
||||
sbx_debug.sbx_log_stack("TERMINATE-PID", pid=pid) # [SBX-DEBUG]
|
||||
normalized = self._coerce_pid(pid)
|
||||
if not normalized:
|
||||
return False
|
||||
if os.name == "nt":
|
||||
# Windows：无 killpg/SIGKILL。先尝试 CTRL_BREAK_EVENT 优雅中断
|
||||
# （子进程以 start_new_session=True 启动，即独立进程组），
|
||||
# 超时后用 taskkill /F /T 强制终止整棵进程树，避免孙进程残留。
|
||||
try:
|
||||
os.kill(normalized, signal.CTRL_BREAK_EVENT)
|
||||
except Exception:
|
||||
pass
|
||||
deadline = time.time() + 2.0
|
||||
while time.time() < deadline:
|
||||
if not self._is_pid_alive(normalized):
|
||||
return True
|
||||
time.sleep(0.05)
|
||||
# Windows：无 killpg/SIGKILL。弃用 CTRL_BREAK_EVENT——实测该事件会
|
||||
# 投递到本进程自身（2026-07 WSL 沙箱排查，见 terminal_ops/run.py 注释），
|
||||
# 直接 taskkill /F /T 强制终止整棵进程树，避免孙进程残留。
|
||||
try:
|
||||
subprocess.run(
|
||||
["taskkill", "/F", "/T", "/PID", str(normalized)],
|
||||
|
||||
@ -1,147 +0,0 @@
|
||||
# modules/sbx_debug.py - 沙箱崩溃排查的临时调试日志模块（诊断完成后移除）
|
||||
#
|
||||
# 双通道日志：
|
||||
# 1) UDP 发送到独立监控进程（后端死亡前消息已送达，监控进程独立存活）
|
||||
# 2) 文件追加 .wsl-exp/sbx_debug.log（每行立即落盘，保底冗余）
|
||||
# 环境变量：
|
||||
# SBX_DEBUG=0 关闭全部调试日志
|
||||
# SBX_DEBUG_PORT=9956 UDP 监控端口
|
||||
# SBX_DEBUG_SWALLOW_CTRL=0 控制台事件不吞掉（记录后恢复默认行为=真实死亡）
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import threading
|
||||
import traceback
|
||||
|
||||
_ENABLED = os.environ.get("SBX_DEBUG", "1").strip().lower() not in {"0", "false", "no", "off"}
|
||||
_PORT = int(os.environ.get("SBX_DEBUG_PORT", "8092") or "8092") # UDP，与后端 TCP 端口不冲突
|
||||
_SWALLOW = os.environ.get("SBX_DEBUG_SWALLOW_CTRL", "1").strip().lower() not in {"0", "false", "no", "off"}
|
||||
|
||||
_LOCK = threading.Lock()
|
||||
_SOCK = None
|
||||
_PROBES_INSTALLED = False
|
||||
|
||||
|
||||
def _log_path() -> str:
|
||||
try:
|
||||
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
except Exception:
|
||||
root = os.getcwd()
|
||||
d = os.path.join(root, ".wsl-exp")
|
||||
try:
|
||||
os.makedirs(d, exist_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
return os.path.join(d, "sbx_debug.log")
|
||||
|
||||
|
||||
def _get_sock():
|
||||
global _SOCK
|
||||
if _SOCK is None:
|
||||
try:
|
||||
_SOCK = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
except Exception:
|
||||
_SOCK = False
|
||||
return _SOCK or None
|
||||
|
||||
|
||||
def sbx_log(event: str, **fields) -> None:
|
||||
"""写一条调试日志（任何异常都吞掉，绝不影响主流程）。"""
|
||||
if not _ENABLED:
|
||||
return
|
||||
try:
|
||||
ts = datetime.datetime.now().strftime("%H:%M:%S.%f")[:-3]
|
||||
parts = [f"{k}={fields[k]!r}" for k in sorted(fields)]
|
||||
line = f"[{ts}] [pid={os.getpid()}] [th={threading.current_thread().name}] {event}"
|
||||
if parts:
|
||||
line += " | " + " ".join(parts)
|
||||
data = (line + "\n").encode("utf-8", "replace")
|
||||
with _LOCK:
|
||||
sock = _get_sock()
|
||||
if sock is not None:
|
||||
try:
|
||||
sock.sendto(data, ("127.0.0.1", _PORT))
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
with open(_log_path(), "a", encoding="utf-8") as f:
|
||||
f.write(line + "\n")
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def sbx_log_stack(event: str, **fields) -> None:
|
||||
"""写一条带调用堆栈的调试日志。"""
|
||||
try:
|
||||
stack = "".join(traceback.format_stack(limit=14)[:-2])
|
||||
except Exception:
|
||||
stack = "<no stack>"
|
||||
fields["stack"] = stack
|
||||
sbx_log(event, **fields)
|
||||
|
||||
|
||||
def console_pids():
|
||||
"""返回当前控制台附加的进程 pid 列表（Windows），用于判断子进程是否共享后端控制台。"""
|
||||
if os.name != "nt":
|
||||
return []
|
||||
try:
|
||||
import ctypes
|
||||
arr = (ctypes.c_ulong * 64)()
|
||||
got = ctypes.windll.kernel32.GetConsoleProcessList(arr, 64)
|
||||
if not got:
|
||||
return []
|
||||
return [int(arr[i]) for i in range(min(got, 64))]
|
||||
except Exception:
|
||||
return ["?"]
|
||||
|
||||
|
||||
def install_ctrl_event_probes(tag: str = "backend") -> None:
|
||||
"""安装 SIGINT/SIGBREAK 探针：控制台事件到达时先记录日志。
|
||||
|
||||
SBX_DEBUG_SWALLOW_CTRL=1（默认）：记录后吞掉事件，进程存活，可继续观察后续行为；
|
||||
=0：记录后恢复原处理器语义（SIGINT→KeyboardInterrupt；SIGBREAK→默认终止）。
|
||||
只能在主线程安装，失败静默。
|
||||
"""
|
||||
global _PROBES_INSTALLED
|
||||
if not _ENABLED or os.name != "nt" or _PROBES_INSTALLED:
|
||||
return
|
||||
if threading.current_thread() is not threading.main_thread():
|
||||
sbx_log("CTRL-PROBES-SKIPPED", reason="not main thread", tag=tag)
|
||||
return
|
||||
|
||||
def _make_handler(sig, name):
|
||||
prev = signal.getsignal(sig)
|
||||
|
||||
def _handler(signum, frame):
|
||||
sbx_log_stack("CTRL-EVENT-RECEIVED", signal=name, tag=tag, swallowed=_SWALLOW)
|
||||
if _SWALLOW:
|
||||
return
|
||||
if callable(prev):
|
||||
prev(signum, frame)
|
||||
return
|
||||
try:
|
||||
signal.signal(sig, signal.SIG_DFL)
|
||||
os.kill(os.getpid(), sig)
|
||||
except Exception:
|
||||
os._exit(128 + int(signum))
|
||||
|
||||
return _handler
|
||||
|
||||
try:
|
||||
signal.signal(signal.SIGINT, _make_handler(signal.SIGINT, "SIGINT(CTRL_C)"))
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
sigbreak = getattr(signal, "SIGBREAK", None)
|
||||
if sigbreak is not None:
|
||||
signal.signal(sigbreak, _make_handler(sigbreak, "SIGBREAK(CTRL_BREAK)"))
|
||||
except Exception:
|
||||
pass
|
||||
_PROBES_INSTALLED = True
|
||||
sbx_log("CTRL-PROBES-INSTALLED", tag=tag, swallow=_SWALLOW, port=_PORT)
|
||||
@ -40,11 +40,6 @@ from modules.host_sandbox_runner import (
|
||||
build_host_sandbox_readonly_plan,
|
||||
host_sandbox_enabled,
|
||||
)
|
||||
from modules import sbx_debug # [SBX-DEBUG]
|
||||
try: # [SBX-DEBUG]
|
||||
sbx_debug.install_ctrl_event_probes("terminal_ops") # [SBX-DEBUG]
|
||||
except Exception: # [SBX-DEBUG]
|
||||
pass # [SBX-DEBUG]
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from modules.user_container_manager import ContainerHandle
|
||||
@ -54,22 +49,38 @@ if TYPE_CHECKING:
|
||||
class RunMixin:
|
||||
"""TerminalOperator run 能力 mixin。"""
|
||||
|
||||
async def _interrupt_subprocess(self, process) -> None:
|
||||
"""超时后的优雅中断。
|
||||
@staticmethod
|
||||
async def _taskkill_tree(process) -> None:
|
||||
"""Windows：taskkill /F /T 强制终止整棵进程树，失败退化为 process.kill()。
|
||||
|
||||
POSIX 用 killpg(SIGINT) 中断整个进程组；Windows 没有 killpg/SIGINT 语义，
|
||||
子进程以 start_new_session=True 启动（映射为 CREATE_NEW_PROCESS_GROUP），
|
||||
可用 CTRL_BREAK_EVENT 发送到该进程组，失败则直接强杀。
|
||||
避免 shell 子进程被杀后孙进程残留为孤儿。
|
||||
"""
|
||||
sbx_debug.sbx_log_stack("INTERRUPT-SUBPROCESS", pid=getattr(process, "pid", None), nt=(os.name == "nt")) # [SBX-DEBUG]
|
||||
if os.name == "nt":
|
||||
try:
|
||||
process.send_signal(signal.CTRL_BREAK_EVENT)
|
||||
await asyncio.to_thread(
|
||||
subprocess.run,
|
||||
["taskkill", "/F", "/T", "/PID", str(process.pid)],
|
||||
capture_output=True,
|
||||
timeout=5,
|
||||
)
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
process.kill()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def _interrupt_subprocess(self, process) -> None:
|
||||
"""超时后的中断。
|
||||
|
||||
POSIX 用 killpg(SIGINT) 中断整个进程组。
|
||||
Windows 弃用 CTRL_BREAK_EVENT：实测（2026-07 WSL 沙箱排查）即使子进程以
|
||||
start_new_session=True（CREATE_NEW_PROCESS_GROUP）启动，控制台事件仍会
|
||||
投递到本进程自身，把后端一并杀死（终端仅显示 ^C、无任何报错）。
|
||||
Windows 控制台事件没有安全的定向语义，超时场景直接 taskkill 杀整棵树。
|
||||
"""
|
||||
if os.name == "nt":
|
||||
await self._taskkill_tree(process)
|
||||
return
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGINT)
|
||||
@ -85,22 +96,8 @@ class RunMixin:
|
||||
POSIX 用 killpg(SIGKILL)；Windows 没有 SIGKILL，用 taskkill /F /T 终止整棵进程树，
|
||||
避免 shell 子进程被杀后孙进程残留为孤儿。
|
||||
"""
|
||||
sbx_debug.sbx_log_stack("KILL-SUBPROCESS", pid=getattr(process, "pid", None), nt=(os.name == "nt")) # [SBX-DEBUG]
|
||||
if os.name == "nt":
|
||||
try:
|
||||
await asyncio.to_thread(
|
||||
subprocess.run,
|
||||
["taskkill", "/F", "/T", "/PID", str(process.pid)],
|
||||
capture_output=True,
|
||||
timeout=5,
|
||||
)
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
process.kill()
|
||||
except Exception:
|
||||
pass
|
||||
await self._taskkill_tree(process)
|
||||
return
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
@ -168,7 +165,6 @@ class RunMixin:
|
||||
network_permission: Optional[str] = None,
|
||||
) -> Dict:
|
||||
start_ts = time.time()
|
||||
sbx_debug.sbx_log("RUN-CMD-ENTER", command=str(command)[:200], timeout=timeout, mode=getattr(self, "host_execution_mode", None), sandbox_write=sandbox_write_access) # [SBX-DEBUG]
|
||||
try:
|
||||
process = None
|
||||
exec_cmd = None
|
||||
@ -242,7 +238,6 @@ class RunMixin:
|
||||
os.close(seccomp_fd)
|
||||
except OSError:
|
||||
pass
|
||||
sbx_debug.sbx_log("SANDBOX-SPAWNED", pid=getattr(process, "pid", None), argv0=(cmd_args[0] if cmd_args else None), argv_tail=str(cmd_args[-3:])[:160], console_pids=sbx_debug.console_pids()) # [SBX-DEBUG]
|
||||
elif use_host_sandbox:
|
||||
return {
|
||||
"success": False,
|
||||
@ -307,13 +302,11 @@ class RunMixin:
|
||||
pass
|
||||
|
||||
timed_out = False
|
||||
sbx_debug.sbx_log("WAIT-BEGIN", pid=getattr(process, "pid", None), timeout=timeout) # [SBX-DEBUG]
|
||||
try:
|
||||
try:
|
||||
await asyncio.wait_for(process.wait(), timeout=timeout)
|
||||
except asyncio.TimeoutError:
|
||||
timed_out = True
|
||||
sbx_debug.sbx_log_stack("TIMEOUT-INTERRUPT", pid=getattr(process, "pid", None), timeout=timeout) # [SBX-DEBUG]
|
||||
await self._interrupt_subprocess(process)
|
||||
try:
|
||||
await asyncio.wait_for(process.wait(), timeout=2)
|
||||
@ -322,7 +315,6 @@ class RunMixin:
|
||||
await process.wait()
|
||||
except asyncio.CancelledError:
|
||||
# 用户主动停止任务或会话断开，立即终止子进程
|
||||
sbx_debug.sbx_log_stack("CANCELLED-KILL", pid=getattr(process, "pid", None)) # [SBX-DEBUG]
|
||||
await self._kill_subprocess(process)
|
||||
raise
|
||||
finally:
|
||||
@ -331,7 +323,6 @@ class RunMixin:
|
||||
# （Task was destroyed but it is pending / unclosed transport 告警）
|
||||
await _finish_reader_tasks(force=timed_out)
|
||||
|
||||
sbx_debug.sbx_log("WAIT-END", pid=getattr(process, "pid", None), returncode=getattr(process, "returncode", None), timed_out=timed_out, stdout_len=sum(len(c) for c in stdout_buf), stderr_len=sum(len(c) for c in stderr_buf)) # [SBX-DEBUG]
|
||||
# 非超时场景下兜底再读一次，防止剩余缓冲未被读取
|
||||
if not timed_out:
|
||||
try:
|
||||
@ -398,7 +389,6 @@ class RunMixin:
|
||||
"elapsed_ms": int((time.time() - start_ts) * 1000)
|
||||
}
|
||||
except Exception as exc:
|
||||
sbx_debug.sbx_log_stack("RUN-EXCEPTION", error=str(exc)[:200]) # [SBX-DEBUG]
|
||||
return {
|
||||
"success": False,
|
||||
"status": "error",
|
||||
|
||||
Loading…
Reference in New Issue
Block a user