fix(modules): 修复 Windows UTF-8 编码链(对照 Codex #7290 类问题)

- persistent_terminal 解码写死 GBK 改为 UTF-8 优先 + GBK 回退逐块自适应,GBK 边界感知 + _pending_bytes 暂存半字,进程结束冲刷

- start.py:Windows 默认 cmd.exe 启动后注入 chcp 65001,与 PYTHONIOENCODING=utf-8 对齐,消除输出/解码自相矛盾

- 5 处 subprocess text=True 补 encoding=utf-8(后台命令x3、MCP stdio、版本管理、病毒扫描、容器管理),照 server/status/git.py 示范

- main.py:补 PYTHONUTF8/PYTHONIOENCODING 兜底 + stdout/stderr reconfigure;修复 print 中被双重编码固化的 emoji

- docx 验证器 XML 改二进制读取,由 lxml 按声明自解码
This commit is contained in:
JOJO 2026-07-31 22:39:51 +08:00
parent 89effa71a8
commit 2c2c5aba4d
10 changed files with 129 additions and 11 deletions

View File

@ -760,7 +760,9 @@ class BaseSchemaValidator:
)
schema = lxml.etree.XMLSchema(xsd_doc)
with open(xml_file, "r") as f:
# 二进制模式交由 lxml 按 XML 声明自解码;文本模式在 Windows 会按
# localecp936预解码 UTF-8 字节,含中文的文档会解析出错或乱码
with open(xml_file, "rb") as f:
xml_doc = lxml.etree.parse(f)
xml_doc, _ = self._remove_template_tags_from_text_nodes(xml_doc)

17
main.py
View File

@ -250,7 +250,7 @@ class AgentSystem:
await self.main_terminal.save_state()
print(f"{OUTPUT_FORMATS['success']} 系统已安全退出")
print("\n👋 再见!\n")
print("\n👋 再见!\n")
async def main():
"""主函数"""
@ -269,10 +269,21 @@ if __name__ == "__main__":
os.system("chcp 65001 > nul") # 设置控制台代码页为UTF-8
except:
pass
# 后代 Python 进程默认编码兜底chcp 只改控制台显示,不改 Python
# 的 open()/stdout 默认编码(仍是 locale 的 cp936这两个变量会
# 被 agent 启动的所有子 Python 进程继承,避免脚本按 GBK 读写/输出
os.environ.setdefault("PYTHONUTF8", "1")
os.environ.setdefault("PYTHONIOENCODING", "utf-8")
# 当前进程 stdout/stderr 同样强制 UTF-8防止 print 中文按 cp936 输出
for _stream in (sys.stdout, sys.stderr):
try:
_stream.reconfigure(encoding="utf-8", errors="replace")
except (AttributeError, ValueError):
pass
asyncio.run(main())
except KeyboardInterrupt:
print("\n\n👋 再见!")
print("\n\n👋 再见!")
sys.exit(0)
except Exception as e:
print(f"\n{OUTPUT_FORMATS['error']} 程序异常退出: {e}")

View File

@ -248,6 +248,7 @@ class BackgroundCommandManager:
env=plan.env,
start_new_session=True,
text=True,
encoding="utf-8",
errors="replace",
bufsize=1,
pass_fds=pass_fds,
@ -268,6 +269,7 @@ class BackgroundCommandManager:
env=env,
start_new_session=True,
text=True,
encoding="utf-8",
errors="replace",
bufsize=1,
)
@ -279,6 +281,7 @@ class BackgroundCommandManager:
env=env,
start_new_session=True,
text=True,
encoding="utf-8",
errors="replace",
bufsize=1,
)

View File

@ -247,12 +247,16 @@ class _StdioMCPClient:
return
launch_cmd, cwd, env = self._prepare_launch()
try:
# MCP 协议JSON-RPC over stdio字节级要求 UTF-8必须显式指定
# 否则 Windows 上 text=True 默认按 localecp936编解码中文 payload 会损坏
self.process = subprocess.Popen(
launch_cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
encoding="utf-8",
errors="replace",
bufsize=1,
cwd=cwd or None,
env=env,

View File

@ -122,7 +122,8 @@ class PersistentTerminalBase:
self.output_queue = queue.Queue()
self.reader_thread = None
self.is_reading = False
self._decoder = None
# _read_output 自适应解码io.py用于暂存被块边界切断的不完整多字节序列
self._pending_bytes = b''
# 状态标志
self.is_interactive = False # 是否在等待输入

View File

@ -63,20 +63,46 @@ except ImportError:
)
def _gbk_complete_prefix_len(data: bytes) -> int:
"""返回 data 中按 GBK 可完整解析的前缀长度。
GBKASCII 单字节0x00-0x7F双字节 lead 0x81-0xFE + trail 0x40-0xFE不含 0x7F
末尾孤立的 lead byte 视为被块边界切断的半字不计入前缀留给下一块拼合
"""
i = 0
n = len(data)
while i < n:
b = data[i]
if b <= 0x7F:
i += 1
elif 0x81 <= b <= 0xFE:
if i + 1 < n and 0x40 <= data[i + 1] <= 0xFE and data[i + 1] != 0x7F:
i += 2
else:
break # 孤立 lead byte可能是边界切断的半字
else:
i += 1 # 0x80/0xFF 非法字节,按单字节跳过(由 replace 兜底)
return i
class IoMixin:
"""PersistentTerminal io 能力 mixin。"""
def _read_output(self):
"""后台线程:持续读取输出(修复版,正确处理编码)"""
if self._decoder is None:
encoding = 'gbk' if self.is_windows else 'utf-8'
self._decoder = codecs.getincrementaldecoder(encoding)(errors='replace')
"""后台线程:持续读取输出。
Windows 下不再写死 GBKcmd 会话启动时已注入 chcp 65001 start.py
PYTHONIOENCODING=utf-8 使子进程 Python 程序输出 UTF-8但用户可能
手动 chcp 936或运行只输出 GBK 的旧程序因此逐块自适应解码
先试 UTF-8失败则整块按 GBK块边界切断的多字节序列用
_pending_bytes 暂存与下一块拼合后再解 _decode_chunk
"""
while self.is_reading and self.process:
try:
# 读取任意字节块,不依赖换行符
chunk = self.process.stdout.read(1024)
if chunk:
text = self._decoder.decode(chunk)
text = self._decode_chunk(chunk)
if text:
self.output_queue.put(text)
self._process_output(text)
@ -87,12 +113,63 @@ class IoMixin:
else:
# 没有输出,短暂休眠
time.sleep(0.01)
except Exception as e:
# 不要因为单个错误而停止
print(f"[Terminal] 读取输出警告: {e}")
time.sleep(0.01)
continue
# 进程结束后冲刷残留的不完整字节(多为被截断的多字节序列半字)
pending = getattr(self, '_pending_bytes', b'')
if pending:
fallback = 'gbk' if self.is_windows else 'utf-8'
text = pending.decode(fallback, errors='replace')
self._pending_bytes = b''
if text:
self.output_queue.put(text)
self._process_output(text)
def _decode_chunk(self, chunk: bytes) -> str:
"""把 stdout 字节块解码为文本WindowsUTF-8 优先、GBK 回退)。
判定依据UTF-8 有严格的多字节结构校验GBK 字节流极少恰好构成
合法 UTF-8因此UTF-8 能解是强信号解不了再按 GBK
两种路径都会把块末尾不完整的多字节序列暂存 _pending_bytes
与下一块拼合后再解避免 read 边界切断汉字
已知限制业界通病VS Code/Windows Terminal 同样无法处理
极少数 GBK 汉字对的字节恰好也是合法 UTF-8目录= C4BF C2BC
会解为 Ŀ¼此时若后续块紧跟 UTF-8 内容会连锁误判终端会话已
注入 chcp 65001start.py正常路径全程 UTF-8GBK 仅为兜底
该巧合仅影响显示不丢数据故不做罕见字符惩罚式启发式猜测
"""
data = getattr(self, '_pending_bytes', b'') + chunk
self._pending_bytes = b''
if not data:
return ''
if not self.is_windows:
return data.decode('utf-8', errors='replace')
try:
return data.decode('utf-8')
except UnicodeDecodeError as exc:
# 末尾 1~3 字节可能是不完整 UTF-8 序列read 边界切断),暂存待拼
if exc.start >= len(data) - 3:
head, pending = data[:exc.start], data[exc.start:]
try:
text = head.decode('utf-8')
self._pending_bytes = pending
return text
except UnicodeDecodeError:
pass # head 也含非法序列 → 并非 UTF-8 流,落入 GBK 路径
# 非法序列位于中部(或 UTF-8 边界暂存失败)→ 确定不是 UTF-8
# (如 cmd 内置命令的 GBK 输出),按 GBK 边界感知解码
return self._decode_gbk_boundary(data)
def _decode_gbk_boundary(self, data: bytes) -> str:
"""按 GBK 解码,末尾不完整的双字节序列(被切断的半字)暂存待拼。"""
safe_len = _gbk_complete_prefix_len(data)
self._pending_bytes = data[safe_len:]
return data[:safe_len].decode('gbk', errors='replace')
def _decode_output(self, data):
"""安全地解码输出"""

View File

@ -215,6 +215,18 @@ class StartMixin:
bufsize=0,
env=env
)
if self.is_windows and not self.host_shell_command:
# 默认 cmd.exe注入 chcp 65001把会话统一切换到 UTF-8 代码页。
# 1) 与上方 PYTHONIOENCODING=utf-8 对齐(子进程 Python 程序输出 UTF-8
# 2) 让 cmd 内置命令dir/echo 等)也输出 UTF-8配合 io.py 的
# UTF-8 优先自适应解码,避免中文输出乱码;
# 3) 仅默认 shell 注入——用户自定义 shell如 pwsh默认即为 UTF-8。
try:
process.stdin.write(b'chcp 65001 >nul\r\n')
process.stdin.flush()
except (OSError, ValueError, BrokenPipeError):
# 注入失败不致命io.py 的 GBK 回退仍能处理默认代码页输出
pass
return process
def _start_docker_terminal(self):

View File

@ -183,6 +183,8 @@ class UploadQuarantineManager:
command,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=self.clamav_timeout,
check=False,
)

View File

@ -343,6 +343,8 @@ class UserContainerManager:
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
encoding="utf-8",
errors="replace",
check=False,
)
if result.returncode != 0:

View File

@ -84,6 +84,10 @@ class ConversationVersioningManager:
cmd,
capture_output=True,
text=True,
# git 输出固定为 UTF-8Windows text=True 默认按 localecp936
# 解码会让中文路径/提交信息乱码(同 server/status/git.py 的处理)
encoding="utf-8",
errors="replace",
timeout=timeout_seconds,
)
except subprocess.TimeoutExpired as exc: