fix(modules): 修复 Windows UTF-8 编码链(对照 Codex #7290 类问题)
- persistent_terminal 解码写死 GBK 改为 UTF-8 优先 + GBK 回退逐块自适应,GBK 边界感知 + _pending_bytes 暂存半字,进程结束冲刷 - start.py:Windows 默认 cmd.exe 启动后注入 chcp 65001,与 PYTHONIOENCODING=utf-8 对齐,消除输出/解码自相矛盾 - 5 处 subprocess text=True 补 encoding=utf-8(后台命令x3、MCP stdio、版本管理、病毒扫描、容器管理),照 server/status/git.py 示范 - main.py:补 PYTHONUTF8/PYTHONIOENCODING 兜底 + stdout/stderr reconfigure;修复 print 中被双重编码固化的 emoji - docx 验证器 XML 改二进制读取,由 lxml 按声明自解码
This commit is contained in:
parent
89effa71a8
commit
2c2c5aba4d
@ -760,7 +760,9 @@ class BaseSchemaValidator:
|
||||
)
|
||||
schema = lxml.etree.XMLSchema(xsd_doc)
|
||||
|
||||
with open(xml_file, "r") as f:
|
||||
# 二进制模式交由 lxml 按 XML 声明自解码;文本模式在 Windows 会按
|
||||
# locale(cp936)预解码 UTF-8 字节,含中文的文档会解析出错或乱码
|
||||
with open(xml_file, "rb") as f:
|
||||
xml_doc = lxml.etree.parse(f)
|
||||
|
||||
xml_doc, _ = self._remove_template_tags_from_text_nodes(xml_doc)
|
||||
|
||||
17
main.py
17
main.py
@ -250,7 +250,7 @@ class AgentSystem:
|
||||
await self.main_terminal.save_state()
|
||||
|
||||
print(f"{OUTPUT_FORMATS['success']} 系统已安全退出")
|
||||
print("\nð 再见!\n")
|
||||
print("\n👋 再见!\n")
|
||||
|
||||
async def main():
|
||||
"""主函数"""
|
||||
@ -269,10 +269,21 @@ if __name__ == "__main__":
|
||||
os.system("chcp 65001 > nul") # 设置控制台代码页为UTF-8
|
||||
except:
|
||||
pass
|
||||
|
||||
# 后代 Python 进程默认编码兜底:chcp 只改控制台显示,不改 Python
|
||||
# 的 open()/stdout 默认编码(仍是 locale 的 cp936);这两个变量会
|
||||
# 被 agent 启动的所有子 Python 进程继承,避免脚本按 GBK 读写/输出
|
||||
os.environ.setdefault("PYTHONUTF8", "1")
|
||||
os.environ.setdefault("PYTHONIOENCODING", "utf-8")
|
||||
# 当前进程 stdout/stderr 同样强制 UTF-8,防止 print 中文按 cp936 输出
|
||||
for _stream in (sys.stdout, sys.stderr):
|
||||
try:
|
||||
_stream.reconfigure(encoding="utf-8", errors="replace")
|
||||
except (AttributeError, ValueError):
|
||||
pass
|
||||
|
||||
asyncio.run(main())
|
||||
except KeyboardInterrupt:
|
||||
print("\n\nð 再见!")
|
||||
print("\n\n👋 再见!")
|
||||
sys.exit(0)
|
||||
except Exception as e:
|
||||
print(f"\n{OUTPUT_FORMATS['error']} 程序异常退出: {e}")
|
||||
|
||||
@ -248,6 +248,7 @@ class BackgroundCommandManager:
|
||||
env=plan.env,
|
||||
start_new_session=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
bufsize=1,
|
||||
pass_fds=pass_fds,
|
||||
@ -268,6 +269,7 @@ class BackgroundCommandManager:
|
||||
env=env,
|
||||
start_new_session=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
bufsize=1,
|
||||
)
|
||||
@ -279,6 +281,7 @@ class BackgroundCommandManager:
|
||||
env=env,
|
||||
start_new_session=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
bufsize=1,
|
||||
)
|
||||
|
||||
@ -247,12 +247,16 @@ class _StdioMCPClient:
|
||||
return
|
||||
launch_cmd, cwd, env = self._prepare_launch()
|
||||
try:
|
||||
# MCP 协议(JSON-RPC over stdio)字节级要求 UTF-8,必须显式指定,
|
||||
# 否则 Windows 上 text=True 默认按 locale(cp936)编解码,中文 payload 会损坏
|
||||
self.process = subprocess.Popen(
|
||||
launch_cmd,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
bufsize=1,
|
||||
cwd=cwd or None,
|
||||
env=env,
|
||||
|
||||
@ -122,7 +122,8 @@ class PersistentTerminalBase:
|
||||
self.output_queue = queue.Queue()
|
||||
self.reader_thread = None
|
||||
self.is_reading = False
|
||||
self._decoder = None
|
||||
# _read_output 自适应解码(io.py)用于暂存被块边界切断的不完整多字节序列
|
||||
self._pending_bytes = b''
|
||||
|
||||
# 状态标志
|
||||
self.is_interactive = False # 是否在等待输入
|
||||
|
||||
@ -63,20 +63,46 @@ except ImportError:
|
||||
)
|
||||
|
||||
|
||||
def _gbk_complete_prefix_len(data: bytes) -> int:
|
||||
"""返回 data 中按 GBK 可完整解析的前缀长度。
|
||||
|
||||
GBK:ASCII 单字节(0x00-0x7F);双字节 lead 0x81-0xFE + trail 0x40-0xFE(不含 0x7F)。
|
||||
末尾孤立的 lead byte 视为被块边界切断的半字,不计入前缀(留给下一块拼合)。
|
||||
"""
|
||||
i = 0
|
||||
n = len(data)
|
||||
while i < n:
|
||||
b = data[i]
|
||||
if b <= 0x7F:
|
||||
i += 1
|
||||
elif 0x81 <= b <= 0xFE:
|
||||
if i + 1 < n and 0x40 <= data[i + 1] <= 0xFE and data[i + 1] != 0x7F:
|
||||
i += 2
|
||||
else:
|
||||
break # 孤立 lead byte(可能是边界切断的半字)
|
||||
else:
|
||||
i += 1 # 0x80/0xFF 非法字节,按单字节跳过(由 replace 兜底)
|
||||
return i
|
||||
|
||||
|
||||
class IoMixin:
|
||||
"""PersistentTerminal io 能力 mixin。"""
|
||||
|
||||
def _read_output(self):
|
||||
"""后台线程:持续读取输出(修复版,正确处理编码)"""
|
||||
if self._decoder is None:
|
||||
encoding = 'gbk' if self.is_windows else 'utf-8'
|
||||
self._decoder = codecs.getincrementaldecoder(encoding)(errors='replace')
|
||||
"""后台线程:持续读取输出。
|
||||
|
||||
Windows 下不再写死 GBK:cmd 会话启动时已注入 chcp 65001(见 start.py),
|
||||
且 PYTHONIOENCODING=utf-8 使子进程 Python 程序输出 UTF-8;但用户可能
|
||||
手动 chcp 回 936、或运行只输出 GBK 的旧程序。因此逐块自适应解码:
|
||||
先试 UTF-8,失败则整块按 GBK;块边界切断的多字节序列用
|
||||
_pending_bytes 暂存,与下一块拼合后再解(见 _decode_chunk)。
|
||||
"""
|
||||
while self.is_reading and self.process:
|
||||
try:
|
||||
# 读取任意字节块,不依赖换行符
|
||||
chunk = self.process.stdout.read(1024)
|
||||
if chunk:
|
||||
text = self._decoder.decode(chunk)
|
||||
text = self._decode_chunk(chunk)
|
||||
if text:
|
||||
self.output_queue.put(text)
|
||||
self._process_output(text)
|
||||
@ -87,12 +113,63 @@ class IoMixin:
|
||||
else:
|
||||
# 没有输出,短暂休眠
|
||||
time.sleep(0.01)
|
||||
|
||||
|
||||
except Exception as e:
|
||||
# 不要因为单个错误而停止
|
||||
print(f"[Terminal] 读取输出警告: {e}")
|
||||
time.sleep(0.01)
|
||||
continue
|
||||
# 进程结束后冲刷残留的不完整字节(多为被截断的多字节序列半字)
|
||||
pending = getattr(self, '_pending_bytes', b'')
|
||||
if pending:
|
||||
fallback = 'gbk' if self.is_windows else 'utf-8'
|
||||
text = pending.decode(fallback, errors='replace')
|
||||
self._pending_bytes = b''
|
||||
if text:
|
||||
self.output_queue.put(text)
|
||||
self._process_output(text)
|
||||
|
||||
def _decode_chunk(self, chunk: bytes) -> str:
|
||||
"""把 stdout 字节块解码为文本(Windows:UTF-8 优先、GBK 回退)。
|
||||
|
||||
判定依据:UTF-8 有严格的多字节结构校验,GBK 字节流极少恰好构成
|
||||
合法 UTF-8,因此「UTF-8 能解」是强信号;解不了再按 GBK。
|
||||
两种路径都会把块末尾不完整的多字节序列暂存 _pending_bytes,
|
||||
与下一块拼合后再解,避免 read 边界切断汉字。
|
||||
|
||||
已知限制(业界通病,VS Code/Windows Terminal 同样无法处理):
|
||||
极少数 GBK 汉字对的字节恰好也是合法 UTF-8(如「目录」= C4BF C2BC
|
||||
会解为 Ŀ¼),此时若后续块紧跟 UTF-8 内容会连锁误判。终端会话已
|
||||
注入 chcp 65001(start.py),正常路径全程 UTF-8,GBK 仅为兜底,
|
||||
该巧合仅影响显示、不丢数据,故不做「罕见字符惩罚」式启发式猜测。
|
||||
"""
|
||||
data = getattr(self, '_pending_bytes', b'') + chunk
|
||||
self._pending_bytes = b''
|
||||
if not data:
|
||||
return ''
|
||||
if not self.is_windows:
|
||||
return data.decode('utf-8', errors='replace')
|
||||
try:
|
||||
return data.decode('utf-8')
|
||||
except UnicodeDecodeError as exc:
|
||||
# 末尾 1~3 字节可能是不完整 UTF-8 序列(read 边界切断),暂存待拼
|
||||
if exc.start >= len(data) - 3:
|
||||
head, pending = data[:exc.start], data[exc.start:]
|
||||
try:
|
||||
text = head.decode('utf-8')
|
||||
self._pending_bytes = pending
|
||||
return text
|
||||
except UnicodeDecodeError:
|
||||
pass # head 也含非法序列 → 并非 UTF-8 流,落入 GBK 路径
|
||||
# 非法序列位于中部(或 UTF-8 边界暂存失败)→ 确定不是 UTF-8
|
||||
# (如 cmd 内置命令的 GBK 输出),按 GBK 边界感知解码
|
||||
return self._decode_gbk_boundary(data)
|
||||
|
||||
def _decode_gbk_boundary(self, data: bytes) -> str:
|
||||
"""按 GBK 解码,末尾不完整的双字节序列(被切断的半字)暂存待拼。"""
|
||||
safe_len = _gbk_complete_prefix_len(data)
|
||||
self._pending_bytes = data[safe_len:]
|
||||
return data[:safe_len].decode('gbk', errors='replace')
|
||||
|
||||
def _decode_output(self, data):
|
||||
"""安全地解码输出"""
|
||||
|
||||
@ -215,6 +215,18 @@ class StartMixin:
|
||||
bufsize=0,
|
||||
env=env
|
||||
)
|
||||
if self.is_windows and not self.host_shell_command:
|
||||
# 默认 cmd.exe:注入 chcp 65001,把会话统一切换到 UTF-8 代码页。
|
||||
# 1) 与上方 PYTHONIOENCODING=utf-8 对齐(子进程 Python 程序输出 UTF-8);
|
||||
# 2) 让 cmd 内置命令(dir/echo 等)也输出 UTF-8,配合 io.py 的
|
||||
# UTF-8 优先自适应解码,避免中文输出乱码;
|
||||
# 3) 仅默认 shell 注入——用户自定义 shell(如 pwsh)默认即为 UTF-8。
|
||||
try:
|
||||
process.stdin.write(b'chcp 65001 >nul\r\n')
|
||||
process.stdin.flush()
|
||||
except (OSError, ValueError, BrokenPipeError):
|
||||
# 注入失败不致命:io.py 的 GBK 回退仍能处理默认代码页输出
|
||||
pass
|
||||
return process
|
||||
|
||||
def _start_docker_terminal(self):
|
||||
|
||||
@ -183,6 +183,8 @@ class UploadQuarantineManager:
|
||||
command,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
timeout=self.clamav_timeout,
|
||||
check=False,
|
||||
)
|
||||
|
||||
@ -343,6 +343,8 @@ class UserContainerManager:
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
|
||||
@ -84,6 +84,10 @@ class ConversationVersioningManager:
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
# git 输出固定为 UTF-8;Windows text=True 默认按 locale(cp936)
|
||||
# 解码会让中文路径/提交信息乱码(同 server/status/git.py 的处理)
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
timeout=timeout_seconds,
|
||||
)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user