JOJO
c76431a8f1
- extract_webpage/save_webpage 命中白名单域名时本机直提：GitHub 代码页走 jsDelivr CDN 直链（备 GitHub API），其余白名单页用 trafilatura 正文识别， 失败自动回退 Tavily；内置白名单 github.com，个人空间可关闭/追加域名 - trafilatura 进 requirements（可选依赖，缺失时静默降级全走 Tavily） - recall_project_memory 返回记忆全文后标记为已读，可直接 edit_file Co-authored-by: Astrion powered by Kimi-K3 <astrion-agent@users.noreply.github.com>
370 lines
13 KiB
Python
370 lines
13 KiB
Python
# modules/webpage_extractor.py - 网页内容提取模块
|
||
#
|
||
# 提取分两层：
|
||
# 1. 白名单域名走本机直提（免费、零配额）：GitHub 代码文件页有专属直链适配
|
||
# （jsDelivr CDN → GitHub API 备用），其余白名单页面用 trafilatura 提取正文。
|
||
# 2. 未命中白名单（或直提失败）回退 Tavily 云端提取。
|
||
#
|
||
# trafilatura 为可选依赖：未安装时通用直提静默失效，仅 GitHub 直链仍可用。
|
||
|
||
import base64
|
||
import re
|
||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||
from urllib.parse import urlparse
|
||
|
||
import httpx
|
||
|
||
from utils.logger import setup_logger
|
||
from modules.i18n import tr
|
||
|
||
logger = setup_logger(__name__)
|
||
|
||
try:
|
||
import trafilatura as _trafilatura
|
||
except ImportError: # 可选依赖
|
||
_trafilatura = None
|
||
|
||
# 内置直提白名单域名（个人空间可追加；子域名自动匹配）
|
||
BUILTIN_DIRECT_EXTRACT_DOMAINS: Tuple[str, ...] = ("github.com",)
|
||
|
||
_DIRECT_REQUEST_HEADERS = {
|
||
"User-Agent": (
|
||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36"
|
||
),
|
||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||
}
|
||
_DIRECT_TIMEOUT = 30
|
||
|
||
|
||
async def tavily_extract(urls: Union[str, List[str]], api_key: str, extract_depth: str = "basic", max_urls: int = 1) -> Dict[str, Any]:
|
||
"""
|
||
执行Tavily网页内容提取
|
||
|
||
Args:
|
||
urls: 要提取的URL（字符串或列表）
|
||
api_key: Tavily API密钥
|
||
extract_depth: 提取深度 (basic/advanced)
|
||
max_urls: 最大提取URL数量
|
||
|
||
Returns:
|
||
提取结果字典
|
||
"""
|
||
if not api_key:
|
||
return {"error": tr("webpage.api_key_missing")}
|
||
|
||
# 确保urls是列表
|
||
if isinstance(urls, str):
|
||
urls = [urls]
|
||
|
||
# 限制URL数量
|
||
urls = urls[:max_urls]
|
||
|
||
try:
|
||
async with httpx.AsyncClient() as client:
|
||
response = await client.post(
|
||
"https://api.tavily.com/extract",
|
||
json={
|
||
"urls": urls,
|
||
"extract_depth": extract_depth,
|
||
"include_images": False,
|
||
},
|
||
headers={
|
||
"Authorization": f"Bearer {api_key}",
|
||
"Content-Type": "application/json",
|
||
},
|
||
timeout=60,
|
||
)
|
||
|
||
if response.status_code == 200:
|
||
return response.json()
|
||
else:
|
||
return {"error": tr("webpage.api_request_failed", status_code=response.status_code)}
|
||
|
||
except httpx.TimeoutException:
|
||
return {"error": tr("webpage.timeout")}
|
||
except httpx.RequestError as e:
|
||
return {"error": tr("webpage.network_error", error=str(e))}
|
||
except Exception as e:
|
||
logger.error(f"网页提取异常: {e}")
|
||
return {"error": tr("webpage.extract_error", error=str(e))}
|
||
|
||
|
||
def format_extract_results(results: Dict[str, Any]) -> str:
|
||
"""
|
||
格式化提取结果为简洁版本
|
||
|
||
Args:
|
||
results: tavily_extract返回的结果
|
||
|
||
Returns:
|
||
格式化后的内容字符串
|
||
"""
|
||
if "error" in results:
|
||
return tr("webpage.format_failed", error=results["error"])
|
||
|
||
if not results.get("results"):
|
||
return tr("webpage.no_content")
|
||
|
||
formatted_parts = []
|
||
|
||
# 成功提取的结果
|
||
for i, result in enumerate(results["results"], 1):
|
||
url = result.get("url", "N/A")
|
||
raw_content = result.get("raw_content", "").strip()
|
||
|
||
if raw_content:
|
||
content_length = len(raw_content)
|
||
formatted_parts.append(f"🌐 网页内容 ({content_length} 字符):")
|
||
formatted_parts.append(f"📍 URL: {url}")
|
||
formatted_parts.append("=" * 50)
|
||
formatted_parts.append(raw_content)
|
||
formatted_parts.append("=" * 50)
|
||
else:
|
||
formatted_parts.append(f"⚠️ URL {url} 提取到空内容")
|
||
|
||
# 失败的URL（如果有）
|
||
if results.get("failed_results"):
|
||
formatted_parts.append("\n❌ 提取失败的URL:")
|
||
for failed in results["failed_results"]:
|
||
formatted_parts.append(f"- {failed.get('url', 'N/A')}: {failed.get('error', tr('webpage.unknown_error'))}")
|
||
|
||
return "\n".join(formatted_parts)
|
||
|
||
|
||
# ============================================================
|
||
# 白名单直提
|
||
# ============================================================
|
||
|
||
def _normalize_domain(raw: Any) -> str:
|
||
"""把用户输入规范化为小写裸域名（容忍粘贴完整 URL / 前导点 / 路径）。"""
|
||
if not isinstance(raw, str):
|
||
return ""
|
||
d = raw.strip().lower()
|
||
if not d:
|
||
return ""
|
||
if "://" in d:
|
||
d = _url_hostname(d) or d
|
||
d = d.split("/")[0].split("?")[0].strip(".")
|
||
if not re.fullmatch(r"[a-z0-9.-]+", d or "") or "." not in d:
|
||
return ""
|
||
return d
|
||
|
||
|
||
def _url_hostname(url: str) -> str:
|
||
try:
|
||
return (urlparse(str(url)).hostname or "").lower()
|
||
except Exception:
|
||
return ""
|
||
|
||
|
||
def resolve_direct_extract_config(personalization: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
||
"""从 personalization 配置解析直提设置。
|
||
|
||
Returns:
|
||
{"enabled": bool, "domains": [...]} —— domains 已合并内置域名并规范化。
|
||
"""
|
||
personalization = personalization or {}
|
||
enabled = bool(personalization.get("webpage_direct_extract_enabled", True))
|
||
extra = personalization.get("webpage_direct_extract_domains")
|
||
if not isinstance(extra, list):
|
||
extra = []
|
||
domains: List[str] = []
|
||
for item in list(BUILTIN_DIRECT_EXTRACT_DOMAINS) + extra:
|
||
nd = _normalize_domain(item)
|
||
if nd and nd not in domains:
|
||
domains.append(nd)
|
||
return {"enabled": enabled, "domains": domains}
|
||
|
||
|
||
def is_whitelisted_url(url: str, domains: List[str]) -> bool:
|
||
"""域名等于白名单条目或为其子域名即命中。"""
|
||
host = _url_hostname(url)
|
||
if not host:
|
||
return False
|
||
return any(host == d or host.endswith("." + d) for d in domains)
|
||
|
||
|
||
_GITHUB_BLOB_RE = re.compile(
|
||
r"^https?://(?:www\.)?github\.com/([^/?#]+)/([^/?#]+)/blob/([^/?#]+)/([^?#]+?)(?:[?#].*)?$",
|
||
re.IGNORECASE,
|
||
)
|
||
|
||
|
||
def parse_github_blob_url(url: str) -> Optional[Tuple[str, str, str, str]]:
|
||
"""解析 GitHub blob 页面 URL，返回 (owner, repo, branch, path)；非 blob 页返回 None。
|
||
|
||
注意：branch 按单段处理（覆盖 main/master 等常见场景）；含斜杠的分支名
|
||
会解析失败并自然降级到通用提取，不会报错。
|
||
"""
|
||
m = _GITHUB_BLOB_RE.match(str(url).strip())
|
||
if not m:
|
||
return None
|
||
return m.group(1), m.group(2), m.group(3), m.group(4)
|
||
|
||
|
||
async def _fetch_jsdelivr_raw(client: httpx.AsyncClient, owner: str, repo: str, branch: str, path: str) -> Optional[str]:
|
||
"""经 jsDelivr CDN 拿 GitHub 文件原文（免费、无限速，替代被墙的 raw.githubusercontent.com）。"""
|
||
cdn_url = f"https://cdn.jsdelivr.net/gh/{owner}/{repo}@{branch}/{path}"
|
||
try:
|
||
resp = await client.get(cdn_url, timeout=_DIRECT_TIMEOUT)
|
||
if resp.status_code == 200 and resp.text:
|
||
return resp.text
|
||
logger.info(f"jsDelivr 直链返回 {resp.status_code}: {cdn_url}")
|
||
except Exception as e:
|
||
logger.info(f"jsDelivr 直链失败 {cdn_url}: {e}")
|
||
return None
|
||
|
||
|
||
async def _fetch_github_api_raw(client: httpx.AsyncClient, owner: str, repo: str, branch: str, path: str) -> Optional[str]:
|
||
"""经 GitHub 官方 contents API 拿文件原文（备用；匿名限 60 次/小时，>1MB 文件不返回内容）。"""
|
||
api_url = f"https://api.github.com/repos/{owner}/{repo}/contents/{path}?ref={branch}"
|
||
try:
|
||
resp = await client.get(api_url, headers={"Accept": "application/vnd.github+json"}, timeout=_DIRECT_TIMEOUT)
|
||
if resp.status_code != 200:
|
||
logger.info(f"GitHub API 返回 {resp.status_code}: {api_url}")
|
||
return None
|
||
data = resp.json()
|
||
if isinstance(data, dict) and data.get("encoding") == "base64" and data.get("content"):
|
||
return base64.b64decode(data["content"]).decode("utf-8", errors="replace")
|
||
except Exception as e:
|
||
logger.info(f"GitHub API 失败 {api_url}: {e}")
|
||
return None
|
||
|
||
|
||
def _trafilatura_extract(html: str, url: str) -> Optional[str]:
|
||
"""trafilatura 通用正文提取（markdown 输出，保留标题/代码块结构）。"""
|
||
if _trafilatura is None:
|
||
return None
|
||
try:
|
||
text = _trafilatura.extract(
|
||
html,
|
||
url=url,
|
||
output_format="markdown",
|
||
include_links=False,
|
||
include_images=False,
|
||
)
|
||
if text and text.strip():
|
||
return text.strip()
|
||
except Exception as e:
|
||
logger.info(f"trafilatura 提取失败 {url}: {e}")
|
||
return None
|
||
|
||
|
||
async def _direct_extract(client: httpx.AsyncClient, url: str) -> Tuple[Optional[str], str]:
|
||
"""白名单直提主流程。返回 (内容, method)；全部失败返回 (None, "")。"""
|
||
blob = parse_github_blob_url(url)
|
||
if blob:
|
||
owner, repo, branch, path = blob
|
||
content = await _fetch_jsdelivr_raw(client, owner, repo, branch, path)
|
||
if content is not None:
|
||
return content, "jsdelivr"
|
||
content = await _fetch_github_api_raw(client, owner, repo, branch, path)
|
||
if content is not None:
|
||
return content, "github_api"
|
||
# 直链均失败 → 继续走通用提取兜底
|
||
|
||
if _trafilatura is not None:
|
||
try:
|
||
resp = await client.get(url, timeout=_DIRECT_TIMEOUT)
|
||
if resp.status_code == 200 and resp.text:
|
||
text = _trafilatura_extract(resp.text, url)
|
||
if text:
|
||
return text, "trafilatura"
|
||
else:
|
||
logger.info(f"直提抓取返回 {resp.status_code}: {url}")
|
||
except Exception as e:
|
||
logger.info(f"直提抓取失败 {url}: {e}")
|
||
return None, ""
|
||
|
||
|
||
def _format_single_result(url: str, content: str, method: Optional[str] = None) -> str:
|
||
"""格式化单条提取结果（沿用 🌐 骨架，附提取方式标注）。"""
|
||
header = f"🌐 网页内容 ({len(content)} 字符)"
|
||
if method:
|
||
header += f" [{tr('webpage.method_label', method=tr(f'webpage.method_{method}'))}]"
|
||
return "\n".join([
|
||
header + ":",
|
||
f"📍 URL: {url}",
|
||
"=" * 50,
|
||
content,
|
||
"=" * 50,
|
||
])
|
||
|
||
|
||
async def extract_single_url(
|
||
url: str,
|
||
api_key: Optional[str],
|
||
extract_depth: str = "basic",
|
||
direct_config: Optional[Dict[str, Any]] = None,
|
||
) -> Dict[str, Any]:
|
||
"""统一单 URL 提取：白名单直提优先，失败/未命中回退 Tavily。
|
||
|
||
Args:
|
||
url: 目标 URL
|
||
api_key: Tavily API 密钥（可为 None；直提命中时不需要）
|
||
extract_depth: Tavily 提取深度
|
||
direct_config: resolve_direct_extract_config() 的返回；None 视为关闭直提
|
||
|
||
Returns:
|
||
{"success": bool, "url": str, "content": str, "method": str} 或
|
||
{"success": False, "url": str, "error": str, "method": str}
|
||
"""
|
||
direct_config = direct_config or {}
|
||
|
||
if direct_config.get("enabled") and is_whitelisted_url(url, direct_config.get("domains") or []):
|
||
content, method = None, ""
|
||
try:
|
||
async with httpx.AsyncClient(headers=_DIRECT_REQUEST_HEADERS, follow_redirects=True) as client:
|
||
content, method = await _direct_extract(client, url)
|
||
except Exception as e:
|
||
logger.info(f"白名单直提异常 {url}: {e}")
|
||
if content is not None:
|
||
return {"success": True, "url": url, "content": content, "method": method}
|
||
logger.info(f"白名单直提未取到内容，回退 Tavily: {url}")
|
||
|
||
results = await tavily_extract(url, api_key, extract_depth, 1)
|
||
if "error" in results:
|
||
return {"success": False, "url": url, "error": results["error"], "method": "tavily"}
|
||
for item in results.get("results") or []:
|
||
raw = (item.get("raw_content") or "").strip()
|
||
if raw:
|
||
return {"success": True, "url": url, "content": raw, "method": "tavily"}
|
||
return {"success": False, "url": url, "error": tr("webpage.no_content"), "method": "tavily"}
|
||
|
||
|
||
async def extract_webpage_content(
|
||
urls: Union[str, List[str]],
|
||
api_key: str,
|
||
extract_depth: str = "basic",
|
||
max_urls: int = 1,
|
||
direct_config: Optional[Dict[str, Any]] = None,
|
||
) -> Tuple[str, str]:
|
||
"""
|
||
完整的网页内容提取流程（白名单直提优先，Tavily 兜底）
|
||
|
||
Args:
|
||
urls: 要提取的URL（字符串或列表）
|
||
api_key: Tavily API密钥
|
||
extract_depth: 提取深度 (basic/advanced)
|
||
max_urls: 最大提取URL数量
|
||
direct_config: 直提配置（resolve_direct_extract_config 返回）；None=关闭直提
|
||
|
||
Returns:
|
||
(完整内容, 完整内容) - 为了兼容性返回相同内容两份
|
||
"""
|
||
if isinstance(urls, str):
|
||
urls = [urls]
|
||
urls = urls[:max_urls]
|
||
|
||
formatted_parts: List[str] = []
|
||
for url in urls:
|
||
result = await extract_single_url(url, api_key, extract_depth=extract_depth, direct_config=direct_config)
|
||
if result.get("success"):
|
||
formatted_parts.append(_format_single_result(url, result["content"], result.get("method")))
|
||
else:
|
||
formatted_parts.append(tr("webpage.format_failed", error=result.get("error")))
|
||
|
||
formatted_content = "\n".join(formatted_parts)
|
||
return formatted_content, formatted_content
|