From e2f0cd17ea41eff86064cb56b8e0603f7a40aeb4 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 22 Oct 2026 04:18:27 -0600 Subject: [PATCH] init --- .gitattributes | 25 ++ .gitignore | 9 + README.md | 173 +++++++++++++ examples/demo.json | 15 ++ examples/template.csv | 2 + run.py | 5 + scripts/export_sqlite.py | 54 ++++ scripts/mac_probe.py | 90 +++++++ scripts/package_release.py | 35 +++ start-mac.command | 9 + start-windows.cmd | 4 + tests/test_workbench.py | 287 +++++++++++++++++++++ wechat_tool/__init__.py | 1 + wechat_tool/analysis.py | 228 +++++++++++++++++ wechat_tool/importer.py | 161 ++++++++++++ wechat_tool/server.py | 456 ++++++++++++++++++++++++++++++++++ wechat_tool/static/app.js | 123 +++++++++ wechat_tool/static/index.html | 39 +++ wechat_tool/static/style.css | 1 + wechat_tool/store.py | 93 +++++++ 20 files changed, 1810 insertions(+) create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 README.md create mode 100644 examples/demo.json create mode 100644 examples/template.csv create mode 100644 run.py create mode 100644 scripts/export_sqlite.py create mode 100644 scripts/mac_probe.py create mode 100644 scripts/package_release.py create mode 100644 start-mac.command create mode 100644 start-windows.cmd create mode 100644 tests/test_workbench.py create mode 100644 wechat_tool/__init__.py create mode 100644 wechat_tool/analysis.py create mode 100644 wechat_tool/importer.py create mode 100644 wechat_tool/server.py create mode 100644 wechat_tool/static/app.js create mode 100644 wechat_tool/static/index.html create mode 100644 wechat_tool/static/style.css create mode 100644 wechat_tool/store.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..d0cbabd --- /dev/null +++ b/.gitattributes @@ -0,0 +1,25 @@ +# Normalize text to LF in the repository and working tree on all platforms. +* text=auto eol=lf + +# Windows launchers require CRLF in the working tree. +*.cmd text eol=crlf +*.bat text eol=crlf + +# Mac / Unix launchers must retain LF. +*.command text eol=lf +*.sh text eol=lf + +# Never apply newline conversion to binary assets or databases. +*.zip binary +*.png binary +*.jpg binary +*.jpeg binary +*.gif binary +*.webp binary +*.ico binary +*.pdf binary +*.db binary +*.sqlite binary +*.sqlite3 binary +*.woff binary +*.woff2 binary diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..552cb7b --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +data/ +__pycache__/ +*.pyc +.venv/ +.idea/ +dist/ +.DS_Store +mac-wechat-diagnostic*.json +/artifacts/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..7ec8cf9 --- /dev/null +++ b/README.md @@ -0,0 +1,173 @@ +# 知微 · 微信知识需求分析工具 + +一个本地运行的 Python + SQLite 工具。从群聊和私聊里发现反复提问、风险/事故、重复踩坑、后续复用线索，将其整理为带原文证据的知识需求卡，对应 CREATES 的 Choose → Retrieve → Extract / Validate。 + +## 交付状态 + +- 已实现：可读文件导入、明文 SQLite 表导入、去重和错误行报告、覆盖统计、时间与会话筛选、本地规则分析、脱敏预览、兼容模型 API、后台任务、原文回溯、人工修正、主题合并/拆分、CSV / Markdown 导出。 +- 已验证：Windows / Python 3.11 的自动化测试；浏览器端样例操作；本机模拟 API 的完整脱敏与证据校验流程。 +- **尚未完成：Mac 微信 4.1.0.18 加密数据库的直接读取与解密。** 当前执行环境是 Windows，未连接目标 Mac；未声称或验证该版本的解密兼容性。应用不能仅凭选择微信目录就自动获取全部聊天。使用真实数据前，仍需在目标 Mac 取得可读导出。 +- 已提供 Mac 只读诊断脚本，帮助确认版本、CPU、数据位置、SQLite / 加密文件头与 WAL 状态；不获取密钥，不修改微信数据或系统保护设置。 +- Intel / Apple Silicon 使用相同 Python 分析代码；本版无需 pip 安装第三方依赖。 + +## 在 Mac 上启动 + +需要 **Python 3.10 或以上**。将整个项目目录或发布 ZIP 复制到 Mac 并解压，在终端进入目录： + +```bash +python3 --version +python3 run.py +``` + +浏览器会打开 。关闭终端或按 Ctrl+C 停止服务。也可在目录内运行： + +```bash +bash start-mac.command +``` + +如果希望以后双击启动，可先执行 `chmod +x start-mac.command`。Python 版本过低时，先安装新版 Python。端口被占用则运行 `python3 run.py --port 8766`。不要通过 `file://` 打开 HTML，也不要换成 `localhost`，服务校验 Host，只接受启动时的 `127.0.0.1:端口`。 + +Windows 开发环境：`python run.py` 或双击 `start-windows.cmd`。 + +## 第一次体验 + +1. 在“数据概览”点击“体验虚构样例”：13 条文字/媒体占位记录、两个群聊和一个私聊。示例中的人名、客户与业务都是虚构的，不是用户真实记录。 +2. 进入“发现知识需求”，时间留空、会话类型选“全部会话”、勾选“跨会话汇总相似问题”，点击“本地发现线索”。无需密钥，不调用网络。 +3. 会得到 4 张候选主题卡，其中“测试比例怎么确定”包含 3 次讨论、跨群聊与私聊。点击“查看证据与审核”查看原文、修改摘要、记录边界或审核。 +4. 多张卡片可勾选后合并。单卡包含多个事件时，可在详情勾选部分事件拆出新主题。合并/拆分会重新标为待审核，旧摘要或备注保留在新卡备注。 +5. 导出 CSV（Excel 可打开）或 Markdown。导出包含本地会话名与分析内容，**不是对外分享的脱敏报告**；分享前请检查。 + +## 从 Mac 获取接入诊断 + +请在目标 Mac 的项目目录运行： + +```bash +python3 scripts/mac_probe.py +``` + +生成 `mac-wechat-diagnostic.json`（如果已存在，拒绝覆盖）。报告仅包含本机路径、大小、修改时间、文件头判定、应用版本、CPU 与访问错误，不包含消息正文。 + +若微信数据在自定义位置，可明确指定目录： + +```bash +python3 scripts/mac_probe.py --root "/你的微信数据目录" --output mac-diagnostic-2.json +``` + +默认检查微信沙盒 Documents、Application Support 和用户 Documents/xwechat_files，最多 8 层、300 个候选数据库，不跟随符号链接，不扫描整块磁盘。路径可能因版本与设置不同而变化；未找到文件不代表没有记录。权限受限会写入 warnings，不会自动提升权限。报告含本地路径，分享前自行检查。 + +`sqlite_header` 仅代表有普通 SQLite 文件头；`encrypted_or_unknown` 不是对加密算法的确认。原生数据库中的联系人关联、XML、压缩文本与 BLOB 等仍可能需要版本适配。不要为了运行本工具关闭 SIP 或安装来路不明的提取器。 + +当前版本明确不依赖 chatlog 原仓库。2026-10-22 核查时，[原仓库](https://github.com/sjzar/chatlog)仅保留项目移除通知，不能据旧教程假设它仍可安装或支持 4.1.0.18。 + +## 导入真实记录 + +支持 JSON、JSONL / NDJSON、CSV、TSV、约定格式的 TXT、明文 SQLite（`.db` / `.sqlite` / `.sqlite3`）。浏览器单文件上限 45 MB，单次上限 50 万条。超限需分文件；每次分析最多选择 20 万条。 + +标准 JSON 为对象数组，或包含 `messages` / `items` / `data` 数组的对象： + +```json +[ + { + "id": "消息唯一ID", + "conversation": "测试项目群", + "conversation_id": "project@chatroom", + "kind": "group", + "sender": "张三", + "sender_id": "user_a", + "timestamp": "2026-10-01T09:00:00+08:00", + "content": "测试比例怎么确定？", + "message_type": "text", + "quote": "" + } +] +``` + +CSV 表头使用同名字段，见 `examples/template.csv`。导入界面可填写实际列名映射；时间、发送人、内容必须可解析。单会话文件缺少会话信息时，填写默认会话名称和类型；混合会话记录须在每行提供会话字段。 + +- `kind` 为 `group` / `private`。支持“群聊”/“私聊”。缺失时，ID 以 `@chatroom` 结尾识别为群，其余为私聊；请核对分类。 +- `timestamp` 支持 ISO 8601、常见年月日时间、10 位秒或 13 位毫秒 Unix 时间戳。 +- 无时区时间按设置中的源时区解释，默认中国时间 `+08:00`，**不使用运行电脑的时区猜测**。统一以 UTC 保存。筛选和界面显示按同一设置的固定 UTC 偏移。跨夏令时数据请使用带时区的源时间；更改设置不会重新解释已导入数据。 +- 唯一消息 ID 按会话去重。没有消息 ID 时，用会话、发送人、时间、内容、类型、引用去重；真实同秒同内容消息可能合并，结果会提示。不要混合多个账号但复用同样的会话/消息 ID；如需多账号，请使用不同 `--data-dir`。 +- 发送人缺失、文本内容为空、时间无效、BLOB 等会拒绝对应行，并返回错误行号，其他有效行仍会导入。 +- 非文字内容保留类型与占位文本，不自动 OCR、转写语音或解析附件。 +- 不支持未经转换的 HTML、任意 TXT 布局、微信官方备份包、加密数据库。拒绝这些格式，避免把乱码当消息。 + +TXT 支持： + +```text +[2026-10-01 09:00:00] 张三: 测试比例怎么确定？ +这一行仍属于张三的消息。 +[2026-10-01 09:02:00] 李四: 请先确认型号。 +``` + +### SQLite 与 WAL + +上传数据库时只会读取你选中的文件，浏览器不会自动附带 `-wal`。**不要直接复制运行中微信的 `.db` 文件然后视为完整导出。** 对已经明文、字段可读的数据库，可在 Mac 本地用只读 SQLite backup API 导出表（会包含已提交的 WAL 内容）： + +```bash +python3 scripts/export_sqlite.py "/路径/可读数据库.db" "消息表名" "/路径/messages.jsonl" +``` + +脚本不修改源库、不覆盖已有输出。输出保留源列名，可在导入页面配置字段映射。压缩/二进制内容会拒绝导出，避免误导。数据库没有 sender、conversation 等完整字段时，仍需先关联联系人/群成员信息；本版不对未知微信原生 schema 做猜测。 + +## 分析与覆盖范围 + +时间段可覆盖全部已导入的群聊和私聊；也可指定会话类型、包含/排除会话。包含列表不选代表全部。时间起止均包含边界；填到分钟时边界精确到该分钟的第 0 秒。不填时间即全部时间。结束日期若通过接口只提供日期，则包含当天 23:59:59。 + +关键词筛选会剔除不匹配的消息，因此可能破坏上下文。希望分析完整讨论时请留空。 + +本地分析先找提问、失败、风险、返工、未来任务等词语，再在同会话内按 30 分钟间隔、最多 30 条消息形成事件候选；用中文二元词片段与英文词相似度合并问题。**规则是可审查的起点，不是准确语义理解**：会漏掉隐含问题、延迟回复和未记录的背景，也可能混入交错话题。模型模式基于相同事件候选进一步提炼，不代表读懂了全部未入选闲聊。原文图片/语音没有转写时，结论可能缺证据。 + +“独立事件数”是窗口分段估计，“回答者”只是可能提供经验的人，“重复踩坑”是文本线索，都不能直接作为事故因果、专家能力或个人绩效结论。 + +本地优先级为启发式：每个事件 15 分、风险 20 分、返工 15 分、未来复用 10 分，上限 100。模型可调整分类与策略，但分数仍来自本地事件和关键词；仅供排序，不是业务损失评估。人工审核不等于实践验证。 + +分析时冻结事件与消息快照，后续导入不会改写历史卡片证据。新增记录须新建分析任务。历史状态、人工编辑和卡片保存在 SQLite，刷新或重启后仍可从分析历史打开。 + +## 配置模型 API 与脱敏 + +在“模型与脱敏”填写完整兼容接口地址（以 `/chat/completions` 结尾）、模型名和 API Key。本工具不绑定特定供应商；需要支持常规 `messages` 请求并返回 `choices[0].message.content`。模型须能遵守 JSON 与 ID 引用要求；响应不合规时整个任务失败，不保存部分卡片。 + +- 远程接口必须 HTTPS；本机 `localhost` / `127.0.0.1` 可用 HTTP。 +- API Key 仅在 Python 进程内存；重启后重新输入。也可通过 `WECHAT_MODEL_API_KEY` 环境变量提供。不会写入 SQLite、浏览器 localStorage、日志或导出。 +- 保存接口、预览均不发外部请求。点击“发送这些内容”才把预览的请求体发送给配置的服务；服务商的数据保留政策由你选用的服务决定。 +- 脱敏覆盖全库已知发送人名/ID、会话名/ID、常见电话号码/邮箱/身份证/IP/网址/凭据与自定义词。代号由本地随机盐生成并保持稳定，映射不发送。时间、消息类型、证据数字 ID 和未替换正文会发送。 +- **这不是可证明不可逆的匿名化**。自然语言中未知的人名、地址、业务秘密和组合身份线索可能残留；请补充敏感词并检查完整预览。预览最多 30 分钟有效，配置变更后失效，一份预览只能发送一次。 +- 每批最多约 48000 JSON 字符、30 个事件；超大单事件拒绝发送而非截断。字符数不是精确 token/费用估计，服务商较小上下文窗口可能报错。 +- 多批串行发送；**不会自动跨批语义合并**，结果中可人工合并。后台页面可等待；关闭浏览器不停止任务，停止 Python 服务会中断。重启后把未完成任务标记失败，不自动重新发送。 +- 不自动重试、跟随 HTTP 重定向或读取系统代理，避免重复计费与转发脱敏数据。单次 API 等待最长 120 秒。需要代理的环境可使用明确配置的本地兼容网关。 +- 模型只获得脱敏事件，没有本机文件/命令/网络工具。响应必须引用当前批次存在的事件和消息，事件不能重复或遗漏，证据必须属于该卡的事件；否则拒绝整次结果。 + +## 本地数据与备份 + +默认数据在项目下 `data/workbench.sqlite3`，包括原文、配置（不含 API Key）、代号盐、分析快照和人工修正。目录在支持 POSIX 权限的系统上设置为 0700。数据库本身未做静态加密，建议放在你的受保护 Mac 用户目录。 + +仅绑定 `127.0.0.1`，校验 Host/Origin 和写入令牌，不加载外部 CDN 或遥测。聊天/模型输出以纯文本 DOM 渲染，CSV 中以 `= + - @` 开头的内容会转义以避免公式执行。该工具是单用户桌面工作台，不是多人服务。 + +备份：先 Ctrl+C 停止服务，再复制整个 `data` 目录（包含可能存在的 WAL/SHM），或用 SQLite backup API。恢复时停止服务后放回原目录。用 `python3 run.py --data-dir "/其他/数据目录"` 建立独立工作空间；不要在两个进程中同时打开同一数据目录。 + +## 开发与验证 + +```bash +python3 -m unittest discover -s tests -v +python3 -m compileall -q wechat_tool scripts run.py +``` + +测试使用临时数据目录与本机模拟 API，不调用真实模型、不读取用户微信数据。覆盖时区、去重、错误行、跨会话聚类、媒体缺口、脱敏、完整批次、模型虚构证据拒绝、WAL 快照、只读诊断、CSRF、导入 HTTP、历史快照固定、人工合并/拆分/审核和导出。 + +源码结构： + +```text +run.py 启动入口 +wechat_tool/store.py SQLite 与筛选 +wechat_tool/importer.py 文件解析、字段映射、去重 +wechat_tool/analysis.py 事件、聚类、脱敏与模型结果校验 +wechat_tool/server.py 本机 HTTP 服务、任务与导出 +wechat_tool/static/ 无构建依赖的界面 +scripts/mac_probe.py Mac 环境与数据库头诊断 +scripts/export_sqlite.py 明文 SQLite / WAL 完整表导出 +examples/ 虚构样例与导入模板 +tests/ 无网络依赖的自动化测试 +``` + +后续真实 Mac 验收仍需：确认 4.1.0.18 的可读数据来源，核对所选时间段的消息数/首尾/随机样本，确认群成员身份和引用解析，使用你选择的真实模型 API 验证响应，再扩大到半年的多群数据。这个接入限制是待完成事项，不是已通过的兼容性声明。 diff --git a/examples/demo.json b/examples/demo.json new file mode 100644 index 0000000..b6992d1 --- /dev/null +++ b/examples/demo.json @@ -0,0 +1,15 @@ +[ + {"id":"d01","conversation":"演示·测试项目群","conversation_id":"demo-test@chatroom","kind":"group","sender":"演示张工","sender_id":"demo-zhang","timestamp":"2026-10-01 09:00:00","content":"测试比例怎么确定？参数表里没有写清楚。"}, + {"id":"d02","conversation":"演示·测试项目群","conversation_id":"demo-test@chatroom","kind":"group","sender":"演示李工","sender_id":"demo-li","timestamp":"2026-10-01 09:03:00","content":"先确认设备型号和负载，A 型设备按试验方案 v2 的比例执行。其他型号要重新验证。"}, + {"id":"d03","conversation":"演示·测试项目群","conversation_id":"demo-test@chatroom","kind":"group","sender":"演示张工","sender_id":"demo-zhang","timestamp":"2026-10-01 09:06:00","content":"收到，我先核对型号。"}, + {"id":"d04","conversation":"演示·测试项目群","conversation_id":"demo-test@chatroom","kind":"group","sender":"演示王工","sender_id":"demo-wang","timestamp":"2026-10-05 10:00:00","content":"请问测试比例怎么确定？新同事找不到那份方案。"}, + {"id":"d05","conversation":"演示·测试项目群","conversation_id":"demo-test@chatroom","kind":"group","sender":"演示李工","sender_id":"demo-li","timestamp":"2026-10-05 10:02:00","content":"我再发一次方案，A 型设备用 v2。客户是虚构客户甲，联系人电话 13800138000，邮箱 test@example.com。"}, + {"id":"d06","conversation":"演示·与陈工私聊","conversation_id":"demo-chen-chat","kind":"private","sender":"演示陈工","sender_id":"demo-chen","timestamp":"2026-10-08 14:00:00","content":"测试比例怎么确定？我这边也是 A 型设备。"}, + {"id":"d07","conversation":"演示·与陈工私聊","conversation_id":"demo-chen-chat","kind":"private","sender":"演示李工","sender_id":"demo-li","timestamp":"2026-10-08 14:03:00","content":"同型号先参考 v2，并确认现场负载相同；现场差异需要补充试验。"}, + {"id":"d08","conversation":"演示·上线支持群","conversation_id":"demo-release@chatroom","kind":"group","sender":"演示周工","sender_id":"demo-zhou","timestamp":"2026-10-09 16:00:00","content":"生产上线报错，差点把测试机的问题判断成系统故障，需要检查设备健康状态。"}, + {"id":"d09","conversation":"演示·上线支持群","conversation_id":"demo-release@chatroom","kind":"group","sender":"演示李工","sender_id":"demo-li","timestamp":"2026-10-09 16:05:00","content":"先核对测试机日志和硬件状态，再判断系统。内部链接 https://internal.example.com/log?token=secret。"}, + {"id":"d10","conversation":"演示·测试项目群","conversation_id":"demo-test@chatroom","kind":"group","sender":"演示张工","sender_id":"demo-zhang","timestamp":"2026-10-12 11:00:00","content":"这次比例又填错，三轮测试重来。上次也有这个问题，返工要避免。"}, + {"id":"d11","conversation":"演示·测试项目群","conversation_id":"demo-test@chatroom","kind":"group","sender":"演示王工","sender_id":"demo-wang","timestamp":"2026-10-12 11:02:00","content":"参数表截图在这里。","message_type":"image"}, + {"id":"d12","conversation":"演示·上线支持群","conversation_id":"demo-release@chatroom","kind":"group","sender":"演示周工","sender_id":"demo-zhou","timestamp":"2026-10-15 15:00:00","content":"接下来还有三个系统要上线，明年还要用到这套测试方法，计划整理检查清单。"}, + {"id":"d13","conversation":"演示·与陈工私聊","conversation_id":"demo-chen-chat","kind":"private","sender":"演示陈工","sender_id":"demo-chen","timestamp":"2026-10-16 08:00:00","content":"早上好，今天的会议改到下午。"} +] diff --git a/examples/template.csv b/examples/template.csv new file mode 100644 index 0000000..da63c8b --- /dev/null +++ b/examples/template.csv @@ -0,0 +1,2 @@ +id,conversation,conversation_id,kind,sender,sender_id,timestamp,content,message_type,quote +example-001,示例工作群,example@chatroom,group,示例人员,user-example,2026-10-01 09:00:00,测试比例怎么确定？,text, diff --git a/run.py b/run.py new file mode 100644 index 0000000..aeb1af3 --- /dev/null +++ b/run.py @@ -0,0 +1,5 @@ +"""Local WeChat knowledge workbench. Python 3.10+, standard library only.""" +from wechat_tool.server import main + +if __name__ == '__main__': + main() diff --git a/scripts/export_sqlite.py b/scripts/export_sqlite.py new file mode 100644 index 0000000..07ac0b1 --- /dev/null +++ b/scripts/export_sqlite.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Snapshot a readable SQLite table to JSONL without modifying its source.""" +import argparse +import json +import sqlite3 +from pathlib import Path + + +def export(source, table, output): + source, output = Path(source).resolve(), Path(output).resolve() + if source == output or output.exists(): + raise ValueError('输出文件已存在或与源文件相同，请使用新的输出路径') + with source.open('rb') as file: + if file.read(16) != b'SQLite format 3\x00': + raise ValueError('源文件不是明文 SQLite；此脚本不解密数据库') + db = sqlite3.connect(source.as_uri() + '?mode=ro', uri=True, timeout=15) + snapshot = sqlite3.connect(':memory:') + created = False + try: + # Backup API includes committed WAL contents; never copy only a live .db. + db.backup(snapshot) + snapshot.row_factory = sqlite3.Row + names = [r[0] for r in snapshot.execute("SELECT name FROM sqlite_master WHERE type='table'")] + if table not in names: + raise ValueError('消息表不存在，可用表：' + ', '.join(names)) + quoted = '"' + table.replace('"', '""') + '"' + with output.open('x', encoding='utf-8') as file: + created = True + try: + output.chmod(0o600) + except OSError: + pass + for row in snapshot.execute('SELECT * FROM ' + quoted): + record = dict(row) + if any(isinstance(v, bytes) for v in record.values()): + raise ValueError('表中含二进制字段，需先解码/转换；不会把 BLOB 假装成消息正文') + file.write(json.dumps(record, ensure_ascii=False) + '\n') + except Exception: + if created and output.exists(): + output.unlink() # Only the output file exclusively created by this invocation. + raise + finally: + db.close() + snapshot.close() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('source', type=Path) + parser.add_argument('table') + parser.add_argument('output', type=Path) + args = parser.parse_args() + export(args.source, args.table, args.output) + print('Readable JSONL written to:', args.output.resolve()) diff --git a/scripts/mac_probe.py b/scripts/mac_probe.py new file mode 100644 index 0000000..fac25fb --- /dev/null +++ b/scripts/mac_probe.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Read-only Mac WeChat storage inventory. Does not decrypt or read messages.""" +import argparse +import json +import os +import platform +import plistlib +from datetime import datetime, timezone +from pathlib import Path + + +def probe(roots=None, max_files=300): + home = Path.home() + default_roots = [ + home / 'Library/Containers/com.tencent.xinWeChat/Data/Documents', + home / 'Library/Containers/com.tencent.xinWeChat/Data/Library/Application Support/com.tencent.xinWeChat', + home / 'Documents/xwechat_files', + ] + report = {'created_utc': datetime.now(timezone.utc).isoformat(), 'platform': platform.system(), + 'architecture': platform.machine(), 'wechat_versions': [], 'roots': [], 'databases': [], 'warnings': [], + 'notes': ['只读取文件名、大小、修改时间和前 16 字节；不读取消息正文、不获取密钥、不修改微信文件。', + '检测到 SQLite 文件头不代表消息内容已解码或可用。', + '未覆盖的自定义存储位置可通过 --root 指定。没有找到不代表没有记录。']} + for app in (Path('/Applications/WeChat.app'), home / 'Applications/WeChat.app'): + info = app / 'Contents/Info.plist' + if info.is_file(): + try: + with info.open('rb') as file: + data = plistlib.load(file) + report['wechat_versions'].append({'app': str(app), 'version': data.get('CFBundleShortVersionString'), 'build': data.get('CFBundleVersion')}) + except (OSError, ValueError, plistlib.InvalidFileException): + report['warnings'].append('无法读取应用版本：' + str(info)) + visited = set() + def error(exc): + report['warnings'].append('无法访问目录（可能需要文件访问权限）：' + str(exc.filename)) + for root in roots or default_roots: + root = Path(root).expanduser().absolute() + report['roots'].append({'path': str(root), 'exists': root.exists()}) + if not root.exists(): + continue + for directory, dirs, files in os.walk(root, followlinks=False, onerror=error): + relative = Path(directory).relative_to(root) + dirs[:] = [d for d in dirs if not (Path(directory)/d).is_symlink() and d not in ('Cache', 'Caches', 'FileStorage', 'video', 'image', 'temp')] + if len(relative.parts) >= 8: + dirs[:] = [] + for name in files: + if not name.lower().endswith(('.db', '.sqlite', '.sqlite3')): + continue + path = Path(directory) / name + if path.is_symlink() or str(path) in visited: + continue + visited.add(str(path)) + if len(report['databases']) >= max_files: + report['warnings'].append(f'达到 {max_files} 个数据库上限，报告不完整。') + return report + try: + stat = path.stat() + with path.open('rb') as file: + header = file.read(16) + report['databases'].append({'path': str(path), 'size': stat.st_size, + 'modified_utc': datetime.fromtimestamp(stat.st_mtime, timezone.utc).isoformat(), + 'format': 'sqlite_header' if header == b'SQLite format 3\x00' else 'encrypted_or_unknown', + 'wal_present': Path(str(path) + '-wal').exists()}) + except OSError: + report['warnings'].append('无法读取数据库头：' + str(path)) + return report + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--root', action='append', help='额外/指定存储根目录，可重复；指定后替代默认目录') + parser.add_argument('--output', type=Path, default=Path('mac-wechat-diagnostic.json')) + args = parser.parse_args() + if platform.system() != 'Darwin' and not args.root: + parser.error('请在目标 Mac 上运行。测试时可显式指定 --root。') + report = probe(args.root) + # Exclusive creation: never overwrite an existing user file or database. + with args.output.open('x', encoding='utf-8') as file: + json.dump(report, file, ensure_ascii=False, indent=2) + try: + args.output.chmod(0o600) + except OSError: + pass + print('Diagnostic written to:', args.output.resolve()) + print('Database files:', len(report['databases'])) + print('Report contains local paths. Review before sharing.') + + +if __name__ == '__main__': + main() diff --git a/scripts/package_release.py b/scripts/package_release.py new file mode 100644 index 0000000..2154f69 --- /dev/null +++ b/scripts/package_release.py @@ -0,0 +1,35 @@ +"""Create a portable source ZIP, excluding all user data and credentials.""" +import stat +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +FILES = ['run.py', 'README.md', '.gitignore', '.gitattributes', 'start-mac.command', 'start-windows.cmd'] +DIRS = ['wechat_tool', 'scripts', 'examples', 'tests'] + + +def main(): + out = ROOT / 'dist' / 'zhiwei-wechat-analysis-v0.1.zip' + out.parent.mkdir(exist_ok=True) + paths = [ROOT / p for p in FILES] + for directory in DIRS: + paths.extend(p for p in (ROOT / directory).rglob('*') if p.is_file() and '__pycache__' not in p.parts and p.suffix != '.pyc') + with zipfile.ZipFile(out, 'w', compression=zipfile.ZIP_DEFLATED) as archive: + for path in sorted(paths): + name = 'zhiwei-wechat-analysis/' + path.relative_to(ROOT).as_posix() + info = zipfile.ZipInfo(name) + info.create_system = 3 + mode = 0o755 if path.name == 'start-mac.command' else 0o644 + info.external_attr = (stat.S_IFREG | mode) << 16 + info.compress_type = zipfile.ZIP_DEFLATED + # Match .gitattributes even when packaging without a Git checkout. + # All files selected above are text; binary artifacts are excluded. + data = path.read_bytes().replace(b'\r\n', b'\n').replace(b'\r', b'\n') + if path.suffix in ('.cmd', '.bat'): + data = data.replace(b'\n', b'\r\n') + archive.writestr(info, data) + print(out) + + +if __name__ == '__main__': + main() diff --git a/start-mac.command b/start-mac.command new file mode 100644 index 0000000..7ed0138 --- /dev/null +++ b/start-mac.command @@ -0,0 +1,9 @@ +#!/bin/bash +cd -- "$(dirname -- "$0")" || exit 1 +if ! command -v python3 >/dev/null 2>&1; then + echo "Python 3.10+ is required. Install Python, then run this launcher again." + read -r -p "Press Enter to close..." + exit 1 +fi +python3 run.py +read -r -p "Press Enter to close..." diff --git a/start-windows.cmd b/start-windows.cmd new file mode 100644 index 0000000..9f8d6ed --- /dev/null +++ b/start-windows.cmd @@ -0,0 +1,4 @@ +@echo off +cd /d "%~dp0" +python run.py +pause diff --git a/tests/test_workbench.py b/tests/test_workbench.py new file mode 100644 index 0000000..1c80e28 --- /dev/null +++ b/tests/test_workbench.py @@ -0,0 +1,287 @@ +import base64 +import csv +import io +import json +import sqlite3 +import tempfile +import threading +import time +import unittest +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +from wechat_tool import store +from wechat_tool.importer import import_rows, normalize, parse_time, read_rows +from wechat_tool.analysis import Redactor, extract_events, local_cards, validate_model_cards, prepare_plan +from wechat_tool.server import Workbench, sqlite_rows, validate_api, call_model +from scripts.mac_probe import probe +from scripts.export_sqlite import export + +ROOT = Path(__file__).resolve().parent.parent +DEMO = json.loads((ROOT / 'examples/demo.json').read_text(encoding='utf-8')) + + +class CoreTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.db = store.connect(self.temp.name) + + def tearDown(self): + self.db.close() + self.temp.cleanup() + + def demo(self): + import_rows(self.db, DEMO, 'demo', {}) + return store.messages(self.db, {}) + + def test_timezone_and_epoch(self): + self.assertEqual(parse_time('2026-10-01 09:00:00'), '2026-10-01T01:00:00Z') + self.assertEqual(parse_time('2026-10-01T03:00:00-06:00'), '2026-10-01T09:00:00Z') + self.assertEqual(parse_time('1767225600000'), '2026-01-01T00:00:00Z') + with self.assertRaises(ValueError): + parse_time('2026-10-01', '+14:01') + + def test_import_dedup_and_rejection(self): + a = import_rows(self.db, DEMO + [{'content': '缺失字段'}], 'demo', {}) + b = import_rows(self.db, DEMO, 'same-other-name', {}) + self.assertEqual((a['added'], a['rejected'], b['duplicates']), (13, 1, 13)) + self.assertEqual(a['errors'][0]['row'], 14) + + def test_id_scoped_by_conversation_and_multiline(self): + rows = [dict(DEMO[0]), dict(DEMO[0], conversation_id='other')] + self.assertEqual(import_rows(self.db, rows, 'x', {})['added'], 2) + txt = '[2026-10-01 09:00:00] 张三: 第一行\n第二行\n[2026-10-01 09:02:00] 李四: 回复' + parsed = read_rows(txt.encode(), 'x.txt') + self.assertEqual(parsed[0]['content'], '第一行\n第二行') + self.assertEqual(len(parsed), 2) + + def test_encodings_and_alias_mapping(self): + content = '时间,发送人,内容\n2026-10-01 09:00:00,张三,怎么操作？' + self.assertEqual(read_rows(content.encode('gb18030'), 'x.csv')[0]['发送人'], '张三') + self.assertEqual(read_rows(content.encode('utf-16'), 'x.csv')[0]['发送人'], '张三') + msg = normalize({'when': '2026-10-01', 'who': 'a', 'body': 'b'}, {'conversation': 'x', 'mapping': {'timestamp': 'when', 'sender': 'who', 'content': 'body'}}) + self.assertEqual(msg['content'], 'b') + with self.assertRaises(ValueError): + normalize(dict(DEMO[0], content=b'compressed'), {}) + + def test_filters_all_private_exclude_and_keyword(self): + self.demo() + self.assertEqual(len(store.messages(self.db, {'start': '2026-10-08', 'end': '2026-10-08'})), 2) + self.assertEqual(len(store.messages(self.db, {'kind': 'private'})), 3) + self.assertEqual(len(store.messages(self.db, {'conversations': ['demo-test@chatroom']})), 7) + self.assertEqual(len(store.messages(self.db, {'exclude': ['demo-test@chatroom']})), 6) + self.assertEqual(len(store.messages(self.db, {'search': '测试比例'})), 3) + self.assertEqual(len(store.messages(self.db, {'search': '%'})), 0) + + def test_cross_conversation_clustering_and_media(self): + rows = self.demo() + events = extract_events(rows) + cards = local_cards(events) + repeated = next(c for c in cards if c['event_count'] == 3) + self.assertEqual(len(repeated['conversations']), 2) + self.assertIn('repeat', repeated['signals']) + self.assertTrue(all(len(c['conversations']) == 1 for c in local_cards(events, False))) + self.assertTrue(any('非文字' in c['boundaries'] for c in cards)) + self.assertFalse(any('会议' in c['title'] for c in cards)) + + def test_redactor_all_sources_and_custom(self): + rows = self.demo() + redactor = Redactor(rows, 'secret', ['虚构客户甲']) + value = redactor.redact('演示李工 demo-li 演示·测试项目群 虚构客户甲 13800138000 test@example.com https://example.com/?key=abc 密码:hello 192.168.1.1') + for original in ['演示李工', 'demo-li', '演示·测试项目群', '虚构客户甲', '13800138000', 'test@example.com', 'example.com', 'hello', '192.168.1.1']: + self.assertNotIn(original, value) + self.assertEqual(redactor.redact('演示李工'), Redactor(rows, 'secret').redact('演示李工')) + + def test_batches_do_not_truncate(self): + rows = self.demo() + events = extract_events(rows) + plan = prepare_plan(events, Redactor(rows, 'salt')) + self.assertEqual(sum(len(b['events']) for b in plan), len(events)) + events[0]['messages'][0]['content'] = '长' * 50000 + with self.assertRaises(ValueError): + prepare_plan(events, Redactor(rows, 'salt')) + + def test_model_hallucination_and_missing_event_rejected(self): + events = extract_events(self.demo()) + valid = {'cards': [dict(title='测试', summary='候选', event_ids=[e['id'] for e in events], evidence_ids=[events[0]['messages'][0]['id']], signals=['repeat'], strategy='轻量记录', status='待核实', answer='待验证', boundaries='需确认')]} + self.assertEqual(len(validate_model_cards(valid, events)), 1) + valid['cards'][0]['evidence_ids'] = [999999] + with self.assertRaises(ValueError): + validate_model_cards(valid, events) + with self.assertRaises(ValueError): + validate_model_cards({'cards': []}, events) + + def test_sqlite_import_and_readonly_export_with_wal(self): + source = Path(self.temp.name) / 'source.db' + src = sqlite3.connect(source) + src.execute('PRAGMA journal_mode=WAL') + src.execute('CREATE TABLE chat (id TEXT, content TEXT)') + src.execute('INSERT INTO chat VALUES (?,?)', ('a', 'hello')) + src.commit() + out = Path(self.temp.name) / 'readable.jsonl' + export(source, 'chat', out) + self.assertEqual(json.loads(out.read_text())['content'], 'hello') + self.assertEqual(src.execute('SELECT COUNT(*) FROM chat').fetchone()[0], 1) + snapshot_path = Path(self.temp.name) / 'snapshot.db' + snapshot = sqlite3.connect(snapshot_path) + src.backup(snapshot) + snapshot.close() + self.assertEqual(sqlite_rows(snapshot_path.read_bytes(), schema_only=True)[0]['table'], 'chat') + with self.assertRaises(ValueError): + sqlite_rows(b'encrypted') + with self.assertRaises(ValueError): + export(source, 'chat', source) + src.close() + + def test_probe_is_header_only_and_does_not_overwrite(self): + source = Path(self.temp.name) / 'wx.db' + source.write_bytes(b'not encrypted header' + b'private' * 100) + original = source.read_bytes() + report = probe([self.temp.name]) + found = next(d for d in report['databases'] if d['path'] == str(source.absolute())) + self.assertEqual(found['format'], 'encrypted_or_unknown') + self.assertNotIn('private', json.dumps(report)) + self.assertEqual(source.read_bytes(), original) + + def test_api_endpoint_validation(self): + for url in ['http://remote.example/v1/chat/completions', 'https://user:pass@example.com/chat/completions', 'https://example.com/chat/completions?key=secret']: + with self.assertRaises(ValueError): + validate_api({'endpoint': url, 'model': 'x'}) + with self.assertRaises(ValueError) as error: + call_model({'endpoint': 'http://127.0.0.1/chat/completions'}, 'private-key\nvalue', {}) + self.assertNotIn('private-key', str(error.exception)) + + +class FakeModel(BaseHTTPRequestHandler): + calls = [] + invalid = False + + def log_message(self, *args): + pass + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers['Content-Length']))) + type(self).calls.append(body) + batch = json.loads(body['messages'][1]['content']) + cards = [dict(title='脱敏主题', summary='提炼的经验线索', event_ids=[e['id']], evidence_ids=[e['messages'][0]['id']], signals=[], status='待核实', strategy='轻量记录', answer='需要人工验证', boundaries='适用条件待确认') for e in batch['events']] + if self.invalid: + cards[0]['evidence_ids'] = [987654321] + value = json.dumps({'choices': [{'message': {'content': json.dumps({'cards': cards}, ensure_ascii=False)}}]}).encode() + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(value))) + self.end_headers() + self.wfile.write(value) + + +class HttpTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.app = Workbench(('127.0.0.1', 0), self.temp.name) + self.thread = threading.Thread(target=self.app.serve_forever, daemon=True) + self.thread.start() + self.model = ThreadingHTTPServer(('127.0.0.1', 0), FakeModel) + threading.Thread(target=self.model.serve_forever, daemon=True).start() + FakeModel.calls, FakeModel.invalid = [], False + self.base = f'http://127.0.0.1:{self.app.server_port}' + + def tearDown(self): + self.app.shutdown() + self.app.server_close() + self.model.shutdown() + self.model.server_close() + self.thread.join() + self.temp.cleanup() + + def request(self, path, body=None, headers=None, raw=False): + opts = {'Content-Type': 'application/json', 'X-Workbench-Token': self.app.token} + if headers: + opts.update(headers) + req = urllib.request.Request(self.base+path, data=json.dumps(body).encode() if body is not None else None, headers=opts) + with urllib.request.urlopen(req) as resp: + data = resp.read() + return data if raw else json.loads(data) + + def configure(self): + return self.request('/api/settings', {'api': {'endpoint': f'http://127.0.0.1:{self.model.server_port}/v1/chat/completions', 'model': 'mock'}, 'key': 'test-secret-key', 'custom': ['虚构客户甲'], 'utc_offset': '+08:00'}) + + def wait_run(self, rid): + for _ in range(60): + result = self.request('/api/run?id='+rid) + if result['status'] != 'running': + return result + time.sleep(.05) + self.fail('mock model task timed out') + + def test_csrf_and_host_protection(self): + with self.assertRaises(urllib.error.HTTPError) as error: + self.request('/api/demo', {}, {'X-Workbench-Token': ''}) + self.assertEqual(error.exception.code, 403) + with self.assertRaises(urllib.error.HTTPError): + self.request('/api/overview', headers={'Host': 'evil.example'}) + with self.assertRaises(urllib.error.HTTPError): + self.request('/api/demo', {}, {'Origin': 'https://evil.example'}) + + def test_complete_local_review_split_merge_export(self): + self.request('/api/demo', {}) + run = self.request('/api/analyze', {'filters': {'start': '2026-10-01', 'end': '2026-10-31'}, 'cross': True}) + result = self.request('/api/run?id='+run['id']) + card = next(c for c in result['cards'] if c['event_count'] == 3) + self.request('/api/card', {'run_id': run['id'], 'ids': [card['id']], 'action': 'edit', 'title': '=HYPERLINK("bad")', 'reviewed': True}) + csv_text = self.request('/api/export?id='+run['id']+'&format=csv', raw=True).decode('utf-8-sig') + self.assertIn("'=HYPERLINK", csv_text) + self.assertIn('微信知识需求清单', self.request('/api/export?id='+run['id']+'&format=md', raw=True).decode()) + context = self.request('/api/context?run_id='+run['id']+'&card_id='+card['id']) + self.assertEqual(len(context), 3) + # Later imports in the same time window must not rewrite historical evidence. + extra = dict(DEMO[1], id='new-id', timestamp='2026-10-01 09:04:00', content='后来补充的记录') + self.request('/api/import', {'filename': 'late.json', 'data': base64.b64encode(json.dumps([extra]).encode()).decode()}) + self.assertEqual(context, self.request('/api/context?run_id='+run['id']+'&card_id='+card['id'])) + self.request('/api/card', {'run_id': run['id'], 'ids': [card['id']], 'action': 'split', 'event_ids': [card['event_ids'][0]]}) + split = self.request('/api/run?id='+run['id']) + self.assertEqual(len(split['cards']), len(result['cards'])+1) + ids = [c['id'] for c in split['cards'][-2:]] + self.request('/api/card', {'run_id': run['id'], 'ids': ids, 'action': 'merge'}) + merged = self.request('/api/run?id='+run['id']) + self.assertEqual(len(merged['cards']), len(result['cards'])) + + def test_preview_exact_send_key_not_persisted(self): + self.request('/api/demo', {}) + self.configure() + preview = self.request('/api/preview', {'filters': {}, 'cross': True}) + self.assertFalse(FakeModel.calls) + serialized = json.dumps(preview, ensure_ascii=False) + for original in ['演示李工', 'demo-li', '虚构客户甲', '13800138000', 'test@example.com', 'test-secret-key', 'internal.example.com']: + self.assertNotIn(original, serialized) + job = self.request('/api/send', {'plan_id': preview['plan_id']}) + result = self.wait_run(job['id']) + self.assertEqual(result['status'], 'complete', result['error']) + self.assertEqual(FakeModel.calls, preview['requests']) + with self.assertRaises(urllib.error.HTTPError): + self.request('/api/send', {'plan_id': preview['plan_id']}) + db = self.app.db() + self.assertNotIn('test-secret-key', '\n'.join(db.iterdump())) + db.close() + + def test_invalid_model_evidence_fails_atomically(self): + self.request('/api/demo', {}) + self.configure() + FakeModel.invalid = True + preview = self.request('/api/preview', {'filters': {}}) + result = self.wait_run(self.request('/api/send', {'plan_id': preview['plan_id']})['id']) + self.assertEqual(result['status'], 'failed') + self.assertEqual(result['cards'], []) + self.assertIn('证据', result['error']) + + def test_json_upload_and_rejected_encrypted_db(self): + payload = {'filename': 'chat.json', 'data': base64.b64encode(json.dumps(DEMO).encode()).decode(), 'options': {}} + self.assertEqual(self.request('/api/import', payload)['added'], 13) + payload['filename'] = 'encrypted.db' + with self.assertRaises(urllib.error.HTTPError): + self.request('/api/schema', payload) + + +if __name__ == '__main__': + unittest.main() diff --git a/wechat_tool/__init__.py b/wechat_tool/__init__.py new file mode 100644 index 0000000..0f4a2f5 --- /dev/null +++ b/wechat_tool/__init__.py @@ -0,0 +1 @@ +"""Local-first chat analysis.""" diff --git a/wechat_tool/analysis.py b/wechat_tool/analysis.py new file mode 100644 index 0000000..f86b241 --- /dev/null +++ b/wechat_tool/analysis.py @@ -0,0 +1,228 @@ +import hashlib +import json +import math +import re +from collections import Counter +from datetime import datetime + +SIGNALS = {'repeat': '反复提问', 'risk': '风险/事故线索', 'rework': '重复踩坑线索', 'future': '后续复用线索'} +STRATEGIES = ['正式资产', '轻量记录', '专家带教', '暂不投入'] +QUESTION = re.compile(r'[?？]|怎么|如何|为什么|是否|求助|请教|谁知道|有没有|怎么办|不清楚|不会|帮忙|报错|失败|异常|出错') +RISK = re.compile(r'事故|故障|损失|误判|错误|风险|差点|出错|报错|失败|异常|生产|回滚') +REWORK = re.compile(r'返工|重做|重来|又.{0,8}(错|失败|出问题)|再次|上次.{0,8}(错|问题)|重复踩') +FUTURE = re.compile(r'下次|明年|下个月|后续|接下来|以后|还要|将要|计划|即将') +STOP = re.compile(r'请问|请教|大家|这个|那个|一下|怎么|如何|什么|有没有|能不能|是不是|谢谢|麻烦|帮忙|知道|可以|还是') + + +def terms(text): + text = STOP.sub('', text.lower()) + chunks = re.findall(r'[a-z0-9_]{2,}|[\u4e00-\u9fff]+', text) + result = set() + for chunk in chunks: + if re.fullmatch('[\u4e00-\u9fff]+', chunk): + result.update(chunk[i:i+2] for i in range(len(chunk)-1)) + else: + result.add(chunk) + return result + + +def focus_terms(text): + # A question's qualifiers often follow the question mark; keep its core stable. + return terms(re.split(r'[?？。!！\n]', text, maxsplit=1)[0]) + + +def is_question(text): + return bool(QUESTION.search(re.sub(r'https?://\S+', '', text))) + + +def similarity(a, b): + return len(a & b) / math.sqrt(len(a) * len(b)) if a and b else 0 + + +def extract_events(messages): + """Question-led time windows, with bounded context and explicit uncertainty.""" + by_conv = {} + for msg in messages: + by_conv.setdefault(msg['conversation_id'], []).append(msg) + events = [] + for cid, rows in by_conv.items(): + current = None + for msg in rows: + moment = datetime.fromisoformat(msg['timestamp'].replace('Z', '+00:00')) + interesting = bool(is_question(msg['content']) or RISK.search(msg['content']) or REWORK.search(msg['content']) or FUTURE.search(msg['content'])) + gap = (moment - current['_last']).total_seconds() if current else float('inf') + related = current and (not interesting or msg['quote'] or similarity(focus_terms(msg['content']), current['_terms']) >= .24) + if current and gap <= 1800 and len(current['messages']) < 30 and related: + current['messages'].append(msg) + current['_last'] = moment + elif interesting: + current = dict(id=f"E{msg['id']}", conversation_id=cid, conversation=msg['conversation'], + title=msg['content'][:100], messages=[msg], _last=moment, _terms=focus_terms(msg['content'])) + events.append(current) + else: + current = None + for event in events: + del event['_last'] + del event['_terms'] + return sorted(events, key=lambda e: e['messages'][0]['timestamp']) + + +def make_card(events, title=None): + msgs = [m for e in events for m in e['messages']] + text = '\n'.join(m['content'] for m in msgs) + signals = [] + if len(events) > 1: + signals.append('repeat') + for key, pattern in [('risk', RISK), ('rework', REWORK), ('future', FUTURE)]: + if pattern.search(text): + signals.append(key) + responders = sorted(set(m['sender'] for e in events for m in e['messages'][1:] + if m['message_type'] == 'text' and not is_question(m['content']) + and m['sender_id'] != e['messages'][0]['sender_id'] and len(m['content']) >= 12)) + gaps = sorted(set(m['message_type'] for m in msgs if m['message_type'] != 'text')) + return dict(id=hashlib.sha256('|'.join(e['id'] for e in events).encode()).hexdigest()[:16], + title=title or events[0]['title'], summary='基于时间窗口和文本相似度的候选主题，需人工核对。', + signals=signals, event_ids=[e['id'] for e in events], + evidence_ids=sorted(set(m['id'] for m in msgs)), + event_count=len(events), conversations=sorted(set(m['conversation'] for m in msgs)), + participants=sorted(set(m['sender'] for m in msgs)), responders=responders, + first=min(m['timestamp'] for m in msgs), last=max(m['timestamp'] for m in msgs), + status='待核实', strategy='正式资产' if len(events) >= 3 or 'risk' in signals else '轻量记录', + priority=min(100, 15*len(events) + 20*('risk' in signals) + 15*('rework' in signals) + 10*('future' in signals)), + answer='尚未验证。候选答复请查看原文，不能据此认定问题已经解决。', + boundaries='自动按 30 分钟窗口收集上下文；可能包含交错话题或漏掉延迟回复。' + (' 包含非文字消息：' + '、'.join(gaps) if gaps else ''), + reviewed=False, notes='') + + +def local_cards(events, cross=True): + groups, signatures, inverted = [], [], {} + for event in events: + sig = focus_terms(event['title']) + candidates = set() + for term in sig: + candidates.update(inverted.get(term, [])) + best, score = None, .42 + for idx in candidates: + if not cross and groups[idx][0]['conversation_id'] != event['conversation_id']: + continue + value = similarity(sig, signatures[idx]) + if value > score: + best, score = idx, value + if best is None: + best = len(groups) + groups.append([event]) + signatures.append(sig) + for term in sig: + inverted.setdefault(term, []).append(best) + else: + groups[best].append(event) + return sorted([make_card(group) for group in groups], key=lambda c: (-c['priority'], -c['event_count'])) + + +class Redactor: + def __init__(self, all_messages, secret, custom=None): + self.secret = secret + self.pairs = {} + for msg in all_messages: + for kind, fields in [('人员', ['sender', 'sender_id']), ('会话', ['conversation', 'conversation_id'])]: + values = [msg.get(field, '') for field in fields] + canonical = next((v for v in reversed(values) if v), '') + alias = self.alias(kind, canonical) + for value in values: + if value: + self.pairs[value] = alias + for value in custom or []: + if value.strip(): + self.pairs[value.strip()] = self.alias('敏感词', value.strip()) + self.pattern = re.compile('|'.join(re.escape(v) for v in sorted(self.pairs, key=len, reverse=True))) if self.pairs else None + + def alias(self, kind, value): + # Salted aliases cannot be guessed from a public hash of a person's name. + import hmac + return '[' + kind + '_' + hmac.new(self.secret.encode(), value.encode(), hashlib.sha256).hexdigest()[:10] + ']' + + def redact(self, text): + text = str(text) + if self.pattern: + text = self.pattern.sub(lambda m: self.pairs[m[0]], text) + patterns = [ + ('凭据', r'(?i)\b(?:sk-[a-z0-9_-]{12,}|bearer\s+[a-z0-9._-]{10,})'), + ('邮箱', r'[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}'), + ('链接', r'(?i)https?://[^\s<>\]\)）]+'), + ('身份证', r'(? 48000: + raise ValueError('某个讨论事件超过 48000 字符，请缩小时间范围或在本地拆分后重试；工具不会截断发送') + if current and (size + length > 48000 or len(current) >= 30): + batches.append(current) + current, size = [], 0 + current.append(item) + size += length + if current: + batches.append(current) + return [{'cross_conversation': bool(cross), 'events': batch} for batch in batches] + + +def validate_model_cards(output, events, cross=True): + if not isinstance(output, dict) or not isinstance(output.get('cards'), list): + raise ValueError('模型没有返回 cards 数组') + by_id = {e['id']: e for e in events} + seen, cards = set(), [] + for raw in output['cards']: + if not isinstance(raw, dict): + raise ValueError('模型卡片格式错误') + ids = raw.get('event_ids') + if not isinstance(ids, list) or not ids or any(not isinstance(i, str) or i not in by_id or i in seen for i in ids) or len(ids) != len(set(ids)): + raise ValueError('模型引用了未知、重复或为空的事件') + group = [by_id[i] for i in ids] + if not cross and len({e['conversation_id'] for e in group}) > 1: + raise ValueError('模型违反了按会话分析的限制') + valid = {m['id'] for e in group for m in e['messages']} + evidence = raw.get('evidence_ids') + if not isinstance(evidence, list) or not evidence or any(type(i) is not int or i not in valid for i in evidence): + raise ValueError('模型引用了无效的原文证据') + card = make_card(group) + for key in ('title', 'summary', 'answer', 'boundaries'): + if not isinstance(raw.get(key), str) or not raw[key].strip(): + raise ValueError('模型缺少字段：' + key) + card[key] = raw[key][:6000] + if raw.get('strategy') not in STRATEGIES: + raise ValueError('模型给出了无效的处理策略') + card['strategy'] = raw['strategy'] + allowed = ('已有经验待验证', '答案冲突', '未见明确答案', '待核实') + card['status'] = raw.get('status') if raw.get('status') in allowed else '待核实' + signals = raw.get('signals', []) + if not isinstance(signals, list) or any(s not in SIGNALS for s in signals): + raise ValueError('模型信号格式错误') + card['signals'] = sorted(set(signals) - ({'repeat'} if len(group) < 2 else set())) + card['evidence_ids'] = sorted(set(evidence)) + # Local responders remain only candidates, never AI-certified experts. + card['boundaries'] += ' 模型结论待人工审核；分批分析可能漏掉批次间的相似问题。' + cards.append(card) + seen.update(ids) + if seen != set(by_id): + raise ValueError('模型漏掉了部分事件，本次结果未保存') + return cards diff --git a/wechat_tool/importer.py b/wechat_tool/importer.py new file mode 100644 index 0000000..57478ef --- /dev/null +++ b/wechat_tool/importer.py @@ -0,0 +1,161 @@ +"""Explicit adapters for readable exports. Never modifies a WeChat database.""" +import csv +import hashlib +import io +import json +import re +from datetime import datetime, timedelta, timezone + +ALIASES = { + 'source_id': ['id', 'msg_id', 'msgId', 'MsgSvrID', 'msgSvrId', '消息ID'], + 'conversation': ['conversation', 'chat_name', 'talkerName', 'room_name', '群名', '会话'], + 'conversation_id': ['conversation_id', 'talker', 'StrTalker', 'chat_id', 'room_id', '会话ID'], + 'kind': ['kind', 'chat_type', '会话类型'], + 'sender': ['sender', 'sender_name', 'senderName', 'NickName', '发送人'], + 'sender_id': ['sender_id', 'senderUsername', 'from_user', '发送人ID'], + 'timestamp': ['timestamp', 'time', 'createTime', 'CreateTime', 'datetime', '时间'], + 'content': ['content', 'text', 'StrContent', 'Content', '内容'], + 'message_type': ['message_type', 'type', 'Type', '消息类型'], + 'quote': ['quote', '引用'], +} + + +def offset_zone(offset): + if not re.fullmatch(r'[+-](?:0\d|1[0-4]):[0-5]\d', str(offset)): + raise ValueError('时区偏移格式应为 +08:00 或 -06:00') + hours, minutes = map(int, offset[1:].split(':')) + if hours == 14 and minutes: + raise ValueError('时区偏移不能超过 14 小时') + return timezone((1 if offset[0] == '+' else -1) * timedelta(hours=hours, minutes=minutes)) + + +def parse_time(value, offset='+08:00'): + zone = offset_zone(offset) + value = str(value).strip() + if re.fullmatch(r'\d{10}(?:\.\d+)?|\d{13}', value): + number = float(value) + if number > 100_000_000_000: + number /= 1000 + dt = datetime.fromtimestamp(number, timezone.utc) + else: + value = value.replace('/', '-').replace('年', '-').replace('月', '-').replace('日', '') + dt = datetime.fromisoformat(value.replace('Z', '+00:00')) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=zone) + # One canonical UTC representation, independent of the importing computer. + return dt.astimezone(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') + + +def decode(data): + encodings = ('utf-16',) if data.startswith((b'\xff\xfe', b'\xfe\xff')) else ('utf-8-sig', 'gb18030') + for encoding in encodings: + try: + return data.decode(encoding) + except UnicodeError: + continue + raise ValueError('无法识别文件编码，请另存为 UTF-8') + + +def read_rows(data, filename): + text = decode(data) + ext = filename.lower().rsplit('.', 1)[-1] + if ext == 'json': + rows = json.loads(text) + if isinstance(rows, dict): + rows = rows.get('messages', rows.get('items', rows.get('data'))) + if isinstance(rows, dict): + rows = rows.get('messages', rows.get('items')) + if not isinstance(rows, list): + raise ValueError('JSON 应为消息数组或包含 messages 数组的对象') + elif ext in ('jsonl', 'ndjson'): + rows = [json.loads(line) for line in text.splitlines() if line.strip()] + elif ext in ('csv', 'tsv'): + rows = list(csv.DictReader(io.StringIO(text), delimiter='\t' if ext == 'tsv' else ',')) + elif ext == 'txt': + # Deliberately strict: no guessing ambiguous sender/time boundaries. + rows = [] + current = None + for line in text.splitlines(): + match = re.match(r'^\[(\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2})\]\s+([^:：]+)[:：]\s?(.*)$', line) + if match: + current = dict(timestamp=match[1], sender=match[2], content=match[3]) + rows.append(current) + elif current: + current['content'] += '\n' + line + elif line.strip(): + raise ValueError('TXT 格式应为：[2026-10-01 09:00:00] 张三: 消息内容。其他格式请先转换为 CSV/JSON。') + else: + raise ValueError('支持 JSON / JSONL / CSV / TSV / TXT；SQLite 请使用数据库导入页') + if any(not isinstance(row, dict) for row in rows): + raise ValueError('每条消息必须是对象') + return rows + + +def normalize(row, options): + mapping = options.get('mapping') or {} + + def get(field, default=''): + keys = [mapping[field]] if mapping.get(field) else ALIASES[field] + for key in keys: + if row.get(key) is not None and str(row[key]).strip(): + val = row[key] + if isinstance(val, bytes): + raise ValueError('字段 ' + key + ' 含二进制内容，需先解码为可读文本') + return json.dumps(val, ensure_ascii=False) if isinstance(val, (dict, list)) else str(val).strip() + return default + + cid = get('conversation_id', options.get('conversation_id', options.get('conversation', ''))) + name = get('conversation', options.get('conversation', cid)) + if not cid: + cid = name + if not cid: + raise ValueError('缺少会话名称/ID，请填写默认会话或字段映射') + kind = get('kind', options.get('kind', '')) + if kind in ('群', '群聊', 'chatroom'): + kind = 'group' + if kind in ('私聊', '个人', 'direct'): + kind = 'private' + if not kind: + kind = 'group' if cid.endswith('@chatroom') else 'private' + if kind not in ('group', 'private'): + raise ValueError('会话类型应为 group 或 private') + sender = get('sender', get('sender_id')) + if not sender: + raise ValueError('缺少发送人，请配置 sender 字段映射') + content = get('content') + typ = get('message_type', 'text') + typ = 'text' if typ.lower() in ('1', 'text', '文本', '文字') else typ + if not content and typ == 'text': + raise ValueError('文字消息内容为空') + msg = dict(source_id=get('source_id'), conversation=name or cid, conversation_id=cid, + kind=kind, sender=sender, sender_id=get('sender_id', sender), + timestamp=parse_time(get('timestamp'), options.get('utc_offset', '+08:00')), + content=content, message_type=typ, quote=get('quote')) + # Authoritative IDs are scoped by conversation; no-ID files use content identity. + identity = [cid, msg['source_id']] if msg['source_id'] else [cid, msg['sender_id'], msg['timestamp'], content, typ, msg['quote']] + msg['fingerprint'] = hashlib.sha256(json.dumps(identity, ensure_ascii=False).encode()).hexdigest() + return msg + + +def import_rows(db, rows, source, options): + added = duplicates = rejected = without_ids = 0 + errors = [] + for index, row in enumerate(rows, 1): + try: + msg = normalize(row, options) + except (ValueError, TypeError, OverflowError, OSError) as exc: + rejected += 1 + if len(errors) < 30: + errors.append({'row': index, 'reason': str(exc)}) + continue + msg['source'] = source + without_ids += not bool(msg['source_id']) + fields = list(msg) + result = db.execute('INSERT OR IGNORE INTO messages (' + ','.join(fields) + ') VALUES (' + ','.join('?' for _ in fields) + ')', [msg[f] for f in fields]) + if result.rowcount: + added += 1 + else: + duplicates += 1 + db.commit() + return dict(added=added, duplicates=duplicates, rejected=rejected, errors=errors, total=len(rows), + without_ids=without_ids, note='无消息 ID 的记录按会话、发送人、时间、内容、类型与引用去重；同秒完全相同的真实消息可能被合并。' if without_ids else '') diff --git a/wechat_tool/server.py b/wechat_tool/server.py new file mode 100644 index 0000000..ee8e18e --- /dev/null +++ b/wechat_tool/server.py @@ -0,0 +1,456 @@ +import argparse +import base64 +import csv +import io +import json +import os +import secrets +import sqlite3 +import tempfile +import threading +import urllib.error +import urllib.parse +import urllib.request +import webbrowser +from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +from . import store +from .analysis import (SYSTEM_PROMPT, STRATEGIES, SIGNALS, Redactor, extract_events, + local_cards, make_card, prepare_plan, validate_model_cards) +from .importer import import_rows, read_rows, offset_zone + +STATIC = Path(__file__).parent / 'static' +MAX_UPLOAD = 64 * 1024 * 1024 + + +def now(): + return datetime.now(timezone.utc).isoformat() + + +def validate_api(config): + endpoint = str(config.get('endpoint', '')).strip() + parsed = urllib.parse.urlsplit(endpoint) + if parsed.username or parsed.password or parsed.query or parsed.fragment: + raise ValueError('API 地址不能包含账号、密码、查询参数或片段') + if parsed.scheme != 'https' and not (parsed.scheme == 'http' and parsed.hostname in ('127.0.0.1', 'localhost', '::1')): + raise ValueError('远程模型 API 必须使用 HTTPS，本地模型可使用 localhost HTTP') + if not parsed.hostname or not parsed.path.endswith('/chat/completions'): + raise ValueError('请填写完整的兼容接口地址，以 /chat/completions 结尾') + model = str(config.get('model', '')).strip() + if not model or len(model) > 200: + raise ValueError('请填写模型名称') + return {'endpoint': endpoint, 'model': model} + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *args, **kwargs): + raise ValueError('API 重定向已拒绝，避免脱敏文本和密钥被转发到其他地址') + + +def call_model(config, key, body): + if key and (not key.isascii() or any(ord(char) < 33 or ord(char) > 126 for char in key)): + raise ValueError('API 密钥含无效字符，请重新填写（不包含空格或换行）') + headers = {'Content-Type': 'application/json', 'Accept': 'application/json'} + if key: + headers['Authorization'] = 'Bearer ' + key + req = urllib.request.Request(config['endpoint'], data=json.dumps(body, ensure_ascii=False).encode(), headers=headers) + try: + # Do not implicitly route chat text through system HTTP proxies. + opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect()) + with opener.open(req, timeout=120) as response: + data = response.read(8 * 1024 * 1024 + 1) + if len(data) > 8 * 1024 * 1024: + raise ValueError('模型响应过大') + result = json.loads(data) + if not isinstance(result, dict): + raise ValueError('模型接口返回格式错误') + choices = result.get('choices') + if not isinstance(choices, list) or not choices: + raise ValueError('模型接口未返回 choices') + text = choices[0].get('message', {}).get('content') + if not isinstance(text, str): + raise ValueError('模型没有返回文本结果') + text = text.strip() + if text.startswith('```'): + text = text.split('\n', 1)[1].rsplit('```', 1)[0].strip() + return json.loads(text) + except urllib.error.HTTPError as exc: + # Never persist provider response bodies: they can echo the prompt/key. + raise ValueError(f'模型 API 返回 HTTP {exc.code}，请检查地址、密钥、模型和额度') from None + except (urllib.error.URLError, TimeoutError, OSError): + raise ValueError('模型 API 连接失败或超时，请检查网络与接口地址') from None + except (json.JSONDecodeError, IndexError, KeyError, AttributeError): + raise ValueError('模型响应不是可用的 JSON，未保存不完整分析') from None + + +def sqlite_rows(data, table=None, schema_only=False): + if not data.startswith(b'SQLite format 3\x00'): + raise ValueError('这不是明文 SQLite 数据库。微信加密数据库需先在 Mac 上取得可读导出；本工具不冒充已完成解密。') + with tempfile.TemporaryDirectory(prefix='wechat-import-') as directory: + path = Path(directory) / 'input.sqlite3' + path.write_bytes(data) + db = sqlite3.connect(path.as_uri() + '?mode=ro&immutable=1', uri=True) + db.row_factory = sqlite3.Row + try: + db.execute('PRAGMA query_only=ON') + tables = [r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")] + def quoted(name): + return '"' + name.replace('"', '""') + '"' + if schema_only: + return [{'table': name, 'columns': [r[1] for r in db.execute('PRAGMA table_info(' + quoted(name) + ')')]} for name in tables] + if table not in tables: + raise ValueError('请选择数据库中存在的消息表') + rows = db.execute('SELECT * FROM ' + quoted(table) + ' LIMIT 500001').fetchall() + if len(rows) > 500000: + raise ValueError('单次最多导入 50 万条，请拆分文件') + return [dict(r) for r in rows] + finally: + db.close() + + +class Workbench(ThreadingHTTPServer): + daemon_threads = True + + def __init__(self, address, data): + super().__init__(address, Handler) + self.data = Path(data).resolve() + self.token = secrets.token_urlsafe(32) + self.lock = threading.RLock() + self.plans = {} + self.api_key = os.environ.get('WECHAT_MODEL_API_KEY', '') + db = store.connect(self.data) + if not store.setting(db, 'salt'): + store.set_setting(db, 'salt', secrets.token_hex(32)) + # Interrupted API jobs cannot be silently resumed or automatically re-sent. + db.execute("UPDATE runs SET status='failed',error='服务重启中断了任务；不会自动重新发送，请重新预览后执行' WHERE status='running'") + db.commit() + db.close() + + def db(self): + return store.connect(self.data) + + +class Handler(BaseHTTPRequestHandler): + server_version = 'WeChatWorkbench/0.1' + + def log_message(self, *args): + pass # Neither chat payloads nor credentials belong in request logs. + + def respond(self, value, status=200, mime='application/json; charset=utf-8', filename=None): + data = json.dumps(value, ensure_ascii=False).encode() if mime.startswith('application/json') else value.encode() if isinstance(value, str) else value + self.send_response(status) + self.send_header('Content-Type', mime) + self.send_header('Content-Length', str(len(data))) + self.send_header('Cache-Control', 'no-store') + self.send_header('X-Content-Type-Options', 'nosniff') + self.send_header('Referrer-Policy', 'no-referrer') + self.send_header('Content-Security-Policy', "default-src 'self'; script-src 'self'; style-src 'self'; connect-src 'self'; img-src 'self' data:; frame-ancestors 'none'; base-uri 'none'; form-action 'self'") + if filename: + self.send_header('Content-Disposition', 'attachment; filename="' + filename + '"') + self.end_headers() + self.wfile.write(data) + + def check_request(self, mutation=False): + expected = f'127.0.0.1:{self.server.server_port}' + if self.headers.get('Host') != expected: + raise PermissionError('只允许通过本机 127.0.0.1 访问') + origin = self.headers.get('Origin') + if origin and origin != 'http://' + expected: + raise PermissionError('拒绝跨站请求') + if self.headers.get('Sec-Fetch-Site') in ('cross-site', 'same-site'): + raise PermissionError('拒绝跨站请求') + if mutation and not secrets.compare_digest(self.headers.get('X-Workbench-Token', ''), self.server.token): + raise PermissionError('页面会话失效，请刷新') + + def do_GET(self): + self.dispatch(False) + + def do_POST(self): + self.dispatch(True) + + def dispatch(self, post): + db = None + try: + self.check_request(post) + parts = urllib.parse.urlsplit(self.path) + path = parts.path + if not post and path in ('/', '/app.js', '/style.css'): + file = STATIC / ('index.html' if path == '/' else path[1:]) + text = file.read_text(encoding='utf-8').replace('__SESSION_TOKEN__', self.server.token) + return self.respond(text, mime={'/': 'text/html; charset=utf-8', '/app.js': 'text/javascript; charset=utf-8', '/style.css': 'text/css; charset=utf-8'}[path]) + body = {} + if post: + length = int(self.headers.get('Content-Length', '0')) + if not 0 < length <= MAX_UPLOAD: + raise ValueError('请求为空或超过 64 MB（文件 base64 后约增大三分之一）') + if 'application/json' not in self.headers.get('Content-Type', ''): + raise ValueError('请求必须是 JSON') + body = json.loads(self.rfile.read(length)) + if not isinstance(body, dict): + raise ValueError('请求必须是对象') + query = urllib.parse.parse_qs(parts.query) + db = self.server.db() + if not post: + return self.get_route(db, path, query) + return self.post_route(db, path, body) + except PermissionError as exc: + self.respond({'error': str(exc)}, 403) + except (ValueError, TypeError, sqlite3.Error, KeyError, OverflowError) as exc: + # Type/key/sql errors do not contain user payloads in supported routes. + error = str(exc) if isinstance(exc, ValueError) else '请求格式或数据库结构不符合要求，请检查字段和输入' + self.respond({'error': error}, 400) + except (BrokenPipeError, ConnectionResetError): + pass + except Exception: + self.respond({'error': '内部错误，数据未发送到新的地址。请重启后重试或运行测试诊断。'}, 500) + finally: + if db: + db.close() + + def get_route(self, db, path, query): + if path == '/api/overview': + result = store.overview(db) + result['runs'] = [dict(r) for r in db.execute('SELECT id,created,mode,status,error,coverage FROM runs ORDER BY created DESC')] + for r in result['runs']: + r['coverage'] = json.loads(r['coverage']) + return self.respond(result) + if path == '/api/settings': + return self.respond({'api': store.setting(db, 'api', {}), 'custom': store.setting(db, 'custom', []), + 'utc_offset': store.setting(db, 'utc_offset', '+08:00'), 'has_key': bool(self.server.api_key)}) + if path == '/api/run': + run = self.get_run(db, query.get('id', [''])[0]) + return self.respond(run) + if path == '/api/evidence': + ids = [int(i) for i in query.get('ids', [''])[0].split(',') if i] + if len(ids) > 1000: + raise ValueError('一次最多读取 1000 条证据') + rows = [dict(r) for r in db.execute('SELECT * FROM messages WHERE id IN (' + ','.join('?' for _ in ids) + ') ORDER BY timestamp,id', ids)] if ids else [] + return self.respond(rows) + if path == '/api/context': + run = self.get_run(db, query.get('run_id', [''])[0]) + card = next((c for c in run['cards'] if c['id'] == query.get('card_id', [''])[0]), None) + if not card: + raise ValueError('卡片不存在') + events = self.run_events(db, run['id']) + return self.respond([e for e in events if e['id'] in card['event_ids']]) + if path == '/api/export': + run = self.get_run(db, query.get('id', [''])[0]) + fmt = query.get('format', ['md'])[0] + if fmt == 'csv': + out = io.StringIO(newline='') + writer = csv.writer(out) + writer.writerow(['主题', '摘要', '事件数', '优先级(启发式)', '信号', '状态', '策略', '审核', '证据ID', '会话', '答复候选', '适用边界', '备注', '经验提供者候选']) + for c in run['cards']: + vals = [c['title'], c['summary'], c['event_count'], c['priority'], '、'.join(SIGNALS[s] for s in c['signals']), c['status'], c['strategy'], '已审核' if c['reviewed'] else '待审核', ','.join(map(str, c['evidence_ids'])), '、'.join(c['conversations']), c['answer'], c['boundaries'], c.get('notes', ''), '、'.join(c['responders'])] + # Neutralize spreadsheet formula injection from messages/model output. + writer.writerow(["'" + str(v) if str(v).lstrip().startswith(('=', '+', '-', '@', '\t', '\r')) else v for v in vals]) + return self.respond('\ufeff' + out.getvalue(), mime='text/csv; charset=utf-8', filename='knowledge-needs.csv') + lines = ['# 微信知识需求清单', '', '生成时间：' + run['created'], '', '覆盖范围：' + json.dumps(run['coverage'], ensure_ascii=False), '', '模型结果和本地规则均为候选判断；人工审核不等于实践验证。', ''] + for c in run['cards']: + lines.extend(['## ' + c['title'].replace('\n', ' '), '', c['summary'], '', f"- 独立事件候选数：{c['event_count']}；优先级（启发式）：{c['priority']}", '- 信号：' + '、'.join(SIGNALS[s] for s in c['signals']), '- 状态：' + c['status'], '- 处理策略：' + c['strategy'], '- 人工审核：' + ('已审核' if c['reviewed'] else '待审核'), '- 证据消息 ID：' + ', '.join(map(str, c['evidence_ids'])), '', '### 经验候选', '', c['answer'], '', '### 边界与缺口', '', c['boundaries'], '', c.get('notes', ''), '']) + lines.extend(['经验提供者候选（需核实）：' + '、'.join(c['responders']), '']) + return self.respond('\n'.join(lines), mime='text/markdown; charset=utf-8', filename='knowledge-needs.md') + return self.respond({'error': '不存在的接口'}, 404) + + def get_run(self, db, run_id): + row = db.execute('SELECT * FROM runs WHERE id=?', (run_id,)).fetchone() + if not row: + raise ValueError('分析任务不存在') + result = dict(row) + result.pop('events', None) + for field in ('cards', 'filters', 'coverage'): + result[field] = json.loads(result[field]) + return result + + def run_events(self, db, run_id): + row = db.execute('SELECT events FROM runs WHERE id=?', (run_id,)).fetchone() + if not row: + raise ValueError('分析任务不存在') + return json.loads(row[0]) + + def selected(self, db, filters): + rows = store.messages(db, filters) + if not rows: + raise ValueError('筛选范围内没有记录') + if len(rows) > 200000: + raise ValueError('一次最多分析 20 万条，请缩小时间范围') + events = extract_events(rows) + coverage = {'messages': len(rows), 'events': len(events), 'conversations': len({m['conversation_id'] for m in rows}), + 'first': rows[0]['timestamp'], 'last': rows[-1]['timestamp'], 'media': sum(m['message_type'] != 'text' for m in rows), + 'source': '仅本机已导入记录；时间窗口为讨论候选，不代表完整问答或因果关系'} + return rows, events, coverage + + def save_run(self, db, mode, filters, coverage, cards, events, status='complete'): + rid = secrets.token_hex(12) + db.execute('INSERT INTO runs (id,created,mode,filters,cards,coverage,status,error,events) VALUES (?,?,?,?,?,?,?,?,?)', (rid, now(), mode, json.dumps(filters, ensure_ascii=False), json.dumps(cards, ensure_ascii=False), json.dumps(coverage, ensure_ascii=False), status, '', json.dumps(events, ensure_ascii=False))) + db.commit() + return rid + + def post_route(self, db, path, body): + if path == '/api/import': + data = base64.b64decode(body['data'], validate=True) + options = body.get('options') or {} + offset_zone(options.get('utc_offset', '+08:00')) + filename = Path(body['filename']).name + if filename.lower().endswith(('.db', '.sqlite', '.sqlite3')): + rows = sqlite_rows(data, options.get('table')) + else: + rows = read_rows(data, filename) + if len(rows) > 500000: + raise ValueError('单次最多导入 50 万条') + return self.respond(import_rows(db, rows, filename, options)) + if path == '/api/schema': + return self.respond(sqlite_rows(base64.b64decode(body['data'], validate=True), schema_only=True)) + if path == '/api/demo': + rows = json.loads((Path(__file__).parent.parent / 'examples' / 'demo.json').read_text(encoding='utf-8')) + return self.respond(import_rows(db, rows, 'demo.json（虚构样例）', {'utc_offset': '+08:00'})) + if path == '/api/settings': + api = validate_api(body['api']) if body.get('api', {}).get('endpoint') else {} + custom = body.get('custom', []) + if not isinstance(custom, list) or any(not isinstance(x, str) for x in custom) or len(custom) > 2000: + raise ValueError('自定义敏感词必须是字符串数组，最多 2000 项') + offset_zone(body.get('utc_offset', '+08:00')) + new_key = str(body.get('key', '')).strip() + if new_key and (not new_key.isascii() or any(ord(c) < 33 or ord(c) > 126 for c in new_key)): + raise ValueError('API 密钥含无效字符，请重新填写（不包含空格或换行）') + store.set_setting(db, 'api', api) + store.set_setting(db, 'custom', custom) + store.set_setting(db, 'utc_offset', body.get('utc_offset', '+08:00')) + with self.server.lock: + if body.get('clear_key'): + self.server.api_key = '' + elif body.get('key'): + self.server.api_key = new_key + self.server.plans.clear() + return self.respond({'saved': True}) + if path in ('/api/analyze', '/api/preview'): + filters = body.get('filters') or {} + rows, events, coverage = self.selected(db, filters) + cross = bool(body.get('cross', True)) + filters = dict(filters, cross=cross) + if path == '/api/analyze': + rid = self.save_run(db, 'local', filters, coverage, local_cards(events, cross), events) + return self.respond({'id': rid}) + if not events: + raise ValueError('当前规则未找到问题或复用线索；可查看原文或调整范围') + config = validate_api(store.setting(db, 'api', {})) + # Collect names from all imported conversations, including excluded ones. + contacts = [dict(r) for r in db.execute('SELECT DISTINCT sender,sender_id,conversation,conversation_id FROM messages')] + redactor = Redactor(contacts, store.setting(db, 'salt'), store.setting(db, 'custom', [])) + batches = prepare_plan(events, redactor, cross) + bodies = [{'model': config['model'], 'messages': [{'role': 'system', 'content': SYSTEM_PROMPT}, {'role': 'user', 'content': json.dumps(batch, ensure_ascii=False)}], 'temperature': 0.2} for batch in batches] + plan_id = secrets.token_hex(16) + plan = dict(config=config, bodies=bodies, events=events, filters=filters, coverage=coverage, created=datetime.now().timestamp()) + with self.server.lock: + # Preview plaintext is redacted. Original event snapshot stays in memory. + if len(self.server.plans) >= 10: + self.server.plans.clear() + self.server.plans[plan_id] = plan + return self.respond({'plan_id': plan_id, 'endpoint': config['endpoint'], 'coverage': coverage, 'batches': len(bodies), 'characters': sum(len(json.dumps(b, ensure_ascii=False)) for b in bodies), 'requests': bodies}) + if path == '/api/send': + with self.server.lock: + plan = self.server.plans.pop(body.get('plan_id', ''), None) + key = self.server.api_key + if not plan or datetime.now().timestamp() - plan['created'] > 1800: + raise ValueError('预览已失效或已发送，请重新生成预览') + coverage = dict(plan['coverage'], batches=len(plan['bodies']), cross_batch_merge=False) + rid = self.save_run(db, 'api', plan['filters'], coverage, [], plan['events'], 'running') + threading.Thread(target=self.run_ai, args=(rid, plan, key), daemon=True).start() + return self.respond({'id': rid}) + if path == '/api/card': + with self.server.lock: + run = self.get_run(db, body['run_id']) + if run['status'] != 'complete': + raise ValueError('任务尚未完成') + cards = run['cards'] + action = body.get('action', 'edit') + indices = [i for i, c in enumerate(cards) if c['id'] in body.get('ids', [])] + if not indices: + raise ValueError('请选择卡片') + if action == 'edit': + if len(indices) != 1: + raise ValueError('一次编辑一张卡片') + card = cards[indices[0]] + for field in ('title', 'summary', 'answer', 'boundaries', 'notes'): + if field in body: + card[field] = str(body[field])[:10000] + if body.get('strategy') in STRATEGIES: + card['strategy'] = body['strategy'] + card['reviewed'] = bool(body.get('reviewed')) + elif action in ('merge', 'split'): + event_map = {e['id']: e for e in self.run_events(db, run['id'])} + if action == 'merge': + if len(indices) < 2: + raise ValueError('合并至少需要两张卡片') + events = [event_map[eid] for i in indices for eid in cards[i]['event_ids']] + if not run['filters'].get('cross', True) and len({e['conversation_id'] for e in events}) > 1: + raise ValueError('该任务按会话分析，不可跨会话合并') + new = make_card(events, body.get('title') or cards[indices[0]]['title']) + new['notes'] = '\n\n'.join(c['title'] + '\n' + c['summary'] + '\n' + c.get('notes', '') for i,c in enumerate(cards) if i in indices) + additions = [new] + else: + if len(indices) != 1: + raise ValueError('一次拆分一张卡片') + original = cards[indices[0]] + ids = original['event_ids'] + chosen = body.get('event_ids', []) + if not chosen or not set(chosen) < set(ids): + raise ValueError('请选择要拆出的部分事件，不能选空或全选') + additions = [make_card([event_map[e] for e in ids if (e in chosen) == included]) for included in (True, False)] + for new in additions: + new['notes'] = '拆分前备注：' + original.get('notes', '') + cards = [c for i,c in enumerate(cards) if i not in indices] + additions + else: + raise ValueError('未知编辑操作') + db.execute('UPDATE runs SET cards=? WHERE id=?', (json.dumps(cards, ensure_ascii=False), run['id'])) + db.commit() + return self.respond({'saved': True}) + return self.respond({'error': '不存在的接口'}, 404) + + def run_ai(self, rid, plan, key): + db = self.server.db() + try: + cards = [] + event_map = {e['id']: e for e in plan['events']} + for body in plan['bodies']: + batch = json.loads(body['messages'][1]['content']) + events = [event_map[e['id']] for e in batch['events']] + result = call_model(plan['config'], key, body) + cards.extend(validate_model_cards(result, events, plan['filters']['cross'])) + db.execute('UPDATE runs SET cards=?,status=? WHERE id=?', (json.dumps(cards, ensure_ascii=False), 'complete', rid)) + db.commit() + except Exception as exc: + reason = str(exc) if isinstance(exc, ValueError) else '分析失败；未保存不完整卡片。请检查接口后重新预览执行。' + db.execute('UPDATE runs SET status=?,error=? WHERE id=?', ('failed', reason[:500], rid)) + db.commit() + finally: + db.close() + + +def main(): + parser = argparse.ArgumentParser(description='微信知识需求分析工具（仅绑定本机）') + parser.add_argument('--port', type=int, default=8765) + parser.add_argument('--data-dir', type=Path, default=store.DEFAULT_DATA) + parser.add_argument('--no-browser', action='store_true') + args = parser.parse_args() + try: + server = Workbench(('127.0.0.1', args.port), args.data_dir) + except OSError: + parser.exit(1, '端口被占用。请关闭旧进程，或使用 --port 8766。\n') + url = f'http://127.0.0.1:{server.server_port}' + print(f'WeChat knowledge workbench: {url}', flush=True) + print(f'Local data: {server.data}', flush=True) + if not args.no_browser: + webbrowser.open(url) + try: + server.serve_forever() + except KeyboardInterrupt: + print('\nStopped.') + finally: + server.server_close() + + +if __name__ == '__main__': + main() diff --git a/wechat_tool/static/app.js b/wechat_tool/static/app.js new file mode 100644 index 0000000..e15779f --- /dev/null +++ b/wechat_tool/static/app.js @@ -0,0 +1,123 @@ +'use strict'; +const $ = s => document.querySelector(s); +const $$ = s => Array.from(document.querySelectorAll(s)); +const token = $('meta[name="workbench-token"]').content; +let overview = null, settings = {}, currentRun = null, currentPlan = null, fileData = null, currentCard = null, poll = null; +const signals = {repeat:'反复提问',risk:'风险 / 事故线索',rework:'重复踩坑线索',future:'后续复用线索'}; +const fields = {source_id:'消息 ID', conversation:'会话名称', conversation_id:'会话 ID', kind:'会话类型', sender:'发送人名称', sender_id:'发送人 ID', timestamp:'时间', content:'内容', message_type:'消息类型', quote:'引用'}; +const titles = {overview:'看见对话中的知识需求',import:'导入你的聊天记录',analyze:'发现值得沉淀的知识',settings:'模型接口与隐私设置'}; + +function el(tag, text, className) { const n = document.createElement(tag); if(text !== undefined) n.textContent = text; if(className) n.className=className; return n; } +function notify(text, error=false) { const n=$('#notice'); n.textContent=text; n.hidden=false; n.classList.toggle('error',error); } +async function api(path, body) { + const opts=body===undefined?{}:{method:'POST',headers:{'Content-Type':'application/json','X-Workbench-Token':token},body:JSON.stringify(body)}; + const response=await fetch(path,opts); const data=await response.json(); if(!response.ok)throw Error(data.error||'请求失败'); return data; +} +async function busy(button, work) {button.disabled=true;try{await work();}catch(e){notify(e.message,true);}finally{button.disabled=false;}} +function page(name) {if(!titles[name])name='overview';$$('.page').forEach(n=>n.hidden=n.id!=='page-'+name);$$('nav button').forEach(n=>n.classList.toggle('active',n.dataset.page===name));$('#page-title').textContent=titles[name];location.hash=name;} +function displayTime(time) { + if(!time)return '—'; const offset=settings.utc_offset||'+08:00'; const sign=offset[0]==='-'?-1:1; + const minutes=sign*(Number(offset.slice(1,3))*60+Number(offset.slice(4,6))); + const dt=new Date(new Date(time).getTime()+minutes*60000); + return dt.toISOString().slice(0,19).replace('T',' '); +} +function selected(select) {return Array.from(select.selectedOptions).map(n=>n.value);} +function filters() { + const start=$('#start').value,end=$('#end').value; + if(start&&end&&start>end)throw Error('开始时间不能晚于结束时间'); + return {start,end,kind:$('#filter-kind').value,conversations:selected($('#include-conversations')),exclude:selected($('#exclude-conversations')),search:$('#filter-search').value.trim(),utc_offset:settings.utc_offset||'+08:00'}; +} +function invalidatePlan() {currentPlan=null;$('#preview-panel').hidden=true;} + +async function refresh() { + overview=await api('/api/overview'); + const stats=$('#stats');stats.replaceChildren(); + const metrics=[['已导入消息',overview.count,'可重复导入，自动去重'],['会话',overview.conversations.length,'群聊与私聊'],['群聊',overview.conversations.filter(c=>c.kind==='group').length,'可跨群汇总相似问题'],['分析记录',overview.runs.length,'保留审核与修正']]; + for(const [name,num,note] of metrics){const node=el('div',undefined,'stat');node.append(el('small',name),el('strong',num.toLocaleString()),el('span',note));stats.append(node);} + $('#conversations').replaceChildren();$('#empty').hidden=overview.count>0; + for(const c of overview.conversations){const tr=el('tr');for(const text of [c.conversation,c.kind==='group'?'群聊':'私聊',c.count,c.media,displayTime(c.first),displayTime(c.last)])tr.append(el('td',text));$('#conversations').append(tr);} + for(const selector of ['#include-conversations','#exclude-conversations']){const select=$(selector),old=new Set(selected(select));select.replaceChildren();for(const c of overview.conversations){const opt=el('option',`${c.conversation} · ${c.kind==='group'?'群':'私聊'} · ${c.count} 条`);opt.value=c.conversation_id;opt.selected=old.has(opt.value);select.append(opt);}} + const history=$('#history');history.replaceChildren(); + for(const r of overview.runs){const b=el('button');b.append(el('span',`${displayTime(r.created)} · ${r.mode==='api'?'模型分析':'本地规则'}`),el('small',`${r.coverage.messages} 条记录 · ${r.coverage.conversations} 个会话 · ${{complete:'已完成',running:'进行中',failed:'失败'}[r.status]}`));b.addEventListener('click',()=>loadRun(r.id).catch(e=>notify(e.message,true)));history.append(b);} + if(!overview.runs.length)history.append(el('p','还没有分析记录。','muted')); +} + +async function loadSettings() { + settings=await api('/api/settings');$('#api-endpoint').value=settings.api.endpoint||'';$('#api-model').value=settings.api.model||'';$('#custom-words').value=(settings.custom||[]).join('\n');$('#utc-offset').value=settings.utc_offset||'+08:00';$('#key-state').textContent=settings.has_key?'当前运行进程中已配置密钥；留空保存会保留它。':'当前没有内存密钥；本地免密接口可留空。';$('#timezone-label').textContent='筛选与显示时区：UTC'+settings.utc_offset+'。不填时间代表全部已导入记录。'; +} + +async function loadFile(file) { + fileData=null;$('#import-button').disabled=true;$('#sqlite-options').hidden=true; + if(!file)return; + if(file.size>45*1024*1024)throw Error('单个文件不能超过 45 MB，请拆分后导入'); + const bytes=new Uint8Array(await file.arrayBuffer());let binary='';for(let i=0;iloadRun(id).catch(e=>notify(e.message,true)),2500); +} + +function renderCards() { + const container=$('#cards');container.replaceChildren();if(!currentRun)return; + const search=$('#card-search').value.toLowerCase(),signal=$('#card-signal').value; + const cards=currentRun.cards.filter(c=>(!search||(c.title+' '+c.summary).toLowerCase().includes(search))&&(!signal||c.signals.includes(signal))); + for(const c of cards){ + const n=el('article',undefined,'card'),top=el('div',undefined,'card-top'),label=el('label'),check=el('input');check.type='checkbox';check.value=c.id;check.className='card-select';check.setAttribute('aria-label','选择主题：'+c.title);label.append(check,el('h3',c.title));top.append(label,el('span','优先级 '+c.priority,'score'));n.append(top); + const tags=el('div',undefined,'tags');for(const s of c.signals)tags.append(el('span',signals[s],'tag '+s));tags.append(el('span',c.strategy,'tag'));n.append(tags,el('p',c.summary),el('div',`${c.event_count} 个独立讨论候选 · ${c.conversations.length} 个会话 · ${c.participants.length} 位参与人`,'card-meta')); + if(c.responders.length)n.append(el('div','经验提供者候选：'+c.responders.join('、'),'card-meta')); + const bottom=el('div',undefined,'card-bottom'),b=el('button','查看证据与审核 ↗');b.addEventListener('click',()=>openCard(c.id).catch(e=>notify(e.message,true)));bottom.append(el('small',(c.reviewed?'已人工审核':'待人工审核')+' · '+c.status),b);n.append(bottom);container.append(n); + } + if(!cards.length)container.append(el('div',currentRun.status==='complete'?'没有匹配的候选主题。当前规则可能未识别出问题，也可以扩大时间范围。':'任务尚未生成完整结果。','empty')); +} + +async function openCard(id) { + currentCard=currentRun.cards.find(c=>c.id===id);if(!currentCard)return; + const card=currentCard;$('#detail-title').textContent=card.title;const content=$('#detail-content');content.replaceChildren(); + const grid=el('div',undefined,'review-grid'); + for(const [field,name] of [['title','主题'],['summary','摘要'],['answer','经验候选（尚需验证）'],['boundaries','适用边界与证据缺口'],['notes','人工备注']]){const label=el('label',name),input=el(field==='title'?'input':'textarea');input.id='edit-'+field;input.value=card[field]||'';if(input.tagName==='TEXTAREA')input.rows=3;label.append(input);grid.append(label);} + const strategyLabel=el('label','处理策略'),strategy=el('select');strategy.id='edit-strategy';for(const v of ['正式资产','轻量记录','专家带教','暂不投入']){const op=el('option',v);op.value=v;op.selected=v===card.strategy;strategy.append(op);}strategyLabel.append(strategy);grid.append(strategyLabel); + const label=el('label',undefined,'check'),reviewed=el('input');reviewed.type='checkbox';reviewed.id='edit-reviewed';reviewed.checked=card.reviewed;label.append(reviewed,document.createTextNode('已人工审核（不代表经过实践验证）'));grid.append(label); + const save=el('button','保存修正','primary');save.addEventListener('click',()=>busy(save,async()=>{const body={run_id:currentRun.id,ids:[id],action:'edit',strategy:strategy.value,reviewed:reviewed.checked};for(const field of ['title','summary','answer','boundaries','notes'])body[field]=$('#edit-'+field).value;await api('/api/card',body);await loadRun(currentRun.id);$('#detail-dialog').close();notify('已保存人工修正。');}));grid.append(save);content.append(grid); + content.append(el('h3','原始讨论与证据','detail-evidence-title'),el('p','下方展示候选事件的全部已收集上下文。★ 为该卡引用的消息。事件窗口可能遗漏延迟回复或混入其他话题。经验提供者仅按他人的较长非问句回复识别，不代表专家认定。','muted')); + const loading=el('p','正在读取本地原文…','muted');content.append(loading);$('#detail-dialog').showModal(); + const events=await api('/api/context?run_id='+currentRun.id+'&card_id='+id);loading.remove(); + const splitButton=el('button','将勾选的讨论拆成新主题');splitButton.disabled=events.length<2;const splitChecks=[]; + for(const event of events){const box=el('div',undefined,'event'),l=el('label'),cb=el('input');cb.type='checkbox';cb.value=event.id;splitChecks.push(cb);l.append(cb,document.createTextNode(`${event.id} · ${event.conversation} · ${displayTime(event.messages[0].timestamp)}`));box.append(l); + for(const m of event.messages){const msg=el('div',undefined,'message');msg.append(el('div',`${card.evidence_ids.includes(m.id)?'★ ':''}#${m.id} · ${m.sender} · ${displayTime(m.timestamp)} · ${m.message_type}`,'msg-meta'),el('p',m.content||'［非文字消息，无文字内容］'));if(m.quote)msg.append(el('blockquote',m.quote));box.append(msg);}content.append(box); + } + splitButton.addEventListener('click',()=>busy(splitButton,async()=>{const ids=splitChecks.filter(c=>c.checked).map(c=>c.value);await api('/api/card',{run_id:currentRun.id,ids:[id],action:'split',event_ids:ids});await loadRun(currentRun.id);$('#detail-dialog').close();notify('已拆分主题，请重新审核。');}));content.append(splitButton); +} + +for(const [field,label] of Object.entries(fields)){const l=el('label',label),input=el('input');input.dataset.mapping=field;input.placeholder='自动匹配';l.append(input);$('#mapping-fields').append(l);} +$$('[data-page]').forEach(n=>n.addEventListener('click',()=>page(n.dataset.page))); +$$('[data-go]').forEach(n=>n.addEventListener('click',()=>page(n.dataset.go))); +window.addEventListener('hashchange',()=>page(location.hash.slice(1))); +$$('[data-action="refresh"]').forEach(n=>n.addEventListener('click',()=>busy(n,refresh))); +$$('[data-action="demo"]').forEach(n=>n.addEventListener('click',()=>busy(n,async()=>{const r=await api('/api/demo',{});await refresh();notify(`已导入虚构样例 ${r.added} 条，重复 ${r.duplicates} 条。可直接进入“发现知识需求”。`);}))); +$('#import-file').addEventListener('change',e=>loadFile(e.target.files[0]).catch(error=>{fileData=null;$('#import-button').disabled=true;notify(error.message,true);})); +$('#import-button').addEventListener('click',e=>busy(e.target,async()=>{if(!fileData)throw Error('请先选择文件');const mapping={};$$('[data-mapping]').forEach(n=>{if(n.value.trim())mapping[n.dataset.mapping]=n.value.trim();});const options={conversation:$('#default-conversation').value.trim(),kind:$('#default-kind').value,utc_offset:settings.utc_offset||'+08:00',mapping,table:$('#sqlite-table').value};const result=await api('/api/import',{...fileData,options});$('#import-result').hidden=false;$('#import-result').textContent=JSON.stringify(result,null,2);invalidatePlan();await refresh();notify(`导入完成：新增 ${result.added}，重复 ${result.duplicates}，拒绝 ${result.rejected}。${result.rejected?'请查看下方具体行号和原因。':''}`);})); +$('#save-settings').addEventListener('click',e=>busy(e.target,async()=>{await api('/api/settings',{api:{endpoint:$('#api-endpoint').value.trim(),model:$('#api-model').value.trim()},key:$('#api-key').value,clear_key:$('#clear-key').checked,utc_offset:$('#utc-offset').value.trim(),custom:$('#custom-words').value.split('\n').map(x=>x.trim()).filter(Boolean)});$('#api-key').value='';$('#clear-key').checked=false;invalidatePlan();await loadSettings();await refresh();notify('配置已保存，密钥未写入磁盘。');})); +$('#analyze-local').addEventListener('click',e=>busy(e.target,async()=>{const r=await api('/api/analyze',{filters:filters(),cross:$('#cross').checked});await loadRun(r.id);await refresh();notify('本地分析完成。请查看原文，核对主题与适用条件。');})); +$('#preview-api').addEventListener('click',e=>busy(e.target,async()=>{invalidatePlan();const p=await api('/api/preview',{filters:filters(),cross:$('#cross').checked});currentPlan=p.plan_id;$('#preview-summary').textContent=`目标：${p.endpoint} · ${p.batches} 批 · 约 ${p.characters.toLocaleString()} 字符 · ${p.coverage.messages} 条原始记录 / ${p.coverage.events} 个讨论候选`;$('#preview-text').value=JSON.stringify(p.requests,null,2);$('#preview-panel').hidden=false;$('#preview-panel').scrollIntoView({behavior:'smooth',block:'start'});})); +$('#send-api').addEventListener('click',e=>busy(e.target,async()=>{if(!currentPlan)throw Error('请重新生成预览');const id=currentPlan;invalidatePlan();const r=await api('/api/send',{plan_id:id});await loadRun(r.id);await refresh();notify('已开始发送预览中的脱敏请求。请保持本地服务运行。');})); +$('#close-preview').addEventListener('click',invalidatePlan); +['start','end','filter-kind','include-conversations','exclude-conversations','filter-search','cross'].forEach(id=>$('#'+id).addEventListener('change',invalidatePlan)); +$('#card-search').addEventListener('input',renderCards);$('#card-signal').addEventListener('change',renderCards); +$('#close-detail').addEventListener('click',()=>$('#detail-dialog').close()); +$('#merge-cards').addEventListener('click',()=>{const ids=$$('.card-select:checked').map(n=>n.value);if(ids.length<2)return notify('请至少勾选两张主题卡片。',true);$('#merge-title').value=currentRun.cards.find(c=>c.id===ids[0]).title;$('#merge-dialog').showModal();}); +$('#cancel-merge').addEventListener('click',()=>$('#merge-dialog').close()); +$('#confirm-merge').addEventListener('click',e=>busy(e.target,async()=>{await api('/api/card',{run_id:currentRun.id,ids:$$('.card-select:checked').map(n=>n.value),action:'merge',title:$('#merge-title').value.trim()});await loadRun(currentRun.id);$('#merge-dialog').close();notify('已合并主题，原摘要保留在备注中，请重新审核。');})); +async function init(){await loadSettings();await refresh();page(location.hash.slice(1)||'overview');} +init().catch(e=>notify(e.message,true)); diff --git a/wechat_tool/static/index.html b/wechat_tool/static/index.html new file mode 100644 index 0000000..b9d8946 --- /dev/null +++ b/wechat_tool/static/index.html @@ -0,0 +1,39 @@ + + +知微 · 微信知识需求分析 + + +
+
WECHAT KNOWLEDGE WORKBENCH

看见对话中的知识需求

LOCAL FIRST · 本地优先
+ +
+
从真实问题出发

哪些问题，
值得被认真整理？

发现重复提问、风险线索与可复用经验。
每一个判断，都能回到原始对话。

+
+

数据覆盖范围

仅展示已导入的记录。最早与最晚日期之间可能存在缺失。

会话类型消息数非文字最早记录最近记录
还没有导入记录。先导入文件，或体验一组虚构样例。
+

分析历史

保留筛选范围、分析方式与人工修正。

+
+ + + +
+
EVIDENCE & REVIEW

 +

合并选中主题

原有摘要会保留在备注，合并后的主题需要重新审核。

 + + diff --git a/wechat_tool/static/style.css b/wechat_tool/static/style.css new file mode 100644 index 0000000..2a4c5f3 --- /dev/null +++ b/wechat_tool/static/style.css @@ -0,0 +1 @@ +:root{--bg:#f6f7f3;--ink:#203d39;--muted:#73817b;--accent:#236b58;--line:#dfe6df;--white:#fff;--light:#edf3ed;--warn:#9a6b27}*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--ink);font-family:-apple-system,BlinkMacSystemFont,"Segoe UI","PingFang SC","Microsoft YaHei",sans-serif;font-size:14px;line-height:1.65}button,input,select,textarea{font:inherit}button,.button{border:1px solid var(--line);background:white;border-radius:8px;padding:9px 15px;cursor:pointer;color:var(--ink);text-decoration:none;display:inline-block;transition:background .15s}button:hover,.button:hover{background:var(--light)}button:disabled{opacity:.45;cursor:wait}button.primary{background:var(--accent);color:white;border-color:var(--accent)}button.primary:hover{background:#195640}button:focus-visible,a:focus-visible{outline:3px solid #7cb9a8;outline-offset:3px}[hidden]{display:none!important}.sidebar{width:225px;position:fixed;inset:0 auto 0 0;background:#eef1eb;border-right:1px solid var(--line);padding:35px 20px;display:flex;flex-direction:column}.brand{display:flex;gap:12px;align-items:center;color:var(--ink);text-decoration:none;font-size:25px;font-weight:650;margin:0 10px 48px}.mark{display:grid;place-items:center;width:43px;height:43px;border-radius:13px;background:var(--accent);color:white;font-size:22px}.brand small{display:block;font-size:11px;font-weight:400;color:var(--muted);letter-spacing:2px}.nav-label{font-size:11px;letter-spacing:2px;color:var(--muted);margin:0 15px 14px}nav button{display:flex;align-items:center;gap:13px;text-align:left;width:100%;margin:6px 0;border:none;background:none;padding:13px 15px;font-size:17px}nav button span{font-size:14px}nav button.active{background:#dfe9df;color:#205a47;font-weight:600}.sidebar-bottom{margin-top:auto;padding:30px 10px 0;font-size:12px;color:var(--accent)}.sidebar-bottom p,.sidebar-bottom small{color:var(--muted);font-size:11px}.dot{display:inline-block;width:7px;height:7px;border-radius:50%;background:#4a8d65;margin-right:5px}main{max-width:1550px;margin-left:225px;padding:35px 48px 60px}header{display:flex;align-items:center;justify-content:space-between;margin-bottom:35px;gap:15px}.eyebrow{font-size:10px;letter-spacing:2px;color:var(--accent);font-weight:650}h1{font-size:24px;font-weight:600;letter-spacing:.5px;margin:6px 0}h2{font-size:21px;font-weight:600;margin:0 0 8px}h3{font-size:16px;margin:22px 0 12px}p{margin:8px 0 16px}.pill{font-size:10px;letter-spacing:1px;border:1px solid var(--line);border-radius:30px;padding:7px 12px;color:var(--accent);white-space:nowrap}.hero{position:relative;display:grid;grid-template-columns:1fr 1fr;gap:25px;padding:36px 40px;background:#e8eee3;border:1px solid #d9e2d6;border-radius:15px;overflow:hidden}.hero h2{font-family:Georgia,"Songti SC",SimSun,serif;font-size:34px;line-height:1.5;font-weight:500;margin:16px 0}.hero p{color:#63776c;font-size:13px;line-height:1.9}.hero button{margin-top:12px}.hero-art{position:relative;min-height:270px;display:grid;place-items:center}.orbit{position:absolute;border:1px solid #bacbbb;border-radius:50%;transform:rotate(-20deg)}.outer{width:290px;height:235px}.inner{width:210px;height:290px;transform:rotate(35deg)}.art-center{border:1px solid #aac4af;background:#dce8d8;border-radius:50%;width:112px;height:112px;display:flex;flex-direction:column;justify-content:center;text-align:center;font-family:Georgia,"Songti SC",serif;font-size:40px}.art-center small{font-size:10px;font-family:inherit;letter-spacing:2px}.art-note{position:absolute;background:#f7f9f2;padding:9px 17px;border:1px solid #dce6d8;border-radius:7px;font-size:12px;box-shadow:0 5px 20px #304b3510}.art-note.a{left:0;top:28px}.art-note.b{right:0;top:130px}.art-note.c{bottom:10px;left:25px}.stats{display:grid;grid-template-columns:repeat(4,1fr);gap:16px;margin:24px 0}.stat{background:white;border:1px solid var(--line);border-radius:10px;padding:20px 23px}.stat small{color:var(--muted);font-size:12px}.stat strong{font-size:29px;font-weight:500;display:block;margin-top:4px}.stat span{font-size:11px;color:var(--muted)}.panel{background:white;border:1px solid var(--line);border-radius:12px;padding:24px;margin:22px 0}.panel-head{display:flex;align-items:center;justify-content:space-between;gap:15px;margin-bottom:20px}.panel-head h2{font-size:17px}.panel-head p{font-size:12px;color:var(--muted);margin-bottom:0}.table-scroll{overflow:auto}table{border-collapse:collapse;width:100%;text-align:left;white-space:nowrap;font-size:12px}th{font-size:11px;color:var(--muted);font-weight:500;background:#f7f9f5}th,td{padding:13px 12px;border-bottom:1px solid #edf0eb}td:first-child{font-weight:500;max-width:230px;overflow:hidden;text-overflow:ellipsis}.empty{padding:36px;text-align:center;color:var(--muted)}.history{display:grid;gap:10px}.history button{text-align:left;display:flex;align-items:center;justify-content:space-between;gap:15px}.muted,.intro p{color:var(--muted);font-size:12px}.intro{margin-bottom:24px}.intro h2{font-size:27px}.callout{border:1px solid #e2d6b8;background:#faf6e9;color:#7c6339;border-radius:8px;padding:15px 19px;font-size:12px;margin:16px 0;overflow-wrap:anywhere}.form-panel{padding:28px}.form-grid{display:grid;grid-template-columns:1fr 1fr;gap:20px;margin:16px 0}.form-grid.three{grid-template-columns:1fr 1fr 1fr}label{display:flex;flex-direction:column;gap:7px;font-size:12px;color:#50655c}input,select,textarea{width:100%;padding:10px 12px;background:#fff;border:1px solid #d9e1d8;border-radius:7px;color:var(--ink);outline:none;min-width:0}input:focus,select:focus,textarea:focus{border-color:#4b9a7d;box-shadow:0 0 0 3px #e4f0e9}textarea{resize:vertical;line-height:1.7}select[multiple]{height:140px}input[type=checkbox]{width:16px;height:16px;accent-color:var(--accent);flex-shrink:0}.check{flex-direction:row;align-items:center;gap:10px}.drop{text-align:center;align-items:center;background:#f6f8f3;border:1px dashed #a9beaa;border-radius:10px;padding:30px;cursor:pointer}.drop span{font-size:28px;color:var(--accent)}.drop strong{font-size:15px}.drop small{font-size:11px;color:var(--muted)}.drop input{max-width:340px;margin-top:8px}.actions{display:flex;gap:10px;align-items:center;flex-wrap:wrap;margin-top:20px}details{border-top:1px solid var(--line);padding:16px 0}summary{cursor:pointer;color:var(--accent);font-size:13px}pre{white-space:pre-wrap;overflow-wrap:anywhere;background:#f3f6f0;border-radius:7px;padding:18px;font-size:12px;max-height:420px;overflow:auto}code{font-size:12px;background:#edf1e9;padding:2px 5px;border-radius:3px}#notice{padding:13px 18px;background:#e5f1e6;border:1px solid #c6deca;border-radius:8px;margin-bottom:20px;font-size:13px;position:sticky;top:10px;z-index:3;overflow-wrap:anywhere}#notice.error{background:#fbede7;color:#984f33;border-color:#e7c8ba}.results-head{display:flex;justify-content:space-between;align-items:center;gap:15px;margin-top:42px}.results-head h2{margin-top:7px}.results-head p{color:var(--muted);font-size:12px}.result-tools{display:grid;grid-template-columns:1fr 180px auto;align-items:end;gap:14px;margin:20px 0}.cards{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:18px}.card{background:white;border:1px solid var(--line);border-radius:12px;padding:24px;display:flex;flex-direction:column;overflow:hidden}.card-top{display:flex;gap:10px;justify-content:space-between}.card-top label{flex-direction:row;align-items:flex-start}.card h3{font-size:17px;margin:0 0 12px;line-height:1.6;overflow-wrap:anywhere}.card p{font-size:12px;color:#66796d;overflow-wrap:anywhere}.tags{display:flex;gap:6px;flex-wrap:wrap;margin:10px 0}.tag{background:#edf3ed;color:#3d7359;padding:3px 8px;border-radius:4px;font-size:10px}.tag.risk{background:#fbefe3;color:#a26f32}.card-bottom{display:flex;justify-content:space-between;align-items:center;gap:10px;margin-top:auto;padding-top:18px;border-top:1px solid #edf1e9}.card-bottom small{font-size:11px;color:var(--muted)}.card .score{font-size:11px;white-space:nowrap;color:#8c9b89}.card-meta{font-size:11px;color:var(--muted);margin:12px 0}dialog{border:1px solid var(--line);border-radius:14px;width:min(950px,94vw);max-height:90vh;overflow:auto;color:var(--ink);padding:28px;background:var(--bg)}dialog::backdrop{background:#173d3255;backdrop-filter:blur(3px)}.dialog-head{display:flex;justify-content:space-between;gap:15px;align-items:flex-start;margin-bottom:22px}.dialog-head h2{margin-top:8px;font-size:22px}.event{border:1px solid var(--line);border-radius:9px;padding:18px;margin:16px 0;background:white}.event h3{font-size:13px;margin:0 0 12px}.message{border-left:2px solid #d3e2d1;padding:3px 0 3px 14px;margin:14px 0}.message .msg-meta{font-size:11px;color:var(--muted)}.message p{white-space:pre-wrap;margin:5px 0;font-size:13px;overflow-wrap:anywhere}.message blockquote{font-size:12px;background:#f1f4ed;margin:5px 0;padding:8px 12px;white-space:pre-wrap;overflow-wrap:anywhere}.review-grid{display:grid;gap:14px}.detail-evidence-title{margin-top:34px}.event label{flex-direction:row;align-items:center}footer{margin-left:225px;padding:0 48px 30px;font-size:11px;color:#929b91}#merge-dialog{max-width:500px}#preview-text{font-family:ui-monospace,monospace;font-size:12px}#file-info{margin:12px 0}@media(min-width:1650px){main{margin-right:auto}}@media(max-width:1100px){main{padding:28px;margin-left:195px}.sidebar{width:195px;padding:28px 12px}.hero{padding:28px;grid-template-columns:1.1fr 1fr}.hero h2{font-size:29px}.outer{width:230px}.inner{width:180px}.art-note{padding:6px 10px;font-size:10px}.stats{gap:10px}.stat{padding:15px}footer{margin-left:195px}.form-grid.three{grid-template-columns:1fr 1fr}.cards{grid-template-columns:1fr}.pill{display:none}}@media(max-width:760px){.sidebar{position:static;width:auto;padding:15px;border-right:none;border-bottom:1px solid var(--line)}.brand{margin:0 0 10px;font-size:20px}.brand small,.nav-label,.sidebar-bottom{display:none}nav{display:flex;gap:5px;overflow:auto}nav button{padding:9px;font-size:14px;gap:5px;white-space:nowrap;width:auto;margin:0}nav button span{font-size:12px}main{margin:0;padding:24px 16px}header{margin-bottom:22px}h1{font-size:21px}.hero{grid-template-columns:1fr}.hero-art{display:none}.stats{grid-template-columns:1fr 1fr}.form-grid,.form-grid.three{grid-template-columns:1fr}.panel,.form-panel{padding:18px}.result-tools{grid-template-columns:1fr}.results-head{display:block}.history button{align-items:flex-start;flex-direction:column}footer{margin:0;padding:0 16px 25px}.panel-head{align-items:flex-start}dialog{padding:18px}.card{padding:20px}} diff --git a/wechat_tool/store.py b/wechat_tool/store.py new file mode 100644 index 0000000..d68140b --- /dev/null +++ b/wechat_tool/store.py @@ -0,0 +1,93 @@ +import json +import os +import sqlite3 +from pathlib import Path + +DEFAULT_DATA = Path(__file__).resolve().parent.parent / 'data' + + +def connect(root=None): + root = Path(root or os.environ.get('WECHAT_ANALYSIS_DATA', DEFAULT_DATA)) + root.mkdir(parents=True, exist_ok=True) + try: + root.chmod(0o700) + except OSError: + pass + db = sqlite3.connect(root / 'workbench.sqlite3', timeout=30) + db.row_factory = sqlite3.Row + db.execute('PRAGMA foreign_keys=ON') + db.execute('PRAGMA journal_mode=WAL') + db.executescript(''' + CREATE TABLE IF NOT EXISTS messages ( + id INTEGER PRIMARY KEY, fingerprint TEXT UNIQUE NOT NULL, + source_id TEXT NOT NULL, conversation TEXT NOT NULL, + conversation_id TEXT NOT NULL, kind TEXT NOT NULL, + sender TEXT NOT NULL, sender_id TEXT NOT NULL, + timestamp TEXT NOT NULL, content TEXT NOT NULL, + message_type TEXT NOT NULL, quote TEXT NOT NULL, source TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS msg_time ON messages(timestamp); + CREATE INDEX IF NOT EXISTS msg_conv ON messages(conversation_id, timestamp); + CREATE TABLE IF NOT EXISTS runs ( + id TEXT PRIMARY KEY, created TEXT NOT NULL, mode TEXT NOT NULL, + filters TEXT NOT NULL, cards TEXT NOT NULL, coverage TEXT NOT NULL, + status TEXT NOT NULL, error TEXT NOT NULL DEFAULT '' + ); + CREATE TABLE IF NOT EXISTS settings (key TEXT PRIMARY KEY, value TEXT NOT NULL); + ''') + if 'events' not in {row[1] for row in db.execute('PRAGMA table_info(runs)')}: + db.execute("ALTER TABLE runs ADD COLUMN events TEXT NOT NULL DEFAULT '[]'") + db.commit() + return db + + +def setting(db, key, default=None): + row = db.execute('SELECT value FROM settings WHERE key=?', (key,)).fetchone() + return json.loads(row[0]) if row else default + + +def set_setting(db, key, value): + db.execute('INSERT OR REPLACE INTO settings VALUES (?,?)', (key, json.dumps(value, ensure_ascii=False))) + db.commit() + + +def where(filters): + clauses, args = ['1=1'], [] + # Stored wall-clock times are normalized to the configured source UTC offset. + from .importer import parse_time + for key, op in [('start', '>='), ('end', '<=')]: + val = filters.get(key) + if val: + if len(val) == 10: + val += ' 00:00:00' if key == 'start' else ' 23:59:59' + clauses.append(f'timestamp {op} ?') + args.append(parse_time(val, filters.get('utc_offset', '+08:00'))) + if filters.get('kind') in ('group', 'private'): + clauses.append('kind=?') + args.append(filters['kind']) + for key, negate in [('conversations', False), ('exclude', True)]: + values = filters.get(key) or [] + if not isinstance(values, list) or len(values) > 500: + raise ValueError('会话筛选必须是数组，最多 500 项') + if values: + clauses.append('conversation_id ' + ('NOT IN' if negate else 'IN') + '(' + ','.join('?' for _ in values) + ')') + args.extend(values) + if filters.get('search'): + clauses.append('content LIKE ? ESCAPE \'\\\'') + term = filters['search'].replace('\\', '\\\\').replace('%', '\\%').replace('_', '\\_') + args.append('%' + term + '%') + return ' AND '.join(clauses), args + + +def messages(db, filters): + clause, args = where(filters) + return [dict(row) for row in db.execute('SELECT * FROM messages WHERE ' + clause + ' ORDER BY timestamp,id', args)] + + +def overview(db): + rows = db.execute('''SELECT conversation_id, MAX(conversation) AS conversation, kind, + COUNT(*) AS count, MIN(timestamp) AS first, MAX(timestamp) AS last, + SUM(CASE WHEN message_type!='text' THEN 1 ELSE 0 END) AS media + FROM messages GROUP BY conversation_id,kind ORDER BY count DESC''').fetchall() + total = db.execute('SELECT COUNT(*),MIN(timestamp),MAX(timestamp) FROM messages').fetchone() + return {'count': total[0], 'first': total[1], 'last': total[2], 'conversations': [dict(r) for r in rows]}