feat(ai-labeling): AI 打标实验工作区初始化 + 模型选型完成

2026-06-11 18:34:55 +08:00
parent 9a86c936ee
commit 202e02c254
22 changed files with 2762 additions and 0 deletions
@@ -0,0 +1,96 @@
+"""
+summarize.py - 汇总打标结果命中情况
+用法: python summarize.py --model m3
+"""
+
+import sys
+sys.stdout.reconfigure(encoding='utf-8')
+sys.stderr.reconfigure(encoding='utf-8')
+
+import json
+import glob
+import argparse
+from pathlib import Path
+
+BASE_DIR = Path(__file__).parent.parent
+EXPERIMENTS_DIR = BASE_DIR / "experiments"
+
+
+def load_json(path):
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except Exception:
+        return None
+
+
+def latest_per_ep(files):
+    """同 ep 有多个文件时取时间戳最新的。"""
+    latest = {}
+    for f in files:
+        name = f.name
+        # 文件名格式: 20260611_154037_m3_ep04.json
+        parts = name.replace(".json", "").split("_")
+        if len(parts) >= 4:
+            ts = parts[0] + parts[1]  # yyyymmddHHMMSS
+            ep = int(parts[-1].replace("ep", ""))
+            key = ep
+            if key not in latest or ts > latest[key][1]:
+                latest[key] = (f, ts)
+    return {ep: info[0] for ep, info in latest.items()}
+
+
+def run(model):
+    pattern = str(EXPERIMENTS_DIR / f"*_{model}_*.json")
+    files = sorted(Path(p) for p in glob.glob(pattern))
+    if not files:
+        print(f"未找到 {pattern}")
+        return
+
+    ep_files = latest_per_ep(files)
+    rows = []
+    parse_fail = 0
+
+    for ep in sorted(ep_files.keys()):
+        data = load_json(ep_files[ep])
+        if data is None:
+            parse_fail += 1
+            rows.append({"ep": ep, "title": "?", "gt": "?", "pred": "解析失败", "hit": False, "conf": "?"})
+            continue
+
+        gt = data.get("ground_truth", {})
+        result = data.get("result")
+
+        title = gt.get("title", "?")
+        gt_val = gt.get("narrative_structure", "?")
+        pred_val = result.get("narrative_structure") if result else None
+        conf = result.get("confidence", "?") if result else "?"
+        hit = pred_val == gt_val if pred_val is not None else False
+
+        rows.append({"ep": ep, "title": title, "gt": gt_val, "pred": pred_val or "解析失败", "hit": hit, "conf": conf})
+
+    # 打印每行
+    for r in rows:
+        mark = "✓" if r["hit"] else "✗"
+        conf_str = f'置信度:{r["conf"]}' if r["conf"] != "?" else ""
+        print(f'  ep{r["ep"]:02d} {r["title"]:<10} | 标准:{r["gt"]:<8} | {model}:{r["pred"]:<8} | {mark} {conf_str}')
+
+    # 汇总
+    total = len(rows)
+    hits = sum(1 for r in rows if r["hit"])
+    hi_conf = [r for r in rows if r["conf"] == "高"]
+    mid_low = [r for r in rows if r["conf"] in ("中", "低")]
+    hi_hit = sum(1 for r in hi_conf if r["hit"])
+    ml_hit = sum(1 for r in mid_low if r["hit"])
+
+    print(f"\n  ===== {model} 命中情况 =====")
+    print(f"  narrative_structure 命中: {hits}/{total} = {hits*100//total}%")
+    print(f"  自评\"高\"置信的命中率: {hi_hit}/{len(hi_conf)}")
+    print(f"  自评\"中/低\"置信的命中率: {ml_hit}/{len(mid_low)}")
+    print(f"  解析失败: {parse_fail} 期")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model", required=True, help="模型键名，如 m3 / deepseek-v4")
+    args = parser.parse_args()
+    run(args.model)