feat(ai-labeling): AI 打标实验工作区初始化 + 模型选型完成
This commit is contained in:
simonkoson
@@ -0,0 +1,119 @@
|
||||
"""
|
||||
import_transcripts.py - 将源目录的 10 期文稿(docx/伪docx)导入到 benchmark-set/transcripts/
|
||||
用法: python scripts/import_transcripts.py
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from docx import Document
|
||||
|
||||
# ===== 配置 =====
|
||||
SRC_DIR = Path(r"E:\TPS-我和顾问的便签\刘瑞桦收集")
|
||||
DST_DIR = Path(__file__).parent.parent / "benchmark-set" / "transcripts"
|
||||
DST_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 映射表: 源文件名 → 目标文件名
|
||||
# （注意：源文件名中的标点/空格与预期不同，已据实更新）
|
||||
FILE_MAP = [
|
||||
("硅基大脑终稿.docx", "ep03_硅基大脑.md"),
|
||||
("潜艇的仿生之路.docx", "ep04_潜艇仿生.md"),
|
||||
("马年军事图鉴0125.docx", "ep07_马年图鉴.md"),
|
||||
("\u300a枪械射速决定论\u300b0225.docx", "ep10_射速决定论.md"),
|
||||
("完稿 武器进化论\uff1a空战颠覆者.docx", "ep11_空战颠覆者.md"),
|
||||
("逆袭战局的组装武器.docx", "ep12_逆袭战局.md"),
|
||||
("枪械设计中的长短智慧总稿.docx", "ep13_长短智慧.md"),
|
||||
("\u201cX\u201d系列新成员\uff1aX-76飞机-最终稿.docx", "ep14_X76飞机.md"),
|
||||
("空中坚盾\u2014\u2014解码现代防空网.docx", "ep15_空中坚盾.md"),
|
||||
# .doc 文件（改 .docx 后重试）
|
||||
("舰证不凡_docx版.docx", "ep05_舰证不凡.md"),
|
||||
]
|
||||
# ===== 配置 =====
|
||||
|
||||
|
||||
def is_real_docx(path: Path) -> bool:
|
||||
"""通过前 4 字节判断是否为真 docx（ZIP 格式）"""
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
header = f.read(4)
|
||||
return header == b"PK\x03\x04"
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def extract_text_real_docx(path: Path) -> str:
|
||||
"""用 python-docx 提取真 docx 段落文本"""
|
||||
doc = Document(str(path))
|
||||
paragraphs = []
|
||||
for para in doc.paragraphs:
|
||||
text = para.text.strip()
|
||||
if text:
|
||||
paragraphs.append(text)
|
||||
return "\n\n".join(paragraphs)
|
||||
|
||||
|
||||
def extract_text_fake_docx(path: Path) -> str:
|
||||
"""伪 docx（实际是 UTF-8 纯文本），直接读取"""
|
||||
return path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def clean_text(text: str) -> str:
|
||||
"""清洗文本：去掉 **Markdown 强调**，合并连续空行"""
|
||||
# 去掉 ** ... ** 强调符号，保留中间文字
|
||||
text = re.sub(r"\*\*(.+?)\*\*", r"\1", text)
|
||||
# 去掉 * ... * 斜体（如果还有残留）
|
||||
text = re.sub(r"\*(.+?)\*", r"\1", text)
|
||||
# 连续 3 个以上空行合并为 2 个
|
||||
text = re.sub(r"\n{3,}", "\n\n", text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
def process_file(src_name: str, dst_name: str) -> tuple[bool, str]:
|
||||
"""处理单个文件，返回 (是否成功, 消息)"""
|
||||
src_path = SRC_DIR / src_name
|
||||
if not src_path.exists():
|
||||
return False, f"源文件不存在: {src_name}"
|
||||
|
||||
try:
|
||||
if is_real_docx(src_path):
|
||||
raw_text = extract_text_real_docx(src_path)
|
||||
else:
|
||||
raw_text = extract_text_fake_docx(src_path)
|
||||
|
||||
clean_text_str = clean_text(raw_text)
|
||||
dst_path = DST_DIR / dst_name
|
||||
dst_path.write_text(clean_text_str, encoding="utf-8")
|
||||
char_count = len(clean_text_str)
|
||||
return True, f"OK ({char_count} 字符)"
|
||||
except Exception as e:
|
||||
return False, f"解析失败: {e}"
|
||||
|
||||
|
||||
def main():
|
||||
print(f"源目录: {SRC_DIR}")
|
||||
print(f"目标目录: {DST_DIR}")
|
||||
print(f"待处理: {len(FILE_MAP)} 期\n")
|
||||
|
||||
results = []
|
||||
for src_name, dst_name in FILE_MAP:
|
||||
ok, msg = process_file(src_name, dst_name)
|
||||
results.append((src_name, dst_name, ok, msg))
|
||||
status = "[OK]" if ok else "[FAIL]"
|
||||
print(f" {status} {src_name} -> {dst_name} {msg}")
|
||||
|
||||
# 汇总
|
||||
success = sum(1 for r in results if r[2])
|
||||
fail = len(results) - success
|
||||
print(f"\n===== 汇总 =====")
|
||||
print(f"成功: {success}/{len(results)}")
|
||||
if fail > 0:
|
||||
print("失败文件:")
|
||||
for src, dst, ok, msg in results:
|
||||
if not ok:
|
||||
print(f" - {src}")
|
||||
else:
|
||||
print("全部成功!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,3 @@
|
||||
openai==1.55.0
|
||||
python-dotenv>=1.0.0
|
||||
python-docx>=1.1.0
|
||||
@@ -0,0 +1,148 @@
|
||||
"""
|
||||
run_labeling.py - 单期或批量 AI 打标脚本
|
||||
用法:
|
||||
单期: python run_labeling.py --ep 4 --model m3
|
||||
批量: python run_labeling.py --all --model m3
|
||||
"""
|
||||
|
||||
import sys
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
sys.stderr.reconfigure(encoding='utf-8')
|
||||
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from openai import OpenAI
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
BASE_DIR = Path(__file__).parent.parent
|
||||
TRANSCRIPTS_DIR = BASE_DIR / "benchmark-set" / "transcripts"
|
||||
PROMPTS_DIR = BASE_DIR / "prompts"
|
||||
EXPERIMENTS_DIR = BASE_DIR / "experiments"
|
||||
GROUND_TRUTH = BASE_DIR / "benchmark-set" / "ground-truth.json"
|
||||
|
||||
MODEL_CONFIG = {
|
||||
"m3": {
|
||||
"base_url": "https://api.minimaxi.com/v1",
|
||||
"model_name": "MiniMax-M3",
|
||||
"api_key_env": "MINIMAX_API_KEY",
|
||||
},
|
||||
"deepseek-v4-pro": {
|
||||
"base_url": "https://api.deepseek.com",
|
||||
"model_name": "deepseek-v4-pro",
|
||||
"api_key_env": "DEEPSEEK_API_KEY",
|
||||
},
|
||||
"mimo-v2.5-pro": {
|
||||
"base_url": "https://api.xiaomimimo.com/v1",
|
||||
"model_name": "mimo-v2.5-pro",
|
||||
"api_key_env": "MIMO_API_KEY",
|
||||
},
|
||||
}
|
||||
|
||||
ALL_EPISODES = [3, 4, 5, 7, 10, 11, 12, 13, 14, 15]
|
||||
|
||||
|
||||
def load_prompt(field):
|
||||
if field == "narrative":
|
||||
return (PROMPTS_DIR / "prompt2_narrative.md").read_text(encoding="utf-8")
|
||||
raise ValueError(f"Unknown field: {field}")
|
||||
|
||||
|
||||
def load_transcript(ep):
|
||||
pattern = f"ep{ep:02d}_*.md"
|
||||
files = list(TRANSCRIPTS_DIR.glob(pattern))
|
||||
if not files:
|
||||
raise FileNotFoundError(f"No transcript found for ep{ep:02d} in {TRANSCRIPTS_DIR}")
|
||||
return files[0].read_text(encoding="utf-8"), files[0].name
|
||||
|
||||
|
||||
def load_ground_truth(ep):
|
||||
data = json.loads(GROUND_TRUTH.read_text(encoding="utf-8"))
|
||||
for episode in data["episodes"]:
|
||||
if episode["ep"] == ep:
|
||||
return episode
|
||||
return None
|
||||
|
||||
|
||||
def parse_prompt(template, transcript):
|
||||
parts = template.split("## USER")
|
||||
system_prompt = parts[0].replace("# Prompt 2:叙事结构判别", "").strip()
|
||||
user_prompt = parts[1].strip().replace("{transcript}", transcript)
|
||||
return system_prompt, user_prompt
|
||||
|
||||
|
||||
def extract_json_from_response(raw: str) -> dict:
|
||||
"""从模型响应中提取 JSON，兼容推理模型的<think>...输出。"""
|
||||
# 先去掉<think>...标签及其内容
|
||||
text = re.sub(r'<think>.*?', '', raw, flags=re.DOTALL)
|
||||
text = text.strip()
|
||||
# 去掉markdown代码块
|
||||
text = re.sub(r'^```(?:json)?\s*', '', text)
|
||||
text = re.sub(r'\s*```$', '', text)
|
||||
text = text.strip()
|
||||
# 从第一个 { 开始，到最后一个 } 结束
|
||||
first_brace = text.find('{')
|
||||
last_brace = text.rfind('}')
|
||||
if first_brace != -1 and last_brace != -1 and last_brace >= first_brace:
|
||||
json_str = text[first_brace:last_brace + 1]
|
||||
return json.loads(json_str)
|
||||
# 兜底：直接尝试解析
|
||||
return json.loads(text)
|
||||
|
||||
|
||||
def call_model(model_key, system_prompt, user_prompt):
|
||||
config = MODEL_CONFIG[model_key]
|
||||
client = OpenAI(
|
||||
api_key=os.environ[config["api_key_env"]],
|
||||
base_url=config["base_url"],
|
||||
)
|
||||
response = client.chat.completions.create(
|
||||
model=config["model_name"],
|
||||
messages=[
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_prompt},
|
||||
],
|
||||
temperature=0.0,
|
||||
)
|
||||
raw = response.choices[0].message.content
|
||||
return extract_json_from_response(raw)
|
||||
|
||||
|
||||
def run_labeling(ep, model_key):
|
||||
transcript, fname = load_transcript(ep)
|
||||
template = load_prompt("narrative")
|
||||
system_prompt, user_prompt = parse_prompt(template, transcript)
|
||||
result = call_model(model_key, system_prompt, user_prompt)
|
||||
gt = load_ground_truth(ep)
|
||||
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
out = EXPERIMENTS_DIR / f"{ts}_{model_key}_ep{ep:02d}.json"
|
||||
out.write_text(
|
||||
json.dumps({"episode": ep, "filename": fname, "result": result, "ground_truth": gt}, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(f"完成 ep{ep:02d} -> {out.name}")
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="AI 打标脚本")
|
||||
parser.add_argument("--ep", type=int, help="单期编号")
|
||||
parser.add_argument("--all", action="store_true", help="跑全部")
|
||||
parser.add_argument("--model", default="m3", help="模型键名")
|
||||
args = parser.parse_args()
|
||||
if args.all:
|
||||
for ep in ALL_EPISODES:
|
||||
run_labeling(ep, args.model)
|
||||
elif args.ep:
|
||||
run_labeling(args.ep, args.model)
|
||||
else:
|
||||
parser.print_help()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,96 @@
|
||||
"""
|
||||
summarize.py - 汇总打标结果命中情况
|
||||
用法: python summarize.py --model m3
|
||||
"""
|
||||
|
||||
import sys
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
sys.stderr.reconfigure(encoding='utf-8')
|
||||
|
||||
import json
|
||||
import glob
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
BASE_DIR = Path(__file__).parent.parent
|
||||
EXPERIMENTS_DIR = BASE_DIR / "experiments"
|
||||
|
||||
|
||||
def load_json(path):
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def latest_per_ep(files):
|
||||
"""同 ep 有多个文件时取时间戳最新的。"""
|
||||
latest = {}
|
||||
for f in files:
|
||||
name = f.name
|
||||
# 文件名格式: 20260611_154037_m3_ep04.json
|
||||
parts = name.replace(".json", "").split("_")
|
||||
if len(parts) >= 4:
|
||||
ts = parts[0] + parts[1] # yyyymmddHHMMSS
|
||||
ep = int(parts[-1].replace("ep", ""))
|
||||
key = ep
|
||||
if key not in latest or ts > latest[key][1]:
|
||||
latest[key] = (f, ts)
|
||||
return {ep: info[0] for ep, info in latest.items()}
|
||||
|
||||
|
||||
def run(model):
|
||||
pattern = str(EXPERIMENTS_DIR / f"*_{model}_*.json")
|
||||
files = sorted(Path(p) for p in glob.glob(pattern))
|
||||
if not files:
|
||||
print(f"未找到 {pattern}")
|
||||
return
|
||||
|
||||
ep_files = latest_per_ep(files)
|
||||
rows = []
|
||||
parse_fail = 0
|
||||
|
||||
for ep in sorted(ep_files.keys()):
|
||||
data = load_json(ep_files[ep])
|
||||
if data is None:
|
||||
parse_fail += 1
|
||||
rows.append({"ep": ep, "title": "?", "gt": "?", "pred": "解析失败", "hit": False, "conf": "?"})
|
||||
continue
|
||||
|
||||
gt = data.get("ground_truth", {})
|
||||
result = data.get("result")
|
||||
|
||||
title = gt.get("title", "?")
|
||||
gt_val = gt.get("narrative_structure", "?")
|
||||
pred_val = result.get("narrative_structure") if result else None
|
||||
conf = result.get("confidence", "?") if result else "?"
|
||||
hit = pred_val == gt_val if pred_val is not None else False
|
||||
|
||||
rows.append({"ep": ep, "title": title, "gt": gt_val, "pred": pred_val or "解析失败", "hit": hit, "conf": conf})
|
||||
|
||||
# 打印每行
|
||||
for r in rows:
|
||||
mark = "✓" if r["hit"] else "✗"
|
||||
conf_str = f'置信度:{r["conf"]}' if r["conf"] != "?" else ""
|
||||
print(f' ep{r["ep"]:02d} {r["title"]:<10} | 标准:{r["gt"]:<8} | {model}:{r["pred"]:<8} | {mark} {conf_str}')
|
||||
|
||||
# 汇总
|
||||
total = len(rows)
|
||||
hits = sum(1 for r in rows if r["hit"])
|
||||
hi_conf = [r for r in rows if r["conf"] == "高"]
|
||||
mid_low = [r for r in rows if r["conf"] in ("中", "低")]
|
||||
hi_hit = sum(1 for r in hi_conf if r["hit"])
|
||||
ml_hit = sum(1 for r in mid_low if r["hit"])
|
||||
|
||||
print(f"\n ===== {model} 命中情况 =====")
|
||||
print(f" narrative_structure 命中: {hits}/{total} = {hits*100//total}%")
|
||||
print(f" 自评\"高\"置信的命中率: {hi_hit}/{len(hi_conf)}")
|
||||
print(f" 自评\"中/低\"置信的命中率: {ml_hit}/{len(mid_low)}")
|
||||
print(f" 解析失败: {parse_fail} 期")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--model", required=True, help="模型键名，如 m3 / deepseek-v4")
|
||||
args = parser.parse_args()
|
||||
run(args.model)
|
||||
Reference in New Issue
Block a user