simonkoson/tps-dashboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ai-labeling: ground-truth v0.3.0 + 脚本扩展 --field 支持 Prompt 1

Browse Source 
- ground-truth.json 新增 4 分类字段骨架（program_format/equipment_domain/scene_tags/tech_tags）
- equipment_domain 类型修正为数组（对齐 v2 快照多选定义）
- run_labeling.py 新增 --field 参数，文件名含 field 标记
- summarize.py 新增 --field 参数 + 老文件向后兼容
- CLAUDE.md 补入 v1-v4 快照精华：象限图规格、6 字段枚举、三层交付策略

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
simonkoson
2026-06-24 17:53:05 +08:00
parent bdc59bf787
commit 30931d4446
4 changed files with 174 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ai-labeling/scripts/run_labeling.py
+29 -11
View File
@@ -41,11 +41,16 @@ MODEL_CONFIG = {
ALL_EPISODES = [3, 4, 5, 7, 10, 11, 12, 13, 14, 15]
FIELD_PROMPT_MAP = {
"narrative": "prompt2_narrative.md",
"classification": "prompt1_classification.md",
}
def load_prompt(field):
if field == "narrative":
return (PROMPTS_DIR / "prompt2_narrative.md").read_text(encoding="utf-8")
raise ValueError(f"Unknown field: {field}")
if field not in FIELD_PROMPT_MAP:
raise ValueError(f"Unknown field: {field}, valid: {list(FIELD_PROMPT_MAP.keys())}")
return (PROMPTS_DIR / FIELD_PROMPT_MAP[field]).read_text(encoding="utf-8")
def load_transcript(ep):
@@ -65,8 +70,19 @@ def load_ground_truth(ep):
def parse_prompt(template, transcript):
"""按 ## SYSTEM / ## USER 分隔符拆解 prompt。
自动剥离 ## SYSTEM 标签之前的标题行。
"""
parts = template.split("## USER")
system_prompt = parts[0].replace("# Prompt 2:叙事结构判别", "").strip()
# parts[0] 是 system 部分，可能包含标题行 + ## SYSTEM 标签
system_raw = parts[0]
# 如果有 ## SYSTEM 标签，取它之后的内容；否则去除标题行
if "## SYSTEM" in system_raw:
system_prompt = system_raw.split("## SYSTEM", 1)[1].strip()
else:
# 没有 ## SYSTEM 标签时，去掉第一行（标题行）作为降级处理
lines = system_raw.strip().splitlines()
system_prompt = "\n".join(lines[1:]).strip() if len(lines) > 1 else system_raw.strip()
user_prompt = parts[1].strip().replace("{transcript}", transcript)
return system_prompt, user_prompt
@@ -108,19 +124,19 @@ def call_model(model_key, system_prompt, user_prompt):
return extract_json_from_response(raw)
def run_labeling(ep, model_key):
def run_labeling(ep, model_key, field="narrative"):
transcript, fname = load_transcript(ep)
template = load_prompt("narrative")
template = load_prompt(field)
system_prompt, user_prompt = parse_prompt(template, transcript)
result = call_model(model_key, system_prompt, user_prompt)
gt = load_ground_truth(ep)
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
out = EXPERIMENTS_DIR / f"{ts}_{model_key}_ep{ep:02d}.json"
out = EXPERIMENTS_DIR / f"{ts}_{model_key}_{field}_ep{ep:02d}.json"
out.write_text(
json.dumps({"episode": ep, "filename": fname, "result": result, "ground_truth": gt}, ensure_ascii=False, indent=2),
json.dumps({"episode": ep, "filename": fname, "field": field, "result": result, "ground_truth": gt}, ensure_ascii=False, indent=2),
encoding="utf-8",
)
print(f"完成 ep{ep:02d} -> {out.name}")
print(f"完成 ep{ep:02d} [{field}] -> {out.name}")
return result
@@ -129,12 +145,14 @@ def main():
parser.add_argument("--ep", type=int, help="单期编号")
parser.add_argument("--all", action="store_true", help="跑全部")
parser.add_argument("--model", default="mimo-v2.5-pro", help="模型键名")
parser.add_argument("--field", default="narrative", choices=["narrative", "classification"],
help="打标字段: narrative(叙事结构) / classification(4分类)")
args = parser.parse_args()
if args.all:
for ep in ALL_EPISODES:
run_labeling(ep, args.model)
run_labeling(ep, args.model, args.field)
elif args.ep:
run_labeling(args.ep, args.model)
run_labeling(args.ep, args.model, args.field)
else:
parser.print_help()
ai-labeling/scripts/summarize.py
+48 -27
View File
@@ -39,11 +39,21 @@ def latest_per_ep(files):
return {ep: info[0] for ep, info in latest.items()}
def run(model):
pattern = str(EXPERIMENTS_DIR / f"*_{model}_*.json")
files = sorted(Path(p) for p in glob.glob(pattern))
def run(model, field="narrative"):
# 文件匹配：新格式有 field 标记，老格式（无 field）向后兼容
pattern_new = str(EXPERIMENTS_DIR / f"*_{model}_{field}_*.json")
files_new = sorted(Path(p) for p in glob.glob(pattern_new))
# 如果是 narrative 且新格式文件为空，尝试匹配老格式（无 field 标记）
if field == "narrative" and not files_new:
pattern_old = str(EXPERIMENTS_DIR / f"*_{model}_ep*.json")
files_old = [f for f in sorted(Path(p) for p in glob.glob(pattern_old))
if len(f.name.replace(".json", "").split("_")) == 3] # 老格式: ts_model_epNN.json
files = files_old
else:
files = files_new
if not files:
print(f"未找到 {pattern}")
print(f"未找到 {pattern_new}")
return
ep_files = latest_per_ep(files)
@@ -61,36 +71,47 @@ def run(model):
result = data.get("result")
title = gt.get("title", "?")
gt_val = gt.get("narrative_structure", "?")
pred_val = result.get("narrative_structure") if result else None
conf = result.get("confidence", "?") if result else "?"
hit = pred_val == gt_val if pred_val is not None else False
rows.append({"ep": ep, "title": title, "gt": gt_val, "pred": pred_val or "解析失败", "hit": hit, "conf": conf})
if field == "narrative":
gt_val = gt.get("narrative_structure", "?")
pred_val = result.get("narrative_structure") if result else None
conf = result.get("confidence", "?") if result else "?"
hit = pred_val == gt_val if pred_val is not None else False
rows.append({"ep": ep, "title": title, "gt": gt_val, "pred": pred_val or "解析失败", "hit": hit, "conf": conf})
elif field == "classification":
# 骨架预留：等 ground-truth 标注就绪后实现具体比对逻辑
rows.append({"ep": ep, "title": title, "gt": "?", "pred": "classification 待实现", "hit": False, "conf": "?"})
# 打印每行
for r in rows:
mark = "✓" if r["hit"] else "✗"
conf_str = f'置信度:{r["conf"]}' if r["conf"] != "?" else ""
print(f' ep{r["ep"]:02d} {r["title"]:<10} | 标准:{r["gt"]:<8} | {model}:{r["pred"]:<8} | {mark} {conf_str}')
if field == "narrative":
# 打印每行
for r in rows:
mark = "✓" if r["hit"] else "✗"
conf_str = f'置信度:{r["conf"]}' if r["conf"] != "?" else ""
print(f' ep{r["ep"]:02d} {r["title"]:<10} | 标准:{r["gt"]:<8} | {model}:{r["pred"]:<8} | {mark} {conf_str}')
# 汇总
total = len(rows)
hits = sum(1 for r in rows if r["hit"])
hi_conf = [r for r in rows if r["conf"] == "高"]
mid_low = [r for r in rows if r["conf"] in ("中", "低")]
hi_hit = sum(1 for r in hi_conf if r["hit"])
ml_hit = sum(1 for r in mid_low if r["hit"])
# 汇总
total = len(rows)
hits = sum(1 for r in rows if r["hit"])
hi_conf = [r for r in rows if r["conf"] == "高"]
mid_low = [r for r in rows if r["conf"] in ("中", "低")]
hi_hit = sum(1 for r in hi_conf if r["hit"])
ml_hit = sum(1 for r in mid_low if r["hit"])
print(f"\n ===== {model} 命中情况 =====")
print(f" narrative_structure 命中: {hits}/{total} = {hits*100//total}%")
print(f" 自评\"高\"置信的命中率: {hi_hit}/{len(hi_conf)}")
print(f" 自评\"中/低\"置信的命中率: {ml_hit}/{len(mid_low)}")
print(f" 解析失败: {parse_fail} 期")
print(f"\n ===== {model} 命中情况 =====")
print(f" narrative_structure 命中: {hits}/{total} = {hits*100//total}%")
print(f" 自评\"高\"置信的命中率: {hi_hit}/{len(hi_conf)}")
print(f" 自评\"中/低\"置信的命中率: {ml_hit}/{len(mid_low)}")
print(f" 解析失败: {parse_fail} 期")
elif field == "classification":
print(f"\n ===== {model} classification =====")
print(f" classification 比对逻辑待实现（等 ground-truth 标注就绪）")
print(f" 解析失败: {parse_fail} 期")
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--model", required=True, help="模型键名，如 mimo-v2.5-pro / deepseek-v4-pro")
parser.add_argument("--field", default="narrative", choices=["narrative", "classification"],
help="打标字段: narrative(叙事结构) / classification(4分类)")
args = parser.parse_args()
run(args.model)
run(args.model, args.field)