simonkoson
a87f453326
ep002（潜艇仿生）全部产物入库：B稿v2(733行)/ASR(411句)/融合B稿/ 融合A稿.docx/c4_alignment.csv等。fusion_align.py修复align_batch 崩溃bug。CLAUDE.md更新ep002完工状态、新增分段偏差根因和MiMo批次 失败率两条关键决策。 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
43 lines
1.1 KiB
Python
43 lines
1.1 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""去重 ocr_raw.jsonl: 每个 idx 只保留第一条"""
|
|
import json, os
|
|
from pathlib import Path
|
|
|
|
WORK_DIR = Path(__file__).resolve().parent
|
|
src = WORK_DIR / "ocr_raw.jsonl"
|
|
dst = WORK_DIR / "ocr_raw_clean.jsonl"
|
|
|
|
sys_out_ok = False
|
|
try:
|
|
import sys
|
|
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
|
sys_out_ok = True
|
|
except:
|
|
pass
|
|
|
|
lines = [l for l in src.read_text(encoding="utf-8").strip().split("\n") if l.strip()]
|
|
seen = set()
|
|
deduped = []
|
|
for l in lines:
|
|
rec = json.loads(l)
|
|
idx = rec.get("idx")
|
|
if idx not in seen:
|
|
seen.add(idx)
|
|
deduped.append(l)
|
|
|
|
deduped.sort(key=lambda x: json.loads(x).get("idx", 0))
|
|
|
|
# write to clean file
|
|
dst.write_text("\n".join(deduped) + "\n", encoding="utf-8")
|
|
print(f"deduped: {len(lines)} -> {len(deduped)} lines")
|
|
|
|
# verify clean file
|
|
v = dst.read_text(encoding="utf-8").strip().split("\n")
|
|
print(f"verify clean: {len(v)} lines")
|
|
|
|
# replace original
|
|
os.replace(str(dst), str(src))
|
|
|
|
# verify original
|
|
v2 = src.read_text(encoding="utf-8").strip().split("\n")
|
|
print(f"verify original: {len(v2)} lines, {src.stat().st_size//1024}KB") |