43 lines
1.1 KiB
Python
43 lines
1.1 KiB
Python
|
# -*- coding: utf-8 -*-
|
||
|
|
"""去重 ocr_raw.jsonl: 每个 idx 只保留第一条"""
|
||
|
|
import json, os
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
WORK_DIR = Path(__file__).resolve().parent
|
||
|
|
src = WORK_DIR / "ocr_raw.jsonl"
|
||
|
|
dst = WORK_DIR / "ocr_raw_clean.jsonl"
|
||
|
|
|
||
|
|
sys_out_ok = False
|
||
|
|
try:
|
||
|
|
import sys
|
||
|
|
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||
|
|
sys_out_ok = True
|
||
|
|
except:
|
||
|
|
pass
|
||
|
|
|
||
|
|
lines = [l for l in src.read_text(encoding="utf-8").strip().split("\n") if l.strip()]
|
||
|
|
seen = set()
|
||
|
|
deduped = []
|
||
|
|
for l in lines:
|
||
|
|
rec = json.loads(l)
|
||
|
|
idx = rec.get("idx")
|
||
|
|
if idx not in seen:
|
||
|
|
seen.add(idx)
|
||
|
|
deduped.append(l)
|
||
|
|
|
||
|
|
deduped.sort(key=lambda x: json.loads(x).get("idx", 0))
|
||
|
|
|
||
|
|
# write to clean file
|
||
|
|
dst.write_text("\n".join(deduped) + "\n", encoding="utf-8")
|
||
|
|
print(f"deduped: {len(lines)} -> {len(deduped)} lines")
|
||
|
|
|
||
|
|
# verify clean file
|
||
|
|
v = dst.read_text(encoding="utf-8").strip().split("\n")
|
||
|
|
print(f"verify clean: {len(v)} lines")
|
||
|
|
|
||
|
|
# replace original
|
||
|
|
os.replace(str(dst), str(src))
|
||
|
|
|
||
|
|
# verify original
|
||
|
|
v2 = src.read_text(encoding="utf-8").strip().split("\n")
|
||
|
|
print(f"verify original: {len(v2)} lines, {src.stat().st_size//1024}KB")
|