120 lines
4.1 KiB
Python
120 lines
4.1 KiB
Python
|
"""
|
|||
|
|
import_transcripts.py - 将源目录的 10 期文稿(docx/伪docx)导入到 benchmark-set/transcripts/
|
|||
|
|
用法: python scripts/import_transcripts.py
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
import re
|
|||
|
|
import sys
|
|||
|
|
from pathlib import Path
|
|||
|
|
from docx import Document
|
|||
|
|
|
|||
|
|
# ===== 配置 =====
|
|||
|
|
SRC_DIR = Path(r"E:\TPS-我和顾问的便签\刘瑞桦收集")
|
|||
|
|
DST_DIR = Path(__file__).parent.parent / "benchmark-set" / "transcripts"
|
|||
|
|
DST_DIR.mkdir(parents=True, exist_ok=True)
|
|||
|
|
|
|||
|
|
# 映射表: 源文件名 → 目标文件名
|
|||
|
|
# （注意：源文件名中的标点/空格与预期不同，已据实更新）
|
|||
|
|
FILE_MAP = [
|
|||
|
|
("硅基大脑终稿.docx", "ep03_硅基大脑.md"),
|
|||
|
|
("潜艇的仿生之路.docx", "ep04_潜艇仿生.md"),
|
|||
|
|
("马年军事图鉴0125.docx", "ep07_马年图鉴.md"),
|
|||
|
|
("\u300a枪械射速决定论\u300b0225.docx", "ep10_射速决定论.md"),
|
|||
|
|
("完稿 武器进化论\uff1a空战颠覆者.docx", "ep11_空战颠覆者.md"),
|
|||
|
|
("逆袭战局的组装武器.docx", "ep12_逆袭战局.md"),
|
|||
|
|
("枪械设计中的长短智慧总稿.docx", "ep13_长短智慧.md"),
|
|||
|
|
("\u201cX\u201d系列新成员\uff1aX-76飞机-最终稿.docx", "ep14_X76飞机.md"),
|
|||
|
|
("空中坚盾\u2014\u2014解码现代防空网.docx", "ep15_空中坚盾.md"),
|
|||
|
|
# .doc 文件（改 .docx 后重试）
|
|||
|
|
("舰证不凡_docx版.docx", "ep05_舰证不凡.md"),
|
|||
|
|
]
|
|||
|
|
# ===== 配置 =====
|
|||
|
|
|
|||
|
|
|
|||
|
|
def is_real_docx(path: Path) -> bool:
|
|||
|
|
"""通过前 4 字节判断是否为真 docx（ZIP 格式）"""
|
|||
|
|
try:
|
|||
|
|
with open(path, "rb") as f:
|
|||
|
|
header = f.read(4)
|
|||
|
|
return header == b"PK\x03\x04"
|
|||
|
|
except Exception:
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
|
|||
|
|
def extract_text_real_docx(path: Path) -> str:
|
|||
|
|
"""用 python-docx 提取真 docx 段落文本"""
|
|||
|
|
doc = Document(str(path))
|
|||
|
|
paragraphs = []
|
|||
|
|
for para in doc.paragraphs:
|
|||
|
|
text = para.text.strip()
|
|||
|
|
if text:
|
|||
|
|
paragraphs.append(text)
|
|||
|
|
return "\n\n".join(paragraphs)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def extract_text_fake_docx(path: Path) -> str:
|
|||
|
|
"""伪 docx（实际是 UTF-8 纯文本），直接读取"""
|
|||
|
|
return path.read_text(encoding="utf-8")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def clean_text(text: str) -> str:
|
|||
|
|
"""清洗文本：去掉 **Markdown 强调**，合并连续空行"""
|
|||
|
|
# 去掉 ** ... ** 强调符号，保留中间文字
|
|||
|
|
text = re.sub(r"\*\*(.+?)\*\*", r"\1", text)
|
|||
|
|
# 去掉 * ... * 斜体（如果还有残留）
|
|||
|
|
text = re.sub(r"\*(.+?)\*", r"\1", text)
|
|||
|
|
# 连续 3 个以上空行合并为 2 个
|
|||
|
|
text = re.sub(r"\n{3,}", "\n\n", text)
|
|||
|
|
return text.strip()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def process_file(src_name: str, dst_name: str) -> tuple[bool, str]:
|
|||
|
|
"""处理单个文件，返回 (是否成功, 消息)"""
|
|||
|
|
src_path = SRC_DIR / src_name
|
|||
|
|
if not src_path.exists():
|
|||
|
|
return False, f"源文件不存在: {src_name}"
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
if is_real_docx(src_path):
|
|||
|
|
raw_text = extract_text_real_docx(src_path)
|
|||
|
|
else:
|
|||
|
|
raw_text = extract_text_fake_docx(src_path)
|
|||
|
|
|
|||
|
|
clean_text_str = clean_text(raw_text)
|
|||
|
|
dst_path = DST_DIR / dst_name
|
|||
|
|
dst_path.write_text(clean_text_str, encoding="utf-8")
|
|||
|
|
char_count = len(clean_text_str)
|
|||
|
|
return True, f"OK ({char_count} 字符)"
|
|||
|
|
except Exception as e:
|
|||
|
|
return False, f"解析失败: {e}"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def main():
|
|||
|
|
print(f"源目录: {SRC_DIR}")
|
|||
|
|
print(f"目标目录: {DST_DIR}")
|
|||
|
|
print(f"待处理: {len(FILE_MAP)} 期\n")
|
|||
|
|
|
|||
|
|
results = []
|
|||
|
|
for src_name, dst_name in FILE_MAP:
|
|||
|
|
ok, msg = process_file(src_name, dst_name)
|
|||
|
|
results.append((src_name, dst_name, ok, msg))
|
|||
|
|
status = "[OK]" if ok else "[FAIL]"
|
|||
|
|
print(f" {status} {src_name} -> {dst_name} {msg}")
|
|||
|
|
|
|||
|
|
# 汇总
|
|||
|
|
success = sum(1 for r in results if r[2])
|
|||
|
|
fail = len(results) - success
|
|||
|
|
print(f"\n===== 汇总 =====")
|
|||
|
|
print(f"成功: {success}/{len(results)}")
|
|||
|
|
if fail > 0:
|
|||
|
|
print("失败文件:")
|
|||
|
|
for src, dst, ok, msg in results:
|
|||
|
|
if not ok:
|
|||
|
|
print(f" - {src}")
|
|||
|
|
else:
|
|||
|
|
print("全部成功!")
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
main()
|