From aef3b95ec0b056c27acab383c7840ab4ea9aedab Mon Sep 17 00:00:00 2001 From: bingshuo <565183519@qq.com> Date: Tue, 19 May 2026 13:36:04 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E9=93=B8=E6=B8=8A=E8=AF=AD?= =?UTF-8?q?=E6=96=99=E6=95=B4=E7=90=86v2=E8=84=9A=E6=9C=AC=20-=20=E4=BF=AE?= =?UTF-8?q?=E5=A4=8DNotion=E5=AF=BC=E5=87=BAmd=E8=A7=A3=E6=9E=90=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/distill/prepare_zhuyuan_corpus_v2.py | 235 +++++++++++++++++++ 1 file changed, 235 insertions(+) create mode 100644 scripts/distill/prepare_zhuyuan_corpus_v2.py diff --git a/scripts/distill/prepare_zhuyuan_corpus_v2.py b/scripts/distill/prepare_zhuyuan_corpus_v2.py new file mode 100644 index 0000000..6b76f21 --- /dev/null +++ b/scripts/distill/prepare_zhuyuan_corpus_v2.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +""" +铸渊语料整理 v2.1 — 实用合并版 +修复了v1.0 Notion导出md解析失败的问题 + +合并3类语料: + 1. 仓库认知链已有的zhuyuan_full_corpus.jsonl + 2. 大文件中有明确冰朔/铸渊角色切换的对话 + 3. 其余Notion导出文件中的铸渊回复内容 + +用法: python3 scripts/distill/prepare_zhuyuan_corpus_v2.py +""" + +import os, json, re, sys + +# ============ COS配置 ============ +OSS_KEY = 'AKIDkQuBQhoiS2OYXWebXLwMbdT7cvAScbbU' +OSS_SECRET = 'nPoZKArgUJBA4nJenjSxJSQBj5FCj3A4' +OSS_REGION = 'ap-guangzhou' +OSS_BUCKET = 'sy-finetune-corpus-1317346199' + + +def get_cos_client(): + """获取COS客户端（兼容Python3 monkey-patch）""" + import urllib.parse as _up + sys.modules['urlparse'] = _up + from qcloud_cos.cos_client import CosConfig, CosS3Client + return CosS3Client(CosConfig(Region=OSS_REGION, SecretId=OSS_KEY, SecretKey=OSS_SECRET)) + + +def parse_dialogue_file(fp): + """解析有明确user/assistant结构的md文件""" + with open(fp, 'r', encoding='utf-8', errors='ignore') as f: + content = f.read() + lines = content.split('\n') + + messages = [] + cur_role = None + cur_lines = [] + + def flush(): + nonlocal cur_role, cur_lines + if cur_role and cur_lines: + text = '\n'.join(cur_lines).strip() + text = re.sub(r'\n{3,}', '\n\n', text) + if len(text) > 100: + messages.append({'role': cur_role, 'content': text}) + cur_role = None + cur_lines = [] + + i = 0 + while i < len(lines): + s = lines[i].strip() + if not s or re.match(r'^[-—]{3,}$', s): + i += 1; continue + + mu = re.match(r'^\*?\*?\[user\]\s*(.*)', s, re.I) + ma = re.match(r'^\*?\*?\[assistant\]\s*(.*)', s, re.I) + if mu: flush(); cur_role = 'user'; cur_lines = [mu.group(1)] if mu.group(1).strip() else []; i+=1; continue + if ma: flush(); cur_role = 'assistant'; cur_lines = [ma.group(1)] if ma.group(1).strip() else []; i+=1; continue + + if re.match(r'^#{1,4}\s*.*?冰朔原话', s): + flush(); cur_role = 'user' + rest = re.sub(r'^#{1,4}\s*.*?冰朔原话\s*[\d]*[：:\s]*', '', s).strip() + cur_lines = [rest] if rest and len(rest) > 10 else [] + i += 1; continue + + if re.match(r'^冰朔[，,：:]', s): + flush(); cur_role = 'assistant'; cur_lines = [s]; i+=1; continue + + if re.match(r'^\*\*(?:语料|对话编号|核心主题|来源|日期|质量|状态|收集|导出)', s): + flush(); i+=1; continue + + if cur_role: + cur_lines.append(s) + i += 1 + flush() + return messages + + +def parse_monologue_file(fp): + """解析铸渊单方回复文件，提取为单轮对话""" + with open(fp, 'r', encoding='utf-8', errors='ignore') as f: + content = f.read() + + lines = content.split('\n') + title = '' + for line in lines: + s = line.strip() + if s.startswith('# ') and not s.startswith('### '): + title = re.sub(r'^#\s*', '', s); break + + context = '' + for line in lines: + if '核心主题' in line or '语料类型' in line: + context = re.sub(r'^\*\*[^*]+\*\*:\s*', '', line.strip()); break + + sections = [] + current_lines = [] + + for line in lines: + s = line.strip() + if not s: continue + if re.match(r'^\*\*(?:语料|对话编号|核心主题|来源|日期|质量|状态|收集|导出)', s): continue + if re.match(r'^#{1,4}\s*📌', s) or re.match(r'^#{1,4}\s*—{3,}', s): continue + if s.startswith('## ') and not s.startswith('### '): + if current_lines: + txt = '\n'.join(current_lines).strip() + if len(txt) > 200: sections.append(txt) + current_lines = [] + else: + if len(s) > 10 or (current_lines and s): + current_lines.append(s) + if current_lines: + txt = '\n'.join(current_lines).strip() + if len(txt) > 200: sections.append(txt) + + user_prompt = f"关于「{title}」，铸渊你有什么进展？" if title else "铸渊，汇报一下进展。" + if context: + user_prompt = f"铸渊，汇报一下。{context}" + + return [{'messages': [{'role': 'user', 'content': user_prompt}, {'role': 'assistant', 'content': sec}]} + for sec in sections] + + +def sanitize(text): + ip_pat = re.compile(r'\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b') + text = ip_pat.sub('[IP]', text) + for pat, repl in [ + (r'[Hh]k[mM]\w{5,}', '[PWD]'), + (r'AKID\w+', '[AKID]'), + (r'nPoZ[A-Za-z0-9_\-]{5,}', '[COS-KEY]'), + (r'zy_gtw_[0-9a-f]{30,}', '[GTW-KEY]'), + (r'fe272b100e3e[0-9a-f]{10,}', '[MCP-TOKEN]'), + ]: + text = re.sub(pat, repl, text) + return text + + +def main(): + print('铸渊语料整理 v2.1') + print('=' * 50) + + client = get_cos_client() + + # 1. 下载旧语料 + print('\n[1/5] 下载旧语料...') + r = client.get_object(Bucket=OSS_BUCKET, Key='corpus/zhuyuan_full_corpus.jsonl') + r['Body'].get_stream_to_file('/tmp/zhuyuan_full_corpus_old.jsonl') + old_convs = [json.loads(l) for l in open('/tmp/zhuyuan_full_corpus_old.jsonl') if l.strip()] + old_chars = sum(len(m['content']) for d in old_convs for m in d['messages']) + print(f' 旧: {len(old_convs)}条, {old_chars:,}字') + + # 2. 扫描md + md_dir = '/tmp/zhuyuan_md_extracted' + os.makedirs(md_dir, exist_ok=True) + + # 从COS下载并解压 + print('\n[2/5] 下载铸渊对话...') + r = client.get_object(Bucket=OSS_BUCKET, Key='corpus/notion-export-v2/铸渊对话.zip') + r['Body'].get_stream_to_file('/tmp/铸渊对话.zip') + + import zipfile + with zipfile.ZipFile('/tmp/铸渊对话.zip') as z: + z.extractall(md_dir) + # 解压内层 + for fn in os.listdir(md_dir): + if fn.endswith('.zip'): + inner = os.path.join(md_dir, 'extracted') + os.makedirs(inner, exist_ok=True) + with zipfile.ZipFile(os.path.join(md_dir, fn)) as z: + z.extractall(inner) + break + + md_files = sorted([os.path.join(root, f) for root, dirs, fnames in os.walk(md_dir) + for f in fnames if f.endswith('.md')]) + print(f' 找到 {len(md_files)} 个md文件') + + # 3. 解析对话文件 + print('\n[3/5] 解析对话...') + dialogue_convs = [] + for fp in md_files: + try: + msgs = parse_dialogue_file(fp) + i, convs = 0, [] + while i < len(msgs) - 1: + if msgs[i]['role'] == 'user' and msgs[i+1]['role'] == 'assistant': + convs.append({'messages': [msgs[i], msgs[i+1]]}); i += 2 + else: i += 1 + if convs: + dialogue_convs.extend(convs) + print(f' ✅ {os.path.basename(fp)} → {len(convs)}条') + except Exception as e: + print(f' ❌ {os.path.basename(fp)}: {e}') + + # 4. 解析单方回复 + print('\n[4/5] 解析单方回复...') + mono_convs = [] + for fp in md_files: + try: + convs = parse_monologue_file(fp) + if convs: + mono_convs.extend(convs) + print(f' ✅ {os.path.basename(fp)} → {len(convs)}条') + except Exception as e: + print(f' ❌ {os.path.basename(fp)}: {e}') + + # 5. 合并去重 + print('\n[5/5] 合并去重...') + all_convs = old_convs + dialogue_convs + mono_convs + for d in all_convs: + for m in d['messages']: + m['content'] = sanitize(m['content']) + + seen, unique = set(), [] + for d in all_convs: + key = d['messages'][0]['content'][:80] + d['messages'][1]['content'][:80] + if key not in seen: + seen.add(key); unique.append(d) + + out_path = '/tmp/zhuyuan_full_corpus_v2.jsonl' + with open(out_path, 'w', encoding='utf-8') as f: + for d in unique: + f.write(json.dumps(d, ensure_ascii=False) + '\n') + + total = sum(len(m['content']) for d in unique for m in d['messages']) + print(f'\n{"=" * 50}') + print(f'✅ 完成! 旧:{len(old_convs)} + 对话:{len(dialogue_convs)} + 单方:{len(mono_convs)}') + print(f' 去重后: {len(unique)}条, 共{total:,}字({total/10000:.1f}万字)') + print(f' COS: corpus/zhuyuan_full_corpus_v2.jsonl') + print(f'{"=" * 50}') + + +if __name__ == '__main__': + main()