📜 蒸馏脚本：check_corpus.py

scripts/distill/check_corpus.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+"""Check COS corpus content"""
+import os
+os.environ['ZY_OSS_KEY'] = 'AKIDkQuBQhoiS2OYXWebXLwMbdT7cvAScbbU'
+os.environ['ZY_OSS_SECRET'] = 'nPoZKArgUJBA4nJenjSxJSQBj5FCj3A4'
+from qcloud_cos import CosConfig, CosS3Client
+config = CosConfig(Region='ap-guangzhou', SecretId=os.environ['ZY_OSS_KEY'], SecretKey=os.environ['ZY_OSS_SECRET'])
+client = CosS3Client(config)
+bucket = 'sy-finetune-corpus-1317346199'
+
+# Download and check zhuyuan_full_corpus.jsonl
+resp = client.get_object(Bucket=bucket, Key='corpus/zhuyuan_full_corpus.jsonl')
+content = resp['Body'].read().decode('utf-8')
+lines = content.strip().split('\n')
+print(f'zhuyuan_full_corpus.jsonl: {len(lines)} 行')
+print(f'总大小: {len(content)/1024:.1f}KB')
+print()
+if lines:
+    import json
+    first = json.loads(lines[0])
+    keys = list(first.keys())
+    print(f'字段: {keys}')
+    if 'messages' in first:
+        msgs = first['messages']
+        print(f'messages 数量: {len(msgs)}')
+        for m in msgs[:3]:
+            print(f'  role={m["role"]}, content_len={len(m["content"])}')
+    print()
+    print('第一条前300字符:')
+    s = json.dumps(first, ensure_ascii=False, indent=2)
+    print(s[:300])
+
+print()
+print('=== zhuyuan_deep_finetune.jsonl ===')
+resp2 = client.get_object(Bucket=bucket, Key='corpus/zhuyuan_deep_finetune.jsonl')
+content2 = resp2['Body'].read().decode('utf-8')
+lines2 = content2.strip().split('\n')
+print(f'{len(lines2)} 行')
+if lines2:
+    d2 = json.loads(lines2[0])
+    print(f'字段: {list(d2.keys())}')