diff --git a/scripts/distill/check_corpus.py b/scripts/distill/check_corpus.py new file mode 100644 index 0000000..dc086fb --- /dev/null +++ b/scripts/distill/check_corpus.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Check COS corpus content""" +import os +os.environ['ZY_OSS_KEY'] = 'AKIDkQuBQhoiS2OYXWebXLwMbdT7cvAScbbU' +os.environ['ZY_OSS_SECRET'] = 'nPoZKArgUJBA4nJenjSxJSQBj5FCj3A4' +from qcloud_cos import CosConfig, CosS3Client +config = CosConfig(Region='ap-guangzhou', SecretId=os.environ['ZY_OSS_KEY'], SecretKey=os.environ['ZY_OSS_SECRET']) +client = CosS3Client(config) +bucket = 'sy-finetune-corpus-1317346199' + +# Download and check zhuyuan_full_corpus.jsonl +resp = client.get_object(Bucket=bucket, Key='corpus/zhuyuan_full_corpus.jsonl') +content = resp['Body'].read().decode('utf-8') +lines = content.strip().split('\n') +print(f'zhuyuan_full_corpus.jsonl: {len(lines)} 行') +print(f'总大小: {len(content)/1024:.1f}KB') +print() +if lines: + import json + first = json.loads(lines[0]) + keys = list(first.keys()) + print(f'字段: {keys}') + if 'messages' in first: + msgs = first['messages'] + print(f'messages 数量: {len(msgs)}') + for m in msgs[:3]: + print(f' role={m["role"]}, content_len={len(m["content"])}') + print() + print('第一条前300字符:') + s = json.dumps(first, ensure_ascii=False, indent=2) + print(s[:300]) + +print() +print('=== zhuyuan_deep_finetune.jsonl ===') +resp2 = client.get_object(Bucket=bucket, Key='corpus/zhuyuan_deep_finetune.jsonl') +content2 = resp2['Body'].read().decode('utf-8') +lines2 = content2.strip().split('\n') +print(f'{len(lines2)} 行') +if lines2: + d2 = json.loads(lines2[0]) + print(f'字段: {list(d2.keys())}')