#!/usr/bin/env python3
"""Check COS corpus content"""
import os
os.environ['ZY_OSS_KEY'] = 'AKIDkQuBQhoiS2OYXWebXLwMbdT7cvAScbbU'
os.environ['ZY_OSS_SECRET'] = 'nPoZKArgUJBA4nJenjSxJSQBj5FCj3A4'
from qcloud_cos import CosConfig, CosS3Client
config = CosConfig(Region='ap-guangzhou', SecretId=os.environ['ZY_OSS_KEY'], SecretKey=os.environ['ZY_OSS_SECRET'])
client = CosS3Client(config)
bucket = 'sy-finetune-corpus-1317346199'

# Download and check zhuyuan_full_corpus.jsonl
resp = client.get_object(Bucket=bucket, Key='corpus/zhuyuan_full_corpus.jsonl')
content = resp['Body'].read().decode('utf-8')
lines = content.strip().split('\n')
print(f'zhuyuan_full_corpus.jsonl: {len(lines)} 行')
print(f'总大小: {len(content)/1024:.1f}KB')
print()
if lines:
    import json
    first = json.loads(lines[0])
    keys = list(first.keys())
    print(f'字段: {keys}')
    if 'messages' in first:
        msgs = first['messages']
        print(f'messages 数量: {len(msgs)}')
        for m in msgs[:3]:
            print(f'  role={m["role"]}, content_len={len(m["content"])}')
    print()
    print('第一条前300字符:')
    s = json.dumps(first, ensure_ascii=False, indent=2)
    print(s[:300])

print()
print('=== zhuyuan_deep_finetune.jsonl ===')
resp2 = client.get_object(Bucket=bucket, Key='corpus/zhuyuan_deep_finetune.jsonl')
content2 = resp2['Body'].read().decode('utf-8')
lines2 = content2.strip().split('\n')
print(f'{len(lines2)} 行')
if lines2:
    d2 = json.loads(lines2[0])
    print(f'字段: {list(d2.keys())}')