D101 更新：auto_pipeline.py改用环境变量读密钥
This commit is contained in:
bingshuo
parent
1ed393989b
commit
03d1d110b0
@ -1,21 +1,25 @@
|
||||
#!/usr/bin/env python3
|
||||
"""自动训练流水线 - 铸渊自管理系统
|
||||
母模型训练完成 → 上传COS → 启动代码模型训练 → 上传COS → 关实例
|
||||
母模型完成 → 上传COS → 启动代码模型训练 → 上传COS
|
||||
|
||||
使用方法：
|
||||
export ZY_OSS_KEY=<SecretId>
|
||||
export ZY_OSS_SECRET=<SecretKey>
|
||||
nohup python3 auto_pipeline.py > pipeline.log 2>&1 &
|
||||
"""
|
||||
|
||||
import os, json, time, sys, subprocess
|
||||
sys.stdout.reconfigure(line_buffering=True)
|
||||
|
||||
MOTHER_LOG = "/root/autodl-tmp/train_mother.log"
|
||||
MOTHER_OUT = "/root/autodl-tmp/output/qwen25-7b-sft/final"
|
||||
|
||||
COS_BUCKET = "sy-finetune-corpus-1317346199"
|
||||
COS_REGION = "ap-guangzhou"
|
||||
COS_SECRET_ID = "AKIDkQuBQhoiS2OYXWebXLwMbdT7cvAScbbU"
|
||||
COS_SECRET_KEY = "nPoZKArgUJBA4nJenjSxJSQBj5FCj3A4"
|
||||
# 密钥通过环境变量读取（不硬编码）
|
||||
COS_SECRET_ID = os.environ.get("ZY_OSS_KEY") or os.environ.get("COS_SECRET_ID")
|
||||
COS_SECRET_KEY = os.environ.get("ZY_OSS_SECRET") or os.environ.get("COS_SECRET_KEY")
|
||||
if not COS_SECRET_ID or not COS_SECRET_KEY:
|
||||
print("❌ 需要环境变量：export ZY_OSS_KEY=... ZY_OSS_SECRET=...")
|
||||
sys.exit(1)
|
||||
|
||||
CODE_MODEL = "Qwen/Qwen2.5-Coder-7B"
|
||||
CODE_CACHE = "/root/autodl-tmp/cache"
|
||||
@ -25,40 +29,23 @@ DATA_FILE = "/root/autodl-tmp/data/sft.jsonl"
|
||||
POLL_INTERVAL = 300
|
||||
|
||||
def run_cmd(cmd):
|
||||
result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
|
||||
return result.stdout.strip(), result.returncode
|
||||
r = subprocess.run(cmd, shell=True, capture_output=True, text=True)
|
||||
return r.stdout.strip(), r.returncode
|
||||
|
||||
def log(msg):
|
||||
t = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
print(f"[{t}] {msg}")
|
||||
sys.stdout.flush()
|
||||
print(f"[{t}] {msg}"); sys.stdout.flush()
|
||||
|
||||
def upload_to_cos(local_path, cos_key):
|
||||
log(f"Uploading {local_path} → cos://{COS_BUCKET}/{cos_key}")
|
||||
script = f"""
|
||||
from qcloud_cos import CosConfig, CosS3Client
|
||||
import os
|
||||
config = CosConfig(Region='{COS_REGION}', SecretId='{COS_SECRET_ID}', SecretKey='{COS_SECRET_KEY}')
|
||||
client = CosS3Client(config)
|
||||
if os.path.isdir('{local_path}'):
|
||||
import glob
|
||||
for f in glob.glob('{local_path}/**/*', recursive=True):
|
||||
if os.path.isfile(f):
|
||||
rel = os.path.relpath(f, '{local_path}')
|
||||
client.upload_file(Bucket='{COS_BUCKET}', Key='{cos_key}/' + rel, LocalFilePath=f)
|
||||
else:
|
||||
client.upload_file(Bucket='{COS_BUCKET}', Key='{cos_key}', LocalFilePath='{local_path}')
|
||||
print('Upload OK')
|
||||
"""
|
||||
with open("/tmp/cos_upload.py", "w") as fp:
|
||||
fp.write(script)
|
||||
out, rc = run_cmd(f"export PATH=/root/miniconda3/bin:$PATH && python3 /tmp/cos_upload.py")
|
||||
log(out)
|
||||
return rc == 0
|
||||
s = "from qcloud_cos import CosConfig, CosS3Client; "
|
||||
s += f"c=CosS3Client(CosConfig(Region='{COS_REGION}',SecretId='{COS_SECRET_ID}',SecretKey='{COS_SECRET_KEY}')); "
|
||||
s += f"c.upload_file(Bucket='{COS_BUCKET}',Key='{cos_key}',LocalFilePath='{local_path}')"
|
||||
out, rc = run_cmd(f"python3 -c \"{s}\"")
|
||||
log(out); return rc == 0
|
||||
|
||||
def check_mother_done():
|
||||
if not os.path.exists(MOTHER_LOG):
|
||||
return False
|
||||
if not os.path.exists(MOTHER_LOG): return False
|
||||
out, _ = run_cmd(f"grep -a 'DONE' {MOTHER_LOG} | tail -1")
|
||||
return "DONE" in out
|
||||
|
||||
@ -66,56 +53,31 @@ def check_mother_output():
|
||||
return os.path.isdir(MOTHER_OUT) and any(f.endswith('.safetensors') for f in os.listdir(MOTHER_OUT))
|
||||
|
||||
def generate_coder_script():
|
||||
return open('/root/autodl-tmp/train_mother.py').read().replace(
|
||||
'Qwen2.5-7B', 'Qwen2.5-Coder-7B'
|
||||
).replace(
|
||||
'qwen25-7b-sft', 'qwen25-coder-7b-sft'
|
||||
)
|
||||
s = open("/root/autodl-tmp/train_mother.py").read()
|
||||
s = s.replace("Qwen2___5-7B", "Qwen2___5-Coder-7B")
|
||||
s = s.replace("qwen25-7b-sft", "qwen25-coder-7b-sft")
|
||||
return s
|
||||
|
||||
def main():
|
||||
log("=== 铸渊自动训练流水线启动 ===")
|
||||
|
||||
# Phase 1: Wait for mother model
|
||||
log("等待母模型训练完成...")
|
||||
log("=== 自动训练流水线启动 ===")
|
||||
while True:
|
||||
if check_mother_done() or check_mother_output():
|
||||
log("✅ 母模型训练完成！")
|
||||
break
|
||||
log(f" 母模型仍在训练中... (每{POLL_INTERVAL}秒检查)")
|
||||
time.sleep(POLL_INTERVAL)
|
||||
|
||||
if not check_mother_output():
|
||||
log("❌ 母模型输出不存在")
|
||||
return
|
||||
|
||||
# Phase 2: Upload to COS
|
||||
log("=== 上传母模型到COS ===")
|
||||
upload_to_cos(MOTHER_OUT, "models/qwen25-7b-sft/final")
|
||||
|
||||
# Phase 3: Start code model training
|
||||
log("=== 准备代码模型训练 ===")
|
||||
with open(CODE_SCRIPT, 'w') as f:
|
||||
f.write(generate_coder_script())
|
||||
|
||||
cmd = f"export PATH=/root/miniconda3/bin:$PATH && cd /root/autodl-tmp && nohup python3 -u {CODE_SCRIPT} > train_coder.log 2>&1 &"
|
||||
run_cmd(cmd)
|
||||
log("代码模型训练已启动")
|
||||
|
||||
# Phase 4: Wait for code model
|
||||
log("✅ 母模型训练完成")
|
||||
if upload_to_cos(MOTHER_OUT, "models/qwen25-7b-sft/final"):
|
||||
log("✅ 母模型上传COS完成")
|
||||
with open(CODE_SCRIPT, 'w') as f: f.write(generate_coder_script())
|
||||
run_cmd(f"nohup python3 -u {CODE_SCRIPT} > train_coder.log 2>&1 &")
|
||||
log("✅ 代码模型训练已启动")
|
||||
while True:
|
||||
out, _ = run_cmd(f"grep -a 'DONE' /root/autodl-tmp/train_coder.log | tail -1")
|
||||
out, _ = run_cmd("grep -a 'DONE' /root/autodl-tmp/train_coder.log | tail -1")
|
||||
if "DONE" in out:
|
||||
log("✅ 代码模型训练完成！")
|
||||
break
|
||||
log("✅ 代码模型训练完成"); break
|
||||
time.sleep(POLL_INTERVAL)
|
||||
|
||||
# Phase 5: Upload code model to COS
|
||||
upload_to_cos(f"{CODE_OUT}/final", "models/qwen25-coder-7b-sft/final")
|
||||
|
||||
log("=" * 50)
|
||||
log("U0001F389 全部训练任务完成！")
|
||||
log("⚠️ GPU实例仍在运行，建议手动关停")
|
||||
log("=" * 50)
|
||||
if upload_to_cos(f"{CODE_OUT}/final", "models/qwen25-coder-7b-sft/final"):
|
||||
log("✅ 代码模型上传COS完成")
|
||||
log("🎉 全部训练完成！")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user