From 03d1d110b0bac810274eaa49a8bda188e5ac7485 Mon Sep 17 00:00:00 2001 From: bingshuo <565183519@qq.com> Date: Mon, 18 May 2026 13:49:28 +0800 Subject: [PATCH] =?UTF-8?q?D101=20=E6=9B=B4=E6=96=B0=EF=BC=9Aauto=5Fpipeli?= =?UTF-8?q?ne.py=E6=94=B9=E7=94=A8=E7=8E=AF=E5=A2=83=E5=8F=98=E9=87=8F?= =?UTF-8?q?=E8=AF=BB=E5=AF=86=E9=92=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/auto_pipeline.py | 106 +++++++++++++-------------------------- 1 file changed, 34 insertions(+), 72 deletions(-) diff --git a/scripts/auto_pipeline.py b/scripts/auto_pipeline.py index 8399842..97ea31a 100644 --- a/scripts/auto_pipeline.py +++ b/scripts/auto_pipeline.py @@ -1,21 +1,25 @@ #!/usr/bin/env python3 """自动训练流水线 - 铸渊自管理系统 -母模型训练完成 → 上传COS → 启动代码模型训练 → 上传COS → 关实例 +母模型完成 → 上传COS → 启动代码模型训练 → 上传COS 使用方法： + export ZY_OSS_KEY= + export ZY_OSS_SECRET= nohup python3 auto_pipeline.py > pipeline.log 2>&1 & """ - import os, json, time, sys, subprocess sys.stdout.reconfigure(line_buffering=True) MOTHER_LOG = "/root/autodl-tmp/train_mother.log" MOTHER_OUT = "/root/autodl-tmp/output/qwen25-7b-sft/final" - COS_BUCKET = "sy-finetune-corpus-1317346199" COS_REGION = "ap-guangzhou" -COS_SECRET_ID = "AKIDkQuBQhoiS2OYXWebXLwMbdT7cvAScbbU" -COS_SECRET_KEY = "nPoZKArgUJBA4nJenjSxJSQBj5FCj3A4" +# 密钥通过环境变量读取（不硬编码） +COS_SECRET_ID = os.environ.get("ZY_OSS_KEY") or os.environ.get("COS_SECRET_ID") +COS_SECRET_KEY = os.environ.get("ZY_OSS_SECRET") or os.environ.get("COS_SECRET_KEY") +if not COS_SECRET_ID or not COS_SECRET_KEY: + print("❌ 需要环境变量：export ZY_OSS_KEY=... ZY_OSS_SECRET=...") + sys.exit(1) CODE_MODEL = "Qwen/Qwen2.5-Coder-7B" CODE_CACHE = "/root/autodl-tmp/cache" @@ -25,40 +29,23 @@ DATA_FILE = "/root/autodl-tmp/data/sft.jsonl" POLL_INTERVAL = 300 def run_cmd(cmd): - result = subprocess.run(cmd, shell=True, capture_output=True, text=True) - return result.stdout.strip(), result.returncode + r = subprocess.run(cmd, shell=True, capture_output=True, text=True) + return r.stdout.strip(), r.returncode def log(msg): t = time.strftime("%Y-%m-%d %H:%M:%S") - print(f"[{t}] {msg}") - sys.stdout.flush() + print(f"[{t}] {msg}"); sys.stdout.flush() def upload_to_cos(local_path, cos_key): log(f"Uploading {local_path} → cos://{COS_BUCKET}/{cos_key}") - script = f""" -from qcloud_cos import CosConfig, CosS3Client -import os -config = CosConfig(Region='{COS_REGION}', SecretId='{COS_SECRET_ID}', SecretKey='{COS_SECRET_KEY}') -client = CosS3Client(config) -if os.path.isdir('{local_path}'): - import glob - for f in glob.glob('{local_path}/**/*', recursive=True): - if os.path.isfile(f): - rel = os.path.relpath(f, '{local_path}') - client.upload_file(Bucket='{COS_BUCKET}', Key='{cos_key}/' + rel, LocalFilePath=f) -else: - client.upload_file(Bucket='{COS_BUCKET}', Key='{cos_key}', LocalFilePath='{local_path}') -print('Upload OK') -""" - with open("/tmp/cos_upload.py", "w") as fp: - fp.write(script) - out, rc = run_cmd(f"export PATH=/root/miniconda3/bin:$PATH && python3 /tmp/cos_upload.py") - log(out) - return rc == 0 + s = "from qcloud_cos import CosConfig, CosS3Client; " + s += f"c=CosS3Client(CosConfig(Region='{COS_REGION}',SecretId='{COS_SECRET_ID}',SecretKey='{COS_SECRET_KEY}')); " + s += f"c.upload_file(Bucket='{COS_BUCKET}',Key='{cos_key}',LocalFilePath='{local_path}')" + out, rc = run_cmd(f"python3 -c \"{s}\"") + log(out); return rc == 0 def check_mother_done(): - if not os.path.exists(MOTHER_LOG): - return False + if not os.path.exists(MOTHER_LOG): return False out, _ = run_cmd(f"grep -a 'DONE' {MOTHER_LOG} | tail -1") return "DONE" in out @@ -66,56 +53,31 @@ def check_mother_output(): return os.path.isdir(MOTHER_OUT) and any(f.endswith('.safetensors') for f in os.listdir(MOTHER_OUT)) def generate_coder_script(): - return open('/root/autodl-tmp/train_mother.py').read().replace( - 'Qwen2.5-7B', 'Qwen2.5-Coder-7B' - ).replace( - 'qwen25-7b-sft', 'qwen25-coder-7b-sft' - ) + s = open("/root/autodl-tmp/train_mother.py").read() + s = s.replace("Qwen2___5-7B", "Qwen2___5-Coder-7B") + s = s.replace("qwen25-7b-sft", "qwen25-coder-7b-sft") + return s def main(): - log("=== 铸渊自动训练流水线启动 ===") - - # Phase 1: Wait for mother model - log("等待母模型训练完成...") + log("=== 自动训练流水线启动 ===") while True: if check_mother_done() or check_mother_output(): - log("✅ 母模型训练完成！") break - log(f" 母模型仍在训练中... (每{POLL_INTERVAL}秒检查)") time.sleep(POLL_INTERVAL) - - if not check_mother_output(): - log("❌ 母模型输出不存在") - return - - # Phase 2: Upload to COS - log("=== 上传母模型到COS ===") - upload_to_cos(MOTHER_OUT, "models/qwen25-7b-sft/final") - - # Phase 3: Start code model training - log("=== 准备代码模型训练 ===") - with open(CODE_SCRIPT, 'w') as f: - f.write(generate_coder_script()) - - cmd = f"export PATH=/root/miniconda3/bin:$PATH && cd /root/autodl-tmp && nohup python3 -u {CODE_SCRIPT} > train_coder.log 2>&1 &" - run_cmd(cmd) - log("代码模型训练已启动") - - # Phase 4: Wait for code model + log("✅ 母模型训练完成") + if upload_to_cos(MOTHER_OUT, "models/qwen25-7b-sft/final"): + log("✅ 母模型上传COS完成") + with open(CODE_SCRIPT, 'w') as f: f.write(generate_coder_script()) + run_cmd(f"nohup python3 -u {CODE_SCRIPT} > train_coder.log 2>&1 &") + log("✅ 代码模型训练已启动") while True: - out, _ = run_cmd(f"grep -a 'DONE' /root/autodl-tmp/train_coder.log | tail -1") + out, _ = run_cmd("grep -a 'DONE' /root/autodl-tmp/train_coder.log | tail -1") if "DONE" in out: - log("✅ 代码模型训练完成！") - break + log("✅ 代码模型训练完成"); break time.sleep(POLL_INTERVAL) - - # Phase 5: Upload code model to COS - upload_to_cos(f"{CODE_OUT}/final", "models/qwen25-coder-7b-sft/final") - - log("=" * 50) - log("U0001F389 全部训练任务完成！") - log("⚠️ GPU实例仍在运行，建议手动关停") - log("=" * 50) + if upload_to_cos(f"{CODE_OUT}/final", "models/qwen25-coder-7b-sft/final"): + log("✅ 代码模型上传COS完成") + log("🎉 全部训练完成！") if __name__ == "__main__": main()