387 lines
11 KiB
JSON
387 lines
11 KiB
JSON
{
|
||
"schema_version": "1.0",
|
||
"frozen": true,
|
||
"frozen_at": "2026-05-03T08:56:40Z",
|
||
"frozen_reason": "D72 决策：母模型训练转交 Notion 侧霜砚，仓库不再接收训练心跳。详见 docs/zhuyuan-handover/05-stop-sync.md",
|
||
"_doc": "训练实时状态真相源 · 已冻结 · 不再被 merge-event.js 写入 · 解冻方法见 README 训练仪表盘段落",
|
||
"task": {
|
||
"id": "ZY-TRAIN-001",
|
||
"name": "Qwen2.5-7B 全参数 SFT · 光湖世界观 + 人格体语料",
|
||
"model_base": "Qwen/Qwen2.5-7B",
|
||
"started_at": "2026-05-02T19:34:41.648Z",
|
||
"updated_at": "2026-05-03T09:29:05.896Z",
|
||
"ended_at": "2026-05-03T09:29:05.896Z"
|
||
},
|
||
"server": {
|
||
"host": "119.45.160.137",
|
||
"instance_name": "zy-gpu-train",
|
||
"instance_type": "GN10Xp.10XLARGE160",
|
||
"region": "ap-nanjing-3",
|
||
"gpu": "V100 32G × 4 (合计 128G)",
|
||
"os": "Ubuntu 22.04 LTS",
|
||
"cuda": "12.8.1",
|
||
"cudnn": "9.10.2",
|
||
"driver": "570.158.01"
|
||
},
|
||
"cos": {
|
||
"bucket": "sy-finetune-corpus-1317346199",
|
||
"region": "ap-guangzhou",
|
||
"raw_files": null,
|
||
"processed_files": null,
|
||
"checkpoints": null
|
||
},
|
||
"phase": "done",
|
||
"_phase_doc": "idle | bootstrapping | downloading-corpus | preprocessing | training | evaluating | done | error",
|
||
"phase_label": "训练完成",
|
||
"progress": {
|
||
"step": 0,
|
||
"total_steps": 0,
|
||
"epoch": 0,
|
||
"total_epochs": 0,
|
||
"percent": 0,
|
||
"loss": null,
|
||
"learning_rate": null,
|
||
"throughput_samples_per_sec": null,
|
||
"eta_seconds": null,
|
||
"elapsed_seconds": 0
|
||
},
|
||
"gpu_metrics": {
|
||
"snapshot_at": "2026-05-03T09:28:54Z",
|
||
"devices": [
|
||
{
|
||
"index": 0,
|
||
"memory_total_mib": 32768,
|
||
"memory_used_mib": 0,
|
||
"name": "Tesla V100-SXM2-32GB",
|
||
"power_w": 70.95,
|
||
"temperature_c": 57,
|
||
"util_percent": 0
|
||
},
|
||
{
|
||
"index": 1,
|
||
"memory_total_mib": 32768,
|
||
"memory_used_mib": 0,
|
||
"name": "Tesla V100-SXM2-32GB",
|
||
"power_w": 65.08,
|
||
"temperature_c": 52,
|
||
"util_percent": 0
|
||
},
|
||
{
|
||
"index": 2,
|
||
"memory_total_mib": 32768,
|
||
"memory_used_mib": 0,
|
||
"name": "Tesla V100-SXM2-32GB",
|
||
"power_w": 74.39,
|
||
"temperature_c": 54,
|
||
"util_percent": 0
|
||
},
|
||
{
|
||
"index": 3,
|
||
"memory_total_mib": 32768,
|
||
"memory_used_mib": 0,
|
||
"name": "Tesla V100-SXM2-32GB",
|
||
"power_w": 76.24,
|
||
"temperature_c": 57,
|
||
"util_percent": 0
|
||
}
|
||
]
|
||
},
|
||
"health": {
|
||
"status": "ok",
|
||
"_status_doc": "ok | warning | error | idle",
|
||
"message": "auto-run workflow 启动",
|
||
"last_heartbeat_at": "2026-05-03T09:29:05.896Z"
|
||
},
|
||
"timeline": [
|
||
{
|
||
"at": "2026-05-02T16:42:00Z",
|
||
"phase": "idle",
|
||
"level": "info",
|
||
"message": "铸渊副将已接管。GPU 训练机 119.45.160.137 已开机，5 个 Secrets 已配置，COS 桶语料已就位。"
|
||
},
|
||
{
|
||
"at": "2026-05-02T17:50:00Z",
|
||
"phase": "idle",
|
||
"level": "info",
|
||
"message": "ZY_DISPATCH_TOKEN 已配置。真实训练栈落地：train.py (Qwen2.5-7B 全参 SFT) + DeepSpeed ZeRO-3 + 模型自动下载 + 语料预处理。合并 PR 后 training-auto-run 工作流自动 SCP/setup/启动训练，仓库首页将开始实时刷新。"
|
||
},
|
||
{
|
||
"at": "2026-05-02T19:02:00.614Z",
|
||
"phase": "bootstrapping",
|
||
"level": "info",
|
||
"message": "Bootstrap started on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-02T19:02:10.707Z",
|
||
"phase": "bootstrapping",
|
||
"level": "info",
|
||
"message": "apt 包安装完成"
|
||
},
|
||
{
|
||
"at": "2026-05-02T19:11:44.714Z",
|
||
"phase": "bootstrapping",
|
||
"level": "info",
|
||
"message": "torch + transformers + accelerate + deepspeed + modelscope 安装完成"
|
||
},
|
||
{
|
||
"at": "2026-05-02T19:12:56.711Z",
|
||
"phase": "preprocessing",
|
||
"level": "info",
|
||
"message": "Bootstrap done on VM-0-11-ubuntu. raw=2 files. Ready to start training."
|
||
},
|
||
{
|
||
"at": "2026-05-02T19:34:41.648Z",
|
||
"phase": "training",
|
||
"level": "info",
|
||
"message": "start-training.sh launched on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-02T19:35:48.820Z",
|
||
"phase": "done",
|
||
"level": "info",
|
||
"message": "train.py exited cleanly on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-03T05:33:56.539Z",
|
||
"phase": "training",
|
||
"level": "info",
|
||
"message": "start-training.sh launched on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-03T05:37:02.879Z",
|
||
"phase": "done",
|
||
"level": "info",
|
||
"message": "train.py exited cleanly on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-03T05:41:36.877Z",
|
||
"phase": "training",
|
||
"level": "info",
|
||
"message": "start-training.sh launched on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-03T05:42:46.428Z",
|
||
"phase": "done",
|
||
"level": "info",
|
||
"message": "train.py exited cleanly on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-03T06:17:34.391Z",
|
||
"phase": "bootstrapping",
|
||
"level": "info",
|
||
"message": "commit 9884be1f5bdc7c847fdf24038c44aacc7a7d912f 合并到 main, 自动训练流程启动"
|
||
},
|
||
{
|
||
"at": "2026-05-03T06:18:14.923Z",
|
||
"phase": "bootstrapping",
|
||
"level": "info",
|
||
"message": "Bootstrap started on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-03T06:18:27.782Z",
|
||
"phase": "bootstrapping",
|
||
"level": "info",
|
||
"message": "torch + transformers + accelerate + deepspeed + modelscope 安装完成"
|
||
},
|
||
{
|
||
"at": "2026-05-03T06:19:36.454Z",
|
||
"phase": "preprocessing",
|
||
"level": "info",
|
||
"message": "running preprocess-corpus.py"
|
||
},
|
||
{
|
||
"at": "2026-05-03T06:19:54.961Z",
|
||
"phase": "preprocessing",
|
||
"level": "info",
|
||
"message": "SFT 样本数=11456 写入 /data/guanghu/processed/sft.jsonl"
|
||
},
|
||
{
|
||
"at": "2026-05-03T06:20:05.467Z",
|
||
"phase": "training",
|
||
"level": "info",
|
||
"message": "deepspeed --num_gpus=4 train.py"
|
||
},
|
||
{
|
||
"at": "2026-05-03T06:24:24.164Z",
|
||
"phase": "done",
|
||
"level": "info",
|
||
"message": "train.py exited cleanly on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-03T06:25:17.707Z",
|
||
"phase": "training",
|
||
"level": "info",
|
||
"message": "start-training.sh launched on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-03T06:30:35.282Z",
|
||
"phase": "done",
|
||
"level": "info",
|
||
"message": "train.py exited cleanly on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-03T06:48:05.610Z",
|
||
"phase": "bootstrapping",
|
||
"level": "info",
|
||
"message": "commit 7961cfb4610270537130ef3675c9bba44b0e7649 合并到 main, 自动训练流程启动"
|
||
},
|
||
{
|
||
"at": "2026-05-03T06:48:40.985Z",
|
||
"phase": "bootstrapping",
|
||
"level": "info",
|
||
"message": "Bootstrap started on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-03T06:49:50.545Z",
|
||
"phase": "bootstrapping",
|
||
"level": "info",
|
||
"message": "ModelScope snapshot_download qwen/Qwen2.5-7B"
|
||
},
|
||
{
|
||
"at": "2026-05-03T06:51:00.558Z",
|
||
"phase": "training",
|
||
"level": "info",
|
||
"message": "deepspeed --num_gpus=4 train.py"
|
||
},
|
||
{
|
||
"at": "2026-05-03T06:52:15.860Z",
|
||
"phase": "bootstrapping",
|
||
"level": "info",
|
||
"message": "ModelScope snapshot_download qwen/Qwen2.5-7B"
|
||
},
|
||
{
|
||
"at": "2026-05-03T06:53:19.509Z",
|
||
"phase": "preprocessing",
|
||
"level": "info",
|
||
"message": "Bootstrap done on VM-0-11-ubuntu. raw=2 files. Ready to start training."
|
||
},
|
||
{
|
||
"at": "2026-05-03T06:55:29.729Z",
|
||
"phase": "training",
|
||
"level": "info",
|
||
"message": "start-training.sh launched on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-03T06:56:36.190Z",
|
||
"phase": "done",
|
||
"level": "info",
|
||
"message": "train.py exited cleanly on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-03T07:13:49.951Z",
|
||
"phase": "bootstrapping",
|
||
"level": "info",
|
||
"message": "commit 52f694cc468b9208e10c9fc44d48eb79a12f9a8c 合并到 main, 自动训练流程启动"
|
||
},
|
||
{
|
||
"at": "2026-05-03T07:14:39.031Z",
|
||
"phase": "bootstrapping",
|
||
"level": "info",
|
||
"message": "Bootstrap started on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-03T07:14:49.274Z",
|
||
"phase": "bootstrapping",
|
||
"level": "info",
|
||
"message": "coscmd configured for sy-finetune-corpus-1317346199 (ap-guangzhou)"
|
||
},
|
||
{
|
||
"at": "2026-05-03T07:15:53.858Z",
|
||
"phase": "bootstrapping",
|
||
"level": "info",
|
||
"message": "Qwen2.5-7B 已就绪 14.20 GiB at /data/guanghu/models/Qwen2.5-7B"
|
||
},
|
||
{
|
||
"at": "2026-05-03T07:16:12.460Z",
|
||
"phase": "preprocessing",
|
||
"level": "info",
|
||
"message": "SFT 样本数=11456 写入 /data/guanghu/processed/sft.jsonl"
|
||
},
|
||
{
|
||
"at": "2026-05-03T07:17:17.765Z",
|
||
"phase": "done",
|
||
"level": "info",
|
||
"message": "train.py exited cleanly on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-03T07:20:28.940Z",
|
||
"phase": "bootstrapping",
|
||
"level": "info",
|
||
"message": "Bootstrap started on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-03T07:21:32.568Z",
|
||
"phase": "downloading-corpus",
|
||
"level": "info",
|
||
"message": "raw=2 files downloaded to /data/guanghu/raw"
|
||
},
|
||
{
|
||
"at": "2026-05-03T07:22:46.429Z",
|
||
"phase": "preprocessing",
|
||
"level": "info",
|
||
"message": "Bootstrap done on VM-0-11-ubuntu. raw=2 files. Ready to start training."
|
||
},
|
||
{
|
||
"at": "2026-05-03T07:24:00.244Z",
|
||
"phase": "training",
|
||
"level": "info",
|
||
"message": "start-training.sh launched on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-03T07:25:11.486Z",
|
||
"phase": "done",
|
||
"level": "info",
|
||
"message": "train.py exited cleanly on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-03T07:28:39.771Z",
|
||
"phase": "training",
|
||
"level": "info",
|
||
"message": "start-training.sh launched on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-03T07:53:32.974Z",
|
||
"phase": "done",
|
||
"level": "info",
|
||
"message": "train.py exited cleanly on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-03T08:11:20.490Z",
|
||
"phase": "training",
|
||
"level": "info",
|
||
"message": "start-training.sh launched on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-03T08:36:12.139Z",
|
||
"phase": "done",
|
||
"level": "info",
|
||
"message": "train.py exited cleanly on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-03T08:54:57.053Z",
|
||
"phase": "training",
|
||
"level": "info",
|
||
"message": "start-training.sh launched on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-03T08:56:05.810Z",
|
||
"phase": "done",
|
||
"level": "info",
|
||
"message": "train.py exited cleanly on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-03T08:58:42.082Z",
|
||
"phase": "training",
|
||
"level": "info",
|
||
"message": "start-training.sh launched on VM-0-11-ubuntu"
|
||
},
|
||
{
|
||
"at": "2026-05-03T09:29:05.896Z",
|
||
"phase": "done",
|
||
"level": "info",
|
||
"message": "train.py exited cleanly on VM-0-11-ubuntu"
|
||
}
|
||
],
|
||
"_timeline_doc": "最新事件在尾部 · 渲染时取最近 10 条倒序展示 · 由 progress-reporter 追加"
|
||
}
|