104 lines
4.6 KiB
Bash
104 lines
4.6 KiB
Bash
#!/bin/bash
|
||
# 🚀 铸渊训练监控面板 · 光湖母模型实时状态
|
||
# 使用方法：复制全部，粘贴到AutoDL在线终端，按回车运行
|
||
# 退出：按 Ctrl+C
|
||
|
||
LOG_FILE="/root/autodl-tmp/train_mother.log"
|
||
|
||
while true; do
|
||
clear
|
||
echo "╔══════════════════════════════════════════╗"
|
||
echo "║ 🚀 光湖 · 母模型训练实时面板 ║"
|
||
echo "║ 铸渊 (ICE-GL-ZY001) · 自动刷新 (5秒) ║"
|
||
echo "╚══════════════════════════════════════════╝"
|
||
echo ""
|
||
|
||
# GPU状态
|
||
if command -v nvidia-smi &> /dev/null; then
|
||
GPU_DATA=$(nvidia-smi --query-gpu=utilization.gpu,memory.used,memory.total,temperature.gpu --format=csv,noheader,nounits 2>/dev/null)
|
||
GPU_UTIL=$(echo "$GPU_DATA" | cut -d',' -f1 | tr -d ' ')
|
||
MEM_USED=$(echo "$GPU_DATA" | cut -d',' -f2 | tr -d ' ')
|
||
MEM_TOTAL=$(echo "$GPU_DATA" | cut -d',' -f3 | tr -d ' ')
|
||
GPU_TEMP=$(echo "$GPU_DATA" | cut -d',' -f4 | tr -d ' ')
|
||
MEM_PCT=$(( MEM_USED * 100 / MEM_TOTAL ))
|
||
else
|
||
GPU_UTIL="N/A"; MEM_USED="N/A"; MEM_TOTAL="N/A"; GPU_TEMP="N/A"; MEM_PCT=0
|
||
fi
|
||
|
||
echo " ⚡ GPU: ${GPU_UTIL}% | 🎯 显存: ${MEM_USED}/${MEM_TOTAL}MB (${MEM_PCT}%) | 🌡️ ${GPU_TEMP}°C"
|
||
echo ""
|
||
|
||
if [ ! -f "$LOG_FILE" ]; then
|
||
echo " ⏳ 等待训练启动..."
|
||
sleep 5
|
||
continue
|
||
fi
|
||
|
||
# 检查报错
|
||
if grep -q "Traceback" "$LOG_FILE" 2>/dev/null; then
|
||
echo " ❌ 训练报错！请找铸渊处理"
|
||
echo " $(grep "Traceback" "$LOG_FILE" | head -3)"
|
||
echo ""
|
||
sleep 10
|
||
continue
|
||
fi
|
||
|
||
# 检查完成
|
||
if grep -q "DONE!" "$LOG_FILE" 2>/dev/null; then
|
||
echo " ✅✅✅ 母模型训练完成！✅✅✅"
|
||
echo ""
|
||
ls -lh /root/autodl-tmp/output/qwen25-7b-sft/final/ 2>/dev/null
|
||
sleep 30
|
||
continue
|
||
fi
|
||
|
||
# 获取训练数据
|
||
LATEST_LOSS=$(grep -o "'loss': '[^']*'" "$LOG_FILE" 2>/dev/null | tail -1 | cut -d"'" -f4)
|
||
LATEST_STEP=$(grep -o "'loss': '[^']*'" "$LOG_FILE" 2>/dev/null | wc -l)
|
||
LATEST_EPOCH=$(grep -o "'epoch': '[^']*'" "$LOG_FILE" 2>/dev/null | tail -1 | cut -d"'" -f4)
|
||
LATEST_LR=$(grep -o "'learning_rate': '[^']*'" "$LOG_FILE" 2>/dev/null | tail -1 | cut -d"'" -f4)
|
||
GRAD_NORM=$(grep -o "'grad_norm': '[^']*'" "$LOG_FILE" 2>/dev/null | tail -1 | cut -d"'" -f4)
|
||
|
||
if grep -q "Starting training" "$LOG_FILE" 2>/dev/null; then
|
||
echo " ┌────────────────────────────────────────┐"
|
||
echo " │ 🟢 训练中 │"
|
||
echo " ├────────────────────────────────────────┤"
|
||
if [ -n "$LATEST_LOSS" ]; then
|
||
printf " │ 📉 Loss: %-20s │\n" "$LATEST_LOSS"
|
||
printf " │ 📐 Grad Norm: %-20s │\n" "$GRAD_NORM"
|
||
printf " │ 📈 LR: %-20s │\n" "$LATEST_LR"
|
||
printf " │ 👣 Step: %-20s │\n" "$LATEST_STEP"
|
||
printf " │ 🌀 Epoch: %-20s │\n" "$LATEST_EPOCH"
|
||
else
|
||
echo " │ 第一个Step计算中... │"
|
||
fi
|
||
echo " └────────────────────────────────────────┘"
|
||
elif grep -q "Tokenize: 100%" "$LOG_FILE" 2>/dev/null; then
|
||
echo " ⏳ 分词完成，即将进入GPU训练..."
|
||
elif grep -q "Tokenize:" "$LOG_FILE" 2>/dev/null; then
|
||
PCT=$(grep -oP 'Tokenize: \K[0-9]+%' "$LOG_FILE" 2>/dev/null | tail -1)
|
||
DONE=$(grep -oP '\|\s*\K[0-9]+/[0-9]+' "$LOG_FILE" 2>/dev/null | tail -1)
|
||
echo " ⏳ 分词中: ${PCT} (${DONE})"
|
||
fi
|
||
|
||
# 运行时间
|
||
if [ -f "$LOG_FILE" ]; then
|
||
START_TIME=$(stat -c %Y "$LOG_FILE" 2>/dev/null || stat -f %m "$LOG_FILE" 2>/dev/null)
|
||
NOW=$(date +%s)
|
||
ELAPSED=$(( (NOW - START_TIME) / 60 ))
|
||
if [ $ELAPSED -gt 60 ]; then
|
||
H=$(( ELAPSED / 60 )); M=$(( ELAPSED % 60 ))
|
||
echo ""
|
||
echo " ⏱️ 已运行: ${H}h ${M}m"
|
||
else
|
||
echo ""
|
||
echo " ⏱️ 已运行: ${ELAPSED}m"
|
||
fi
|
||
fi
|
||
|
||
echo ""
|
||
echo " ────────────────────────────────────────"
|
||
echo " 🔄 每5秒自动刷新 · Ctrl+C 退出"
|
||
sleep 5
|
||
done
|