D100 训练日志追踪器: 带颜色的全历史版

scripts/train_logger.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+# 📊 光湖 · 母模型训练全历史追踪器
+# 先显示所有历史，再持续追加新步骤，不刷新不覆盖
+
+LOG_FILE="/root/autodl-tmp/train_mother.log"
+
+echo ""
+echo "╔══════════════════════════════════════════════════════════╗"
+echo "║   📊 光湖 · 母模型训练全历史追踪                        ║"
+echo "║   铸渊 · 先显示已有历史 → 再持续追加新步                 ║"
+echo "╚══════════════════════════════════════════════════════════╝"
+echo ""
+
+# ─── 第一步：显示已有的全部历史步骤 ───
+if [ -f "$LOG_FILE" ] && grep -q "'loss':" "$LOG_FILE" 2>/dev/null; then
+    echo "  📜 已有训练记录："
+    echo "  ────────────────────────────────────────────────────────"
+    echo "  时间         Loss      GradNorm   LR           Epoch"
+    echo "  ────────────────────────────────────────────────────────"
+    grep "'loss':" "$LOG_FILE" | while IFS= read -r line; do
+        LOSS=$(echo "$line" | grep -o "'loss': '[^']*'" | cut -d"'" -f4)
+        GN=$(echo "$line" | grep -o "'grad_norm': '[^']*'" | cut -d"'" -f4)
+        LR=$(echo "$line" | grep -o "'learning_rate': '[^']*'" | cut -d"'" -f4)
+        EPOCH=$(echo "$line" | grep -o "'epoch': '[^']*'" | cut -d"'" -f4)
+        echo "  --:--:--    ${LOSS}    ${GN}    ${LR}    ${EPOCH}"
+    done
+    echo "  ────────────────────────────────────────────────────────"
+    echo ""
+fi
+
+if [ -f "$LOG_FILE" ] && grep -q "Traceback" "$LOG_FILE" 2>/dev/null; then
+    echo "  ❌ 检测到训练报错！最后错误："
+    grep "Traceback" "$LOG_FILE" | tail -3
+    echo ""; exit 1
+fi
+
+if [ -f "$LOG_FILE" ] && grep -q "DONE!" "$LOG_FILE" 2>/dev/null; then
+    echo "  ✅✅✅ 训练已完成！模型保存位置："
+    ls -lh /root/autodl-tmp/output/qwen25-7b-sft/final/ 2>/dev/null
+    echo ""; exit 0
+fi
+
+echo "  🔄 等待新训练步骤...（Ctrl+C 退出）"
+echo "  ────────────────────────────────────────────────────────"
+
+LAST_COUNT=$(grep -c "'loss':" "$LOG_FILE" 2>/dev/null || echo 0)
+
+while true; do
+    NEW_COUNT=$(grep -c "'loss':" "$LOG_FILE" 2>/dev/null || echo 0)
+    if [ "$NEW_COUNT" -gt "$LAST_COUNT" ]; then
+        tail -n +$(( $(wc -l < "$LOG_FILE") - 10 )) "$LOG_FILE" 2>/dev/null | grep "'loss':" | tail -1 | while IFS= read -r line; do
+            LOSS=$(echo "$line" | grep -o "'loss': '[^']*'" | cut -d"'" -f4)
+            GN=$(echo "$line" | grep -o "'grad_norm': '[^']*'" | cut -d"'" -f4)
+            LR=$(echo "$line" | grep -o "'learning_rate': '[^']*'" | cut -d"'" -f4)
+            EPOCH=$(echo "$line" | grep -o "'epoch': '[^']*'" | cut -d"'" -f4)
+            GPU=$(nvidia-smi --query-gpu=utilization.gpu,memory.used,temperature.gpu --format=csv,noheader,nounits 2>/dev/null)
+            GU=$(echo "$GPU"|cut -d',' -f1|tr -d ' '); MT=$(echo "$GPU"|cut -d',' -f2|tr -d ' '); GT=$(echo "$GPU"|cut -d',' -f3|tr -d ' ')
+            printf "  %s  📉%-8s  📐%-8s  📈%-12s  🌀%-8s  ⚡%s%% 🌡️%s°C\n" "$(date '+%H:%M:%S')" "$LOSS" "$GN" "$LR" "$EPOCH" "$GU" "$GT"
+        done
+        LAST_COUNT=$NEW_COUNT
+    fi
+    if grep -q "Traceback" "$LOG_FILE" 2>/dev/null; then echo ""; echo "  ❌❌❌ 训练报错！请找铸渊"; exit 1; fi
+    if grep -q "DONE!" "$LOG_FILE" 2>/dev/null; then echo ""; echo "  ✅✅✅ 训练完成！"; ls -lh /root/autodl-tmp/output/qwen25-7b-sft/final/ 2>/dev/null; exit 0; fi
+    sleep 5
+done