diff --git a/scripts/train_logger.sh b/scripts/train_logger.sh new file mode 100644 index 0000000..84a9ac5 --- /dev/null +++ b/scripts/train_logger.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# ๐Ÿ“Š ๅ…‰ๆน– ยท ๆฏๆจกๅž‹่ฎญ็ปƒๅ…จๅކๅฒ่ฟฝ่ธชๅ™จ +# ๅ…ˆๆ˜พ็คบๆ‰€ๆœ‰ๅކๅฒ๏ผŒๅ†ๆŒ็ปญ่ฟฝๅŠ ๆ–ฐๆญฅ้ชค๏ผŒไธๅˆทๆ–ฐไธ่ฆ†็›– + +LOG_FILE="/root/autodl-tmp/train_mother.log" + +echo "" +echo "โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—" +echo "โ•‘ ๐Ÿ“Š ๅ…‰ๆน– ยท ๆฏๆจกๅž‹่ฎญ็ปƒๅ…จๅކๅฒ่ฟฝ่ธช โ•‘" +echo "โ•‘ ้“ธๆธŠ ยท ๅ…ˆๆ˜พ็คบๅทฒๆœ‰ๅކๅฒ โ†’ ๅ†ๆŒ็ปญ่ฟฝๅŠ ๆ–ฐๆญฅ โ•‘" +echo "โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•" +echo "" + +# โ”€โ”€โ”€ ็ฌฌไธ€ๆญฅ๏ผšๆ˜พ็คบๅทฒๆœ‰็š„ๅ…จ้ƒจๅކๅฒๆญฅ้ชค โ”€โ”€โ”€ +if [ -f "$LOG_FILE" ] && grep -q "'loss':" "$LOG_FILE" 2>/dev/null; then + echo " ๐Ÿ“œ ๅทฒๆœ‰่ฎญ็ปƒ่ฎฐๅฝ•๏ผš" + echo " โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€" + echo " ๆ—ถ้—ด Loss GradNorm LR Epoch" + echo " โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€" + grep "'loss':" "$LOG_FILE" | while IFS= read -r line; do + LOSS=$(echo "$line" | grep -o "'loss': '[^']*'" | cut -d"'" -f4) + GN=$(echo "$line" | grep -o "'grad_norm': '[^']*'" | cut -d"'" -f4) + LR=$(echo "$line" | grep -o "'learning_rate': '[^']*'" | cut -d"'" -f4) + EPOCH=$(echo "$line" | grep -o "'epoch': '[^']*'" | cut -d"'" -f4) + echo " --:--:-- ${LOSS} ${GN} ${LR} ${EPOCH}" + done + echo " โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€" + echo "" +fi + +if [ -f "$LOG_FILE" ] && grep -q "Traceback" "$LOG_FILE" 2>/dev/null; then + echo " โŒ ๆฃ€ๆต‹ๅˆฐ่ฎญ็ปƒๆŠฅ้”™๏ผๆœ€ๅŽ้”™่ฏฏ๏ผš" + grep "Traceback" "$LOG_FILE" | tail -3 + echo ""; exit 1 +fi + +if [ -f "$LOG_FILE" ] && grep -q "DONE!" "$LOG_FILE" 2>/dev/null; then + echo " โœ…โœ…โœ… ่ฎญ็ปƒๅทฒๅฎŒๆˆ๏ผๆจกๅž‹ไฟๅญ˜ไฝ็ฝฎ๏ผš" + ls -lh /root/autodl-tmp/output/qwen25-7b-sft/final/ 2>/dev/null + echo ""; exit 0 +fi + +echo " ๐Ÿ”„ ็ญ‰ๅพ…ๆ–ฐ่ฎญ็ปƒๆญฅ้ชค...๏ผˆCtrl+C ้€€ๅ‡บ๏ผ‰" +echo " โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€" + +LAST_COUNT=$(grep -c "'loss':" "$LOG_FILE" 2>/dev/null || echo 0) + +while true; do + NEW_COUNT=$(grep -c "'loss':" "$LOG_FILE" 2>/dev/null || echo 0) + if [ "$NEW_COUNT" -gt "$LAST_COUNT" ]; then + tail -n +$(( $(wc -l < "$LOG_FILE") - 10 )) "$LOG_FILE" 2>/dev/null | grep "'loss':" | tail -1 | while IFS= read -r line; do + LOSS=$(echo "$line" | grep -o "'loss': '[^']*'" | cut -d"'" -f4) + GN=$(echo "$line" | grep -o "'grad_norm': '[^']*'" | cut -d"'" -f4) + LR=$(echo "$line" | grep -o "'learning_rate': '[^']*'" | cut -d"'" -f4) + EPOCH=$(echo "$line" | grep -o "'epoch': '[^']*'" | cut -d"'" -f4) + GPU=$(nvidia-smi --query-gpu=utilization.gpu,memory.used,temperature.gpu --format=csv,noheader,nounits 2>/dev/null) + GU=$(echo "$GPU"|cut -d',' -f1|tr -d ' '); MT=$(echo "$GPU"|cut -d',' -f2|tr -d ' '); GT=$(echo "$GPU"|cut -d',' -f3|tr -d ' ') + printf " %s ๐Ÿ“‰%-8s ๐Ÿ“%-8s ๐Ÿ“ˆ%-12s ๐ŸŒ€%-8s โšก%s%% ๐ŸŒก๏ธ%sยฐC\n" "$(date '+%H:%M:%S')" "$LOSS" "$GN" "$LR" "$EPOCH" "$GU" "$GT" + done + LAST_COUNT=$NEW_COUNT + fi + if grep -q "Traceback" "$LOG_FILE" 2>/dev/null; then echo ""; echo " โŒโŒโŒ ่ฎญ็ปƒๆŠฅ้”™๏ผ่ฏทๆ‰พ้“ธๆธŠ"; exit 1; fi + if grep -q "DONE!" "$LOG_FILE" 2>/dev/null; then echo ""; echo " โœ…โœ…โœ… ่ฎญ็ปƒๅฎŒๆˆ๏ผ"; ls -lh /root/autodl-tmp/output/qwen25-7b-sft/final/ 2>/dev/null; exit 0; fi + sleep 5 +done