fetch_train v4: fix phase/epoch/step detection for distill training

homepage/fetch_train.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """
-fetch_train.py v3 — 蒸馏+GPU实时进度同步
+fetch_train.py v4 — 蒸馏+GPU实时进度同步 (FIXED: phase/epoch/step detection)
 """
 import subprocess, json, re
 from datetime import datetime, timezone
@@ -17,29 +17,51 @@ def ssh(cmd_list):
         return ''
 
 def get_distill_status():
-    raw = ssh(['tail','-50','/root/autodl-tmp/distill_mother.log'])
+    # Use tail -300 to ensure we catch epoch print + loss lines
+    raw = ssh(['tail','-300','/root/autodl-tmp/distill_mother.log'])
     if not raw:
         return {}, 'no_log'
     lines = raw.replace('\r', '\n').split('\n')
 
-    if 'DONE' in raw:
-        return {'phase': 'completed'}, 'done'
-    if 'Train' in raw:
+    # Phase detection: look for tqdm progress bar pattern EpX: X%|
+    if 'ALL DONE' in raw or 'ALL VERIFIED' in raw:
+        return {'phase': 'completed'}, 'completed'
+    
+    # Check if tqdm is running (EpX: XX%| pattern)
+    has_progress = False
+    for line in lines:
+        if re.search(r'Ep\d+:\s+\d+%\|', line):
+            has_progress = True
+            break
+    
+    if has_progress:
+        phase = 'training'
+    elif 'Train' in raw or 'Training' in raw:
         phase = 'training'
     elif 'Load models' in raw or 'Loading weights' in raw:
         phase = 'loading_models'
     elif 'Tokenize' in raw:
         phase = 'tokenizing'
+    elif 'Save final' in raw or 'Upload' in raw:
+        phase = 'saving'
     else:
-        phase = 'unknown'
+        phase = 'running'
 
+    # Epoch: try "EpX" from tqdm first, then "Epoch X/Y" from print
     epoch = None; total_epoch = 3
     for line in reversed(lines):
-        m = re.search(r'Epoch\s+(\d+)/(\d+)', line)
+        m = re.search(r'Ep(\d+):', line)
         if m:
-            epoch, total_epoch = int(m.group(1)), int(m.group(2))
+            epoch = int(m.group(1))
             break
+    if epoch is None:
+        for line in reversed(lines):
+            m = re.search(r'Epoch\s+(\d+)/(\d+)', line)
+            if m:
+                epoch, total_epoch = int(m.group(1)), int(m.group(2))
+                break
 
+    # Step & total: from tqdm progress bar  step/15781
     step = None; total_steps = None
     for line in reversed(lines):
         m = re.search(r'(\d+)/(\d+)\s+\[', line)
@@ -47,13 +69,15 @@ def get_distill_status():
             step, total_steps = int(m.group(1)), int(m.group(2))
             break
 
+    # Loss: from "step=X loss=Y.ZZZZ" lines
     loss = '--'
     for line in reversed(lines):
-        m = re.search(r"loss[=:]?\s*'?([0-9.]+)'?", line)
+        m = re.search(r"loss=([0-9.]+)", line)
         if m:
             loss = m.group(1)
             break
 
+    # ETA: from tqdm progress <HH:MM:SS>
     eta = '--'
     for line in reversed(lines):
         m = re.search(r'<([0-9]+:[0-9]+:[0-9]+)', line)
@@ -82,21 +106,27 @@ def main():
     info.update(status)
     info['gpu'] = get_gpu_status()
 
-    if info.get('phase') == 'completed':
+    if info.get('phase') in ('completed', 'done'):
         info['display_step'] = '完成'
         info['display_pct'] = 100.0
-    elif info.get('step') and info.get('total_steps') and info.get('epoch'):
-        ep_progress = (info['epoch'] - 1) * 100 / info['total_epoch']
-        ep_step = info['step'] / info['total_steps'] * 100 / info['total_epoch']
+    elif info.get('step') and info.get('total_steps'):
+        # Calculate overall progress: (epoch-1)*100/3 + step/total*100/3
+        ep = info.get('epoch') or 1
+        total_ep = info.get('total_epoch') or 3
+        ep_progress = (ep - 1) * 100 / total_ep
+        ep_step = info['step'] / info['total_steps'] * 100 / total_ep
         info['display_pct'] = round(ep_progress + ep_step, 1)
-        info['display_step'] = f"Ep{info['epoch']}/{info['total_epoch']} · {info['step']}/{info['total_steps']}"
+        if info.get('epoch'):
+            info['display_step'] = f"Ep{info['epoch']}/{info['total_epoch']} · {info['step']}/{info['total_steps']}"
+        else:
+            info['display_step'] = f"{info['step']}/{info['total_steps']}"
     else:
-        info['display_step'] = phase
+        info['display_step'] = phase if phase != 'no_log' else '等待数据...'
         info['display_pct'] = 0
 
     info['updated'] = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.000000+00:00Z')
     with open(DATA_FILE, 'w') as f:
-        json.dump(info, f)
+        json.dump(info, f, ensure_ascii=False)
     print(json.dumps(info, ensure_ascii=False, indent=2))
 
 if __name__ == '__main__':