26 lines
996 B
Python
26 lines
996 B
Python
#!/usr/bin/env python3
|
|
import subprocess, json, re
|
|
from datetime import datetime, timezone
|
|
|
|
DATA_FILE = '/opt/guanghulab-repo/homepage/training-status.json'
|
|
SSH = ['sshpass','-p','HkM43lFVUIsc','ssh','-o','StrictHostKeyChecking=no',
|
|
'-o','ConnectTimeout=10','root@connect.westd.seetacloud.com','-p','23647']
|
|
|
|
try:
|
|
r = subprocess.run(SSH + ['tail','-5','/root/autodl-tmp/train_coder.log'],
|
|
capture_output=True, text=True, timeout=25).stdout
|
|
sm = re.search(r'(\d+)/(\d+)', r.split(chr(10))[-1] if chr(10) in r[-5:] else r)
|
|
lm = re.search(r"'loss':\s*'([\d.]+)'", r)
|
|
step = int(sm.group(1)) if sm else 0
|
|
total = int(sm.group(2)) if sm else 11838
|
|
loss = lm.group(1) if lm else '--'
|
|
except:
|
|
step, total, loss = 0, 11838, '--'
|
|
|
|
data = {'step': step, 'total': total, 'loss': loss,
|
|
'updated': datetime.now(timezone.utc).isoformat()+'Z'}
|
|
|
|
with open(DATA_FILE, 'w') as f:
|
|
json.dump(data, f)
|
|
print(f"Step {step} / {total}, Loss: {loss}")
|