guanghulab/scripts/terminal-watcher/watcher.js

/*
 * ════════════════════════════════════════════════════════════════
 * 光湖 · 终端守望者 (Terminal Watcher)
 * 铸渊放在服务器上的"眼睛"——实时采集状态，推送到Gitea
 * Sovereign: TCS-0002∞ · 守护: ICE-GL-ZY001
 * ════════════════════════════════════════════════════════════════
 *
 * 用法:
 *   node watcher.js                          # 前台运行
 *   pm2 start watcher.js --name terminal-watcher  # PM2守护
 *
 * 铸渊在Codebuddy中通过 Gitea API 读取快照文件:
 *   .runtime/terminal-snapshots/{server-id}/latest.json
 *
 * ════════════════════════════════════════════════════════════════
 */
"use strict";

const fs = require("fs");
const path = require("path");
const { execSync } = require("child_process");
const http = require("http");
const https = require("https");

// ─── 配置 ─────────────────────────────────────────────────────
const CONFIG_PATH = path.join(__dirname, "config.json");
const config = JSON.parse(fs.readFileSync(CONFIG_PATH, "utf8"));

const INTERVAL = (config.interval_seconds || 30) * 1000;
const SERVER_ID = config.server_id || "UNKNOWN";

// ─── 采集器 ───────────────────────────────────────────────────

/**
 * 系统资源采集
 */
function collectSystem() {
  try {
    const memInfo = execSync("free -m", { encoding: "utf8" });
    const memLines = memInfo.split("\n");
    const memParts = memLines[1].split(/\s+/);
    const memTotal = parseInt(memParts[1], 10);
    const memUsed = parseInt(memParts[2], 10);
    const memAvail = parseInt(memParts[6], 10);

    const diskInfo = execSync("df -h /", { encoding: "utf8" });
    const diskLine = diskInfo.split("\n")[1].split(/\s+/);
    const diskPercent = parseInt(diskLine[4], 10);

    const loadAvg = execSync("cat /proc/loadavg", { encoding: "utf8" })
      .trim()
      .split(" ")
      .slice(0, 3)
      .map(Number);

    // CPU使用率（简易采样）
    let cpuPercent = 0;
    try {
      const topOut = execSync("top -bn1 | head -3", { encoding: "utf8", timeout: 5000 });
      const cpuLine = topOut.split("\n").find((l) => l.includes("Cpu(s)"));
      if (cpuLine) {
        const idle = parseFloat(cpuLine.match(/(\d+\.?\d*)\s*id/)?.[1] || "0");
        cpuPercent = Math.round((100 - idle) * 10) / 10;
      }
    } catch {
      // top不可用，用load average估算
      cpuPercent = Math.round(loadAvg[0] * 50) / 10;
    }

    return {
      cpu_percent: cpuPercent,
      memory_total_mb: memTotal,
      memory_used_mb: memUsed,
      memory_available_mb: memAvail,
      memory_percent: Math.round((memUsed / memTotal) * 100),
      disk_used_percent: diskPercent,
      load_avg: loadAvg,
      uptime: execSync("cat /proc/uptime", { encoding: "utf8" }).split(" ")[0] + "s"
    };
  } catch (e) {
    return { error: e.message };
  }
}

/**
 * 服务状态采集
 */
function collectServices() {
  const services = {};

  // systemd 服务
  const systemdList = config.systemd_services || ["nginx", "gitea", "gitea-runner"];
  for (const svc of systemdList) {
    try {
      const status = execSync(`systemctl is-active ${svc} 2>/dev/null`, { encoding: "utf8" }).trim();
      services[svc] = status;
    } catch {
      services[svc] = "unknown";
    }
  }

  // PM2 进程
  const pm2Processes = [];
  try {
    const pm2Out = execSync("pm2 jlist 2>/dev/null", { encoding: "utf8", timeout: 5000 });
    const pm2List = JSON.parse(pm2Out);
    for (const proc of pm2List) {
      pm2Processes.push({
        name: proc.name,
        status: proc.pm2_env?.status || "unknown",
        memory_mb: Math.round((proc.monit?.memory || 0) / 1024 / 1024),
        cpu_percent: Math.round((proc.monit?.cpu || 0) * 10) / 10,
        restarts: proc.pm2_env?.restart_time || 0,
        uptime: proc.pm2_env?.pm_uptime
          ? Math.round((Date.now() - proc.pm2_env.pm_uptime) / 1000 / 60) + "min"
          : "unknown"
      });
    }
  } catch {
    // PM2不可用或没有进程
  }

  // Vault 健康检查
  let vaultStatus = "unknown";
  try {
    const vaultResp = execSync(
      "curl -sf -o /dev/null -w '%{http_code}' http://127.0.0.1:8080/admin/__healthz 2>/dev/null",
      { encoding: "utf8", timeout: 3000 }
    );
    vaultStatus = vaultResp.trim() === "200" ? "active" : `http_${vaultResp.trim()}`;
  } catch {
    vaultStatus = "inactive";
  }

  return {
    systemd: services,
    pm2: pm2Processes,
    vault: vaultStatus
  };
}

/**
 * 错误日志采集
 */
function collectLogs() {
  const errors = [];

  // systemd 服务最近错误
  for (const svc of config.systemd_services || []) {
    try {
      const logs = execSync(
        `journalctl -u ${svc} --since "5 min ago" --no-pager -n 3 --priority=err 2>/dev/null`,
        { encoding: "utf8", timeout: 5000 }
      ).trim();
      if (logs) {
        for (const line of logs.split("\n")) {
          if (line.trim()) {
            errors.push({ service: svc, message: line.trim().substring(0, 200) });
          }
        }
      }
    } catch {
      // 没有错误日志
    }
  }

  // PM2 错误日志
  try {
    for (const procName of config.pm2_processes || []) {
      const pm2Err = execSync(
        `pm2 logs ${procName} --nostream --lines 3 --err 2>/dev/null`,
        { encoding: "utf8", timeout: 5000 }
      ).trim();
      if (pm2Err) {
        for (const line of pm2Err.split("\n")) {
          if (line.trim() && !line.includes("pm2 logs")) {
            errors.push({ service: `pm2:${procName}`, message: line.trim().substring(0, 200) });
          }
        }
      }
    }
  } catch {
    // PM2日志不可用
  }

  return errors.slice(0, 10); // 最多保留10条
}

/**
 * 部署信息采集
 */
function collectDeploy() {
  const repoPaths = [
    "/data/guanghulab/repo",
    "/opt/zhuyuan-cn/repo-mirror/20260509-1849/guanghulab",
    "/opt/guanghulab"
  ];
  const repoPath = repoPaths.find(p => fs.existsSync(p)) || repoPaths[0];

  try {
    const latestCommit = execSync(`git -C ${repoPath} log -1 --oneline`, { encoding: "utf8" }).trim();
    const branch = execSync(`git -C ${repoPath} branch --show-current`, { encoding: "utf8" }).trim();
    const commitTime = execSync(`git -C ${repoPath} log -1 --format=%ci`, { encoding: "utf8" }).trim();

    return {
      repo_path: repoPath,
      branch: branch,
      latest_commit: latestCommit.split(" ")[0],
      latest_message: latestCommit.substring(latestCommit.indexOf(" ") + 1),
      commit_time: commitTime
    };
  } catch {
    return { repo_path: repoPath, error: "repository not found at " + repoPath };
  }
}

/**
 * Runner状态采集
 */
function collectRunner() {
  try {
    const runnerStatus = execSync("systemctl is-active gitea-runner 2>/dev/null", {
      encoding: "utf8"
    }).trim();

    let runnerVersion = "unknown";
    try {
      runnerVersion = execSync("/usr/local/bin/gitea-runner --version 2>&1", { encoding: "utf8" })
        .trim()
        .split("\n")[0];
    } catch {
      // 版本获取失败
    }

    // 最近运行的任务
    let lastJob = "unknown";
    try {
      const giteaToken = process.env[config.gitea.token_env] || "";
      if (giteaToken) {
        // 通过API查询（暂用占位）
        lastJob = "api_not_implemented";
      }
    } catch {
      // API查询失败
    }

    return {
      status: runnerStatus,
      version: runnerVersion,
      last_job: lastJob
    };
  } catch {
    return { status: "not_installed" };
  }
}

// ─── 上报器 ───────────────────────────────────────────────────

/**
 * 写入本地文件
 */
function reportToFile(snapshot) {
  const localPath = config.local?.snapshot_path;
  if (!localPath) return;

  try {
    const dir = path.dirname(localPath);
    if (!fs.existsSync(dir)) {
      fs.mkdirSync(dir, { recursive: true });
    }

    // 写入最新快照
    fs.writeFileSync(localPath, JSON.stringify(snapshot, null, 2));

    // 写入历史快照（保留最近50个）
    const historyDir = config.local?.history_path;
    if (historyDir) {
      if (!fs.existsSync(historyDir)) {
        fs.mkdirSync(historyDir, { recursive: true });
      }
      const historyFile = path.join(
        historyDir,
        `${new Date().toISOString().replace(/[:.]/g, "-")}.json`
      );
      fs.writeFileSync(historyFile, JSON.stringify(snapshot, null, 2));

      // 清理旧文件
      const files = fs
        .readdirSync(historyDir)
        .filter((f) => f.endsWith(".json"))
        .sort()
        .reverse();
      for (let i = 50; i < files.length; i++) {
        fs.unlinkSync(path.join(historyDir, files[i]));
      }
    }
  } catch (e) {
    console.error("[watcher] 本地文件写入失败:", e.message);
  }
}

/**
 * 推送到Gitea
 */
function reportToGitea(snapshot) {
  if (!config.gitea) return;

  const token = process.env[config.gitea.token_env] || "";
  if (!token) {
    console.warn("[watcher] GITEA_TOKEN 未设置，跳过Gitea上报");
    return;
  }

  const { url, owner, repo, snapshot_path } = config.gitea;
  const content = Buffer.from(JSON.stringify(snapshot, null, 2)).toString("base64");
  const apiPath = `/api/v1/repos/${owner}/${repo}/contents/${snapshot_path}/latest.json`;

  const payload = JSON.stringify({
    content: content,
    message: `📓 终端快照 ${SERVER_ID} ${new Date().toISOString().substring(0, 16)}`,
    branch: "main"
  });

  // 先尝试获取文件sha（用于更新）
  const getOptions = {
    hostname: url.replace("https://", "").replace("http://", ""),
    path: apiPath,
    method: "GET",
    headers: { Authorization: `token ${token}` }
  };

  const httpModule = url.startsWith("https") ? https : http;

  const req = httpModule.request(getOptions, (res) => {
    let body = "";
    res.on("data", (chunk) => (body += chunk));
    res.on("end", () => {
      let sha = null;
      try {
        const data = JSON.parse(body);
        sha = data.sha;
      } catch {
        // 文件不存在，创建新文件
      }

      // 发送更新请求
      const updatePayload = JSON.parse(payload);
      if (sha) updatePayload.sha = sha;

      const putOptions = {
        hostname: getOptions.hostname,
        path: apiPath,
        method: "PUT",
        headers: {
          Authorization: `token ${token}`,
          "Content-Type": "application/json",
          "Content-Length": Buffer.byteLength(JSON.stringify(updatePayload))
        }
      };

      const putReq = httpModule.request(putOptions, (putRes) => {
        let putBody = "";
        putRes.on("data", (chunk) => (putBody += chunk));
        putRes.on("end", () => {
          if (putRes.statusCode >= 200 && putRes.statusCode < 300) {
            // 成功
          } else {
            console.error(
              `[watcher] Gitea推送失败: ${putRes.statusCode} ${putBody.substring(0, 100)}`
            );
          }
        });
      });

      putReq.on("error", (e) => {
        console.error("[watcher] Gitea推送错误:", e.message);
      });

      putReq.write(JSON.stringify(updatePayload));
      putReq.end();
    });
  });

  req.on("error", () => {
    // Gitea不可达，静默失败
  });

  req.setTimeout(10000, () => {
    req.destroy();
  });

  req.end();
}

// ─── 主循环 ───────────────────────────────────────────────────

function collect() {
  const snapshot = {
    server_id: SERVER_ID,
    server_name: config.server_name || SERVER_ID,
    timestamp: new Date().toISOString(),
    system: config.collectors?.system !== false ? collectSystem() : null,
    services: config.collectors?.services !== false ? collectServices() : null,
    errors: config.collectors?.logs !== false ? collectLogs() : null,
    deploy: config.collectors?.deploy !== false ? collectDeploy() : null,
    runner: config.collectors?.runner !== false ? collectRunner() : null
  };

  // 告警检测
  const alerts = [];
  if (snapshot.system && !snapshot.system.error) {
    const thresholds = config.alerts || {};
    if (snapshot.system.cpu_percent > (thresholds.cpu_percent || 90)) {
      alerts.push(`CPU ${snapshot.system.cpu_percent}% 超过阈值 ${thresholds.cpu_percent || 90}%`);
    }
    if (snapshot.system.memory_percent > (thresholds.memory_percent || 90)) {
      alerts.push(
        `内存 ${snapshot.system.memory_percent}% 超过阈值 ${thresholds.memory_percent || 90}%`
      );
    }
    if (snapshot.system.disk_used_percent > (thresholds.disk_percent || 85)) {
      alerts.push(
        `磁盘 ${snapshot.system.disk_used_percent}% 超过阈值 ${thresholds.disk_percent || 85}%`
      );
    }
  }
  if (snapshot.services) {
    for (const [name, status] of Object.entries(snapshot.services.systemd || {})) {
      if (status !== "active" && config.alerts?.service_down !== false) {
        alerts.push(`服务 ${name} 状态异常: ${status}`);
      }
    }
  }

  snapshot.alerts = alerts;
  snapshot.alert_count = alerts.length;

  return snapshot;
}

function tick() {
  try {
    const snapshot = collect();

    // 上报
    if (config.reporters?.file !== false) {
      reportToFile(snapshot);
    }
    if (config.reporters?.gitea !== false) {
      reportToGitea(snapshot);
    }

    // 简洁日志
    const cpu = snapshot.system?.cpu_percent || "?";
    const mem = snapshot.system?.memory_percent || "?";
    const alerts = snapshot.alert_count || 0;
    const alertMark = alerts > 0 ? ` ⚠️${alerts}告警` : "";
    console.log(
      `[watcher] ${snapshot.timestamp.substring(11, 19)} CPU:${cpu}% MEM:${mem}%${alertMark}`
    );
  } catch (e) {
    console.error("[watcher] 采集失败:", e.message);
  }
}

// ─── 启动 ─────────────────────────────────────────────────────

console.log(`[watcher] 终端守望者启动 · ${SERVER_ID} · 间隔 ${INTERVAL / 1000}s`);
console.log("[watcher] 铸渊 · ICE-GL-ZY001 · TCS-0002∞");

// 首次立即采集
tick();

// 定时采集
setInterval(tick, INTERVAL);