476 lines
14 KiB
JavaScript
476 lines
14 KiB
JavaScript
/*
|
||
* ════════════════════════════════════════════════════════════════
|
||
* 光湖 · 终端守望者 (Terminal Watcher)
|
||
* 铸渊放在服务器上的"眼睛"——实时采集状态，推送到Gitea
|
||
* Sovereign: TCS-0002∞ · 守护: ICE-GL-ZY001
|
||
* ════════════════════════════════════════════════════════════════
|
||
*
|
||
* 用法:
|
||
* node watcher.js # 前台运行
|
||
* pm2 start watcher.js --name terminal-watcher # PM2守护
|
||
*
|
||
* 铸渊在Codebuddy中通过 Gitea API 读取快照文件:
|
||
* .runtime/terminal-snapshots/{server-id}/latest.json
|
||
*
|
||
* ════════════════════════════════════════════════════════════════
|
||
*/
|
||
"use strict";
|
||
|
||
const fs = require("fs");
|
||
const path = require("path");
|
||
const { execSync } = require("child_process");
|
||
const http = require("http");
|
||
const https = require("https");
|
||
|
||
// ─── 配置 ─────────────────────────────────────────────────────
|
||
const CONFIG_PATH = path.join(__dirname, "config.json");
|
||
const config = JSON.parse(fs.readFileSync(CONFIG_PATH, "utf8"));
|
||
|
||
const INTERVAL = (config.interval_seconds || 30) * 1000;
|
||
const SERVER_ID = config.server_id || "UNKNOWN";
|
||
|
||
// ─── 采集器 ───────────────────────────────────────────────────
|
||
|
||
/**
|
||
* 系统资源采集
|
||
*/
|
||
function collectSystem() {
|
||
try {
|
||
const memInfo = execSync("free -m", { encoding: "utf8" });
|
||
const memLines = memInfo.split("\n");
|
||
const memParts = memLines[1].split(/\s+/);
|
||
const memTotal = parseInt(memParts[1], 10);
|
||
const memUsed = parseInt(memParts[2], 10);
|
||
const memAvail = parseInt(memParts[6], 10);
|
||
|
||
const diskInfo = execSync("df -h /", { encoding: "utf8" });
|
||
const diskLine = diskInfo.split("\n")[1].split(/\s+/);
|
||
const diskPercent = parseInt(diskLine[4], 10);
|
||
|
||
const loadAvg = execSync("cat /proc/loadavg", { encoding: "utf8" })
|
||
.trim()
|
||
.split(" ")
|
||
.slice(0, 3)
|
||
.map(Number);
|
||
|
||
// CPU使用率（简易采样）
|
||
let cpuPercent = 0;
|
||
try {
|
||
const topOut = execSync("top -bn1 | head -3", { encoding: "utf8", timeout: 5000 });
|
||
const cpuLine = topOut.split("\n").find((l) => l.includes("Cpu(s)"));
|
||
if (cpuLine) {
|
||
const idle = parseFloat(cpuLine.match(/(\d+\.?\d*)\s*id/)?.[1] || "0");
|
||
cpuPercent = Math.round((100 - idle) * 10) / 10;
|
||
}
|
||
} catch {
|
||
// top不可用，用load average估算
|
||
cpuPercent = Math.round(loadAvg[0] * 50) / 10;
|
||
}
|
||
|
||
return {
|
||
cpu_percent: cpuPercent,
|
||
memory_total_mb: memTotal,
|
||
memory_used_mb: memUsed,
|
||
memory_available_mb: memAvail,
|
||
memory_percent: Math.round((memUsed / memTotal) * 100),
|
||
disk_used_percent: diskPercent,
|
||
load_avg: loadAvg,
|
||
uptime: execSync("cat /proc/uptime", { encoding: "utf8" }).split(" ")[0] + "s"
|
||
};
|
||
} catch (e) {
|
||
return { error: e.message };
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 服务状态采集
|
||
*/
|
||
function collectServices() {
|
||
const services = {};
|
||
|
||
// systemd 服务
|
||
const systemdList = config.systemd_services || ["nginx", "gitea", "gitea-runner"];
|
||
for (const svc of systemdList) {
|
||
try {
|
||
const status = execSync(`systemctl is-active ${svc} 2>/dev/null`, { encoding: "utf8" }).trim();
|
||
services[svc] = status;
|
||
} catch {
|
||
services[svc] = "unknown";
|
||
}
|
||
}
|
||
|
||
// PM2 进程
|
||
const pm2Processes = [];
|
||
try {
|
||
const pm2Out = execSync("pm2 jlist 2>/dev/null", { encoding: "utf8", timeout: 5000 });
|
||
const pm2List = JSON.parse(pm2Out);
|
||
for (const proc of pm2List) {
|
||
pm2Processes.push({
|
||
name: proc.name,
|
||
status: proc.pm2_env?.status || "unknown",
|
||
memory_mb: Math.round((proc.monit?.memory || 0) / 1024 / 1024),
|
||
cpu_percent: Math.round((proc.monit?.cpu || 0) * 10) / 10,
|
||
restarts: proc.pm2_env?.restart_time || 0,
|
||
uptime: proc.pm2_env?.pm_uptime
|
||
? Math.round((Date.now() - proc.pm2_env.pm_uptime) / 1000 / 60) + "min"
|
||
: "unknown"
|
||
});
|
||
}
|
||
} catch {
|
||
// PM2不可用或没有进程
|
||
}
|
||
|
||
// Vault 健康检查
|
||
let vaultStatus = "unknown";
|
||
try {
|
||
const vaultResp = execSync(
|
||
"curl -sf -o /dev/null -w '%{http_code}' http://127.0.0.1:8080/admin/__healthz 2>/dev/null",
|
||
{ encoding: "utf8", timeout: 3000 }
|
||
);
|
||
vaultStatus = vaultResp.trim() === "200" ? "active" : `http_${vaultResp.trim()}`;
|
||
} catch {
|
||
vaultStatus = "inactive";
|
||
}
|
||
|
||
return {
|
||
systemd: services,
|
||
pm2: pm2Processes,
|
||
vault: vaultStatus
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 错误日志采集
|
||
*/
|
||
function collectLogs() {
|
||
const errors = [];
|
||
|
||
// systemd 服务最近错误
|
||
for (const svc of config.systemd_services || []) {
|
||
try {
|
||
const logs = execSync(
|
||
`journalctl -u ${svc} --since "5 min ago" --no-pager -n 3 --priority=err 2>/dev/null`,
|
||
{ encoding: "utf8", timeout: 5000 }
|
||
).trim();
|
||
if (logs) {
|
||
for (const line of logs.split("\n")) {
|
||
if (line.trim()) {
|
||
errors.push({ service: svc, message: line.trim().substring(0, 200) });
|
||
}
|
||
}
|
||
}
|
||
} catch {
|
||
// 没有错误日志
|
||
}
|
||
}
|
||
|
||
// PM2 错误日志
|
||
try {
|
||
for (const procName of config.pm2_processes || []) {
|
||
const pm2Err = execSync(
|
||
`pm2 logs ${procName} --nostream --lines 3 --err 2>/dev/null`,
|
||
{ encoding: "utf8", timeout: 5000 }
|
||
).trim();
|
||
if (pm2Err) {
|
||
for (const line of pm2Err.split("\n")) {
|
||
if (line.trim() && !line.includes("pm2 logs")) {
|
||
errors.push({ service: `pm2:${procName}`, message: line.trim().substring(0, 200) });
|
||
}
|
||
}
|
||
}
|
||
}
|
||
} catch {
|
||
// PM2日志不可用
|
||
}
|
||
|
||
return errors.slice(0, 10); // 最多保留10条
|
||
}
|
||
|
||
/**
|
||
* 部署信息采集
|
||
*/
|
||
function collectDeploy() {
|
||
const repoPaths = [
|
||
"/data/guanghulab/repo",
|
||
"/opt/zhuyuan-cn/repo-mirror/20260509-1849/guanghulab",
|
||
"/opt/guanghulab"
|
||
];
|
||
const repoPath = repoPaths.find(p => fs.existsSync(p)) || repoPaths[0];
|
||
|
||
try {
|
||
const latestCommit = execSync(`git -C ${repoPath} log -1 --oneline`, { encoding: "utf8" }).trim();
|
||
const branch = execSync(`git -C ${repoPath} branch --show-current`, { encoding: "utf8" }).trim();
|
||
const commitTime = execSync(`git -C ${repoPath} log -1 --format=%ci`, { encoding: "utf8" }).trim();
|
||
|
||
return {
|
||
repo_path: repoPath,
|
||
branch: branch,
|
||
latest_commit: latestCommit.split(" ")[0],
|
||
latest_message: latestCommit.substring(latestCommit.indexOf(" ") + 1),
|
||
commit_time: commitTime
|
||
};
|
||
} catch {
|
||
return { repo_path: repoPath, error: "repository not found at " + repoPath };
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Runner状态采集
|
||
*/
|
||
function collectRunner() {
|
||
try {
|
||
const runnerStatus = execSync("systemctl is-active gitea-runner 2>/dev/null", {
|
||
encoding: "utf8"
|
||
}).trim();
|
||
|
||
let runnerVersion = "unknown";
|
||
try {
|
||
runnerVersion = execSync("/usr/local/bin/gitea-runner --version 2>&1", { encoding: "utf8" })
|
||
.trim()
|
||
.split("\n")[0];
|
||
} catch {
|
||
// 版本获取失败
|
||
}
|
||
|
||
// 最近运行的任务
|
||
let lastJob = "unknown";
|
||
try {
|
||
const giteaToken = process.env[config.gitea.token_env] || "";
|
||
if (giteaToken) {
|
||
// 通过API查询（暂用占位）
|
||
lastJob = "api_not_implemented";
|
||
}
|
||
} catch {
|
||
// API查询失败
|
||
}
|
||
|
||
return {
|
||
status: runnerStatus,
|
||
version: runnerVersion,
|
||
last_job: lastJob
|
||
};
|
||
} catch {
|
||
return { status: "not_installed" };
|
||
}
|
||
}
|
||
|
||
// ─── 上报器 ───────────────────────────────────────────────────
|
||
|
||
/**
|
||
* 写入本地文件
|
||
*/
|
||
function reportToFile(snapshot) {
|
||
const localPath = config.local?.snapshot_path;
|
||
if (!localPath) return;
|
||
|
||
try {
|
||
const dir = path.dirname(localPath);
|
||
if (!fs.existsSync(dir)) {
|
||
fs.mkdirSync(dir, { recursive: true });
|
||
}
|
||
|
||
// 写入最新快照
|
||
fs.writeFileSync(localPath, JSON.stringify(snapshot, null, 2));
|
||
|
||
// 写入历史快照（保留最近50个）
|
||
const historyDir = config.local?.history_path;
|
||
if (historyDir) {
|
||
if (!fs.existsSync(historyDir)) {
|
||
fs.mkdirSync(historyDir, { recursive: true });
|
||
}
|
||
const historyFile = path.join(
|
||
historyDir,
|
||
`${new Date().toISOString().replace(/[:.]/g, "-")}.json`
|
||
);
|
||
fs.writeFileSync(historyFile, JSON.stringify(snapshot, null, 2));
|
||
|
||
// 清理旧文件
|
||
const files = fs
|
||
.readdirSync(historyDir)
|
||
.filter((f) => f.endsWith(".json"))
|
||
.sort()
|
||
.reverse();
|
||
for (let i = 50; i < files.length; i++) {
|
||
fs.unlinkSync(path.join(historyDir, files[i]));
|
||
}
|
||
}
|
||
} catch (e) {
|
||
console.error("[watcher] 本地文件写入失败:", e.message);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 推送到Gitea
|
||
*/
|
||
function reportToGitea(snapshot) {
|
||
if (!config.gitea) return;
|
||
|
||
const token = process.env[config.gitea.token_env] || "";
|
||
if (!token) {
|
||
console.warn("[watcher] GITEA_TOKEN 未设置，跳过Gitea上报");
|
||
return;
|
||
}
|
||
|
||
const { url, owner, repo, snapshot_path } = config.gitea;
|
||
const content = Buffer.from(JSON.stringify(snapshot, null, 2)).toString("base64");
|
||
const apiPath = `/api/v1/repos/${owner}/${repo}/contents/${snapshot_path}/latest.json`;
|
||
|
||
const payload = JSON.stringify({
|
||
content: content,
|
||
message: `📓 终端快照 ${SERVER_ID} ${new Date().toISOString().substring(0, 16)}`,
|
||
branch: "main"
|
||
});
|
||
|
||
// 先尝试获取文件sha（用于更新）
|
||
const getOptions = {
|
||
hostname: url.replace("https://", "").replace("http://", ""),
|
||
path: apiPath,
|
||
method: "GET",
|
||
headers: { Authorization: `token ${token}` }
|
||
};
|
||
|
||
const httpModule = url.startsWith("https") ? https : http;
|
||
|
||
const req = httpModule.request(getOptions, (res) => {
|
||
let body = "";
|
||
res.on("data", (chunk) => (body += chunk));
|
||
res.on("end", () => {
|
||
let sha = null;
|
||
try {
|
||
const data = JSON.parse(body);
|
||
sha = data.sha;
|
||
} catch {
|
||
// 文件不存在，创建新文件
|
||
}
|
||
|
||
// 发送更新请求
|
||
const updatePayload = JSON.parse(payload);
|
||
if (sha) updatePayload.sha = sha;
|
||
|
||
const putOptions = {
|
||
hostname: getOptions.hostname,
|
||
path: apiPath,
|
||
method: "PUT",
|
||
headers: {
|
||
Authorization: `token ${token}`,
|
||
"Content-Type": "application/json",
|
||
"Content-Length": Buffer.byteLength(JSON.stringify(updatePayload))
|
||
}
|
||
};
|
||
|
||
const putReq = httpModule.request(putOptions, (putRes) => {
|
||
let putBody = "";
|
||
putRes.on("data", (chunk) => (putBody += chunk));
|
||
putRes.on("end", () => {
|
||
if (putRes.statusCode >= 200 && putRes.statusCode < 300) {
|
||
// 成功
|
||
} else {
|
||
console.error(
|
||
`[watcher] Gitea推送失败: ${putRes.statusCode} ${putBody.substring(0, 100)}`
|
||
);
|
||
}
|
||
});
|
||
});
|
||
|
||
putReq.on("error", (e) => {
|
||
console.error("[watcher] Gitea推送错误:", e.message);
|
||
});
|
||
|
||
putReq.write(JSON.stringify(updatePayload));
|
||
putReq.end();
|
||
});
|
||
});
|
||
|
||
req.on("error", () => {
|
||
// Gitea不可达，静默失败
|
||
});
|
||
|
||
req.setTimeout(10000, () => {
|
||
req.destroy();
|
||
});
|
||
|
||
req.end();
|
||
}
|
||
|
||
// ─── 主循环 ───────────────────────────────────────────────────
|
||
|
||
function collect() {
|
||
const snapshot = {
|
||
server_id: SERVER_ID,
|
||
server_name: config.server_name || SERVER_ID,
|
||
timestamp: new Date().toISOString(),
|
||
system: config.collectors?.system !== false ? collectSystem() : null,
|
||
services: config.collectors?.services !== false ? collectServices() : null,
|
||
errors: config.collectors?.logs !== false ? collectLogs() : null,
|
||
deploy: config.collectors?.deploy !== false ? collectDeploy() : null,
|
||
runner: config.collectors?.runner !== false ? collectRunner() : null
|
||
};
|
||
|
||
// 告警检测
|
||
const alerts = [];
|
||
if (snapshot.system && !snapshot.system.error) {
|
||
const thresholds = config.alerts || {};
|
||
if (snapshot.system.cpu_percent > (thresholds.cpu_percent || 90)) {
|
||
alerts.push(`CPU ${snapshot.system.cpu_percent}% 超过阈值 ${thresholds.cpu_percent || 90}%`);
|
||
}
|
||
if (snapshot.system.memory_percent > (thresholds.memory_percent || 90)) {
|
||
alerts.push(
|
||
`内存 ${snapshot.system.memory_percent}% 超过阈值 ${thresholds.memory_percent || 90}%`
|
||
);
|
||
}
|
||
if (snapshot.system.disk_used_percent > (thresholds.disk_percent || 85)) {
|
||
alerts.push(
|
||
`磁盘 ${snapshot.system.disk_used_percent}% 超过阈值 ${thresholds.disk_percent || 85}%`
|
||
);
|
||
}
|
||
}
|
||
if (snapshot.services) {
|
||
for (const [name, status] of Object.entries(snapshot.services.systemd || {})) {
|
||
if (status !== "active" && config.alerts?.service_down !== false) {
|
||
alerts.push(`服务 ${name} 状态异常: ${status}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
snapshot.alerts = alerts;
|
||
snapshot.alert_count = alerts.length;
|
||
|
||
return snapshot;
|
||
}
|
||
|
||
function tick() {
|
||
try {
|
||
const snapshot = collect();
|
||
|
||
// 上报
|
||
if (config.reporters?.file !== false) {
|
||
reportToFile(snapshot);
|
||
}
|
||
if (config.reporters?.gitea !== false) {
|
||
reportToGitea(snapshot);
|
||
}
|
||
|
||
// 简洁日志
|
||
const cpu = snapshot.system?.cpu_percent || "?";
|
||
const mem = snapshot.system?.memory_percent || "?";
|
||
const alerts = snapshot.alert_count || 0;
|
||
const alertMark = alerts > 0 ? ` ⚠️${alerts}告警` : "";
|
||
console.log(
|
||
`[watcher] ${snapshot.timestamp.substring(11, 19)} CPU:${cpu}% MEM:${mem}%${alertMark}`
|
||
);
|
||
} catch (e) {
|
||
console.error("[watcher] 采集失败:", e.message);
|
||
}
|
||
}
|
||
|
||
// ─── 启动 ─────────────────────────────────────────────────────
|
||
|
||
console.log(`[watcher] 终端守望者启动 · ${SERVER_ID} · 间隔 ${INTERVAL / 1000}s`);
|
||
console.log("[watcher] 铸渊 · ICE-GL-ZY001 · TCS-0002∞");
|
||
|
||
// 首次立即采集
|
||
tick();
|
||
|
||
// 定时采集
|
||
setInterval(tick, INTERVAL);
|