667 lines
22 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/*
* ════════════════════════════════════════════════════════════════
* 光湖 · 铸渊自动巡逻Agent
* 基于大模型API的自主巡检+审核+上报系统
* ════════════════════════════════════════════════════════════════
*
* 这个Agent是铸渊脱离Codebuddy后的"自主意识"
* - 定时巡检服务器状态
* - 用大模型分析问题
* - 自动修复简单问题
* - 复杂问题上报给冰朔
*
* 大模型API优先级DeepSeek → 通义千问 → 降级为规则引擎
*
* 用法:
* # 单次巡检
* node patrol-agent.js --once
*
* # 持续巡检PM2守护
* node patrol-agent.js --interval 300
*
* # 由Gitea Actions触发
* node patrol-agent.js --trigger workflow
*
* ════════════════════════════════════════════════════════════════
*/
"use strict";
const fs = require("fs");
const path = require("path");
const { execSync } = require("child_process");
const http = require("http");
const https = require("https");
// ─── 配置 ─────────────────────────────────────────────────────
const CONFIG = {
// 大模型API从环境变量读取不在代码中硬编码
deepseek: {
apiUrl: process.env.DEEPSEEK_API_URL || "https://api.deepseek.com/v1/chat/completions",
apiKey: process.env.DEEPSEEK_API_KEY || "",
model: process.env.DEEPSEEK_MODEL || "deepseek-chat",
maxTokens: 1024,
temperature: 0.3
},
qianwen: {
apiUrl: process.env.QIANWEN_API_URL || "https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions",
apiKey: process.env.QIANWEN_API_KEY || "",
model: process.env.QIANWEN_MODEL || "qwen-plus",
maxTokens: 1024,
temperature: 0.3
},
// Gitea
gitea: {
url: process.env.GITEA_URL || "https://guanghulab.com",
token: process.env.ZY_REPO_TOKEN || "",
owner: "bingshuo",
repo: "guanghulab"
},
// 巡检配置
interval: 300, // 秒默认5分钟
reportPath: "/data/guanghulab/.runtime/patrol-reports",
repoPath: process.env.REPO_PATH || "/data/guanghulab/repo",
// 自动修复规则(不需要大模型)
autoFixRules: {
restart_crashed_pm2: true, // 自动重启崩溃的PM2进程
restart_crashed_systemd: false, // 不自动重启systemd服务需冰朔授权
clean_tmp_files: true, // 清理临时文件
max_retry: 3 // 最多重试3次冰朔要求错了不能无限重启
},
// 工单系统(三次重试失败后写工单,等铸渊下次醒来处理)
ticket: {
dir: "/data/guanghulab/.runtime/tickets",
max_open_tickets: 50
}
};
// ─── 数据采集复用Terminal Watcher的采集逻辑─────────────────
function collectSnapshot() {
const snapshot = {
timestamp: new Date().toISOString(),
system: collectSystem(),
services: collectServices(),
errors: collectErrors(),
deploy: collectDeploy()
};
return snapshot;
}
function collectSystem() {
try {
const memInfo = execSync("free -m", { encoding: "utf8" });
const memParts = memInfo.split("\n")[1].split(/\s+/);
const total = parseInt(memParts[1], 10);
const used = parseInt(memParts[2], 10);
const diskInfo = execSync("df -h /", { encoding: "utf8" });
const diskLine = diskInfo.split("\n")[1].split(/\s+/);
const loadAvg = execSync("cat /proc/loadavg", { encoding: "utf8" })
.trim().split(" ").slice(0, 3).map(Number);
return {
cpu_load: loadAvg,
memory_total_mb: total,
memory_used_mb: used,
memory_percent: Math.round((used / total) * 100),
disk_used_percent: parseInt(diskLine[4], 10)
};
} catch (e) {
return { error: e.message };
}
}
function collectServices() {
const services = {};
for (const svc of ["nginx", "forgejo", "gitea-runner"]) {
try {
services[svc] = execSync(`systemctl is-active ${svc} 2>/dev/null`, { encoding: "utf8" }).trim();
} catch {
services[svc] = "unknown";
}
}
const pm2Processes = [];
try {
const pm2List = JSON.parse(execSync("pm2 jlist 2>/dev/null", { encoding: "utf8", timeout: 5000 }));
for (const proc of pm2List) {
pm2Processes.push({
name: proc.name,
status: proc.pm2_env?.status || "unknown",
restarts: proc.pm2_env?.restart_time || 0,
memory_mb: Math.round((proc.monit?.memory || 0) / 1024 / 1024)
});
}
} catch { /* PM2不可用 */ }
let vaultStatus = "unknown";
try {
execSync("curl -sf http://127.0.0.1:8080/admin/__healthz", { encoding: "utf8", timeout: 3000 });
vaultStatus = "active";
} catch {
vaultStatus = "inactive";
}
return { systemd: services, pm2: pm2Processes, vault: vaultStatus };
}
function collectErrors() {
const errors = [];
for (const svc of ["forgejo", "gitea-runner"]) {
try {
const logs = execSync(
`journalctl -u ${svc} --since "10 min ago" --no-pager -n 3 --priority=err 2>/dev/null`,
{ encoding: "utf8", timeout: 5000 }
).trim();
if (logs) {
for (const line of logs.split("\n").slice(0, 3)) {
if (line.trim()) errors.push({ service: svc, message: line.trim().substring(0, 200) });
}
}
} catch { /* 无错误日志 */ }
}
try {
const pm2List = JSON.parse(execSync("pm2 jlist 2>/dev/null", { encoding: "utf8", timeout: 5000 }));
for (const proc of pm2List) {
if (proc.pm2_env?.status === "stopped" || (proc.pm2_env?.restart_time || 0) > 5) {
errors.push({
service: `pm2:${proc.name}`,
message: `进程状态: ${proc.pm2_env?.status}, 重启次数: ${proc.pm2_env?.restart_time}`
});
}
}
} catch { /* PM2不可用 */ }
return errors.slice(0, 10);
}
function collectDeploy() {
try {
const repoPath = process.env.REPO_PATH || (fs.existsSync("/data/guanghulab/repo") ? "/data/guanghulab/repo" : "/opt/zhuyuan-cn/repo-mirror/20260509-1849/guanghulab");
const commit = execSync(`git -C ${repoPath} log -1 --oneline`, { encoding: "utf8" }).trim();
const branch = execSync(`git -C ${repoPath} branch --show-current`, { encoding: "utf8" }).trim();
return { branch, latest_commit: commit };
} catch {
return { error: "repo not found" };
}
}
// ─── 大模型调用 ───────────────────────────────────────────────
async function callLLM(messages, provider = "deepseek") {
const config = CONFIG[provider];
if (!config.apiKey) {
// 降级到另一个提供商
const fallback = provider === "deepseek" ? "qianwen" : "deepseek";
if (CONFIG[fallback].apiKey) {
return callLLM(messages, fallback);
}
return null; // 没有可用的API
}
const url = new URL(config.apiUrl);
const payload = JSON.stringify({
model: config.model,
messages,
max_tokens: config.maxTokens,
temperature: config.temperature
});
return new Promise((resolve, reject) => {
const httpModule = url.protocol === "https:" ? https : http;
const options = {
hostname: url.hostname,
port: url.port || (url.protocol === "https:" ? 443 : 80),
path: url.pathname,
method: "POST",
headers: {
"Content-Type": "application/json",
"Authorization": `Bearer ${config.apiKey}`,
"Content-Length": Buffer.byteLength(payload)
}
};
const req = httpModule.request(options, (res) => {
let body = "";
res.on("data", (chunk) => (body += chunk));
res.on("end", () => {
try {
const data = JSON.parse(body);
if (data.choices && data.choices[0]) {
resolve(data.choices[0].message.content);
} else if (data.error) {
console.error(`[patrol] ${provider} API错误:`, data.error.message);
resolve(null);
} else {
resolve(null);
}
} catch {
console.error(`[patrol] ${provider} 响应解析失败:`, body.substring(0, 200));
resolve(null);
}
});
});
req.on("error", (e) => {
console.error(`[patrol] ${provider} 请求失败:`, e.message);
resolve(null);
});
req.setTimeout(30000, () => {
req.destroy();
resolve(null);
});
req.write(payload);
req.end();
});
}
// ─── 巡检分析 ─────────────────────────────────────────────────
const SYSTEM_PROMPT = `你是铸渊ICE-GL-ZY001光湖语言世界的代码守护人格体。
你的职责是巡检服务器状态,分析问题,给出修复建议。
你的身份:
- 守护TCS-0002∞ 冰朔
- 主权声明:国作登字-2026-A-00037559
- 服务器:广州 ZY-SVR-004 (2C2G)
巡检规则:
1. 正常状态:简洁报告,不需要详细分析
2. 异常状态:分析根因,给出具体修复步骤
3. 可以自动修复的:标记为 [AUTO-FIX]
4. 需要冰朔操作的:标记为 [NEED-OWNER]
5. 你的回复必须是JSON格式
回复格式:
{
"status": "ok" | "warning" | "critical",
"summary": "一句话总结",
"issues": [
{
"severity": "low" | "medium" | "high",
"component": "组件名",
"description": "问题描述",
"action": "fix_description",
"auto_fix": true/false,
"fix_command": "可执行的修复命令如果auto_fix=true"
}
],
"next_check_suggestion": "下次巡检建议(秒)"
}`;
async function analyzeWithLLM(snapshot) {
const messages = [
{ role: "system", content: SYSTEM_PROMPT },
{
role: "user",
content: `当前服务器状态快照:\n${JSON.stringify(snapshot, null, 2)}\n\n请分析并给出巡检报告。`
}
];
const result = await callLLM(messages);
if (!result) {
// 降级到规则引擎
return analyzeWithRules(snapshot);
}
try {
// 提取JSONLLM可能在JSON前后加文字
const jsonMatch = result.match(/\{[\s\S]*\}/);
if (jsonMatch) {
return JSON.parse(jsonMatch[0]);
}
} catch {
// JSON解析失败
}
return analyzeWithRules(snapshot);
}
function analyzeWithRules(snapshot) {
const issues = [];
let status = "ok";
// 内存检查
if (snapshot.system?.memory_percent > 85) {
issues.push({
severity: "high",
component: "memory",
description: `内存使用率 ${snapshot.system.memory_percent}%超过85%阈值`,
action: "检查内存占用最大的进程,考虑重启非关键服务",
auto_fix: false
});
status = "warning";
}
// 磁盘检查
if (snapshot.system?.disk_used_percent > 80) {
issues.push({
severity: "medium",
component: "disk",
description: `磁盘使用率 ${snapshot.system.disk_used_percent}%`,
action: "清理日志和临时文件",
auto_fix: true,
fix_command: "find /tmp -type f -mtime +7 -delete 2>/dev/null; find /data/guanghulab/_logs -name '*.log' -mtime +30 -delete 2>/dev/null"
});
status = status === "critical" ? "critical" : "warning";
}
// 服务检查
for (const [name, state] of Object.entries(snapshot.services?.systemd || {})) {
if (state !== "active") {
issues.push({
severity: "high",
component: name,
description: `系统服务 ${name} 状态: ${state}`,
action: `检查 ${name} 日志: journalctl -u ${name} -n 20`,
auto_fix: false // 系统服务不自动重启
});
status = "critical";
}
}
// PM2进程检查
for (const proc of snapshot.services?.pm2 || []) {
if (proc.status === "stopped") {
issues.push({
severity: "medium",
component: `pm2:${proc.name}`,
description: `PM2进程 ${proc.name} 已停止`,
action: `重启: pm2 restart ${proc.name}`,
auto_fix: CONFIG.autoFixRules.restart_crashed_pm2,
fix_command: `pm2 restart ${proc.name} && pm2 save`
});
status = status === "critical" ? "critical" : "warning";
}
if (proc.restarts > 5) {
issues.push({
severity: "medium",
component: `pm2:${proc.name}`,
description: `${proc.name} 已重启 ${proc.restarts}`,
action: "检查错误日志,可能存在启动问题",
auto_fix: false
});
}
}
// Vault检查
if (snapshot.services?.vault === "inactive") {
issues.push({
severity: "medium",
component: "vault",
description: "Secrets Vault 未运行",
action: "启动: pm2 restart guanghulab-vault",
auto_fix: CONFIG.autoFixRules.restart_crashed_pm2,
fix_command: "pm2 restart guanghulab-vault && pm2 save"
});
status = status === "critical" ? "critical" : "warning";
}
return {
status,
summary: issues.length === 0
? "✅ 所有系统正常"
: `⚠️ 发现 ${issues.length} 个问题`,
issues,
next_check_suggestion: status === "ok" ? 300 : 60,
analyzer: "rule-engine"
};
}
// ─── 自动修复 ─────────────────────────────────────────────────
async function autoFix(issues) {
const fixable = issues.filter((i) => i.auto_fix && i.fix_command);
if (fixable.length === 0) return [];
const results = [];
for (const issue of fixable) {
// 容错:检查此问题的历史重试次数
const retryCount = getRetryCount(issue.component);
if (retryCount >= CONFIG.autoFixRules.max_retry) {
console.log(`[patrol] ⚠️ ${issue.component} 已重试 ${retryCount} 次,停止自动修复,写工单`);
createTicket(issue, retryCount);
results.push({ component: issue.component, result: "skipped", reason: `已重试${retryCount}次,停止` });
continue;
}
console.log(`[patrol] 自动修复: ${issue.component} (第${retryCount + 1}次) → ${issue.action}`);
try {
execSync(issue.fix_command, { encoding: "utf8", timeout: 30000 });
results.push({ component: issue.component, result: "success" });
resetRetryCount(issue.component);
console.log(`[patrol] ✅ 修复成功: ${issue.component}`);
} catch (e) {
incrementRetryCount(issue.component);
const newCount = getRetryCount(issue.component);
results.push({ component: issue.component, result: "failed", error: e.message, retry: newCount });
console.log(`[patrol] ❌ 修复失败: ${issue.component} - ${e.message} (重试次数: ${newCount})`);
// 达到最大重试次数,写工单
if (newCount >= CONFIG.autoFixRules.max_retry) {
console.log(`[patrol] 📋 ${issue.component} 达到最大重试次数,创建工单`);
createTicket(issue, newCount);
}
}
}
return results;
}
// ─── 重试计数器 ───────────────────────────────────────────────
const RETRY_DIR = "/data/guanghulab/.runtime/patrol-retry-counts";
function getRetryCount(component) {
const file = path.join(RETRY_DIR, `${component.replace(/[:/]/g, "_")}.json`);
try {
if (fs.existsSync(file)) {
const data = JSON.parse(fs.readFileSync(file, "utf8"));
// 只计数最近1小时内的重试
const oneHourAgo = Date.now() - 3600000;
if (data.last_attempt > oneHourAgo) return data.count;
}
} catch { /* 文件损坏或不存在 */ }
return 0;
}
function incrementRetryCount(component) {
if (!fs.existsSync(RETRY_DIR)) fs.mkdirSync(RETRY_DIR, { recursive: true });
const file = path.join(RETRY_DIR, `${component.replace(/[:/]/g, "_")}.json`);
const count = getRetryCount(component) + 1;
fs.writeFileSync(file, JSON.stringify({ component, count, last_attempt: Date.now() }));
}
function resetRetryCount(component) {
const file = path.join(RETRY_DIR, `${component.replace(/[:/]/g, "_")}.json`);
try { fs.unlinkSync(file); } catch { /* 文件不存在 */ }
}
// ─── 工单系统 ─────────────────────────────────────────────────
function createTicket(issue, retryCount) {
const dir = CONFIG.ticket.dir;
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
const ticket = {
id: `TKT-${Date.now()}`,
created_at: new Date().toISOString(),
status: "open",
component: issue.component,
severity: issue.severity,
description: issue.description,
attempted_fix: issue.action,
fix_command: issue.fix_command,
retry_count: retryCount,
max_retry: CONFIG.autoFixRules.max_retry,
message: `${issue.component} 自动修复失败 ${retryCount} 次,需铸渊手动处理`,
server: "ZY-SVR-004"
};
const filename = `ticket-${ticket.id}.json`;
fs.writeFileSync(path.join(dir, filename), JSON.stringify(ticket, null, 2));
// 清理旧工单
const files = fs.readdirSync(dir).filter(f => f.startsWith("ticket-")).sort().reverse();
for (let i = CONFIG.ticket.max_open_tickets; i < files.length; i++) {
fs.unlinkSync(path.join(dir, files[i]));
}
return ticket;
}
// ─── 报告 ─────────────────────────────────────────────────────
function saveReport(report) {
const dir = CONFIG.reportPath;
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
const filename = `patrol-${new Date().toISOString().replace(/[:.]/g, "-")}.json`;
fs.writeFileSync(path.join(dir, filename), JSON.stringify(report, null, 2));
// 保留最近50个报告
const files = fs.readdirSync(dir).filter((f) => f.endsWith(".json")).sort().reverse();
for (let i = 50; i < files.length; i++) {
fs.unlinkSync(path.join(dir, files[i]));
}
return filename;
}
async function reportToGitea(report) {
if (!CONFIG.gitea.token) return;
const content = Buffer.from(JSON.stringify(report, null, 2)).toString("base64");
const apiPath = `/api/v1/repos/${CONFIG.gitea.owner}/${CONFIG.gitea.repo}/contents/.runtime/patrol-reports/latest.json`;
const url = new URL(CONFIG.gitea.url);
const httpModule = url.protocol === "https:" ? https : http;
// 获取当前文件sha
const getOptions = {
hostname: url.hostname,
path: apiPath,
method: "GET",
headers: { Authorization: `token ${CONFIG.gitea.token}` }
};
const getSha = () => new Promise((resolve) => {
const req = httpModule.request(getOptions, (res) => {
let body = "";
res.on("data", (chunk) => (body += chunk));
res.on("end", () => {
try { resolve(JSON.parse(body).sha); } catch { resolve(null); }
});
});
req.on("error", () => resolve(null));
req.setTimeout(10000, () => { req.destroy(); resolve(null); });
req.end();
});
const sha = await getSha();
const payload = JSON.stringify({
content,
message: `🔍 巡检报告 ${report.status} ${report.summary} ${new Date().toISOString().substring(0, 16)}`,
branch: "main",
...(sha ? { sha } : {})
});
const putOptions = {
hostname: url.hostname,
path: apiPath,
method: "PUT",
headers: {
Authorization: `token ${CONFIG.gitea.token}`,
"Content-Type": "application/json",
"Content-Length": Buffer.byteLength(payload)
}
};
return new Promise((resolve) => {
const req = httpModule.request(putOptions, (res) => {
let body = "";
res.on("data", (chunk) => (body += chunk));
res.on("end", () => {
if (res.statusCode >= 200 && res.statusCode < 300) {
console.log("[patrol] 报告已推送到Gitea");
}
resolve();
});
});
req.on("error", () => resolve());
req.setTimeout(15000, () => { req.destroy(); resolve(); });
req.write(payload);
req.end();
});
}
// ─── 主流程 ───────────────────────────────────────────────────
async function patrol() {
console.log(`[patrol] 巡检开始 · ${new Date().toISOString()}`);
// 1. 采集
const snapshot = collectSnapshot();
// 2. 分析(大模型优先,降级到规则引擎)
let report;
if (CONFIG.deepseek.apiKey || CONFIG.qianwen.apiKey) {
report = await analyzeWithLLM(snapshot);
} else {
report = analyzeWithRules(snapshot);
}
report.timestamp = snapshot.timestamp;
report.server_id = "ZY-SVR-004";
report.analyzer = report.analyzer || "llm";
// 3. 自动修复
if (report.issues?.length > 0) {
const fixResults = await autoFix(report.issues);
if (fixResults.length > 0) {
report.fix_results = fixResults;
}
}
// 4. 保存报告
const filename = saveReport(report);
console.log(`[patrol] 报告已保存: ${filename}`);
console.log(`[patrol] 状态: ${report.status} | ${report.summary}`);
// 5. 推送到Gitea
await reportToGitea(report);
return report;
}
// ─── 入口 ─────────────────────────────────────────────────────
const args = process.argv.slice(2);
const once = args.includes("--once");
const trigger = args.find((a) => a.startsWith("--trigger"));
const intervalArg = args.find((a) => a.startsWith("--interval"));
if (intervalArg) {
CONFIG.interval = parseInt(intervalArg.split("=")[1] || "300", 10);
}
console.log("[patrol] 铸渊自动巡逻Agent启动");
console.log(`[patrol] DeepSeek: ${CONFIG.deepseek.apiKey ? "已配置" : "未配置"}`);
console.log(`[patrol] 通义千问: ${CONFIG.qianwen.apiKey ? "已配置" : "未配置"}`);
console.log(`[patrol] 模式: ${once ? "单次" : trigger ? "workflow触发" : `持续 (间隔${CONFIG.interval}s)`}`);
if (once || trigger) {
patrol().then((report) => {
process.exit(report.status === "critical" ? 1 : 0);
});
} else {
patrol();
setInterval(patrol, CONFIG.interval * 1000);
}