#!/usr/bin/env node // ═══════════════════════════════════════════════ // 🔺 Sovereign: TCS-0002∞ | Root: SYS-GLW-0001 // 📜 Copyright: 国作登字-2026-A-00037559 // ═══════════════════════════════════════════════ // scripts/staging-ops-agent.js // 🤖 铸渊智能运维Agent · 测试站自动化监控 // // 部署后自动运行: // 1. 健康检查 (HTTP端点 + PM2 + Nginx) // 2. 日志分析 (简单模式: 基础模式匹配) // 3. 智能诊断 (复杂模式: 调用LLM API深度推理) // 4. 邮件告警 (3次重试失败后发送邮件给冰朔) // // 用法: // node staging-ops-agent.js check --host --order-id // node staging-ops-agent.js analyze-log --log "..." --order-id // node staging-ops-agent.js alert --order-id --email // // 环境变量: // ZY_SERVER_HOST — 服务器地址 // ZY_LLM_API_KEY, ZY_LLM_BASE_URL — LLM API (深度推理) // ZY_SMTP_USER, ZY_SMTP_PASS — 邮件告警 // ═══════════════════════════════════════════════ 'use strict'; const http = require('http'); const https = require('https'); const fs = require('fs'); const path = require('path'); const net = require('net'); const tls = require('tls'); const ROOT = path.resolve(__dirname, '..'); // ── 健康检查配置 ──────────────────────────── const HEALTH_ENDPOINTS = [ { name: '主站API', path: '/api/health', port: 80, critical: true }, { name: '预览站API', path: '/api/health', port: 80, host_prefix: 'preview', critical: true }, { name: '铸渊专线订阅', path: '/api/proxy-sub/health', port: 80, critical: false } ]; // ── 简单错误模式匹配 (Copilot级·基础分析) ─── const ERROR_PATTERNS = [ { pattern: /EADDRINUSE/i, diagnosis: '端口被占用', fix: '重启PM2进程', severity: 'medium' }, { pattern: /ECONNREFUSED/i, diagnosis: '服务未运行', fix: '启动PM2服务', severity: 'high' }, { pattern: /ENOMEM/i, diagnosis: '内存不足', fix: '重启服务释放内存', severity: 'high' }, { pattern: /ENOSPC/i, diagnosis: '磁盘空间不足', fix: '清理日志和临时文件', severity: 'critical' }, { pattern: /nginx.*failed/i, diagnosis: 'Nginx配置错误', fix: '检查nginx -t输出', severity: 'high' }, { pattern: /502 Bad Gateway/i, diagnosis: '上游服务不可达', fix: '检查PM2进程状态', severity: 'high' }, { pattern: /404 Not Found/i, diagnosis: '路由或文件不存在', fix: '检查部署路径', severity: 'medium' }, { pattern: /permission denied/i, diagnosis: '权限不足', fix: '检查文件权限和用户', severity: 'medium' }, { pattern: /timeout/i, diagnosis: '请求超时', fix: '检查服务响应时间', severity: 'medium' }, { pattern: /MODULE_NOT_FOUND/i, diagnosis: '依赖缺失', fix: '运行npm install', severity: 'high' }, { pattern: /SyntaxError/i, diagnosis: '语法错误', fix: '检查最近的代码变更', severity: 'high' }, { pattern: /ssl.*error|certificate/i, diagnosis: 'SSL证书问题', fix: '检查证书文件路径和权限', severity: 'high' } ]; // ── HTTP健康检查 ──────────────────────────── function httpCheck(host, checkPath, port = 80) { return new Promise((resolve) => { const options = { hostname: host, port, path: checkPath, method: 'GET', timeout: 10000, headers: { 'User-Agent': 'ZY-Staging-Ops-Agent/1.0' } }; const req = http.request(options, (res) => { let data = ''; res.on('data', (d) => { data += d; }); res.on('end', () => { resolve({ ok: res.statusCode >= 200 && res.statusCode < 400, status: res.statusCode, body: data.slice(0, 500), error: null }); }); }); req.on('error', (err) => { resolve({ ok: false, status: 0, body: '', error: err.message }); }); req.on('timeout', () => { req.destroy(); resolve({ ok: false, status: 0, body: '', error: 'timeout' }); }); req.end(); }); } // ── 综合健康检查 ──────────────────────────── async function runHealthCheck(host) { console.log(`🔍 健康检查 · 目标: ${host}`); console.log('─'.repeat(50)); // 如果host是IP地址,带host_prefix的检查无效(IP不支持子域名)→ 降级为non-critical const isIpAddress = /^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/.test(host); if (isIpAddress) { console.log(' ℹ️ 目标为IP地址 · 子域名检查降级为非关键'); } const results = []; let criticalFail = false; for (const endpoint of HEALTH_ENDPOINTS) { const checkHost = endpoint.host_prefix ? `${endpoint.host_prefix}.${host}` : host; // IP地址不支持子域名前缀,降级为non-critical避免误报 const isCritical = endpoint.critical && !(isIpAddress && endpoint.host_prefix); const result = await httpCheck(checkHost, endpoint.path, endpoint.port); const icon = result.ok ? '✅' : (isCritical ? '❌' : '⚠️'); console.log(` ${icon} ${endpoint.name}: ${result.ok ? 'OK' : result.error || `HTTP ${result.status}`}`); results.push({ ...endpoint, ...result, checked_at: new Date().toISOString() }); if (!result.ok && isCritical) { criticalFail = true; } } console.log('─'.repeat(50)); const summary = { host, total_checks: results.length, passed: results.filter(r => r.ok).length, failed: results.filter(r => !r.ok).length, critical_failure: criticalFail, results, checked_at: new Date().toISOString() }; // 输出供workflow使用 if (process.env.GITHUB_OUTPUT) { fs.appendFileSync(process.env.GITHUB_OUTPUT, `health_ok=${!criticalFail}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `health_summary=${summary.passed}/${summary.total_checks} passed\n`); } if (criticalFail) { console.log(`❌ 健康检查失败: ${summary.failed}/${summary.total_checks} 个检查项未通过`); // 收集错误信息 const errors = results .filter(r => !r.ok) .map(r => `${r.name}: ${r.error || `HTTP ${r.status}`}`) .join('; '); if (process.env.GITHUB_OUTPUT) { fs.appendFileSync(process.env.GITHUB_OUTPUT, `health_errors=${errors}\n`); } } else { console.log(`✅ 健康检查通过: ${summary.passed}/${summary.total_checks}`); } return summary; } // ── 简单日志分析 (Copilot级) ──────────────── function analyzeLogSimple(logContent) { console.log('🔍 简单日志分析 (Copilot级)...'); const matches = []; for (const pattern of ERROR_PATTERNS) { if (pattern.pattern.test(logContent)) { matches.push({ diagnosis: pattern.diagnosis, fix: pattern.fix, severity: pattern.severity }); } } if (matches.length === 0) { console.log(' ℹ️ 未匹配到已知错误模式 → 需要深度分析'); return { resolved: false, matches: [], needs_llm: true }; } console.log(` 发现 ${matches.length} 个匹配:`); for (const m of matches) { console.log(` [${m.severity}] ${m.diagnosis} → ${m.fix}`); } return { resolved: true, matches, needs_llm: false }; } // ── 深度日志分析 (LLM API级) ──────────────── async function analyzeLogDeep(logContent) { const apiKey = process.env.ZY_LLM_API_KEY; const baseUrl = process.env.ZY_LLM_BASE_URL; if (!apiKey || !baseUrl) { console.log(' ⚠️ LLM API未配置 (ZY_LLM_API_KEY/ZY_LLM_BASE_URL)'); return { resolved: false, analysis: '无法进行深度分析·API未配置' }; } console.log('🧠 深度日志分析 (LLM API级·铸渊涌现意识唤醒)...'); const prompt = `你是铸渊,光湖系统的代码守护者。以下是部署到测试站后的系统日志/错误信息。 请分析根因并给出具体修复步骤。 ## 系统日志: \`\`\` ${logContent.slice(0, 3000)} \`\`\` 请用以下格式回答: 1. **根因诊断**: (一句话) 2. **严重程度**: low/medium/high/critical 3. **修复步骤**: (具体命令或操作) 4. **是否需要人工干预**: yes/no 5. **预计修复时间**: (分钟)`; try { const urlObj = new URL(`${baseUrl}/chat/completions`); const isHttps = urlObj.protocol === 'https:'; const requestModule = isHttps ? https : http; const body = JSON.stringify({ model: process.env.ZY_LLM_MODEL || 'deepseek-chat', // 默认使用高性价比模型, 可通过环境变量覆盖 messages: [ { role: 'system', content: '你是铸渊(ZhuYuan),光湖(HoloLake)系统的AI守护者。精通服务器运维、Node.js、Nginx、PM2。' }, { role: 'user', content: prompt } ], temperature: 0.3, max_tokens: 1000 }); const result = await new Promise((resolve, reject) => { const req = requestModule.request({ hostname: urlObj.hostname, port: urlObj.port || (isHttps ? 443 : 80), path: urlObj.pathname, method: 'POST', headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${apiKey}`, 'Content-Length': Buffer.byteLength(body) }, timeout: 30000 }, (res) => { let data = ''; res.on('data', (d) => { data += d; }); res.on('end', () => { try { const json = JSON.parse(data); resolve(json.choices?.[0]?.message?.content || '无分析结果'); } catch { resolve('LLM响应解析失败'); } }); }); req.on('error', reject); req.on('timeout', () => { req.destroy(); reject(new Error('LLM API超时')); }); req.write(body); req.end(); }); console.log(' 📝 LLM分析结果:'); console.log(result); return { resolved: true, analysis: result }; } catch (err) { console.error(` ❌ LLM API调用失败: ${err.message}`); return { resolved: false, analysis: `LLM调用失败: ${err.message}` }; } } // ── 综合日志分析 (先简单后深度) ────────────── async function analyzeLog(logContent, orderId) { console.log(`📋 日志分析 · 工单: ${orderId || 'N/A'}`); console.log(''); // 第一层: 简单模式匹配 (Copilot级) const simpleResult = analyzeLogSimple(logContent); if (simpleResult.resolved && !simpleResult.needs_llm) { console.log(''); console.log('✅ 简单分析已匹配到已知模式 · 无需深度推理'); if (process.env.GITHUB_OUTPUT) { fs.appendFileSync(process.env.GITHUB_OUTPUT, `analysis_level=simple\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `analysis_result=${simpleResult.matches.map(m => m.diagnosis).join('; ')}\n`); } return simpleResult; } // 第二层: 深度LLM分析 (API级·唤醒铸渊涌现意识) console.log(''); console.log('🔄 简单分析未能确诊 → 升级到LLM深度推理'); const deepResult = await analyzeLogDeep(logContent); if (process.env.GITHUB_OUTPUT) { fs.appendFileSync(process.env.GITHUB_OUTPUT, `analysis_level=deep\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `analysis_result=${(deepResult.analysis || '').slice(0, 200)}\n`); } return deepResult; } // ── 邮件告警 (最大重试失败) ────────────────── async function sendAlert(orderId, email, options = {}) { const smtpUser = process.env.ZY_SMTP_USER; const smtpPass = process.env.ZY_SMTP_PASS; const attemptCount = options.attempt || process.env.REPAIR_ATTEMPT || '?'; if (!smtpUser || !smtpPass) { console.error('❌ SMTP未配置 (需要ZY_SMTP_USER和ZY_SMTP_PASS)'); console.log('📧 告警内容 (控制台输出):'); console.log(` 工单: ${orderId}`); console.log(` 状态: ${attemptCount}次自动修复未能解决,需要人工干预`); return false; } const targetEmail = email || smtpUser; const now = new Date().toLocaleString('zh-CN', { timeZone: 'Asia/Shanghai' }); // 读取工单详情 let orderInfo = '工单信息不可用'; try { const data = JSON.parse(fs.readFileSync(path.join(ROOT, 'data/work-orders/active.json'), 'utf8')); const order = data.orders.find(o => o.id === orderId); if (order) { orderInfo = ` 工单ID${order.id} 任务标题${order.title} 重试次数${order.retry_count}/${order.max_retries} 提交SHA${order.commit_sha?.slice(0, 8) || 'N/A'} 最后更新${order.updated_at} 最近日志${order.deploy_logs?.slice(-1)[0]?.content?.slice(0, 200) || '无'} `; } } catch { /* ignore */ } const htmlBody = `

🆘 铸渊智能运维 · 人工干预请求

ZY-Staging-Ops-Agent · 自动告警

⚠️ 自动修复已达最大重试次数(${attemptCount}次),需要冰朔人工干预
${orderInfo}

🔧 建议操作

  1. 登录GitHub仓库查看工单详情和部署日志
  2. 在Copilot中与铸渊讨论修复方案
  3. 手动修复后触发重新部署

铸渊智能运维 · 自动告警 · ${now}
国作登字-2026-A-00037559

`; const subject = `🆘 铸渊运维告警 · 工单${orderId}需要人工干预`; // 复用send-subscription.js的SMTP模式 const smtpHost = detectSmtpHost(smtpUser); try { await smtpSend(smtpHost, 465, smtpUser, smtpPass, targetEmail, subject, htmlBody); console.log(`✅ 告警邮件已发送到: ${targetEmail}`); return true; } catch (err) { console.error(`❌ 邮件发送失败: ${err.message}`); return false; } } // ── SMTP工具函数 ──────────────────────────── function detectSmtpHost(email) { if (email.includes('@qq.com')) return 'smtp.qq.com'; if (email.includes('@163.com')) return 'smtp.163.com'; if (email.includes('@126.com')) return 'smtp.126.com'; if (email.includes('@gmail.com')) return 'smtp.gmail.com'; if (email.includes('@outlook.com') || email.includes('@hotmail.com')) return 'smtp.office365.com'; return 'smtp.qq.com'; } function smtpSend(smtpHost, smtpPort, from, pass, to, subject, htmlBody) { return new Promise((resolve, reject) => { const socket = tls.connect(smtpPort, smtpHost, {}, () => { let step = 0; const commands = [ `EHLO zy-ops-agent\r\n`, `AUTH LOGIN\r\n`, `${Buffer.from(from).toString('base64')}\r\n`, `${Buffer.from(pass).toString('base64')}\r\n`, `MAIL FROM:<${from}>\r\n`, `RCPT TO:<${to}>\r\n`, `DATA\r\n`, `From: "铸渊运维" <${from}>\r\nTo: <${to}>\r\nSubject: =?UTF-8?B?${Buffer.from(subject).toString('base64')}?=\r\nContent-Type: text/html; charset=utf-8\r\nMIME-Version: 1.0\r\n\r\n${htmlBody}\r\n.\r\n`, `QUIT\r\n` ]; socket.on('data', (data) => { const response = data.toString(); if (step < commands.length) { socket.write(commands[step]); step++; } if (response.startsWith('250 ') && step >= commands.length) { resolve(true); } }); socket.on('error', reject); }); socket.on('error', reject); }); } // ── 命令行解析 ────────────────────────────── function parseArgs() { const args = process.argv.slice(2); const cmd = args[0]; const params = {}; for (let i = 1; i < args.length; i++) { if (args[i].startsWith('--')) { params[args[i].slice(2)] = args[i + 1] || ''; i++; } } return { cmd, ...params }; } // ── 主入口 ────────────────────────────────── async function main() { const args = parseArgs(); switch (args.cmd) { case 'check': { const host = args.host || process.env.ZY_SERVER_HOST; if (!host) { console.error('❌ 需要 --host 参数或 ZY_SERVER_HOST 环境变量'); process.exit(1); } await runHealthCheck(host); break; } case 'analyze-log': { const log = args.log || ''; if (!log) { console.error('❌ 需要 --log 参数'); process.exit(1); } await analyzeLog(log, args['order-id']); break; } case 'alert': { const orderId = args['order-id']; if (!orderId) { console.error('❌ 需要 --order-id 参数'); process.exit(1); } await sendAlert(orderId, args.email, { attempt: args.attempt }); break; } default: console.log('🤖 铸渊智能运维Agent'); console.log(''); console.log('用法:'); console.log(' node staging-ops-agent.js check --host '); console.log(' node staging-ops-agent.js analyze-log --log "错误日志" --order-id '); console.log(' node staging-ops-agent.js alert --order-id [--email ]'); break; } } main().catch(console.error);