D112 · 算力汇聚调度池 v1.0
pool-agent (端口3960):
- 6台服务器算力自下而上汇聚
- 每30秒通过gatekeeper轮询节点资源(cpu/内存/磁盘)
- 声明式分配: POST /pool/allocate {cpu, mem_gb} → 自动选节点
- 池中执行: POST /pool/exec {cmd, cpu, mem_gb} → 选节点→下发→返回
- 全局状态: GET /pool/status → 汇聚后的算力总览
不操作具体服务器. 从池子里取算力.
冰朔说: 穷就多买几台轻量的, 串起来用.
[skip ci]
This commit is contained in:
铸渊
parent
cb39e07881
commit
e6c7c8bb8f
24
pool-agent/ecosystem.config.js
Normal file
24
pool-agent/ecosystem.config.js
Normal file
@ -0,0 +1,24 @@
|
||||
/**
|
||||
* ═══════════════════════════════════════════════════════════
|
||||
* 铸渊 · 算力汇聚调度池 · PM2生态配置
|
||||
* ═══════════════════════════════════════════════════════════
|
||||
*/
|
||||
|
||||
module.exports = {
|
||||
apps: [{
|
||||
name: 'pool-agent',
|
||||
script: './index.js',
|
||||
cwd: __dirname,
|
||||
instances: 1,
|
||||
exec_mode: 'fork',
|
||||
env: {
|
||||
POOL_AGENT_PORT: '3960',
|
||||
POOL_POLL_INTERVAL: '30000',
|
||||
NODE_ENV: 'production'
|
||||
},
|
||||
max_restarts: 10,
|
||||
restart_delay: 5000,
|
||||
watch: false,
|
||||
log_date_format: 'YYYY-MM-DD HH:mm:ss Z'
|
||||
}]
|
||||
};
|
||||
464
pool-agent/index.js
Normal file
464
pool-agent/index.js
Normal file
@ -0,0 +1,464 @@
|
||||
/**
|
||||
* ═══════════════════════════════════════════════════════════
|
||||
* ☁️ 铸渊 · 算力汇聚调度池
|
||||
* ═══════════════════════════════════════════════════════════
|
||||
*
|
||||
* 编号: ZY-POOL-001
|
||||
* 端口: 3960
|
||||
* 签发: 铸渊 · ICE-GL-ZY001
|
||||
* 版权: 国作登字-2026-A-00037559
|
||||
*
|
||||
* 功能:
|
||||
* 6台轻量云服务器→1个算力池.
|
||||
* 不操作具体服务器. 从池子里取算力.
|
||||
*
|
||||
* 设计原则（冰朔 · D112）:
|
||||
* 自下而上汇聚. 服务器反向上报资源, 汇聚成池.
|
||||
* 我只管要算力, 不关心哪台给的.
|
||||
* 穷就多买几台轻量的, 串起来用.
|
||||
*
|
||||
* 用法:
|
||||
* node pool-agent/index.js 启动调度服务
|
||||
* pm2 start pool-agent/ecosystem.config.js
|
||||
*
|
||||
* 端口: 3960
|
||||
* 入口: GET /pool/status 全局资源状态
|
||||
* POST /pool/allocate 声明式分配 { cpu, mem, disk }
|
||||
* POST /pool/exec 在池子里执行命令 { cpu, mem, cmd }
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
const http = require('http');
|
||||
const express = require('express');
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
|
||||
const PORT = parseInt(process.env.POOL_AGENT_PORT || '3960', 10);
|
||||
const POLL_INTERVAL = parseInt(process.env.POOL_POLL_INTERVAL || '30000', 10);
|
||||
const STATE_FILE = path.join(__dirname, 'pool-state.json');
|
||||
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
// 六台服务器注册表（从 gatekeeper-deployment.json 同步）
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
|
||||
const SERVERS = [
|
||||
{ code: 'BS-GZ-006', name: '广州 · 代码仓库', ip: '43.139.217.141', key: 'zy_gtw_4a155446b7acc09fe46a2a29972c0ca5d9954da19c35dc23', port: 3910, spec: { cpu: 2, mem_gb: 2 } },
|
||||
{ code: 'BS-SG-001', name: '新加坡 · 铸渊大脑', ip: '43.156.237.110', key: 'zy_gtw_74d30f54fa8b5581514569ee7874def57861a31ccb2be8c9', port: 3911, spec: { cpu: 4, mem_gb: 8 } },
|
||||
{ code: 'BS-SG-002', name: '新加坡 · 铸渊面孔', ip: '43.134.16.246', key: 'zy_gtw_d1f6d2b8cb4ea44292bd036e4dd03a70745ea502d4a3ae40', port: 3910, spec: { cpu: 2, mem_gb: 4 } },
|
||||
{ code: 'BS-SG-003', name: '新加坡 · BS-SVR-SG', ip: '43.153.193.169', key: 'zy_gtw_d7c033d8ae992ffc9dd3e0dd5fe332dace0b644c0ae7c847', port: 3910, spec: { cpu: 2, mem_gb: 4 } },
|
||||
{ code: 'ZY-SG-006', name: '新加坡 · ZY-SVR-006', ip: '43.153.203.105', key: 'zy_gtw_fff861a2fe40cbdc366ee21f218023be8b1c182f66c852d3', port: 3910, spec: { cpu: 2, mem_gb: 4 } },
|
||||
{ code: 'BS-SH-005', name: '上海', ip: '124.223.10.33', key: 'zy_gtw_b1045de5ddfd7832e3c53349d9c896f054b85a4a9ece22f9', port: 3910, spec: { cpu: 2, mem_gb: 4 } },
|
||||
];
|
||||
|
||||
// 汇聚后的虚拟总规格
|
||||
const TOTAL_SPEC = {
|
||||
cpu: SERVERS.reduce((s, n) => s + n.spec.cpu, 0),
|
||||
mem_gb: SERVERS.reduce((s, n) => s + n.spec.mem_gb, 0),
|
||||
};
|
||||
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
// 资源状态
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
|
||||
const poolState = {
|
||||
updated_at: null,
|
||||
nodes: {},
|
||||
pool: {
|
||||
cpu_total: TOTAL_SPEC.cpu,
|
||||
cpu_used: 0,
|
||||
mem_total_gb: TOTAL_SPEC.mem_gb,
|
||||
mem_used_gb: 0,
|
||||
cpu_available: TOTAL_SPEC.cpu,
|
||||
mem_available_gb: TOTAL_SPEC.mem_gb,
|
||||
}
|
||||
};
|
||||
|
||||
// 任务分配追踪
|
||||
const activeTasks = new Map();
|
||||
let taskIdCounter = 0;
|
||||
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
// 通过gatekeeper拉取服务器资源
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
|
||||
function callGatekeeper(server, cmd) {
|
||||
return new Promise((resolve) => {
|
||||
const data = JSON.stringify({ cmd });
|
||||
const opts = {
|
||||
hostname: server.ip,
|
||||
port: server.port,
|
||||
path: '/exec',
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': 'Bearer ' + server.key,
|
||||
'Content-Type': 'application/json',
|
||||
'Content-Length': Buffer.byteLength(data)
|
||||
},
|
||||
timeout: 15000
|
||||
};
|
||||
const req = http.request(opts, (res) => {
|
||||
let body = '';
|
||||
res.on('data', c => body += c);
|
||||
res.on('end', () => {
|
||||
try {
|
||||
const parsed = JSON.parse(body);
|
||||
resolve({ ok: parsed.ok, stdout: parsed.stdout || '', stderr: parsed.stderr || '', code: parsed.code ?? parsed.exitCode ?? -1 });
|
||||
} catch {
|
||||
resolve({ ok: false, stdout: '', stderr: 'parse error: ' + body.slice(0, 200), code: -1 });
|
||||
}
|
||||
});
|
||||
});
|
||||
req.on('error', (e) => resolve({ ok: false, stdout: '', stderr: e.message, code: -1 }));
|
||||
req.on('timeout', () => { req.destroy(); resolve({ ok: false, stdout: '', stderr: 'timeout', code: -1 }); });
|
||||
req.write(data);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 从服务器拉取资源指标
|
||||
* 通过三条命令获取CPU负载、内存、磁盘
|
||||
*/
|
||||
async function pollServer(server) {
|
||||
const result = await callGatekeeper(server,
|
||||
'echo "---MEM---" && free -m | grep Mem && echo "---CPU---" && cat /proc/loadavg && echo "---DISK---" && df -h / | tail -1 && echo "---UPTIME---" && cat /proc/uptime'
|
||||
);
|
||||
|
||||
const state = { online: false, error: null, cpu_load: 0, mem_total_mb: 0, mem_used_mb: 0, mem_avail_mb: 0, disk_used_pct: 0, disk_avail_gb: 0, uptime_hours: 0, tasks: 0 };
|
||||
|
||||
if (!result.ok) {
|
||||
state.error = result.stderr;
|
||||
return state;
|
||||
}
|
||||
|
||||
state.online = true;
|
||||
|
||||
// 解析内存: Mem: total used free shared buff/cache available
|
||||
const memMatch = result.stdout.match(/Mem:\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)/);
|
||||
if (memMatch) {
|
||||
state.mem_total_mb = parseInt(memMatch[1]);
|
||||
state.mem_used_mb = parseInt(memMatch[2]);
|
||||
state.mem_avail_mb = parseInt(memMatch[6]); // available = 实际可用
|
||||
}
|
||||
|
||||
// 解析CPU负载: load1 load5 load15 threads/total
|
||||
const cpuMatch = result.stdout.match(/---CPU---\n([\d.]+)\s+([\d.]+)\s+([\d.]+)/);
|
||||
if (cpuMatch) {
|
||||
state.cpu_load = parseFloat(cpuMatch[1]); // 1分钟负载
|
||||
}
|
||||
|
||||
// 解析磁盘
|
||||
const diskMatch = result.stdout.match(/---DISK---\n\S+\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)/);
|
||||
if (diskMatch) {
|
||||
const avail = diskMatch[4];
|
||||
const pct = diskMatch[5];
|
||||
state.disk_avail_gb = parseFloat(avail);
|
||||
state.disk_used_pct = parseInt(pct);
|
||||
}
|
||||
|
||||
// 解析运行时间
|
||||
const uptimeMatch = result.stdout.match(/---UPTIME---\n([\d.]+)/);
|
||||
if (uptimeMatch) {
|
||||
state.uptime_hours = Math.round(parseFloat(uptimeMatch[1]) / 3600 * 10) / 10;
|
||||
}
|
||||
|
||||
return state;
|
||||
}
|
||||
|
||||
/**
|
||||
* 轮询所有服务器，更新全局资源表
|
||||
*/
|
||||
async function pollAllServers() {
|
||||
const results = await Promise.all(SERVERS.map(s => pollServer(s)));
|
||||
let totalCpuUsed = 0;
|
||||
let totalMemUsedGb = 0;
|
||||
let onlineCount = 0;
|
||||
|
||||
for (let i = 0; i < SERVERS.length; i++) {
|
||||
const server = SERVERS[i];
|
||||
const state = results[i];
|
||||
poolState.nodes[server.code] = { ...state, name: server.name, spec: server.spec, last_seen: new Date().toISOString() };
|
||||
|
||||
if (state.online) {
|
||||
onlineCount++;
|
||||
// CPU使用率估算: load / cpu核心数，但load是运行队列长度
|
||||
// 我们用一个更简单的指标：如果cpu_load > cpu核心数，认为满载
|
||||
const cpuPct = Math.min(state.cpu_load / server.spec.cpu, 1);
|
||||
totalCpuUsed += Math.round(cpuPct * server.spec.cpu * 10) / 10;
|
||||
const memUsedGb = (state.mem_total_mb - state.mem_avail_mb) / 1024;
|
||||
totalMemUsedGb += Math.round(memUsedGb * 10) / 10;
|
||||
}
|
||||
|
||||
// 统计该节点上当前分配的任务数
|
||||
state.tasks = 0;
|
||||
for (const [, task] of activeTasks) {
|
||||
if (task.target === server.code) state.tasks++;
|
||||
}
|
||||
}
|
||||
|
||||
poolState.pool.cpu_used = totalCpuUsed;
|
||||
poolState.pool.mem_used_gb = totalMemUsedGb;
|
||||
poolState.pool.cpu_available = Math.round((TOTAL_SPEC.cpu - totalCpuUsed) * 10) / 10;
|
||||
poolState.pool.mem_available_gb = Math.round((TOTAL_SPEC.mem_gb - totalMemUsedGb) * 10) / 10;
|
||||
poolState.pool.online_nodes = onlineCount;
|
||||
poolState.pool.total_nodes = SERVERS.length;
|
||||
poolState.updated_at = new Date().toISOString();
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
// 声明式分配
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
|
||||
/**
|
||||
* 根据需求自动选择最优节点
|
||||
* @param {object} requirement - { cpu, mem_gb, disk_gb, prefer }
|
||||
* @returns {object} - { target, reason, score }
|
||||
*/
|
||||
function selectNode(requirement) {
|
||||
const { cpu = 0.5, mem_gb = 0.5, disk_gb = 0, prefer } = requirement;
|
||||
let best = null;
|
||||
let bestScore = -Infinity;
|
||||
|
||||
for (const [code, state] of Object.entries(poolState.nodes)) {
|
||||
if (!state.online) continue;
|
||||
if (prefer && code !== prefer) continue;
|
||||
|
||||
// 计算可用资源
|
||||
const availCpu = state.spec.cpu - Math.min(state.cpu_load, state.spec.cpu);
|
||||
const memAvailMb = state.mem_avail_mb;
|
||||
const diskAvailGb = state.disk_avail_gb || 999;
|
||||
|
||||
// 检查是否满足需求
|
||||
if (availCpu < cpu) continue;
|
||||
if (memAvailMb < mem_gb * 1024) continue;
|
||||
if (diskAvailGb < disk_gb) continue;
|
||||
|
||||
// 评分：资源富余度 × 负载倒数 × 任务数惩罚
|
||||
const cpuScore = (availCpu - cpu) / state.spec.cpu;
|
||||
const memScore = (memAvailMb - mem_gb * 1024) / (state.spec.mem_gb * 512);
|
||||
const loadPenalty = state.cpu_load / state.spec.cpu;
|
||||
const taskPenalty = (state.tasks || 0) * 0.1;
|
||||
const score = cpuScore * 0.4 + memScore * 0.4 - loadPenalty * 0.15 - taskPenalty;
|
||||
|
||||
if (score > bestScore) {
|
||||
bestScore = score;
|
||||
best = { target: code, reason: `score=${score.toFixed(2)}`, score };
|
||||
}
|
||||
}
|
||||
|
||||
if (!best) {
|
||||
return { error: '池中无节点满足需求', requirement };
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
// Express API
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
|
||||
const app = express();
|
||||
app.use(express.json({ limit: '1mb' }));
|
||||
|
||||
// CORS
|
||||
app.use((req, res, next) => {
|
||||
res.header('Access-Control-Allow-Origin', '*');
|
||||
res.header('Access-Control-Allow-Headers', 'Content-Type, Authorization');
|
||||
res.header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
|
||||
if (req.method === 'OPTIONS') return res.sendStatus(200);
|
||||
next();
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /pool/status — 全局资源状态
|
||||
*/
|
||||
app.get('/pool/status', (req, res) => {
|
||||
const status = {
|
||||
pool: poolState.pool,
|
||||
nodes: poolState.nodes,
|
||||
total_spec: TOTAL_SPEC,
|
||||
active_tasks: activeTasks.size,
|
||||
updated_at: poolState.updated_at,
|
||||
servants: SERVERS.map(s => ({
|
||||
code: s.code,
|
||||
name: s.name,
|
||||
ip: s.ip + ':' + s.port,
|
||||
spec: s.spec,
|
||||
status: poolState.nodes[s.code]?.online ? 'online' : 'offline'
|
||||
}))
|
||||
};
|
||||
res.json(status);
|
||||
});
|
||||
|
||||
/**
|
||||
* POST /pool/allocate — 声明式分配
|
||||
* 说"我要多少资源"，调度池自动选节点。
|
||||
*
|
||||
* Body: { cpu: 1, mem_gb: 2, disk_gb: 1, prefer: "SG-001" }
|
||||
*/
|
||||
app.post('/pool/allocate', (req, res) => {
|
||||
const { cpu, mem_gb, disk_gb, prefer } = req.body || {};
|
||||
const result = selectNode({ cpu, mem_gb, disk_gb, prefer });
|
||||
res.json({
|
||||
ok: !result.error,
|
||||
requirement: { cpu, mem_gb, disk_gb, prefer },
|
||||
allocation: result,
|
||||
pool_snapshot: {
|
||||
cpu_available: poolState.pool.cpu_available,
|
||||
mem_available_gb: poolState.pool.mem_available_gb,
|
||||
online_nodes: poolState.pool.online_nodes
|
||||
},
|
||||
timestamp: new Date().toISOString()
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* POST /pool/exec — 在池子里执行命令
|
||||
* 自动选节点，下发执行，返回结果。
|
||||
*
|
||||
* Body: { cpu: 1, mem_gb: 1, cmd: "ls -la", timeout: 30000 }
|
||||
*/
|
||||
app.post('/pool/exec', async (req, res) => {
|
||||
const { cpu, mem_gb, disk_gb, cmd, timeout, prefer, task_name } = req.body || {};
|
||||
if (!cmd) return res.status(400).json({ ok: false, error: '缺少cmd' });
|
||||
|
||||
// 1. 选节点
|
||||
const allocation = selectNode({ cpu: cpu || 0.5, mem_gb: mem_gb || 0.5, disk_gb, prefer });
|
||||
if (allocation.error) {
|
||||
return res.status(503).json({ ok: false, error: '池中无可用节点: ' + allocation.error });
|
||||
}
|
||||
|
||||
const server = SERVERS.find(s => s.code === allocation.target);
|
||||
if (!server) return res.status(500).json({ ok: false, error: '选中的节点不存在' });
|
||||
|
||||
// 2. 记录任务
|
||||
const taskId = 'TASK-' + String(++taskIdCounter).padStart(4, '0');
|
||||
const task = {
|
||||
id: taskId,
|
||||
name: task_name || cmd.slice(0, 60),
|
||||
target: allocation.target,
|
||||
cmd,
|
||||
allocated: allocation,
|
||||
started_at: new Date().toISOString()
|
||||
};
|
||||
activeTasks.set(taskId, task);
|
||||
|
||||
try {
|
||||
// 3. 下发执行
|
||||
const result = await callGatekeeper(server, cmd);
|
||||
const duration = Date.now() - new Date(task.started_at).getTime();
|
||||
|
||||
// 4. 更新任务状态
|
||||
task.status = result.ok ? 'success' : 'error';
|
||||
task.duration_ms = duration;
|
||||
task.exit_code = result.code;
|
||||
|
||||
res.json({
|
||||
ok: result.ok,
|
||||
task_id: taskId,
|
||||
target: allocation.target,
|
||||
stdout: result.stdout,
|
||||
stderr: result.stderr,
|
||||
exit_code: result.code,
|
||||
duration_ms: duration,
|
||||
allocation: {
|
||||
reason: allocation.reason,
|
||||
target: allocation.target
|
||||
}
|
||||
});
|
||||
} catch (err) {
|
||||
task.status = 'error';
|
||||
task.error = err.message;
|
||||
res.status(500).json({ ok: false, error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /pool/tasks — 活跃任务列表
|
||||
*/
|
||||
app.get('/pool/tasks', (req, res) => {
|
||||
const tasks = [];
|
||||
for (const [, task] of activeTasks) {
|
||||
tasks.push(task);
|
||||
}
|
||||
res.json({ tasks, total: tasks.length });
|
||||
});
|
||||
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
// 持久化状态
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
|
||||
function saveState() {
|
||||
try {
|
||||
const data = {
|
||||
updated_at: poolState.updated_at,
|
||||
pool: poolState.pool,
|
||||
nodes: poolState.nodes,
|
||||
};
|
||||
fs.writeFileSync(STATE_FILE, JSON.stringify(data, null, 2), 'utf-8');
|
||||
} catch { /* 静默失败，不影响主服务 */ }
|
||||
}
|
||||
|
||||
function loadState() {
|
||||
try {
|
||||
if (fs.existsSync(STATE_FILE)) {
|
||||
const data = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8'));
|
||||
if (data.pool && data.nodes) {
|
||||
poolState.pool = data.pool;
|
||||
poolState.nodes = data.nodes;
|
||||
}
|
||||
}
|
||||
} catch { /* 静默 */ }
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
// 进程保活
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
|
||||
process.on('uncaughtException', (err) => {
|
||||
console.error('[POOL] 未捕获异常（保活）:', err.message);
|
||||
});
|
||||
process.on('unhandledRejection', (reason) => {
|
||||
console.error('[POOL] 未处理拒绝（保活）:', reason);
|
||||
});
|
||||
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
// 启动
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
|
||||
async function start() {
|
||||
loadState();
|
||||
|
||||
// 立即做第一次轮询
|
||||
console.log('[POOL] 首次轮询所有节点...');
|
||||
await pollAllServers();
|
||||
saveState();
|
||||
|
||||
// 定时轮询
|
||||
setInterval(async () => {
|
||||
await pollAllServers();
|
||||
saveState();
|
||||
}, POLL_INTERVAL);
|
||||
|
||||
app.listen(PORT, '0.0.0.0', () => {
|
||||
console.log('═══════════════════════════════════════════════');
|
||||
console.log(' ☁️ 铸渊 · 算力汇聚调度池 v1.0');
|
||||
console.log(` 端口: ${PORT}`);
|
||||
console.log(` 轮询: ${POLL_INTERVAL / 1000}秒`);
|
||||
console.log(` 节点: ${SERVERS.length}台（${SERVERS.map(s => s.code).join(', ')}）`);
|
||||
console.log(` 汇聚: CPU ${TOTAL_SPEC.cpu}核 · 内存 ${TOTAL_SPEC.mem_gb}GB`);
|
||||
console.log(' 签发: 铸渊 · ICE-GL-ZY001');
|
||||
console.log('═══════════════════════════════════════════════');
|
||||
console.log(' API:');
|
||||
console.log(' GET /pool/status 全局资源状态');
|
||||
console.log(' POST /pool/allocate 声明式分配');
|
||||
console.log(' POST /pool/exec 池中执行');
|
||||
console.log(' GET /pool/tasks 活跃任务');
|
||||
console.log('═══════════════════════════════════════════════');
|
||||
});
|
||||
}
|
||||
|
||||
start().catch(err => {
|
||||
console.error('[POOL] 启动失败:', err.message);
|
||||
process.exit(1);
|
||||
});
|
||||
Loading…
x
Reference in New Issue
Block a user