364 lines
12 KiB
JavaScript
364 lines
12 KiB
JavaScript
#!/usr/bin/env node
|
||
/**
|
||
* ════════════════════════════════════════════════════════════════
|
||
* 光湖 · COS 中转桥 · 新加坡侧拉取 + 推送
|
||
* Sovereign: TCS-0002∞ · ICE-GL∞ · 国作登字-2026-A-00037559
|
||
* 守护: 铸渊 · ICE-GL-ZY001
|
||
* ════════════════════════════════════════════════════════════════
|
||
*
|
||
* 设计理念 (cc-003 动态适配 + cc-004 系统强制自主):
|
||
*
|
||
* 国内服务器下载国外开源软件慢 → 新加坡出站快 (35Mbps)
|
||
* 铸渊触发下载 → 新加坡拉 → 打包推 COS → 广州从 COS 拉
|
||
*
|
||
* 铸渊自己串整条链路，冰朔不需要手动做任何事。
|
||
*
|
||
* 用法:
|
||
* # 拉取单个文件/仓库
|
||
* node sg-fetch-push.js --type npm --name express --version latest
|
||
* node sg-fetch-push.js --type git --url https://github.com/user/repo.git
|
||
* node sg-fetch-push.js --type file --url https://example.com/package.tar.gz
|
||
*
|
||
* # 批量拉取（从配置文件）
|
||
* node sg-fetch-push.js --config ./fetch-manifest.json
|
||
*
|
||
* 凭据:
|
||
* 通过环境变量: TENCENT_COS_SECRET_ID / TENCENT_COS_SECRET_KEY
|
||
* COS 配置: COS_BUCKET / COS_REGION (默认 sy-finetune-corpus-1317346199 / ap-guangzhou)
|
||
*
|
||
* 存储路径:
|
||
* COS: cos-bridge/{type}/{name}/{version}/{filename}
|
||
* 例: cos-bridge/npm/express/4.18.2/express-4.18.2.tgz
|
||
* cos-bridge/git/gitea/1.23.7/gitea-src.tar.gz
|
||
* cos-bridge/file/some-package/1.0/package.tar.gz
|
||
*/
|
||
|
||
'use strict';
|
||
|
||
const { execSync } = require('child_process');
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
const os = require('os');
|
||
const crypto = require('crypto');
|
||
|
||
// ─── 配置 ─────────────────────────────────────────────────────
|
||
const CONFIG = {
|
||
cosBucket: process.env.COS_BUCKET || 'sy-finetune-corpus-1317346199',
|
||
cosRegion: process.env.COS_REGION || 'ap-guangzhou',
|
||
cosSecretId: process.env.TENCENT_COS_SECRET_ID,
|
||
cosSecretKey: process.env.TENCENT_COS_SECRET_KEY,
|
||
cosPrefix: 'cos-bridge',
|
||
tmpDir: process.env.COS_BRIDGE_TMP || path.join(os.tmpdir(), 'cos-bridge'),
|
||
dryRun: !process.env.TENCENT_COS_SECRET_ID,
|
||
};
|
||
|
||
const COPYRIGHT = {
|
||
registration: '国作登字-2026-A-00037559',
|
||
sovereign: '冰朔 · TCS-0002∞',
|
||
system_root: 'SYS-GLW-0001',
|
||
guard: '铸渊 · ICE-GL-ZY001',
|
||
};
|
||
|
||
// ─── 日志 ─────────────────────────────────────────────────────
|
||
function log(level, msg) {
|
||
const ts = new Date().toISOString();
|
||
const prefix = { info: '📋', ok: '✅', warn: '⚠️', err: '❌', bridge: '🌉' };
|
||
console.log(`${prefix[level] || '·'} [${ts}] ${msg}`);
|
||
}
|
||
|
||
// ─── 工具 ─────────────────────────────────────────────────────
|
||
function ensureDir(dir) {
|
||
fs.mkdirSync(dir, { recursive: true });
|
||
}
|
||
|
||
function sha256File(filePath) {
|
||
const hash = crypto.createHash('sha256');
|
||
hash.update(fs.readFileSync(filePath));
|
||
return hash.digest('hex');
|
||
}
|
||
|
||
function exec(cmd, opts = {}) {
|
||
log('info', `执行: ${cmd}`);
|
||
try {
|
||
return execSync(cmd, { encoding: 'utf-8', timeout: 300000, ...opts });
|
||
} catch (e) {
|
||
log('err', `命令失败: ${e.message}`);
|
||
throw e;
|
||
}
|
||
}
|
||
|
||
// ─── 拉取逻辑 ─────────────────────────────────────────────────
|
||
|
||
/**
|
||
* npm 包拉取
|
||
*/
|
||
function fetchNpm(name, version = 'latest') {
|
||
const workDir = path.join(CONFIG.tmpDir, 'npm', name);
|
||
ensureDir(workDir);
|
||
|
||
log('bridge', `🇸🇬 新加坡 → npm 拉取 ${name}@${version}`);
|
||
|
||
// 用 npm pack 拉取 tgz
|
||
const target = exec(`npm pack ${name}@${version} --pack-destination="${workDir}"`).trim();
|
||
const localFile = path.join(workDir, target);
|
||
|
||
// 获取实际版本号
|
||
const resolvedVersion = target.replace(/^[^-]+-/, '').replace(/\.tgz$/, '');
|
||
|
||
log('ok', `拉取完成: ${target} (${(fs.statSync(localFile).size / 1024).toFixed(1)}KB)`);
|
||
|
||
return {
|
||
type: 'npm',
|
||
name,
|
||
version: resolvedVersion,
|
||
localFile,
|
||
cosKey: `${CONFIG.cosPrefix}/npm/${name}/${resolvedVersion}/${target}`,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Git 仓库拉取
|
||
*/
|
||
function fetchGit(url, branch = 'main', name = null) {
|
||
const repoName = name || url.split('/').pop().replace('.git', '');
|
||
const workDir = path.join(CONFIG.tmpDir, 'git', repoName);
|
||
ensureDir(workDir);
|
||
|
||
log('bridge', `🇸🇬 新加坡 → git clone ${url} (branch: ${branch})`);
|
||
|
||
// 浅克隆
|
||
exec(`git clone --depth=1 --branch ${branch} "${url}" "${workDir}/src"`);
|
||
|
||
// 打包
|
||
const archiveFile = path.join(CONFIG.tmpDir, 'git', `${repoName}-${branch}.tar.gz`);
|
||
exec(`cd "${workDir}/src" && git archive --format=tar.gz --prefix="${repoName}/" -o "${archiveFile}" HEAD`);
|
||
|
||
// 获取 commit hash
|
||
const commitHash = exec(`cd "${workDir}/src" && git rev-parse --short HEAD`).trim();
|
||
|
||
log('ok', `打包完成: ${repoName}-${branch}.tar.gz (${(fs.statSync(archiveFile).size / 1024 / 1024).toFixed(1)}MB) @${commitHash}`);
|
||
|
||
return {
|
||
type: 'git',
|
||
name: repoName,
|
||
version: commitHash,
|
||
branch,
|
||
localFile: archiveFile,
|
||
cosKey: `${CONFIG.cosPrefix}/git/${repoName}/${branch}/${repoName}-${commitHash}.tar.gz`,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 通用文件拉取
|
||
*/
|
||
function fetchFile(url, name = null) {
|
||
const fileName = name || url.split('/').pop().split('?')[0];
|
||
const workDir = path.join(CONFIG.tmpDir, 'file');
|
||
ensureDir(workDir);
|
||
|
||
log('bridge', `🇸🇬 新加坡 → wget ${url}`);
|
||
|
||
const localFile = path.join(workDir, fileName);
|
||
exec(`wget -q -O "${localFile}" "${url}"`);
|
||
|
||
log('ok', `下载完成: ${fileName} (${(fs.statSync(localFile).size / 1024 / 1024).toFixed(1)}MB)`);
|
||
|
||
return {
|
||
type: 'file',
|
||
name: fileName,
|
||
version: new Date().toISOString().split('T')[0],
|
||
localFile,
|
||
cosKey: `${CONFIG.cosPrefix}/file/${fileName}/${new Date().toISOString().split('T')[0]}/${fileName}`,
|
||
};
|
||
}
|
||
|
||
// ─── COS 推送 ──────────────────────────────────────────────────
|
||
|
||
/**
|
||
* 推送到 COS
|
||
* 优先使用 coscli（如果安装了），否则用 COS Node SDK
|
||
*/
|
||
async function pushToCos(item) {
|
||
if (CONFIG.dryRun) {
|
||
log('warn', `[DRY-RUN] 将推送 ${item.localFile} → cos://${CONFIG.cosBucket}/${item.cosKey}`);
|
||
return { ...item, cosUrl: `cos://${CONFIG.cosBucket}/${item.cosKey}`, dryRun: true };
|
||
}
|
||
|
||
const sha256 = sha256File(item.localFile);
|
||
const fileSize = fs.statSync(item.localFile).size;
|
||
|
||
log('bridge', `☁️ 推送 → COS: ${item.cosKey} (${(fileSize / 1024 / 1024).toFixed(1)}MB, sha256=${sha256.slice(0, 12)}...)`);
|
||
|
||
// 尝试用 coscli
|
||
try {
|
||
exec(`coscli cp "${item.localFile}" "cos://${CONFIG.cosBucket}/${item.cosKey}"`, { timeout: 600000 });
|
||
log('ok', `COS 推送成功 (coscli)`);
|
||
} catch (e) {
|
||
// coscli 不可用，尝试 coscmd
|
||
try {
|
||
exec(`coscmd upload "${item.localFile}" "${item.cosKey}"`, { timeout: 600000 });
|
||
log('ok', `COS 推送成功 (coscmd)`);
|
||
} catch (e2) {
|
||
log('warn', `coscli 和 coscmd 均不可用，尝试 Node SDK...`);
|
||
await pushViaNodeSdk(item);
|
||
}
|
||
}
|
||
|
||
// 同时推送 sha256 校验文件
|
||
const shaFile = item.localFile + '.sha256';
|
||
fs.writeFileSync(shaFile, `${sha256} ${path.basename(item.localFile)}`);
|
||
|
||
try {
|
||
exec(`coscli cp "${shaFile}" "cos://${CONFIG.cosBucket}/${item.cosKey}.sha256"`, { timeout: 60000 });
|
||
} catch (_) {
|
||
// 校验文件推送失败不影响主流程
|
||
}
|
||
|
||
return {
|
||
...item,
|
||
sha256,
|
||
fileSize,
|
||
cosUrl: `https://${CONFIG.cosBucket}.cos.${CONFIG.cosRegion}.myqcloud.com/${item.cosKey}`,
|
||
pushedAt: new Date().toISOString(),
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Node SDK 推送（兜底）
|
||
*/
|
||
async function pushViaNodeSdk(item) {
|
||
try {
|
||
// 动态加载 cos-nodejs-sdk-v5
|
||
const COS = require('cos-nodejs-sdk-v5');
|
||
const cos = new COS({
|
||
SecretId: CONFIG.cosSecretId,
|
||
SecretKey: CONFIG.cosSecretKey,
|
||
});
|
||
|
||
await new Promise((resolve, reject) => {
|
||
cos.sliceUploadFile({
|
||
Bucket: CONFIG.cosBucket,
|
||
Region: CONFIG.cosRegion,
|
||
Key: item.cosKey,
|
||
FilePath: item.localFile,
|
||
}, (err, data) => {
|
||
if (err) reject(err);
|
||
else resolve(data);
|
||
});
|
||
});
|
||
log('ok', `COS 推送成功 (Node SDK)`);
|
||
} catch (e) {
|
||
log('err', `COS 推送全部失败: ${e.message}`);
|
||
throw new Error('无法推送到 COS: coscli/coscmd/SDK 均不可用');
|
||
}
|
||
}
|
||
|
||
// ─── 清理临时文件 ──────────────────────────────────────────────
|
||
function cleanup() {
|
||
if (process.env.COS_BRIDGE_KEEP_TMP !== '1') {
|
||
log('info', '清理临时文件...');
|
||
try {
|
||
exec(`rm -rf "${CONFIG.tmpDir}"`);
|
||
} catch (_) {}
|
||
}
|
||
}
|
||
|
||
// ─── 主流程 ────────────────────────────────────────────────────
|
||
async function main() {
|
||
const args = process.argv.slice(2);
|
||
const type = getArg(args, '--type');
|
||
const name = getArg(args, '--name');
|
||
const version = getArg(args, '--version') || 'latest';
|
||
const url = getArg(args, '--url');
|
||
const branch = getArg(args, '--branch') || 'main';
|
||
const configPath = getArg(args, '--config');
|
||
|
||
log('bridge', '═══ 光湖 COS 中转桥 · 新加坡侧 ═══');
|
||
log('info', `COS: ${CONFIG.cosBucket} / ${CONFIG.cosRegion}`);
|
||
log('info', `模式: ${CONFIG.dryRun ? 'DRY-RUN (无凭据)' : '正式'}`);
|
||
|
||
let items = [];
|
||
|
||
try {
|
||
if (configPath) {
|
||
// 批量模式
|
||
const manifest = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
|
||
log('info', `批量模式: ${manifest.items.length} 个任务`);
|
||
for (const task of manifest.items) {
|
||
items.push(fetchItem(task));
|
||
}
|
||
} else if (type === 'npm') {
|
||
items.push(fetchNpm(name || 'express', version));
|
||
} else if (type === 'git') {
|
||
items.push(fetchGit(url, branch, name));
|
||
} else if (type === 'file') {
|
||
items.push(fetchFile(url, name));
|
||
} else {
|
||
console.error('用法: node sg-fetch-push.js --type npm|git|file --name NAME [--url URL] [--version VER] [--branch BRANCH]');
|
||
console.error('批量: node sg-fetch-push.js --config ./fetch-manifest.json');
|
||
process.exit(1);
|
||
}
|
||
|
||
// 逐个推送到 COS
|
||
const results = [];
|
||
for (const item of items) {
|
||
const result = await pushToCos(item);
|
||
results.push(result);
|
||
log('ok', `✅ ${result.name}@${result.version} → ${result.cosKey}`);
|
||
}
|
||
|
||
// 输出清单（给广州侧用）
|
||
const manifest = {
|
||
generated_at: new Date().toISOString(),
|
||
generated_by: '铸渊 · ICE-GL-ZY001',
|
||
sovereign: 'TCS-0002∞ 冰朔',
|
||
bridge: '新加坡 → COS → 广州',
|
||
items: results.map(r => ({
|
||
type: r.type,
|
||
name: r.name,
|
||
version: r.version,
|
||
cosKey: r.cosKey,
|
||
sha256: r.sha256,
|
||
fileSize: r.fileSize,
|
||
cosUrl: r.cosUrl,
|
||
})),
|
||
};
|
||
|
||
const manifestPath = path.join(CONFIG.tmpDir || os.tmpdir(), 'cos-bridge-manifest.json');
|
||
ensureDir(path.dirname(manifestPath));
|
||
fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2));
|
||
log('ok', `清单已写入: ${manifestPath}`);
|
||
|
||
// 清单也推到 COS
|
||
if (!CONFIG.dryRun) {
|
||
try {
|
||
const today = new Date().toISOString().split('T')[0];
|
||
exec(`coscli cp "${manifestPath}" "cos://${CONFIG.cosBucket}/${CONFIG.cosPrefix}/manifests/${today}-manifest.json"`);
|
||
log('ok', `清单已推到 COS: cos-bridge/manifests/${today}-manifest.json`);
|
||
} catch (_) {}
|
||
}
|
||
|
||
} finally {
|
||
cleanup();
|
||
}
|
||
}
|
||
|
||
function fetchItem(task) {
|
||
switch (task.type) {
|
||
case 'npm': return fetchNpm(task.name, task.version);
|
||
case 'git': return fetchGit(task.url, task.branch || 'main', task.name);
|
||
case 'file': return fetchFile(task.url, task.name);
|
||
default: throw new Error(`未知类型: ${task.type}`);
|
||
}
|
||
}
|
||
|
||
function getArg(args, flag) {
|
||
const idx = args.indexOf(flag);
|
||
return idx >= 0 && idx + 1 < args.length ? args[idx + 1] : null;
|
||
}
|
||
|
||
main().catch(e => {
|
||
log('err', `中转桥失败: ${e.message}`);
|
||
process.exit(1);
|
||
});
|