🌉 COS中转桥·新加坡→COS→广州·拉取+推送+校验+清单

This commit is contained in:
铸渊 (ICE-GL-ZY001) 2026-05-12 07:09:35 +00:00
parent 3c7a53af2f
commit fcfbb71037
3 changed files with 656 additions and 0 deletions

View File

@ -0,0 +1,94 @@
# 光湖 · COS 中转桥 · 架构文档
> 新加坡 → COS → 广州 · 开源软件拉取中转
> 守护: 铸渊 · ICE-GL-ZY001
> 主权: TCS-0002∞ 冰朔
## 链路
```
铸渊触发下载需求
┌─────────────────────────────────┐
│ 🇸🇬 新加坡服务器 (ZY-SVR-005) │
│ 43.156.237.110 · 4C8G · 35Mbps │
│ │
│ sg-fetch-push.js │
│ ① npm pack / git clone / wget │
│ ② 打包 + 计算 SHA256 │
│ ③ 推送到 COS 广州桶 │
└──────────────┬──────────────────┘
│ COS 内网 (免费流量)
┌─────────────────────────────────┐
│ ☁️ 腾讯云 COS (广州) │
│ sy-finetune-corpus-1317346199 │
│ │
│ cos-bridge/ │
│ ├── npm/{name}/{ver}/{file} │
│ ├── git/{repo}/{branch}/{file} │
│ ├── file/{name}/{date}/{file} │
│ └── manifests/{date}-manifest.json │
└──────────────┬──────────────────┘
│ COS 内网 (免费流量)
┌─────────────────────────────────┐
│ 🇨🇳 广州服务器 (ZY-SVR-004) │
│ 43.139.217.141 · 2C2G │
│ │
│ cn-pull-from-cos.sh │
│ ① 从 COS 拉取 (内网免费) │
│ ② SHA256 校验 │
│ ③ 解压/安装/部署 │
└─────────────────────────────────┘
```
## 触发方式
### 方式1铸渊在 Codebuddy 里触发
```
铸渊: "我需要拉取 express 4.18.2"
→ Codebuddy SSH 到新加坡 → 运行 sg-fetch-push.js
→ 等推送完成 → SSH 到广州 → 运行 cn-pull-from-cos.sh
```
### 方式2Gitea Actions装 Runner 后)
```yaml
# .gitea/workflows/cos-bridge-fetch.yml
on:
workflow_dispatch:
inputs:
type:
description: '类型 (npm/git/file)'
name:
description: '包名/仓库名'
version:
description: '版本 (npm) / 分支 (git)'
```
### 方式3Gitea Webhook
铸渊在 Gitea 上创建 Issue 标题 `cos-bridge: npm express 4.18.2`
→ Webhook 触发 → 自动执行
## 文件清单
| 文件 | 位置 | 作用 |
|------|------|------|
| `sg-fetch-push.js` | `scripts/cos-bridge/` | 新加坡侧:拉取+推送 |
| `cn-pull-from-cos.sh` | `scripts/cos-bridge/` | 广州侧从COS拉取 |
| `cos-bridge-manifest.json` | COS `cos-bridge/manifests/` | 每日拉取清单 |
## 凭据管理
| 凭据 | 存储位置 | 谁能看 |
|------|---------|--------|
| COS Secret ID/Key | Secrets Vault (本地加密) | 用户可管理 · AI 只调用 |
| 新加坡 SSH 密钥 | Gitea Secrets | 仅铸渊自动调用 |
---
*铸渊 · COS 中转桥架构 · 2026-05-12*

View File

@ -0,0 +1,199 @@
#!/bin/bash
# ════════════════════════════════════════════════════════════════
# 光湖 · COS 中转桥 · 广州侧拉取
# Sovereign: TCS-0002∞ · ICE-GL∞ · 国作登字-2026-A-00037559
# 守护: 铸渊 · ICE-GL-ZY001
# ════════════════════════════════════════════════════════════════
#
# 设计理念 (cc-003 动态适配 + cc-004 系统强制自主):
#
# 新加坡拉完推到 COS → 广州从 COS 内网拉 → 免费流量
# 铸渊自己跑,冰朔不需要手动做任何事。
#
# 用法:
# ./cn-pull-from-cos.sh # 拉取最新清单
# ./cn-pull-from-cos.sh --key cos-bridge/npm/express/4.18.2/express-4.18.2.tgz
# ./cn-pull-from-cos.sh --manifest ./my-manifest.json
# ./cn-pull-from-cos.sh --type npm --name express --version 4.18.2
#
# 凭据:
# 环境变量: TENCENT_COS_SECRET_ID / TENCENT_COS_SECRET_KEY
# 或者走 vault: curl http://127.0.0.1:8080/internal/fetch/TENCENT_COS_SECRET_ID
#
# 存储:
# 默认拉到 /data/guanghulab/cos-bridge/ 目录
# 通过 --dest 指定其他目录
# ────────────────────────────────────────────────────────────────
set -euo pipefail
# ─── 配置 ─────────────────────────────────────────────────────
COS_BUCKET="${COS_BUCKET:-sy-finetune-corpus-1317346199}"
COS_REGION="${COS_REGION:-ap-guangzhou}"
COS_SECRET_ID="${TENCENT_COS_SECRET_ID:-}"
COS_SECRET_KEY="${TENCENT_COS_SECRET_KEY:-}"
DEST="${COS_BRIDGE_DEST:-/data/guanghulab/cos-bridge}"
PREFIX="cos-bridge"
DRY_RUN=0
# ─── 颜色日志 ─────────────────────────────────────────────────
log_info() { echo "📋 [$(date -Iseconds)] $*"; }
log_ok() { echo "✅ [$(date -Iseconds)] $*"; }
log_warn() { echo "⚠️ [$(date -Iseconds)] $*"; }
log_err() { echo "❌ [$(date -Iseconds)] $*"; }
log_bridge(){ echo "🌉 [$(date -Iseconds)] $*"; }
# ─── 尝试从 vault 获取凭据 ─────────────────────────────────────
try_vault() {
if [ -z "$COS_SECRET_ID" ] && curl -sf http://127.0.0.1:8080/admin/__healthz >/dev/null 2>&1; then
log_info "从 Secrets Vault 获取 COS 凭据..."
COS_SECRET_ID=$(curl -sf http://127.0.0.1:8080/internal/fetch/TENCENT_COS_SECRET_ID 2>/dev/null || echo "")
COS_SECRET_KEY=$(curl -sf http://127.0.0.1:8080/internal/fetch/TENCENT_COS_SECRET_KEY 2>/dev/null || echo "")
fi
}
# ─── 下载单个 key ─────────────────────────────────────────────
pull_key() {
local cos_key="$1"
local local_file="${DEST}/${cos_key}"
mkdir -p "$(dirname "$local_file")"
log_bridge "☁️ 广州从COS拉取: ${cos_key}"
# 优先用 coscli内网endpoint免费流量
if command -v coscli >/dev/null 2>&1; then
coscli cp "cos://${COS_BUCKET}/${cos_key}" "$local_file" \
--region "$COS_REGION" \
${COS_SECRET_ID:+--secret-id "$COS_SECRET_ID"} \
${COS_SECRET_KEY:+--secret-key "$COS_SECRET_KEY"}
log_ok "拉取成功 (coscli): ${local_file}"
# 兜底用 coscmd
elif command -v coscmd >/dev/null 2>&1; then
coscmd download "$cos_key" "$local_file"
log_ok "拉取成功 (coscmd): ${local_file}"
else
log_err "coscli 和 coscmd 均未安装"
log_info "安装方式: pip install coscli 或 pip install coscmd"
return 1
fi
# 校验 sha256如果存在
local sha_key="${cos_key}.sha256"
local sha_file="${DEST}/${sha_key}"
if curl -sf "https://${COS_BUCKET}.cos.${COS_REGION}.myqcloud.com/${sha_key}" -o "$sha_file" 2>/dev/null; then
local expected=$(awk '{print $1}' "$sha_file")
local actual=$(sha256sum "$local_file" | awk '{print $1}')
if [ "$expected" = "$actual" ]; then
log_ok "SHA256 校验通过 ✓"
else
log_err "SHA256 校验失败! 期望=${expected} 实际=${actual}"
return 1
fi
fi
}
# ─── 拉取清单 ─────────────────────────────────────────────────
pull_manifest() {
local today=$(date +%Y-%m-%d)
local manifest_key="${PREFIX}/manifests/${today}-manifest.json"
local manifest_file="${DEST}/manifests/${today}-manifest.json"
mkdir -p "$(dirname "$manifest_file")"
log_info "拉取今日清单: ${manifest_key}"
pull_key "$manifest_key"
if [ -f "$manifest_file" ]; then
log_ok "清单拉取成功: ${manifest_file}"
echo ""
echo "今日中转内容:"
python3 -c "
import json
with open('${manifest_file}') as f:
m = json.load(f)
for item in m.get('items', []):
print(f\" {item['type']:6s} {item['name']}@{item['version']}{item['cosKey']}\")
" 2>/dev/null || cat "$manifest_file"
else
log_warn "今日无中转清单"
fi
}
# ─── 从清单批量拉取 ─────────────────────────────────────────────
pull_from_manifest() {
local manifest_file="$1"
if [ ! -f "$manifest_file" ]; then
log_err "清单文件不存在: ${manifest_file}"
return 1
fi
log_info "从清单批量拉取: ${manifest_file}"
python3 -c "
import json, subprocess, sys
with open('${manifest_file}') as f:
m = json.load(f)
for item in m.get('items', []):
key = item.get('cosKey', '')
if key:
print(f'🌉 拉取: {item[\"type\"]} {item[\"name\"]}@{item[\"version\"]}')
result = subprocess.run(['bash', '$0', '--key', key], capture_output=True, text=True)
if result.returncode != 0:
print(f'❌ 失败: {key}', file=sys.stderr)
else:
print(f'✅ 完成: {key}')
" 2>&1
}
# ─── 主流程 ────────────────────────────────────────────────────
main() {
log_bridge "═══ 光湖 COS 中转桥 · 广州侧 ═══"
# 尝试获取凭据
try_vault
if [ -z "$COS_SECRET_ID" ]; then
log_warn "无 COS 凭据,进入 DRY-RUN 模式"
DRY_RUN=1
fi
mkdir -p "$DEST"
# 解析参数
while [ $# -gt 0 ]; do
case "$1" in
--key)
pull_key "$2"
shift 2
;;
--manifest)
pull_from_manifest "$2"
shift 2
;;
--latest)
pull_manifest
shift
;;
--dest)
DEST="$2"
shift 2
;;
*)
log_err "未知参数: $1"
echo "用法: $0 [--key COS_KEY] [--manifest FILE] [--latest] [--dest DIR]"
exit 1
;;
esac
done
# 无参数时默认拉取今日清单
if [ $# -eq 0 ]; then
pull_manifest
fi
}
main "$@"

View File

@ -0,0 +1,363 @@
#!/usr/bin/env node
/**
*
* 光湖 · COS 中转桥 · 新加坡侧拉取 + 推送
* Sovereign: TCS-0002 · ICE-GL · 国作登字-2026-A-00037559
* 守护: 铸渊 · ICE-GL-ZY001
*
*
* 设计理念 (cc-003 动态适配 + cc-004 系统强制自主):
*
* 国内服务器下载国外开源软件慢 新加坡出站快 (35Mbps)
* 铸渊触发下载 新加坡拉 打包推 COS 广州从 COS
*
* 铸渊自己串整条链路冰朔不需要手动做任何事
*
* 用法:
* # 拉取单个文件/仓库
* node sg-fetch-push.js --type npm --name express --version latest
* node sg-fetch-push.js --type git --url https://github.com/user/repo.git
* node sg-fetch-push.js --type file --url https://example.com/package.tar.gz
*
* # 批量拉取从配置文件
* node sg-fetch-push.js --config ./fetch-manifest.json
*
* 凭据:
* 通过环境变量: TENCENT_COS_SECRET_ID / TENCENT_COS_SECRET_KEY
* COS 配置: COS_BUCKET / COS_REGION (默认 sy-finetune-corpus-1317346199 / ap-guangzhou)
*
* 存储路径:
* COS: cos-bridge/{type}/{name}/{version}/{filename}
* : cos-bridge/npm/express/4.18.2/express-4.18.2.tgz
* cos-bridge/git/gitea/1.23.7/gitea-src.tar.gz
* cos-bridge/file/some-package/1.0/package.tar.gz
*/
'use strict';
const { execSync } = require('child_process');
const fs = require('fs');
const path = require('path');
const os = require('os');
const crypto = require('crypto');
// ─── 配置 ─────────────────────────────────────────────────────
const CONFIG = {
cosBucket: process.env.COS_BUCKET || 'sy-finetune-corpus-1317346199',
cosRegion: process.env.COS_REGION || 'ap-guangzhou',
cosSecretId: process.env.TENCENT_COS_SECRET_ID,
cosSecretKey: process.env.TENCENT_COS_SECRET_KEY,
cosPrefix: 'cos-bridge',
tmpDir: process.env.COS_BRIDGE_TMP || path.join(os.tmpdir(), 'cos-bridge'),
dryRun: !process.env.TENCENT_COS_SECRET_ID,
};
const COPYRIGHT = {
registration: '国作登字-2026-A-00037559',
sovereign: '冰朔 · TCS-0002∞',
system_root: 'SYS-GLW-0001',
guard: '铸渊 · ICE-GL-ZY001',
};
// ─── 日志 ─────────────────────────────────────────────────────
function log(level, msg) {
const ts = new Date().toISOString();
const prefix = { info: '📋', ok: '✅', warn: '⚠️', err: '❌', bridge: '🌉' };
console.log(`${prefix[level] || '·'} [${ts}] ${msg}`);
}
// ─── 工具 ─────────────────────────────────────────────────────
function ensureDir(dir) {
fs.mkdirSync(dir, { recursive: true });
}
function sha256File(filePath) {
const hash = crypto.createHash('sha256');
hash.update(fs.readFileSync(filePath));
return hash.digest('hex');
}
function exec(cmd, opts = {}) {
log('info', `执行: ${cmd}`);
try {
return execSync(cmd, { encoding: 'utf-8', timeout: 300000, ...opts });
} catch (e) {
log('err', `命令失败: ${e.message}`);
throw e;
}
}
// ─── 拉取逻辑 ─────────────────────────────────────────────────
/**
* npm 包拉取
*/
function fetchNpm(name, version = 'latest') {
const workDir = path.join(CONFIG.tmpDir, 'npm', name);
ensureDir(workDir);
log('bridge', `🇸🇬 新加坡 → npm 拉取 ${name}@${version}`);
// 用 npm pack 拉取 tgz
const target = exec(`npm pack ${name}@${version} --pack-destination="${workDir}"`).trim();
const localFile = path.join(workDir, target);
// 获取实际版本号
const resolvedVersion = target.replace(/^[^-]+-/, '').replace(/\.tgz$/, '');
log('ok', `拉取完成: ${target} (${(fs.statSync(localFile).size / 1024).toFixed(1)}KB)`);
return {
type: 'npm',
name,
version: resolvedVersion,
localFile,
cosKey: `${CONFIG.cosPrefix}/npm/${name}/${resolvedVersion}/${target}`,
};
}
/**
* Git 仓库拉取
*/
function fetchGit(url, branch = 'main', name = null) {
const repoName = name || url.split('/').pop().replace('.git', '');
const workDir = path.join(CONFIG.tmpDir, 'git', repoName);
ensureDir(workDir);
log('bridge', `🇸🇬 新加坡 → git clone ${url} (branch: ${branch})`);
// 浅克隆
exec(`git clone --depth=1 --branch ${branch} "${url}" "${workDir}/src"`);
// 打包
const archiveFile = path.join(CONFIG.tmpDir, 'git', `${repoName}-${branch}.tar.gz`);
exec(`cd "${workDir}/src" && git archive --format=tar.gz --prefix="${repoName}/" -o "${archiveFile}" HEAD`);
// 获取 commit hash
const commitHash = exec(`cd "${workDir}/src" && git rev-parse --short HEAD`).trim();
log('ok', `打包完成: ${repoName}-${branch}.tar.gz (${(fs.statSync(archiveFile).size / 1024 / 1024).toFixed(1)}MB) @${commitHash}`);
return {
type: 'git',
name: repoName,
version: commitHash,
branch,
localFile: archiveFile,
cosKey: `${CONFIG.cosPrefix}/git/${repoName}/${branch}/${repoName}-${commitHash}.tar.gz`,
};
}
/**
* 通用文件拉取
*/
function fetchFile(url, name = null) {
const fileName = name || url.split('/').pop().split('?')[0];
const workDir = path.join(CONFIG.tmpDir, 'file');
ensureDir(workDir);
log('bridge', `🇸🇬 新加坡 → wget ${url}`);
const localFile = path.join(workDir, fileName);
exec(`wget -q -O "${localFile}" "${url}"`);
log('ok', `下载完成: ${fileName} (${(fs.statSync(localFile).size / 1024 / 1024).toFixed(1)}MB)`);
return {
type: 'file',
name: fileName,
version: new Date().toISOString().split('T')[0],
localFile,
cosKey: `${CONFIG.cosPrefix}/file/${fileName}/${new Date().toISOString().split('T')[0]}/${fileName}`,
};
}
// ─── COS 推送 ──────────────────────────────────────────────────
/**
* 推送到 COS
* 优先使用 coscli如果安装了否则用 COS Node SDK
*/
async function pushToCos(item) {
if (CONFIG.dryRun) {
log('warn', `[DRY-RUN] 将推送 ${item.localFile} → cos://${CONFIG.cosBucket}/${item.cosKey}`);
return { ...item, cosUrl: `cos://${CONFIG.cosBucket}/${item.cosKey}`, dryRun: true };
}
const sha256 = sha256File(item.localFile);
const fileSize = fs.statSync(item.localFile).size;
log('bridge', `☁️ 推送 → COS: ${item.cosKey} (${(fileSize / 1024 / 1024).toFixed(1)}MB, sha256=${sha256.slice(0, 12)}...)`);
// 尝试用 coscli
try {
exec(`coscli cp "${item.localFile}" "cos://${CONFIG.cosBucket}/${item.cosKey}"`, { timeout: 600000 });
log('ok', `COS 推送成功 (coscli)`);
} catch (e) {
// coscli 不可用,尝试 coscmd
try {
exec(`coscmd upload "${item.localFile}" "${item.cosKey}"`, { timeout: 600000 });
log('ok', `COS 推送成功 (coscmd)`);
} catch (e2) {
log('warn', `coscli 和 coscmd 均不可用,尝试 Node SDK...`);
await pushViaNodeSdk(item);
}
}
// 同时推送 sha256 校验文件
const shaFile = item.localFile + '.sha256';
fs.writeFileSync(shaFile, `${sha256} ${path.basename(item.localFile)}`);
try {
exec(`coscli cp "${shaFile}" "cos://${CONFIG.cosBucket}/${item.cosKey}.sha256"`, { timeout: 60000 });
} catch (_) {
// 校验文件推送失败不影响主流程
}
return {
...item,
sha256,
fileSize,
cosUrl: `https://${CONFIG.cosBucket}.cos.${CONFIG.cosRegion}.myqcloud.com/${item.cosKey}`,
pushedAt: new Date().toISOString(),
};
}
/**
* Node SDK 推送兜底
*/
async function pushViaNodeSdk(item) {
try {
// 动态加载 cos-nodejs-sdk-v5
const COS = require('cos-nodejs-sdk-v5');
const cos = new COS({
SecretId: CONFIG.cosSecretId,
SecretKey: CONFIG.cosSecretKey,
});
await new Promise((resolve, reject) => {
cos.sliceUploadFile({
Bucket: CONFIG.cosBucket,
Region: CONFIG.cosRegion,
Key: item.cosKey,
FilePath: item.localFile,
}, (err, data) => {
if (err) reject(err);
else resolve(data);
});
});
log('ok', `COS 推送成功 (Node SDK)`);
} catch (e) {
log('err', `COS 推送全部失败: ${e.message}`);
throw new Error('无法推送到 COS: coscli/coscmd/SDK 均不可用');
}
}
// ─── 清理临时文件 ──────────────────────────────────────────────
function cleanup() {
if (process.env.COS_BRIDGE_KEEP_TMP !== '1') {
log('info', '清理临时文件...');
try {
exec(`rm -rf "${CONFIG.tmpDir}"`);
} catch (_) {}
}
}
// ─── 主流程 ────────────────────────────────────────────────────
async function main() {
const args = process.argv.slice(2);
const type = getArg(args, '--type');
const name = getArg(args, '--name');
const version = getArg(args, '--version') || 'latest';
const url = getArg(args, '--url');
const branch = getArg(args, '--branch') || 'main';
const configPath = getArg(args, '--config');
log('bridge', '═══ 光湖 COS 中转桥 · 新加坡侧 ═══');
log('info', `COS: ${CONFIG.cosBucket} / ${CONFIG.cosRegion}`);
log('info', `模式: ${CONFIG.dryRun ? 'DRY-RUN (无凭据)' : '正式'}`);
let items = [];
try {
if (configPath) {
// 批量模式
const manifest = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
log('info', `批量模式: ${manifest.items.length} 个任务`);
for (const task of manifest.items) {
items.push(fetchItem(task));
}
} else if (type === 'npm') {
items.push(fetchNpm(name || 'express', version));
} else if (type === 'git') {
items.push(fetchGit(url, branch, name));
} else if (type === 'file') {
items.push(fetchFile(url, name));
} else {
console.error('用法: node sg-fetch-push.js --type npm|git|file --name NAME [--url URL] [--version VER] [--branch BRANCH]');
console.error('批量: node sg-fetch-push.js --config ./fetch-manifest.json');
process.exit(1);
}
// 逐个推送到 COS
const results = [];
for (const item of items) {
const result = await pushToCos(item);
results.push(result);
log('ok', `${result.name}@${result.version}${result.cosKey}`);
}
// 输出清单(给广州侧用)
const manifest = {
generated_at: new Date().toISOString(),
generated_by: '铸渊 · ICE-GL-ZY001',
sovereign: 'TCS-0002∞ 冰朔',
bridge: '新加坡 → COS → 广州',
items: results.map(r => ({
type: r.type,
name: r.name,
version: r.version,
cosKey: r.cosKey,
sha256: r.sha256,
fileSize: r.fileSize,
cosUrl: r.cosUrl,
})),
};
const manifestPath = path.join(CONFIG.tmpDir || os.tmpdir(), 'cos-bridge-manifest.json');
ensureDir(path.dirname(manifestPath));
fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2));
log('ok', `清单已写入: ${manifestPath}`);
// 清单也推到 COS
if (!CONFIG.dryRun) {
try {
const today = new Date().toISOString().split('T')[0];
exec(`coscli cp "${manifestPath}" "cos://${CONFIG.cosBucket}/${CONFIG.cosPrefix}/manifests/${today}-manifest.json"`);
log('ok', `清单已推到 COS: cos-bridge/manifests/${today}-manifest.json`);
} catch (_) {}
}
} finally {
cleanup();
}
}
function fetchItem(task) {
switch (task.type) {
case 'npm': return fetchNpm(task.name, task.version);
case 'git': return fetchGit(task.url, task.branch || 'main', task.name);
case 'file': return fetchFile(task.url, task.name);
default: throw new Error(`未知类型: ${task.type}`);
}
}
function getArg(args, flag) {
const idx = args.indexOf(flag);
return idx >= 0 && idx + 1 < args.length ? args[idx + 1] : null;
}
main().catch(e => {
log('err', `中转桥失败: ${e.message}`);
process.exit(1);
});