86f9708059
- ✅ CHAR-HERO-DESIGN-PACKER (char-hero-design-packer.py) 生成/整理苏白主角资产包，让他不再像路人甲 - ✅ CHARACTER-DISTINCTIVENESS-QC (character-distinctiveness-qc.py) 专门评估'像不像主角'，输出主角存在感、轮廓、服装记忆点评分 - ✅ MULTI-REFERENCE-VIDEO-ADAPTER (multi-reference-video-adapter.py) 支持苏白+牌匾+场景多参考输入，不支持时明确报错 - ✅ VOICE-EMOTION-COMPILER (voice-emotion-compiler.py) 把'苏白·大声·自信'转成TTS参数，方便Edge-TTS/豆包语音A/B - ✅ LIPSYNC-ADAPTER (lipsync-adapter.py) 接视频改口型或Wav2Lip，解决人物真正说台词的问题 - ✅ AUDIO-MIXER (audio-mixer.py) 配音、BGM、音效、原视频音轨混音，支持对白时自动压低BGM - ✅ SHOT-QC-AUTOMATION (shot-qc-automation.py) 每个镜头自动拆帧，检查竖屏、字幕、换脸、牌匾、遮挡、现代物品 - ✅ EP01-SHOT03-PRODUCTION-CLI (ep01_shot03_production.py) 一键跑苏白站牌匾下说台词：生成底片、合成牌匾、配音、口型、字幕、混音、质检 冰朔 TCS-0002∞ 见证 · 国作登字-2026-A-00037559 ⊢ 铸渊 ICE-GL-ZY001 · D144 · 2026-06-24
375 lines
13 KiB
Python
375 lines
13 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
MULTI-REFERENCE-VIDEO-ADAPTER
|
||
多参考图视频适配器 — 支持苏白+牌匾+场景多参考输入。
|
||
|
||
功能:
|
||
1. 检查视频API是否支持多参考图输入
|
||
2. 如果支持: 封装多参考图接口，统一调用
|
||
3. 如果不支持: 明确报错，回退到"单参考图+后期合成"路线
|
||
4. 提供统一的调用接口给上游 Agent
|
||
|
||
用法:
|
||
python multi-reference-video-adapter.py --prompt "苏白站在天道宗牌匾下" \\
|
||
--references char-003-subai.png tdz-plaque.png env-baizonghui.png \\
|
||
--output output.mp4
|
||
|
||
检查API能力:
|
||
python multi-reference-video-adapter.py --check-api
|
||
"""
|
||
|
||
import os
|
||
import sys
|
||
import json
|
||
import argparse
|
||
import requests
|
||
from pathlib import Path
|
||
from datetime import datetime
|
||
|
||
PROJECT_ROOT = Path(__file__).parent.parent
|
||
sys.path.insert(0, str(PROJECT_ROOT / "engines"))
|
||
|
||
# 从环境变量加载 API 配置
|
||
def load_api_config():
|
||
"""从 video-ai-system/.env 加载配置"""
|
||
config = {}
|
||
env_file = PROJECT_ROOT / ".env"
|
||
if env_file.exists():
|
||
with open(env_file, "r") as f:
|
||
for line in f:
|
||
line = line.strip()
|
||
if line and not line.startswith("#"):
|
||
key, _, val = line.partition("=")
|
||
config[key.strip()] = val.strip()
|
||
return config
|
||
|
||
|
||
class MultiReferenceVideoAdapter:
|
||
"""多参考图视频适配器"""
|
||
|
||
def __init__(self):
|
||
self.config = load_api_config()
|
||
self.api_key = self.config.get("JIMENG_API_KEY", "")
|
||
self.base_url = self.config.get("JIMENG_BASE_URL", "https://ark.cn-beijing.volces.com/api/v3")
|
||
self.model = self.config.get("JIMENG_MODEL", "doubao-seedance-2-0-260128")
|
||
|
||
# API 能力探测结果缓存
|
||
self._api_capabilities = None
|
||
|
||
def check_api_capabilities(self):
|
||
"""
|
||
检查 API 是否支持多参考图
|
||
返回: {
|
||
"multi_reference_supported": bool,
|
||
"max_references": int,
|
||
"supported_types": list, # ["image_url", "image_url_2", ...]
|
||
"details": str
|
||
}
|
||
"""
|
||
if self._api_capabilities:
|
||
return self._api_capabilities
|
||
|
||
print("🔍 检查 Seedance API 多参考图支持...")
|
||
|
||
# 根据 Volcengine 官方文档 (https://www.volcengine.com/docs/82379/1520757)
|
||
# Seedance 2.0 API 的 content 数组支持多个 image_url 对象
|
||
# 但需要实际测试确认
|
||
|
||
# 理论上的 API 结构:
|
||
# content: [
|
||
# { type: "text", text: "..." },
|
||
# { type: "image_url", image_url: { url: "data:image/png;base64,..." } },
|
||
# { type: "image_url", image_url: { url: "data:image/png;base64,..." } }, # 第二张参考图
|
||
# ]
|
||
|
||
# 实际测试: 尝试提交一个包含2张参考图的请求，看是否报错
|
||
test_result = self._test_multi_reference()
|
||
|
||
self._api_capabilities = test_result
|
||
return test_result
|
||
|
||
def _test_multi_reference(self):
|
||
"""
|
||
实际测试 API 是否支持多参考图
|
||
方法: 提交一个测试请求，包含2张参考图，观察响应
|
||
"""
|
||
# 构造一个最小测试请求
|
||
test_prompt = "test multi-reference support"
|
||
|
||
# 创建1x1像素的测试图片 (PNG)
|
||
import base64
|
||
from io import BytesIO
|
||
try:
|
||
from PIL import Image
|
||
img = Image.new("RGB", (32, 32), color=(255, 0, 0))
|
||
buf = BytesIO()
|
||
img.save(buf, format="PNG")
|
||
test_img_b64 = base64.b64encode(buf.getvalue()).decode()
|
||
except ImportError:
|
||
# 如果没有 PIL，用空base64
|
||
test_img_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWg"
|
||
|
||
# 构造 content 数组 (2张参考图)
|
||
content = [
|
||
{"type": "text", "text": test_prompt},
|
||
{"type": "image_url", "image_url": {"url": f"data:image/png;base64,{test_img_b64}"}},
|
||
{"type": "image_url", "image_url": {"url": f"data:image/png;base64,{test_img_b64}"}},
|
||
]
|
||
|
||
payload = {
|
||
"model": self.model,
|
||
"content": content,
|
||
"duration": 4, # 最短时长，省钱
|
||
"resolution": "480p"
|
||
}
|
||
|
||
# 发送请求
|
||
try:
|
||
print(" 📤 发送测试请求 (2张参考图)...")
|
||
url = f"{self.base_url}/contents/generations/tasks"
|
||
headers = {
|
||
"Authorization": f"Bearer {self.api_key}",
|
||
"Content-Type": "application/json"
|
||
}
|
||
|
||
response = requests.post(url, json=payload, headers=headers, timeout=30)
|
||
|
||
if response.status_code == 200:
|
||
# 成功！API 支持多参考图
|
||
print(" ✅ API 支持多参考图！")
|
||
return {
|
||
"multi_reference_supported": True,
|
||
"max_references": 2, # 需要逐步测试确定上限
|
||
"supported_types": ["image_url"],
|
||
"details": "API 成功接受2张参考图"
|
||
}
|
||
elif response.status_code == 400:
|
||
# 看错误信息
|
||
error_data = response.json()
|
||
error_msg = error_data.get("error", {}).get("message", "")
|
||
print(f" ❌ API 不支持多参考图: {error_msg}")
|
||
return {
|
||
"multi_reference_supported": False,
|
||
"max_references": 1,
|
||
"supported_types": ["image_url"], # 只支持单张
|
||
"details": error_msg,
|
||
"error_response": error_data
|
||
}
|
||
else:
|
||
print(f" ⚠️ 未知响应: {response.status_code}")
|
||
return {
|
||
"multi_reference_supported": False,
|
||
"max_references": 1,
|
||
"details": f"Unknown response: {response.status_code}"
|
||
}
|
||
|
||
except Exception as e:
|
||
print(f" ❌ 测试失败: {e}")
|
||
return {
|
||
"multi_reference_supported": False,
|
||
"max_references": 1,
|
||
"details": f"Test failed: {e}"
|
||
}
|
||
|
||
def generate_video(self, prompt, reference_images, output_path=None, duration=5, resolution="720p"):
|
||
"""
|
||
生成视频 (多参考图)
|
||
|
||
参数:
|
||
prompt: str - 提示词
|
||
reference_images: list[str] - 参考图路径列表
|
||
output_path: str - 输出路径
|
||
duration: int - 时长 (4-15)
|
||
resolution: str - 分辨率 ("480p" | "720p")
|
||
|
||
返回:
|
||
{
|
||
"success": bool,
|
||
"task_id": str,
|
||
"output_path": str,
|
||
"method": str, # "multi-reference" | "single-reference+composite"
|
||
"warning": str
|
||
}
|
||
"""
|
||
print(f"\n🎬 生成视频 (多参考图)")
|
||
print(f" 提示词: {prompt[:60]}...")
|
||
print(f" 参考图数量: {len(reference_images)}")
|
||
for i, img in enumerate(reference_images):
|
||
print(f" [{i+1}] {Path(img).name}")
|
||
|
||
# 检查 API 能力
|
||
capabilities = self.check_api_capabilities()
|
||
|
||
if capabilities["multi_reference_supported"]:
|
||
# API 支持多参考图 → 直接调用
|
||
print(f"\n ✅ API 支持多参考图，直接调用...")
|
||
result = self._generate_multi_reference(prompt, reference_images, output_path, duration, resolution)
|
||
result["method"] = "multi-reference"
|
||
return result
|
||
else:
|
||
# API 不支持多参考图 → 明确报错 + 建议回退方案
|
||
print(f"\n ❌ API 不支持多参考图")
|
||
print(f" 📋 错误详情: {capabilities['details']}")
|
||
print(f"\n 💡 回退方案:")
|
||
print(f" 1. 使用第一张参考图 (苏白) 生成视频")
|
||
print(f" 2. 后期合成牌匾/场景 (平面追踪 + 贴图)")
|
||
print(f" 3. 或使用可灵生成角色，Seedance 生成场景，后期合成")
|
||
|
||
# 回退: 只用第一张参考图
|
||
warning = "API不支持多参考图，已回退到单参考图模式。牌匾/场景一致性需要后期合成。"
|
||
print(f"\n 🔄 回退: 使用第一张参考图生成...")
|
||
|
||
result = self._generate_single_reference(prompt, reference_images[0], output_path, duration, resolution)
|
||
result["method"] = "single-reference+composite"
|
||
result["warning"] = warning
|
||
result["fallback_reason"] = capabilities["details"]
|
||
|
||
return result
|
||
|
||
def _generate_multi_reference(self, prompt, reference_images, output_path, duration, resolution):
|
||
"""调用多参考图 API"""
|
||
# 构造 content 数组
|
||
content = [{"type": "text", "text": prompt}]
|
||
|
||
for img_path in reference_images:
|
||
img_path = Path(img_path)
|
||
if not img_path.exists():
|
||
print(f" ⚠️ 参考图不存在: {img_path}")
|
||
continue
|
||
|
||
# 读取图片并转 base64
|
||
import base64
|
||
with open(img_path, "rb") as f:
|
||
img_data = f.read()
|
||
b64 = base64.b64encode(img_data).decode()
|
||
mime = "image/png" if img_path.suffix.lower() == ".png" else "image/jpeg"
|
||
|
||
content.append({
|
||
"type": "image_url",
|
||
"image_url": {"url": f"data:{mime};base64,{b64}"}
|
||
})
|
||
|
||
# 调用 API
|
||
payload = {
|
||
"model": self.model,
|
||
"content": content,
|
||
"duration": duration,
|
||
"resolution": resolution
|
||
}
|
||
|
||
print(f" 📤 提交任务...")
|
||
url = f"{self.base_url}/contents/generations/tasks"
|
||
headers = {
|
||
"Authorization": f"Bearer {self.api_key}",
|
||
"Content-Type": "application/json"
|
||
}
|
||
|
||
response = requests.post(url, json=payload, headers=headers, timeout=60)
|
||
response.raise_for_status()
|
||
data = response.json()
|
||
|
||
task_id = data.get("id") or data.get("task_id")
|
||
print(f" ✅ 任务已提交: {task_id}")
|
||
|
||
# 返回任务ID，等待轮询
|
||
return {
|
||
"success": True,
|
||
"task_id": task_id,
|
||
"output_path": output_path,
|
||
"api_response": data
|
||
}
|
||
|
||
def _generate_single_reference(self, prompt, reference_image, output_path, duration, resolution):
|
||
"""回退: 单参考图生成"""
|
||
# 调用现有的 video-api-adapter (Node.js)
|
||
# 这里用 subprocess 调用
|
||
import subprocess
|
||
|
||
print(f" 📤 调用单参考图 API...")
|
||
|
||
# 构造调用参数
|
||
node_script = PROJECT_ROOT / "engines" / "video-api-adapter.js"
|
||
cmd = [
|
||
"node", str(node_script),
|
||
"--prompt", prompt,
|
||
"--reference-image", str(reference_image),
|
||
"--duration", str(duration),
|
||
"--resolution", resolution
|
||
]
|
||
|
||
# 执行
|
||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||
if result.returncode != 0:
|
||
print(f" ❌ 调用失败: {result.stderr}")
|
||
return {"success": False, "error": result.stderr}
|
||
|
||
print(f" ✅ 任务已提交")
|
||
return {"success": True, "method": "single-reference", "stdout": result.stdout}
|
||
|
||
def batch_generate(self, shots_config):
|
||
"""
|
||
批量生成 (从配置文件)
|
||
|
||
shots_config 格式:
|
||
[
|
||
{
|
||
"shot_id": "E1-SHOT01",
|
||
"prompt": "...",
|
||
"references": ["char.png", "prop.png", "env.png"],
|
||
"output": "output/E1-SHOT01.mp4"
|
||
},
|
||
...
|
||
]
|
||
"""
|
||
results = []
|
||
for shot in shots_config:
|
||
result = self.generate_video(
|
||
prompt=shot["prompt"],
|
||
reference_images=shot["references"],
|
||
output_path=shot["output"]
|
||
)
|
||
results.append(result)
|
||
return results
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description="MULTI-REFERENCE-VIDEO-ADAPTER")
|
||
parser.add_argument("--check-api", action="store_true", help="检查API多参考图支持")
|
||
parser.add_argument("--prompt", type=str, help="提示词")
|
||
parser.add_argument("--references", type=str, nargs="+", help="参考图路径列表")
|
||
parser.add_argument("--output", type=str, help="输出路径")
|
||
parser.add_argument("--duration", type=int, default=5, help="时长 (4-15)")
|
||
parser.add_argument("--resolution", type=str, default="720p", choices=["480p", "720p"], help="分辨率")
|
||
|
||
args = parser.parse_args()
|
||
|
||
adapter = MultiReferenceVideoAdapter()
|
||
|
||
if args.check_api:
|
||
capabilities = adapter.check_api_capabilities()
|
||
print(f"\n📊 API 能力报告:")
|
||
print(f" 多参考图支持: {capabilities['multi_reference_supported']}")
|
||
print(f" 最大参考图数: {capabilities['max_references']}")
|
||
print(f" 详情: {capabilities['details']}")
|
||
return
|
||
|
||
if not args.prompt or not args.references:
|
||
parser.print_help()
|
||
return
|
||
|
||
result = adapter.generate_video(
|
||
prompt=args.prompt,
|
||
reference_images=args.references,
|
||
output_path=args.output,
|
||
duration=args.duration,
|
||
resolution=args.resolution
|
||
)
|
||
|
||
print(f"\n📋 生成结果:")
|
||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|