cang-ying/engines/lipsync-adapter.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
LIPSYNC-ADAPTER
口型适配器 — 接视频改口型或 Wav2Lip，解决人物真正说台词的问题。

功能:
1. 输入视频 + 对白音频
2. 用 Wav2Lip 开源工具做口型同步
3. 支持批量处理
4. 封装为统一接口

依赖:
  pip install librosa opencv-python numpy
  # Wav2Lip 需要单独安装: https://github.com/Rudrabha/Wav2Lip

用法:
  python lipsync-adapter.py --video input.mp4 --audio dialogue.mp3 --output output.mp4
  python lipsync-adapter.py --batch video_list.json
"""

import os
import sys
import json
import argparse
from pathlib import Path
from datetime import datetime

PROJECT_ROOT = Path(__file__).parent.parent
sys.path.insert(0, str(PROJECT_ROOT / "engines"))


class LipSyncAdapter:
    """口型适配器 — API驱动: 火山改口型(主)→Edge-TTS(配音回退)"""

    def __init__(self, allow_fallback=False):
        # 光湖不本地装模型。所有模型走API。
        self.allow_fallback = allow_fallback
        self._cloud_available = None

    def sync_lips(self, video_path, audio_path, output_path=None, allow_fallback=None):
        """
        口型同步 — API驱动

        路线: 火山引擎视频改口型(主) → Edge-TTS配音(回退)

        返回:
          {success, output_path, method, warning/error}
        """
        print(f"\n🎤 口型同步")
        print(f"   视频: {Path(video_path).name}")
        print(f"   音频: {Path(audio_path).name}")

        video_path = Path(video_path)
        audio_path = Path(audio_path)

        if not video_path.exists():
            return {"success": False, "error": f"视频不存在: {video_path}"}
        if not audio_path.exists():
            return {"success": False, "error": f"音频不存在: {audio_path}"}

        if output_path is None:
            output_path = video_path.parent / f"{video_path.stem}_synced{video_path.suffix}"
        else:
            output_path = Path(output_path)
        output_path.parent.mkdir(parents=True, exist_ok=True)

        # Route1: 火山引擎视频改口型API（主路线）
        cloud = self._check_cloud()
        if cloud["available"]:
            print(f"  ☁️  火山引擎视频改口型...")
            result = self._run_cloud_lipsync(video_path, audio_path, output_path)
            if result.get("success"):
                return result
            print(f"  ⚠️  云端改口型失败: {result.get('error')}")

        # Route2: Edge-TTS 配音替代（角色说话时可用）
        print(f"  💡 口型同步路由:")
        print(f"     Route1 火山改口型: {'可用' if cloud.get('available') else '❌ 需开通智能视觉服务'}")
        print(f"     Route2 Edge-TTS:   ✅ (voice-emotion-compiler已通过)")
        print(f"  📌 光湖不本地安装模型。所有模型走API。")

        fallback_enabled = self.allow_fallback if allow_fallback is None else allow_fallback
        if not fallback_enabled:
            return {
                "success": False,
                "error": "口型同步不可用: 火山API未开通 + 未允许回退",
                "method": "not-run",
                "next_step": "开通火山引擎智能视觉服务'视频改口型'，配置密钥后自动可用"
            }

        print(f"  ⚠️  回退模式: 配音替代口型同步")
        return {
            "success": False,
            "error": "口型同步路由不可用，请先开通API",
            "method": "api-required"
        }

    def _check_cloud(self):
        """检查云端改口型服务是否可用"""
        if self._cloud_available is not None:
            return self._cloud_available
        try:
            import importlib.util
            spec = importlib.util.spec_from_file_location(
                'volce',
                str(PROJECT_ROOT / 'engines' / 'volce-lipsync-adapter.py')
            )
            mod = importlib.util.module_from_spec(spec)
            spec.loader.exec_module(mod)
            self._cloud_available = mod.check_available()
        except Exception as e:
            self._cloud_available = {"available": False, "reason": f"云端适配器不可用: {e}"}
        return self._cloud_available

    def _run_cloud_lipsync(self, video_path, audio_path, output_path):
        """通过火山引擎云端改口型"""
        try:
            import importlib.util
            spec = importlib.util.spec_from_file_location(
                'volce',
                str(PROJECT_ROOT / 'engines' / 'volce-lipsync-adapter.py')
            )
            mod = importlib.util.module_from_spec(spec)
            spec.loader.exec_module(mod)
            return mod.submit_lipsync(video_path, audio_path)
        except Exception as e:
            return {"success": False, "error": f"云端适配器调用失败: {e}", "method": "cloud-stub"}

    def batch_sync(self, video_audio_pairs, output_dir):
        """
        批量口型同步

        参数:
          video_audio_pairs: list of (video_path, audio_path)
          output_dir: 输出目录

        返回:
          list of result dicts
        """
        print(f"\n📦 批量口型同步: {len(video_audio_pairs)} 个任务")
        print(f"   输出目录: {output_dir}")

        output_dir = Path(output_dir)
        output_dir.mkdir(parents=True, exist_ok=True)

        results = []

        for i, (video_path, audio_path) in enumerate(video_audio_pairs):
            print(f"\n  进度: [{i+1}/{len(video_audio_pairs)}]")

            output_path = output_dir / f"{Path(video_path).stem}_synced.mp4"
            result = self.sync_lips(video_path, audio_path, output_path)
            results.append(result)

        # 统计
        success_count = sum(1 for r in results if r.get("success"))
        print(f"\n✅ 批量完成: {success_count}/{len(results)} 成功")

        # 保存报告
        report_path = output_dir / "lipsync_report.json"
        with open(report_path, "w", encoding="utf-8") as f:
            json.dump({
                "total": len(results),
                "success": success_count,
                "results": results,
                "generated_at": datetime.now().isoformat()
            }, f, ensure_ascii=False, indent=2)

        print(f"   报告已保存: {report_path}")

        return results

    def check_audio_sync(self, video_path, tolerance_ms=100):
        """
        检查口型同步质量
        简单方法: 检测音频能量峰值，与视频画面变化对比

        返回:
          {
            "synced": bool,
            "offset_ms": float,
            "score": float  # 0-1, 1=完美同步
          }
        """
        print(f"\n🔍 检查口型同步质量: {Path(video_path).name}")

        if not self.available:
            print(f"  ⚠️  Wav2Lip 不可用，跳过质量检查")
            return {"synced": None, "score": None, "warning": "Wav2Lip 不可用"}

        # TODO: 实现口型同步质量检查
        # 1. 提取音频能量包络
        # 2. 检测视频中嘴部区域的运动
        # 3. 计算相关性
        # 4. 返回偏移量和分数

        print(f"  ⚠️  质量检查未实现 (需要 librosa + OpenCV 嘴部检测)")

        return {
            "synced": None,
            "offset_ms": 0,
            "score": None,
            "warning": "Quality check not implemented yet"
        }


def main():
    parser = argparse.ArgumentParser(description="LIPSYNC-ADAPTER")
    parser.add_argument("--video", type=str, help="输入视频路径")
    parser.add_argument("--audio", type=str, help="对白音频路径")
    parser.add_argument("--output", type=str, help="输出视频路径")
    parser.add_argument("--batch", type=str, help="批量处理配置文件 (JSON)")
    parser.add_argument("--check-sync", action="store_true", help="检查口型同步质量")
    parser.add_argument("--allow-fallback-copy", action="store_true",
                        help="仅调试用: Wav2Lip 不可用时复制原视频并标记为 fallback")

    args = parser.parse_args()

    if args.check_sync:
        if not args.video:
            print("❌ --check-sync 需要 --video")
            sys.exit(1)

        adapter = LipSyncAdapter()
        result = adapter.check_audio_sync(args.video)
        print(json.dumps(result, ensure_ascii=False, indent=2))
        sys.exit(0)

    if args.batch:
        # 批量模式
        batch_file = Path(args.batch)
        if not batch_file.exists():
            print(f"❌ 批量配置文件不存在: {batch_file}")
            sys.exit(1)

        with open(batch_file, "r", encoding="utf-8") as f:
            config = json.load(f)

        video_audio_pairs = []
        for item in config.get("tasks", []):
            video_audio_pairs.append((item["video"], item["audio"]))

        output_dir = config.get("output_dir", "./outputs/lipsync/")

        adapter = LipSyncAdapter(allow_fallback=args.allow_fallback_copy)
        results = adapter.batch_sync(video_audio_pairs, output_dir)

        sys.exit(0)

    if not args.video or not args.audio:
        parser.print_help()
        sys.exit(1)

    adapter = LipSyncAdapter(allow_fallback=args.allow_fallback_copy)
    result = adapter.sync_lips(args.video, args.audio, args.output)

    if result["success"]:
        print(f"\n✅ 成功: {result['output_path']}")
        sys.exit(0)
    else:
        print(f"\n❌ 失败: {result.get('error', 'Unknown error')}")
        sys.exit(1)


if __name__ == "__main__":
    main()