即墨市网站建设喀什住房和城乡建设局网站

当前位置： 首页 > news >正文

即墨市网站建设,喀什住房和城乡建设局网站,绍兴做网站鼎成,4399游戏大全EchoMimic是由蚂蚁集团推出的一个 AI 驱动的口型同步技术项目#xff0c;能够通过人像面部特征和音频来帮助人物“对口型”#xff0c;生成逼真的动态肖像视频。 EchoMimic的技术亮点在于其创新的动画生成方法#xff0c;它不仅能够通过音频和面部关键点单独驱动图像动画能够通过人像面部特征和音频来帮助人物“对口型”生成逼真的动态肖像视频。 EchoMimic的技术亮点在于其创新的动画生成方法它不仅能够通过音频和面部关键点单独驱动图像动画还能结合这两种方式通过音频信号和面部关键点的组合来生成逼真的“说话的头部”视频。 EchoMimic支持单独使用音频或面部标志点生成肖像视频也支持将音频和人像照片相结合实现更自然、流畅的对口型效果。 EchoMimic支持多语言包括中文普通话、英语以及适应唱歌等场景。 github项目地址https://github.com/BadToBest/EchoMimic。 一、环境安装 1、python环境 建议安装python版本在3.10以上。 2、pip库安装 pip install torch2.0.1cu118 torchvision0.15.2cu118 torchaudio2.0.2 –index-url https://download.pytorch.org/whl/cu118 pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple 3、模型下载 git lfs install git clone https://huggingface.co/BadToBest/EchoMimic 二、功能测试 1、运行测试 1python代码调用测试audio2video import argparse import os import random import platform import subprocess from datetime import datetime from pathlib import Pathimport cv2 import numpy as np import torch from diffusers import AutoencoderKL, DDIMScheduler from omegaconf import OmegaConf from PIL import Imagefrom src.models.unet_2d_condition import UNet2DConditionModel from src.models.unet_3d_echo import EchoUNet3DConditionModel from src.models.whisper.audio2feature import load_audio_model from src.pipelines.pipeline_echo_mimic import Audio2VideoPipeline from src.utils.util import save_videos_grid, crop_and_pad from src.models.face_locator import FaceLocator from moviepy.editor import VideoFileClip, AudioFileClip from facenet_pytorch import MTCNN# Check and add FFmpeg path if necessary ffmpeg_path os.getenv(FFMPEG_PATH) if ffmpeg_path is None and platform.system() in [Linux, Darwin]:try:result subprocess.run([which, ffmpeg], capture_outputTrue, textTrue)if result.returncode 0:ffmpeg_path result.stdout.strip()print(fFFmpeg is installed at: {ffmpeg_path})else:print(FFmpeg is not installed. Please download ffmpeg-static and export to FFMPEG_PATH.)print(For example: export FFMPEG_PATH/musetalk/ffmpeg-4.4-amd64-static)except Exception as e:print(fError finding ffmpeg: {e}) else:if ffmpeg_path and ffmpeg_path not in os.getenv(PATH):print(Adding FFMPEG_PATH to PATH)os.environ[PATH] f{ffmpeg_path}:{os.environ[PATH]}def parse_args():parser argparse.ArgumentParser()parser.add_argument(–config, typestr, default./configs/prompts/animation.yaml)parser.add_argument(-W, typeint, default512)parser.add_argument(-H, typeint, default512)parser.add_argument(-L, typeint, default1200)parser.add_argument(–seed, typeint, default420)parser.add_argument(–facemusk_dilation_ratio, typefloat, default0.1)parser.add_argument(–facecrop_dilation_ratio, typefloat, default0.5)parser.add_argument(–context_frames, typeint, default12)parser.add_argument(–context_overlap, typeint, default3)parser.add_argument(–cfg, typefloat, default2.5)parser.add_argument(–steps, typeint, default30)parser.add_argument(–sample_rate, typeint, default16000)parser.add_argument(–fps, typeint, default24)parser.add_argument(–device, typestr, defaultcuda)return parser.parse_args()def select_face(det_bboxes, probs):Select the largest face with a detection probability above 0.8.if det_bboxes is None or probs is None:return Nonefiltered_bboxes [det_bboxes[i] for i in range(len(det_bboxes)) if probs[i] 0.8]if not filtered_bboxes:return Nonereturn max(filtered_bboxes, keylambda x: (x[3] - x[1]) * (x[2] - x[0]))def main():args parse_args()config OmegaConf.load(args.config)weight_dtype torch.float16 if config.weight_dtype fp16 else torch.float32device args.deviceif cuda in device and not torch.cuda.is_available():device cpuinfer_config OmegaConf.load(config.inference_config)############# Initialize models #############vae AutoencoderKL.from_pretrained(config.pretrained_vae_path).to(cuda, dtypeweight_dtype)reference_unet UNet2DConditionModel.from_pretrained(config.pretrained_base_model_path, subfolderunet).to(dtypeweight_dtype, devicedevice)reference_unet.load_state_dict(torch.load(config.reference_unet_path, map_locationcpu))unet_kwargs infer_config.unet_additional_kwargs or {}denoising_unet EchoUNet3DConditionModel.from_pretrained_2d(config.pretrained_base_model_path,config.motion_module_path if os.path.exists(config.motion_module_path) else ,subfolderunet,unet_additional_kwargsunet_kwargs).to(dtypeweight_dtype, devicedevice)denoising_unet.load_state_dict(torch.load(config.denoising_unet_path, map_locationcpu), strictFalse)face_locator FaceLocator(320, conditioning_channels1, block_out_channels(16, 32, 96, 256)).to(dtypeweight_dtype, devicecuda)face_locator.load_state_dict(torch.load(config.face_locator_path))audio_processor load_audio_model(model_pathconfig.audio_model_path, devicedevice)face_detector MTCNN(image_size320, margin0, min_face_size20, thresholds[0.6, 0.7, 0.7], factor0.709, post_processTrue, devicedevice)############# Initiate pipeline #############scheduler DDIMScheduler(**OmegaConf.to_container(infer_config.noise_scheduler_kwargs))pipe Audio2VideoPipeline(vaevae,reference_unetreference_unet,denoising_unetdenoising_unet,audio_guideraudio_processor,face_locatorface_locator,schedulerscheduler,).to(cuda, dtypeweight_dtype)date_str datetime.now().strftime(%Y%m%d)time_str datetime.now().strftime(%H%M)save_dir_name f{timestr}–seed{args.seed}-{args.W}x{args.H}save_dir Path(foutput/{date_str}/{save_dir_name})save_dir.mkdir(exist_okTrue, parentsTrue)for ref_image_path, audio_paths in config[test_cases].items():for audio_path in audio_paths:seed args.seed if args.seed is not None and args.seed -1 else random.randint(100, 1000000)generator torch.manual_seed(seed)ref_name Path(ref_image_path).stemaudio_name Path(audio_path).stemfinal_fps args.fps#### Prepare face maskface_img cv2.imread(ref_image_path)face_mask np.zeros((face_img.shape[0], face_img.shape[1]), dtypeuint8)det_bboxes, probs face_detector.detect(face_img)select_bbox select_face(det_bboxes, probs)if select_bbox is None:face_mask[:, :] 255else:xyxy np.round(select_bbox[:4]).astype(int)rb, re, cb, ce xyxy[1], xyxy[3], xyxy[0], xyxy[2]r_pad int((re - rb) * args.facemusk_dilation_ratio)c_pad int((ce - cb) * args.facemusk_dilation_ratio)face_mask[rb - r_pad : re r_pad, cb - c_pad : ce c_pad] 255r_pad_crop int((re - rb) * args.facecrop_dilation_ratio)c_pad_crop int((ce - cb) * args.facecrop_dilation_ratio)crop_rect [max(0, cb - c_pad_crop), max(0, rb - r_pad_crop), min(ce c_pad_crop, face_img.shape[1]), min(re r_pad_crop, face_img.shape[0])]face_img crop_and_pad(face_img, crop_rect)face_mask crop_and_pad(face_mask, crop_rect)face_img cv2.resize(face_img, (args.W, args.H))face_mask cv2.resize(face_mask, (args.W, args.H))ref_image_pil Image.fromarray(face_img[:, :, [2, 1, 0]])face_mask_tensor torch.Tensor(face_mask).to(dtypeweight_dtype, devicecuda).unsqueeze(0).unsqueeze(0).unsqueeze(0) / 255.0video pipe(ref_image_pil,audio_path,face_mask_tensor,widthargs.W,heightargs.H,durationargs.L,num_inference_stepsargs.steps,cfg_scaleargs.cfg,generatorgenerator,audio_sample_rateargs.sample_rate,context_framesargs.context_frames,fpsfinal_fps,context_overlapargs.context_overlap).videosvideo_save_path save_dir / f{refname}{audioname}{args.H}x{args.W}{int(args.cfg)}{time_str}.mp4save_videos_grid(video, str(video_save_path), n_rows1, fpsfinal_fps)# Add audio to generated videowith_audio_path save_dir / f{refname}{audioname}{args.H}x{args.W}{int(args.cfg)}{time_str}_withaudio.mp4video_clip VideoFileClip(str(video_save_path))audio_clip AudioFileClip(audio_path)final_video video_clip.set_audio(audio_clip)final_video.write_videofile(str(with_audio_path), codeclibx264, audio_codecaac)print(fSaved video with audio to {with_audio_path})if name main:main() 2python代码调用测试audio2pose 未完…… 更多详细的内容欢迎关注杰哥新技术