1. 增加配置项audio，用于控制使用的tts模块 2. 增加common/config-template.json，用于一般使用场景

hayate-hsu · hayate-hsu · commit c23258e16ec7 · 2023-12-20T14:11:14.000+08:00
diff --git a/README.md b/README.md
@@ -5,11 +5,13 @@
   - 文本（脚本）与视频相关度较低时（threshold<0.5），检索不到视频。可以尝试降低相关度阈值，或者提供更多相关性视频素材。
   - 文本（脚本）与图片，未作相关度阈值检测，简单返回top-N。
   - 当与文本匹配的视频片段或者图片不足时，系统使用空白（黑色）背景作为填充帧,补足时长。
+  - 可以调整输出视频分辨率&帧率，默认分辨率为1080P（1920x1080，16：9），30fps
 
 ## install(安装)
 ### 安装miniconda 或者其他python管理工具
 
-
+- 官网下载安装文件
+- 配置更改下载源
 
 ### 创建python（建议3.9版本）环境
 
@@ -33,17 +35,59 @@ pip install -r requirements.txt
 
 ## configuration（配置）
 
-### bert 模型
+### 配置文件
 
-### cn_clip 模型
+- 首先加载./data/conf/config.json 目录配置项，加载系统配置。
+- 如果未找到**./data/conf/config.json**，则加载./common/config-template.json 配置文件。
 
-### voc 模型
+```python
+{
+    "version":"v0.1.0",
+    "audio":"ms-tts",				# 音频模块，使用自己训练&部署模型，或者使用第三方接口，默认使用
+
+    "clip_model_name":"ViT-H-14",	# cn_clip 库以及模型文件下载存储目录
+    "download_root":"./clip_cn",
+    "device":"cuda",
+
+    "sentence_size": 256,               # 分词语句最大长度
+    "negativate_class": "other",         
+    
+    "max_size": 224,                   	# 与cn_clip模型相关，ViT-H-14窗口大小为224，这里统一把图片/视频帧缩放至224
+    		
+    "img_top_n": 5,  					# 视频&图片检索默认设置，可由webui控制
+    "video_top_n": 3,                        
+    "threshold": 0.6,                  
+    "d_value": 0.1,                    
+    "max_lenght": 20,
+
+    "image_vs":"./data/vs/image_vs/",	# 向量数据库目录，一个存储图片特征，一个存储视频特征
+    "video_vs":"./data/vs/video_vs/",
+
+    "voc_conf":"./data/conf/voc.json",		# 当audio选项为voc时，加载模型配置文件
+    "voc_model":"G_54000.pth", 				# voc对应的自训练tts模型
+
+    "sample_rate":44100,					# 音频采样率
 
-### config.json
+    "cache_path":"./data/cache",			#缓存目录
+    "output_path": "./data/output",			# 视频&音频 结果输出目录
 
-- openi: 配置token
+    "mirror": "openi",						# 下载bert模型时，指定源，启智https://openi.org.cn/
+    "openi_token": "",						# openi对应的token
 
-## using
+    "port":7800
+}
+```
+
+### bert 模型
 
-![主界面](./images/main.png)
+voc模块，对应的语言模型，支持ZH、EN、JP三语种。当audio配置项为ms-tts时，不需要此模型。
+
+### cn_clip 模型
+
+中文clip模型，用于跨模态检索。更多信息可访问[github项目地址](https://github.com/OFA-Sys/Chinese-CLIP)。
+
+CN-CLIPViT-H/14模型文件可以提前下载，下载地址：[Download](https://clip-cn-beijing.oss-cn-beijing.aliyuncs.com/checkpoints/clip_cn_vit-h-14.pt)。
+
+### voc 模型
 
+自训练的语音模型，当audio配置项为ms-tts时，不需要此模型。
diff --git a/common/conf.py b/common/conf.py
@@ -4,6 +4,7 @@
 import functools
 
 available_setting = {
+    "audio":"voc",                       # 语音转换模型，voc：使用自训练模型；ms-tts: 使用微软在线tts服务
     "device": "cuda",
 
     "clip_model_name": "ViT-H-14",       # chinese clip model： 选择的模型
@@ -38,7 +39,7 @@
     
     # 模型镜像源，默认huggingface，使用openi镜像源需指定openi_token
     "mirror": "openi",
-    "openi_token": "e6ea9886c4b70aaf4b62d6efe444fa574744b5ac",  # openi token
+    "openi_token": "",  # openi token
     
     "tmp_dir":'./tmp/'                  #
 }
@@ -77,7 +78,7 @@ def __getattr__(self, name):
 
 def load_config(config_path):
     if not os.path.exists(config_path):
-        config_path = "./data/conf/config-template.json"
+        config_path = "./common/config-template.json"
 
     config_str = read_file(config_path)
 
diff --git a/common/config-template.json b/common/config-template.json
@@ -0,0 +1,36 @@
+{
+    "version":"v0.1.0",
+
+    "audio":"ms-tts",
+
+    "clip_model_name":"ViT-H-14",
+    "download_root":"./clip_cn",
+    "device":"cuda",
+
+    "sentence_size": 256,               
+    "negativate_class": "other",         
+    
+    "max_size": 224,                     
+    
+    "img_top_n": 5,  
+    "video_top_n": 3,                        
+    "threshold": 0.6,                  
+    "d_value": 0.1,                    
+    "max_lenght": 20,
+
+    "image_vs":"./data/vs/image_vs/",
+    "video_vs":"./data/vs/video_vs/",
+
+    "voc_conf":"./data/conf/voc.json",
+    "voc_model":"D:\\ai\\audio\\Bert-VITS2\\models\\G_54000.pth", 
+
+    "sample_rate":44100,
+
+    "cache_path":"./data/cache",
+    "output_path": "./data/output",
+
+    "mirror": "openi",
+    "openi_token": "",
+
+    "port":7800
+}
diff --git a/voc/audio.py b/voc/audio.py
@@ -50,8 +50,6 @@ def generate_audio(
     
     with torch.no_grad():
         for idx, piece in enumerate(slices):
-            if piece == conf.negativate_class:
-                continue
             audio = infer(
                 piece,
                 sdp_ratio=sdp_ratio,
diff --git a/webui.py b/webui.py
@@ -46,10 +46,18 @@ def find(s_text, video_paths, image_paths, threshold, video_length, d_value, ima
     from worker.search import find_image, find_video
     image_paths = convert_path(image_paths)
     video_paths = convert_path(video_paths)
+    
+    # 检索参数设置
+    kwargs = {}
+    kwargs['threshold'] = threshold
+    kwargs['lenght'] = video_length
+    kwargs['dvalue'] = d_value
+    kwargs['itopN'] = image_top_n
+    kwargs['vtopN'] = video_top_n
 
-    image_s= find_image(s_text, image_paths, threshold, image_top_n)
+    image_s= find_image(s_text, image_paths, **kwargs)
     
-    video_s = find_video(s_text, video_paths, threshold, d_value, video_top_n, video_length)
+    video_s = find_video(s_text, video_paths, **kwargs)
     
     return image_s, video_s
 
@@ -66,8 +74,20 @@ def compose(scripts, video_paths, image_paths, resolution_rate, frame_rate, thre
     
     from worker.compose import synthesis
     
+    # 视频分辨率以及帧率设置
+    kwargs = dict(width=width, height=height, whr=wh_rate, frate=frame_rate)
+    # 检索参数设置
+    kwargs['threshold'] = threshold
+    kwargs['lenght'] = video_length
+    kwargs['dvalue'] = d_value
+    kwargs['itopN'] = image_top_n
+    kwargs['vtopN'] = video_top_n
+    
+    # tts设置
+    kwargs['speaker'] = 'fangqi'
+    kwargs['language'] = 'ZH' 
     
-    result = synthesis(scripts, video_paths, image_paths, width=width, height=height, whr=wh_rate, frate=frame_rate)
+    result = synthesis(scripts, video_paths, image_paths, **kwargs)
     
     return result
 
@@ -122,10 +142,11 @@ def compose(scripts, video_paths, image_paths, resolution_rate, frame_rate, thre
                             - 文案与视频素材尽可能匹配
                             - 当给定文本匹配素材不足时，采用黑色帧填充
                             - 文案目录未给定时，则检索匹配数据库中的全部图片&视频""")
-            with gr.Row():   
-                scripts = gr.Textbox(placeholder='请输入脚本（文本）', lines=3, max_lines=20, label='用于视频剪辑的脚本')
+            with gr.Row():  
+                with gr.Column(): 
+                    scripts = gr.Textbox(placeholder='请输入脚本（文本）', lines=3, max_lines=20, label='用于视频剪辑的脚本')
                 with gr.Column():
-                    resolution_rate = gr.Dropdown(choices=list(resolutions.keys()), value='720p[宽]',label='视频分辨率')
+                    resolution_rate = gr.Dropdown(choices=list(resolutions.keys()), value='720p:1280x720',label='视频分辨率')
                     frame_rate = gr.Slider(minimum=24, maximum=60, value=30, step=1, label="视频帧率")
                 with gr.Column():
                     btn_compose = gr.Button(value='剪辑视频')
diff --git a/worker/compose.py b/worker/compose.py
@@ -12,11 +12,7 @@
 conf = get_conf()
 
 from .text import split_text
-from . import video
-from . import image
-
-
-from voc import audio
+from . import video, image
 
 from worker.search import find_image, find_video
 
@@ -35,21 +31,32 @@ def synthesis(texts, video_folder, img_folder, **kwargs):
     
     logger.info(f'split_text: {docs}')
     
-    audio_results = audio.generate_audio(
-        docs, 
-        sdp_ratio=0.2, 
-        noise_scale=0.6,
-        noise_scale_w=0.8,
-        length_scale=1.0,
-        speaker='fangqi',
-        language='ZH'
+    # 转换音频时，剔除文本拆分时，添加的最后一项（other)
+    if conf.audio == 'voc':
+        from voc import audio
+        audio_results = audio.generate_audio(
+            docs[:-1], 
+            sdp_ratio=0.2, 
+            noise_scale=0.6,
+            noise_scale_w=0.8,
+            length_scale=1.0,
+            speaker=kwargs['speaker'],
+            language=kwargs['language'],
+            )
+    else:
+        # 调用ms tts 接口，生成语音
+        from . import audio
+        audio_results = audio.generate_audio(
+            docs[:-1], _rate=0, _volume=0, 
+            _lang='Auto', _gender='女', 
+            sample_rate=conf.sample_rate,
         )
     
     # 视频搜索
-    video_results = find_video(texts, video_folder)
+    video_results = find_video(texts, video_folder, **kwargs)
     
     # 图片搜索
-    image_results = find_image(texts, img_folder)
+    image_results = find_image(texts, img_folder, **kwargs)
     
     docs_videos = []        # 记录每段文本对应的视频文件
     
@@ -123,7 +130,7 @@ def synthesis(texts, video_folder, img_folder, **kwargs):
     # 拼接处理音频
     audio_file = conf.output_path + f'/audio_{now}.wav'
     audios = [ item[1] for item in audio_results]
-    audio_file = audio.concat_audios(audios[:-1], audio_file)       
+    audio_file = audio.concat_audios(audios, audio_file)       
     
     # 拼接处理视频
     ret_video = concat_fragments(docs_videos, -1, docs, conf.cache_path)
diff --git a/worker/search.py b/worker/search.py
@@ -236,10 +236,13 @@ def update_image_db(model, preprocess, image_paths, db):
         if len(img_fs):
             db.index(img_fs)            # 添加新图片至数据库中
     
-def find_image(text, image_paths, threshold=0.5, topn=3):
+def find_image(text, image_paths, **kwargs):
     '''
     image_paths : 图片目录列表
     '''
+    threshold = kwargs.get('threshold', conf.threshold)
+    topn = kwargs.get('itopN', conf.img_top_n)   
+    
     # 加载模型
     model, preprocess = load_chinese_clip(conf.clip_model_name, conf.download_root)
     # 读取数据库
@@ -318,9 +321,14 @@ def update_video_db(model, preprocess, db, video_paths):
             if video_f:
                 db.index(video_f)   
 
-def find_video(text,video_paths, threshold=0.5, dvalue=0.1, topn=3, length=10):
+def find_video(text,video_paths, **kwargs):
     '''
     '''
+    threshold = kwargs.get('threshold', conf.threshold)
+    dvalue = kwargs.get('dvalue', conf.d_value)   
+    topn = kwargs.get('vtopN', conf.video_top_n)   
+    length = kwargs.get('lenght', conf.max_lenght)   
+      
     # 加载模型
     model, preprocess = load_chinese_clip(conf.clip_model_name, conf.download_root)
     # 读取数据库
@@ -379,26 +387,7 @@ def find_video(text,video_paths, threshold=0.5, dvalue=0.1, topn=3, length=10):
         s_results[kw] = index_list
         
     logger.info('find video results: {}'.format(s_results))
-    # video_r = {}
-    # for kw, index_list in s_results:
-    #     videos = []
-    #     for idx, item in enumerate(index_list):
-    #         left, right = index_list[idx]['leftIndex'], index_list[idx]['rightIndex']
-    #         # duration = right - left 
-    #         start = getTime(left) # 将其转换为标准时间
-            
-    #         max_index = item['maxImage']['index']
-    #         uri = item['maxImage']['uri']
-    #         score = item['maxImage']['score']
-            
-    #         output =  "{}/clip_{}_{}.mp4".format(conf.tmp_dir, kw, idx)
 
-    #         logger.info('cut video:{} from: {} to: {}. maxImage:{}\noutput:{}'.format(uri, left, right, max_index, output))
-    #         cutVideo(start,right-left, uri, output) # 对视频进行切分
-            
-    #         videos.append({'uri':output, 'score':score, 'origin_video':uri, 'range':[left, right]})
-        
-    #     video_r[kw] = videos
     return s_results
 
 
diff --git a/worker/video.py b/worker/video.py
@@ -126,11 +126,11 @@ def imgs_to_video(images, lenght, output, **kwargs):
         input_images += f'-i {dst} '
         
     frame_rate = len(images)/lenght
-   
+    rate = kwargs.get('frate', 30)      #
     # -s 1080x1920
     # 拼接缓存目录的的 图片，进行有序拼接
-    logger.info(f'图片至视频：ffmpeg -framerate {frame_rate} -f image2 -i {cache_path}/image_%02d.jpg -c:v libx264 -t {lenght} -r 30 -pix_fmt yuv420p {output}')
-    os.system(f'ffmpeg -framerate {frame_rate} -f image2 -i {cache_path}/image_%02d.jpg -c:v libx264 -t {lenght} -r 30 -pix_fmt yuv420p {output}')
+    logger.info(f'图片至视频：ffmpeg -framerate {frame_rate} -f image2 -i {cache_path}/image_%02d.jpg -c:v libx264 -t {lenght} -r {rate} -pix_fmt yuv420p {output}')
+    os.system(f'ffmpeg -framerate {frame_rate} -f image2 -i {cache_path}/image_%02d.jpg -c:v libx264 -t {lenght} -r {rate} -pix_fmt yuv420p {output}')
 
 
 def concat_videos(videos, output):
@@ -172,8 +172,10 @@ def compose(video_f, audio_f, output):
     '''
     音频、视频组合
     '''
+    # '-filter_complex "[0:v]=[v];[1:a]=[a];[v][a]concat=n=1:v=1:a=1" -c:v libx264 -c:a acc -movfalgs +faststart'
     logger.info(f'compose video&audio: ffmpeg -i {video_f} -i {audio_f} -c:v copy -c:a aac -strict experimental {output}')
     os.system(f'ffmpeg -i {video_f} -i {audio_f} -c:v copy -c:a aac -strict experimental {output}')
+    # os.system(f'ffmpeg -i {video_f} -i {audio_f} -filter_complex "[0:v]=[v];[1:a]=[a];[v][a]concat=n=1:v=1:a=1" -c:v libx264 -c:a acc -movfalgs +faststart -strict experimental {output}')     
     
 def scale_video(video_path, output, **kwargs):
     '''
