gpustack commited on
Commit
268a094
·
verified ·
1 Parent(s): b81ebe5

Upload folder using huggingface_hub

Browse files
.gitattributes CHANGED
@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ flow.decoder.estimator.fp16.A10.plan filter=lfs diff=lfs merge=lfs -text
37
+ flow.decoder.estimator.fp16.Volta.plan filter=lfs diff=lfs merge=lfs -text
38
+ flow.decoder.estimator.fp16.a800.plan filter=lfs diff=lfs merge=lfs -text
39
+ flow.decoder.estimator.fp16.l20.plan filter=lfs diff=lfs merge=lfs -text
.msc ADDED
Binary file (1.68 kB). View file
 
.mv ADDED
@@ -0,0 +1 @@
 
 
1
+ Revision:master,CreatedAt:1744623950
CosyVoice-BlankEN/config.json ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen2ForCausalLM"
4
+ ],
5
+ "attention_dropout": 0.0,
6
+ "bos_token_id": 151643,
7
+ "eos_token_id": 151645,
8
+ "hidden_act": "silu",
9
+ "hidden_size": 896,
10
+ "initializer_range": 0.02,
11
+ "intermediate_size": 4864,
12
+ "max_position_embeddings": 32768,
13
+ "max_window_layers": 24,
14
+ "model_type": "qwen2",
15
+ "num_attention_heads": 14,
16
+ "num_hidden_layers": 24,
17
+ "num_key_value_heads": 2,
18
+ "rms_norm_eps": 1e-06,
19
+ "rope_theta": 1000000.0,
20
+ "sliding_window": 32768,
21
+ "tie_word_embeddings": true,
22
+ "torch_dtype": "bfloat16",
23
+ "transformers_version": "4.40.1",
24
+ "use_cache": true,
25
+ "use_sliding_window": false,
26
+ "vocab_size": 151936
27
+ }
CosyVoice-BlankEN/generation_config.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 151643,
3
+ "pad_token_id": 151643,
4
+ "do_sample": true,
5
+ "eos_token_id": [
6
+ 151645,
7
+ 151643
8
+ ],
9
+ "repetition_penalty": 1.1,
10
+ "temperature": 0.7,
11
+ "top_p": 0.8,
12
+ "top_k": 20,
13
+ "transformers_version": "4.37.0"
14
+ }
CosyVoice-BlankEN/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
CosyVoice-BlankEN/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:130282af0dfa9fe5840737cc49a0d339d06075f83c5a315c3372c9a0740d0b96
3
+ size 988097824
CosyVoice-BlankEN/tokenizer_config.json ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "151643": {
5
+ "content": "<|endoftext|>",
6
+ "lstrip": false,
7
+ "normalized": false,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ },
12
+ "151644": {
13
+ "content": "<|im_start|>",
14
+ "lstrip": false,
15
+ "normalized": false,
16
+ "rstrip": false,
17
+ "single_word": false,
18
+ "special": true
19
+ },
20
+ "151645": {
21
+ "content": "<|im_end|>",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false,
26
+ "special": true
27
+ }
28
+ },
29
+ "additional_special_tokens": ["<|im_start|>", "<|im_end|>"],
30
+ "bos_token": null,
31
+ "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
32
+ "clean_up_tokenization_spaces": false,
33
+ "eos_token": "<|im_end|>",
34
+ "errors": "replace",
35
+ "model_max_length": 32768,
36
+ "pad_token": "<|endoftext|>",
37
+ "split_special_tokens": false,
38
+ "tokenizer_class": "Qwen2Tokenizer",
39
+ "unk_token": null
40
+ }
CosyVoice-BlankEN/vocab.json ADDED
The diff for this file is too large to render. See raw diff
 
README.md ADDED
@@ -0,0 +1,228 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ pipeline_tag: text-to-speech
3
+ library_name: cosyvoice
4
+ ---
5
+ # CosyVoice
6
+
7
+ ## 👉🏻 [CosyVoice2 Demos](https://funaudiollm.github.io/cosyvoice2/) 👈🏻
8
+ [[CosyVoice2 Paper](https://fun-audio-llm.github.io/pdf/CosyVoice_v1.pdf)][[CosyVoice2 Studio](https://www.modelscope.cn/studios/iic/CosyVoice2-0.5B)]
9
+
10
+ ## 👉🏻 [CosyVoice Demos](https://fun-audio-llm.github.io/) 👈🏻
11
+ [[CosyVoice Paper](https://fun-audio-llm.github.io/pdf/CosyVoice_v1.pdf)][[CosyVoice Studio](https://www.modelscope.cn/studios/iic/CosyVoice-300M)][[CosyVoice Code](https://github.com/FunAudioLLM/CosyVoice)]
12
+
13
+ For `SenseVoice`, visit [SenseVoice repo](https://github.com/FunAudioLLM/SenseVoice) and [SenseVoice space](https://www.modelscope.cn/studios/iic/SenseVoice).
14
+
15
+ ## Roadmap
16
+
17
+ - [x] 2024/12
18
+
19
+ - [x] CosyVoice2-0.5B model release
20
+ - [x] CosyVoice2-0.5B streaming inference with no quality degradation
21
+
22
+ - [x] 2024/07
23
+
24
+ - [x] Flow matching training support
25
+ - [x] WeTextProcessing support when ttsfrd is not avaliable
26
+ - [x] Fastapi server and client
27
+
28
+ - [x] 2024/08
29
+
30
+ - [x] Repetition Aware Sampling(RAS) inference for llm stability
31
+ - [x] Streaming inference mode support, including kv cache and sdpa for rtf optimization
32
+
33
+ - [x] 2024/09
34
+
35
+ - [x] 25hz cosyvoice base model
36
+ - [x] 25hz cosyvoice voice conversion model
37
+
38
+ - [ ] TBD
39
+
40
+ - [ ] CosyVoice2-0.5B bistream inference support
41
+ - [ ] CosyVoice2-0.5B training and finetune recipie
42
+ - [ ] CosyVoice-500M trained with more multi-lingual data
43
+ - [ ] More...
44
+
45
+ ## Install
46
+
47
+ **Clone and install**
48
+
49
+ - Clone the repo
50
+ ``` sh
51
+ git clone --recursive https://github.com/FunAudioLLM/CosyVoice.git
52
+ # If you failed to clone submodule due to network failures, please run following command until success
53
+ cd CosyVoice
54
+ git submodule update --init --recursive
55
+ ```
56
+
57
+ - Install Conda: please see https://docs.conda.io/en/latest/miniconda.html
58
+ - Create Conda env:
59
+
60
+ ``` sh
61
+ conda create -n cosyvoice python=3.10
62
+ conda activate cosyvoice
63
+ # pynini is required by WeTextProcessing, use conda to install it as it can be executed on all platform.
64
+ conda install -y -c conda-forge pynini==2.1.5
65
+ pip install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host=mirrors.aliyun.com
66
+
67
+ # If you encounter sox compatibility issues
68
+ # ubuntu
69
+ sudo apt-get install sox libsox-dev
70
+ # centos
71
+ sudo yum install sox sox-devel
72
+ ```
73
+
74
+ **Model download**
75
+
76
+ We strongly recommend that you download our pretrained `CosyVoice-300M` `CosyVoice-300M-SFT` `CosyVoice-300M-Instruct` model and `CosyVoice-ttsfrd` resource.
77
+
78
+ If you are expert in this field, and you are only interested in training your own CosyVoice model from scratch, you can skip this step.
79
+
80
+ ``` python
81
+ # SDK模型下载
82
+ from modelscope import snapshot_download
83
+ snapshot_download('iic/CosyVoice2-0.5B', local_dir='pretrained_models/CosyVoice2-0.5B')
84
+ snapshot_download('iic/CosyVoice-300M', local_dir='pretrained_models/CosyVoice-300M')
85
+ snapshot_download('iic/CosyVoice-300M-25Hz', local_dir='pretrained_models/CosyVoice-300M-25Hz')
86
+ snapshot_download('iic/CosyVoice-300M-SFT', local_dir='pretrained_models/CosyVoice-300M-SFT')
87
+ snapshot_download('iic/CosyVoice-300M-Instruct', local_dir='pretrained_models/CosyVoice-300M-Instruct')
88
+ snapshot_download('iic/CosyVoice-ttsfrd', local_dir='pretrained_models/CosyVoice-ttsfrd')
89
+ ```
90
+
91
+ ``` sh
92
+ # git模型下载,请确保已安装git lfs
93
+ mkdir -p pretrained_models
94
+ git clone https://www.modelscope.cn/iic/CosyVoice2-0.5B.git pretrained_models/CosyVoice2-0.5B
95
+ git clone https://www.modelscope.cn/iic/CosyVoice-300M.git pretrained_models/CosyVoice-300M
96
+ git clone https://www.modelscope.cn/iic/CosyVoice-300M-25Hz.git pretrained_models/CosyVoice-300M-25Hz
97
+ git clone https://www.modelscope.cn/iic/CosyVoice-300M-SFT.git pretrained_models/CosyVoice-300M-SFT
98
+ git clone https://www.modelscope.cn/iic/CosyVoice-300M-Instruct.git pretrained_models/CosyVoice-300M-Instruct
99
+ git clone https://www.modelscope.cn/iic/CosyVoice-ttsfrd.git pretrained_models/CosyVoice-ttsfrd
100
+ ```
101
+
102
+ Optionaly, you can unzip `ttsfrd` resouce and install `ttsfrd` package for better text normalization performance.
103
+
104
+ Notice that this step is not necessary. If you do not install `ttsfrd` package, we will use WeTextProcessing by default.
105
+
106
+ ``` sh
107
+ cd pretrained_models/CosyVoice-ttsfrd/
108
+ unzip resource.zip -d .
109
+ pip install ttsfrd-0.3.6-cp38-cp38-linux_x86_64.whl
110
+ ```
111
+
112
+ **Basic Usage**
113
+
114
+ For zero_shot/cross_lingual inference, please use `CosyVoice2-0.5B` or `CosyVoice-300M` model.
115
+ For sft inference, please use `CosyVoice-300M-SFT` model.
116
+ For instruct inference, please use `CosyVoice-300M-Instruct` model.
117
+ We strongly recommend using `CosyVoice2-0.5B` model for better streaming performance.
118
+
119
+ First, add `third_party/Matcha-TTS` to your `PYTHONPATH`.
120
+
121
+ ``` sh
122
+ export PYTHONPATH=third_party/Matcha-TTS
123
+ ```
124
+
125
+ ``` python
126
+ from cosyvoice.cli.cosyvoice import CosyVoice, CosyVoice2
127
+ from cosyvoice.utils.file_utils import load_wav
128
+ import torchaudio
129
+
130
+ ## cosyvoice2 usage
131
+ cosyvoice2 = CosyVoice('pretrained_models/CosyVoice-300M-SFT', load_jit=False, load_onnx=False, load_trt=False)
132
+ # sft usage
133
+ prompt_speech_16k = load_wav('zero_shot_prompt.wav', 16000)
134
+ for i, j in enumerate(cosyvoice2.inference_zero_shot('收到好友从远方寄来的生日礼物,那份意外的惊喜与深深的祝福让我心中充满了甜蜜的快乐,笑容如花儿般绽放。', '希望你以后能够做的比我还好呦。', prompt_speech_16k, stream=True)):
135
+ torchaudio.save('zero_shot_{}.wav'.format(i), j['tts_speech'], cosyvoice2.sample_rate)
136
+
137
+ ## cosyvoice usage
138
+ cosyvoice = CosyVoice('pretrained_models/CosyVoice-300M-SFT', load_jit=True, load_onnx=False, fp16=True)
139
+ # sft usage
140
+ print(cosyvoice.list_avaliable_spks())
141
+ # change stream=True for chunk stream inference
142
+ for i, j in enumerate(cosyvoice.inference_sft('你好,我是通义生成式语音大模型,请问有什么可以帮您的吗?', '中文女', stream=False)):
143
+ torchaudio.save('sft_{}.wav'.format(i), j['tts_speech'], cosyvoice.sample_rate)
144
+
145
+ cosyvoice = CosyVoice('pretrained_models/CosyVoice-300M-25Hz') # or change to pretrained_models/CosyVoice-300M for 50Hz inference
146
+ # zero_shot usage, <|zh|><|en|><|jp|><|yue|><|ko|> for Chinese/English/Japanese/Cantonese/Korean
147
+ prompt_speech_16k = load_wav('zero_shot_prompt.wav', 16000)
148
+ for i, j in enumerate(cosyvoice.inference_zero_shot('收到好友从远方寄来的生日礼物,那份意外的惊喜与深深的祝福让我心中充满了甜蜜的快乐,笑容如花儿般绽放。', '希望你以后能够做的比我还好呦。', prompt_speech_16k, stream=False)):
149
+ torchaudio.save('zero_shot_{}.wav'.format(i), j['tts_speech'], cosyvoice.sample_rate)
150
+ # cross_lingual usage
151
+ prompt_speech_16k = load_wav('cross_lingual_prompt.wav', 16000)
152
+ for i, j in enumerate(cosyvoice.inference_cross_lingual('<|en|>And then later on, fully acquiring that company. So keeping management in line, interest in line with the asset that\'s coming into the family is a reason why sometimes we don\'t buy the whole thing.', prompt_speech_16k, stream=False)):
153
+ torchaudio.save('cross_lingual_{}.wav'.format(i), j['tts_speech'], cosyvoice.sample_rate)
154
+ # vc usage
155
+ prompt_speech_16k = load_wav('zero_shot_prompt.wav', 16000)
156
+ source_speech_16k = load_wav('cross_lingual_prompt.wav', 16000)
157
+ for i, j in enumerate(cosyvoice.inference_vc(source_speech_16k, prompt_speech_16k, stream=False)):
158
+ torchaudio.save('vc_{}.wav'.format(i), j['tts_speech'], cosyvoice.sample_rate)
159
+
160
+ cosyvoice = CosyVoice('pretrained_models/CosyVoice-300M-Instruct')
161
+ # instruct usage, support <laughter></laughter><strong></strong>[laughter][breath]
162
+ for i, j in enumerate(cosyvoice.inference_instruct('在面对挑战时,他展现了非凡的<strong>勇气</strong>与<strong>智慧</strong>。', '中文男', 'Theo \'Crimson\', is a fiery, passionate rebel leader. Fights with fervor for justice, but struggles with impulsiveness.', stream=False)):
163
+ torchaudio.save('instruct_{}.wav'.format(i), j['tts_speech'], cosyvoice.sample_rate)
164
+ ```
165
+
166
+ **Start web demo**
167
+
168
+ You can use our web demo page to get familiar with CosyVoice quickly.
169
+ We support sft/zero_shot/cross_lingual/instruct inference in web demo.
170
+
171
+ Please see the demo website for details.
172
+
173
+ ``` python
174
+ # change iic/CosyVoice-300M-SFT for sft inference, or iic/CosyVoice-300M-Instruct for instruct inference
175
+ python3 webui.py --port 50000 --model_dir pretrained_models/CosyVoice-300M
176
+ ```
177
+
178
+ **Advanced Usage**
179
+
180
+ For advanced user, we have provided train and inference scripts in `examples/libritts/cosyvoice/run.sh`.
181
+ You can get familiar with CosyVoice following this recipie.
182
+
183
+ **Build for deployment**
184
+
185
+ Optionally, if you want to use grpc for service deployment,
186
+ you can run following steps. Otherwise, you can just ignore this step.
187
+
188
+ ``` sh
189
+ cd runtime/python
190
+ docker build -t cosyvoice:v1.0 .
191
+ # change iic/CosyVoice-300M to iic/CosyVoice-300M-Instruct if you want to use instruct inference
192
+ # for grpc usage
193
+ docker run -d --runtime=nvidia -p 50000:50000 cosyvoice:v1.0 /bin/bash -c "cd /opt/CosyVoice/CosyVoice/runtime/python/grpc && python3 server.py --port 50000 --max_conc 4 --model_dir iic/CosyVoice-300M && sleep infinity"
194
+ cd grpc && python3 client.py --port 50000 --mode <sft|zero_shot|cross_lingual|instruct>
195
+ # for fastapi usage
196
+ docker run -d --runtime=nvidia -p 50000:50000 cosyvoice:v1.0 /bin/bash -c "cd /opt/CosyVoice/CosyVoice/runtime/python/fastapi && python3 server.py --port 50000 --model_dir iic/CosyVoice-300M && sleep infinity"
197
+ cd fastapi && python3 client.py --port 50000 --mode <sft|zero_shot|cross_lingual|instruct>
198
+ ```
199
+
200
+ ## Discussion & Communication
201
+
202
+ You can directly discuss on [Github Issues](https://github.com/FunAudioLLM/CosyVoice/issues).
203
+
204
+ You can also scan the QR code to join our official Dingding chat group.
205
+
206
+ <img src="./asset/dingding.png" width="250px">
207
+
208
+ ## Acknowledge
209
+
210
+ 1. We borrowed a lot of code from [FunASR](https://github.com/modelscope/FunASR).
211
+ 2. We borrowed a lot of code from [FunCodec](https://github.com/modelscope/FunCodec).
212
+ 3. We borrowed a lot of code from [Matcha-TTS](https://github.com/shivammehta25/Matcha-TTS).
213
+ 4. We borrowed a lot of code from [AcademiCodec](https://github.com/yangdongchao/AcademiCodec).
214
+ 5. We borrowed a lot of code from [WeNet](https://github.com/wenet-e2e/wenet).
215
+
216
+ ## Citations
217
+
218
+ ``` bibtex
219
+ @article{du2024cosyvoice,
220
+ title={Cosyvoice: A scalable multilingual zero-shot text-to-speech synthesizer based on supervised semantic tokens},
221
+ author={Du, Zhihao and Chen, Qian and Zhang, Shiliang and Hu, Kai and Lu, Heng and Yang, Yexin and Hu, Hangrui and Zheng, Siqi and Gu, Yue and Ma, Ziyang and others},
222
+ journal={arXiv preprint arXiv:2407.05407},
223
+ year={2024}
224
+ }
225
+ ```
226
+
227
+ ## Disclaimer
228
+ The content provided above is for academic purposes only and is intended to demonstrate technical capabilities. Some examples are sourced from the internet. If any content infringes on your rights, please contact us to request its removal.
asset/dingding.png ADDED
campplus.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a6ac6a63997761ae2997373e2ee1c47040854b4b759ea41ec48e4e42df0f4d73
3
+ size 28303423
configuration.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"framework":"Pytorch","task":"text-to-speech"}
cosyvoice.yaml ADDED
@@ -0,0 +1,140 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # set random seed, so that you may reproduce your result.
2
+ __set_seed1: !apply:random.seed [1986]
3
+ __set_seed2: !apply:numpy.random.seed [1986]
4
+ __set_seed3: !apply:torch.manual_seed [1986]
5
+ __set_seed4: !apply:torch.cuda.manual_seed_all [1986]
6
+
7
+ # fixed params
8
+ sample_rate: 24000
9
+ llm_input_size: 896
10
+ llm_output_size: 896
11
+ spk_embed_dim: 192
12
+ qwen_pretrain_path: ''
13
+
14
+ # model params
15
+ # for all class/function included in this repo, we use !<name> or !<new> for intialization, so that user may find all corresponding class/function according to one single yaml.
16
+ # for system/third_party class/function, we do not require this.
17
+ llm: !new:cosyvoice.llm.llm.Qwen2LM
18
+ llm_input_size: !ref <llm_input_size>
19
+ llm_output_size: !ref <llm_output_size>
20
+ speech_token_size: 6561
21
+ length_normalized_loss: True
22
+ lsm_weight: 0
23
+ llm: !new:cosyvoice.llm.llm.Qwen2Encoder
24
+ pretrain_path: !ref <qwen_pretrain_path>
25
+ sampling: !name:cosyvoice.utils.common.ras_sampling
26
+ top_p: 0.8
27
+ top_k: 25
28
+ win_size: 10
29
+ tau_r: 0.1
30
+
31
+ flow: !new:cosyvoice.flow.flow.CausalMaskedDiffWithXvec
32
+ input_size: 512
33
+ output_size: 80
34
+ spk_embed_dim: !ref <spk_embed_dim>
35
+ output_type: 'mel'
36
+ vocab_size: 6561
37
+ input_frame_rate: 25
38
+ only_mask_loss: True
39
+ token_mel_ratio: 2
40
+ pre_lookahead_len: 3
41
+ encoder: !new:cosyvoice.transformer.upsample_encoder.UpsampleConformerEncoder
42
+ output_size: 512
43
+ attention_heads: 8
44
+ linear_units: 2048
45
+ num_blocks: 6
46
+ dropout_rate: 0.1
47
+ positional_dropout_rate: 0.1
48
+ attention_dropout_rate: 0.1
49
+ normalize_before: True
50
+ input_layer: 'linear'
51
+ pos_enc_layer_type: 'rel_pos_espnet'
52
+ selfattention_layer_type: 'rel_selfattn'
53
+ input_size: 512
54
+ use_cnn_module: False
55
+ macaron_style: False
56
+ decoder: !new:cosyvoice.flow.flow_matching.CausalConditionalCFM
57
+ in_channels: 240
58
+ n_spks: 1
59
+ spk_emb_dim: 80
60
+ cfm_params: !new:omegaconf.DictConfig
61
+ content:
62
+ sigma_min: 1e-06
63
+ solver: 'euler'
64
+ t_scheduler: 'cosine'
65
+ training_cfg_rate: 0.2
66
+ inference_cfg_rate: 0.7
67
+ reg_loss_type: 'l1'
68
+ estimator: !new:cosyvoice.flow.decoder.ConditionalDecoder
69
+ in_channels: 320
70
+ out_channels: 80
71
+ causal: True
72
+ channels: [256]
73
+ dropout: 0.0
74
+ attention_head_dim: 64
75
+ n_blocks: 4
76
+ num_mid_blocks: 12
77
+ num_heads: 8
78
+ act_fn: 'gelu'
79
+
80
+ hift: !new:cosyvoice.hifigan.generator.HiFTGenerator
81
+ in_channels: 80
82
+ base_channels: 512
83
+ nb_harmonics: 8
84
+ sampling_rate: !ref <sample_rate>
85
+ nsf_alpha: 0.1
86
+ nsf_sigma: 0.003
87
+ nsf_voiced_threshold: 10
88
+ upsample_rates: [8, 5, 3]
89
+ upsample_kernel_sizes: [16, 11, 7]
90
+ istft_params:
91
+ n_fft: 16
92
+ hop_len: 4
93
+ resblock_kernel_sizes: [3, 7, 11]
94
+ resblock_dilation_sizes: [[1, 3, 5], [1, 3, 5], [1, 3, 5]]
95
+ source_resblock_kernel_sizes: [7, 7, 11]
96
+ source_resblock_dilation_sizes: [[1, 3, 5], [1, 3, 5], [1, 3, 5]]
97
+ lrelu_slope: 0.1
98
+ audio_limit: 0.99
99
+ f0_predictor: !new:cosyvoice.hifigan.f0_predictor.ConvRNNF0Predictor
100
+ num_class: 1
101
+ in_channels: 80
102
+ cond_channels: 512
103
+
104
+ # processor functions
105
+ parquet_opener: !name:cosyvoice.dataset.processor.parquet_opener
106
+ get_tokenizer: !name:cosyvoice.tokenizer.tokenizer.get_qwen_tokenizer
107
+ token_path: !ref <qwen_pretrain_path>
108
+ skip_special_tokens: True
109
+ allowed_special: 'all'
110
+ tokenize: !name:cosyvoice.dataset.processor.tokenize
111
+ get_tokenizer: !ref <get_tokenizer>
112
+ allowed_special: !ref <allowed_special>
113
+ filter: !name:cosyvoice.dataset.processor.filter
114
+ max_length: 40960
115
+ min_length: 0
116
+ token_max_length: 200
117
+ token_min_length: 1
118
+ resample: !name:cosyvoice.dataset.processor.resample
119
+ resample_rate: !ref <sample_rate>
120
+ feat_extractor: !name:matcha.utils.audio.mel_spectrogram
121
+ n_fft: 1920
122
+ num_mels: 80
123
+ sampling_rate: !ref <sample_rate>
124
+ hop_size: 480
125
+ win_size: 1920
126
+ fmin: 0
127
+ fmax: 8000
128
+ center: False
129
+ compute_fbank: !name:cosyvoice.dataset.processor.compute_fbank
130
+ feat_extractor: !ref <feat_extractor>
131
+ parse_embedding: !name:cosyvoice.dataset.processor.parse_embedding
132
+ normalize: True
133
+ shuffle: !name:cosyvoice.dataset.processor.shuffle
134
+ shuffle_size: 1000
135
+ sort: !name:cosyvoice.dataset.processor.sort
136
+ sort_size: 500 # sort_size should be less than shuffle_size
137
+ batch: !name:cosyvoice.dataset.processor.batch
138
+ batch_type: 'dynamic'
139
+ max_frames_in_batch: 2000
140
+ padding: !name:cosyvoice.dataset.processor.padding
flow.decoder.estimator.fp16.A10.plan ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6f6b9073bd9e7b8ac5bef0a21431391cbc32376b9265ec73935d6f28a0d32d01
3
+ size 168597292
flow.decoder.estimator.fp16.Volta.plan ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f231edf01451fafbc3dc0498a51feb3a264afad43275536c8151fff954ef3c56
3
+ size 161799540
flow.decoder.estimator.fp16.a800.plan ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:14105d80375213591e11d0c8a941966b683fd08a94d885796e1892d7002dba68
3
+ size 165970980
flow.decoder.estimator.fp16.l20.plan ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:012df9e730e36e1cb61bf2780378c15ae92c536ae87518b7a54a90026cb99385
3
+ size 166520788
flow.decoder.estimator.fp32.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:51aed3efa2c153898ea53a780893c920e968dab1d7aec25402bd6c9815d94702
3
+ size 286521895
flow.encoder.fp32.zip ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7cf2fdb6cc83952d20a4faade946ab867068b1f5531c95f7b61db932c44c5a43
3
+ size 192366262
flow.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ff4c2f867674411e0a08cee702996df13fa67c1cd864c06108da88d16d088541
3
+ size 450575567
hift.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1d4af0d661a416c69544eec83ff9c070dc80c37ee53ef44af3a37d910c95bc21
3
+ size 83364158
llm.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b144ef55b51ce8cfb79a73c90dbba0bdaba4e451c0ebcfab20f769264f84a608
3
+ size 2023316821
speech_tokenizer_v2.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d43342aa12163a80bf07bffb94c9de2e120a8df2f9917cd2f642e7f4219c6f71
3
+ size 496082973