Update model

README.md

@@ -0,0 +1,292 @@
+---
+tags:
+- espnet
+- audio
+- universa
+language: multilingual
+datasets:
+- universa_unite
+license: cc-by-4.0
+---
+
+## ESPnet2 universa model
+
+### `espnet/arecho_scale_v0`
+
+This model was trained by ftshijt using universa_unite recipe in [espnet](https://github.com/espnet/espnet/).
+
+### Demo: How to use in ESPnet2
+
+Follow the [ESPnet installation instructions](https://espnet.github.io/espnet/installation.html)
+if you haven't done that already.
+
+```bash
+cd espnet
+git checkout 69996dc206e556ec48db77b6cc385ff1d32895b3
+pip install -e .
+cd egs2/universa_unite/uni_versa1
+./run.sh --skip_data_prep false --skip_train true --download_model espnet/arecho_scale_v0
+```
+
+
+
+## universa config
+
+<details><summary>expand</summary>
+
+```
+config: conf/train_aruniversa_wavlm.yaml
+print_config: false
+log_level: INFO
+drop_last_iter: false
+dry_run: false
+iterator_type: sequence
+valid_iterator_type: null
+output_dir: exp/universa_universa_ar_overall_scale_token_wavlm
+ngpu: 1
+seed: 777
+num_workers: 1
+num_att_plot: 0
+dist_backend: nccl
+dist_init_method: env://
+dist_world_size: null
+dist_rank: null
+local_rank: 0
+dist_master_addr: null
+dist_master_port: null
+dist_launcher: null
+multiprocessing_distributed: false
+unused_parameters: false
+sharded_ddp: false
+use_deepspeed: false
+deepspeed_config: null
+gradient_as_bucket_view: true
+ddp_comm_hook: null
+cudnn_enabled: true
+cudnn_benchmark: false
+cudnn_deterministic: false
+use_tf32: false
+collect_stats: false
+write_collected_feats: false
+max_epoch: 100
+patience: null
+val_scheduler_criterion:
+- valid
+- loss
+early_stopping_criterion:
+- valid
+- loss
+- min
+best_model_criterion:
+-   - train
+    - loss
+    - min
+-   - valid
+    - loss
+    - min
+-   - train
+    - acc
+    - max
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 1
+nbest_averaging_interval: 0
+grad_clip: -1
+grad_clip_type: 2.0
+grad_noise: false
+accum_grad: 2
+no_forward_run: false
+resume: true
+train_dtype: float32
+use_amp: false
+log_interval: 50
+use_matplotlib: true
+use_tensorboard: true
+create_graph_in_tensorboard: false
+use_wandb: false
+wandb_project: null
+wandb_id: null
+wandb_entity: null
+wandb_name: null
+wandb_model_log_interval: -1
+detect_anomaly: false
+use_adapter: false
+adapter: lora
+save_strategy: all
+adapter_conf: {}
+pretrain_path: null
+init_param: []
+ignore_init_mismatch: false
+freeze_param:
+- frontend.upstream
+num_iters_per_epoch: null
+batch_size: 16
+valid_batch_size: null
+batch_bins: 1000000
+valid_batch_bins: null
+category_sample_size: 10
+train_shape_file:
+- exp/universa_stats_overall_scale/train/audio_shape
+- exp/universa_stats_overall_scale/train/ref_audio_shape
+valid_shape_file:
+- exp/universa_stats_overall_scale/valid/audio_shape
+- exp/universa_stats_overall_scale/valid/ref_audio_shape
+batch_type: sorted
+valid_batch_type: null
+fold_length:
+- 256000
+sort_in_batch: descending
+shuffle_within_batch: false
+sort_batch: descending
+multiple_iterator: false
+chunk_length: 500
+chunk_shift_ratio: 0.5
+num_cache_chunks: 1024
+chunk_excluded_key_prefixes: []
+chunk_default_fs: null
+chunk_max_abs_length: null
+chunk_discard_short_samples: true
+train_data_path_and_name_and_type:
+-   - dump/raw/overall_scale/wav.scp
+    - audio
+    - kaldi_ark
+-   - dump/raw/overall_scale/metric.scp
+    - metrics
+    - metric
+-   - dump/raw/overall_scale/ref_wav.scp
+    - ref_audio
+    - kaldi_ark
+valid_data_path_and_name_and_type:
+-   - dump/raw/overall_dev/wav.scp
+    - audio
+    - kaldi_ark
+-   - dump/raw/overall_dev/metric.scp
+    - metrics
+    - metric
+-   - dump/raw/overall_dev/ref_wav.scp
+    - ref_audio
+    - kaldi_ark
+multi_task_dataset: false
+allow_variable_data_keys: false
+max_cache_size: 0.0
+max_cache_fd: 32
+allow_multi_rates: false
+valid_max_cache_size: null
+exclude_weight_decay: false
+exclude_weight_decay_conf: {}
+optim: adamw
+optim_conf:
+    lr: 0.001
+scheduler: warmuplr
+scheduler_conf:
+    warmup_steps: 25000
+metric2id: dump/raw/overall_scale/metric2id
+metric2type: dump/raw/overall_scale/metric2type
+metric_pad_value: -100
+token_list: null
+metric_token_info: data/token_list/metric_500_percentile_overall_scale_w-numerical/tokens.json
+metric_token_pad_value: 0
+tokenize_numerical_metric: true
+init: null
+model_conf: {}
+use_ref_audio: true
+use_ref_text: false
+use_preprocessor: true
+token_type: bpe
+bpemodel: null
+non_linguistic_symbols: null
+cleaner: null
+g2p: null
+sequential_metric: true
+randomize_sequential_metric: true
+frontend: s3prl
+frontend_conf:
+    frontend_conf:
+        upstream: wavlm_large
+    download_dir: ./hub
+    multilayer_feature: true
+universa: ar_universa
+universa_conf:
+    embedding_dim: 256
+    audio_encoder_type: transformer
+    audio_encoder_params:
+        num_blocks: 4
+        attention_heads: 4
+        linear_units: 1024
+        dropout_rate: 0.1
+        positional_dropout_rate: 0.1
+        attention_dropout_rate: 0.1
+        input_layer: conv2d
+        normalize_before: true
+        concat_after: false
+        positionwise_layer_type: linear
+        positionwise_conv_kernel_size: 1
+        layer_drop_rate: 0.1
+        qk_norm: false
+        use_flash_attn: false
+    cross_attention_type: multihead
+    cross_attention_params:
+        n_head: 2
+        dropout_rate: 0.1
+    metric_decoder_params:
+        num_blocks: 4
+        attention_heads: 4
+        linear_units: 1024
+        dropout_rate: 0.1
+        positional_dropout_rate: 0.1
+        src_attention_dropout_rate: 0.1
+        self_attention_dropout_rate: 0.1
+        input_layer: embed
+        normalize_before: true
+        concat_after: false
+        layer_drop_rate: 0.1
+        qk_norm: false
+        use_flash_attn: false
+    use_rope: true
+    lsm_weight: 0.1
+    sym_sos: <sos>
+    sym_eos: <eos>
+required:
+- output_dir
+- metric2id
+version: '202503'
+distributed: false
+```
+
+</details>
+
+
+
+### Citing ESPnet
+
+```BibTex
+@inproceedings{watanabe2018espnet,
+  author={Shinji Watanabe and Takaaki Hori and Shigeki Karita and Tomoki Hayashi and Jiro Nishitoba and Yuya Unno and Nelson Yalta and Jahn Heymann and Matthew Wiesner and Nanxin Chen and Adithya Renduchintala and Tsubasa Ochiai},
+  title={{ESPnet}: End-to-End Speech Processing Toolkit},
+  year={2018},
+  booktitle={Proceedings of Interspeech},
+  pages={2207--2211},
+  doi={10.21437/Interspeech.2018-1456},
+  url={http://dx.doi.org/10.21437/Interspeech.2018-1456}
+}
+
+
+
+
+
+
+```
+
+or arXiv:
+
+```bibtex
+@misc{watanabe2018espnet,
+  title={ESPnet: End-to-End Speech Processing Toolkit},
+  author={Shinji Watanabe and Takaaki Hori and Shigeki Karita and Tomoki Hayashi and Jiro Nishitoba and Yuya Unno and Nelson Yalta and Jahn Heymann and Matthew Wiesner and Nanxin Chen and Adithya Renduchintala and Tsubasa Ochiai},
+  year={2018},
+  eprint={1804.00015},
+  archivePrefix={arXiv},
+  primaryClass={cs.CL}
+}
+```

dump/raw/overall_scale/metric2id

@@ -0,0 +1,87 @@
+asr_match_error_rate
+asvspoof_score
+audiobox_aesthetics_CE
+audiobox_aesthetics_CU
+audiobox_aesthetics_PC
+audiobox_aesthetics_PQ
+cer
+ci_sdr
+dns_overall
+dns_p808
+emotion_similarity
+f0corr
+f0rmse
+language
+mcd
+nisqa_col_pred
+nisqa_dis_pred
+nisqa_loud_pred
+nisqa_mos_pred
+nisqa_noi_pred
+nisqa_real_mos
+nomad
+noresqa_score
+pam_score
+pesq
+plcmos
+pred_text_length
+pysepm_c_bak
+pysepm_cd
+pysepm_c_ovl
+pysepm_c_sig
+pysepm_csii_high
+pysepm_csii_low
+pysepm_csii_mid
+pysepm_fwsegsnr
+pysepm_llr
+pysepm_ncm
+pysepm_wss
+qwen_channel_type
+qwen_language
+qwen_laughter_crying
+qwen_pitch_range
+qwen_recording_quality
+qwen_speaker_age
+qwen_speaker_count
+qwen_speaker_gender
+qwen_speaking_style
+qwen_speech_background_environment
+qwen_speech_clarity
+qwen_speech_emotion
+qwen_speech_impairment
+qwen_speech_purpose
+qwen_speech_rate
+qwen_speech_register
+qwen_speech_volume_level
+qwen_vocabulary_complexity
+qwen_voice_pitch
+qwen_voice_type
+real_language
+ref_text_length
+rir_room_size
+rt60
+sar
+scoreq_nr
+scoreq_ref
+sdr
+se_ci_sdr
+se_sar
+se_sdr
+se_si_snr
+sheet_ssqa
+singmos
+si_snr
+snr_simulation
+speaking_rate
+speech_bert
+speech_bleu
+speech_token_distance
+spk_similarity
+srmr
+stoi
+urgent_mos
+utmos
+utmosv2
+visqol
+voicemos_real_mos
+wer

dump/raw/overall_scale/metric2type

@@ -0,0 +1,87 @@
+asr_match_error_rate numerical
+asvspoof_score numerical
+audiobox_aesthetics_CE numerical
+audiobox_aesthetics_CU numerical
+audiobox_aesthetics_PC numerical
+audiobox_aesthetics_PQ numerical
+cer numerical
+ci_sdr numerical
+dns_overall numerical
+dns_p808 numerical
+emotion_similarity numerical
+f0corr numerical
+f0rmse numerical
+language categorical
+mcd numerical
+nisqa_col_pred numerical
+nisqa_dis_pred numerical
+nisqa_loud_pred numerical
+nisqa_mos_pred numerical
+nisqa_noi_pred numerical
+nisqa_real_mos numerical
+nomad numerical
+noresqa_score numerical
+pam_score numerical
+pesq numerical
+plcmos numerical
+pred_text_length numerical
+pysepm_c_bak numerical
+pysepm_cd numerical
+pysepm_c_ovl numerical
+pysepm_c_sig numerical
+pysepm_csii_high numerical
+pysepm_csii_low numerical
+pysepm_csii_mid numerical
+pysepm_fwsegsnr numerical
+pysepm_llr numerical
+pysepm_ncm numerical
+pysepm_wss numerical
+qwen_channel_type categorical
+qwen_language categorical
+qwen_laughter_crying categorical
+qwen_pitch_range categorical
+qwen_recording_quality categorical
+qwen_speaker_age categorical
+qwen_speaker_count numerical
+qwen_speaker_gender categorical
+qwen_speaking_style categorical
+qwen_speech_background_environment categorical
+qwen_speech_clarity categorical
+qwen_speech_emotion categorical
+qwen_speech_impairment categorical
+qwen_speech_purpose categorical
+qwen_speech_rate categorical
+qwen_speech_register categorical
+qwen_speech_volume_level categorical
+qwen_vocabulary_complexity categorical
+qwen_voice_pitch categorical
+qwen_voice_type categorical
+real_language categorical
+ref_text_length numerical
+rir_room_size categorical
+rt60 numerical
+sar numerical
+scoreq_nr numerical
+scoreq_ref numerical
+sdr numerical
+se_ci_sdr numerical
+se_sar numerical
+se_sdr numerical
+se_si_snr numerical
+sheet_ssqa numerical
+singmos numerical
+si_snr numerical
+snr_simulation numerical
+speaking_rate numerical
+speech_bert numerical
+speech_bleu numerical
+speech_token_distance numerical
+spk_similarity numerical
+srmr numerical
+stoi numerical
+urgent_mos numerical
+utmos numerical
+utmosv2 numerical
+visqol numerical
+voicemos_real_mos numerical
+wer numerical

exp/universa_universa_ar_overall_scale_token_wavlm/6epoch.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f44093ca75bac69aa1fe7eea7c50c664ffb6e3d31d157c2b4b552d7e681acb57
+size 2325327716

exp/universa_universa_ar_overall_scale_token_wavlm/config.yaml

@@ -0,0 +1,217 @@
+config: conf/train_aruniversa_wavlm.yaml
+print_config: false
+log_level: INFO
+drop_last_iter: false
+dry_run: false
+iterator_type: sequence
+valid_iterator_type: null
+output_dir: exp/universa_universa_ar_overall_scale_token_wavlm
+ngpu: 1
+seed: 777
+num_workers: 1
+num_att_plot: 0
+dist_backend: nccl
+dist_init_method: env://
+dist_world_size: null
+dist_rank: null
+local_rank: 0
+dist_master_addr: null
+dist_master_port: null
+dist_launcher: null
+multiprocessing_distributed: false
+unused_parameters: false
+sharded_ddp: false
+use_deepspeed: false
+deepspeed_config: null
+gradient_as_bucket_view: true
+ddp_comm_hook: null
+cudnn_enabled: true
+cudnn_benchmark: false
+cudnn_deterministic: false
+use_tf32: false
+collect_stats: false
+write_collected_feats: false
+max_epoch: 100
+patience: null
+val_scheduler_criterion:
+- valid
+- loss
+early_stopping_criterion:
+- valid
+- loss
+- min
+best_model_criterion:
+-   - train
+    - loss
+    - min
+-   - valid
+    - loss
+    - min
+-   - train
+    - acc
+    - max
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 1
+nbest_averaging_interval: 0
+grad_clip: -1
+grad_clip_type: 2.0
+grad_noise: false
+accum_grad: 2
+no_forward_run: false
+resume: true
+train_dtype: float32
+use_amp: false
+log_interval: 50
+use_matplotlib: true
+use_tensorboard: true
+create_graph_in_tensorboard: false
+use_wandb: false
+wandb_project: null
+wandb_id: null
+wandb_entity: null
+wandb_name: null
+wandb_model_log_interval: -1
+detect_anomaly: false
+use_adapter: false
+adapter: lora
+save_strategy: all
+adapter_conf: {}
+pretrain_path: null
+init_param: []
+ignore_init_mismatch: false
+freeze_param:
+- frontend.upstream
+num_iters_per_epoch: null
+batch_size: 16
+valid_batch_size: null
+batch_bins: 1000000
+valid_batch_bins: null
+category_sample_size: 10
+train_shape_file:
+- exp/universa_stats_overall_scale/train/audio_shape
+- exp/universa_stats_overall_scale/train/ref_audio_shape
+valid_shape_file:
+- exp/universa_stats_overall_scale/valid/audio_shape
+- exp/universa_stats_overall_scale/valid/ref_audio_shape
+batch_type: sorted
+valid_batch_type: null
+fold_length:
+- 256000
+sort_in_batch: descending
+shuffle_within_batch: false
+sort_batch: descending
+multiple_iterator: false
+chunk_length: 500
+chunk_shift_ratio: 0.5
+num_cache_chunks: 1024
+chunk_excluded_key_prefixes: []
+chunk_default_fs: null
+chunk_max_abs_length: null
+chunk_discard_short_samples: true
+train_data_path_and_name_and_type:
+-   - dump/raw/overall_scale/wav.scp
+    - audio
+    - kaldi_ark
+-   - dump/raw/overall_scale/metric.scp
+    - metrics
+    - metric
+-   - dump/raw/overall_scale/ref_wav.scp
+    - ref_audio
+    - kaldi_ark
+valid_data_path_and_name_and_type:
+-   - dump/raw/overall_dev/wav.scp
+    - audio
+    - kaldi_ark
+-   - dump/raw/overall_dev/metric.scp
+    - metrics
+    - metric
+-   - dump/raw/overall_dev/ref_wav.scp
+    - ref_audio
+    - kaldi_ark
+multi_task_dataset: false
+allow_variable_data_keys: false
+max_cache_size: 0.0
+max_cache_fd: 32
+allow_multi_rates: false
+valid_max_cache_size: null
+exclude_weight_decay: false
+exclude_weight_decay_conf: {}
+optim: adamw
+optim_conf:
+    lr: 0.001
+scheduler: warmuplr
+scheduler_conf:
+    warmup_steps: 25000
+metric2id: dump/raw/overall_scale/metric2id
+metric2type: dump/raw/overall_scale/metric2type
+metric_pad_value: -100
+token_list: null
+metric_token_info: data/token_list/metric_500_percentile_overall_scale_w-numerical/tokens.json
+metric_token_pad_value: 0
+tokenize_numerical_metric: true
+init: null
+model_conf: {}
+use_ref_audio: true
+use_ref_text: false
+use_preprocessor: true
+token_type: bpe
+bpemodel: null
+non_linguistic_symbols: null
+cleaner: null
+g2p: null
+sequential_metric: true
+randomize_sequential_metric: true
+frontend: s3prl
+frontend_conf:
+    frontend_conf:
+        upstream: wavlm_large
+    download_dir: ./hub
+    multilayer_feature: true
+universa: ar_universa
+universa_conf:
+    embedding_dim: 256
+    audio_encoder_type: transformer
+    audio_encoder_params:
+        num_blocks: 4
+        attention_heads: 4
+        linear_units: 1024
+        dropout_rate: 0.1
+        positional_dropout_rate: 0.1
+        attention_dropout_rate: 0.1
+        input_layer: conv2d
+        normalize_before: true
+        concat_after: false
+        positionwise_layer_type: linear
+        positionwise_conv_kernel_size: 1
+        layer_drop_rate: 0.1
+        qk_norm: false
+        use_flash_attn: false
+    cross_attention_type: multihead
+    cross_attention_params:
+        n_head: 2
+        dropout_rate: 0.1
+    metric_decoder_params:
+        num_blocks: 4
+        attention_heads: 4
+        linear_units: 1024
+        dropout_rate: 0.1
+        positional_dropout_rate: 0.1
+        src_attention_dropout_rate: 0.1
+        self_attention_dropout_rate: 0.1
+        input_layer: embed
+        normalize_before: true
+        concat_after: false
+        layer_drop_rate: 0.1
+        qk_norm: false
+        use_flash_attn: false
+    use_rope: true
+    lsm_weight: 0.1
+    sym_sos: <sos>
+    sym_eos: <eos>
+required:
+- output_dir
+- metric2id
+version: '202503'
+distributed: false

meta.yaml

@@ -0,0 +1,8 @@
+espnet: '202503'
+files:
+  model_file: exp/universa_universa_ar_overall_scale_token_wavlm/6epoch.pth
+python: "3.9.18 | packaged by conda-forge | (main, Dec 23 2023, 17:20:25) \n[GCC 12.3.0]"
+timestamp: 1749800845.586997
+torch: 2.6.0.dev20241210+cu124
+yaml_files:
+  train_config: exp/universa_universa_ar_overall_scale_token_wavlm/config.yaml