Add tokenizers

.gitattributes

@@ -35,3 +35,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 pytorch_model-00001-of-00002.bin filter=lfs diff=lfs merge=lfs -text
 pytorch_model-00002-of-00002.bin filter=lfs diff=lfs merge=lfs -text
+speechtokenizer/SpeechTokenizer.pt filter=lfs diff=lfs merge=lfs -text
+wavtokenizer/WavTokenizer_small_600_24k_4096.ckpt filter=lfs diff=lfs merge=lfs -text

speechtokenizer/SpeechTokenizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d04593b6c9a4b475f91ca481141a6ef5b23e6ac112f347dd2b2717f193c1c728
+size 481906997

speechtokenizer/config.json

@@ -0,0 +1,49 @@
+{
+    "resblock": "1",
+    "num_gpus": 3,
+    "batch_size": 60,
+    "learning_rate": 0.0001,
+    "adam_b1": 0.5,
+    "adam_b2": 0.9,
+    "lr_decay": 0.98,
+    "seed": 1234,
+    "lambda_distill": 0.15,
+
+    "n_filters": 64,
+    "strides": [8,5,4,2],
+    "dimension": 1024,
+    "semantic_dimension": 768,
+    "bidirectional": true,
+    "dilation_base": 2,
+    "residual_kernel_size": 3,
+    "n_residual_layers": 1,
+    "lstm_layers": 2,
+    "activation": "ELU",
+
+
+    "segment_size": 48000,
+    "num_mels": 80,
+    "num_freq": 1025,
+    "n_fft": 1024,
+    "hop_size": 240,
+    "win_size": 1024,
+
+    "sampling_rate": 16000,
+    "sample_rate": 16000,
+
+    "codebook_size": 1024,
+    "n_q": 8,
+
+    "fmin": 0,
+    "fmax": 8000,
+    "fmax_for_loss": null,
+
+    "num_workers": 12,
+
+    "dist_config": {
+        "dist_backend": "nccl",
+        "dist_url": "tcp://localhost:54322",
+        "world_size": 1
+    }
+}
+

wavtokenizer/WavTokenizer_small_600_24k_4096.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d44c40fbb83d2d42329ac098e252a31b5708fb7b3bf864d108dd3ed26911d004
+size 1589082492

wavtokenizer/config.yaml

@@ -0,0 +1,93 @@
+seed_everything: 3407
+
+data:
+  class_path: decoder.dataset.VocosDataModule
+  init_args:
+    train_params:
+      filelist_path: ./WavTokenizer/data/train/libritts_train
+      sampling_rate: 24000
+      num_samples: 72000
+      batch_size: 40  # 20
+      num_workers: 8
+
+    val_params:
+      filelist_path: ./WavTokenizer/data/infer/librttts_val
+      sampling_rate: 24000
+      num_samples: 72000
+      batch_size: 5   # 10
+      num_workers: 8
+
+model:
+  class_path: decoder.experiment.WavTokenizer
+  init_args:
+    sample_rate: 24000
+    initial_learning_rate: 2e-4
+    mel_loss_coeff: 45
+    mrd_loss_coeff: 1.0
+    num_warmup_steps: 0 # Optimizers warmup steps
+    pretrain_mel_steps: 0  # 0 means GAN objective from the first iteration
+
+    # automatic evaluation
+    evaluate_utmos: true
+    evaluate_pesq: true
+    evaluate_periodicty: true
+
+    resume: false
+    resume_config: ./WavTokenizer/configs/wavtokenizer_smalldata_frame40_3s_nq1_code16384_dim512_kmeans800_attn.yaml
+    resume_model: ./version_3/checkpoints/xxx.ckpt
+
+    feature_extractor:
+      class_path: decoder.feature_extractors.EncodecFeatures
+      init_args:
+        encodec_model: encodec_24khz
+        bandwidths: [6.6, 6.6, 6.6, 6.6]
+        train_codebooks: true
+        num_quantizers: 1  
+        dowmsamples: [6, 5, 5, 4]
+        vq_bins: 4096
+        vq_kmeans: 200
+
+    backbone:
+      class_path: decoder.models.VocosBackbone
+      init_args:
+        input_channels: 512
+        dim: 768
+        intermediate_dim: 2304
+        num_layers: 12
+        adanorm_num_embeddings: 4  
+
+    head:
+      class_path: decoder.heads.ISTFTHead
+      init_args:
+        dim: 768
+        n_fft: 2400 
+        hop_length: 600
+        padding: same
+
+trainer:
+  logger:
+    class_path: pytorch_lightning.loggers.TensorBoardLogger
+    init_args:
+      save_dir: ./WavTokenizer/result/train/wavtokenizer_smalldata_frame40_3s_nq1_code4096_dim512_kmeans200_attn/
+  callbacks:
+    - class_path: pytorch_lightning.callbacks.LearningRateMonitor
+    - class_path: pytorch_lightning.callbacks.ModelSummary
+      init_args:
+        max_depth: 2
+    - class_path: pytorch_lightning.callbacks.ModelCheckpoint
+      init_args:
+        monitor: val_loss
+        filename: wavtokenizer_checkpoint_{epoch}_{step}_{val_loss:.4f}
+        save_top_k: 10
+        save_last: true
+    - class_path: decoder.helpers.GradNormCallback
+
+  # Lightning calculates max_steps across all optimizer steps (rather than number of batches)
+  # This equals to 1M steps per generator and 1M per discriminator
+  max_steps: 20000000
+  # You might want to limit val batches when evaluating all the metrics, as they are time-consuming
+  limit_val_batches: 200
+  accelerator: gpu
+  strategy: ddp
+  devices: [0,1,2,3,4,5,6,7]
+  log_every_n_steps: 1000