Upload 6 files

checkpoints/log_VoxCeleb2_lip_tfgridnet-isam_3spk /config.yaml

@@ -0,0 +1,59 @@
+## Config file
+
+# Log 
+seed: 777
+use_cuda: 1           # 1 for True, 0 for False
+
+# dataset
+speaker_no: 3
+mix_lst_path: ./data/VoxCeleb2_non_repeat/mixture_data_list_3mix.csv
+audio_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/audio_clean/
+reference_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/orig/
+# mix_lst_path: ./data/LRS2/mixture_data_list_3mix.csv
+# audio_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/LRS2/audio_clean/
+# reference_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/LRS2/mvlrs_v1/
+# mix_lst_path: ./data/LRS3/mixture_data_list_3mix.csv
+# audio_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/LRS3/audio_clean/
+# reference_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/LRS3/orig/
+audio_sr: 16000
+ref_sr: 25
+
+# dataloader
+num_workers: 2
+batch_size: 1         # 4-GPU training with a total effective batch size of 8
+accu_grad: 1
+effec_batch_size: 2   # per GPU, only used if accu_grad is set to 1, must be multiple times of batch size
+max_length: 3         # truncate the utterances in dataloader, in seconds 
+
+# network settings
+init_from: 'checkpoints/log_VoxCeleb2_lip_tfgridnet_3spk'       # 'None' or a log name 'log_2024-07-22(18:12:13)'
+causal: 0             # 1 for True, 0 for False
+network_reference:
+  cue: lip            # lip or speech or gesture or EEG
+  backbone: resnet18  # resnet18 or shufflenetV2 or blazenet64
+  emb_size: 256       # resnet18:256
+network_audio:
+  backbone: av_tfgridnet_isam
+  n_fft: 256
+  stride: 128
+  window: "hann"
+  use_builtin_complex: False
+  n_srcs:  1
+  n_imics: 1
+  n_layers: 6
+  lstm_hidden_units: 192
+  attn_n_head: 4
+  attn_qk_output_channel: 4
+  emb_dim: 48
+  emb_ks: 4
+  emb_hs: 1
+  activation: "prelu"
+  isam: 1
+
+# optimizer
+spk_att_dropout: 1    # 0 for always use speaker attention
+loss_type: ss_sisdr      # "snr", "sisdr", "hybrid"
+lr_warmup: 1
+init_learning_rate: 0.0005
+max_epoch: 150
+clip_grad_norm: 5

checkpoints/log_VoxCeleb2_lip_tfgridnet-isam_3spk /last_best_checkpoint.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4351584fa30020fbda626f9f63c859a35bfc1c663509e721fb8ff46d65cf6c79
+size 162585486

checkpoints/log_VoxCeleb2_lip_tfgridnet-isam_3spk /last_checkpoint.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90f3c8209585d03d8ef148cbff650530144f5e4b844dd557fb7c8684ed520646
+size 162576686

checkpoints/log_VoxCeleb2_lip_tfgridnet-isam_3spk /log_2024-12-23(15:50:02).txt

@@ -0,0 +1,771 @@
+## Config file
+
+# Log 
+seed: 777
+use_cuda: 1           # 1 for True, 0 for False
+
+# dataset
+speaker_no: 3
+mix_lst_path: ./data/VoxCeleb2_non_repeat/mixture_data_list_3mix.csv
+audio_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/audio_clean/
+reference_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/orig/
+audio_sr: 16000
+ref_sr: 25
+
+# dataloader
+num_workers: 2
+batch_size: 1         # 4-GPU training with a total effective batch size of 8
+accu_grad: 1
+effec_batch_size: 2   # per GPU, only used if accu_grad is set to 1, must be multiple times of batch size
+max_length: 3         # truncate the utterances in dataloader, in seconds 
+
+# network settings
+init_from: 'checkpoints/log_VoxCeleb2_lip_tfgridnet_3spk'       # 'None' or a log name 'log_2024-07-22(18:12:13)'
+causal: 0             # 1 for True, 0 for False
+network_reference:
+  cue: lip            # lip or speech or gesture or EEG
+  backbone: resnet18  # resnet18 or shufflenetV2 or blazenet64
+  emb_size: 256       # resnet18:256
+network_audio:
+  backbone: av_tfgridnet_att_ss
+  n_fft: 256
+  stride: 128
+  window: "hann"
+  use_builtin_complex: False
+  n_srcs:  1
+  n_imics: 1
+  n_layers: 6
+  lstm_hidden_units: 192
+  attn_n_head: 4
+  attn_qk_output_channel: 4
+  emb_dim: 48
+  emb_ks: 4
+  emb_hs: 1
+  activation: "prelu"
+
+# optimizer
+spk_att_dropout: 1    # 0 for always use speaker attention
+loss_type: ss_sisdr      # "snr", "sisdr", "hybrid"
+lr_warmup: 1
+init_learning_rate: 0.0005
+max_epoch: 150
+clip_grad_norm: 5
+W1223 15:50:36.042814 140017896986432 torch/distributed/run.py:779] 
+W1223 15:50:36.042814 140017896986432 torch/distributed/run.py:779] *****************************************
+W1223 15:50:36.042814 140017896986432 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W1223 15:50:36.042814 140017896986432 torch/distributed/run.py:779] *****************************************
+[W1223 15:50:57.603307732 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
+[W1223 15:50:57.603314732 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
+[W1223 15:50:57.604096578 Utils.hpp:135] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
+[W1223 15:50:57.603320695 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
+[W1223 15:50:57.604115033 Utils.hpp:135] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
+[W1223 15:50:57.604131007 Utils.hpp:135] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
+[W1223 15:50:57.603696860 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
+[W1223 15:50:57.604146436 Utils.hpp:135] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
+started on checkpoints/log_2024-12-23(15:50:02)
+
+namespace(accu_grad=1, audio_direc='/mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/audio_clean/', audio_sr=16000, batch_size=1, causal=0, checkpoint_dir='checkpoints/log_2024-12-23(15:50:02)', clip_grad_norm=5.0, config=[<yamlargparse.Path object at 0x7fefcdc30e80>], device=device(type='cuda'), distributed=True, effec_batch_size=2, evaluate_only=0, init_from='checkpoints/log_VoxCeleb2_lip_tfgridnet_3spk', init_learning_rate=0.0005, local_rank=0, loss_type='ss_sisdr', lr_warmup=1, max_epoch=150, max_length=3, mix_lst_path='./data/VoxCeleb2_non_repeat/mixture_data_list_3mix.csv', network_audio=namespace(activation='prelu', attn_n_head=4, attn_qk_output_channel=4, backbone='av_tfgridnet_att_ss', emb_dim=48, emb_hs=1, emb_ks=4, lstm_hidden_units=192, n_fft=256, n_imics=1, n_layers=6, n_srcs=1, stride=128, use_builtin_complex=False, window='hann'), network_reference=namespace(backbone='resnet18', cue='lip', emb_size=256), num_workers=2, ref_sr=25, reference_direc='/mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/orig/', seed=777, speaker_no=3, spk_att_dropout=1, train_from_last_checkpoint=0, use_cuda=1, world_size=4)
+network_wrapper(
+  (sep_network): av_TFGridNetV3_att_ss(
+    (enc): STFTEncoder(
+      (stft): Stft(n_fft=256, win_length=256, hop_length=128, center=True, normalized=False, onesided=True)
+    )
+    (dec): STFTDecoder(
+      (stft): Stft(n_fft=256, win_length=256, hop_length=128, center=True, normalized=False, onesided=True)
+    )
+    (conv): Sequential(
+      (0): Conv2d(2, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+      (1): GroupNorm(1, 48, eps=1e-05, affine=True)
+    )
+    (blocks): ModuleList(
+      (0-5): 6 x GridNetV3Block(
+        (intra_norm): LayerNorm((48,), eps=1e-05, elementwise_affine=True)
+        (intra_rnn): LSTM(192, 192, batch_first=True, bidirectional=True)
+        (intra_linear): ConvTranspose1d(384, 48, kernel_size=(4,), stride=(1,))
+        (inter_norm): LayerNorm((48,), eps=1e-05, elementwise_affine=True)
+        (inter_rnn): LSTM(192, 192, batch_first=True, bidirectional=True)
+        (inter_linear): ConvTranspose1d(384, 48, kernel_size=(4,), stride=(1,))
+        (attn_conv_Q): Conv2d(48, 16, kernel_size=(1, 1), stride=(1, 1))
+        (attn_norm_Q): AllHeadPReLULayerNormalization4DC(
+          (act): PReLU(num_parameters=4)
+        )
+        (attn_conv_K): Conv2d(48, 16, kernel_size=(1, 1), stride=(1, 1))
+        (attn_norm_K): AllHeadPReLULayerNormalization4DC(
+          (act): PReLU(num_parameters=4)
+        )
+        (attn_conv_V): Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1))
+        (attn_norm_V): AllHeadPReLULayerNormalization4DC(
+          (act): PReLU(num_parameters=4)
+        )
+        (attn_concat_proj): Sequential(
+          (0): Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1))
+          (1): PReLU(num_parameters=1)
+          (2): LayerNormalization()
+        )
+        (spk_att): TransformerEncoder(
+          (layers): ModuleList(
+            (0): TransformerEncoderLayer(
+              (self_attn): MultiheadAttention(
+                (out_proj): NonDynamicallyQuantizableLinear(in_features=48, out_features=48, bias=True)
+              )
+              (linear1): Linear(in_features=48, out_features=192, bias=True)
+              (dropout): Dropout(p=0.1, inplace=False)
+              (linear2): Linear(in_features=192, out_features=48, bias=True)
+              (norm1): LayerNorm((48,), eps=1e-05, elementwise_affine=True)
+              (norm2): LayerNorm((48,), eps=1e-05, elementwise_affine=True)
+              (dropout1): Dropout(p=0.1, inplace=False)
+              (dropout2): Dropout(p=0.1, inplace=False)
+            )
+          )
+        )
+        (spk_norm): GroupNorm(1, 48, eps=1e-08, affine=True)
+      )
+    )
+    (deconv): ConvTranspose2d(48, 2, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+    (av_conv): ModuleList(
+      (0-5): 6 x Linear(in_features=304, out_features=48, bias=True)
+    )
+  )
+  (ref_encoder): Visual_encoder(
+    (v_frontend): VisualFrontend(
+      (frontend3D): Sequential(
+        (0): Conv3d(1, 64, kernel_size=(5, 7, 7), stride=(1, 2, 2), padding=(2, 3, 3), bias=False)
+        (1): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+        (2): ReLU()
+        (3): MaxPool3d(kernel_size=(1, 3, 3), stride=(1, 2, 2), padding=(0, 1, 1), dilation=1, ceil_mode=False)
+      )
+      (resnet): ResNet(
+        (layer1): ResNetLayer(
+          (conv1a): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (bn1a): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2a): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (downsample): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
+          (outbna): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv1b): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (bn1b): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2b): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (outbnb): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+        )
+        (layer2): ResNetLayer(
+          (conv1a): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
+          (bn1a): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2a): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (downsample): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)
+          (outbna): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv1b): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (bn1b): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2b): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (outbnb): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+        )
+        (layer3): ResNetLayer(
+          (conv1a): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
+          (bn1a): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2a): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (downsample): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)
+          (outbna): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv1b): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (bn1b): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2b): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (outbnb): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+        )
+        (layer4): ResNetLayer(
+          (conv1a): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
+          (bn1a): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2a): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (downsample): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)
+          (outbna): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv1b): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (bn1b): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2b): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (outbnb): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+        )
+        (avgpool): AvgPool2d(kernel_size=(4, 4), stride=(1, 1), padding=0)
+      )
+    )
+    (v_ds): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+    (visual_conv): Sequential(
+      (0): VisualConv1D(
+        (relu_0): ReLU()
+        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (relu): ReLU()
+        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+        (prelu): PReLU(num_parameters=1)
+        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+      )
+      (1): VisualConv1D(
+        (relu_0): ReLU()
+        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (relu): ReLU()
+        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+        (prelu): PReLU(num_parameters=1)
+        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+      )
+      (2): VisualConv1D(
+        (relu_0): ReLU()
+        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (relu): ReLU()
+        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+        (prelu): PReLU(num_parameters=1)
+        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+      )
+      (3): VisualConv1D(
+        (relu_0): ReLU()
+        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (relu): ReLU()
+        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+        (prelu): PReLU(num_parameters=1)
+        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+      )
+      (4): VisualConv1D(
+        (relu_0): ReLU()
+        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (relu): ReLU()
+        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+        (prelu): PReLU(num_parameters=1)
+        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+      )
+    )
+  )
+)
+
+Total number of parameters: 20950309 
+
+
+Total number of trainable parameters: 9765221 
+
+dlchqhe6f3ef1ed0-master-0:28:28 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlchqhe6f3ef1ed0-master-0:28:28 [0] NCCL INFO Bootstrap : Using eth0:22.5.232.161<0>
+dlchqhe6f3ef1ed0-master-0:28:28 [0] NCCL INFO Plugin name set by env to libnccl-net-none.so
+dlchqhe6f3ef1ed0-master-0:28:28 [0] NCCL INFO NET/Plugin : dlerror=libnccl-net-none.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net-none.so), using internal implementation
+dlchqhe6f3ef1ed0-master-0:28:28 [0] NCCL INFO cudaDriverVersion 11040
+dlchqhe6f3ef1ed0-master-0:31:31 [3] NCCL INFO cudaDriverVersion 11040
+dlchqhe6f3ef1ed0-master-0:30:30 [2] NCCL INFO cudaDriverVersion 11040
+dlchqhe6f3ef1ed0-master-0:29:29 [1] NCCL INFO cudaDriverVersion 11040
+NCCL version 2.20.5+cuda11.8
+dlchqhe6f3ef1ed0-master-0:29:29 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlchqhe6f3ef1ed0-master-0:31:31 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlchqhe6f3ef1ed0-master-0:30:30 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlchqhe6f3ef1ed0-master-0:29:29 [1] NCCL INFO Bootstrap : Using eth0:22.5.232.161<0>
+dlchqhe6f3ef1ed0-master-0:30:30 [2] NCCL INFO Bootstrap : Using eth0:22.5.232.161<0>
+dlchqhe6f3ef1ed0-master-0:31:31 [3] NCCL INFO Bootstrap : Using eth0:22.5.232.161<0>
+dlchqhe6f3ef1ed0-master-0:29:29 [1] NCCL INFO Plugin name set by env to libnccl-net-none.so
+dlchqhe6f3ef1ed0-master-0:30:30 [2] NCCL INFO Plugin name set by env to libnccl-net-none.so
+dlchqhe6f3ef1ed0-master-0:31:31 [3] NCCL INFO Plugin name set by env to libnccl-net-none.so
+dlchqhe6f3ef1ed0-master-0:30:30 [2] NCCL INFO NET/Plugin : dlerror=libnccl-net-none.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net-none.so), using internal implementation
+dlchqhe6f3ef1ed0-master-0:31:31 [3] NCCL INFO NET/Plugin : dlerror=libnccl-net-none.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net-none.so), using internal implementation
+dlchqhe6f3ef1ed0-master-0:29:29 [1] NCCL INFO NET/Plugin : dlerror=libnccl-net-none.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net-none.so), using internal implementation
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO NCCL_IB_HCA set to mlx5
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO NCCL_IB_HCA set to mlx5
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO NCCL_IB_HCA set to mlx5
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO NCCL_IB_HCA set to mlx5
+libibverbs: Warning: couldn't load driver 'libhfi1verbs-rdmav25.so': libhfi1verbs-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libhfi1verbs-rdmav25.so': libhfi1verbs-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libhfi1verbs-rdmav25.so': libhfi1verbs-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libhfi1verbs-rdmav25.so': libhfi1verbs-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'librxe-rdmav25.so': librxe-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'librxe-rdmav25.so': librxe-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'librxe-rdmav25.so': librxe-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'librxe-rdmav25.so': librxe-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libmthca-rdmav25.so': libmthca-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libmthca-rdmav25.so': libmthca-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libmthca-rdmav25.so': libmthca-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libmthca-rdmav25.so': libmthca-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libvmw_pvrdma-rdmav25.so': libvmw_pvrdma-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libvmw_pvrdma-rdmav25.so': libvmw_pvrdma-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libvmw_pvrdma-rdmav25.so': libvmw_pvrdma-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libvmw_pvrdma-rdmav25.so': libvmw_pvrdma-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libhns-rdmav25.so': libhns-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libhns-rdmav25.so': libhns-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libhns-rdmav25.so': libhns-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libhns-rdmav25.so': libhns-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libipathverbs-rdmav25.so': libipathverbs-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libipathverbs-rdmav25.so': libipathverbs-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libipathverbs-rdmav25.so': libipathverbs-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libipathverbs-rdmav25.so': libipathverbs-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libsiw-rdmav25.so': libsiw-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libsiw-rdmav25.so': libsiw-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libsiw-rdmav25.so': libsiw-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libsiw-rdmav25.so': libsiw-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libbnxt_re-rdmav25.so': libbnxt_re-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libbnxt_re-rdmav25.so': libbnxt_re-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libbnxt_re-rdmav25.so': libbnxt_re-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libbnxt_re-rdmav25.so': libbnxt_re-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libocrdma-rdmav25.so': libocrdma-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libocrdma-rdmav25.so': libocrdma-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libocrdma-rdmav25.so': libocrdma-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libocrdma-rdmav25.so': libocrdma-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libmlx4-rdmav25.so': libmlx4-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libmlx4-rdmav25.so': libmlx4-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libmlx4-rdmav25.so': libmlx4-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libmlx4-rdmav25.so': libmlx4-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libqedr-rdmav25.so': libqedr-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libqedr-rdmav25.so': libqedr-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libqedr-rdmav25.so': libqedr-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libqedr-rdmav25.so': libqedr-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libcxgb4-rdmav25.so': libcxgb4-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libcxgb4-rdmav25.so': libcxgb4-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libcxgb4-rdmav25.so': libcxgb4-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libcxgb4-rdmav25.so': libcxgb4-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libi40iw-rdmav25.so': libi40iw-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libi40iw-rdmav25.so': libi40iw-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libi40iw-rdmav25.so': libi40iw-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libi40iw-rdmav25.so': libi40iw-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libefa-rdmav25.so': libefa-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libefa-rdmav25.so': libefa-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libefa-rdmav25.so': libefa-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libefa-rdmav25.so': libefa-rdmav25.so: cannot open shared object file: No such file or directory
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [RO]; OOB eth0:22.5.232.161<0>
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Using non-device net plugin version 0
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Using network IB
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [RO]; OOB eth0:22.5.232.161<0>
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [RO]; OOB eth0:22.5.232.161<0>
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Using non-device net plugin version 0
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Using network IB
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Using non-device net plugin version 0
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Using network IB
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [RO]; OOB eth0:22.5.232.161<0>
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Using non-device net plugin version 0
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Using network IB
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO comm 0x8620bf0 rank 3 nranks 4 cudaDev 3 nvmlDev 3 busId 40 commId 0x7a7db4e36deb53ea - Init START
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO comm 0x86c7e10 rank 2 nranks 4 cudaDev 2 nvmlDev 2 busId 30 commId 0x7a7db4e36deb53ea - Init START
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO comm 0x9352850 rank 1 nranks 4 cudaDev 1 nvmlDev 1 busId 20 commId 0x7a7db4e36deb53ea - Init START
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO comm 0x81df340 rank 0 nranks 4 cudaDev 0 nvmlDev 0 busId 10 commId 0x7a7db4e36deb53ea - Init START
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Setting affinity for GPU 3 to ffffff
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Setting affinity for GPU 0 to ffffff
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Setting affinity for GPU 2 to ffffff
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Setting affinity for GPU 1 to ffffff
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO comm 0x81df340 rank 0 nRanks 4 nNodes 1 localRanks 4 localRank 0 MNNVL 0
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO comm 0x8620bf0 rank 3 nRanks 4 nNodes 1 localRanks 4 localRank 3 MNNVL 0
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO comm 0x9352850 rank 1 nRanks 4 nNodes 1 localRanks 4 localRank 1 MNNVL 0
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Channel 00/12 :    0   1   2   3
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO comm 0x86c7e10 rank 2 nRanks 4 nNodes 1 localRanks 4 localRank 2 MNNVL 0
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Channel 01/12 :    0   1   2   3
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Channel 02/12 :    0   1   2   3
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Channel 03/12 :    0   1   2   3
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Channel 04/12 :    0   1   2   3
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Channel 05/12 :    0   1   2   3
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Channel 06/12 :    0   1   2   3
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Channel 07/12 :    0   1   2   3
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Channel 08/12 :    0   1   2   3
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Trees [0] -1/-1/-1->3->2 [1] -1/-1/-1->3->2 [2] -1/-1/-1->3->2 [3] -1/-1/-1->3->2 [4] -1/-1/-1->3->2 [5] -1/-1/-1->3->2 [6] -1/-1/-1->3->2 [7] -1/-1/-1->3->2 [8] -1/-1/-1->3->2 [9] -1/-1/-1->3->2 [10] -1/-1/-1->3->2 [11] -1/-1/-1->3->2
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Channel 09/12 :    0   1   2   3
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Channel 10/12 :    0   1   2   3
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO P2P Chunksize set to 524288
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Channel 11/12 :    0   1   2   3
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 [2] 2/-1/-1->1->0 [3] 2/-1/-1->1->0 [4] 2/-1/-1->1->0 [5] 2/-1/-1->1->0 [6] 2/-1/-1->1->0 [7] 2/-1/-1->1->0 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->0 [10] 2/-1/-1->1->0 [11] 2/-1/-1->1->0
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO P2P Chunksize set to 524288
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 3/-1/-1->2->1 [4] 3/-1/-1->2->1 [5] 3/-1/-1->2->1 [6] 3/-1/-1->2->1 [7] 3/-1/-1->2->1 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 3/-1/-1->2->1 [11] 3/-1/-1->2->1
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] 1/-1/-1->0->-1 [5] 1/-1/-1->0->-1 [6] 1/-1/-1->0->-1 [7] 1/-1/-1->0->-1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO P2P Chunksize set to 524288
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO P2P Chunksize set to 524288
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Channel 00/0 : 3[3] -> 0[0] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Channel 01/0 : 3[3] -> 0[0] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Channel 02/0 : 3[3] -> 0[0] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Channel 02/0 : 2[2] -> 3[3] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Channel 03/0 : 3[3] -> 0[0] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Channel 03/0 : 2[2] -> 3[3] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Channel 04/0 : 3[3] -> 0[0] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Channel 04/0 : 2[2] -> 3[3] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Channel 05/0 : 3[3] -> 0[0] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Channel 02/0 : 1[1] -> 2[2] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Channel 05/0 : 2[2] -> 3[3] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Channel 06/0 : 3[3] -> 0[0] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Channel 06/0 : 2[2] -> 3[3] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Channel 07/0 : 3[3] -> 0[0] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Channel 04/0 : 1[1] -> 2[2] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Channel 08/0 : 3[3] -> 0[0] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Channel 05/0 : 1[1] -> 2[2] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Channel 08/0 : 2[2] -> 3[3] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Channel 09/0 : 3[3] -> 0[0] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Channel 06/0 : 1[1] -> 2[2] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Channel 10/0 : 3[3] -> 0[0] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Channel 10/0 : 2[2] -> 3[3] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Channel 11/0 : 3[3] -> 0[0] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Channel 08/0 : 1[1] -> 2[2] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Channel 11/0 : 2[2] -> 3[3] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Channel 10/0 : 1[1] -> 2[2] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Connected all rings
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Channel 00/0 : 3[3] -> 2[2] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Connected all rings
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Connected all rings
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Connected all rings
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Channel 01/0 : 3[3] -> 2[2] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Channel 02/0 : 3[3] -> 2[2] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Channel 03/0 : 3[3] -> 2[2] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Channel 04/0 : 3[3] -> 2[2] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Channel 05/0 : 3[3] -> 2[2] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Channel 06/0 : 3[3] -> 2[2] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Channel 07/0 : 3[3] -> 2[2] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Channel 08/0 : 3[3] -> 2[2] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Channel 09/0 : 3[3] -> 2[2] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Channel 10/0 : 3[3] -> 2[2] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Channel 11/0 : 3[3] -> 2[2] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Channel 00/0 : 2[2] -> 1[1] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Channel 00/0 : 1[1] -> 0[0] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Channel 01/0 : 2[2] -> 1[1] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Channel 01/0 : 1[1] -> 0[0] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Channel 02/0 : 2[2] -> 1[1] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Channel 02/0 : 1[1] -> 0[0] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Channel 03/0 : 2[2] -> 1[1] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Channel 03/0 : 1[1] -> 0[0] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Channel 04/0 : 2[2] -> 1[1] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Channel 04/0 : 1[1] -> 0[0] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Channel 05/0 : 2[2] -> 1[1] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Channel 05/0 : 1[1] -> 0[0] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Channel 06/0 : 2[2] -> 1[1] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Channel 06/0 : 1[1] -> 0[0] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Channel 07/0 : 2[2] -> 1[1] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Channel 07/0 : 1[1] -> 0[0] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Channel 08/0 : 2[2] -> 1[1] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Channel 08/0 : 1[1] -> 0[0] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Channel 09/0 : 2[2] -> 1[1] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Channel 09/0 : 1[1] -> 0[0] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Channel 10/0 : 2[2] -> 1[1] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Channel 10/0 : 1[1] -> 0[0] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Channel 11/0 : 2[2] -> 1[1] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Channel 11/0 : 1[1] -> 0[0] via P2P/IPC/read
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO Connected all trees
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO 12 coll channels, 0 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO Connected all trees
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO Connected all trees
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO 12 coll channels, 0 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO Connected all trees
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO 12 coll channels, 0 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO 12 coll channels, 0 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer
+dlchqhe6f3ef1ed0-master-0:31:60 [3] NCCL INFO comm 0x8620bf0 rank 3 nranks 4 cudaDev 3 nvmlDev 3 busId 40 commId 0x7a7db4e36deb53ea - Init COMPLETE
+dlchqhe6f3ef1ed0-master-0:29:57 [1] NCCL INFO comm 0x9352850 rank 1 nranks 4 cudaDev 1 nvmlDev 1 busId 20 commId 0x7a7db4e36deb53ea - Init COMPLETE
+dlchqhe6f3ef1ed0-master-0:30:59 [2] NCCL INFO comm 0x86c7e10 rank 2 nranks 4 cudaDev 2 nvmlDev 2 busId 30 commId 0x7a7db4e36deb53ea - Init COMPLETE
+dlchqhe6f3ef1ed0-master-0:28:58 [0] NCCL INFO comm 0x81df340 rank 0 nranks 4 cudaDev 0 nvmlDev 0 busId 10 commId 0x7a7db4e36deb53ea - Init COMPLETE
+[rank2]:[W1223 15:51:28.859981038 Utils.hpp:110] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
+[rank3]:[W1223 15:51:28.859981116 Utils.hpp:110] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
+[rank1]:[W1223 15:51:28.860002924 Utils.hpp:110] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
+[rank0]:[W1223 15:51:28.860009929 Utils.hpp:110] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
+module.sep_network.blocks.0.spk_att.layers.0.self_attn.in_proj_weight not loaded
+module.sep_network.blocks.0.spk_att.layers.0.self_attn.in_proj_bias not loaded
+module.sep_network.blocks.0.spk_att.layers.0.self_attn.out_proj.weight not loaded
+module.sep_network.blocks.0.spk_att.layers.0.self_attn.out_proj.bias not loaded
+module.sep_network.blocks.0.spk_att.layers.0.linear1.weight not loaded
+module.sep_network.blocks.0.spk_att.layers.0.linear1.bias not loaded
+module.sep_network.blocks.0.spk_att.layers.0.linear2.weight not loaded
+module.sep_network.blocks.0.spk_att.layers.0.linear2.bias not loaded
+module.sep_network.blocks.0.spk_att.layers.0.norm1.weight not loaded
+module.sep_network.blocks.0.spk_att.layers.0.norm1.bias not loaded
+module.sep_network.blocks.0.spk_att.layers.0.norm2.weight not loaded
+module.sep_network.blocks.0.spk_att.layers.0.norm2.bias not loaded
+module.sep_network.blocks.0.spk_norm.weight not loaded
+module.sep_network.blocks.0.spk_norm.bias not loaded
+module.sep_network.blocks.1.spk_att.layers.0.self_attn.in_proj_weight not loaded
+module.sep_network.blocks.1.spk_att.layers.0.self_attn.in_proj_bias not loaded
+module.sep_network.blocks.1.spk_att.layers.0.self_attn.out_proj.weight not loaded
+module.sep_network.blocks.1.spk_att.layers.0.self_attn.out_proj.bias not loaded
+module.sep_network.blocks.1.spk_att.layers.0.linear1.weight not loaded
+module.sep_network.blocks.1.spk_att.layers.0.linear1.bias not loaded
+module.sep_network.blocks.1.spk_att.layers.0.linear2.weight not loaded
+module.sep_network.blocks.1.spk_att.layers.0.linear2.bias not loaded
+module.sep_network.blocks.1.spk_att.layers.0.norm1.weight not loaded
+module.sep_network.blocks.1.spk_att.layers.0.norm1.bias not loaded
+module.sep_network.blocks.1.spk_att.layers.0.norm2.weight not loaded
+module.sep_network.blocks.1.spk_att.layers.0.norm2.bias not loaded
+module.sep_network.blocks.1.spk_norm.weight not loaded
+module.sep_network.blocks.1.spk_norm.bias not loaded
+module.sep_network.blocks.2.spk_att.layers.0.self_attn.in_proj_weight not loaded
+module.sep_network.blocks.2.spk_att.layers.0.self_attn.in_proj_bias not loaded
+module.sep_network.blocks.2.spk_att.layers.0.self_attn.out_proj.weight not loaded
+module.sep_network.blocks.2.spk_att.layers.0.self_attn.out_proj.bias not loaded
+module.sep_network.blocks.2.spk_att.layers.0.linear1.weight not loaded
+module.sep_network.blocks.2.spk_att.layers.0.linear1.bias not loaded
+module.sep_network.blocks.2.spk_att.layers.0.linear2.weight not loaded
+module.sep_network.blocks.2.spk_att.layers.0.linear2.bias not loaded
+module.sep_network.blocks.2.spk_att.layers.0.norm1.weight not loaded
+module.sep_network.blocks.2.spk_att.layers.0.norm1.bias not loaded
+module.sep_network.blocks.2.spk_att.layers.0.norm2.weight not loaded
+module.sep_network.blocks.2.spk_att.layers.0.norm2.bias not loaded
+module.sep_network.blocks.2.spk_norm.weight not loaded
+module.sep_network.blocks.2.spk_norm.bias not loaded
+module.sep_network.blocks.3.spk_att.layers.0.self_attn.in_proj_weight not loaded
+module.sep_network.blocks.3.spk_att.layers.0.self_attn.in_proj_bias not loaded
+module.sep_network.blocks.3.spk_att.layers.0.self_attn.out_proj.weight not loaded
+module.sep_network.blocks.3.spk_att.layers.0.self_attn.out_proj.bias not loaded
+module.sep_network.blocks.3.spk_att.layers.0.linear1.weight not loaded
+module.sep_network.blocks.3.spk_att.layers.0.linear1.bias not loaded
+module.sep_network.blocks.3.spk_att.layers.0.linear2.weight not loaded
+module.sep_network.blocks.3.spk_att.layers.0.linear2.bias not loaded
+module.sep_network.blocks.3.spk_att.layers.0.norm1.weight not loaded
+module.sep_network.blocks.3.spk_att.layers.0.norm1.bias not loaded
+module.sep_network.blocks.3.spk_att.layers.0.norm2.weight not loaded
+module.sep_network.blocks.3.spk_att.layers.0.norm2.bias not loaded
+module.sep_network.blocks.3.spk_norm.weight not loaded
+module.sep_network.blocks.3.spk_norm.bias not loaded
+module.sep_network.blocks.4.spk_att.layers.0.self_attn.in_proj_weight not loaded
+module.sep_network.blocks.4.spk_att.layers.0.self_attn.in_proj_bias not loaded
+module.sep_network.blocks.4.spk_att.layers.0.self_attn.out_proj.weight not loaded
+module.sep_network.blocks.4.spk_att.layers.0.self_attn.out_proj.bias not loaded
+module.sep_network.blocks.4.spk_att.layers.0.linear1.weight not loaded
+module.sep_network.blocks.4.spk_att.layers.0.linear1.bias not loaded
+module.sep_network.blocks.4.spk_att.layers.0.linear2.weight not loaded
+module.sep_network.blocks.4.spk_att.layers.0.linear2.bias not loaded
+module.sep_network.blocks.4.spk_att.layers.0.norm1.weight not loaded
+module.sep_network.blocks.4.spk_att.layers.0.norm1.bias not loaded
+module.sep_network.blocks.4.spk_att.layers.0.norm2.weight not loaded
+module.sep_network.blocks.4.spk_att.layers.0.norm2.bias not loaded
+module.sep_network.blocks.4.spk_norm.weight not loaded
+module.sep_network.blocks.4.spk_norm.bias not loaded
+module.sep_network.blocks.5.spk_att.layers.0.self_attn.in_proj_weight not loaded
+module.sep_network.blocks.5.spk_att.layers.0.self_attn.in_proj_bias not loaded
+module.sep_network.blocks.5.spk_att.layers.0.self_attn.out_proj.weight not loaded
+module.sep_network.blocks.5.spk_att.layers.0.self_attn.out_proj.bias not loaded
+module.sep_network.blocks.5.spk_att.layers.0.linear1.weight not loaded
+module.sep_network.blocks.5.spk_att.layers.0.linear1.bias not loaded
+module.sep_network.blocks.5.spk_att.layers.0.linear2.weight not loaded
+module.sep_network.blocks.5.spk_att.layers.0.linear2.bias not loaded
+module.sep_network.blocks.5.spk_att.layers.0.norm1.weight not loaded
+module.sep_network.blocks.5.spk_att.layers.0.norm1.bias not loaded
+module.sep_network.blocks.5.spk_att.layers.0.norm2.weight not loaded
+module.sep_network.blocks.5.spk_att.layers.0.norm2.bias not loaded
+module.sep_network.blocks.5.spk_norm.weight not loaded
+module.sep_network.blocks.5.spk_norm.bias not loaded
+Init model from checkpoints/log_VoxCeleb2_lip_tfgridnet_3spk, and start new training
+[rank1]:[W1223 15:51:50.762137069 reducer.cpp:1400] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
+[rank0]:[W1223 15:51:50.762144793 reducer.cpp:1400] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
+[rank3]:[W1223 15:51:50.762153452 reducer.cpp:1400] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
+[rank2]:[W1223 15:51:50.762154877 reducer.cpp:1400] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
+Train Summary | End of Epoch 1 | Time 12225.14s | Train Loss -10.991
+Valid Summary | End of Epoch 1 | Time 1985.94s | Valid Loss -9.535
+Test Summary | End of Epoch 1 | Time 1191.17s | Test Loss -9.332
+Fund new best model, dict saved
+Train Summary | End of Epoch 2 | Time 12245.66s | Train Loss -11.196
+Valid Summary | End of Epoch 2 | Time 1984.22s | Valid Loss -9.752
+Test Summary | End of Epoch 2 | Time 1191.02s | Test Loss -9.545
+Fund new best model, dict saved
+Train Summary | End of Epoch 3 | Time 12255.68s | Train Loss -11.276
+Valid Summary | End of Epoch 3 | Time 1984.68s | Valid Loss -9.986
+Test Summary | End of Epoch 3 | Time 1190.82s | Test Loss -9.660
+Fund new best model, dict saved
+Train Summary | End of Epoch 4 | Time 12232.89s | Train Loss -11.290
+Valid Summary | End of Epoch 4 | Time 1984.51s | Valid Loss -9.968
+Test Summary | End of Epoch 4 | Time 1190.85s | Test Loss -9.738
+Train Summary | End of Epoch 5 | Time 12251.83s | Train Loss -11.241
+Valid Summary | End of Epoch 5 | Time 1984.37s | Valid Loss -10.024
+Test Summary | End of Epoch 5 | Time 1190.96s | Test Loss -9.794
+Fund new best model, dict saved
+Train Summary | End of Epoch 6 | Time 12265.31s | Train Loss -11.113
+Valid Summary | End of Epoch 6 | Time 1985.26s | Valid Loss -9.982
+Test Summary | End of Epoch 6 | Time 1190.49s | Test Loss -9.781
+Train Summary | End of Epoch 7 | Time 12263.74s | Train Loss -10.994
+Valid Summary | End of Epoch 7 | Time 1984.31s | Valid Loss -9.952
+Test Summary | End of Epoch 7 | Time 1190.49s | Test Loss -9.796
+Train Summary | End of Epoch 8 | Time 12242.87s | Train Loss -10.854
+Valid Summary | End of Epoch 8 | Time 1984.14s | Valid Loss -10.036
+Test Summary | End of Epoch 8 | Time 1191.18s | Test Loss -9.791
+Fund new best model, dict saved
+Train Summary | End of Epoch 9 | Time 12236.27s | Train Loss -10.703
+Valid Summary | End of Epoch 9 | Time 1984.02s | Valid Loss -9.937
+Test Summary | End of Epoch 9 | Time 1190.37s | Test Loss -9.650
+Train Summary | End of Epoch 10 | Time 12223.76s | Train Loss -10.582
+Valid Summary | End of Epoch 10 | Time 1983.95s | Valid Loss -9.851
+Test Summary | End of Epoch 10 | Time 1190.40s | Test Loss -9.663
+Train Summary | End of Epoch 11 | Time 12222.04s | Train Loss -10.333
+Valid Summary | End of Epoch 11 | Time 1984.19s | Valid Loss -9.653
+Test Summary | End of Epoch 11 | Time 1191.05s | Test Loss -9.510
+Train Summary | End of Epoch 12 | Time 12240.55s | Train Loss -10.279
+Valid Summary | End of Epoch 12 | Time 1983.65s | Valid Loss -9.600
+Test Summary | End of Epoch 12 | Time 1190.74s | Test Loss -9.431
+Train Summary | End of Epoch 13 | Time 12235.76s | Train Loss -10.031
+Valid Summary | End of Epoch 13 | Time 1983.34s | Valid Loss -9.060
+Test Summary | End of Epoch 13 | Time 1190.27s | Test Loss -8.872
+reload weights and optimizer from last best checkpoint
+Learning rate adjusted to: 0.000255
+Train Summary | End of Epoch 14 | Time 12226.64s | Train Loss -11.080
+Valid Summary | End of Epoch 14 | Time 1983.85s | Valid Loss -10.234
+Test Summary | End of Epoch 14 | Time 1190.61s | Test Loss -9.979
+Fund new best model, dict saved
+Train Summary | End of Epoch 15 | Time 12229.66s | Train Loss -11.199
+Valid Summary | End of Epoch 15 | Time 1984.06s | Valid Loss -10.375
+Test Summary | End of Epoch 15 | Time 1190.50s | Test Loss -10.135
+Fund new best model, dict saved
+Train Summary | End of Epoch 16 | Time 12232.49s | Train Loss -11.242
+Valid Summary | End of Epoch 16 | Time 1983.78s | Valid Loss -10.281
+Test Summary | End of Epoch 16 | Time 1190.14s | Test Loss -10.068
+Train Summary | End of Epoch 17 | Time 12238.05s | Train Loss -11.275
+Valid Summary | End of Epoch 17 | Time 1983.20s | Valid Loss -10.382
+Test Summary | End of Epoch 17 | Time 1190.36s | Test Loss -10.152
+Fund new best model, dict saved
+Train Summary | End of Epoch 18 | Time 12245.93s | Train Loss -11.297
+Valid Summary | End of Epoch 18 | Time 1983.33s | Valid Loss -10.459
+Test Summary | End of Epoch 18 | Time 1190.37s | Test Loss -10.193
+Fund new best model, dict saved
+Train Summary | End of Epoch 19 | Time 12239.15s | Train Loss -11.333
+Valid Summary | End of Epoch 19 | Time 1983.50s | Valid Loss -10.302
+Test Summary | End of Epoch 19 | Time 1190.54s | Test Loss -10.117
+Train Summary | End of Epoch 20 | Time 12237.59s | Train Loss -11.366
+Valid Summary | End of Epoch 20 | Time 1983.43s | Valid Loss -10.481
+Test Summary | End of Epoch 20 | Time 1190.16s | Test Loss -10.247
+Fund new best model, dict saved
+Train Summary | End of Epoch 21 | Time 12237.17s | Train Loss -11.364
+Valid Summary | End of Epoch 21 | Time 1983.54s | Valid Loss -10.396
+Test Summary | End of Epoch 21 | Time 1189.99s | Test Loss -10.263
+Train Summary | End of Epoch 22 | Time 12247.11s | Train Loss -11.387
+Valid Summary | End of Epoch 22 | Time 1983.62s | Valid Loss -10.483
+Test Summary | End of Epoch 22 | Time 1190.28s | Test Loss -10.264
+Fund new best model, dict saved
+Train Summary | End of Epoch 23 | Time 12239.50s | Train Loss -11.386
+Valid Summary | End of Epoch 23 | Time 1984.03s | Valid Loss -10.445
+Test Summary | End of Epoch 23 | Time 1190.35s | Test Loss -10.239
+Train Summary | End of Epoch 24 | Time 12244.29s | Train Loss -11.395
+Valid Summary | End of Epoch 24 | Time 1983.04s | Valid Loss -10.464
+Test Summary | End of Epoch 24 | Time 1190.10s | Test Loss -10.245
+Train Summary | End of Epoch 25 | Time 12242.35s | Train Loss -11.402
+Valid Summary | End of Epoch 25 | Time 1983.03s | Valid Loss -10.567
+Test Summary | End of Epoch 25 | Time 1189.98s | Test Loss -10.172
+Fund new best model, dict saved
+Train Summary | End of Epoch 26 | Time 12242.38s | Train Loss -11.430
+Valid Summary | End of Epoch 26 | Time 1983.15s | Valid Loss -10.517
+Test Summary | End of Epoch 26 | Time 1190.06s | Test Loss -10.234
+Train Summary | End of Epoch 27 | Time 12241.62s | Train Loss -11.407
+Valid Summary | End of Epoch 27 | Time 1982.98s | Valid Loss -10.536
+Test Summary | End of Epoch 27 | Time 1190.38s | Test Loss -10.245
+Train Summary | End of Epoch 28 | Time 12243.78s | Train Loss -11.443
+Valid Summary | End of Epoch 28 | Time 1982.87s | Valid Loss -10.544
+Test Summary | End of Epoch 28 | Time 1189.98s | Test Loss -10.239
+Train Summary | End of Epoch 29 | Time 12245.88s | Train Loss -11.440
+Valid Summary | End of Epoch 29 | Time 1982.87s | Valid Loss -10.550
+Test Summary | End of Epoch 29 | Time 1190.15s | Test Loss -10.319
+Train Summary | End of Epoch 30 | Time 12239.32s | Train Loss -11.457
+Valid Summary | End of Epoch 30 | Time 1982.92s | Valid Loss -10.515
+Test Summary | End of Epoch 30 | Time 1189.95s | Test Loss -10.311
+reload weights and optimizer from last best checkpoint
+Learning rate adjusted to: 0.000128
+Train Summary | End of Epoch 31 | Time 12240.18s | Train Loss -11.540
+Valid Summary | End of Epoch 31 | Time 1983.06s | Valid Loss -10.636
+Test Summary | End of Epoch 31 | Time 1189.96s | Test Loss -10.376
+Fund new best model, dict saved
+Train Summary | End of Epoch 32 | Time 12236.81s | Train Loss -11.609
+Valid Summary | End of Epoch 32 | Time 1983.10s | Valid Loss -10.657
+Test Summary | End of Epoch 32 | Time 1190.04s | Test Loss -10.379
+Fund new best model, dict saved
+Train Summary | End of Epoch 33 | Time 12239.32s | Train Loss -11.645
+Valid Summary | End of Epoch 33 | Time 1983.07s | Valid Loss -10.643
+Test Summary | End of Epoch 33 | Time 1190.25s | Test Loss -10.411
+Train Summary | End of Epoch 34 | Time 12236.30s | Train Loss -11.641
+Valid Summary | End of Epoch 34 | Time 1983.17s | Valid Loss -10.679
+Test Summary | End of Epoch 34 | Time 1190.24s | Test Loss -10.401
+Fund new best model, dict saved
+Train Summary | End of Epoch 35 | Time 12245.94s | Train Loss -11.678
+Valid Summary | End of Epoch 35 | Time 1983.96s | Valid Loss -10.720
+Test Summary | End of Epoch 35 | Time 1190.20s | Test Loss -10.445
+Fund new best model, dict saved
+Train Summary | End of Epoch 36 | Time 12254.78s | Train Loss -11.681
+Valid Summary | End of Epoch 36 | Time 1983.80s | Valid Loss -10.655
+Test Summary | End of Epoch 36 | Time 1190.39s | Test Loss -10.401
+Train Summary | End of Epoch 37 | Time 12244.39s | Train Loss -11.702
+Valid Summary | End of Epoch 37 | Time 1982.89s | Valid Loss -10.654
+Test Summary | End of Epoch 37 | Time 1189.95s | Test Loss -10.361
+Train Summary | End of Epoch 38 | Time 12244.42s | Train Loss -11.710
+Valid Summary | End of Epoch 38 | Time 1983.02s | Valid Loss -10.683
+Test Summary | End of Epoch 38 | Time 1190.16s | Test Loss -10.420
+Train Summary | End of Epoch 39 | Time 12245.65s | Train Loss -11.723
+Valid Summary | End of Epoch 39 | Time 1982.96s | Valid Loss -10.716
+Test Summary | End of Epoch 39 | Time 1190.09s | Test Loss -10.468
+# 1spk
+Avg SISNR:i tensor([14.2587], device='cuda:0')
+Avg SNRi: 14.708940179199223
+Avg PESQi: 0.9282668574651083
+Avg STOIi: 0.3682121250644893
+# 2spk
+Avg SISNR:i tensor([14.7764], device='cuda:0')
+Avg SNRi: 15.152641806291069
+Avg PESQi: 0.9619782056808471
+Avg STOIi: 0.3763378789653155
+# 3spk 
+Avg SISNR:i tensor([15.5649], device='cuda:0')
+Avg SNRi: 15.93704020578726
+Avg PESQi: 1.0807508533398311
+Avg STOIi: 0.3897622864471111
+
+
+# on LRS2
+# 1spk 
+Avg SISNR:i tensor([13.8892], device='cuda:0')
+Avg SNRi: 14.385947986215816
+Avg PESQi: 0.8105871801376343
+Avg STOIi: 0.35947340789408305
+# 2spk 
+Avg SISNR:i tensor([14.4489], device='cuda:0')
+Avg SNRi: 14.8332929594773
+Avg PESQi: 0.8476730751593907
+Avg STOIi: 0.36752675748255176
+
+# 3spk 
+Avg SISNR:i tensor([15.7151], device='cuda:0')
+Avg SNRi: 16.10634403873235
+Avg PESQi: 1.0045358400742213
+Avg STOIi: 0.3877202131964405
+
+
+# on LRS3
+# 1spk 
+Avg SISNR:i tensor([15.1143], device='cuda:0')
+Avg SNRi: 15.476579012213097
+Avg PESQi: 1.0222730774879456
+Avg STOIi: 0.3547696576788025
+# 2spk 
+Avg SISNR:i tensor([15.8752], device='cuda:0')
+Avg SNRi: 16.128778737789872
+Avg PESQi: 1.0782776288191478
+Avg STOIi: 0.36668242221201147
+# 3spk 
+Total 3000 samples evaluated.
+Total 1 samples not evaluated for pesq.
+Avg SISNR:i tensor([17.2220], device='cuda:0')
+Avg SNRi: 17.448678353901542
+Avg PESQi: 1.2627000254789988
+Avg STOIi: 0.38352159958506526

checkpoints/log_VoxCeleb2_lip_tfgridnet-isam_3spk /tensorboard/events.out.tfevents.1734940287.dlchqhe6f3ef1ed0-master-0.28.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3376efcc409b958e793ea56a2df50a80676d187d8a1149eacd6156546581fce8
+size 5860