Delete checkpoints/log_VoxCeleb2_tfgridnet-isam_2spk

checkpoints/log_VoxCeleb2_tfgridnet-isam_2spk/config.yaml

@@ -1,59 +0,0 @@
-## Config file
-
-# Log 
-seed: 777
-use_cuda: 1           # 1 for True, 0 for False
-
-# dataset
-speaker_no: 2
-mix_lst_path: ./data/VoxCeleb2_non_repeat/mixture_data_list_2mix.csv
-audio_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/audio_clean/
-reference_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/orig/
-# mix_lst_path: ./data/LRS2/mixture_data_list_2mix.csv
-# audio_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/LRS2/audio_clean/
-# reference_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/LRS2/mvlrs_v1/
-# mix_lst_path: ./data/LRS3/mixture_data_list_2mix.csv
-# audio_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/LRS3/audio_clean/
-# reference_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/LRS3/orig/
-audio_sr: 16000
-ref_sr: 25
-
-# dataloader
-num_workers: 2
-batch_size: 1         # 4-GPU training with a total effective batch size of 8
-accu_grad: 1
-effec_batch_size: 2   # per GPU, only used if accu_grad is set to 1, must be multiple times of batch size
-max_length: 3         # truncate the utterances in dataloader, in seconds 
-
-# network settings
-init_from: 'checkpoints/log_VoxCeleb2_lip_tfgridnet_2spk'       # 'None' or a log name 'log_2024-07-22(18:12:13)'
-causal: 0             # 1 for True, 0 for False
-network_reference:
-  cue: lip            # lip or speech or gesture or EEG
-  backbone: resnet18  # resnet18 or shufflenetV2 or blazenet64
-  emb_size: 256       # resnet18:256
-network_audio:
-  backbone: av_tfgridnet_isam
-  n_fft: 256
-  stride: 128
-  window: "hann"
-  use_builtin_complex: False
-  n_srcs:  1
-  n_imics: 1
-  n_layers: 6
-  lstm_hidden_units: 192
-  attn_n_head: 4
-  attn_qk_output_channel: 4
-  emb_dim: 48
-  emb_ks: 4
-  emb_hs: 1
-  activation: "prelu"
-  isam: 1
-
-# optimizer
-spk_att_dropout: 1    # 0 for always use speaker attention
-loss_type: ss_sisdr      # "snr", "sisdr", "hybrid"
-lr_warmup: 1
-init_learning_rate: 0.0005
-max_epoch: 150
-clip_grad_norm: 5

checkpoints/log_VoxCeleb2_tfgridnet-isam_2spk/last_best_checkpoint.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:99ab899c261721b450ae9c494b71cf33c3e08f962c458760c14cfc5cf4659c89
-size 162585486

checkpoints/log_VoxCeleb2_tfgridnet-isam_2spk/last_checkpoint.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e2bd23023c6ba194a49e46a5c16798184fe1a5b7f38b224914f4a49a6f5a0cb1
-size 162576686

checkpoints/log_VoxCeleb2_tfgridnet-isam_2spk/log_2024-12-23(15:50:05).txt

@@ -1,762 +0,0 @@
-## Config file
-
-# Log 
-seed: 777
-use_cuda: 1           # 1 for True, 0 for False
-
-# dataset
-speaker_no: 2
-mix_lst_path: ./data/VoxCeleb2_non_repeat/mixture_data_list_2mix.csv
-audio_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/audio_clean/
-reference_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/orig/
-audio_sr: 16000
-ref_sr: 25
-
-# dataloader
-num_workers: 2
-batch_size: 1         # 4-GPU training with a total effective batch size of 8
-accu_grad: 1
-effec_batch_size: 2   # per GPU, only used if accu_grad is set to 1, must be multiple times of batch size
-max_length: 3         # truncate the utterances in dataloader, in seconds 
-
-# network settings
-init_from: 'checkpoints/log_VoxCeleb2_lip_tfgridnet_2spk'       # 'None' or a log name 'log_2024-07-22(18:12:13)'
-causal: 0             # 1 for True, 0 for False
-network_reference:
-  cue: lip            # lip or speech or gesture or EEG
-  backbone: resnet18  # resnet18 or shufflenetV2 or blazenet64
-  emb_size: 256       # resnet18:256
-network_audio:
-  backbone: av_tfgridnet_att_ss
-  n_fft: 256
-  stride: 128
-  window: "hann"
-  use_builtin_complex: False
-  n_srcs:  1
-  n_imics: 1
-  n_layers: 6
-  lstm_hidden_units: 192
-  attn_n_head: 4
-  attn_qk_output_channel: 4
-  emb_dim: 48
-  emb_ks: 4
-  emb_hs: 1
-  activation: "prelu"
-
-# optimizer
-spk_att_dropout: 1    # 0 for always use speaker attention
-loss_type: ss_sisdr      # "snr", "sisdr", "hybrid"
-lr_warmup: 1
-init_learning_rate: 0.0005
-max_epoch: 150
-clip_grad_norm: 5
-W1223 15:50:36.040409 140547448362816 torch/distributed/run.py:779] 
-W1223 15:50:36.040409 140547448362816 torch/distributed/run.py:779] *****************************************
-W1223 15:50:36.040409 140547448362816 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
-W1223 15:50:36.040409 140547448362816 torch/distributed/run.py:779] *****************************************
-[W1223 15:50:57.605401514 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
-[W1223 15:50:57.606194396 Utils.hpp:135] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
-[W1223 15:50:57.605414589 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
-[W1223 15:50:57.606215894 Utils.hpp:135] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
-[W1223 15:50:57.605418359 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
-[W1223 15:50:57.606241166 Utils.hpp:135] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
-[W1223 15:50:57.605453172 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
-[W1223 15:50:57.606286008 Utils.hpp:135] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
-started on checkpoints/log_2024-12-23(15:50:05)
-
-namespace(accu_grad=1, audio_direc='/mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/audio_clean/', audio_sr=16000, batch_size=1, causal=0, checkpoint_dir='checkpoints/log_2024-12-23(15:50:05)', clip_grad_norm=5.0, config=[<yamlargparse.Path object at 0x7fdc6754ee80>], device=device(type='cuda'), distributed=True, effec_batch_size=2, evaluate_only=0, init_from='checkpoints/log_VoxCeleb2_lip_tfgridnet_2spk', init_learning_rate=0.0005, local_rank=0, loss_type='ss_sisdr', lr_warmup=1, max_epoch=150, max_length=3, mix_lst_path='./data/VoxCeleb2_non_repeat/mixture_data_list_2mix.csv', network_audio=namespace(activation='prelu', attn_n_head=4, attn_qk_output_channel=4, backbone='av_tfgridnet_att_ss', emb_dim=48, emb_hs=1, emb_ks=4, lstm_hidden_units=192, n_fft=256, n_imics=1, n_layers=6, n_srcs=1, stride=128, use_builtin_complex=False, window='hann'), network_reference=namespace(backbone='resnet18', cue='lip', emb_size=256), num_workers=2, ref_sr=25, reference_direc='/mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/orig/', seed=777, speaker_no=2, spk_att_dropout=1, train_from_last_checkpoint=0, use_cuda=1, world_size=4)
-network_wrapper(
-  (sep_network): av_TFGridNetV3_att_ss(
-    (enc): STFTEncoder(
-      (stft): Stft(n_fft=256, win_length=256, hop_length=128, center=True, normalized=False, onesided=True)
-    )
-    (dec): STFTDecoder(
-      (stft): Stft(n_fft=256, win_length=256, hop_length=128, center=True, normalized=False, onesided=True)
-    )
-    (conv): Sequential(
-      (0): Conv2d(2, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
-      (1): GroupNorm(1, 48, eps=1e-05, affine=True)
-    )
-    (blocks): ModuleList(
-      (0-5): 6 x GridNetV3Block(
-        (intra_norm): LayerNorm((48,), eps=1e-05, elementwise_affine=True)
-        (intra_rnn): LSTM(192, 192, batch_first=True, bidirectional=True)
-        (intra_linear): ConvTranspose1d(384, 48, kernel_size=(4,), stride=(1,))
-        (inter_norm): LayerNorm((48,), eps=1e-05, elementwise_affine=True)
-        (inter_rnn): LSTM(192, 192, batch_first=True, bidirectional=True)
-        (inter_linear): ConvTranspose1d(384, 48, kernel_size=(4,), stride=(1,))
-        (attn_conv_Q): Conv2d(48, 16, kernel_size=(1, 1), stride=(1, 1))
-        (attn_norm_Q): AllHeadPReLULayerNormalization4DC(
-          (act): PReLU(num_parameters=4)
-        )
-        (attn_conv_K): Conv2d(48, 16, kernel_size=(1, 1), stride=(1, 1))
-        (attn_norm_K): AllHeadPReLULayerNormalization4DC(
-          (act): PReLU(num_parameters=4)
-        )
-        (attn_conv_V): Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1))
-        (attn_norm_V): AllHeadPReLULayerNormalization4DC(
-          (act): PReLU(num_parameters=4)
-        )
-        (attn_concat_proj): Sequential(
-          (0): Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1))
-          (1): PReLU(num_parameters=1)
-          (2): LayerNormalization()
-        )
-        (spk_att): TransformerEncoder(
-          (layers): ModuleList(
-            (0): TransformerEncoderLayer(
-              (self_attn): MultiheadAttention(
-                (out_proj): NonDynamicallyQuantizableLinear(in_features=48, out_features=48, bias=True)
-              )
-              (linear1): Linear(in_features=48, out_features=192, bias=True)
-              (dropout): Dropout(p=0.1, inplace=False)
-              (linear2): Linear(in_features=192, out_features=48, bias=True)
-              (norm1): LayerNorm((48,), eps=1e-05, elementwise_affine=True)
-              (norm2): LayerNorm((48,), eps=1e-05, elementwise_affine=True)
-              (dropout1): Dropout(p=0.1, inplace=False)
-              (dropout2): Dropout(p=0.1, inplace=False)
-            )
-          )
-        )
-        (spk_norm): GroupNorm(1, 48, eps=1e-08, affine=True)
-      )
-    )
-    (deconv): ConvTranspose2d(48, 2, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
-    (av_conv): ModuleList(
-      (0-5): 6 x Linear(in_features=304, out_features=48, bias=True)
-    )
-  )
-  (ref_encoder): Visual_encoder(
-    (v_frontend): VisualFrontend(
-      (frontend3D): Sequential(
-        (0): Conv3d(1, 64, kernel_size=(5, 7, 7), stride=(1, 2, 2), padding=(2, 3, 3), bias=False)
-        (1): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-        (2): ReLU()
-        (3): MaxPool3d(kernel_size=(1, 3, 3), stride=(1, 2, 2), padding=(0, 1, 1), dilation=1, ceil_mode=False)
-      )
-      (resnet): ResNet(
-        (layer1): ResNetLayer(
-          (conv1a): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (bn1a): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-          (conv2a): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (downsample): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
-          (outbna): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-          (conv1b): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (bn1b): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-          (conv2b): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (outbnb): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-        )
-        (layer2): ResNetLayer(
-          (conv1a): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
-          (bn1a): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-          (conv2a): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (downsample): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)
-          (outbna): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-          (conv1b): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (bn1b): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-          (conv2b): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (outbnb): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-        )
-        (layer3): ResNetLayer(
-          (conv1a): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
-          (bn1a): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-          (conv2a): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (downsample): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)
-          (outbna): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-          (conv1b): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (bn1b): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-          (conv2b): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (outbnb): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-        )
-        (layer4): ResNetLayer(
-          (conv1a): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
-          (bn1a): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-          (conv2a): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (downsample): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)
-          (outbna): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-          (conv1b): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (bn1b): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-          (conv2b): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (outbnb): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-        )
-        (avgpool): AvgPool2d(kernel_size=(4, 4), stride=(1, 1), padding=0)
-      )
-    )
-    (v_ds): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
-    (visual_conv): Sequential(
-      (0): VisualConv1D(
-        (relu_0): ReLU()
-        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
-        (relu): ReLU()
-        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
-        (prelu): PReLU(num_parameters=1)
-        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
-      )
-      (1): VisualConv1D(
-        (relu_0): ReLU()
-        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
-        (relu): ReLU()
-        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
-        (prelu): PReLU(num_parameters=1)
-        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
-      )
-      (2): VisualConv1D(
-        (relu_0): ReLU()
-        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
-        (relu): ReLU()
-        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
-        (prelu): PReLU(num_parameters=1)
-        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
-      )
-      (3): VisualConv1D(
-        (relu_0): ReLU()
-        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
-        (relu): ReLU()
-        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
-        (prelu): PReLU(num_parameters=1)
-        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
-      )
-      (4): VisualConv1D(
-        (relu_0): ReLU()
-        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
-        (relu): ReLU()
-        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
-        (prelu): PReLU(num_parameters=1)
-        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
-      )
-    )
-  )
-)
-
-Total number of parameters: 20950309 
-
-
-Total number of trainable parameters: 9765221 
-
-dlcdanw1zq2cucwx-master-0:28:28 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
-dlcdanw1zq2cucwx-master-0:28:28 [0] NCCL INFO Bootstrap : Using eth0:22.5.96.241<0>
-dlcdanw1zq2cucwx-master-0:28:28 [0] NCCL INFO Plugin name set by env to libnccl-net-none.so
-dlcdanw1zq2cucwx-master-0:28:28 [0] NCCL INFO NET/Plugin : dlerror=libnccl-net-none.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net-none.so), using internal implementation
-dlcdanw1zq2cucwx-master-0:28:28 [0] NCCL INFO cudaDriverVersion 11040
-dlcdanw1zq2cucwx-master-0:30:30 [2] NCCL INFO cudaDriverVersion 11040
-dlcdanw1zq2cucwx-master-0:31:31 [3] NCCL INFO cudaDriverVersion 11040
-dlcdanw1zq2cucwx-master-0:29:29 [1] NCCL INFO cudaDriverVersion 11040
-NCCL version 2.20.5+cuda11.8
-dlcdanw1zq2cucwx-master-0:30:30 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
-dlcdanw1zq2cucwx-master-0:31:31 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
-dlcdanw1zq2cucwx-master-0:29:29 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
-dlcdanw1zq2cucwx-master-0:31:31 [3] NCCL INFO Bootstrap : Using eth0:22.5.96.241<0>
-dlcdanw1zq2cucwx-master-0:30:30 [2] NCCL INFO Bootstrap : Using eth0:22.5.96.241<0>
-dlcdanw1zq2cucwx-master-0:31:31 [3] NCCL INFO Plugin name set by env to libnccl-net-none.so
-dlcdanw1zq2cucwx-master-0:29:29 [1] NCCL INFO Bootstrap : Using eth0:22.5.96.241<0>
-dlcdanw1zq2cucwx-master-0:30:30 [2] NCCL INFO Plugin name set by env to libnccl-net-none.so
-dlcdanw1zq2cucwx-master-0:29:29 [1] NCCL INFO Plugin name set by env to libnccl-net-none.so
-dlcdanw1zq2cucwx-master-0:30:30 [2] NCCL INFO NET/Plugin : dlerror=libnccl-net-none.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net-none.so), using internal implementation
-dlcdanw1zq2cucwx-master-0:31:31 [3] NCCL INFO NET/Plugin : dlerror=libnccl-net-none.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net-none.so), using internal implementation
-dlcdanw1zq2cucwx-master-0:29:29 [1] NCCL INFO NET/Plugin : dlerror=libnccl-net-none.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net-none.so), using internal implementation
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO NCCL_IB_HCA set to mlx5
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO NCCL_IB_HCA set to mlx5
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO NCCL_IB_HCA set to mlx5
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO NCCL_IB_HCA set to mlx5
-libibverbs: Warning: couldn't load driver 'libhfi1verbs-rdmav25.so': libhfi1verbs-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libhfi1verbs-rdmav25.so': libhfi1verbs-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libhfi1verbs-rdmav25.so': libhfi1verbs-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libhfi1verbs-rdmav25.so': libhfi1verbs-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'librxe-rdmav25.so': librxe-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'librxe-rdmav25.so': librxe-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'librxe-rdmav25.so': librxe-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'librxe-rdmav25.so': librxe-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libmthca-rdmav25.so': libmthca-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libmthca-rdmav25.so': libmthca-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libmthca-rdmav25.so': libmthca-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libmthca-rdmav25.so': libmthca-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libvmw_pvrdma-rdmav25.so': libvmw_pvrdma-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libvmw_pvrdma-rdmav25.so': libvmw_pvrdma-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libvmw_pvrdma-rdmav25.so': libvmw_pvrdma-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libvmw_pvrdma-rdmav25.so': libvmw_pvrdma-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libhns-rdmav25.so': libhns-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libhns-rdmav25.so': libhns-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libhns-rdmav25.so': libhns-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libhns-rdmav25.so': libhns-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libipathverbs-rdmav25.so': libipathverbs-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libipathverbs-rdmav25.so': libipathverbs-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libipathverbs-rdmav25.so': libipathverbs-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libipathverbs-rdmav25.so': libipathverbs-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libsiw-rdmav25.so': libsiw-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libsiw-rdmav25.so': libsiw-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libsiw-rdmav25.so': libsiw-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libsiw-rdmav25.so': libsiw-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libbnxt_re-rdmav25.so': libbnxt_re-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libbnxt_re-rdmav25.so': libbnxt_re-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libbnxt_re-rdmav25.so': libbnxt_re-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libbnxt_re-rdmav25.so': libbnxt_re-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libocrdma-rdmav25.so': libocrdma-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libocrdma-rdmav25.so': libocrdma-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libocrdma-rdmav25.so': libocrdma-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libocrdma-rdmav25.so': libocrdma-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libmlx4-rdmav25.so': libmlx4-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libmlx4-rdmav25.so': libmlx4-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libmlx4-rdmav25.so': libmlx4-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libmlx4-rdmav25.so': libmlx4-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libqedr-rdmav25.so': libqedr-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libqedr-rdmav25.so': libqedr-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libqedr-rdmav25.so': libqedr-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libqedr-rdmav25.so': libqedr-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libcxgb4-rdmav25.so': libcxgb4-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libcxgb4-rdmav25.so': libcxgb4-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libcxgb4-rdmav25.so': libcxgb4-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libcxgb4-rdmav25.so': libcxgb4-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libi40iw-rdmav25.so': libi40iw-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libi40iw-rdmav25.so': libi40iw-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libi40iw-rdmav25.so': libi40iw-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libi40iw-rdmav25.so': libi40iw-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libefa-rdmav25.so': libefa-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libefa-rdmav25.so': libefa-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libefa-rdmav25.so': libefa-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libefa-rdmav25.so': libefa-rdmav25.so: cannot open shared object file: No such file or directory
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [RO]; OOB eth0:22.5.96.241<0>
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Using non-device net plugin version 0
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Using network IB
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [RO]; OOB eth0:22.5.96.241<0>
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Using non-device net plugin version 0
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Using network IB
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [RO]; OOB eth0:22.5.96.241<0>
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Using non-device net plugin version 0
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Using network IB
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [RO]; OOB eth0:22.5.96.241<0>
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Using non-device net plugin version 0
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Using network IB
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO comm 0x866fc40 rank 1 nranks 4 cudaDev 1 nvmlDev 1 busId 20 commId 0xafe80e26c24ecacf - Init START
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO comm 0x9d94400 rank 2 nranks 4 cudaDev 2 nvmlDev 2 busId 30 commId 0xafe80e26c24ecacf - Init START
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO comm 0x6be36a0 rank 3 nranks 4 cudaDev 3 nvmlDev 3 busId 40 commId 0xafe80e26c24ecacf - Init START
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO comm 0x87bffd0 rank 0 nranks 4 cudaDev 0 nvmlDev 0 busId 10 commId 0xafe80e26c24ecacf - Init START
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Setting affinity for GPU 1 to ffffff
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Setting affinity for GPU 2 to ffffff
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Setting affinity for GPU 0 to ffffff
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO comm 0x9d94400 rank 2 nRanks 4 nNodes 1 localRanks 4 localRank 2 MNNVL 0
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO comm 0x6be36a0 rank 3 nRanks 4 nNodes 1 localRanks 4 localRank 3 MNNVL 0
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO comm 0x87bffd0 rank 0 nRanks 4 nNodes 1 localRanks 4 localRank 0 MNNVL 0
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO comm 0x866fc40 rank 1 nRanks 4 nNodes 1 localRanks 4 localRank 1 MNNVL 0
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Channel 00/12 :    0   1   2   3
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 3/-1/-1->2->1 [4] 3/-1/-1->2->1 [5] 3/-1/-1->2->1 [6] 3/-1/-1->2->1 [7] 3/-1/-1->2->1 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 3/-1/-1->2->1 [11] 3/-1/-1->2->1
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Channel 01/12 :    0   1   2   3
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO P2P Chunksize set to 524288
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 [2] 2/-1/-1->1->0 [3] 2/-1/-1->1->0 [4] 2/-1/-1->1->0 [5] 2/-1/-1->1->0 [6] 2/-1/-1->1->0 [7] 2/-1/-1->1->0 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->0 [10] 2/-1/-1->1->0 [11] 2/-1/-1->1->0
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Channel 02/12 :    0   1   2   3
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Trees [0] -1/-1/-1->3->2 [1] -1/-1/-1->3->2 [2] -1/-1/-1->3->2 [3] -1/-1/-1->3->2 [4] -1/-1/-1->3->2 [5] -1/-1/-1->3->2 [6] -1/-1/-1->3->2 [7] -1/-1/-1->3->2 [8] -1/-1/-1->3->2 [9] -1/-1/-1->3->2 [10] -1/-1/-1->3->2 [11] -1/-1/-1->3->2
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Channel 03/12 :    0   1   2   3
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO P2P Chunksize set to 524288
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO P2P Chunksize set to 524288
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Channel 04/12 :    0   1   2   3
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Channel 05/12 :    0   1   2   3
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Channel 06/12 :    0   1   2   3
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Channel 07/12 :    0   1   2   3
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Channel 08/12 :    0   1   2   3
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Channel 09/12 :    0   1   2   3
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Channel 10/12 :    0   1   2   3
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Channel 11/12 :    0   1   2   3
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] 1/-1/-1->0->-1 [5] 1/-1/-1->0->-1 [6] 1/-1/-1->0->-1 [7] 1/-1/-1->0->-1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO P2P Chunksize set to 524288
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Channel 00/0 : 3[3] -> 0[0] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Channel 02/0 : 1[1] -> 2[2] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Channel 01/0 : 3[3] -> 0[0] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Channel 02/0 : 2[2] -> 3[3] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Channel 02/0 : 3[3] -> 0[0] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Channel 03/0 : 2[2] -> 3[3] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Channel 04/0 : 1[1] -> 2[2] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Channel 03/0 : 3[3] -> 0[0] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Channel 04/0 : 2[2] -> 3[3] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Channel 05/0 : 1[1] -> 2[2] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Channel 04/0 : 3[3] -> 0[0] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Channel 06/0 : 1[1] -> 2[2] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Channel 05/0 : 2[2] -> 3[3] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Channel 05/0 : 3[3] -> 0[0] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Channel 06/0 : 3[3] -> 0[0] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Channel 06/0 : 2[2] -> 3[3] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Channel 08/0 : 1[1] -> 2[2] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Channel 07/0 : 3[3] -> 0[0] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Channel 08/0 : 3[3] -> 0[0] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Channel 08/0 : 2[2] -> 3[3] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Channel 10/0 : 1[1] -> 2[2] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Channel 09/0 : 3[3] -> 0[0] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Channel 10/0 : 3[3] -> 0[0] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Channel 10/0 : 2[2] -> 3[3] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Channel 11/0 : 3[3] -> 0[0] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Channel 11/0 : 2[2] -> 3[3] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Connected all rings
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Connected all rings
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Channel 00/0 : 3[3] -> 2[2] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Connected all rings
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Connected all rings
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Channel 01/0 : 3[3] -> 2[2] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Channel 02/0 : 3[3] -> 2[2] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Channel 03/0 : 3[3] -> 2[2] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Channel 04/0 : 3[3] -> 2[2] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Channel 05/0 : 3[3] -> 2[2] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Channel 06/0 : 3[3] -> 2[2] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Channel 07/0 : 3[3] -> 2[2] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Channel 08/0 : 3[3] -> 2[2] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Channel 09/0 : 3[3] -> 2[2] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Channel 10/0 : 3[3] -> 2[2] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Channel 11/0 : 3[3] -> 2[2] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Channel 00/0 : 2[2] -> 1[1] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Channel 00/0 : 1[1] -> 0[0] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Channel 01/0 : 2[2] -> 1[1] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Channel 01/0 : 1[1] -> 0[0] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Channel 02/0 : 2[2] -> 1[1] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Channel 02/0 : 1[1] -> 0[0] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Channel 03/0 : 2[2] -> 1[1] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Channel 03/0 : 1[1] -> 0[0] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Channel 04/0 : 2[2] -> 1[1] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Channel 04/0 : 1[1] -> 0[0] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Channel 05/0 : 2[2] -> 1[1] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Channel 05/0 : 1[1] -> 0[0] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Channel 06/0 : 2[2] -> 1[1] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Channel 06/0 : 1[1] -> 0[0] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Channel 07/0 : 2[2] -> 1[1] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Channel 07/0 : 1[1] -> 0[0] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Channel 08/0 : 2[2] -> 1[1] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Channel 08/0 : 1[1] -> 0[0] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Channel 09/0 : 2[2] -> 1[1] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Channel 09/0 : 1[1] -> 0[0] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Channel 10/0 : 2[2] -> 1[1] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Channel 10/0 : 1[1] -> 0[0] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Channel 11/0 : 2[2] -> 1[1] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Channel 11/0 : 1[1] -> 0[0] via P2P/IPC/read
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO Connected all trees
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO 12 coll channels, 0 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO Connected all trees
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO Connected all trees
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO 12 coll channels, 0 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO 12 coll channels, 0 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO Connected all trees
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO 12 coll channels, 0 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer
-dlcdanw1zq2cucwx-master-0:28:57 [0] NCCL INFO comm 0x87bffd0 rank 0 nranks 4 cudaDev 0 nvmlDev 0 busId 10 commId 0xafe80e26c24ecacf - Init COMPLETE
-dlcdanw1zq2cucwx-master-0:29:60 [1] NCCL INFO comm 0x866fc40 rank 1 nranks 4 cudaDev 1 nvmlDev 1 busId 20 commId 0xafe80e26c24ecacf - Init COMPLETE
-dlcdanw1zq2cucwx-master-0:30:58 [2] NCCL INFO comm 0x9d94400 rank 2 nranks 4 cudaDev 2 nvmlDev 2 busId 30 commId 0xafe80e26c24ecacf - Init COMPLETE
-dlcdanw1zq2cucwx-master-0:31:59 [3] NCCL INFO comm 0x6be36a0 rank 3 nranks 4 cudaDev 3 nvmlDev 3 busId 40 commId 0xafe80e26c24ecacf - Init COMPLETE
-[rank1]:[W1223 15:51:28.897884818 Utils.hpp:110] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
-[rank0]:[W1223 15:51:28.897922529 Utils.hpp:110] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
-[rank2]:[W1223 15:51:28.897929604 Utils.hpp:110] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
-[rank3]:[W1223 15:51:28.898014217 Utils.hpp:110] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
-module.sep_network.blocks.0.spk_att.layers.0.self_attn.in_proj_weight not loaded
-module.sep_network.blocks.0.spk_att.layers.0.self_attn.in_proj_bias not loaded
-module.sep_network.blocks.0.spk_att.layers.0.self_attn.out_proj.weight not loaded
-module.sep_network.blocks.0.spk_att.layers.0.self_attn.out_proj.bias not loaded
-module.sep_network.blocks.0.spk_att.layers.0.linear1.weight not loaded
-module.sep_network.blocks.0.spk_att.layers.0.linear1.bias not loaded
-module.sep_network.blocks.0.spk_att.layers.0.linear2.weight not loaded
-module.sep_network.blocks.0.spk_att.layers.0.linear2.bias not loaded
-module.sep_network.blocks.0.spk_att.layers.0.norm1.weight not loaded
-module.sep_network.blocks.0.spk_att.layers.0.norm1.bias not loaded
-module.sep_network.blocks.0.spk_att.layers.0.norm2.weight not loaded
-module.sep_network.blocks.0.spk_att.layers.0.norm2.bias not loaded
-module.sep_network.blocks.0.spk_norm.weight not loaded
-module.sep_network.blocks.0.spk_norm.bias not loaded
-module.sep_network.blocks.1.spk_att.layers.0.self_attn.in_proj_weight not loaded
-module.sep_network.blocks.1.spk_att.layers.0.self_attn.in_proj_bias not loaded
-module.sep_network.blocks.1.spk_att.layers.0.self_attn.out_proj.weight not loaded
-module.sep_network.blocks.1.spk_att.layers.0.self_attn.out_proj.bias not loaded
-module.sep_network.blocks.1.spk_att.layers.0.linear1.weight not loaded
-module.sep_network.blocks.1.spk_att.layers.0.linear1.bias not loaded
-module.sep_network.blocks.1.spk_att.layers.0.linear2.weight not loaded
-module.sep_network.blocks.1.spk_att.layers.0.linear2.bias not loaded
-module.sep_network.blocks.1.spk_att.layers.0.norm1.weight not loaded
-module.sep_network.blocks.1.spk_att.layers.0.norm1.bias not loaded
-module.sep_network.blocks.1.spk_att.layers.0.norm2.weight not loaded
-module.sep_network.blocks.1.spk_att.layers.0.norm2.bias not loaded
-module.sep_network.blocks.1.spk_norm.weight not loaded
-module.sep_network.blocks.1.spk_norm.bias not loaded
-module.sep_network.blocks.2.spk_att.layers.0.self_attn.in_proj_weight not loaded
-module.sep_network.blocks.2.spk_att.layers.0.self_attn.in_proj_bias not loaded
-module.sep_network.blocks.2.spk_att.layers.0.self_attn.out_proj.weight not loaded
-module.sep_network.blocks.2.spk_att.layers.0.self_attn.out_proj.bias not loaded
-module.sep_network.blocks.2.spk_att.layers.0.linear1.weight not loaded
-module.sep_network.blocks.2.spk_att.layers.0.linear1.bias not loaded
-module.sep_network.blocks.2.spk_att.layers.0.linear2.weight not loaded
-module.sep_network.blocks.2.spk_att.layers.0.linear2.bias not loaded
-module.sep_network.blocks.2.spk_att.layers.0.norm1.weight not loaded
-module.sep_network.blocks.2.spk_att.layers.0.norm1.bias not loaded
-module.sep_network.blocks.2.spk_att.layers.0.norm2.weight not loaded
-module.sep_network.blocks.2.spk_att.layers.0.norm2.bias not loaded
-module.sep_network.blocks.2.spk_norm.weight not loaded
-module.sep_network.blocks.2.spk_norm.bias not loaded
-module.sep_network.blocks.3.spk_att.layers.0.self_attn.in_proj_weight not loaded
-module.sep_network.blocks.3.spk_att.layers.0.self_attn.in_proj_bias not loaded
-module.sep_network.blocks.3.spk_att.layers.0.self_attn.out_proj.weight not loaded
-module.sep_network.blocks.3.spk_att.layers.0.self_attn.out_proj.bias not loaded
-module.sep_network.blocks.3.spk_att.layers.0.linear1.weight not loaded
-module.sep_network.blocks.3.spk_att.layers.0.linear1.bias not loaded
-module.sep_network.blocks.3.spk_att.layers.0.linear2.weight not loaded
-module.sep_network.blocks.3.spk_att.layers.0.linear2.bias not loaded
-module.sep_network.blocks.3.spk_att.layers.0.norm1.weight not loaded
-module.sep_network.blocks.3.spk_att.layers.0.norm1.bias not loaded
-module.sep_network.blocks.3.spk_att.layers.0.norm2.weight not loaded
-module.sep_network.blocks.3.spk_att.layers.0.norm2.bias not loaded
-module.sep_network.blocks.3.spk_norm.weight not loaded
-module.sep_network.blocks.3.spk_norm.bias not loaded
-module.sep_network.blocks.4.spk_att.layers.0.self_attn.in_proj_weight not loaded
-module.sep_network.blocks.4.spk_att.layers.0.self_attn.in_proj_bias not loaded
-module.sep_network.blocks.4.spk_att.layers.0.self_attn.out_proj.weight not loaded
-module.sep_network.blocks.4.spk_att.layers.0.self_attn.out_proj.bias not loaded
-module.sep_network.blocks.4.spk_att.layers.0.linear1.weight not loaded
-module.sep_network.blocks.4.spk_att.layers.0.linear1.bias not loaded
-module.sep_network.blocks.4.spk_att.layers.0.linear2.weight not loaded
-module.sep_network.blocks.4.spk_att.layers.0.linear2.bias not loaded
-module.sep_network.blocks.4.spk_att.layers.0.norm1.weight not loaded
-module.sep_network.blocks.4.spk_att.layers.0.norm1.bias not loaded
-module.sep_network.blocks.4.spk_att.layers.0.norm2.weight not loaded
-module.sep_network.blocks.4.spk_att.layers.0.norm2.bias not loaded
-module.sep_network.blocks.4.spk_norm.weight not loaded
-module.sep_network.blocks.4.spk_norm.bias not loaded
-module.sep_network.blocks.5.spk_att.layers.0.self_attn.in_proj_weight not loaded
-module.sep_network.blocks.5.spk_att.layers.0.self_attn.in_proj_bias not loaded
-module.sep_network.blocks.5.spk_att.layers.0.self_attn.out_proj.weight not loaded
-module.sep_network.blocks.5.spk_att.layers.0.self_attn.out_proj.bias not loaded
-module.sep_network.blocks.5.spk_att.layers.0.linear1.weight not loaded
-module.sep_network.blocks.5.spk_att.layers.0.linear1.bias not loaded
-module.sep_network.blocks.5.spk_att.layers.0.linear2.weight not loaded
-module.sep_network.blocks.5.spk_att.layers.0.linear2.bias not loaded
-module.sep_network.blocks.5.spk_att.layers.0.norm1.weight not loaded
-module.sep_network.blocks.5.spk_att.layers.0.norm1.bias not loaded
-module.sep_network.blocks.5.spk_att.layers.0.norm2.weight not loaded
-module.sep_network.blocks.5.spk_att.layers.0.norm2.bias not loaded
-module.sep_network.blocks.5.spk_norm.weight not loaded
-module.sep_network.blocks.5.spk_norm.bias not loaded
-Init model from checkpoints/log_VoxCeleb2_lip_tfgridnet_2spk, and start new training
-[rank1]:[W1223 15:53:04.527524171 reducer.cpp:1400] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
-[rank2]:[W1223 15:53:04.529782679 reducer.cpp:1400] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
-[rank0]:[W1223 15:53:04.529824121 reducer.cpp:1400] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
-[rank3]:[W1223 15:53:04.530453400 reducer.cpp:1400] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
-Train Summary | End of Epoch 1 | Time 8862.50s | Train Loss -15.642
-Valid Summary | End of Epoch 1 | Time 1299.28s | Valid Loss -14.021
-Test Summary | End of Epoch 1 | Time 779.08s | Test Loss -13.495
-Fund new best model, dict saved
-Train Summary | End of Epoch 2 | Time 8868.78s | Train Loss -15.693
-Valid Summary | End of Epoch 2 | Time 1297.45s | Valid Loss -13.954
-Test Summary | End of Epoch 2 | Time 778.92s | Test Loss -13.553
-Train Summary | End of Epoch 3 | Time 8857.29s | Train Loss -15.673
-Valid Summary | End of Epoch 3 | Time 1297.92s | Valid Loss -13.922
-Test Summary | End of Epoch 3 | Time 778.84s | Test Loss -13.479
-Train Summary | End of Epoch 4 | Time 8841.24s | Train Loss -15.672
-Valid Summary | End of Epoch 4 | Time 1297.81s | Valid Loss -14.050
-Test Summary | End of Epoch 4 | Time 778.69s | Test Loss -13.570
-Fund new best model, dict saved
-Train Summary | End of Epoch 5 | Time 8842.92s | Train Loss -15.658
-Valid Summary | End of Epoch 5 | Time 1297.78s | Valid Loss -13.980
-Test Summary | End of Epoch 5 | Time 778.76s | Test Loss -13.672
-Train Summary | End of Epoch 6 | Time 8863.22s | Train Loss -15.625
-Valid Summary | End of Epoch 6 | Time 1297.63s | Valid Loss -14.042
-Test Summary | End of Epoch 6 | Time 778.93s | Test Loss -13.626
-Train Summary | End of Epoch 7 | Time 8874.34s | Train Loss -15.628
-Valid Summary | End of Epoch 7 | Time 1297.79s | Valid Loss -14.223
-Test Summary | End of Epoch 7 | Time 778.60s | Test Loss -13.703
-Fund new best model, dict saved
-Train Summary | End of Epoch 8 | Time 8878.31s | Train Loss -15.633
-Valid Summary | End of Epoch 8 | Time 1297.68s | Valid Loss -14.197
-Test Summary | End of Epoch 8 | Time 778.55s | Test Loss -13.741
-Train Summary | End of Epoch 9 | Time 8879.85s | Train Loss -15.644
-Valid Summary | End of Epoch 9 | Time 1297.92s | Valid Loss -14.241
-Test Summary | End of Epoch 9 | Time 778.76s | Test Loss -13.799
-Fund new best model, dict saved
-Train Summary | End of Epoch 10 | Time 8885.21s | Train Loss -15.669
-Valid Summary | End of Epoch 10 | Time 1297.27s | Valid Loss -14.229
-Test Summary | End of Epoch 10 | Time 778.51s | Test Loss -13.791
-Train Summary | End of Epoch 11 | Time 8884.38s | Train Loss -15.687
-Valid Summary | End of Epoch 11 | Time 1297.56s | Valid Loss -14.257
-Test Summary | End of Epoch 11 | Time 778.75s | Test Loss -13.766
-Fund new best model, dict saved
-Train Summary | End of Epoch 12 | Time 8861.05s | Train Loss -15.689
-Valid Summary | End of Epoch 12 | Time 1297.83s | Valid Loss -14.225
-Test Summary | End of Epoch 12 | Time 778.37s | Test Loss -13.740
-Train Summary | End of Epoch 13 | Time 8854.73s | Train Loss -15.715
-Valid Summary | End of Epoch 13 | Time 1297.07s | Valid Loss -14.338
-Test Summary | End of Epoch 13 | Time 778.98s | Test Loss -13.852
-Fund new best model, dict saved
-Train Summary | End of Epoch 14 | Time 8861.60s | Train Loss -15.720
-Valid Summary | End of Epoch 14 | Time 1297.61s | Valid Loss -14.320
-Test Summary | End of Epoch 14 | Time 778.73s | Test Loss -13.842
-Train Summary | End of Epoch 15 | Time 8867.45s | Train Loss -15.740
-Valid Summary | End of Epoch 15 | Time 1297.15s | Valid Loss -14.264
-Test Summary | End of Epoch 15 | Time 778.45s | Test Loss -13.831
-Train Summary | End of Epoch 16 | Time 8871.73s | Train Loss -15.750
-Valid Summary | End of Epoch 16 | Time 1297.58s | Valid Loss -14.301
-Test Summary | End of Epoch 16 | Time 778.59s | Test Loss -13.901
-Train Summary | End of Epoch 17 | Time 8874.12s | Train Loss -15.739
-Valid Summary | End of Epoch 17 | Time 1297.21s | Valid Loss -14.338
-Test Summary | End of Epoch 17 | Time 778.18s | Test Loss -13.897
-Fund new best model, dict saved
-Train Summary | End of Epoch 18 | Time 8866.66s | Train Loss -15.756
-Valid Summary | End of Epoch 18 | Time 1297.23s | Valid Loss -14.248
-Test Summary | End of Epoch 18 | Time 778.35s | Test Loss -13.823
-Train Summary | End of Epoch 19 | Time 8866.16s | Train Loss -15.784
-Valid Summary | End of Epoch 19 | Time 1296.46s | Valid Loss -14.318
-Test Summary | End of Epoch 19 | Time 778.56s | Test Loss -13.898
-Train Summary | End of Epoch 20 | Time 8861.71s | Train Loss -15.780
-Valid Summary | End of Epoch 20 | Time 1297.90s | Valid Loss -14.337
-Test Summary | End of Epoch 20 | Time 778.23s | Test Loss -13.892
-Train Summary | End of Epoch 21 | Time 8861.37s | Train Loss -15.811
-Valid Summary | End of Epoch 21 | Time 1296.68s | Valid Loss -14.383
-Test Summary | End of Epoch 21 | Time 778.57s | Test Loss -13.913
-Fund new best model, dict saved
-Train Summary | End of Epoch 22 | Time 8868.58s | Train Loss -15.815
-Valid Summary | End of Epoch 22 | Time 1297.38s | Valid Loss -14.341
-Test Summary | End of Epoch 22 | Time 778.10s | Test Loss -13.883
-Train Summary | End of Epoch 23 | Time 8876.67s | Train Loss -15.816
-Valid Summary | End of Epoch 23 | Time 1297.54s | Valid Loss -14.315
-Test Summary | End of Epoch 23 | Time 778.88s | Test Loss -13.974
-Train Summary | End of Epoch 24 | Time 8873.42s | Train Loss -15.821
-Valid Summary | End of Epoch 24 | Time 1297.77s | Valid Loss -14.415
-Test Summary | End of Epoch 24 | Time 778.40s | Test Loss -13.877
-Fund new best model, dict saved
-Train Summary | End of Epoch 25 | Time 8853.35s | Train Loss -15.834
-Valid Summary | End of Epoch 25 | Time 1297.67s | Valid Loss -14.356
-Test Summary | End of Epoch 25 | Time 778.28s | Test Loss -13.917
-Train Summary | End of Epoch 26 | Time 8879.37s | Train Loss -15.844
-Valid Summary | End of Epoch 26 | Time 1297.45s | Valid Loss -14.384
-Test Summary | End of Epoch 26 | Time 778.64s | Test Loss -13.973
-Train Summary | End of Epoch 27 | Time 8872.84s | Train Loss -15.854
-Valid Summary | End of Epoch 27 | Time 1297.92s | Valid Loss -14.402
-Test Summary | End of Epoch 27 | Time 779.00s | Test Loss -14.017
-Train Summary | End of Epoch 28 | Time 8875.05s | Train Loss -15.856
-Valid Summary | End of Epoch 28 | Time 1297.66s | Valid Loss -14.351
-Test Summary | End of Epoch 28 | Time 778.75s | Test Loss -13.939
-Train Summary | End of Epoch 29 | Time 8869.04s | Train Loss -15.884
-Valid Summary | End of Epoch 29 | Time 1297.87s | Valid Loss -14.430
-Test Summary | End of Epoch 29 | Time 778.77s | Test Loss -13.954
-Fund new best model, dict saved
-Train Summary | End of Epoch 30 | Time 8864.22s | Train Loss -15.890
-Valid Summary | End of Epoch 30 | Time 1297.79s | Valid Loss -14.398
-Test Summary | End of Epoch 30 | Time 778.88s | Test Loss -14.004
-Train Summary | End of Epoch 31 | Time 8862.20s | Train Loss -15.906
-Valid Summary | End of Epoch 31 | Time 1297.63s | Valid Loss -14.396
-Test Summary | End of Epoch 31 | Time 778.78s | Test Loss -13.985
-Train Summary | End of Epoch 32 | Time 8872.91s | Train Loss -15.911
-Valid Summary | End of Epoch 32 | Time 1297.77s | Valid Loss -14.425
-Test Summary | End of Epoch 32 | Time 778.33s | Test Loss -13.933
-Train Summary | End of Epoch 33 | Time 8867.16s | Train Loss -15.911
-Valid Summary | End of Epoch 33 | Time 1297.78s | Valid Loss -14.423
-Test Summary | End of Epoch 33 | Time 778.53s | Test Loss -13.952
-Train Summary | End of Epoch 34 | Time 8863.09s | Train Loss -15.916
-Valid Summary | End of Epoch 34 | Time 1297.82s | Valid Loss -14.470
-Test Summary | End of Epoch 34 | Time 778.80s | Test Loss -13.885
-Fund new best model, dict saved
-Train Summary | End of Epoch 35 | Time 8865.77s | Train Loss -15.936
-Valid Summary | End of Epoch 35 | Time 1297.80s | Valid Loss -14.486
-Test Summary | End of Epoch 35 | Time 778.59s | Test Loss -13.990
-Fund new best model, dict saved
-Train Summary | End of Epoch 36 | Time 8864.53s | Train Loss -15.931
-Valid Summary | End of Epoch 36 | Time 1297.82s | Valid Loss -14.413
-Test Summary | End of Epoch 36 | Time 778.79s | Test Loss -13.959
-Train Summary | End of Epoch 37 | Time 8860.11s | Train Loss -15.934
-Valid Summary | End of Epoch 37 | Time 1298.28s | Valid Loss -14.503
-Test Summary | End of Epoch 37 | Time 778.33s | Test Loss -13.980
-Fund new best model, dict saved
-Train Summary | End of Epoch 38 | Time 8855.04s | Train Loss -15.957
-Valid Summary | End of Epoch 38 | Time 1298.32s | Valid Loss -14.387
-Test Summary | End of Epoch 38 | Time 778.98s | Test Loss -13.980
-Train Summary | End of Epoch 39 | Time 8862.85s | Train Loss -15.964
-Valid Summary | End of Epoch 39 | Time 1297.98s | Valid Loss -14.462
-Test Summary | End of Epoch 39 | Time 779.50s | Test Loss -14.014
-Train Summary | End of Epoch 40 | Time 8861.48s | Train Loss -15.956
-Valid Summary | End of Epoch 40 | Time 1297.55s | Valid Loss -14.344
-Test Summary | End of Epoch 40 | Time 778.53s | Test Loss -13.998
-Train Summary | End of Epoch 41 | Time 8861.38s | Train Loss -15.955
-Valid Summary | End of Epoch 41 | Time 1297.55s | Valid Loss -14.369
-Test Summary | End of Epoch 41 | Time 779.16s | Test Loss -13.946
-Train Summary | End of Epoch 42 | Time 8853.68s | Train Loss -15.969
-Valid Summary | End of Epoch 42 | Time 1297.52s | Valid Loss -14.472
-Test Summary | End of Epoch 42 | Time 778.78s | Test Loss -14.000
-reload weights and optimizer from last best checkpoint
-Learning rate adjusted to: 0.000255
-Train Summary | End of Epoch 43 | Time 8855.22s | Train Loss -16.093
-Valid Summary | End of Epoch 43 | Time 1297.53s | Valid Loss -14.584
-Test Summary | End of Epoch 43 | Time 778.81s | Test Loss -14.144
-Fund new best model, dict saved
-Train Summary | End of Epoch 44 | Time 8839.27s | Train Loss -16.155
-Valid Summary | End of Epoch 44 | Time 1297.71s | Valid Loss -14.520
-Test Summary | End of Epoch 44 | Time 778.86s | Test Loss -14.054
-Train Summary | End of Epoch 45 | Time 8840.93s | Train Loss -16.158
-Valid Summary | End of Epoch 45 | Time 1298.07s | Valid Loss -14.566
-Test Summary | End of Epoch 45 | Time 778.17s | Test Loss -14.039
-Train Summary | End of Epoch 46 | Time 8846.57s | Train Loss -16.194
-Valid Summary | End of Epoch 46 | Time 1297.09s | Valid Loss -14.602
-Test Summary | End of Epoch 46 | Time 779.00s | Test Loss -14.058
-Fund new best model, dict saved
-Train Summary | End of Epoch 47 | Time 8848.27s | Train Loss -16.203
-Valid Summary | End of Epoch 47 | Time 1297.44s | Valid Loss -14.561
-Test Summary | End of Epoch 47 | Time 778.99s | Test Loss -14.038
-Train Summary | End of Epoch 48 | Time 8855.13s | Train Loss -16.208
-Valid Summary | End of Epoch 48 | Time 1298.17s | Valid Loss -14.567
-Test Summary | End of Epoch 48 | Time 779.06s | Test Loss -14.059
-Train Summary | End of Epoch 49 | Time 8860.44s | Train Loss -16.230
-Valid Summary | End of Epoch 49 | Time 1297.85s | Valid Loss -14.600
-Test Summary | End of Epoch 49 | Time 778.57s | Test Loss -14.109
-Train Summary | End of Epoch 50 | Time 8849.69s | Train Loss -16.234
-Valid Summary | End of Epoch 50 | Time 1297.78s | Valid Loss -14.520
-Test Summary | End of Epoch 50 | Time 778.36s | Test Loss -14.044
-Train Summary | End of Epoch 51 | Time 8845.04s | Train Loss -16.262
-Valid Summary | End of Epoch 51 | Time 1296.91s | Valid Loss -14.605
-Test Summary | End of Epoch 51 | Time 778.81s | Test Loss -14.112
-Fund new best model, dict saved
-Train Summary | End of Epoch 52 | Time 8846.18s | Train Loss -16.262
-Valid Summary | End of Epoch 52 | Time 1297.45s | Valid Loss -14.570
-Test Summary | End of Epoch 52 | Time 778.92s | Test Loss -14.142
-Train Summary | End of Epoch 53 | Time 8848.28s | Train Loss -16.273
-Valid Summary | End of Epoch 53 | Time 1297.74s | Valid Loss -14.576
-Test Summary | End of Epoch 53 | Time 778.71s | Test Loss -14.089
-Train Summary | End of Epoch 54 | Time 8827.56s | Train Loss -16.303
-Valid Summary | End of Epoch 54 | Time 1298.30s | Valid Loss -14.529
-Test Summary | End of Epoch 54 | Time 779.41s | Test Loss -14.063
-Train Summary | End of Epoch 55 | Time 8853.03s | Train Loss -16.310
-Valid Summary | End of Epoch 55 | Time 1297.30s | Valid Loss -14.512
-Test Summary | End of Epoch 55 | Time 778.55s | Test Loss -14.077

checkpoints/log_VoxCeleb2_tfgridnet-isam_2spk/log_2024-12-31(09:48:03).txt

@@ -1,632 +0,0 @@
-## Config file
-
-# Log 
-seed: 777
-use_cuda: 1           # 1 for True, 0 for False
-
-# dataset
-speaker_no: 2
-mix_lst_path: ./data/VoxCeleb2_non_repeat/mixture_data_list_2mix.csv
-audio_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/audio_clean/
-reference_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/orig/
-audio_sr: 16000
-ref_sr: 25
-
-# dataloader
-num_workers: 2
-batch_size: 1         # 4-GPU training with a total effective batch size of 8
-accu_grad: 1
-effec_batch_size: 2   # per GPU, only used if accu_grad is set to 1, must be multiple times of batch size
-max_length: 3         # truncate the utterances in dataloader, in seconds 
-
-# network settings
-init_from: 'checkpoints/log_VoxCeleb2_lip_tfgridnet_2spk'       # 'None' or a log name 'log_2024-07-22(18:12:13)'
-causal: 0             # 1 for True, 0 for False
-network_reference:
-  cue: lip            # lip or speech or gesture or EEG
-  backbone: resnet18  # resnet18 or shufflenetV2 or blazenet64
-  emb_size: 256       # resnet18:256
-network_audio:
-  backbone: av_tfgridnet_att_ss
-  n_fft: 256
-  stride: 128
-  window: "hann"
-  use_builtin_complex: False
-  n_srcs:  1
-  n_imics: 1
-  n_layers: 6
-  lstm_hidden_units: 192
-  attn_n_head: 4
-  attn_qk_output_channel: 4
-  emb_dim: 48
-  emb_ks: 4
-  emb_hs: 1
-  activation: "prelu"
-
-# optimizer
-spk_att_dropout: 1    # 0 for always use speaker attention
-loss_type: ss_sisdr      # "snr", "sisdr", "hybrid"
-lr_warmup: 1
-init_learning_rate: 0.0005
-max_epoch: 150
-clip_grad_norm: 5
-W1231 09:48:39.901772 139973816584000 torch/distributed/run.py:779] 
-W1231 09:48:39.901772 139973816584000 torch/distributed/run.py:779] *****************************************
-W1231 09:48:39.901772 139973816584000 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
-W1231 09:48:39.901772 139973816584000 torch/distributed/run.py:779] *****************************************
-[W1231 09:49:05.069551764 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
-[W1231 09:49:05.069566273 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
-[W1231 09:49:05.070374617 Utils.hpp:135] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
-[W1231 09:49:05.070402090 Utils.hpp:135] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
-[W1231 09:49:05.069566736 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
-[W1231 09:49:05.070424006 Utils.hpp:135] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
-[W1231 09:49:05.069610510 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
-[W1231 09:49:05.070456802 Utils.hpp:135] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
-started on checkpoints/log_2024-12-23(15:50:05)
-
-namespace(accu_grad=1, audio_direc='/mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/audio_clean/', audio_sr=16000, batch_size=1, causal=0, checkpoint_dir='checkpoints/log_2024-12-23(15:50:05)', clip_grad_norm=5.0, config=[<yamlargparse.Path object at 0x7f87241d2c40>], device=device(type='cuda'), distributed=True, effec_batch_size=2, evaluate_only=0, init_from='checkpoints/log_VoxCeleb2_lip_tfgridnet_2spk', init_learning_rate=0.0005, local_rank=0, loss_type='ss_sisdr', lr_warmup=1, max_epoch=150, max_length=3, mix_lst_path='./data/VoxCeleb2_non_repeat/mixture_data_list_2mix.csv', network_audio=namespace(activation='prelu', attn_n_head=4, attn_qk_output_channel=4, backbone='av_tfgridnet_att_ss', emb_dim=48, emb_hs=1, emb_ks=4, lstm_hidden_units=192, n_fft=256, n_imics=1, n_layers=6, n_srcs=1, stride=128, use_builtin_complex=False, window='hann'), network_reference=namespace(backbone='resnet18', cue='lip', emb_size=256), num_workers=2, ref_sr=25, reference_direc='/mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/orig/', seed=777, speaker_no=2, spk_att_dropout=1, train_from_last_checkpoint=1, use_cuda=1, world_size=4)
-network_wrapper(
-  (sep_network): av_TFGridNetV3_att_ss(
-    (enc): STFTEncoder(
-      (stft): Stft(n_fft=256, win_length=256, hop_length=128, center=True, normalized=False, onesided=True)
-    )
-    (dec): STFTDecoder(
-      (stft): Stft(n_fft=256, win_length=256, hop_length=128, center=True, normalized=False, onesided=True)
-    )
-    (conv): Sequential(
-      (0): Conv2d(2, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
-      (1): GroupNorm(1, 48, eps=1e-05, affine=True)
-    )
-    (blocks): ModuleList(
-      (0-5): 6 x GridNetV3Block(
-        (intra_norm): LayerNorm((48,), eps=1e-05, elementwise_affine=True)
-        (intra_rnn): LSTM(192, 192, batch_first=True, bidirectional=True)
-        (intra_linear): ConvTranspose1d(384, 48, kernel_size=(4,), stride=(1,))
-        (inter_norm): LayerNorm((48,), eps=1e-05, elementwise_affine=True)
-        (inter_rnn): LSTM(192, 192, batch_first=True, bidirectional=True)
-        (inter_linear): ConvTranspose1d(384, 48, kernel_size=(4,), stride=(1,))
-        (attn_conv_Q): Conv2d(48, 16, kernel_size=(1, 1), stride=(1, 1))
-        (attn_norm_Q): AllHeadPReLULayerNormalization4DC(
-          (act): PReLU(num_parameters=4)
-        )
-        (attn_conv_K): Conv2d(48, 16, kernel_size=(1, 1), stride=(1, 1))
-        (attn_norm_K): AllHeadPReLULayerNormalization4DC(
-          (act): PReLU(num_parameters=4)
-        )
-        (attn_conv_V): Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1))
-        (attn_norm_V): AllHeadPReLULayerNormalization4DC(
-          (act): PReLU(num_parameters=4)
-        )
-        (attn_concat_proj): Sequential(
-          (0): Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1))
-          (1): PReLU(num_parameters=1)
-          (2): LayerNormalization()
-        )
-        (spk_att): TransformerEncoder(
-          (layers): ModuleList(
-            (0): TransformerEncoderLayer(
-              (self_attn): MultiheadAttention(
-                (out_proj): NonDynamicallyQuantizableLinear(in_features=48, out_features=48, bias=True)
-              )
-              (linear1): Linear(in_features=48, out_features=192, bias=True)
-              (dropout): Dropout(p=0.1, inplace=False)
-              (linear2): Linear(in_features=192, out_features=48, bias=True)
-              (norm1): LayerNorm((48,), eps=1e-05, elementwise_affine=True)
-              (norm2): LayerNorm((48,), eps=1e-05, elementwise_affine=True)
-              (dropout1): Dropout(p=0.1, inplace=False)
-              (dropout2): Dropout(p=0.1, inplace=False)
-            )
-          )
-        )
-        (spk_norm): GroupNorm(1, 48, eps=1e-08, affine=True)
-      )
-    )
-    (deconv): ConvTranspose2d(48, 2, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
-    (av_conv): ModuleList(
-      (0-5): 6 x Linear(in_features=304, out_features=48, bias=True)
-    )
-  )
-  (ref_encoder): Visual_encoder(
-    (v_frontend): VisualFrontend(
-      (frontend3D): Sequential(
-        (0): Conv3d(1, 64, kernel_size=(5, 7, 7), stride=(1, 2, 2), padding=(2, 3, 3), bias=False)
-        (1): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-        (2): ReLU()
-        (3): MaxPool3d(kernel_size=(1, 3, 3), stride=(1, 2, 2), padding=(0, 1, 1), dilation=1, ceil_mode=False)
-      )
-      (resnet): ResNet(
-        (layer1): ResNetLayer(
-          (conv1a): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (bn1a): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-          (conv2a): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (downsample): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
-          (outbna): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-          (conv1b): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (bn1b): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-          (conv2b): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (outbnb): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-        )
-        (layer2): ResNetLayer(
-          (conv1a): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
-          (bn1a): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-          (conv2a): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (downsample): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)
-          (outbna): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-          (conv1b): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (bn1b): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-          (conv2b): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (outbnb): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-        )
-        (layer3): ResNetLayer(
-          (conv1a): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
-          (bn1a): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-          (conv2a): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (downsample): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)
-          (outbna): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-          (conv1b): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (bn1b): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-          (conv2b): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (outbnb): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-        )
-        (layer4): ResNetLayer(
-          (conv1a): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
-          (bn1a): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-          (conv2a): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (downsample): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)
-          (outbna): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-          (conv1b): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (bn1b): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-          (conv2b): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-          (outbnb): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
-        )
-        (avgpool): AvgPool2d(kernel_size=(4, 4), stride=(1, 1), padding=0)
-      )
-    )
-    (v_ds): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
-    (visual_conv): Sequential(
-      (0): VisualConv1D(
-        (relu_0): ReLU()
-        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
-        (relu): ReLU()
-        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
-        (prelu): PReLU(num_parameters=1)
-        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
-      )
-      (1): VisualConv1D(
-        (relu_0): ReLU()
-        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
-        (relu): ReLU()
-        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
-        (prelu): PReLU(num_parameters=1)
-        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
-      )
-      (2): VisualConv1D(
-        (relu_0): ReLU()
-        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
-        (relu): ReLU()
-        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
-        (prelu): PReLU(num_parameters=1)
-        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
-      )
-      (3): VisualConv1D(
-        (relu_0): ReLU()
-        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
-        (relu): ReLU()
-        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
-        (prelu): PReLU(num_parameters=1)
-        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
-      )
-      (4): VisualConv1D(
-        (relu_0): ReLU()
-        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
-        (relu): ReLU()
-        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
-        (prelu): PReLU(num_parameters=1)
-        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
-        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
-      )
-    )
-  )
-)
-
-Total number of parameters: 20950309 
-
-
-Total number of trainable parameters: 9765221 
-
-dlc10xm9l399lwkq-master-0:26:26 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
-dlc10xm9l399lwkq-master-0:26:26 [0] NCCL INFO Bootstrap : Using eth0:22.6.236.79<0>
-dlc10xm9l399lwkq-master-0:26:26 [0] NCCL INFO Plugin name set by env to libnccl-net-none.so
-dlc10xm9l399lwkq-master-0:26:26 [0] NCCL INFO NET/Plugin : dlerror=libnccl-net-none.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net-none.so), using internal implementation
-dlc10xm9l399lwkq-master-0:26:26 [0] NCCL INFO cudaDriverVersion 11040
-dlc10xm9l399lwkq-master-0:29:29 [3] NCCL INFO cudaDriverVersion 11040
-dlc10xm9l399lwkq-master-0:27:27 [1] NCCL INFO cudaDriverVersion 11040
-dlc10xm9l399lwkq-master-0:28:28 [2] NCCL INFO cudaDriverVersion 11040
-NCCL version 2.20.5+cuda11.8
-dlc10xm9l399lwkq-master-0:29:29 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
-dlc10xm9l399lwkq-master-0:27:27 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
-dlc10xm9l399lwkq-master-0:28:28 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
-dlc10xm9l399lwkq-master-0:29:29 [3] NCCL INFO Bootstrap : Using eth0:22.6.236.79<0>
-dlc10xm9l399lwkq-master-0:28:28 [2] NCCL INFO Bootstrap : Using eth0:22.6.236.79<0>
-dlc10xm9l399lwkq-master-0:27:27 [1] NCCL INFO Bootstrap : Using eth0:22.6.236.79<0>
-dlc10xm9l399lwkq-master-0:29:29 [3] NCCL INFO Plugin name set by env to libnccl-net-none.so
-dlc10xm9l399lwkq-master-0:28:28 [2] NCCL INFO Plugin name set by env to libnccl-net-none.so
-dlc10xm9l399lwkq-master-0:27:27 [1] NCCL INFO Plugin name set by env to libnccl-net-none.so
-dlc10xm9l399lwkq-master-0:29:29 [3] NCCL INFO NET/Plugin : dlerror=libnccl-net-none.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net-none.so), using internal implementation
-dlc10xm9l399lwkq-master-0:28:28 [2] NCCL INFO NET/Plugin : dlerror=libnccl-net-none.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net-none.so), using internal implementation
-dlc10xm9l399lwkq-master-0:27:27 [1] NCCL INFO NET/Plugin : dlerror=libnccl-net-none.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net-none.so), using internal implementation
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO NCCL_IB_HCA set to mlx5
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO NCCL_IB_HCA set to mlx5
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO NCCL_IB_HCA set to mlx5
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO NCCL_IB_HCA set to mlx5
-libibverbs: Warning: couldn't load driver 'libhfi1verbs-rdmav25.so': libhfi1verbs-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libhfi1verbs-rdmav25.so': libhfi1verbs-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libhfi1verbs-rdmav25.so': libhfi1verbs-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libhfi1verbs-rdmav25.so': libhfi1verbs-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'librxe-rdmav25.so': librxe-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'librxe-rdmav25.so': librxe-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'librxe-rdmav25.so': librxe-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'librxe-rdmav25.so': librxe-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libmthca-rdmav25.so': libmthca-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libmthca-rdmav25.so': libmthca-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libmthca-rdmav25.so': libmthca-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libmthca-rdmav25.so': libmthca-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libvmw_pvrdma-rdmav25.so': libvmw_pvrdma-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libvmw_pvrdma-rdmav25.so': libvmw_pvrdma-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libvmw_pvrdma-rdmav25.so': libvmw_pvrdma-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libvmw_pvrdma-rdmav25.so': libvmw_pvrdma-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libhns-rdmav25.so': libhns-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libhns-rdmav25.so': libhns-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libhns-rdmav25.so': libhns-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libhns-rdmav25.so': libhns-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libipathverbs-rdmav25.so': libipathverbs-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libipathverbs-rdmav25.so': libipathverbs-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libipathverbs-rdmav25.so': libipathverbs-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libipathverbs-rdmav25.so': libipathverbs-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libsiw-rdmav25.so': libsiw-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libsiw-rdmav25.so': libsiw-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libsiw-rdmav25.so': libsiw-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libsiw-rdmav25.so': libsiw-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libbnxt_re-rdmav25.so': libbnxt_re-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libbnxt_re-rdmav25.so': libbnxt_re-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libbnxt_re-rdmav25.so': libbnxt_re-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libbnxt_re-rdmav25.so': libbnxt_re-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libocrdma-rdmav25.so': libocrdma-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libocrdma-rdmav25.so': libocrdma-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libocrdma-rdmav25.so': libocrdma-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libocrdma-rdmav25.so': libocrdma-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libmlx4-rdmav25.so': libmlx4-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libmlx4-rdmav25.so': libmlx4-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libmlx4-rdmav25.so': libmlx4-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libmlx4-rdmav25.so': libmlx4-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libqedr-rdmav25.so': libqedr-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libqedr-rdmav25.so': libqedr-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libqedr-rdmav25.so': libqedr-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libqedr-rdmav25.so': libqedr-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libcxgb4-rdmav25.so': libcxgb4-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libcxgb4-rdmav25.so': libcxgb4-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libcxgb4-rdmav25.so': libcxgb4-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libcxgb4-rdmav25.so': libcxgb4-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libi40iw-rdmav25.so': libi40iw-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libi40iw-rdmav25.so': libi40iw-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libi40iw-rdmav25.so': libi40iw-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libi40iw-rdmav25.so': libi40iw-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libefa-rdmav25.so': libefa-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libefa-rdmav25.so': libefa-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libefa-rdmav25.so': libefa-rdmav25.so: cannot open shared object file: No such file or directory
-libibverbs: Warning: couldn't load driver 'libefa-rdmav25.so': libefa-rdmav25.so: cannot open shared object file: No such file or directory
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [RO]; OOB eth0:22.6.236.79<0>
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Using non-device net plugin version 0
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Using network IB
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [RO]; OOB eth0:22.6.236.79<0>
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [RO]; OOB eth0:22.6.236.79<0>
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Using non-device net plugin version 0
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Using network IB
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Using non-device net plugin version 0
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Using network IB
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [RO]; OOB eth0:22.6.236.79<0>
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Using non-device net plugin version 0
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Using network IB
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO comm 0x8e64ad0 rank 1 nranks 4 cudaDev 1 nvmlDev 1 busId 20 commId 0xf279881ae65b16f2 - Init START
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO comm 0x933e770 rank 0 nranks 4 cudaDev 0 nvmlDev 0 busId 10 commId 0xf279881ae65b16f2 - Init START
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO comm 0x792e140 rank 3 nranks 4 cudaDev 3 nvmlDev 3 busId 40 commId 0xf279881ae65b16f2 - Init START
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO comm 0x754a850 rank 2 nranks 4 cudaDev 2 nvmlDev 2 busId 30 commId 0xf279881ae65b16f2 - Init START
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Setting affinity for GPU 0 to ffffff
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Setting affinity for GPU 2 to ffffff
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Setting affinity for GPU 1 to ffffff
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO comm 0x933e770 rank 0 nRanks 4 nNodes 1 localRanks 4 localRank 0 MNNVL 0
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO comm 0x8e64ad0 rank 1 nRanks 4 nNodes 1 localRanks 4 localRank 1 MNNVL 0
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO comm 0x792e140 rank 3 nRanks 4 nNodes 1 localRanks 4 localRank 3 MNNVL 0
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO comm 0x754a850 rank 2 nRanks 4 nNodes 1 localRanks 4 localRank 2 MNNVL 0
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Channel 00/12 :    0   1   2   3
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 [2] 2/-1/-1->1->0 [3] 2/-1/-1->1->0 [4] 2/-1/-1->1->0 [5] 2/-1/-1->1->0 [6] 2/-1/-1->1->0 [7] 2/-1/-1->1->0 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->0 [10] 2/-1/-1->1->0 [11] 2/-1/-1->1->0
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Channel 01/12 :    0   1   2   3
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO P2P Chunksize set to 524288
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Trees [0] -1/-1/-1->3->2 [1] -1/-1/-1->3->2 [2] -1/-1/-1->3->2 [3] -1/-1/-1->3->2 [4] -1/-1/-1->3->2 [5] -1/-1/-1->3->2 [6] -1/-1/-1->3->2 [7] -1/-1/-1->3->2 [8] -1/-1/-1->3->2 [9] -1/-1/-1->3->2 [10] -1/-1/-1->3->2 [11] -1/-1/-1->3->2
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Channel 02/12 :    0   1   2   3
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 3/-1/-1->2->1 [4] 3/-1/-1->2->1 [5] 3/-1/-1->2->1 [6] 3/-1/-1->2->1 [7] 3/-1/-1->2->1 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 3/-1/-1->2->1 [11] 3/-1/-1->2->1
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO P2P Chunksize set to 524288
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO P2P Chunksize set to 524288
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Channel 03/12 :    0   1   2   3
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Channel 04/12 :    0   1   2   3
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Channel 05/12 :    0   1   2   3
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Channel 06/12 :    0   1   2   3
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Channel 07/12 :    0   1   2   3
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Channel 08/12 :    0   1   2   3
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Channel 09/12 :    0   1   2   3
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Channel 10/12 :    0   1   2   3
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Channel 11/12 :    0   1   2   3
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] 1/-1/-1->0->-1 [5] 1/-1/-1->0->-1 [6] 1/-1/-1->0->-1 [7] 1/-1/-1->0->-1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO P2P Chunksize set to 524288
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Channel 00/0 : 3[3] -> 0[0] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Channel 01/0 : 3[3] -> 0[0] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Channel 02/0 : 1[1] -> 2[2] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Channel 02/0 : 3[3] -> 0[0] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Channel 02/0 : 2[2] -> 3[3] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Channel 03/0 : 3[3] -> 0[0] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Channel 04/0 : 1[1] -> 2[2] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Channel 03/0 : 2[2] -> 3[3] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Channel 04/0 : 3[3] -> 0[0] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Channel 05/0 : 1[1] -> 2[2] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Channel 04/0 : 2[2] -> 3[3] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Channel 05/0 : 3[3] -> 0[0] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Channel 06/0 : 1[1] -> 2[2] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Channel 05/0 : 2[2] -> 3[3] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Channel 06/0 : 3[3] -> 0[0] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Channel 06/0 : 2[2] -> 3[3] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Channel 07/0 : 3[3] -> 0[0] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Channel 08/0 : 1[1] -> 2[2] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Channel 08/0 : 3[3] -> 0[0] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Channel 08/0 : 2[2] -> 3[3] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Channel 09/0 : 3[3] -> 0[0] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Channel 10/0 : 1[1] -> 2[2] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Channel 10/0 : 3[3] -> 0[0] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Channel 10/0 : 2[2] -> 3[3] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Channel 11/0 : 3[3] -> 0[0] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Channel 11/0 : 2[2] -> 3[3] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Connected all rings
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Connected all rings
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Channel 00/0 : 3[3] -> 2[2] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Connected all rings
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Connected all rings
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Channel 01/0 : 3[3] -> 2[2] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Channel 02/0 : 3[3] -> 2[2] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Channel 03/0 : 3[3] -> 2[2] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Channel 04/0 : 3[3] -> 2[2] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Channel 05/0 : 3[3] -> 2[2] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Channel 06/0 : 3[3] -> 2[2] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Channel 07/0 : 3[3] -> 2[2] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Channel 08/0 : 3[3] -> 2[2] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Channel 09/0 : 3[3] -> 2[2] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Channel 10/0 : 3[3] -> 2[2] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Channel 11/0 : 3[3] -> 2[2] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Channel 00/0 : 1[1] -> 0[0] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Channel 00/0 : 2[2] -> 1[1] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Channel 01/0 : 1[1] -> 0[0] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Channel 01/0 : 2[2] -> 1[1] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Channel 02/0 : 1[1] -> 0[0] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Channel 02/0 : 2[2] -> 1[1] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Channel 03/0 : 1[1] -> 0[0] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Channel 03/0 : 2[2] -> 1[1] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Channel 04/0 : 1[1] -> 0[0] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Channel 04/0 : 2[2] -> 1[1] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Channel 05/0 : 1[1] -> 0[0] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Channel 05/0 : 2[2] -> 1[1] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Channel 06/0 : 1[1] -> 0[0] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Channel 06/0 : 2[2] -> 1[1] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Channel 07/0 : 1[1] -> 0[0] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Channel 07/0 : 2[2] -> 1[1] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Channel 08/0 : 1[1] -> 0[0] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Channel 08/0 : 2[2] -> 1[1] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Channel 09/0 : 1[1] -> 0[0] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Channel 09/0 : 2[2] -> 1[1] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Channel 10/0 : 1[1] -> 0[0] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Channel 10/0 : 2[2] -> 1[1] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Channel 11/0 : 1[1] -> 0[0] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Channel 11/0 : 2[2] -> 1[1] via P2P/IPC/read
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO Connected all trees
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO 12 coll channels, 0 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO Connected all trees
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO 12 coll channels, 0 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO Connected all trees
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO Connected all trees
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO 12 coll channels, 0 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO 12 coll channels, 0 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer
-dlc10xm9l399lwkq-master-0:27:57 [1] NCCL INFO comm 0x8e64ad0 rank 1 nranks 4 cudaDev 1 nvmlDev 1 busId 20 commId 0xf279881ae65b16f2 - Init COMPLETE
-dlc10xm9l399lwkq-master-0:26:56 [0] NCCL INFO comm 0x933e770 rank 0 nranks 4 cudaDev 0 nvmlDev 0 busId 10 commId 0xf279881ae65b16f2 - Init COMPLETE
-dlc10xm9l399lwkq-master-0:28:59 [2] NCCL INFO comm 0x754a850 rank 2 nranks 4 cudaDev 2 nvmlDev 2 busId 30 commId 0xf279881ae65b16f2 - Init COMPLETE
-dlc10xm9l399lwkq-master-0:29:58 [3] NCCL INFO comm 0x792e140 rank 3 nranks 4 cudaDev 3 nvmlDev 3 busId 40 commId 0xf279881ae65b16f2 - Init COMPLETE
-[rank0]:[W1231 09:49:44.279868977 Utils.hpp:110] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
-[rank2]:[W1231 09:49:44.279870560 Utils.hpp:110] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
-[rank3]:[W1231 09:49:44.280285689 Utils.hpp:110] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
-[rank1]:[W1231 09:49:44.280413742 Utils.hpp:110] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
-Resume training from epoch: 56
-[rank1]:[W1231 09:51:26.949995860 reducer.cpp:1400] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
-[rank2]:[W1231 09:51:26.954996096 reducer.cpp:1400] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
-[rank0]:[W1231 09:51:26.955118250 reducer.cpp:1400] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
-[rank3]:[W1231 09:51:26.957005274 reducer.cpp:1400] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
-Train Summary | End of Epoch 56 | Time 8839.06s | Train Loss -16.312
-Valid Summary | End of Epoch 56 | Time 1303.07s | Valid Loss -14.529
-Test Summary | End of Epoch 56 | Time 781.34s | Test Loss -14.061
-reload weights and optimizer from last best checkpoint
-Learning rate adjusted to: 0.000128
-Learning rate is: 0.000128
-Train Summary | End of Epoch 57 | Time 8856.34s | Train Loss -16.327
-Valid Summary | End of Epoch 57 | Time 1301.12s | Valid Loss -14.620
-Test Summary | End of Epoch 57 | Time 781.09s | Test Loss -14.115
-Learning rate is: 0.000128
-Fund new best model, dict saved
-Train Summary | End of Epoch 58 | Time 8854.23s | Train Loss -16.346
-Valid Summary | End of Epoch 58 | Time 1301.64s | Valid Loss -14.590
-Test Summary | End of Epoch 58 | Time 780.97s | Test Loss -14.113
-Learning rate is: 0.000128
-Train Summary | End of Epoch 59 | Time 8854.80s | Train Loss -16.353
-Valid Summary | End of Epoch 59 | Time 1301.66s | Valid Loss -14.589
-Test Summary | End of Epoch 59 | Time 781.22s | Test Loss -14.091
-Learning rate is: 0.000128
-Train Summary | End of Epoch 60 | Time 8863.27s | Train Loss -16.371
-Valid Summary | End of Epoch 60 | Time 1301.48s | Valid Loss -14.578
-Test Summary | End of Epoch 60 | Time 781.07s | Test Loss -14.179
-Learning rate is: 0.000128
-Train Summary | End of Epoch 61 | Time 8878.44s | Train Loss -16.381
-Valid Summary | End of Epoch 61 | Time 1301.09s | Valid Loss -14.561
-Test Summary | End of Epoch 61 | Time 781.19s | Test Loss -14.117
-Learning rate is: 0.000128
-Train Summary | End of Epoch 62 | Time 8874.16s | Train Loss -16.381
-Valid Summary | End of Epoch 62 | Time 1301.83s | Valid Loss -14.627
-Test Summary | End of Epoch 62 | Time 781.20s | Test Loss -14.076
-Learning rate is: 0.000128
-Fund new best model, dict saved
-Train Summary | End of Epoch 63 | Time 8872.86s | Train Loss -16.386
-Valid Summary | End of Epoch 63 | Time 1301.73s | Valid Loss -14.573
-Test Summary | End of Epoch 63 | Time 781.22s | Test Loss -14.034
-Learning rate is: 0.000128
-Train Summary | End of Epoch 64 | Time 8872.70s | Train Loss -16.407
-Valid Summary | End of Epoch 64 | Time 1302.35s | Valid Loss -14.552
-Test Summary | End of Epoch 64 | Time 781.45s | Test Loss -14.107
-Learning rate is: 0.000128
-Train Summary | End of Epoch 65 | Time 8871.09s | Train Loss -16.411
-Valid Summary | End of Epoch 65 | Time 1301.10s | Valid Loss -14.493
-Test Summary | End of Epoch 65 | Time 781.20s | Test Loss -14.088
-Learning rate is: 0.000128
-Train Summary | End of Epoch 66 | Time 8868.35s | Train Loss -16.424
-Valid Summary | End of Epoch 66 | Time 1301.48s | Valid Loss -14.576
-Test Summary | End of Epoch 66 | Time 780.88s | Test Loss -14.107
-Learning rate is: 0.000128
-Train Summary | End of Epoch 67 | Time 8872.21s | Train Loss -16.428
-Valid Summary | End of Epoch 67 | Time 1302.63s | Valid Loss -14.600
-Test Summary | End of Epoch 67 | Time 780.67s | Test Loss -14.124
-reload weights and optimizer from last best checkpoint
-Learning rate adjusted to: 0.000064
-Learning rate is: 0.000064
-Train Summary | End of Epoch 68 | Time 8866.46s | Train Loss -16.420
-Valid Summary | End of Epoch 68 | Time 1301.06s | Valid Loss -14.629
-Test Summary | End of Epoch 68 | Time 781.03s | Test Loss -14.045
-Learning rate is: 0.000064
-Fund new best model, dict saved
-Train Summary | End of Epoch 69 | Time 8869.51s | Train Loss -16.423
-Valid Summary | End of Epoch 69 | Time 1301.42s | Valid Loss -14.610
-Test Summary | End of Epoch 69 | Time 781.07s | Test Loss -14.122
-Learning rate is: 0.000064
-Train Summary | End of Epoch 70 | Time 8870.52s | Train Loss -16.439
-Valid Summary | End of Epoch 70 | Time 1301.36s | Valid Loss -14.595
-Test Summary | End of Epoch 70 | Time 781.50s | Test Loss -14.017
-Learning rate is: 0.000064
-Train Summary | End of Epoch 71 | Time 8872.92s | Train Loss -16.443
-Valid Summary | End of Epoch 71 | Time 1302.03s | Valid Loss -14.576
-Test Summary | End of Epoch 71 | Time 781.03s | Test Loss -14.149
-Learning rate is: 0.000064
-Train Summary | End of Epoch 72 | Time 8868.30s | Train Loss -16.452
-Valid Summary | End of Epoch 72 | Time 1300.98s | Valid Loss -14.580
-Test Summary | End of Epoch 72 | Time 780.55s | Test Loss -14.096
-Learning rate is: 0.000064
-Train Summary | End of Epoch 73 | Time 8867.89s | Train Loss -16.449
-Valid Summary | End of Epoch 73 | Time 1301.70s | Valid Loss -14.556
-Test Summary | End of Epoch 73 | Time 780.97s | Test Loss -14.083
-reload weights and optimizer from last best checkpoint
-Learning rate adjusted to: 0.000032
-Learning rate is: 0.000032
-Train Summary | End of Epoch 74 | Time 8867.13s | Train Loss -16.444
-Valid Summary | End of Epoch 74 | Time 1301.02s | Valid Loss -14.603
-Test Summary | End of Epoch 74 | Time 781.03s | Test Loss -14.174
-Learning rate is: 0.000032
-Train Summary | End of Epoch 75 | Time 8848.91s | Train Loss -16.446
-Valid Summary | End of Epoch 75 | Time 1302.16s | Valid Loss -14.586
-Test Summary | End of Epoch 75 | Time 781.19s | Test Loss -14.069
-Learning rate is: 0.000032
-Train Summary | End of Epoch 76 | Time 8854.80s | Train Loss -16.460
-Valid Summary | End of Epoch 76 | Time 1300.81s | Valid Loss -14.595
-Test Summary | End of Epoch 76 | Time 781.01s | Test Loss -14.063
-Learning rate is: 0.000032
-Train Summary | End of Epoch 77 | Time 8863.14s | Train Loss -16.455
-Valid Summary | End of Epoch 77 | Time 1300.81s | Valid Loss -14.602
-Test Summary | End of Epoch 77 | Time 780.15s | Test Loss -14.056
-Learning rate is: 0.000032
-Train Summary | End of Epoch 78 | Time 8867.28s | Train Loss -16.456
-Valid Summary | End of Epoch 78 | Time 1300.89s | Valid Loss -14.569
-Test Summary | End of Epoch 78 | Time 780.83s | Test Loss -14.124
-No imporvement for 10 epochs, early stopping.
-Start evaluation
-Avg SISNR:i tensor([13.8622], device='cuda:0')
-Avg SNRi: 14.180312131890753
-Avg PESQi: 1.4813443135023117
-Avg STOIi: 0.27852386519253225
-# 2spk
-Avg SISNR:i tensor([14.4913], device='cuda:0')
-Avg SNRi: 14.766350478813006
-Avg PESQi: 1.566092278043429
-Avg STOIi: 0.2884288031001122
-
-
-# lrs2 
-# 1spk 
-Avg SISNR:i tensor([14.3695], device='cuda:0')
-Avg SNRi: 14.712913455144225
-Avg PESQi: 1.4528894378741581
-Avg STOIi: 0.27694796861127857
-# 2spk 
-Avg SISNR:i tensor([15.0343], device='cuda:0')
-Avg SNRi: 15.362836239165572
-Avg PESQi: 1.5522074782848359
-Avg STOIi: 0.28548176812938764
-
-
-# lrs3 
-# 1spk 
-Avg SISNR:i tensor([16.2440], device='cuda:0')
-Avg SNRi: 16.48747834483003
-Avg PESQi: 1.7686368883450827
-Avg STOIi: 0.2731847089622733
-# 2spk 
-Avg SISNR:i tensor([16.9063], device='cuda:0')
-Avg SNRi: 17.114254503063478
-Avg PESQi: 1.8624962186813354
-Avg STOIi: 0.279682808481817

checkpoints/log_VoxCeleb2_tfgridnet-isam_2spk/tensorboard/events.out.tfevents.1734940287.dlcdanw1zq2cucwx-master-0.28.0

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8e29766adb4a58b0df338860869285d8dbc93e0fa0c642ade14dd326772d9706
-size 8228

checkpoints/log_VoxCeleb2_tfgridnet-isam_2spk/tensorboard/events.out.tfevents.1735609783.dlc10xm9l399lwkq-master-0.26.0

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8e7ae671ffc26ca42f44f0ce092ff6f6fb160e081580b2a680bfbe4e5761751f
-size 3344