Model Loss functions becomes NaN when using multi-gpu

[bmt621]

Hello guys,

I've been trying to debug this code for a week now, but to no aveil, I am running into loss values of NaN when I try to train my model on multi-gpu, but works fine on 1 gpu, I don't know where might be the problem, when I tried to peak into the model tensor outpus, I discovered that at some iterations, the dataset at second gpu outputs all NaN values but tensors at first GPU are all floating numbers.

please if there's any expert that can help I'll really appreciate.

I am dealing with text to speech synthesis datasets, the input is a token for the text, and the output is a mel-spectrogram which contains 80 number of mels, 1024 n_fft, and hop length of 256.

I want to perform an experiment on the librispeech datasets, however, if also any one wants to contribute to the repo I'm working on, should feel free and let me know.

Thanks.

This is my sample code, and if you need to get much more access to the code please I will be happy to share also.

from model_utils import *
from utils import load_configs
import pytorch_lightning as pl
from losses import TTS_Loss
from textmel_loader import TextMelLoader1
from torch.utils.data import DataLoader
import numpy as np



class TTS_Model(nn.Module):
    def __init__(self,configs):

        super(TTS_Model,self).__init__()
        
        self.configs = configs
        self.input_embed = nn.Embedding(self.configs['EncDec_Configs']['vocab_size']+1,self.configs['EncDec_Configs']['embed_dim'])

        self.wpe = PositionalEncoding(self.configs['EncDec_Configs']['embed_dim'],self.configs['EncDec_Configs']['dropout'],max_len=self.configs['EncDec_Configs']['max_length'])
        
        self.encode = nn.TransformerEncoder(nn.TransformerEncoderLayer(configs['EncDec_Configs']['embed_dim'],self.configs['EncDec_Configs']['n_head'],self.configs['EncDec_Configs']['d_ff'],batch_first=True),
                                            self.configs['EncDec_Configs']['n_encoder_layer'])
        
        self.decode = nn.TransformerDecoder(nn.TransformerDecoderLayer(self.configs['EncDec_Configs']['embed_dim'],self.configs['EncDec_Configs']['n_head'],self.configs['EncDec_Configs']['d_ff'],batch_first=True),
                                            self.configs['EncDec_Configs']['n_decoder_layer'])
        
        self.encoder_prenet = EncoderPrenet(self.configs['EncDec_Configs']['embed_dim'],dropout=self.configs['EncDec_Configs']['dropout'])
        
        self.decoder_inprenet = Decoder_InPrenet(self.configs['Audio_Configs']['num_mels'],self.configs['EncDec_Configs']['embed_dim'],dropout=self.configs['EncDec_Configs']['dropout'])
        self.decoder_outprenet = Decoder_OutPrenet(self.configs['EncDec_Configs']['embed_dim'],self.configs['Audio_Configs']['num_mels'],dropout=self.configs['EncDec_Configs']['dropout'])
        
        self.start = nn.Embedding(1,self.configs['Audio_Configs']['num_mels'])
        self.head = HeadPredictor(self.configs,dropout=self.configs['EncDec_Configs']['dropout'])
        

    def forward(self,x,mel):

        raise NotImplementedError

    def encoder(self,src):

        """ pass input token to produce memory

        Args:
            src (_type_): mel spectrogram (B, T)
            
        """
        _, src_t = src.shape

        src_padding = self.make_src_pad_mask(src,self.configs['EncDec_Configs']['pad_idx'])

        src_tok_emb = self.input_embed(src)
        
        src_tok_emb = self.encoder_prenet(src_tok_emb)

        src_tok_emb = self.wpe(src_tok_emb)

        memory = self.encode(src_tok_emb,src_key_padding_mask=src_padding)

        return memory
    

    def decoder(self,mel,memory,tgt_padding_mask=None):
        """ 
        the decoder receives mel spectrograms (B, T, n_mel) an memory of the encoder layers,
        the docoder then passes it to decoder layer and head layer to predict the spectrograms,
        and the gate output.

        Args:
            mel (_type_): mel spectrogram (B, T, n_mel)
            mel_mask (_type_): 
            memory  (_type_): memory of encoder (B, T, n_hid)
            
        """
        

        _,tgt_t,_ = mel.shape

        self.register_buffer('attn_mask',torch.triu(torch.full((tgt_t,tgt_t), float('-inf')).to(mel.device),diagonal=1))
        
        mel_in = self.decoder_inprenet(mel)
        
        mel_in = self.wpe(mel_in)

        out = self.decode(mel_in,memory,tgt_mask=self.attn_mask,tgt_key_padding_mask=tgt_padding_mask)

        mel_lin, mel_fin, gate = self.head(out)

        return mel_lin, mel_fin, gate
    

    def parse_mel_and_padding_mask(self,mel,tgt_padding_mask=None):
        
        start_mel = self.start(torch.tensor([0]).to(mel.device)).unsqueeze(1).expand([mel.shape[0],1,mel.shape[-1]]) # Get the start Embeddings
        
        mel = torch.cat([start_mel,mel],dim=1)
        
        if tgt_padding_mask != None:
            tgt_padding_mask = torch.cat([torch.tensor(0).unsqueeze(0).expand(tgt_padding_mask.shape[0],1).to(mel.device),tgt_padding_mask],dim=1)
        
        return mel, tgt_padding_mask.float()


    def make_src_pad_mask(self,src,pad_idx):
        src_padding = (src == pad_idx).float()
        
        return src_padding

    

    



class Trainer(pl.LightningModule):

    def __init__(self,configs,batch_size, max_epochs):
        super(Trainer,self).__init__()
        
        self.model = TTS_Model(configs)
        self.loss_fn = TTS_Loss(weight=configs['Training_Configs']['weight'])

        self.configs = configs


    def forward(self,input_id,mel_input,mel_padding):
        memory = self.model.encoder(input_id)
        output = self.model.decoder(mel_input,memory,mel_padding)
        
        return output
    
    def training_step(self,batch,batch_idx):

        input_id, output_mel, mel_padding = batch
        output_mel,mel_padding = self.model.parse_mel_and_padding_mask(output_mel,mel_padding)

        mel_input = output_mel[:,:-1,:]
        mel_output = output_mel[:,1:,:]
        mel_padding = mel_padding[:,:-1]
        

        pred = self.forward(input_id,mel_input,mel_padding)
        mel1, mel2, gate_pred = pred


        print('mel1: ',mel1)
        print('mel2: ',mel2)
        print('gate pred: ',gate_pred)

        target = (mel_output,mel_padding)
                
        loss = self.loss_fn(pred,target)
        
        self.log("train_loss",loss,prog_bar=True,logger=True,on_step=True,on_epoch=True)

        return loss


    def validation_step(self,batch,batch_idx):
        
        input_id, output_mel, mel_padding = batch
        
        output_mel, mel_padding = self.model.parse_mel_and_padding_mask(output_mel,mel_padding)

        mel_input = output_mel[:,:-1,:]
        mel_output = output_mel[:,1:,:]
        mel_padding = mel_padding[:,:-1]

        pred = self.forward(input_id,mel_input,mel_padding)
        target = (mel_output,mel_padding)
                
        loss = self.loss_fn(pred,target)
        
        metrics = {'val_loss':loss}

        self.log_dict(metrics,prog_bar=True,on_step=True,on_epoch=True,logger=True)

        return loss


    def configure_optimizers(self):
        
        optimizer = torch.optim.AdamW(self.parameters(),lr=self.configs['Training_Configs']['lr'],betas=(0.9, 0.98), eps=1e-9)
        # We don't return the lr scheduler because we need to apply it per iteration, not per epoch

        # Set warmup to 10% of total training iterations (adjust as needed)
        warmup_ratio = 0.05
        batch_size = self.configs['Training_Configs']['train_batch_size']
        max_epochs = self.configs['Training_Configs']['epoch']

        self.total_train_iterators = (batch_size * max_epochs)
        
        warmup_steps = int(warmup_ratio * self.total_train_iterators)

        """self.lr_scheduler = CosineWarmupScheduler(
            optimizer, warmup=warmup_steps, max_iters=self.total_train_iterators
        )"""
        return optimizer
    

    def optimizer_step(self, *args, **kwargs):
        super().optimizer_step(*args, **kwargs)
        #self.lr_scheduler.step()  # Step per iteration


    """def configure_gradient_clipping(self, optimizer,gradient_clip_val=1, gradient_clip_algorithm="norm"):
        self.clip_gradients(
            optimizer, gradient_clip_val=gradient_clip_val, gradient_clip_algorithm=gradient_clip_algorithm
        )"""
    

class CosineWarmupScheduler(torch.optim.lr_scheduler._LRScheduler):
    def __init__(self, optimizer, warmup, max_iters):
        self.warmup = warmup
        self.max_num_iters = max_iters
        super().__init__(optimizer)

    def get_lr(self):
        lr_factor = self.get_lr_factor(epoch=self.last_epoch)
        return [base_lr * lr_factor for base_lr in self.base_lrs]

    def get_lr_factor(self, epoch):
        lr_factor = 0.5 * (1 + np.cos(np.pi * epoch / self.max_num_iters))
        if epoch <= self.warmup:
            lr_factor *= epoch * 1.0 / self.warmup
        return lr_factor
    

def main():

    configs = load_configs('hparams.yaml')

    train_ds = TextMelLoader1(configs,what='train')
    test_ds = TextMelLoader1(configs,what= 'test')

    train_loader = DataLoader(train_ds,batch_size=configs['Training_Configs']['train_batch_size'],shuffle=True,collate_fn=train_ds.collate,num_workers=6)
    test_loader = DataLoader(test_ds,batch_size=configs['Training_Configs']['test_batch_size'],shuffle=False,collate_fn=test_ds.collate,num_workers=6)
    
    batch_size = len(train_loader)
    max_epochs = configs['Training_Configs']['epoch']

    model = Trainer(configs,batch_size,max_epochs)

    trainer = pl.Trainer(default_root_dir='callbacks',accelerator='gpu',devices=[0,1],max_epochs=max_epochs,gradient_clip_val=1,gradient_clip_algorithm='norm',detect_anomaly=True)
    
    trainer.fit(model,train_dataloaders=train_loader,val_dataloaders=test_loader)


if __name__ == "__main__":
    main()

[bmt621]

This is the output terminal of the model

        ...,

        [[8.5059],
         [7.0590],
         [6.6325],
         ...,
         [8.6990],
         [8.8243],
         [8.8800]],

        [[8.2098],
         [7.1830],
         [6.9496],
         ...,
         [8.7324],
         [9.5356],
         [9.1277]],

        [[7.9836],
         [6.9741],
         [7.6025],
         ...,
         [8.7032],
         [9.0171],
         [9.1814]]], device='cuda:0', grad_fn=<ViewBackward0>)
mel1:  tensor([[[-4.9884, -1.5409, -5.0648,  ..., -2.5985, -3.5240, -2.7660],
         [-5.7336, -1.7388, -5.1402,  ..., -2.6406, -3.6311, -1.8710],
         [-4.6297, -1.8799, -4.7710,  ..., -1.2502, -2.2766, -1.7270],
         ...,
         [-5.4363, -1.5452, -4.0014,  ..., -1.3293, -2.7649, -1.8893],
         [-4.2600, -2.4688, -4.4258,  ..., -1.2707, -2.4754, -2.3815],
         [-3.8967, -1.0034, -3.9172,  ..., -2.2047, -3.7327, -1.9109]],

        [[-4.5635, -1.7109, -4.5046,  ..., -2.3803, -2.9769, -2.1610],
         [-5.1448, -2.4769, -4.1390,  ..., -0.6731, -2.5241, -1.8009],
         [-5.5880, -1.6439, -3.3579,  ..., -2.6309, -4.0569, -2.3283],
         ...,
         [-6.5007, -1.2457, -4.0366,  ..., -2.1569, -3.4386, -2.0832],
         [-4.1053, -1.9734, -4.4610,  ..., -2.5131, -2.3053, -2.4151],
         [-7.2219, -3.5396, -4.0966,  ..., -2.7761, -4.1764, -2.3055]],

        [[-6.7539, -1.6804, -5.1220,  ..., -1.7438, -5.0347, -3.0818],
         [-5.7484, -1.6623, -4.9055,  ..., -3.1252, -3.4022, -3.4192],
         [-5.3388, -1.2595, -5.2539,  ..., -1.4861, -3.5202, -2.7390],
         ...,
         [-5.2920, -0.9981, -4.8467,  ..., -2.2151, -3.7936, -3.7568],
         [-5.4554, -2.4706, -3.7217,  ..., -2.6489, -2.5144, -2.9372],
         [-4.1735, -1.4009, -5.3510,  ..., -1.8421, -2.3268, -3.2954]],

        ...,

        [[-5.6374, -1.7298, -6.1246,  ..., -1.5326, -3.8852, -1.4034],
         [-5.1004, -1.9723, -4.7442,  ..., -2.4933, -2.5889, -2.2618],
         [-4.8285, -1.7660, -4.2603,  ..., -1.3787, -2.1026, -2.5428],
         ...,
         [-3.6687, -1.7850, -3.7021,  ..., -2.3615, -2.2335, -2.1362],
         [-5.8319, -2.1694, -4.7262,  ..., -1.5712, -0.9611, -2.2790],
         [-4.2562, -0.9180, -4.7454,  ..., -1.7181, -2.3141, -2.0880]],

        [[-4.9499, -1.8667, -5.1503,  ..., -1.2253, -3.4637, -1.7901],
         [-4.7766, -2.0239, -4.6474,  ..., -1.6091, -1.5420, -3.3627],
         [-5.5642, -2.1460, -4.2860,  ..., -2.1353, -4.2000, -2.3017],
         ...,
         [-5.7457, -1.8145, -5.0864,  ..., -3.2450, -3.3706, -3.2901],
         [-4.9854, -2.0279, -4.3242,  ..., -1.8446, -1.9390, -2.8786],
         [-5.9864, -2.3211, -4.2784,  ..., -2.2322, -3.0749, -2.5804]],

        [[-4.5818, -1.9168, -2.7206,  ..., -0.7405, -2.7996, -1.7181],
         [-6.2994, -3.0330, -4.6531,  ..., -1.0518, -3.4149, -1.9067],
         [-5.8963, -1.1864, -5.2191,  ..., -3.3968, -2.8966, -4.3547],
         ...,
         [-4.7075, -2.5914, -4.2948,  ..., -1.9004, -3.7486, -2.9151],
         [-4.6649, -0.7504, -5.2345,  ..., -2.2625, -2.5098, -3.9085],
         [-7.9993, -1.2826, -4.5913,  ..., -2.3825, -2.1755, -2.3040]]],
       device='cuda:1', grad_fn=<ViewBackward0>)
mel2:  tensor([[[-3.9274, -1.1393, -5.2785,  ..., -3.5141, -3.5224, -3.4842],
         [-6.4213, -1.4006, -5.7724,  ..., -3.0137, -2.6327, -3.3559],
         [-3.4007, -2.4416, -4.7104,  ..., -0.9256, -2.7826, -0.2526],
         ...,
         [-4.3364, -3.5336, -3.4306,  ..., -1.5923, -4.1370, -0.9033],
         [-4.2583, -1.6115, -4.4258,  ..., -2.5330, -2.1568, -3.9227],
         [-3.4125, -1.0034, -3.7791,  ..., -3.2858, -4.5588, -1.6640]],

        [[-5.7773, -1.7652, -3.9504,  ..., -3.0439, -2.2698, -1.9934],
         [-6.9154, -2.5210, -4.3644,  ...,  0.0285, -2.6914, -1.8009],
         [-3.9909, -2.1314, -2.7204,  ..., -0.9779, -4.6965, -3.6536],
         ...,
         [-6.9219, -1.2074, -3.6075,  ..., -1.3697, -2.6172, -0.6180],
         [-3.5163, -0.9328, -4.4610,  ..., -2.7252, -2.3053, -2.4151],
         [-7.2219, -3.4411, -3.5864,  ..., -3.6896, -5.2199, -2.0645]],

        [[-7.3655, -1.4729, -5.1013,  ..., -1.5752, -3.2866, -2.9591],
         [-6.0752, -0.5220, -5.0998,  ..., -1.0819, -2.5676, -1.7312],
         [-3.1986, -0.8842, -5.5762,  ..., -1.4861, -2.5050, -2.7390],
         ...,
         [-5.0798, -1.0112, -6.9025,  ..., -2.5434, -4.8439, -4.1401],
         [-4.7895, -2.3298, -4.0790,  ..., -2.7893, -1.5416, -4.1977],
         [-3.8896, -0.8861, -4.9485,  ..., -1.2538, -2.1691, -3.6368]],

        ...,

        [[-4.8778, -1.4927, -6.7199,  ..., -1.6080, -4.5175, -1.1253],
         [-5.7843, -2.0700, -4.8329,  ..., -2.1609, -3.6090, -1.2030],
         [-4.1107, -2.0833, -4.6741,  ..., -1.2443, -1.0768, -0.8272],
         ...,
         [-3.4289, -0.0842, -3.4881,  ..., -3.0110, -2.7868, -1.6242],
         [-4.8621, -3.4186, -5.2199,  ..., -2.1102, -0.9611, -3.1965],
         [-3.0441, -0.9180, -4.6498,  ..., -2.4138, -2.4511, -2.4053]],

        [[-4.8455, -2.3686, -5.1503,  ..., -1.9349, -2.5022, -1.2992],
         [-4.1757, -2.0239, -5.5830,  ..., -1.0777, -1.1596, -1.9491],
         [-6.5007, -3.7302, -4.7240,  ..., -2.8703, -2.9442, -1.5494],
         ...,
         [-4.7991, -1.5615, -5.9187,  ..., -4.0526, -4.2220, -3.2901],
         [-4.6506, -2.1720, -3.9907,  ..., -3.4848, -3.1754, -2.8219],
         [-6.0490, -2.3211, -5.0508,  ..., -2.8976, -4.0585, -2.9438]],

        [[-5.3165, -1.9168, -2.3715,  ..., -1.4593, -3.6692, -2.1535],
         [-6.2994, -2.6859, -3.5024,  ...,  0.0257, -3.8527, -3.4017],
         [-5.8963, -3.4826, -5.8582,  ..., -4.2347, -5.4275, -4.9906],
         ...,
         [-3.6292, -2.0066, -4.2948,  ..., -1.9004, -4.2288, -1.5753],
         [-5.4924, -2.3215, -4.3679,  ..., -2.2143, -2.9648, -5.6602],
         [-8.8751, -1.2826, -4.3175,  ..., -2.4699, -2.0855, -1.1172]]],
       device='cuda:1', grad_fn=<AddBackward0>)
gate pred:  tensor([[[9.0608],
         [7.5030],
         [7.8003],
         ...,
         [8.8862],
         [9.3799],
         [8.8652]],

        [[8.7899],
         [7.0769],
         [7.3285],
         ...,
         [8.7310],
         [9.0610],
         [9.0852]],

        [[8.3062],
         [6.9441],
         [7.5635],
         ...,
         [9.4233],
         [9.0525],
         [8.8826]],

        ...,

        [[8.8262],
         [7.6986],
         [7.0370],
         ...,
         [9.2783],
         [9.4123],
         [9.1084]],

        [[8.7526],
         [7.2238],
         [7.6079],
         ...,
         [8.8004],
         [9.0540],
         [9.2850]],

        [[7.7275],
         [7.0758],
         [6.2250],
         ...,
         [7.6019],
         [6.9380],
         [7.8266]]], device='cuda:1', grad_fn=<ViewBackward0>)
Epoch 0:   1%|▎                                                                  | 2/377 [00:14<45:28,  7.28s/it, loss=45.1, v_num=1, train_loss_step=34.50]mel1:  tensor([[[nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         ...,
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan]],

        [[nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         ...,
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan]],

        [[nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         ...,
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan]],

        ...,

        [[nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         ...,
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan]],

        [[nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         ...,
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan]],

        [[nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         ...,
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:1',
       grad_fn=<ViewBackward0>)
mel2:  tensor([[[nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         ...,
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan]],

        [[nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         ...,
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan]],

        [[nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         ...,
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan]],

        ...,

        [[nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         ...,
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan]],

        [[nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         ...,
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan]],

        [[nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         ...,
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:1',
       grad_fn=<AddBackward0>)
gate pred:  tensor([[[nan],
         [nan],
         [nan],
         ...,
         [nan],
         [nan],
         [nan]],

        [[nan],
         [nan],
         [nan],
         ...,
         [nan],
         [nan],
         [nan]],

        [[nan],
         [nan],
         [nan],
         ...,
         [nan],
         [nan],
         [nan]],

        ...,

        [[nan],
         [nan],
         [nan],
         ...,
         [nan],
         [nan],
         [nan]],

        [[nan],
         [nan],
         [nan],
         ...,
         [nan],
         [nan],
         [nan]],

        [[nan],
         [nan],
         [nan],
         ...,
         [nan],
         [nan],
         [nan]]], device='cuda:1', grad_fn=<ViewBackward0>)
/home/unicconai/.local/lib/python3.10/site-packages/torch/autograd/__init__.py:200: UserWarning: Error detected in BinaryCrossEntropyWithLogitsBackward0. Traceback of forward call that caused the error:
  File "<string>", line 1, in <module>
  File "/usr/lib/python3.10/multiprocessing/spawn.py", line 116, in spawn_main
    exitcode = _main(fd, parent_sentinel)
  File "/usr/lib/python3.10/multiprocessing/spawn.py", line 129, in _main
    return self._bootstrap(parent_sentinel)
  File "/usr/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
    self.run()
  File "/usr/lib/python3.10/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 69, in _wrap
    fn(i, *args)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/multiprocessing.py", line 139, in _wrapping_function
    results = function(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 650, in _fit_impl
    self._run(model, ckpt_path=self.ckpt_path)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1112, in _run
    results = self._run_stage()
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1191, in _run_stage
    self._run_train()
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1214, in _run_train
    self.fit_loop.run()
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
    self.advance(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 267, in advance
    self._outputs = self.epoch_loop.run(self._data_fetcher)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
    self.advance(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/epoch/training_epoch_loop.py", line 213, in advance
    batch_output = self.batch_loop.run(kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
    self.advance(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/batch/training_batch_loop.py", line 88, in advance
    outputs = self.optimizer_loop.run(optimizers, kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
    self.advance(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 202, in advance
    result = self._run_optimization(kwargs, self._optimizers[self.optim_progress.optimizer_position])
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 249, in _run_optimization
    self._optimizer_step(optimizer, opt_idx, kwargs.get("batch_idx", 0), closure)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 370, in _optimizer_step
    self.trainer._call_lightning_module_hook(
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1356, in _call_lightning_module_hook
    output = fn(*args, **kwargs)
  File "/home/unicconai/DATAHDD01/Ground1/trainer.py", line 203, in optimizer_step
    super().optimizer_step(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/core/module.py", line 1742, in optimizer_step
    optimizer.step(closure=optimizer_closure)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/core/optimizer.py", line 169, in step
    step_output = self._strategy.optimizer_step(self._optimizer, self._optimizer_idx, closure, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 234, in optimizer_step
    return self.precision_plugin.optimizer_step(
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 119, in optimizer_step
    return optimizer.step(closure=closure, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/optim/optimizer.py", line 280, in wrapper
    out = func(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/optim/optimizer.py", line 33, in _use_grad
    ret = func(self, *args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/optim/adamw.py", line 148, in step
    loss = closure()
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 105, in _wrap_closure
    closure_result = closure()
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 149, in __call__
    self._result = self.closure(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 135, in closure
    step_output = self._step_fn()
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 419, in _training_step
    training_step_output = self.trainer._call_strategy_hook("training_step", *kwargs.values())
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1494, in _call_strategy_hook
    output = fn(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/strategies/ddp_spawn.py", line 280, in training_step
    return self.model(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1156, in forward
    output = self._run_ddp_forward(*inputs, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1110, in _run_ddp_forward
    return module_to_run(*inputs[0], **kwargs[0])  # type: ignore[index]
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/overrides/base.py", line 98, in forward
    output = self._forward_module.training_step(*inputs, **kwargs)
  File "/home/unicconai/DATAHDD01/Ground1/trainer.py", line 153, in training_step
    loss = self.loss_fn(pred,target)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/unicconai/DATAHDD01/Ground1/losses.py", line 22, in forward
    gate_loss = self.gate_loss_fn(gate_out.view(gate_out.shape[0],gate_out.shape[1]),gate_target.float())
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/nn/modules/loss.py", line 720, in forward
    return F.binary_cross_entropy_with_logits(input, target,
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/nn/functional.py", line 3165, in binary_cross_entropy_with_logits
    return torch.binary_cross_entropy_with_logits(input, target, weight, pos_weight, reduction_enum)
 (Triggered internally at ../torch/csrc/autograd/python_anomaly_mode.cpp:114.)
  Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
mel1:  tensor([[[-3.6737, -4.4388, -4.1974,  ..., -3.3221, -3.6772, -3.9826],
         [-3.2896, -4.2869, -4.8163,  ..., -3.7687, -2.7521, -4.1529],
         [-3.6435, -4.9859, -4.6466,  ..., -3.3028, -3.5812, -5.7133],
         ...,
         [-2.3688, -3.3139, -3.8303,  ..., -2.8653, -4.0887, -4.2637],
         [-3.2940, -4.4961, -3.6710,  ..., -2.4030, -3.3444, -4.9252],
         [-3.4368, -4.0684, -3.5163,  ..., -3.5467, -1.9612, -5.1070]],

        [[-3.7376, -3.8719, -4.7969,  ..., -3.8687, -2.6388, -4.1079],
         [-4.0198, -4.3327, -3.9137,  ..., -2.3830, -3.9492, -3.0275],
         [-3.1840, -4.4632, -4.1187,  ..., -3.0205, -4.2227, -3.4890],
         ...,
         [-2.7660, -5.1312, -3.1032,  ..., -1.5552, -3.2653, -3.6323],
         [-3.6386, -2.9156, -3.7783,  ..., -3.3586, -4.6999, -4.5302],
         [-4.1057, -4.3188, -3.0821,  ..., -3.1743, -2.8602, -4.7234]],

        [[-3.2824, -3.3503, -4.0521,  ..., -2.3499, -3.4213, -4.9645],
         [-2.5607, -4.0498, -4.2321,  ..., -4.6593, -2.1048, -4.2810],
         [-3.5598, -4.5178, -4.3636,  ..., -3.1830, -2.8124, -5.2206],
         ...,
         [-2.9164, -4.9892, -5.1761,  ..., -3.7800, -3.0219, -4.2786],
         [-3.2871, -3.1808, -3.7218,  ..., -2.2283, -2.7605, -4.1158],
         [-2.0283, -5.0359, -4.0723,  ..., -1.6424, -1.7052, -3.6586]],

        ...,

        [[-2.5087, -3.1404, -3.8999,  ..., -3.4438, -2.5906, -5.2503],
         [-4.5608, -3.6911, -4.5526,  ..., -4.1241, -3.0050, -4.8943],
         [-3.7187, -5.9198, -3.8341,  ..., -3.3498, -2.3064, -4.7206],
         ...,
         [-4.3645, -4.6682, -4.8049,  ..., -3.1054, -2.9969, -4.2599],
         [-3.3657, -4.5014, -4.4181,  ..., -2.1969, -3.0795, -4.3896],
         [-2.4793, -4.0863, -5.2396,  ..., -2.5869, -3.8837, -4.6583]],

        [[-3.1446, -4.9218, -3.4313,  ..., -2.5798, -2.6980, -4.9484],
         [-3.1812, -4.2918, -3.8062,  ..., -2.4020, -3.1682, -4.6596],
         [-3.9589, -3.8272, -5.6857,  ..., -5.0058, -2.2427, -4.0993],
         ...,
         [-2.3539, -3.7514, -5.2302,  ..., -2.1599, -3.6589, -3.7084],
         [-3.7855, -4.5482, -4.0361,  ..., -3.7872, -4.6892, -4.3589],
         [-2.8493, -3.3697, -4.2885,  ..., -4.3837, -3.9416, -3.1004]],

        [[-5.5211, -4.9770, -3.8955,  ..., -2.5983, -2.9595, -3.8463],
         [-3.0943, -4.9072, -4.8974,  ..., -2.9151, -3.8979, -4.3141],
         [-1.6350, -5.4536, -3.5156,  ..., -2.7827, -0.9073, -4.0542],
         ...,
         [-4.3370, -4.2997, -3.4462,  ..., -1.6786, -2.7401, -3.9190],
         [-2.1478, -3.3616, -4.5292,  ..., -3.9777, -2.4254, -4.7707],
         [-3.2601, -3.3411, -3.7299,  ..., -3.5424, -2.9140, -4.5661]]],
       device='cuda:0', grad_fn=<ViewBackward0>)
mel2:  tensor([[[-3.6737, -4.2227, -3.2627,  ..., -2.6269, -3.7237, -3.6050],
         [-2.0118, -3.9422, -5.2740,  ..., -3.0495, -2.2741, -2.8709],
         [-2.8481, -4.3753, -4.0890,  ..., -2.2044, -2.4787, -5.2597],
         ...,
         [-2.3688, -3.3139, -2.5435,  ..., -0.9596, -2.3182, -3.0674],
         [-2.0684, -3.1707, -2.4704,  ..., -1.5260, -1.4389, -3.4386],
         [-2.9780, -3.1678, -3.3533,  ..., -3.0477, -0.4928, -5.1070]],

        [[-3.7376, -3.8719, -4.1349,  ..., -3.7605, -2.6992, -3.7016],
         [-3.2645, -3.8901, -3.4757,  ..., -1.4946, -2.6056, -1.5933],
         [-2.6647, -4.4632, -4.1080,  ..., -2.3512, -4.4159, -2.8394],
         ...,
         [-1.6691, -3.4361, -2.0379,  ...,  0.0169, -1.2240, -3.6323],
         [-2.4754, -1.8359, -3.0041,  ..., -1.7989, -3.2347, -2.8957],
         [-3.3585, -3.3159, -2.4370,  ..., -2.1597, -0.6378, -3.7043]],

        [[-3.6744, -2.7947, -3.3436,  ..., -2.1181, -2.8327, -4.7893],
         [-1.8971, -3.6933, -4.0952,  ..., -4.6683, -1.8358, -3.8846],
         [-3.3674, -4.1346, -4.7305,  ..., -2.5498, -2.8124, -5.2206],
         ...,
         [-2.6198, -5.4576, -5.2921,  ..., -3.6770, -2.7278, -3.9119],
         [-3.4294, -3.6296, -3.6772,  ..., -2.4316, -2.6590, -4.6120],
         [-2.0283, -5.4738, -4.3711,  ..., -1.8928, -0.8672, -3.6441]],

        ...,

        [[-2.2997, -2.7337, -3.3252,  ..., -2.9289, -2.3216, -4.9538],
         [-4.3377, -3.6410, -4.5526,  ..., -3.7877, -2.5555, -4.4047],
         [-3.0588, -4.8019, -3.8341,  ..., -2.8384, -1.1474, -4.1912],
         ...,
         [-2.8035, -3.5128, -3.5559,  ..., -1.5325, -0.9383, -2.9903],
         [-2.3121, -3.2058, -2.3809,  ..., -2.1969, -0.4593, -3.4148],
         [-1.6623, -3.3980, -5.0106,  ..., -2.0352, -2.3633, -4.6583]],

        [[-2.6171, -4.2185, -3.6560,  ..., -2.0681, -2.3484, -4.2555],
         [-2.5860, -3.7166, -3.8062,  ..., -1.3815, -1.8003, -4.5201],
         [-2.7940, -2.8133, -5.4647,  ..., -4.4507, -0.4949, -3.1798],
         ...,
         [-1.1231, -2.2940, -3.1840,  ..., -0.6263, -2.0772, -1.4253],
         [-2.4211, -4.5482, -2.5539,  ..., -2.6604, -4.6892, -2.9856],
         [-2.0626, -3.3697, -4.1118,  ..., -3.4535, -3.9416, -2.7145]],

        [[-5.5211, -4.8270, -3.1139,  ..., -2.0383, -2.3418, -3.2395],
         [-3.1454, -5.2666, -5.2316,  ..., -2.6279, -3.3461, -4.3141],
         [-1.0367, -5.5609, -3.6919,  ..., -2.7948, -0.5908, -3.9613],
         ...,
         [-3.2751, -3.1657, -3.4462,  ...,  0.2233, -1.5093, -2.4451],
         [-1.7301, -2.6804, -3.4152,  ..., -3.0595, -0.9398, -3.7772],
         [-2.7613, -2.3687, -3.0048,  ..., -2.7431, -1.5044, -3.7297]]],
       device='cuda:0', grad_fn=<AddBackward0>)
gate pred:  tensor([[[-4.4020],
         [-5.1660],
         [-5.0090],
         ...,
         [-2.8767],
         [-3.3255],
         [-3.1444]],

        [[-4.0703],
         [-4.8511],
         [-5.1287],
         ...,
         [-3.4711],
         [-2.8913],
         [-3.3216]],

        [[-4.4903],
         [-5.0030],
         [-4.8219],
         ...,
         [-5.0511],
         [-4.8871],
         [-5.2677]],

        ...,

        [[-4.5807],
         [-5.0055],
         [-4.5069],
         ...,
         [-4.0453],
         [-3.9579],
         [-3.7498]],

        [[-4.2894],
         [-4.9158],
         [-4.8787],
         ...,
         [-3.4418],
         [-3.1622],
         [-3.6099]],

        [[-4.1484],
         [-5.0784],
         [-4.9759],
         ...,
         [-4.2049],
         [-4.3198],
         [-3.7459]]], device='cuda:0', grad_fn=<ViewBackward0>)
Traceback (most recent call last):
  File "/home/unicconai/DATAHDD01/Ground1/trainer.py", line 251, in <module>
    main()
  File "/home/unicconai/DATAHDD01/Ground1/trainer.py", line 247, in main
    trainer.fit(model,train_dataloaders=train_loader,val_dataloaders=test_loader)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 608, in fit
    call._call_and_handle_interrupt(
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 36, in _call_and_handle_interrupt
    return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/multiprocessing.py", line 113, in launch
    mp.start_processes(
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 197, in start_processes
    while not context.join():
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 160, in join
    raise ProcessRaisedException(msg, error_index, failed_process.pid)
torch.multiprocessing.spawn.ProcessRaisedException: 

-- Process 1 terminated with the following error:
Traceback (most recent call last):
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 69, in _wrap
    fn(i, *args)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/multiprocessing.py", line 139, in _wrapping_function
    results = function(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 650, in _fit_impl
    self._run(model, ckpt_path=self.ckpt_path)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1112, in _run
    results = self._run_stage()
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1191, in _run_stage
    self._run_train()
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1214, in _run_train
    self.fit_loop.run()
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
    self.advance(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 267, in advance
    self._outputs = self.epoch_loop.run(self._data_fetcher)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
    self.advance(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/epoch/training_epoch_loop.py", line 213, in advance
    batch_output = self.batch_loop.run(kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
    self.advance(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/batch/training_batch_loop.py", line 88, in advance
    outputs = self.optimizer_loop.run(optimizers, kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
    self.advance(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 202, in advance
    result = self._run_optimization(kwargs, self._optimizers[self.optim_progress.optimizer_position])
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 249, in _run_optimization
    self._optimizer_step(optimizer, opt_idx, kwargs.get("batch_idx", 0), closure)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 370, in _optimizer_step
    self.trainer._call_lightning_module_hook(
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1356, in _call_lightning_module_hook
    output = fn(*args, **kwargs)
  File "/home/unicconai/DATAHDD01/Ground1/trainer.py", line 203, in optimizer_step
    super().optimizer_step(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/core/module.py", line 1742, in optimizer_step
    optimizer.step(closure=optimizer_closure)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/core/optimizer.py", line 169, in step
    step_output = self._strategy.optimizer_step(self._optimizer, self._optimizer_idx, closure, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 234, in optimizer_step
    return self.precision_plugin.optimizer_step(
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 119, in optimizer_step
    return optimizer.step(closure=closure, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/optim/optimizer.py", line 280, in wrapper
    out = func(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/optim/optimizer.py", line 33, in _use_grad
    ret = func(self, *args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/optim/adamw.py", line 148, in step
    loss = closure()
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 105, in _wrap_closure
    closure_result = closure()
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 149, in __call__
    self._result = self.closure(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 144, in closure
    self._backward_fn(step_output.closure_loss)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 305, in backward_fn
    self.trainer._call_strategy_hook("backward", loss, optimizer, opt_idx)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1494, in _call_strategy_hook
    output = fn(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 207, in backward
    self.precision_plugin.backward(closure_loss, self.lightning_module, optimizer, optimizer_idx, *args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 67, in backward
    model.backward(tensor, optimizer, optimizer_idx, *args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/core/module.py", line 1486, in backward
    loss.backward(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/_tensor.py", line 487, in backward
    torch.autograd.backward(
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/autograd/__init__.py", line 200, in backward
    Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
RuntimeError: Function 'BinaryCrossEntropyWithLogitsBackward0' returned nan values in its 0th output.

[rasbt]

Hm, that's weird. Maybe try

• drop_last=True in the training data loader.
• a very small learning rate
• accelerator='gpu',devices=[0] and accelerator='gpu',devices=[1] (sometimes hardware issues can also result in NaNs)

[bmt621]

thanks for the help. but it still does not solve it. after getting a bit deeper, I got this

works fine with src.
works fine with src_padding.
works fine with input_embed.
works fine with encoder prenet
works fine with positional encoding.
works fine with src.
works fine with src_padding.
works fine with input_embed.
works fine with encoder prenet
works fine with positional encoding.
Epoch 0:   0%|▏                                                               | 1/375 [00:14<1:28:03, 14.13s/it, loss=55.1, v_num=23, train_loss_step=55.10]works fine with src.
works fine with src.
works fine with src_padding.
works fine with src_padding.
works fine with input_embed.
works fine with input_embed.
works fine with encoder prenet
works fine with encoder prenet
works fine with positional encoding.
found 2495 NaN when passed to postional encoding on gpu cuda:1. here is it:  tensor([[[ 0.4866,  0.0000,  0.5142,  ...,    -inf,    -inf,    -inf],
         [   -inf,    -inf,    -inf,  ..., -1.2173, -0.5182,  0.0000],
         [ 0.3769, -0.9761,  0.3787,  ..., -1.5793,  2.2152,  0.0000],
         ...,
         [ 1.7059,  1.4806, -0.7680,  ...,  3.1686, -0.7785,  0.8146],
         [ 1.6538,  0.2284, -1.3080,  ...,  2.7032, -0.0457,  0.4524],
         [-0.2355,  0.0000, -0.5987,  ..., -0.5142,  1.5948,  1.6634]],

        [[-1.3544,  0.0422, -1.0196,  ...,    -inf,    -inf,    -inf],
         [   -inf,    -inf,    -inf,  ...,  0.1871, -0.0246, -0.5295],
         [ 1.3340,  0.3426, -1.6562,  ...,  0.4174, -0.0889, -0.3865],
         ...,
         [ 0.3414,  1.1602,  2.8662,  ...,  0.1115, -1.8514,  1.3603],
         [-0.4950,  0.8117,  2.4417,  ...,  0.5528, -1.5063,  1.3418],
         [-0.5341,  1.0792,  2.5675,  ..., -0.1759, -2.0140,  1.5531]],

        [[ 0.2021,  0.4118, -2.2820,  ...,    -inf,    -inf,    -inf],
         [   -inf,    -inf,    -inf,  ..., -0.0051, -1.9306,  2.7405],
         [ 0.7215,  1.7670,  0.0567,  ..., -0.7825, -0.9797, -0.0752],
         ...,
         [-0.7565,  0.0000,  2.5726,  ...,  0.3521, -1.7480,  1.5808],
         [-0.6534,  1.2411,  2.0983,  ...,  0.1834, -1.9297,  1.7794],
         [-0.3179,  2.0919,  1.8541,  ...,  0.0635, -0.0000,  1.8485]],

        ...,

        [[ 0.7104,  1.5277,  1.3966,  ...,    -inf,    -inf,    -inf],
         [    nan,    -inf,    -inf,  ..., -0.4818, -0.3819,  0.2207],
         [ 1.3153, -1.2470, -0.1559,  ..., -0.4127,  1.6908,  0.0000],
         ...,
         [-0.5205,  0.9182,  3.1999,  ..., -0.0237, -1.6786,  1.6730],
         [-0.2554,  0.7506,  2.3096,  ...,  0.1957, -2.0126,  0.0000],
         [-0.0000,  1.3480,  2.5885,  ..., -0.2350, -1.0572,  1.1677]],

        [[-1.4862, -0.8884,  1.5412,  ...,     nan,    -inf,    -inf],
         [   -inf,    -inf,     nan,  ...,  0.0000, -0.4598, -0.5305],
         [-1.0473, -0.5680, -0.8212,  ..., -1.8638,  0.9324,  0.2712],
         ...,
         [ 0.1536,  1.7386,  2.2279,  ...,  0.4891, -1.8380,  2.0124],
         [-0.3326,  1.3920,  2.8015,  ..., -0.3142, -1.9711,  1.8491],
         [-0.8495,  0.6122,  2.0013,  ...,  0.7340, -1.6493,  0.0000]],

        [[ 1.3981, -1.4943, -0.5961,  ...,    -inf,    -inf,    -inf],
         [   -inf,    -inf,    -inf,  ..., -1.8155,  2.4193,  0.0000],
         [-0.0000, -0.0000, -0.1436,  ..., -0.8193, -0.0000,  2.0746],
         ...,
         [-0.0000,  0.8050,  0.0000,  ..., -0.3591, -1.8052,  1.2698],
         [-0.2848,  0.6283,  2.3617,  ...,  0.1663, -1.4633,  2.0116],
         [-0.4634,  1.5026,  2.6911,  ...,  0.4452, -1.5716,  1.7501]]],
       device='cuda:1', grad_fn=<NativeDropoutBackward0>)
found 1771520 NaNs on memory. here is it:  tensor([[[nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         ...,
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan]],

        [[nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         ...,
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan]],

        [[nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         ...,
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan]],

        ...,

        [[nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         ...,
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan]],

        [[nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         ...,
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan]],

        [[nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         ...,
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan],
         [nan, nan, nan,  ..., nan, nan, nan]]], device='cuda:1',
       grad_fn=<NativeLayerNormBackward0>)
/home/unicconai/.local/lib/python3.10/site-packages/torch/autograd/__init__.py:200: UserWarning: Error detected in BinaryCrossEntropyWithLogitsBackward0. Traceback of forward call that caused the error:
  File "<string>", line 1, in <module>
  File "/usr/lib/python3.10/multiprocessing/spawn.py", line 116, in spawn_main
    exitcode = _main(fd, parent_sentinel)
  File "/usr/lib/python3.10/multiprocessing/spawn.py", line 129, in _main
    return self._bootstrap(parent_sentinel)
  File "/usr/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
    self.run()
  File "/usr/lib/python3.10/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 69, in _wrap
    fn(i, *args)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/multiprocessing.py", line 139, in _wrapping_function
    results = function(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 650, in _fit_impl
    self._run(model, ckpt_path=self.ckpt_path)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1112, in _run
    results = self._run_stage()
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1191, in _run_stage
    self._run_train()
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1214, in _run_train
    self.fit_loop.run()
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
    self.advance(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 267, in advance
    self._outputs = self.epoch_loop.run(self._data_fetcher)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
    self.advance(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/epoch/training_epoch_loop.py", line 213, in advance
    batch_output = self.batch_loop.run(kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
    self.advance(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/batch/training_batch_loop.py", line 88, in advance
    outputs = self.optimizer_loop.run(optimizers, kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
    self.advance(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 202, in advance
    result = self._run_optimization(kwargs, self._optimizers[self.optim_progress.optimizer_position])
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 249, in _run_optimization
    self._optimizer_step(optimizer, opt_idx, kwargs.get("batch_idx", 0), closure)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 370, in _optimizer_step
    self.trainer._call_lightning_module_hook(
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1356, in _call_lightning_module_hook
    output = fn(*args, **kwargs)
  File "/home/unicconai/DATAHDD01/Ground1/trainer.py", line 247, in optimizer_step
    super().optimizer_step(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/core/module.py", line 1742, in optimizer_step
    optimizer.step(closure=optimizer_closure)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/core/optimizer.py", line 169, in step
    step_output = self._strategy.optimizer_step(self._optimizer, self._optimizer_idx, closure, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 234, in optimizer_step
    return self.precision_plugin.optimizer_step(
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 119, in optimizer_step
    return optimizer.step(closure=closure, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/optim/optimizer.py", line 280, in wrapper
    out = func(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/optim/optimizer.py", line 33, in _use_grad
    ret = func(self, *args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/optim/adamw.py", line 148, in step
    loss = closure()
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 105, in _wrap_closure
    closure_result = closure()
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 149, in __call__
    self._result = self.closure(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 135, in closure
    step_output = self._step_fn()
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 419, in _training_step
    training_step_output = self.trainer._call_strategy_hook("training_step", *kwargs.values())
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1494, in _call_strategy_hook
    output = fn(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/strategies/ddp_spawn.py", line 280, in training_step
    return self.model(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1156, in forward
    output = self._run_ddp_forward(*inputs, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1110, in _run_ddp_forward
    return module_to_run(*inputs[0], **kwargs[0])  # type: ignore[index]
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/overrides/base.py", line 98, in forward
    output = self._forward_module.training_step(*inputs, **kwargs)
  File "/home/unicconai/DATAHDD01/Ground1/trainer.py", line 197, in training_step
    loss = self.loss_fn(pred,target)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/unicconai/DATAHDD01/Ground1/losses.py", line 22, in forward
    gate_loss = self.gate_loss_fn(gate_out.view(gate_out.shape[0],gate_out.shape[1]),gate_target.float())
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/nn/modules/loss.py", line 720, in forward
    return F.binary_cross_entropy_with_logits(input, target,
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/nn/functional.py", line 3165, in binary_cross_entropy_with_logits
    return torch.binary_cross_entropy_with_logits(input, target, weight, pos_weight, reduction_enum)
 (Triggered internally at ../torch/csrc/autograd/python_anomaly_mode.cpp:114.)
  Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
Traceback (most recent call last):
  File "/home/unicconai/DATAHDD01/Ground1/trainer.py", line 295, in <module>
    main()
  File "/home/unicconai/DATAHDD01/Ground1/trainer.py", line 291, in main
    trainer.fit(model,train_dataloaders=train_loader,val_dataloaders=test_loader)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 608, in fit
    call._call_and_handle_interrupt(
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 36, in _call_and_handle_interrupt
    return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/multiprocessing.py", line 113, in launch
    mp.start_processes(
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 197, in start_processes
    while not context.join():
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 160, in join
    raise ProcessRaisedException(msg, error_index, failed_process.pid)
torch.multiprocessing.spawn.ProcessRaisedException: 

-- Process 1 terminated with the following error:
Traceback (most recent call last):
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 69, in _wrap
    fn(i, *args)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/multiprocessing.py", line 139, in _wrapping_function
    results = function(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 650, in _fit_impl
    self._run(model, ckpt_path=self.ckpt_path)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1112, in _run
    results = self._run_stage()
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1191, in _run_stage
    self._run_train()
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1214, in _run_train
    self.fit_loop.run()
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
    self.advance(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 267, in advance
    self._outputs = self.epoch_loop.run(self._data_fetcher)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
    self.advance(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/epoch/training_epoch_loop.py", line 213, in advance
    batch_output = self.batch_loop.run(kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
    self.advance(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/batch/training_batch_loop.py", line 88, in advance
    outputs = self.optimizer_loop.run(optimizers, kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
    self.advance(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 202, in advance
    result = self._run_optimization(kwargs, self._optimizers[self.optim_progress.optimizer_position])
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 249, in _run_optimization
    self._optimizer_step(optimizer, opt_idx, kwargs.get("batch_idx", 0), closure)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 370, in _optimizer_step
    self.trainer._call_lightning_module_hook(
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1356, in _call_lightning_module_hook
    output = fn(*args, **kwargs)
  File "/home/unicconai/DATAHDD01/Ground1/trainer.py", line 247, in optimizer_step
    super().optimizer_step(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/core/module.py", line 1742, in optimizer_step
    optimizer.step(closure=optimizer_closure)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/core/optimizer.py", line 169, in step
    step_output = self._strategy.optimizer_step(self._optimizer, self._optimizer_idx, closure, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 234, in optimizer_step
    return self.precision_plugin.optimizer_step(
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 119, in optimizer_step
    return optimizer.step(closure=closure, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/optim/optimizer.py", line 280, in wrapper
    out = func(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/optim/optimizer.py", line 33, in _use_grad
    ret = func(self, *args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/optim/adamw.py", line 148, in step
    loss = closure()
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 105, in _wrap_closure
    closure_result = closure()
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 149, in __call__
    self._result = self.closure(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 144, in closure
    self._backward_fn(step_output.closure_loss)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 305, in backward_fn
    self.trainer._call_strategy_hook("backward", loss, optimizer, opt_idx)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1494, in _call_strategy_hook
    output = fn(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 207, in backward
    self.precision_plugin.backward(closure_loss, self.lightning_module, optimizer, optimizer_idx, *args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 67, in backward
    model.backward(tensor, optimizer, optimizer_idx, *args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/pytorch_lightning/core/module.py", line 1486, in backward
    loss.backward(*args, **kwargs)
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/_tensor.py", line 487, in backward
    torch.autograd.backward(
  File "/home/unicconai/.local/lib/python3.10/site-packages/torch/autograd/__init__.py", line 200, in backward
    Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
RuntimeError: Function 'BinaryCrossEntropyWithLogitsBackward0' returned nan values in its 0th output.

unicconai@unicconai-Z790-AORUS-ELITE-AX-DDR4:~/DATAHDD01/Ground1$ /usr/lib/python3.10/multiprocessing/resource_tracker.py:224: UserWarning: resource_tracker: There appear to be 53 leaked semaphore objects to clean up at shutdown
  warnings.warn('resource_tracker: There appear to be %d '
^C
unicconai@unicconai-Z790-AORUS-ELITE-AX-DDR4:~/DATAHDD01/Ground1$ 

it seems it started from PositiionalEncoding. and this is the PE implementation

class PositionalEncoding(nn.Module):
    
    def __init__(self, d_model, dropout, max_len=512):
        super(PositionalEncoding, self).__init__()
        self.dropout = nn.Dropout(p=dropout)

        # Compute the positional encodings once in log space.
        pe = torch.zeros(max_len, d_model)
        position = torch.arange(0, max_len).unsqueeze(1)
        div_term = torch.exp(torch.arange(0, d_model, 2) *
                             -(math.log(10000.0) / d_model))
        pe[:, 0::2] = torch.sin(position * div_term)
        pe[:, 1::2] = torch.cos(position * div_term)
        pe = pe.unsqueeze(0)
        self.register_buffer('pe', pe)

    def forward(self, x):
        x = x + Variable(self.pe[:, :x.size(1)],
                         requires_grad=False)
        return self.dropout(x)

do you have any idea?