list_data_collate differences between MONAI 0.4.0 and 0.5.0

[KSchmidtACR]

Hello,

I have a script that uses MONAI to train a resnet 50 model on numpy array data. This script runs with MONAI 0.4.0, but throws a default_collate error if run with MOANI 0.5.0+. I’ve verified that all data are indeed numpy arrays of the same size, are not empty, and do not contain NaN values. I've been trying to figure out how to make my script compatible with the more recent versions of MONAI, but have not yet been successful. Any help would be greatly appreciated. Here is my script (dataset.json contains file paths and labels for training and validation data):

import torch
import torchvision
from torch.utils import data
from torch.utils.data import DataLoader
import os
import sys
import numpy as np
import json
import numpy as np
import random

from monai.transforms import (
    Compose,
    LoadImaged,
    AsChannelFirstd,
    RandRotated,
    RandFlipd,
    ToNumpyd,
    CastToTyped,
    ToTensord,
    AsDiscreted,
    Activationsd,
    DataStatsd,
    SqueezeDimd
)

from monai.data import CacheDataset, DataLoader, list_data_collate
from monai.networks.layers import Norm
from monai.handlers import (
    StatsHandler,
    TensorBoardStatsHandler,
    TensorBoardImageHandler,
    ValidationHandler,
    LrScheduleHandler,
    CheckpointSaver
)
import monai
from monai.engines import SupervisedTrainer, SupervisedEvaluator
from monai.inferers import SimpleInferer
from ignite.metrics import Accuracy
import logging
import warnings
warnings.filterwarnings("ignore")

from monai.handlers import stopping_fn_from_metric
from ignite.engine import Events
from ignite.handlers import EarlyStopping

import torch
from torch import Tensor
import torch.nn as nn


net = torchvision.models.resnet50()
num_ftrs = net.fc.in_features
net.fc = torch.nn.Linear(num_ftrs, 4)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
net.to(device)

f = open('/home/dataset.json')
data = json.load(f)

training_data = data["training"]
validation_data = data["validation"]

train_transforms = Compose([
            LoadImaged(keys=["image"]),
            AsChannelFirstd(keys=["image"]),
            RandRotated(keys=["image"], prob=1, range_y=45, keep_size=True, mode="nearest"),
            RandFlipd(keys=["image"],spatial_axis=0, prob=0.5),
            RandFlipd(keys=["image"],spatial_axis=1, prob=0.5),
            ToTensord(keys=["image", "label"]),
            DataStatsd(keys=["image"]),
            SqueezeDimd(keys="label", dim=0)])

val_transforms = Compose([
            LoadImaged(keys=["image"]),
            AsChannelFirstd(keys=["image"]),
            ToTensord(keys=["image", "label"]),
            SqueezeDimd(keys="label", dim=0)])

train_ds = CacheDataset(data=training_data, transform=train_transforms)
val_ds = CacheDataset(data=validation_data, transform=val_transforms)
train_loader = DataLoader(train_ds, collate_fn=list_data_collate, batch_size=32, shuffle=True, num_workers=5)
val_loader = DataLoader(val_ds, collate_fn=list_data_collate, batch_size=64, shuffle=True, num_workers=5)

loss = torch.nn.CrossEntropyLoss()
lr = 1e-6
opt = torch.optim.Adam(net.parameters(), lr)
lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(opt, mode='min', factor=0.1, patience=10, verbose=True)


post_transforms = Compose([
        AsDiscreted(keys="pred", argmax=True, to_onehot=True, n_classes=4),
        DataStatsd(keys="label")])


post_transforms = Compose([
        AsDiscreted(keys="pred", argmax=True),
        DataStatsd(keys="label")])

val_handlers = [
            StatsHandler(output_transform=lambda x: None),
            TensorBoardStatsHandler(log_dir="./runs/", output_transform=lambda x: None),
            TensorBoardImageHandler(log_dir="./runs/", batch_transform=lambda x: (x["image"], x["label"]), output_transform=lambda x: x["pred"])
        ]

evaluator = SupervisedEvaluator(
            device=device,
            val_data_loader=val_loader,
            network=net,
            inferer=SimpleInferer(),
            key_val_metric={"val_acc": Accuracy(output_transform=lambda x: (x["pred"], x["label"]))},
            val_handlers=val_handlers
        )

train_handlers = [
            ValidationHandler(validator=evaluator, interval=1, epoch_level=True),
            StatsHandler(tag_name="train_loss", output_transform=lambda x: x["loss"]),
            TensorBoardStatsHandler(log_dir="./runs/", tag_name="train_loss", output_transform=lambda x: x["loss"]),
            CheckpointSaver(save_dir='/home/kschmidt/breast_density_monai_testing', save_dict={"model": net, "optimizer": opt, "lr_scheduler": lr_scheduler}, save_key_metric=True, save_final=True)
        ]


trainer = SupervisedTrainer(
            device=device,
            max_epochs=5,
            train_data_loader=train_loader,
            network=net,
            optimizer=opt,
            loss_function=loss,
            inferer=SimpleInferer(),
            amp=False,
            post_transform = post_transforms,
            key_train_metric={"train_acc": Accuracy(output_transform=lambda x: (x["pred"], x["label"]))},
            train_handlers=train_handlers,
        )

trainer.run()

And the error:

Original Traceback (most recent call last):
  File "/home/kschmidt/miniconda3/envs/test/lib/python3.9/site-packages/monai/data/utils.py", line 264, in list_data_collate
    ret[k] = default_collate([d[k] for d in data])
  File "/home/kschmidt/miniconda3/envs/test/lib/python3.9/site-packages/torch/utils/data/_utils/collate.py", line 83, in default_collate
    return [default_collate(samples) for samples in transposed]
  File "/home/kschmidt/miniconda3/envs/test/lib/python3.9/site-packages/torch/utils/data/_utils/collate.py", line 83, in <listcomp>
    return [default_collate(samples) for samples in transposed]
  File "/home/kschmidt/miniconda3/envs/test/lib/python3.9/site-packages/torch/utils/data/_utils/collate.py", line 73, in default_collate
    return {key: default_collate([d[key] for d in batch]) for key in elem}
  File "/home/kschmidt/miniconda3/envs/test/lib/python3.9/site-packages/torch/utils/data/_utils/collate.py", line 73, in <dictcomp>
    return {key: default_collate([d[key] for d in batch]) for key in elem}
  File "/home/kschmidt/miniconda3/envs/test/lib/python3.9/site-packages/torch/utils/data/_utils/collate.py", line 85, in default_collate
    raise TypeError(default_collate_err_msg_format.format(elem_type))
TypeError: default_collate: batch must contain tensors, numpy arrays, numbers, dicts or lists; found <class 'NoneType'>

[rijobro]

So it's complaining that something in the data is None. The question is what (and why it didn't previously throw an error).

My initial guess is that it's probably in the metadata when the image is loaded. To test, could you try using LoadImaged(..., image_only=True) and see what that does? You could also remove as many transformations as possible, and when you've got it working, add them back in.

You could also disable collation: collate_fn: lambda x: x, and then inspect train_loader and val_loader for any None values.

[KSchmidtACR]

I've tested my script with piecemeal additions of transforms. LoadImaged, RandRotated, and RandFlipd function without a default_collate error (though there is and error thrown eventually because data is not the correct shape without using AsChannelFirstd). The issue appears to be with ToTensord but the orig_size value doesn't seem to be generated in the ToTensord or ToTensor functions. Do you know where this is generated?

Also, using AsChannelFirstd adds ToTensord to image_transforms and label_transforms (even if it is not explicitly defined as a transform) and then causes the original error. Looking at the AsChannelFirstd function I can't understand why this happens.

I looked into MONAI 0.4.0 more closely too. The reason I don't get an error using that version is because only image, label, and image_meta_dict (not image_transforms and label_transforms) are passed from list_data_collate to default_collate.

Deleting image_transforms and label_transfroms with DeleteItemsd as suggested throws this error: KeyError: 'Key was missing (label_transforms) and allow_missing_keys==False' and changing allow_missing_keys==True resulted in the original error.

I don't see how the inverse functionality could help here. Doesn't the inverse functionality revert the data back to it's initial state? I'm sure I'm misunderstanding something.

[rijobro]

Hm, very strange. If you could create me a minimum working example, I could take it from there.

You can create synthetic data with something like:

im_fname, seg_fname = [make_nifti_image(i) for i in create_test_image_3d(101, 100, 107)]
data = [{"img": img_fname, "seg": seg_fname} for _ in range(10)]

and then i guess you just need to create transforms, dataset and dataloader and try iterating over the data loader:

transforms = ...
ds = Dataset(data, transforms)
dl = DataLoader(...)
for i in dl:
    pass

[KSchmidtACR]

This script reproduces the error:

import numpy as np
import random
import os

from monai.transforms import (
    Compose,
    LoadImaged,
    AsChannelFirstd,
    RandRotated,
    RandFlipd,
    ToTensord,
    SqueezeDimd,
    AsDiscreted)

from monai.data import CacheDataset, DataLoader
from monai.handlers import StatsHandler, ValidationHandler
from monai.engines import SupervisedTrainer, SupervisedEvaluator
from monai.inferers import SimpleInferer

import torch
import torchvision
from ignite.metrics import Accuracy

current_dir = os.getcwd()
np_dir = os.path.join(current_dir, 'fake_nps')
if not os.path.exists(np_dir):
    os.mkdir(np_dir)

train_list = []

for i in range(10):
    np_img = np.random.rand(224,224,3)
    np.save(os.path.join(np_dir, 'img_' + str(i) + '.npy'), np_img)
    train_list.append(os.path.join(np_dir, 'img_' + str(i) + '.npy')) 

train_data = [{'image': img, 'label': random.randrange(0, 4, 1)} for img in train_list[:6]]
val_data = [{'image': img, 'label': random.randrange(0, 4, 1)} for img in train_list[6:]]

train_transforms = Compose([
            LoadImaged(keys=["image"]),
            AsChannelFirstd(keys=["image"]),
            ToTensord(keys=["image", "label"])])

val_transforms = Compose([
            LoadImaged(keys=["image"]),
            AsChannelFirstd(keys=["image"]),
            ToTensord(keys=["image", "label"])])

train_ds = CacheDataset(data=train_data, transform=train_transforms)
val_ds = CacheDataset(data=val_data, transform=val_transforms)
train_loader = DataLoader(train_ds, batch_size=2, shuffle=True, num_workers=5)
val_loader = DataLoader(val_ds, batch_size=2, shuffle=True, num_workers=5)

net = torchvision.models.resnet50()
num_ftrs = net.fc.in_features
net.fc = torch.nn.Linear(num_ftrs, 4)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
net.to(device)

loss = torch.nn.CrossEntropyLoss()
opt = torch.optim.Adam(net.parameters(), 1e-6)


post_transforms = Compose([AsDiscreted(keys="pred", argmax=True, to_onehot=True, n_classes=4)])


val_handlers = [
            StatsHandler(output_transform=lambda x: None)]
        
evaluator = SupervisedEvaluator(
            device=device,
            val_data_loader=val_loader,
            network=net,
            inferer=SimpleInferer(),
            key_val_metric={"val_acc": Accuracy(output_transform=lambda x: (x["pred"], x["label"]))},
            val_handlers=val_handlers
        )

train_handlers = [
            ValidationHandler(validator=evaluator, interval=1, epoch_level=True),
            StatsHandler(tag_name="train_loss", output_transform=lambda x: x["loss"])
        ]


trainer = SupervisedTrainer(
            device=device,
            max_epochs=2,
            train_data_loader=train_loader,
            network=net,
            optimizer=opt,
            loss_function=loss,
            inferer=SimpleInferer(),
            amp=False,
            post_transform = post_transforms,
            key_train_metric={"train_acc": Accuracy(output_transform=lambda x: (x["pred"], x["label"]))},
            train_handlers=train_handlers,
        )

trainer.run()

[rijobro]

@KSchmidtACR I found the bug and have created a PR. Hopefully the tests will be green, feel free to try it in the meantime or wait for it to be merged.

Superfluous info for an end user, but just in case you were interested, the reason is below...

The problem was that every time an invertible transform is applied, the original size is stored. If the object didn't have the attribute shape (so not a torch.tensor or numpy.array), then default to None. I've changed it to now not default to anything, i.e., don't add orig_size if the object doesn't have the attribute shape.

[KSchmidtACR]

Perfect, thank you for your help!