diff --git a/CE.py b/CE.py new file mode 100644 index 0000000..c7534b1 --- /dev/null +++ b/CE.py @@ -0,0 +1,215 @@ +import numpy as np +from torchvision.datasets import MNIST +from torchvision import transforms +from datasets import UnbalancedMNIST, BalancedBatchSampler +from networks import EmbeddingNet, ClassificationNet,ResNetEmbeddingNet +from metrics import AccumulatedAccuracyMetric,AverageNonzeroTripletsMetric +from skinDatasetFolder import skinDatasetFolder +from covidDataSetFolder import CovidDataset +from losses import OnlineTripletLoss,OnlineContrastiveLoss +from utils import AllTripletSelector,HardestNegativeTripletSelector, RandomNegativeTripletSelector, SemihardNegativeTripletSelector # Strategies for selecting triplets within a minibatch +from utils import BatchHardTripletSelector,AllPositivePairSelector, HardNegativePairSelector # Strategies for selecting pairs within a minibatch +from trainer import fit + +import torch +from torch.optim import lr_scheduler +import torch.optim as optim + + +import numpy as np +import matplotlib +import matplotlib.pyplot as plt +import argparse +import os +def str2bool(v): + """Convert string to Boolean + + Args: + v: True or False but in string + + Returns: + True or False in Boolean + + Raises: + TyreError + """ + + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') + + +parser = argparse.ArgumentParser(description='Triplet For MNIST') +parser.add_argument('--dataset_name',default='covid19', + help='Choose dataset [...]') +parser.add_argument('--rescale',default=False,type=str2bool, + help='rescale dataset') +parser.add_argument('--iterNo',default=1,type=int, + help='Use for choosing fold validation') +parser.add_argument('--cuda_device',default=0,type=int, + help='Choose cuda_device:(0,1,2,3,4,5,6,7)') +parser.add_argument('--EmbeddingMode',default=False,type = str2bool , + help='True for tripletsLoss(embedding) / False for EntropyLoss(classfication)') +parser.add_argument('--dim',default=128,type=int, + help='The dimension of embedding(type int)') +parser.add_argument('--n_classes',default=3,type=int, + help='The number of classes (type int)') +parser.add_argument('--margin',default=0.5,type=float, + help='Margin used in triplet loss (type float)') +parser.add_argument('--logdir',default='result', + help='Path to log tensorboard, pick a UNIQUE name to log') +parser.add_argument('--start_epoch',default=0,type=int + ,help='Start epoch (int)') +parser.add_argument('--n_epoch',default=200,type=int, + help='End_epoch (int)') +parser.add_argument('--batch_size',default=16,type=int, + help='Batch size (int)') +parser.add_argument('--n_sample_classes',default=3,type=int, + help='For a batch sampler to work comine #samples_per_class') +parser.add_argument('--n_samples_per_class',default=10,type=int, + help='For a batch sampler to work comine #n_sample_classes') +parser.add_argument('--TripletSelector',default='SemihardNegativeTripletSelector', + help='Triplet selector chosen in ({},{},{},{},{})' + .format('AllTripletSelector', + 'HardestNegativeTripletSelector', + 'RandomNegativeTripletSelector', + 'SemihardNegativeTripletSelector', + 'BatchHardTripletSelector')) +args = parser.parse_args() + + + +def extract_embeddings(dataloader, model, dimension): + with torch.no_grad(): + model.eval() + embeddings = np.zeros((len(dataloader.dataset), dimension))#num_of_dim + labels = np.zeros(len(dataloader.dataset)) + k = 0 + for images, target in dataloader: + if cuda: + images = images.cuda() + embeddings[k:k+len(images)] = model.get_embedding(images).data.cpu().numpy() + labels[k:k+len(images)] = target.numpy() + k += len(images) + return embeddings, labels + + +if __name__ == '__main__': + print(args) + + torch.cuda.set_device(args.cuda_device) + logdir = args.logdir + + dataset_name = args.dataset_name + + Attr_Dict = { + 'covid19':{'in_channel':1, + 'n_classes':3, + 'train_dataset' : CovidDataset(iterNo=args.iterNo,train=True), + 'test_dataset' : CovidDataset(iterNo=args.iterNo,train=False), + 'resDir':'./covid19Res/iterNo{}'.format(args.iterNo) + } + } + + num_of_dim = args.dim + n_classes = Attr_Dict[dataset_name]['n_classes'] + train_dataset = Attr_Dict[dataset_name]['train_dataset'] + test_dataset = Attr_Dict[dataset_name]['test_dataset'] + + n_sample_classes = args.n_sample_classes + n_samples_per_class = args.n_samples_per_class + train_batch_sampler = BalancedBatchSampler(train_dataset, n_classes=n_sample_classes, n_samples=n_samples_per_class) + test_batch_sampler = BalancedBatchSampler(test_dataset, n_classes=n_sample_classes, n_samples=n_samples_per_class) + + cuda = torch.cuda.is_available() + + kwargs = {'num_workers': 40, 'pin_memory': True} if cuda else {} + batch_size = args.batch_size + train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, **kwargs) + test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, **kwargs) + + online_train_loader = torch.utils.data.DataLoader(train_dataset, batch_sampler=train_batch_sampler, **kwargs) + online_test_loader = torch.utils.data.DataLoader(test_dataset, batch_sampler=test_batch_sampler, **kwargs) + + + start_epoch = args.start_epoch + n_epochs = args.n_epoch + log_interval = 50 + margin = args.margin + + Selector = { + 'AllTripletSelector':AllTripletSelector(), + 'HardestNegativeTripletSelector':HardestNegativeTripletSelector(margin), + 'RandomNegativeTripletSelector':RandomNegativeTripletSelector(margin), + 'SemihardNegativeTripletSelector':SemihardNegativeTripletSelector(margin), + 'BatchHardTripletSelector':BatchHardTripletSelector(margin) + } + + embedding_net = ResNetEmbeddingNet(dataset_name,num_of_dim) + classification_net = ClassificationNet(embedding_net, dimension = num_of_dim ,n_classes = n_classes) + + if args.EmbeddingMode: + loader1 = online_train_loader + loader2 = online_test_loader + model = embedding_net + loss_fn = OnlineTripletLoss(margin, Selector[args.TripletSelector]) + lr = 1e-4 + # optimizer = optim.Adam(model.parameters(), lr=lr) + optimizer = optim.Adam( + model.parameters(), + lr=lr, + betas=(0.9, 0.99), + eps=1e-8, + amsgrad=True) + scheduler = lr_scheduler.StepLR(optimizer, 50, gamma=0.1, last_epoch=-1) + metrics = [AverageNonzeroTripletsMetric()] + logName = 'margin{}_{}d-embedding_{}'.format(margin,num_of_dim,args.TripletSelector) + logName = os.path.join(Attr_Dict[dataset_name]['resDir'],logName) + EmbeddingArgs = (num_of_dim,train_loader,test_loader) + + else: + loader1 = train_loader + loader2 = test_loader + model = classification_net + loss_fn = torch.nn.CrossEntropyLoss() + lr = 1e-4 + # optimizer = optim.Adam(model.parameters(), lr=lr) + optimizer = optim.Adam( + model.parameters(), + lr=lr, + betas=(0.9, 0.99), + eps=1e-8, + amsgrad=True) + scheduler = lr_scheduler.StepLR(optimizer, 100, gamma=0.1, last_epoch=-1) + metrics = [AccumulatedAccuracyMetric()] + logName = '{}d-CE-no_class_weights'.format(num_of_dim) + logName = os.path.join(Attr_Dict[dataset_name]['resDir'],logName) + EmbeddingArgs = () + + + + + + if cuda: + model.cuda() + + logfile = os.path.join(logdir,logName) + fit(dataset_name, + logfile, + loader1, + loader2, + model, + loss_fn, + optimizer, + scheduler, + n_epochs, + cuda, + log_interval, + metrics, + start_epoch, + *EmbeddingArgs) + + diff --git a/CenterLoss.py b/CenterLoss.py new file mode 100644 index 0000000..b4dd727 --- /dev/null +++ b/CenterLoss.py @@ -0,0 +1,68 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import numpy as np + +def calc_centers(embeddings,targets,n_classes): + centers = torch.Tensor([]).cuda() + for lbl in range(n_classes): + mask = targets.eq(lbl) + embeddings_ = embeddings[mask] + center = embeddings_.mean(dim=0) + centers = torch.cat([centers,center.unsqueeze(dim=0)]) + assert centers.shape == (n_classes,embeddings.size()[1]) + return centers + +def diversity_regularizer(centers,n_classes): + c_j = torch.Tensor([]).cuda() + c_k = torch.Tensor([]).cuda()# j < k + labelSet = torch.arange(n_classes).cuda() + for lbl in range(n_classes): + mask = labelSet.eq(lbl) + gt_mask = labelSet.gt(lbl) + repeat_n = torch.sum(gt_mask) + if repeat_n > 0: + c_j = torch.cat([c_j,centers[mask].repeat(repeat_n,1)]) + c_k = torch.cat([c_k,centers[gt_mask]]) + + assert c_j.size() == c_k.size() + mu = (c_j - c_k).pow(2).sum(1).mean() + R_w = ((c_j - c_k).pow(2).sum(1) - mu).pow(2).mean() + + return R_w + +class CenterLoss(nn.Module): + def __init__(self,lambd,n_classes): + super(CenterLoss,self).__init__() + self.lambd = lambd + self.n_classes = n_classes + + def forward(self,embeddings,targets,centers): + repeat_n = self.n_classes - 1 + labelSet = torch.arange(self.n_classes).cuda() + + center_mat = torch.Tensor([]).cuda() + exc_center_mat = torch.Tensor([]).cuda() + + data_mat = torch.Tensor(embeddings.cpu().data.numpy().repeat(repeat_n,axis=0)).cuda() + for i in range(embeddings.size()[0]): + lbl = targets[i] + exc_center_mask = labelSet.ne(lbl) + center_mask = labelSet.eq(lbl) + center_mat = torch.cat([center_mat,centers[center_mask].repeat(repeat_n,1)]) + exc_center_mat = torch.cat([exc_center_mat,centers[exc_center_mask]]) + + #print('data:{},center:{},excenter:{}'.format(data_mat.size(),center_mat.size(),exc_center_mat.size())) + assert center_mat.size() == exc_center_mat.size() + assert center_mat.size() == data_mat.size() + + dis_intra = (data_mat - center_mat).pow(2).sum(1) + dis_inter = (data_mat - exc_center_mat).pow(2).sum(1) + L_mm = F.relu(self.lambd + dis_intra - dis_inter).mean() + #R_w = diversity_regularizer(centers,self.n_classes) + loss = L_mm + + return loss + + + diff --git a/FL.py b/FL.py new file mode 100644 index 0000000..f354b53 --- /dev/null +++ b/FL.py @@ -0,0 +1,220 @@ +import numpy as np +from torchvision.datasets import MNIST +from torchvision import transforms +from datasets import UnbalancedMNIST, BalancedBatchSampler +from networks import EmbeddingNet, ClassificationNet,ResNetEmbeddingNet +from metrics import AccumulatedAccuracyMetric,AverageNonzeroTripletsMetric +from skinDatasetFolder import skinDatasetFolder +from covidDataSetFolder import CovidDataset +from focalloss2d import FocalLoss +from losses import OnlineTripletLoss,OnlineContrastiveLoss +from utils import AllTripletSelector,HardestNegativeTripletSelector, RandomNegativeTripletSelector, SemihardNegativeTripletSelector # Strategies for selecting triplets within a minibatch +from utils import BatchHardTripletSelector,AllPositivePairSelector, HardNegativePairSelector # Strategies for selecting pairs within a minibatch +from trainer import fit + +import torch +from torch.optim import lr_scheduler +import torch.optim as optim + + +import numpy as np +import matplotlib +import matplotlib.pyplot as plt +import argparse +import os +def str2bool(v): + """Convert string to Boolean + + Args: + v: True or False but in string + + Returns: + True or False in Boolean + + Raises: + TyreError + """ + + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') + + +parser = argparse.ArgumentParser(description='Triplet For MNIST') +parser.add_argument('--dataset_name',default='covid19', + help='Choose dataset [...]') +parser.add_argument('--rescale',default=False,type=str2bool, + help='rescale dataset') +parser.add_argument('--iterNo',default=1,type=int, + help='Use for choosing fold validation') +parser.add_argument('--cuda_device',default=0,type=int, + help='Choose cuda_device:(0,1,2,3,4,5,6,7)') +parser.add_argument('--EmbeddingMode',default=False,type = str2bool , + help='True for tripletsLoss(embedding) / False for EntropyLoss(classfication)') +parser.add_argument('--dim',default=128,type=int, + help='The dimension of embedding(type int)') +parser.add_argument('--n_classes',default=3,type=int, + help='The number of classes (type int)') +parser.add_argument('--margin',default=0.5,type=float, + help='Margin used in triplet loss (type float)') +parser.add_argument('--logdir',default='result', + help='Path to log tensorboard, pick a UNIQUE name to log') +parser.add_argument('--start_epoch',default=0,type=int + ,help='Start epoch (int)') +parser.add_argument('--n_epoch',default=200,type=int, + help='End_epoch (int)') +parser.add_argument('--batch_size',default=16,type=int, + help='Batch size (int)') +parser.add_argument('--n_sample_classes',default=3,type=int, + help='For a batch sampler to work comine #samples_per_class') +parser.add_argument('--n_samples_per_class',default=10,type=int, + help='For a batch sampler to work comine #n_sample_classes') +parser.add_argument('--TripletSelector',default='SemihardNegativeTripletSelector', + help='Triplet selector chosen in ({},{},{},{},{})' + .format('AllTripletSelector', + 'HardestNegativeTripletSelector', + 'RandomNegativeTripletSelector', + 'SemihardNegativeTripletSelector', + 'BatchHardTripletSelector')) +args = parser.parse_args() + + + +def extract_embeddings(dataloader, model, dimension): + with torch.no_grad(): + model.eval() + embeddings = np.zeros((len(dataloader.dataset), dimension))#num_of_dim + labels = np.zeros(len(dataloader.dataset)) + k = 0 + for images, target in dataloader: + if cuda: + images = images.cuda() + embeddings[k:k+len(images)] = model.get_embedding(images).data.cpu().numpy() + labels[k:k+len(images)] = target.numpy() + k += len(images) + return embeddings, labels + + +if __name__ == '__main__': + print(args) + + torch.cuda.set_device(args.cuda_device) + logdir = args.logdir + + dataset_name = args.dataset_name + + Attr_Dict = { + 'covid19':{'in_channel':1, + 'n_classes':3, + 'train_dataset' : CovidDataset(iterNo=args.iterNo,train=True), + 'test_dataset' : CovidDataset(iterNo=args.iterNo,train=False), + 'resDir':'./covid19Res/iterNo{}'.format(args.iterNo) + } + } + + num_of_dim = args.dim + n_classes = Attr_Dict[dataset_name]['n_classes'] + train_dataset = Attr_Dict[dataset_name]['train_dataset'] + test_dataset = Attr_Dict[dataset_name]['test_dataset'] + + n_sample_classes = args.n_sample_classes + n_samples_per_class = args.n_samples_per_class + train_batch_sampler = BalancedBatchSampler(train_dataset, n_classes=n_sample_classes, n_samples=n_samples_per_class) + test_batch_sampler = BalancedBatchSampler(test_dataset, n_classes=n_sample_classes, n_samples=n_samples_per_class) + + cuda = torch.cuda.is_available() + + kwargs = {'num_workers': 40, 'pin_memory': True} if cuda else {} + batch_size = args.batch_size + train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, **kwargs) + test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, **kwargs) + + online_train_loader = torch.utils.data.DataLoader(train_dataset, batch_sampler=train_batch_sampler, **kwargs) + online_test_loader = torch.utils.data.DataLoader(test_dataset, batch_sampler=test_batch_sampler, **kwargs) + + + start_epoch = args.start_epoch + n_epochs = args.n_epoch + log_interval = 50 + margin = args.margin + + Selector = { + 'AllTripletSelector':AllTripletSelector(), + 'HardestNegativeTripletSelector':HardestNegativeTripletSelector(margin), + 'RandomNegativeTripletSelector':RandomNegativeTripletSelector(margin), + 'SemihardNegativeTripletSelector':SemihardNegativeTripletSelector(margin), + 'BatchHardTripletSelector':BatchHardTripletSelector(margin) + } + + embedding_net = ResNetEmbeddingNet(dataset_name,num_of_dim) + classification_net = ClassificationNet(embedding_net, dimension = num_of_dim ,n_classes = n_classes) + + if args.EmbeddingMode: + loader1 = online_train_loader + loader2 = online_test_loader + model = embedding_net + loss_fn = OnlineTripletLoss(margin, Selector[args.TripletSelector]) + lr = 1e-4 + # optimizer = optim.Adam(model.parameters(), lr=lr) + optimizer = optim.Adam( + model.parameters(), + lr=lr, + betas=(0.9, 0.99), + eps=1e-8, + amsgrad=True) + scheduler = lr_scheduler.StepLR(optimizer, 50, gamma=0.1, last_epoch=-1) + metrics = [AverageNonzeroTripletsMetric()] + logName = 'margin{}_{}d-embedding_{}'.format(margin,num_of_dim,args.TripletSelector) + logName = os.path.join(Attr_Dict[dataset_name]['resDir'],logName) + EmbeddingArgs = (num_of_dim,train_loader,test_loader) + + else: + loader1 = train_loader + loader2 = test_loader + model = classification_net + loss_fn = FocalLoss(class_num = n_classes, gamma = 2) + # if dataset_name == 'skin': + # loss_fn = torch.nn.CrossEntropyLoss(torch.FloatTensor([0.036,0.002,0.084,0.134,0.037,0.391,0.316]).cuda()) + lr = 1e-4 + # optimizer = optim.Adam(model.parameters(), lr=lr) + optimizer = optim.Adam( + model.parameters(), + lr=lr, + betas=(0.9, 0.99), + eps=1e-8, + amsgrad=True) + scheduler = lr_scheduler.StepLR(optimizer, 100, gamma=0.1, last_epoch=-1) + metrics = [AccumulatedAccuracyMetric()] + logName = '{}d-focalloss'.format(num_of_dim) + logName = os.path.join(Attr_Dict[dataset_name]['resDir'],logName) + EmbeddingArgs = () + + + + + if cuda: + model.cuda() + + logfile = os.path.join(logdir,logName) + fit(dataset_name, + logfile, + loader1, + loader2, + model, + loss_fn, + optimizer, + scheduler, + n_epochs, + cuda, + log_interval, + metrics, + start_epoch, + *EmbeddingArgs) + + + colors = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', + '#9467bd', '#8c564b', '#e377c2', '#7f7f7f', + '#bcbd22', '#17becf'] diff --git a/Pneumonia.py b/Pneumonia.py new file mode 100644 index 0000000..d535c26 --- /dev/null +++ b/Pneumonia.py @@ -0,0 +1,106 @@ +import os +import sys +import torch +import pandas as pd +import numpy as np +from PIL import Image +from torchvision import transforms +from torch.utils.data import Dataset +import SimpleITK as sitk + + +class PneumoniaDataset(Dataset): + def __init__(self, root, iterNo = 1,train = True, transform = None): + self.root = os.path.join(root, "pneumonia_data") + if transform is not None: + self.transform = transform + else: + self.transform = transforms.Compose([ + transforms.Resize((224,224)), + transforms.ToTensor(), + transforms.Normalize(mean= [0.4833, 0.4833, 0.4833] ,std=[0.2480, 0.2480, 0.2480] ) + ]) + self.train = train + + self.image_data_dir = os.path.join(self.root, 'stage_2_train_images') + self.imgs_path, self.targets = \ + self.get_data(iterNo, os.path.join(self.root, 'split_data')) + if self.train: + self.train_labels = torch.LongTensor(self.targets) + else: + self.test_labels = torch.LongTensor(self.targets) + + + + self.loader = dcm_loader + classes_name = ['Normal', 'Lung Opacity', '‘No Lung Opacity/Not Normal'] + self.classes = list(range(len(classes_name))) + self.target_img_dict = dict() + targets = np.array(self.targets) + for target in self.classes: + indexes = np.nonzero(targets == target)[0] + self.target_img_dict.update({target: indexes}) + + def __getitem__(self, index): + """ + Args: + index (int): Index + Returns: + tuple: (sample, target) where target is class_index of the target class. + """ + path = self.imgs_path[index] + target = self.targets[index] + img = self.loader(path) + if self.transform is not None: + img = self.transform(img) + return img, target + + def __len__(self): + return len(self.targets) + + def get_data(self, iterNo, data_dir): + + if self.train: + csv = 'pneumonia_split_{}_train.csv'.format(iterNo) + else: + csv = 'pneumonia_split_{}_test.csv'.format(iterNo) + + fn = os.path.join(data_dir, csv) + csvfile = pd.read_csv(fn, index_col=0) + raw_data = csvfile.values + + data = [] + targets = [] + for (path, label) in raw_data: + data.append(os.path.join(self.image_data_dir, path)) + targets.append(label) + + return data, targets + + +def dcm_loader(path): + ds = sitk.ReadImage(path) + img_array = sitk.GetArrayFromImage(ds) + img_bitmap = Image.fromarray(img_array[0]).convert('RGB') + return img_bitmap + +def print_dataset(dataset, print_time): + print(len(dataset)) + from collections import Counter + counter = Counter() + labels = [] + for index, (img, label) in enumerate(dataset): + if index % print_time == 0: + print(img.size(), label) + labels.append(label) + counter.update(labels) + print(counter) + + +if __name__ == "__main__": + root = "../data" + dataset = PneumoniaDataset(root=root, iterNo=5,train=True) + print_dataset(dataset, print_time=10000) + + dataset = PneumoniaDataset(root=root, iterNo=5,train=False) + print_dataset(dataset, print_time=1000) \ No newline at end of file diff --git a/WFL.py b/WFL.py new file mode 100644 index 0000000..b997dac --- /dev/null +++ b/WFL.py @@ -0,0 +1,212 @@ +import numpy as np +from torchvision.datasets import MNIST +from torchvision import transforms +from datasets import UnbalancedMNIST, BalancedBatchSampler +from networks import EmbeddingNet, ClassificationNet,ResNetEmbeddingNet +from metrics import AccumulatedAccuracyMetric,AverageNonzeroTripletsMetric +from skinDatasetFolder import skinDatasetFolder +from covidDataSetFolder import CovidDataset +from focalloss2d import FocalLoss +from losses import OnlineTripletLoss,OnlineContrastiveLoss +from utils import AllTripletSelector,HardestNegativeTripletSelector, RandomNegativeTripletSelector, SemihardNegativeTripletSelector # Strategies for selecting triplets within a minibatch +from utils import BatchHardTripletSelector,AllPositivePairSelector, HardNegativePairSelector # Strategies for selecting pairs within a minibatch +from trainer import fit +import torch +from torch.optim import lr_scheduler +import torch.optim as optim + + +import numpy as np +import matplotlib +import matplotlib.pyplot as plt +import argparse +import os +def str2bool(v): + """Convert string to Boolean + + Args: + v: True or False but in string + + Returns: + True or False in Boolean + + Raises: + TyreError + """ + + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') + + +parser = argparse.ArgumentParser(description='Triplet For MNIST') +parser.add_argument('--dataset_name',default='covid19', + help='Choose dataset [...]') +parser.add_argument('--rescale',default=False,type=str2bool, + help='rescale dataset') +parser.add_argument('--iterNo',default=1,type=int, + help='Use for choosing fold validation') +parser.add_argument('--cuda_device',default=0,type=int, + help='Choose cuda_device:(0,1,2,3,4,5,6,7)') +parser.add_argument('--EmbeddingMode',default=False,type = str2bool , + help='True for tripletsLoss(embedding) / False for EntropyLoss(classfication)') +parser.add_argument('--dim',default=128,type=int, + help='The dimension of embedding(type int)') +parser.add_argument('--n_classes',default=3,type=int, + help='The number of classes (type int)') +parser.add_argument('--margin',default=0.5,type=float, + help='Margin used in triplet loss (type float)') +parser.add_argument('--logdir',default='result', + help='Path to log tensorboard, pick a UNIQUE name to log') +parser.add_argument('--start_epoch',default=0,type=int + ,help='Start epoch (int)') +parser.add_argument('--n_epoch',default=200,type=int, + help='End_epoch (int)') +parser.add_argument('--batch_size',default=16,type=int, + help='Batch size (int)') +parser.add_argument('--n_sample_classes',default=3,type=int, + help='For a batch sampler to work comine #samples_per_class') +parser.add_argument('--n_samples_per_class',default=10,type=int, + help='For a batch sampler to work comine #n_sample_classes') +parser.add_argument('--TripletSelector',default='SemihardNegativeTripletSelector', + help='Triplet selector chosen in ({},{},{},{},{})' + .format('AllTripletSelector', + 'HardestNegativeTripletSelector', + 'RandomNegativeTripletSelector', + 'SemihardNegativeTripletSelector', + 'BatchHardTripletSelector')) +args = parser.parse_args() + + + +def extract_embeddings(dataloader, model, dimension): + with torch.no_grad(): + model.eval() + embeddings = np.zeros((len(dataloader.dataset), dimension))#num_of_dim + labels = np.zeros(len(dataloader.dataset)) + k = 0 + for images, target in dataloader: + if cuda: + images = images.cuda() + embeddings[k:k+len(images)] = model.get_embedding(images).data.cpu().numpy() + labels[k:k+len(images)] = target.numpy() + k += len(images) + return embeddings, labels + + +if __name__ == '__main__': + print(args) + + torch.cuda.set_device(args.cuda_device) + logdir = args.logdir + + dataset_name = args.dataset_name + + Attr_Dict = { + 'covid19':{'in_channel':1, + 'n_classes':3, + 'train_dataset' : CovidDataset(iterNo=args.iterNo,train=True), + 'test_dataset' : CovidDataset(iterNo=args.iterNo,train=False), + 'resDir':'./covid19Res/iterNo{}'.format(args.iterNo) + } + } + + num_of_dim = args.dim + n_classes = Attr_Dict[dataset_name]['n_classes'] + train_dataset = Attr_Dict[dataset_name]['train_dataset'] + test_dataset = Attr_Dict[dataset_name]['test_dataset'] + + n_sample_classes = args.n_sample_classes + n_samples_per_class = args.n_samples_per_class + train_batch_sampler = BalancedBatchSampler(train_dataset, n_classes=n_sample_classes, n_samples=n_samples_per_class) + test_batch_sampler = BalancedBatchSampler(test_dataset, n_classes=n_sample_classes, n_samples=n_samples_per_class) + + cuda = torch.cuda.is_available() + + kwargs = {'num_workers': 40, 'pin_memory': True} if cuda else {} + batch_size = args.batch_size + train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, **kwargs) + test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, **kwargs) + + online_train_loader = torch.utils.data.DataLoader(train_dataset, batch_sampler=train_batch_sampler, **kwargs) + online_test_loader = torch.utils.data.DataLoader(test_dataset, batch_sampler=test_batch_sampler, **kwargs) + + + start_epoch = args.start_epoch + n_epochs = args.n_epoch + log_interval = 50 + margin = args.margin + + Selector = { + 'AllTripletSelector':AllTripletSelector(), + 'HardestNegativeTripletSelector':HardestNegativeTripletSelector(margin), + 'RandomNegativeTripletSelector':RandomNegativeTripletSelector(margin), + 'SemihardNegativeTripletSelector':SemihardNegativeTripletSelector(margin), + 'BatchHardTripletSelector':BatchHardTripletSelector(margin) + } + + embedding_net = ResNetEmbeddingNet(dataset_name,num_of_dim) + classification_net = ClassificationNet(embedding_net, dimension = num_of_dim ,n_classes = n_classes) + + + if args.EmbeddingMode: + loader1 = online_train_loader + loader2 = online_test_loader + model = embedding_net + loss_fn = OnlineTripletLoss(margin, Selector[args.TripletSelector]) + lr = 1e-4 + # optimizer = optim.Adam(model.parameters(), lr=lr) + optimizer = optim.Adam( + model.parameters(), + lr=lr, + betas=(0.9, 0.99), + eps=1e-8, + amsgrad=True) + scheduler = lr_scheduler.StepLR(optimizer, 50, gamma=0.1, last_epoch=-1) + metrics = [AverageNonzeroTripletsMetric()] + logName = 'margin{}_{}d-embedding_{}'.format(margin,num_of_dim,args.TripletSelector) + logName = os.path.join(Attr_Dict[dataset_name]['resDir'],logName) + EmbeddingArgs = (num_of_dim,train_loader,test_loader) + + else: + loader1 = train_loader + loader2 = test_loader + model = classification_net + weight_ = torch.FloatTensor([0.01317614, 0.87021505, 0.11660882]).cuda() + loss_fn = FocalLoss(class_num = n_classes, alpha = weight_, gamma = 2) + lr = 1e-4 + optimizer = optim.Adam( + model.parameters(), + lr=lr, + betas=(0.9, 0.99), + eps=1e-8, + amsgrad=True) + scheduler = lr_scheduler.StepLR(optimizer, 100, gamma=0.1, last_epoch=-1) + metrics = [AccumulatedAccuracyMetric()] + logName = '{}d-weight-focalloss'.format(num_of_dim) + logName = os.path.join(Attr_Dict[dataset_name]['resDir'],logName) + EmbeddingArgs = () + + + + if cuda: + model.cuda() + + logfile = os.path.join(logdir,logName) + fit(dataset_name, + logfile, + loader1, + loader2, + model, + loss_fn, + optimizer, + scheduler, + n_epochs, + cuda, + log_interval, + metrics, + start_epoch, + *EmbeddingArgs) diff --git a/centerMain.py b/centerMain.py new file mode 100644 index 0000000..b1bf1f8 --- /dev/null +++ b/centerMain.py @@ -0,0 +1,314 @@ +import numpy as np +from torchvision.datasets import MNIST +from torchvision import transforms +from datasets import UnbalancedMNIST, BalancedBatchSampler +from networks import EmbeddingNet, ClassificationNet,ResNetEmbeddingNet +from skinDatasetFolder import skinDatasetFolder +from covidDataSetFolder import CovidDataset +from losses import OnlineTripletLoss,OnlineContrastiveLoss,OnlineCenterLoss +from utils import AllTripletSelector,HardestNegativeTripletSelector, RandomNegativeTripletSelector, SemihardNegativeTripletSelector # Strategies for selecting triplets within a minibatch +from utils import BatchHardTripletSelector,AllPositivePairSelector, HardNegativePairSelector # Strategies for selecting pairs within a minibatch +from trainer import fit +from CenterLoss import CenterLoss +import torch +from torch.optim import lr_scheduler +import torch.optim as optim +from torch.autograd import Variable +from trainer import getMetrics,plot_confusion_matrix +from sklearn.metrics import precision_score, recall_score ,accuracy_score +from tqdm import tqdm +import numpy as np +import matplotlib +import matplotlib.pyplot as plt +import argparse +import os +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +from tensorboardX import SummaryWriter +import csv + +Name_dict = { + 'MNIST' : ['0','1','2','3','4','5','6','7','8','9'], + 'skin' : ['MEL', 'NV', 'BCC', 'AKIEC', 'BKL', 'DF', 'VASC'], + 'Retina' : ['0','1','2','3','4'], + 'Xray14': ['Atelectasis','Cardiomegaly','Consolidation','Edema','Effusion','Emphysema','Fibrosis','Hernia','Infiltration','Mass', + 'No_Finding','Nodule','Pleural_Thickening','Pneumonia','Pneumothorax'], + 'xray3' : ['Normal', 'Lung Opacity', '‘No Lung Opacity/Not Normal'], + 'covid19' : ['Normal','covid19','Others'] +} + +def str2bool(v): + """Convert string to Boolean + + Args: + v: True or False but in string + + Returns: + True or False in Boolean + + Raises: + TyreError + """ + + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') + + +parser = argparse.ArgumentParser(description='Triplet For MNIST') +parser.add_argument('--dataset_name',default='covid19', + help='Choose dataset [...]') +parser.add_argument('--rescale',default=False,type=str2bool, + help='rescale dataset') +parser.add_argument('--iterNo',default=1,type=int, + help='Use for choosing fold validation') +parser.add_argument('--cuda_device',default=0,type=int, + help='Choose cuda_device:(0,1,2,3,4,5,6,7)') +parser.add_argument('--EmbeddingMode',default=False,type = str2bool , + help='True for tripletsLoss(embedding) / False for EntropyLoss(classfication)') +parser.add_argument('--dim',default=128,type=int, + help='The dimension of embedding(type int)') +parser.add_argument('--n_classes',default=3,type=int, + help='The number of classes (type int)') +parser.add_argument('--margin',default=0.5,type=float, + help='Margin used in triplet loss (type float)') +parser.add_argument('--logdir',default='result', + help='Path to log tensorboard, pick a UNIQUE name to log') +parser.add_argument('--start_epoch',default=0,type=int + ,help='Start epoch (int)') +parser.add_argument('--n_epoch',default=200,type=int, + help='End_epoch (int)') +parser.add_argument('--batch_size',default=16,type=int, + help='Batch size (int)') +parser.add_argument('--n_sample_classes',default=3,type=int, + help='For a batch sampler to work comine #samples_per_class') +parser.add_argument('--n_samples_per_class',default=10,type=int, + help='For a batch sampler to work comine #n_sample_classes') +parser.add_argument('--TripletSelector',default='SemihardNegativeTripletSelector', + help='Triplet selector chosen in ({},{},{},{},{})' + .format('AllTripletSelector', + 'HardestNegativeTripletSelector', + 'RandomNegativeTripletSelector', + 'SemihardNegativeTripletSelector', + 'BatchHardTripletSelector')) +args = parser.parse_args() + +def extract_embeddings(dataloader, model, dimension): + with torch.no_grad(): + model.eval() + embeddings = np.zeros((len(dataloader.dataset), dimension))#num_of_dim + labels = np.zeros(len(dataloader.dataset)) + k = 0 + for images, target in dataloader: + if cuda: + images = images.cuda() + embeddings[k:k+len(images)] = model.get_embedding(images).data.cpu().numpy() + labels[k:k+len(images)] = target.numpy() + k += len(images) + return embeddings, labels + +def calc_centers(dataloader,model,n_classes): + with torch.no_grad(): + model.eval() + centers = torch.tensor([],requires_grad=True).cuda() + embeddings = torch.Tensor([]).cuda() + targets = torch.LongTensor([]).cuda() + for (data,target) in dataloader: + data , target = data.cuda() , target.cuda() + batch_embedding = model(data) + embeddings = torch.cat([embeddings,batch_embedding]) + targets = torch.cat([targets,target]) + + for lbl in range(n_classes): + mask = targets.eq(lbl) + embeddings_ = embeddings[mask] + center = embeddings_.mean(dim=0) + centers = torch.cat([centers,center.unsqueeze(dim=0)]) + assert centers.shape == (n_classes,embeddings.size()[1]) + #print(centers,centers.size(),centers.requires_grad) + return centers + +def CenterPredict(embeddings,centers): + C = centers.size()[0] + n = embeddings.size()[0] + labels = torch.LongTensor([]) + for i in range(n): + dis = (embeddings[i].repeat((C,1))-centers).pow(2).sum(1) + labels = torch.cat([labels,torch.LongTensor([torch.min(dis,0)[1]])]) + return labels + +def GetMetric(y_true,y_pred): + accuracy = accuracy_score(y_true,y_pred) + precision = precision_score(y_true,y_pred,average=None) + recall = recall_score(y_true,y_pred,average=None) + MCA , MCR = precision.mean() , recall.mean() + print(precision,recall) + return accuracy , MCA , MCR + +if __name__ == '__main__': + torch.cuda.set_device(args.cuda_device) + dataset_name = args.dataset_name + train_dataset = CovidDataset(iterNo=args.iterNo,train=True) + test_dataset = CovidDataset(iterNo=args.iterNo,train=False) + + + cuda = torch.cuda.is_available() + + kwargs = {'num_workers': 40, 'pin_memory': True} if cuda else {} + batch_size = args.batch_size + train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, **kwargs) + test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, **kwargs) + + + start_epoch = args.start_epoch + n_epochs = args.n_epoch + n_classes = args.n_classes + margin = args.margin + + + + embedding_net = ResNetEmbeddingNet(dataset_name,args.dim) + #classification_net = ClassificationNet(embedding_net, dimension = num_of_dim ,n_classes = n_classes) + device = torch.device("cuda") + model = embedding_net + pth = './{}_d-checkpoint/iterNo{}.pth'.format(args.dim,args.iterNo) + location = 'cuda:{}'.format(args.cuda_device) + model.load_state_dict(torch.load(pth, map_location=location),strict=False) + model.to(device) + + print('Check point loaded successfully! '+ pth) + # loss_fn = CenterLoss(margin,n_classes) + loss_fn = OnlineCenterLoss(margin) + # loss_fn = OnlineCenterLoss(margin) + lr = 1e-4 + + optimizer = optim.Adam( + model.parameters(), + lr=lr, + betas=(0.9, 0.99), + eps=1e-8, + amsgrad=True) + scheduler = lr_scheduler.StepLR(optimizer, 50, gamma=0.1, last_epoch=-1) + + output_writer_path = os.path.join('./run/result', '{}Res/iterNo{}/Centerloss_margin{}_{}d'.format( + dataset_name, args.iterNo, margin,args.dim) + ) + checkpoint_path = output_writer_path + writer = SummaryWriter(output_writer_path) + csvfileName = os.path.join(output_writer_path,'result.csv') + + best_mf1 = 0 + ################################### + with open(csvfileName,'w',newline='') as csvfile: + csvwriter = csv.writer(csvfile) + Metrics = ['_p','_r','_f1'] + firstRow = [] + for name in Name_dict[args.dataset_name]: + for m in Metrics: + firstRow.append(name+m) + firstRow.extend(['mean_p','mean_r','mean_f1']) + csvwriter.writerow(firstRow) + + for epoch in range(start_epoch,n_epochs): + scheduler.step() + + centers = calc_centers(train_loader,model,n_classes) + centers.requires_grad_() + + model.train() + Loss = 0 + correct_train = [] + pred_train = [] + for batch_idx,(data,target) in enumerate(train_loader): + optimizer.zero_grad() + data , target = data.cuda() , target.cuda() + batch_embedding = model(data) + loss = loss_fn(batch_embedding,target,centers) + loss.backward() + optimizer.step() + Loss += loss.item() + + correct_train.extend(target.data.cpu().numpy()) + pred_train.extend(CenterPredict(batch_embedding,centers).data.cpu().numpy()) + + if batch_idx%10 == 0: + print('Train => epoch:{} | batch:{} | loss:{}'.format(epoch,batch_idx,loss.item())) + train_acc, train_precision, train_recall, train_f1, train_mca, train_mcr, train_mf1 = \ + getMetrics(dataset_name,correct_train, pred_train) + writer.add_scalar('train/loss', Loss, epoch) + writer.add_scalar('train/acc',train_acc,epoch) + writer.add_scalar('train/mca',train_mca,epoch) + writer.add_scalar('train/mcr',train_mcr,epoch) + writer.add_scalar('train/mean_f1',train_mf1,epoch) + for lbl,precision_ in enumerate(train_precision): + writer.add_scalar('train/class_precision_of_CLASS{}'.format(Name_dict[dataset_name][lbl]),precision_,epoch) + for lbl,recall_ in enumerate(train_recall): + writer.add_scalar('train/class_recall_of_CLASS{}'.format(Name_dict[dataset_name][lbl]),recall_,epoch) + for lbl,f1_ in enumerate(train_f1): + writer.add_scalar('train/class_f1_of_CLASS{}'.format(Name_dict[dataset_name][lbl]),f1_,epoch) + + print('Train => [epoch : {} | Loss : {} | ACC :{} | MCA : {} | MCR : {} | Mf1 : {} ]'.format(epoch,Loss,train_acc,train_mca,train_mcr,train_mf1)) + + with torch.no_grad(): + model.eval() + # centers = calc_centers(test_loader,model,n_classes) + #centers.requires_grad_() + Loss = 0 + correct_test = [] + pred_test = [] + for batch_idx,(data,target) in enumerate(test_loader): + data , target = data.cuda() , target.cuda() + batch_embedding = model(data) + loss = loss_fn(batch_embedding,target,centers) + Loss += loss.item() + correct_test.extend(target.data.cpu().numpy()) + pred_test.extend(CenterPredict(batch_embedding,centers).data.cpu().numpy()) + + if batch_idx%10 == 0: + print('Test => epoch:{} | batch:{} | loss:{}'.format(epoch,batch_idx,loss.item())) + + test_acc, test_precision, test_recall, test_f1, test_mca, test_mcr, test_mf1 = \ + getMetrics(dataset_name,correct_test, pred_test) + epochRow = [] + for i in range(len(Name_dict[args.dataset_name])): + epochRow.extend([test_precision[i],test_recall[i],test_f1[i]]) + epochRow.extend([test_mca,test_mcr,test_mf1]) + csvwriter.writerow(epochRow) + + writer.add_scalar('test/loss', Loss, epoch) + writer.add_scalar('test/acc',test_acc,epoch) + writer.add_scalar('test/mca',test_mca,epoch) + writer.add_scalar('test/mcr',test_mcr,epoch) + writer.add_scalar('test/mean_f1',test_mf1,epoch) + for lbl,precision_ in enumerate(test_precision): + writer.add_scalar('test/class_precision_of_CLASS{}'.format(Name_dict[dataset_name][lbl]),precision_,epoch) + for lbl,recall_ in enumerate(test_recall): + writer.add_scalar('test/class_recall_of_CLASS{}'.format(Name_dict[dataset_name][lbl]),recall_,epoch) + for lbl,f1_ in enumerate(test_f1): + writer.add_scalar('test/class_f1_of_CLASS{}'.format(Name_dict[dataset_name][lbl]),f1_,epoch) + + + + if test_mf1>best_mf1: + best_mf1 = test_mf1 + torch.save(model.state_dict(),checkpoint_path+'/Mf1-{:.4f}'.format(best_mf1)+'.pth') + print('*************** Best_f1 Log ***************\nMf1-{:.4f}\tMCA-{:.4f}\tMCR-{:.4f}'.format(test_mf1 , test_mca , test_mcr)) + fig, title = plot_confusion_matrix( dataset_name, correct_test, pred_test,False) + plt.close() + writer.add_figure(title, fig, epoch) + + print('Test => [epoch : {} | Loss : {} | ACC :{} | MCA : {} | MCR : {} | Mf1 : {} ]'.format(epoch,Loss,test_acc,test_mca,test_mcr,test_mf1)) + + if epoch%5 == 0: + test_data_embeddings , correct = extract_embeddings(test_loader,model,args.dim) + writer.add_embedding(test_data_embeddings, + metadata = correct, + global_step = epoch + ) + + if epoch+1 == n_epochs: + print('*************** End_epoch Log ***************\nMf1-{:.4f}\tMCA-{:.4f}\tMCR-{:.4f}'.format(test_mf1 , test_mca , test_mcr)) \ No newline at end of file diff --git a/centerMain2.py b/centerMain2.py new file mode 100644 index 0000000..16c8a83 --- /dev/null +++ b/centerMain2.py @@ -0,0 +1,285 @@ +import numpy as np +from torchvision.datasets import MNIST +from torchvision import transforms +from datasets import UnbalancedMNIST, BalancedBatchSampler +from networks import EmbeddingNet, ClassificationNet,ResNetEmbeddingNet +from skinDatasetFolder import skinDatasetFolder +from losses import OnlineTripletLoss,OnlineContrastiveLoss,OnlineCenterLoss,OnlineCenterLossV2 +from utils import AllTripletSelector,HardestNegativeTripletSelector, RandomNegativeTripletSelector, SemihardNegativeTripletSelector # Strategies for selecting triplets within a minibatch +from utils import BatchHardTripletSelector,AllPositivePairSelector, HardNegativePairSelector # Strategies for selecting pairs within a minibatch +from trainer import fit +from CenterLoss import CenterLoss +import torch +from torch.optim import lr_scheduler +import torch.optim as optim +from torch.autograd import Variable +from trainer import getMetrics,plot_confusion_matrix +from sklearn.metrics import precision_score, recall_score ,accuracy_score +from tqdm import tqdm +import numpy as np +import matplotlib +import matplotlib.pyplot as plt +import argparse +import os +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +from tensorboardX import SummaryWriter + +Name_dict = { + 'MNIST' : ['0','1','2','3','4','5','6','7','8','9'], + 'skin' : ['MEL', 'NV', 'BCC', 'AKIEC', 'BKL', 'DF', 'VASC'] +} + +def str2bool(v): + """Convert string to Boolean + + Args: + v: True or False but in string + + Returns: + True or False in Boolean + + Raises: + TyreError + """ + + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') + +parser = argparse.ArgumentParser(description='Triplet For MNIST') +parser.add_argument('--dataset_name',default='skin', + help='Choose dataset [skin,MNIST,...]') +parser.add_argument('--rescale',default=False,type=str2bool,help='rescale dataset') +parser.add_argument('--iterNo',default=1,type=int,help='Use for choosing skin-image set') +parser.add_argument('--cuda_device',default=0,type=int,help='Choose cuda_device:(0,1,2,3)') +parser.add_argument('--EmbeddingMode',default=True,type = str2bool , + help='True for tripletsLoss(embedding) / False for EntropyLoss(classfication)') +parser.add_argument('--dim',default=128,type=int, + help='The dimension of embedding(type int)') +parser.add_argument('--n_classes',default=10,type=int, + help='The number of classes (type int)') +parser.add_argument('--margin',default=1.,type=float, + help='Margin used in triplet loss (type float)') +parser.add_argument('--logdir',default='result', + help='Path to log tensorboard, pick a UNIQUE name to log') +parser.add_argument('--start_epoch',default=0,type=int,help='Start epoch (int)') +parser.add_argument('--n_epoch',default=200,type=int,help='End_epoch (int)') +parser.add_argument('--batch_size',default=256,type=int,help='Batch size (int)') +parser.add_argument('--probability',default=400./5000.,type=float,help='Leave out probability') +parser.add_argument('--mnist1',default=5,type=int,help='Sampling num1 with probability') +parser.add_argument('--mnist2',default=8,type=int,help='Sampling num2 with probability') + +parser.add_argument('--n_sample_classes',default=10,type=int, + help='For a batch sampler to work comine #samples_per_class') +parser.add_argument('--n_samples_per_class',default=25,type=int, + help='For a batch sampler to work comine #n_sample_classes') +parser.add_argument('--TripletSelector',default='SemihardNegativeTripletSelector', + help='Triplet selector chosen in ({},{},{},{},{})' + .format('AllTripletSelector', + 'HardestNegativeTripletSelector', + 'RandomNegativeTripletSelector', + 'SemihardNegativeTripletSelector', + 'BatchHardTripletSelector')) +args = parser.parse_args() + +def extract_embeddings(dataloader, model, dimension): + with torch.no_grad(): + model.eval() + embeddings = np.zeros((len(dataloader.dataset), dimension))#num_of_dim + labels = np.zeros(len(dataloader.dataset)) + k = 0 + for images, target in dataloader: + if cuda: + images = images.cuda() + embeddings[k:k+len(images)] = model.get_embedding(images).data.cpu().numpy() + labels[k:k+len(images)] = target.numpy() + k += len(images) + return embeddings, labels + +def calc_centers(dataloader,model,n_classes): + with torch.no_grad(): + model.eval() + centers = torch.tensor([],requires_grad=True).cuda() + embeddings = torch.Tensor([]).cuda() + targets = torch.LongTensor([]).cuda() + for (data,target) in dataloader: + data , target = data.cuda() , target.cuda() + batch_embedding = model(data) + embeddings = torch.cat([embeddings,batch_embedding]) + targets = torch.cat([targets,target]) + + for lbl in range(n_classes): + mask = targets.eq(lbl) + embeddings_ = embeddings[mask] + center = embeddings_.mean(dim=0) + centers = torch.cat([centers,center.unsqueeze(dim=0)]) + assert centers.shape == (n_classes,embeddings.size()[1]) + #print(centers,centers.size(),centers.requires_grad) + return centers + +def CenterPredict(embeddings,centers): + C = centers.size()[0] + n = embeddings.size()[0] + labels = torch.LongTensor([]) + for i in range(n): + dis = (embeddings[i].repeat((C,1))-centers).pow(2).sum(1) + labels = torch.cat([labels,torch.LongTensor([torch.min(dis,0)[1]])]) + return labels + +def GetMetric(y_true,y_pred): + accuracy = accuracy_score(y_true,y_pred) + precision = precision_score(y_true,y_pred,average=None) + recall = recall_score(y_true,y_pred,average=None) + MCA , MCR = precision.mean() , recall.mean() + print(precision,recall) + return accuracy , MCA , MCR + +if __name__ == '__main__': + torch.cuda.set_device(args.cuda_device) + dataset_name = args.dataset_name + train_dataset = skinDatasetFolder(train=True,iterNo=args.iterNo,data_dir='../data') + test_dataset = skinDatasetFolder(train=False,iterNo=args.iterNo,data_dir='../data') + + cuda = torch.cuda.is_available() + + kwargs = {'num_workers': 40, 'pin_memory': True} if cuda else {} + + batch_size = args.batch_size + train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, **kwargs) + test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, **kwargs) + + start_epoch = args.start_epoch + n_epochs = args.n_epoch + n_classes = args.n_classes + margin = args.margin + + + + embedding_net = ResNetEmbeddingNet(dataset_name,args.dim) + #classification_net = ClassificationNet(embedding_net, dimension = num_of_dim ,n_classes = n_classes) + device = torch.device("cuda") + model = embedding_net + pth = './{}_d-checkpoint/iterNo{}.pth'.format(args.dim,args.iterNo) + location = 'cuda:{}'.format(args.cuda_device) + model.load_state_dict(torch.load(pth, map_location=location),strict=False) + model.to(device) + + print('Check point loaded successfully! '+ pth) + # loss_fn = CenterLoss(margin,n_classes) + loss_fn = OnlineCenterLossV2(margin) + # loss_fn = OnlineCenterLoss(margin) + lr = 1e-4 + + optimizer = optim.Adam( + model.parameters(), + lr=lr, + betas=(0.9, 0.99), + eps=1e-8, + amsgrad=True) + scheduler = lr_scheduler.StepLR(optimizer, 50, gamma=0.1, last_epoch=-1) + + output_writer_path = os.path.join('./run', '{}/iter{}/Centerloss_margin{}_{}d'.format(dataset_name,args.iterNo,margin,args.dim)) + checkpoint_path = output_writer_path + writer = SummaryWriter(output_writer_path) + + best_mf1 = 0 + ################################### + + for epoch in range(start_epoch,n_epochs): + scheduler.step() + + centers = calc_centers(train_loader,model,n_classes) + centers.requires_grad_() + + model.train() + Loss = 0 + correct_train = [] + pred_train = [] + for batch_idx,(data,target) in enumerate(train_loader): + optimizer.zero_grad() + data , target = data.cuda() , target.cuda() + batch_embedding = model(data) + loss = loss_fn(batch_embedding,target,centers) + loss.backward() + optimizer.step() + Loss += loss.item() + + correct_train.extend(target.data.cpu().numpy()) + pred_train.extend(CenterPredict(batch_embedding,centers).data.cpu().numpy()) + + if batch_idx%10 == 0: + print('Train => epoch:{} | batch:{} | loss:{}'.format(epoch,batch_idx,loss.item())) + train_acc, train_precision, train_recall, train_f1, train_mca, train_mcr, train_mf1 = \ + getMetrics(dataset_name,correct_train, pred_train) + writer.add_scalar('train/loss', Loss, epoch) + writer.add_scalar('train/acc',train_acc,epoch) + writer.add_scalar('train/mca',train_mca,epoch) + writer.add_scalar('train/mcr',train_mcr,epoch) + writer.add_scalar('train/mean_f1',train_mf1,epoch) + for lbl,precision_ in enumerate(train_precision): + writer.add_scalar('train/class_precision_of_CLASS{}'.format(Name_dict[dataset_name][lbl]),precision_,epoch) + for lbl,recall_ in enumerate(train_recall): + writer.add_scalar('train/class_recall_of_CLASS{}'.format(Name_dict[dataset_name][lbl]),recall_,epoch) + for lbl,f1_ in enumerate(train_f1): + writer.add_scalar('train/class_f1_of_CLASS{}'.format(Name_dict[dataset_name][lbl]),f1_,epoch) + + print('Train => [epoch : {} | Loss : {} | ACC :{} | MCA : {} | MCR : {} | Mf1 : {} ]'.format(epoch,Loss,train_acc,train_mca,train_mcr,train_mf1)) + + with torch.no_grad(): + model.eval() + # centers = calc_centers(test_loader,model,n_classes) + #centers.requires_grad_() + Loss = 0 + correct_test = [] + pred_test = [] + for batch_idx,(data,target) in enumerate(test_loader): + data , target = data.cuda() , target.cuda() + batch_embedding = model(data) + loss = loss_fn(batch_embedding,target,centers) + Loss += loss.item() + correct_test.extend(target.data.cpu().numpy()) + pred_test.extend(CenterPredict(batch_embedding,centers).data.cpu().numpy()) + + if batch_idx%10 == 0: + print('Test => epoch:{} | batch:{} | loss:{}'.format(epoch,batch_idx,loss.item())) + + test_acc, test_precision, test_recall, test_f1, test_mca, test_mcr, test_mf1 = \ + getMetrics(dataset_name,correct_test, pred_test) + writer.add_scalar('test/loss', Loss, epoch) + writer.add_scalar('test/acc',test_acc,epoch) + writer.add_scalar('test/mca',test_mca,epoch) + writer.add_scalar('test/mcr',test_mcr,epoch) + writer.add_scalar('test/mean_f1',test_mf1,epoch) + for lbl,precision_ in enumerate(test_precision): + writer.add_scalar('test/class_precision_of_CLASS{}'.format(Name_dict[dataset_name][lbl]),precision_,epoch) + for lbl,recall_ in enumerate(test_recall): + writer.add_scalar('test/class_recall_of_CLASS{}'.format(Name_dict[dataset_name][lbl]),recall_,epoch) + for lbl,f1_ in enumerate(test_f1): + writer.add_scalar('test/class_f1_of_CLASS{}'.format(Name_dict[dataset_name][lbl]),f1_,epoch) + + + + if test_mf1>best_mf1: + best_mf1 = test_mf1 + torch.save(model.state_dict(),checkpoint_path+'/Mf1-{:.4f}'.format(best_mf1)+'.pth') + print('*************** Best_f1 Log ***************\nMf1-{:.4f}\tMCA-{:.4f}\tMCR-{:.4f}'.format(test_mf1 , test_mca , test_mcr)) + fig, title = plot_confusion_matrix( dataset_name, correct_test, pred_test,False) + plt.close() + writer.add_figure(title, fig, epoch) + + print('Test => [epoch : {} | Loss : {} | ACC :{} | MCA : {} | MCR : {} | Mf1 : {} ]'.format(epoch,Loss,test_acc,test_mca,test_mcr,test_mf1)) + + if epoch%5 == 0: + test_data_embeddings , correct = extract_embeddings(test_loader,model,args.dim) + writer.add_embedding(test_data_embeddings, + metadata = correct, + global_step = epoch + ) + + if epoch+1 == n_epochs: + print('*************** End_epoch Log ***************\nMf1-{:.4f}\tMCA-{:.4f}\tMCR-{:.4f}'.format(test_mf1 , test_mca , test_mcr)) \ No newline at end of file diff --git a/centerMainBatch.py b/centerMainBatch.py new file mode 100644 index 0000000..9ff3f7c --- /dev/null +++ b/centerMainBatch.py @@ -0,0 +1,296 @@ +import numpy as np +from torchvision.datasets import MNIST +from torchvision import transforms +from datasets import UnbalancedMNIST, BalancedBatchSampler +from networks import EmbeddingNet, ClassificationNet,ResNetEmbeddingNet +from skinDatasetFolder import skinDatasetFolder +from losses import OnlineTripletLoss,OnlineContrastiveLoss,OnlineCenterLoss +from utils import AllTripletSelector,HardestNegativeTripletSelector, RandomNegativeTripletSelector, SemihardNegativeTripletSelector # Strategies for selecting triplets within a minibatch +from utils import BatchHardTripletSelector,AllPositivePairSelector, HardNegativePairSelector # Strategies for selecting pairs within a minibatch +from trainer import fit +from CenterLoss import CenterLoss , calc_centers +import torch +from torch.optim import lr_scheduler +import torch.optim as optim +from torch.autograd import Variable +from trainer import getMetrics,plot_confusion_matrix +from sklearn.metrics import precision_score, recall_score ,accuracy_score +from tqdm import tqdm +import numpy as np +import matplotlib +import matplotlib.pyplot as plt +import argparse +import os +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +from tensorboardX import SummaryWriter + +Name_dict = { + 'MNIST' : ['0','1','2','3','4','5','6','7','8','9'], + 'skin' : ['MEL', 'NV', 'BCC', 'AKIEC', 'BKL', 'DF', 'VASC'] +} + +def str2bool(v): + """Convert string to Boolean + + Args: + v: True or False but in string + + Returns: + True or False in Boolean + + Raises: + TyreError + """ + + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') + +parser = argparse.ArgumentParser(description='Triplet For MNIST') +parser.add_argument('--dataset_name',default='skin', + help='Choose dataset [skin,MNIST,...]') +parser.add_argument('--rescale',default=False,type=str2bool,help='rescale dataset') +parser.add_argument('--iterNo',default=1,type=int,help='Use for choosing skin-image set') +parser.add_argument('--cuda_device',default=0,type=int,help='Choose cuda_device:(0,1,2,3)') +parser.add_argument('--EmbeddingMode',default=True,type = str2bool , + help='True for tripletsLoss(embedding) / False for EntropyLoss(classfication)') +parser.add_argument('--dim',default=128,type=int, + help='The dimension of embedding(type int)') +parser.add_argument('--n_classes',default=10,type=int, + help='The number of classes (type int)') +parser.add_argument('--margin',default=1.,type=float, + help='Margin used in triplet loss (type float)') +parser.add_argument('--logdir',default='result', + help='Path to log tensorboard, pick a UNIQUE name to log') +parser.add_argument('--start_epoch',default=0,type=int,help='Start epoch (int)') +parser.add_argument('--n_epoch',default=200,type=int,help='End_epoch (int)') +parser.add_argument('--batch_size',default=256,type=int,help='Batch size (int)') +parser.add_argument('--probability',default=400./5000.,type=float,help='Leave out probability') +parser.add_argument('--mnist1',default=5,type=int,help='Sampling num1 with probability') +parser.add_argument('--mnist2',default=8,type=int,help='Sampling num2 with probability') + +parser.add_argument('--n_sample_classes',default=10,type=int, + help='For a batch sampler to work comine #samples_per_class') +parser.add_argument('--n_samples_per_class',default=25,type=int, + help='For a batch sampler to work comine #n_sample_classes') +parser.add_argument('--TripletSelector',default='SemihardNegativeTripletSelector', + help='Triplet selector chosen in ({},{},{},{},{})' + .format('AllTripletSelector', + 'HardestNegativeTripletSelector', + 'RandomNegativeTripletSelector', + 'SemihardNegativeTripletSelector', + 'BatchHardTripletSelector')) +args = parser.parse_args() + +def extract_embeddings(dataloader, model, dimension): + with torch.no_grad(): + model.eval() + embeddings = np.zeros((len(dataloader.dataset), dimension))#num_of_dim + labels = np.zeros(len(dataloader.dataset)) + k = 0 + for images, target in dataloader: + if cuda: + images = images.cuda() + embeddings[k:k+len(images)] = model.get_embedding(images).data.cpu().numpy() + labels[k:k+len(images)] = target.numpy() + k += len(images) + return embeddings, labels + +def calc_centers_whole(dataloader,model,n_classes): + with torch.no_grad(): + model.eval() + centers = torch.tensor([],requires_grad=True).cuda() + embeddings = torch.Tensor([]).cuda() + targets = torch.LongTensor([]).cuda() + for (data,target) in dataloader: + data , target = data.cuda() , target.cuda() + batch_embedding = model(data) + embeddings = torch.cat([embeddings,batch_embedding]) + targets = torch.cat([targets,target]) + + for lbl in range(n_classes): + mask = targets.eq(lbl) + embeddings_ = embeddings[mask] + center = embeddings_.mean(dim=0) + centers = torch.cat([centers,center.unsqueeze(dim=0)]) + assert centers.shape == (n_classes,embeddings.size()[1]) + #print(centers,centers.size(),centers.requires_grad) + return centers + +def CenterPredict(embeddings,centers): + C = centers.size()[0] + n = embeddings.size()[0] + labels = torch.LongTensor([]) + for i in range(n): + dis = (embeddings[i].repeat((C,1))-centers).pow(2).sum(1) + labels = torch.cat([labels,torch.LongTensor([torch.min(dis,0)[1]])]) + return labels + +def GetMetric(y_true,y_pred): + accuracy = accuracy_score(y_true,y_pred) + precision = precision_score(y_true,y_pred,average=None) + recall = recall_score(y_true,y_pred,average=None) + MCA , MCR = precision.mean() , recall.mean() + print(precision,recall) + return accuracy , MCA , MCR + +if __name__ == '__main__': + torch.cuda.set_device(args.cuda_device) + dataset_name = args.dataset_name + train_dataset = skinDatasetFolder(train=True,iterNo=args.iterNo,data_dir='../data') + test_dataset = skinDatasetFolder(train=False,iterNo=args.iterNo,data_dir='../data') + + cuda = torch.cuda.is_available() + + kwargs = {'num_workers': 40, 'pin_memory': True} if cuda else {} + + batch_size = args.batch_size + + train_batch_sampler = BalancedBatchSampler(train_dataset, n_classes=args.n_sample_classes, n_samples=args.n_samples_per_class) + test_batch_sampler = BalancedBatchSampler(test_dataset, n_classes=args.n_sample_classes, n_samples=args.n_sample_classes) + + train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, **kwargs) + online_train_loader = torch.utils.data.DataLoader(train_dataset, batch_sampler=train_batch_sampler, **kwargs) + test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, **kwargs) + + start_epoch = args.start_epoch + n_epochs = args.n_epoch + n_classes = args.n_classes + margin = args.margin + + + + embedding_net = ResNetEmbeddingNet(dataset_name,args.dim) + #classification_net = ClassificationNet(embedding_net, dimension = num_of_dim ,n_classes = n_classes) + device = torch.device("cuda") + model = embedding_net + pth = './{}_d-checkpoint/iterNo{}.pth'.format(args.dim,args.iterNo) + location = 'cuda:{}'.format(args.cuda_device) + model.load_state_dict(torch.load(pth, map_location=location),strict=False) + model.to(device) + + print('Check point loaded successfully! '+ pth) + # loss_fn = CenterLoss(margin,n_classes) + loss_fn = CenterLoss(margin,n_classes) + # loss_fn = OnlineCenterLoss(margin) + lr = 1e-4 + + optimizer = optim.Adam( + model.parameters(), + lr=lr, + betas=(0.9, 0.99), + eps=1e-8, + amsgrad=True) + scheduler = lr_scheduler.StepLR(optimizer, 50, gamma=0.1, last_epoch=-1) + + output_writer_path = os.path.join('./run', '{}/iter{}/CenterlossBatch_margin{}_{}d'.format(dataset_name,args.iterNo,margin,args.dim)) + checkpoint_path = output_writer_path + writer = SummaryWriter(output_writer_path) + + best_mf1 = 0 + ################################### + + for epoch in range(start_epoch,n_epochs): + scheduler.step() + + # centers = calc_centers(train_loader,model,n_classes) + # centers.requires_grad_() + + model.train() + Loss = 0 + correct_train = [] + pred_train = [] + for batch_idx,(data,target) in enumerate(online_train_loader): + optimizer.zero_grad() + data , target = data.cuda() , target.cuda() + batch_embedding = model(data) + + centers = calc_centers(batch_embedding,target,n_classes) + centers.requires_grad_() + + loss = loss_fn(batch_embedding,target,centers) + loss.backward() + optimizer.step() + Loss += loss.item() + + correct_train.extend(target.data.cpu().numpy()) + pred_train.extend(CenterPredict(batch_embedding,centers).data.cpu().numpy()) + + if batch_idx%10 == 0: + print('Train => epoch:{} | batch:{} | loss:{}'.format(epoch,batch_idx,loss.item())) + train_acc, train_precision, train_recall, train_f1, train_mca, train_mcr, train_mf1 = \ + getMetrics(dataset_name,correct_train, pred_train) + writer.add_scalar('train/loss', Loss, epoch) + writer.add_scalar('train/acc',train_acc,epoch) + writer.add_scalar('train/mca',train_mca,epoch) + writer.add_scalar('train/mcr',train_mcr,epoch) + writer.add_scalar('train/mean_f1',train_mf1,epoch) + for lbl,precision_ in enumerate(train_precision): + writer.add_scalar('train/class_precision_of_CLASS{}'.format(Name_dict[dataset_name][lbl]),precision_,epoch) + for lbl,recall_ in enumerate(train_recall): + writer.add_scalar('train/class_recall_of_CLASS{}'.format(Name_dict[dataset_name][lbl]),recall_,epoch) + for lbl,f1_ in enumerate(train_f1): + writer.add_scalar('train/class_f1_of_CLASS{}'.format(Name_dict[dataset_name][lbl]),f1_,epoch) + + print('Train => [epoch : {} | Loss : {} | ACC :{} | MCA : {} | MCR : {} | Mf1 : {} ]'.format(epoch,Loss,train_acc,train_mca,train_mcr,train_mf1)) + + with torch.no_grad(): + model.eval() + # centers = calc_centers(test_loader,model,n_classes) + #centers.requires_grad_() + Loss = 0 + correct_test = [] + pred_test = [] + centers = calc_centers_whole(train_loader,model,n_classes) + + for batch_idx,(data,target) in enumerate(test_loader): + data , target = data.cuda() , target.cuda() + batch_embedding = model(data) + loss = loss_fn(batch_embedding,target,centers) + Loss += loss.item() + correct_test.extend(target.data.cpu().numpy()) + pred_test.extend(CenterPredict(batch_embedding,centers).data.cpu().numpy()) + + if batch_idx%10 == 0: + print('Test => epoch:{} | batch:{} | loss:{}'.format(epoch,batch_idx,loss.item())) + + test_acc, test_precision, test_recall, test_f1, test_mca, test_mcr, test_mf1 = \ + getMetrics(dataset_name,correct_test, pred_test) + writer.add_scalar('test/loss', Loss, epoch) + writer.add_scalar('test/acc',test_acc,epoch) + writer.add_scalar('test/mca',test_mca,epoch) + writer.add_scalar('test/mcr',test_mcr,epoch) + writer.add_scalar('test/mean_f1',test_mf1,epoch) + for lbl,precision_ in enumerate(test_precision): + writer.add_scalar('test/class_precision_of_CLASS{}'.format(Name_dict[dataset_name][lbl]),precision_,epoch) + for lbl,recall_ in enumerate(test_recall): + writer.add_scalar('test/class_recall_of_CLASS{}'.format(Name_dict[dataset_name][lbl]),recall_,epoch) + for lbl,f1_ in enumerate(test_f1): + writer.add_scalar('test/class_f1_of_CLASS{}'.format(Name_dict[dataset_name][lbl]),f1_,epoch) + + + + if test_mf1>best_mf1: + best_mf1 = test_mf1 + torch.save(model.state_dict(),checkpoint_path+'/Mf1-{:.4f}'.format(best_mf1)+'.pth') + print('*************** Best_f1 Log ***************\nMf1-{:.4f}\tMCA-{:.4f}\tMCR-{:.4f}'.format(test_mf1 , test_mca , test_mcr)) + fig, title = plot_confusion_matrix( dataset_name, correct_test, pred_test,False) + plt.close() + writer.add_figure(title, fig, epoch) + + print('Test => [epoch : {} | Loss : {} | ACC :{} | MCA : {} | MCR : {} | Mf1 : {} ]'.format(epoch,Loss,test_acc,test_mca,test_mcr,test_mf1)) + + if epoch%5 == 0: + test_data_embeddings , correct = extract_embeddings(test_loader,model,args.dim) + writer.add_embedding(test_data_embeddings, + metadata = correct, + global_step = epoch + ) + + if epoch+1 == n_epochs: + print('*************** End_epoch Log ***************\nMf1-{:.4f}\tMCA-{:.4f}\tMCR-{:.4f}'.format(test_mf1 , test_mca , test_mcr)) \ No newline at end of file diff --git a/covidDataSetFolder.py b/covidDataSetFolder.py new file mode 100644 index 0000000..5c44f66 --- /dev/null +++ b/covidDataSetFolder.py @@ -0,0 +1,84 @@ +import os +import sys +import torch +import pandas as pd +import numpy as np +from PIL import Image +from torchvision import transforms +from torch.utils.data import Dataset + +''' + train_0.txt --- mean : 0.49584001302719116 | std : 0.25280070304870605 | label0 : 4425 | label1 : 67 | label2 : 500 + train_1.txt --- mean : 0.49532830715179443 | std : 0.25283026695251465 | label0 : 4426 | label1 : 66 | label2 : 500 + test_0.txt --- mean : 0.49532830715179443 | std : 0.25283026695251465 | label0 : 4426 | label1 : 66 | label2 : 500 + test_1.txt --- mean : 0.49584001302719116 | std : 0.25280070304870605 | label0 : 4425 | label1 : 67 | label2 : 500 +''' +_mean_ = [(0.49584,),(0.49532,)] +_std_ = [(0.252800,),(0.252830,)] +_normalize_ = { + 'train_0.txt' : transforms.Normalize(mean=(0.49584,),std=(0.252800,)) , + 'train_1.txt' : transforms.Normalize(mean=(0.49532,),std=(0.252830,)) , + 'test_0.txt' : transforms.Normalize(mean=(0.49532,),std=(0.252830,)) , + 'test_1.txt' : transforms.Normalize(mean=(0.49584,),std=(0.252800,)) +} + +class CovidDataset(Dataset): + def __init__(self, iterNo = 0, train = True, transform = None): + self.train = train + self.train_file = "train_{}.txt".format(iterNo) + self.test_file = "test_{}.txt".format(iterNo) + self.load_file = self.train_file if self.train else self.test_file + if transform is not None: + self.transform = transform + else: + self.transform = transforms.Compose([ + transforms.Resize((512,512)), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + _normalize_[self.load_file] + ]) if self.train else transforms.Compose([ + transforms.Resize((512,512)), + transforms.ToTensor(), + _normalize_[self.load_file] + ]) + + self.targets = pd.read_csv(self.load_file).values + if self.train: + self.train_labels = torch.LongTensor([dta[1] for dta in self.targets]) + else: + self.test_labels = torch.LongTensor([dta[1] for dta in self.targets]) + + + def __getitem__(self, index): + """ + Args: + index (int): Index + Returns: + tuple: (sample, label) where target is class_index of the target class. + """ + path = self.targets[index][0] + target = self.targets[index][1] + # target = torch.LongTensor([target]) + # class_name = self.targets[index][2] + img = Image.open(path).convert('L') + if self.transform is not None: + img = self.transform(img) + return img, target + + def __len__(self): + return len(self.targets) + + + + +if __name__ == "__main__": + dataset = CovidDataset(iterNo=1,train=False) + print(dataset.__len__(),dataset.load_file) + dataloader = torch.utils.data.DataLoader(dataset, batch_size=4, shuffle=True) + + for idx ,batch in enumerate(dataloader): + (imgs, lbls) = batch + print(imgs.size(), lbls) + if idx>10: + break + diff --git a/datasets.py b/datasets.py new file mode 100644 index 0000000..d877b89 --- /dev/null +++ b/datasets.py @@ -0,0 +1,233 @@ +import random +import numpy as np +from PIL import Image + +from torch.utils.data import Dataset +from torch.utils.data.sampler import BatchSampler + + +class SiameseMNIST(Dataset): + """ + Train: For each sample creates randomly a positive or a negative pair + Test: Creates fixed pairs for testing + """ + + def __init__(self, mnist_dataset): + self.mnist_dataset = mnist_dataset + + self.train = self.mnist_dataset.train + self.transform = self.mnist_dataset.transform + + if self.train: + self.train_labels = self.mnist_dataset.train_labels + self.train_data = self.mnist_dataset.train_data + self.labels_set = set(self.train_labels.numpy()) + self.label_to_indices = {label: np.where(self.train_labels.numpy() == label)[0] + for label in self.labels_set} + else: + # generate fixed pairs for testing + self.test_labels = self.mnist_dataset.test_labels + self.test_data = self.mnist_dataset.test_data + self.labels_set = set(self.test_labels.numpy()) + self.label_to_indices = {label: np.where(self.test_labels.numpy() == label)[0] + for label in self.labels_set} + + random_state = np.random.RandomState(29) + + positive_pairs = [[i, + random_state.choice(self.label_to_indices[self.test_labels[i].item()]), + 1] + for i in range(0, len(self.test_data), 2)] + + negative_pairs = [[i, + random_state.choice(self.label_to_indices[ + np.random.choice( + list(self.labels_set - set([self.test_labels[i].item()])) + ) + ]), + 0] + for i in range(1, len(self.test_data), 2)] + self.test_pairs = positive_pairs + negative_pairs + + def __getitem__(self, index): + if self.train: + target = np.random.randint(0, 2) + img1, label1 = self.train_data[index], self.train_labels[index].item() + if target == 1: + siamese_index = index + while siamese_index == index: + siamese_index = np.random.choice(self.label_to_indices[label1]) + else: + siamese_label = np.random.choice(list(self.labels_set - set([label1]))) + siamese_index = np.random.choice(self.label_to_indices[siamese_label]) + img2 = self.train_data[siamese_index] + else: + img1 = self.test_data[self.test_pairs[index][0]] + img2 = self.test_data[self.test_pairs[index][1]] + target = self.test_pairs[index][2] + + img1 = Image.fromarray(img1.numpy(), mode='L') + img2 = Image.fromarray(img2.numpy(), mode='L') + if self.transform is not None: + img1 = self.transform(img1) + img2 = self.transform(img2) + return (img1, img2), target + + def __len__(self): + return len(self.mnist_dataset) + + +class TripletMNIST(Dataset): + """ + Train: For each sample (anchor) randomly chooses a positive and negative samples + Test: Creates fixed triplets for testing + """ + + def __init__(self, mnist_dataset): + self.mnist_dataset = mnist_dataset + self.train = self.mnist_dataset.train + self.transform = self.mnist_dataset.transform + + if self.train: + self.train_labels = self.mnist_dataset.train_labels + self.train_data = self.mnist_dataset.train_data + self.labels_set = set(self.train_labels.numpy()) + self.label_to_indices = {label: np.where(self.train_labels.numpy() == label)[0] + for label in self.labels_set} + + else: + self.test_labels = self.mnist_dataset.test_labels + self.test_data = self.mnist_dataset.test_data + # generate fixed triplets for testing + self.labels_set = set(self.test_labels.numpy()) + self.label_to_indices = {label: np.where(self.test_labels.numpy() == label)[0] + for label in self.labels_set} + + random_state = np.random.RandomState(29) + + triplets = [[i, + random_state.choice(self.label_to_indices[self.test_labels[i].item()]), + random_state.choice(self.label_to_indices[ + np.random.choice( + list(self.labels_set - set([self.test_labels[i].item()])) + ) + ]) + ] + for i in range(len(self.test_data))] + self.test_triplets = triplets + + def __getitem__(self, index): + if self.train: + img1, label1 = self.train_data[index], self.train_labels[index].item() + positive_index = index + while positive_index == index: + positive_index = np.random.choice(self.label_to_indices[label1]) + negative_label = np.random.choice(list(self.labels_set - set([label1]))) + negative_index = np.random.choice(self.label_to_indices[negative_label]) + img2 = self.train_data[positive_index] + img3 = self.train_data[negative_index] + else: + img1 = self.test_data[self.test_triplets[index][0]] + img2 = self.test_data[self.test_triplets[index][1]] + img3 = self.test_data[self.test_triplets[index][2]] + + img1 = Image.fromarray(img1.numpy(), mode='L') + img2 = Image.fromarray(img2.numpy(), mode='L') + img3 = Image.fromarray(img3.numpy(), mode='L') + if self.transform is not None: + img1 = self.transform(img1) + img2 = self.transform(img2) + img3 = self.transform(img3) + return (img1, img2, img3), [] + + def __len__(self): + return len(self.mnist_dataset) + + +class UnbalancedMNIST(Dataset): + """ + UnbalancedMNIST create an unbalanced data set MNIST + left out certain labels and their data with a setting probality + others keep unchanged + """ + def __init__(self, mnist_dataset, probability, *kargs): + self.mnist_dataset = mnist_dataset + self.probability = probability + self.kargs = kargs + self.transform = mnist_dataset.transform + self.train = self.mnist_dataset.train + if self.train: + self.orginal_labels = self.mnist_dataset.train_labels + else: + self.orginal_labels = self.mnist_dataset.test_labels + + indices = [] + for idx,label in enumerate(list(self.orginal_labels.numpy())): + if label not in self.kargs or random.uniform(0,1) len(self.label_to_indices[class_]): + np.random.shuffle(self.label_to_indices[class_]) + self.used_label_indices_count[class_] = 0 + yield indices + self.count += self.n_classes * self.n_samples + + def __len__(self): + return len(self.dataset) // self.batch_size diff --git a/focalloss2d.py b/focalloss2d.py new file mode 100644 index 0000000..39a4e08 --- /dev/null +++ b/focalloss2d.py @@ -0,0 +1,117 @@ +#################################################### +##### This is focal loss class for multi class ##### +##### University of Tokyo Doi Kento ##### +#################################################### + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Variable +# I refered https://github.com/c0nn3r/RetinaNet/blob/master/focal_loss.py + +class FocalLoss2d(nn.Module): + + def __init__(self, gamma=0, weight=None, size_average=True): + super(FocalLoss2d, self).__init__() + + self.gamma = gamma + self.weight = weight + self.size_average = size_average + + def forward(self, input, target): + if input.dim()>2: + input = input.contiguous().view(input.size(0), input.size(1), -1) + input = input.transpose(1,2) + input = input.contiguous().view(-1, input.size(2)).squeeze() + if target.dim()==4: + target = target.contiguous().view(target.size(0), target.size(1), -1) + target = target.transpose(1,2) + target = target.contiguous().view(-1, target.size(2)).squeeze() + elif target.dim()==3: + target = target.view(-1) + else: + target = target.view(-1, 1) + + # compute the negative likelyhood + target = torch.squeeze(target) + + weight = Variable(self.weight) + #print(input, target) + logpt = -F.cross_entropy(input, target) + pt = torch.exp(logpt) + + # compute the loss + loss = -((1-pt)**self.gamma) * logpt + + # averaging (or not) loss + if self.size_average: + return loss.mean() + else: + return loss.sum() + + + +class FocalLoss(nn.Module): + r""" + This criterion is a implemenation of Focal Loss, which is proposed in + Focal Loss for Dense Object Detection. + + Loss(x, class) = - \alpha (1-softmax(x)[class])^gamma \log(softmax(x)[class]) + + The losses are averaged across observations for each minibatch. + + Args: + alpha(1D Tensor, Variable) : the scalar factor for this criterion + gamma(float, double) : gamma > 0; reduces the relative loss for well-classified examples (p > .5), + putting more focus on hard, misclassified examples + size_average(bool): By default, the losses are averaged over observations for each minibatch. + However, if the field size_average is set to False, the losses are + instead summed for each minibatch. + + + """ + def __init__(self, class_num, alpha=None, gamma=2, size_average=True): + super(FocalLoss, self).__init__() + if alpha is None: + self.alpha = Variable(torch.ones(class_num, 1)) + else: + if isinstance(alpha, Variable): + self.alpha = alpha + else: + self.alpha = Variable(alpha) + self.gamma = gamma + self.class_num = class_num + self.size_average = size_average + + def forward(self, inputs, targets): + N = inputs.size(0) + C = inputs.size(1) + P = F.softmax(inputs) + + class_mask = inputs.data.new(N, C).fill_(0) + class_mask = Variable(class_mask) + ids = targets.view(-1, 1) + class_mask.scatter_(1, ids.data, 1.) + #print(class_mask) + + + if inputs.is_cuda and not self.alpha.is_cuda: + self.alpha = self.alpha.cuda() + alpha = self.alpha[ids.data.view(-1)] + + probs = (P*class_mask).sum(1).view(-1,1) + + log_p = probs.log() + #print('probs size= {}'.format(probs.size())) + #print(probs) + + batch_loss = -alpha*(torch.pow((1-probs), self.gamma))*log_p + #print('-----bacth_loss------') + #print(batch_loss) + + + if self.size_average: + loss = batch_loss.mean() + else: + loss = batch_loss.sum() + return loss \ No newline at end of file diff --git a/losses.py b/losses.py new file mode 100644 index 0000000..8225cfe --- /dev/null +++ b/losses.py @@ -0,0 +1,192 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +def get_pesTriplet(embeddings,labels,centers,lambd): + C = centers.size()[0] + n = embeddings.size()[0] + labelSet = torch.arange(C) + pes_triplet = [] + for i in range(n): + embedding = embeddings[i] + label = labels[i] + dis_to_centers = (embedding.repeat((C,1)) - centers).pow(2).sum(1) + ap_dis = dis_to_centers[label] + loss_val = lambd + ap_dis - dis_to_centers + #print(loss_val) + mask = loss_val.gt(0) + mask[label] = 0 + if torch.sum(mask) > 0: + neg = labelSet[mask] + pes_triplet += [[i,label,ex_label] for ex_label in neg] + if len(pes_triplet)==0: + return None + return torch.LongTensor(pes_triplet).cuda() if embeddings.is_cuda else torch.LongTensor(pes_triplet) + +def get_minTriplet(embeddings,labels,centers,lambd): + C = centers.size()[0] + n = embeddings.size()[0] + labelSet = torch.arange(C) + pes_triplet = [] + for i in range(n): + embedding = embeddings[i] + label = labels[i] + dis_to_centers = (embedding.repeat((C,1)) - centers).pow(2).sum(1) + ap_dis = dis_to_centers[label] + loss_val = lambd + ap_dis - dis_to_centers + #print(loss_val) + argmin = torch.argmax(loss_val) + + if argmin!=label and loss_val[argmin]>0: + pes_triplet += [[i,label,argmin]] + if len(pes_triplet)==0: + return None + return torch.LongTensor(pes_triplet).cuda() if embeddings.is_cuda else torch.LongTensor(pes_triplet) + +class ContrastiveLoss(nn.Module): + """ + Contrastive loss + Takes embeddings of two samples and a target label == 1 if samples are from the same class and label == 0 otherwise + """ + + def __init__(self, margin): + super(ContrastiveLoss, self).__init__() + self.margin = margin + + def forward(self, output1, output2, target, size_average=True): + distances = (output2 - output1).pow(2).sum(1) # squared distances + losses = 0.5 * (target.float() * distances + + (1 + -1 * target).float() * F.relu(self.margin - distances.sqrt()).pow(2)) + return losses.mean() if size_average else losses.sum() + + +class TripletLoss(nn.Module): + """ + Triplet loss + Takes embeddings of an anchor sample, a positive sample and a negative sample + """ + + def __init__(self, margin): + super(TripletLoss, self).__init__() + self.margin = margin + + def forward(self, anchor, positive, negative, size_average=True): + distance_positive = (anchor - positive).pow(2).sum(1) # .pow(.5) + distance_negative = (anchor - negative).pow(2).sum(1) # .pow(.5) + losses = F.relu(distance_positive - distance_negative + self.margin) + return losses.mean() if size_average else losses.sum() + + +class OnlineContrastiveLoss(nn.Module): + """ + Online Contrastive loss + Takes a batch of embeddings and corresponding labels. + Pairs are generated using pair_selector object that take embeddings and targets and return indices of positive + and negative pairs + """ + + def __init__(self, margin, pair_selector): + super(OnlineContrastiveLoss, self).__init__() + self.margin = margin + self.pair_selector = pair_selector + + def forward(self, embeddings, target): + positive_pairs, negative_pairs = self.pair_selector.get_pairs(embeddings, target) + if embeddings.is_cuda: + positive_pairs = positive_pairs.cuda() + negative_pairs = negative_pairs.cuda() + positive_loss = (embeddings[positive_pairs[:, 0]] - embeddings[positive_pairs[:, 1]]).pow(2).sum(1) + negative_loss = F.relu( + self.margin - (embeddings[negative_pairs[:, 0]] - embeddings[negative_pairs[:, 1]]).pow(2).sum( + 1).sqrt()).pow(2) + loss = torch.cat([positive_loss, negative_loss], dim=0) + return loss.mean() + + +class OnlineCenterLoss(nn.Module): + def __init__(self,lambd): + super(OnlineCenterLoss,self).__init__() + self.lambd = lambd + + def forward(self, embeddings, targets, centers): + triplets = get_pesTriplet(embeddings,targets,centers,self.lambd) # A:embedding P:center N:center(false) + if triplets is None: + zero = torch.Tensor([0.]) + zero.requires_grad_() + return zero + #print(triplets) + ap_distances = (embeddings[triplets[:,0]] - centers[triplets[:,1]]).pow(2).sum(1) + an_distances = (embeddings[triplets[:,0]] - centers[triplets[:,2]]).pow(2).sum(1) + + losses = F.relu(ap_distances - an_distances + self.lambd) + return losses.mean() + +class OnlineCenterLossV2(nn.Module): + def __init__(self,lambd): + super(OnlineCenterLossV2,self).__init__() + self.lambd = lambd + + def forward(self, embeddings, targets, centers): + triplets = get_minTriplet(embeddings,targets,centers,self.lambd) # A:embedding P:center N:center(false) + if triplets is None: + zero = torch.Tensor([0.]) + zero.requires_grad_() + return zero + #print(triplets) + ap_distances = (embeddings[triplets[:,0]] - centers[triplets[:,1]]).pow(2).sum(1) + an_distances = (embeddings[triplets[:,0]] - centers[triplets[:,2]]).pow(2).sum(1) + + losses = F.relu(ap_distances - an_distances + self.lambd) + return losses.mean() + +class OnlineTripletLoss(nn.Module): + """ + Online Triplets loss + Takes a batch of embeddings and corresponding labels. + Triplets are generated using triplet_selector object that take embeddings and targets and return indices of + triplets + """ + + def __init__(self, margin, triplet_selector): + super(OnlineTripletLoss, self).__init__() + self.margin = margin + self.triplet_selector = triplet_selector + + def forward(self, embeddings, target): + + triplets = self.triplet_selector.get_triplets(embeddings, target) + + if embeddings.is_cuda: + triplets = triplets.cuda() + + ap_distances = (embeddings[triplets[:, 0]] - embeddings[triplets[:, 1]]).pow(2).sum(1) # .pow(.5) + an_distances = (embeddings[triplets[:, 0]] - embeddings[triplets[:, 2]]).pow(2).sum(1) # .pow(.5) + losses = F.relu(ap_distances - an_distances + self.margin) + + return losses.mean(), len(triplets) + +class LiftedEmbeddingLoss(nn.Module): + def __init__(self, margin, triplet_selector): + super(LiftedEmbeddingLoss, self).__init__() + self.margin = margin + self.triplet_selector = triplet_selector + + def forward(self, embeddings, target): + + triplets = self.triplet_selector.get_triplets(embeddings, target) + + losses = 0 + for i in range(len(triplets)): + triplet = triplets[i] + ap_distances = (embeddings[triplet[0]] - embeddings[triplet[1]]).pow(2).sum(1) # .pow(.5) + an_distances = (embeddings[triplet[0]] - embeddings[triplet[2]]).pow(2).sum(1) # .pow(.5) + + ap_exp_sum = torch.exp(ap_distances).sum() + an_exp_sum = torch.exp(self.margin - an_distances).sum() + + ap = torch.log(ap_exp_sum) + an = torch.log(an_exp_sum) + + losses += F.relu(ap+an) + + return losses, len(triplets) \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 0000000..abebb83 --- /dev/null +++ b/main.py @@ -0,0 +1,210 @@ +import numpy as np +from torchvision.datasets import MNIST +from torchvision import transforms +from datasets import UnbalancedMNIST, BalancedBatchSampler +from networks import EmbeddingNet, ClassificationNet,ResNetEmbeddingNet +from metrics import AccumulatedAccuracyMetric,AverageNonzeroTripletsMetric +from skinDatasetFolder import skinDatasetFolder +from covidDataSetFolder import CovidDataset +from losses import OnlineTripletLoss,OnlineContrastiveLoss +from utils import AllTripletSelector,HardestNegativeTripletSelector, RandomNegativeTripletSelector, SemihardNegativeTripletSelector # Strategies for selecting triplets within a minibatch +from utils import BatchHardTripletSelector,AllPositivePairSelector, HardNegativePairSelector # Strategies for selecting pairs within a minibatch +from trainer import fit + +import torch +from torch.optim import lr_scheduler +import torch.optim as optim + + +import numpy as np +import matplotlib +import matplotlib.pyplot as plt +import argparse +import os +def str2bool(v): + """Convert string to Boolean + + Args: + v: True or False but in string + + Returns: + True or False in Boolean + + Raises: + TyreError + """ + + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') + + +parser = argparse.ArgumentParser(description='Triplet For MNIST') +parser.add_argument('--dataset_name',default='covid19', + help='Choose dataset [...]') +parser.add_argument('--rescale',default=False,type=str2bool, + help='rescale dataset') +parser.add_argument('--iterNo',default=1,type=int, + help='Use for choosing fold validation') +parser.add_argument('--cuda_device',default=0,type=int, + help='Choose cuda_device:(0,1,2,3,4,5,6,7)') +parser.add_argument('--EmbeddingMode',default=False,type = str2bool , + help='True for tripletsLoss(embedding) / False for EntropyLoss(classfication)') +parser.add_argument('--dim',default=128,type=int, + help='The dimension of embedding(type int)') +parser.add_argument('--n_classes',default=3,type=int, + help='The number of classes (type int)') +parser.add_argument('--margin',default=0.5,type=float, + help='Margin used in triplet loss (type float)') +parser.add_argument('--logdir',default='result', + help='Path to log tensorboard, pick a UNIQUE name to log') +parser.add_argument('--start_epoch',default=0,type=int + ,help='Start epoch (int)') +parser.add_argument('--n_epoch',default=200,type=int, + help='End_epoch (int)') +parser.add_argument('--batch_size',default=16,type=int, + help='Batch size (int)') +parser.add_argument('--n_sample_classes',default=3,type=int, + help='For a batch sampler to work comine #samples_per_class') +parser.add_argument('--n_samples_per_class',default=10,type=int, + help='For a batch sampler to work comine #n_sample_classes') +parser.add_argument('--TripletSelector',default='SemihardNegativeTripletSelector', + help='Triplet selector chosen in ({},{},{},{},{})' + .format('AllTripletSelector', + 'HardestNegativeTripletSelector', + 'RandomNegativeTripletSelector', + 'SemihardNegativeTripletSelector', + 'BatchHardTripletSelector')) +args = parser.parse_args() + + + +def extract_embeddings(dataloader, model, dimension): + with torch.no_grad(): + model.eval() + embeddings = np.zeros((len(dataloader.dataset), dimension))#num_of_dim + labels = np.zeros(len(dataloader.dataset)) + k = 0 + for images, target in dataloader: + if cuda: + images = images.cuda() + embeddings[k:k+len(images)] = model.get_embedding(images).data.cpu().numpy() + labels[k:k+len(images)] = target.numpy() + k += len(images) + return embeddings, labels + + +if __name__ == '__main__': + print(args) + + torch.cuda.set_device(args.cuda_device) + logdir = args.logdir + + dataset_name = args.dataset_name + + Attr_Dict = { + 'covid19':{'in_channel':1, + 'n_classes':3, + 'train_dataset' : CovidDataset(iterNo=args.iterNo,train=True), + 'test_dataset' : CovidDataset(iterNo=args.iterNo,train=False), + 'resDir':'./covid19Res/iterNo{}'.format(args.iterNo) + } + } + + num_of_dim = args.dim + n_classes = Attr_Dict[dataset_name]['n_classes'] + train_dataset = Attr_Dict[dataset_name]['train_dataset'] + test_dataset = Attr_Dict[dataset_name]['test_dataset'] + + n_sample_classes = args.n_sample_classes + n_samples_per_class = args.n_samples_per_class + train_batch_sampler = BalancedBatchSampler(train_dataset, n_classes=n_sample_classes, n_samples=n_samples_per_class) + test_batch_sampler = BalancedBatchSampler(test_dataset, n_classes=n_sample_classes, n_samples=n_samples_per_class) + + cuda = torch.cuda.is_available() + + kwargs = {'num_workers': 40, 'pin_memory': True} if cuda else {} + batch_size = args.batch_size + train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, **kwargs) + test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, **kwargs) + + online_train_loader = torch.utils.data.DataLoader(train_dataset, batch_sampler=train_batch_sampler, **kwargs) + online_test_loader = torch.utils.data.DataLoader(test_dataset, batch_sampler=test_batch_sampler, **kwargs) + + + start_epoch = args.start_epoch + n_epochs = args.n_epoch + log_interval = 50 + margin = args.margin + + Selector = { + 'AllTripletSelector':AllTripletSelector(), + 'HardestNegativeTripletSelector':HardestNegativeTripletSelector(margin), + 'RandomNegativeTripletSelector':RandomNegativeTripletSelector(margin), + 'SemihardNegativeTripletSelector':SemihardNegativeTripletSelector(margin), + 'BatchHardTripletSelector':BatchHardTripletSelector(margin) + } + + embedding_net = ResNetEmbeddingNet(dataset_name,num_of_dim) + classification_net = ClassificationNet(embedding_net, dimension = num_of_dim ,n_classes = n_classes) + + + if args.EmbeddingMode: + loader1 = online_train_loader + loader2 = online_test_loader + model = embedding_net + loss_fn = OnlineTripletLoss(margin, Selector[args.TripletSelector]) + lr = 1e-4 + # optimizer = optim.Adam(model.parameters(), lr=lr) + optimizer = optim.Adam( + model.parameters(), + lr=lr, + betas=(0.9, 0.99), + eps=1e-8, + amsgrad=True) + scheduler = lr_scheduler.StepLR(optimizer, 50, gamma=0.1, last_epoch=-1) + metrics = [AverageNonzeroTripletsMetric()] + logName = 'margin{}_{}d-embedding_{}'.format(margin,num_of_dim,args.TripletSelector) + logName = os.path.join(Attr_Dict[dataset_name]['resDir'],logName) + EmbeddingArgs = (num_of_dim,train_loader,test_loader) + + else: + loader1 = train_loader + loader2 = test_loader + model = classification_net + loss_fn = torch.nn.NLLLoss(torch.FloatTensor([0.01317614, 0.87021505, 0.11660882]).cuda()) + lr = 1e-4 + # optimizer = optim.Adam(model.parameters(), lr=lr) + optimizer = optim.Adam( + model.parameters(), + lr=lr, + betas=(0.9, 0.99), + eps=1e-8, + amsgrad=True) + scheduler = lr_scheduler.StepLR(optimizer, 100, gamma=0.1, last_epoch=-1) + metrics = [AccumulatedAccuracyMetric()] + logName = '{}d-WCE'.format(num_of_dim) + logName = os.path.join(Attr_Dict[dataset_name]['resDir'],logName) + EmbeddingArgs = () + + if cuda: + model.cuda() + + logfile = os.path.join(logdir,logName) + fit(dataset_name, + logfile, + loader1, + loader2, + model, + loss_fn, + optimizer, + scheduler, + n_epochs, + cuda, + log_interval, + metrics, + start_epoch, + *EmbeddingArgs) diff --git a/metrics.py b/metrics.py new file mode 100644 index 0000000..7e94f23 --- /dev/null +++ b/metrics.py @@ -0,0 +1,66 @@ +import numpy as np + + +class Metric: + def __init__(self): + pass + + def __call__(self, outputs, target, loss): + raise NotImplementedError + + def reset(self): + raise NotImplementedError + + def value(self): + raise NotImplementedError + + def name(self): + raise NotImplementedError + + +class AccumulatedAccuracyMetric(Metric): + """ + Works with classification model + """ + + def __init__(self): + self.correct = 0 + self.total = 0 + + def __call__(self, outputs, target, loss): + pred = outputs[0].data.max(1, keepdim=True)[1] + self.correct += pred.eq(target[0].data.view_as(pred)).cpu().sum() + self.total += target[0].size(0) + return self.value() + + def reset(self): + self.correct = 0 + self.total = 0 + + def value(self): + return 100 * float(self.correct) / self.total + + def name(self): + return 'Accuracy' + + +class AverageNonzeroTripletsMetric(Metric): + ''' + Counts average number of nonzero triplets found in minibatches + ''' + + def __init__(self): + self.values = [] + + def __call__(self, outputs, target, loss): + self.values.append(loss[1]) + return self.value() + + def reset(self): + self.values = [] + + def value(self): + return np.mean(self.values) + + def name(self): + return 'Average nonzero triplets' diff --git a/networks.py b/networks.py new file mode 100644 index 0000000..c072c28 --- /dev/null +++ b/networks.py @@ -0,0 +1,121 @@ +import torch.nn as nn +import torch.nn.functional as F +import torchvision +import torch + +class ResNetEmbeddingNet(nn.Module): + def __init__(self,dataset_name,dimension): + super(ResNetEmbeddingNet,self).__init__() + self.resnet50 = torchvision.models.resnet50(pretrained=True) + self.resnet50.conv1 = nn.Conv2d(1,64,kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False) + self.resnet50.fc = nn.Linear(2048,dimension,bias=True) ##Arch-Net 3 + self.dataset_name = dataset_name + + #self.bn = nn.BatchNorm1d(1000) + #self.act = nn.ReLU() + #self.embedding = nn.Linear(1000, dimension) + + def l2_norm(self,input): + input_size = input.size() + buffer = torch.pow(input, 2) + normp = torch.sum(buffer, 1).add_(1e-10) + norm = torch.sqrt(normp) + _output = torch.div(input, norm.view(-1, 1).expand_as(input)) + output = _output.view(input_size) + + return output + def forward(self,x): + x = self.resnet50(x) + #x = self.bn(x) + #x = self.act(x) + #x = self.embedding(x) + output = self.l2_norm(x) + return output + + def get_embedding(self,x): + return self.forward(x) + + +class EmbeddingNet(nn.Module): + def __init__(self,dimension): + super(EmbeddingNet, self).__init__() + self.convnet = nn.Sequential(nn.Conv2d(1, 32, 5), nn.PReLU(), + nn.MaxPool2d(2, stride=2), + nn.Conv2d(32, 64, 5), nn.PReLU(), + nn.MaxPool2d(2, stride=2)) + + self.fc = nn.Sequential(nn.Linear(64 * 4 * 4, 256), + nn.PReLU(), + nn.Linear(256, 256), + nn.PReLU(), + nn.Linear(256, dimension)######num_of_dim + ) + + def forward(self, x): + output = self.convnet(x) + output = output.view(output.size()[0], -1) + output = self.fc(output) + return output + + def get_embedding(self, x): + return self.forward(x) + + +class EmbeddingNetL2(EmbeddingNet): + def __init__(self): + super(EmbeddingNetL2, self).__init__() + + def forward(self, x): + output = super(EmbeddingNetL2, self).forward(x) + output /= output.pow(2).sum(1, keepdim=True).sqrt() + return output + + def get_embedding(self, x): + return self.forward(x) + + +class ClassificationNet(nn.Module): + def __init__(self, embedding_net, dimension,n_classes): + super(ClassificationNet, self).__init__() + self.embedding_net = embedding_net + self.n_classes = n_classes + self.nonlinear = nn.ReLU() + self.fc1 = nn.Linear(dimension, n_classes)#####num_of_dims + + def forward(self, x): + output = self.embedding_net(x) + output = self.nonlinear(output) + scores = F.log_softmax(self.fc1(output), dim=-1) + return scores + + def get_embedding(self, x): + return self.nonlinear(self.embedding_net(x)) + + +class SiameseNet(nn.Module): + def __init__(self, embedding_net): + super(SiameseNet, self).__init__() + self.embedding_net = embedding_net + + def forward(self, x1, x2): + output1 = self.embedding_net(x1) + output2 = self.embedding_net(x2) + return output1, output2 + + def get_embedding(self, x): + return self.embedding_net(x) + + +class TripletNet(nn.Module): + def __init__(self, embedding_net): + super(TripletNet, self).__init__() + self.embedding_net = embedding_net + + def forward(self, x1, x2, x3): + output1 = self.embedding_net(x1) + output2 = self.embedding_net(x2) + output3 = self.embedding_net(x3) + return output1, output2, output3 + + def get_embedding(self, x): + return self.embedding_net(x) diff --git a/overCE.py b/overCE.py new file mode 100644 index 0000000..452fe6a --- /dev/null +++ b/overCE.py @@ -0,0 +1,215 @@ +import numpy as np +from torchvision.datasets import MNIST +from torchvision import transforms +from datasets import UnbalancedMNIST, BalancedBatchSampler +from networks import EmbeddingNet, ClassificationNet,ResNetEmbeddingNet +from metrics import AccumulatedAccuracyMetric,AverageNonzeroTripletsMetric +from skinDatasetFolder import skinDatasetFolder +from covidDataSetFolder import CovidDataset +from losses import OnlineTripletLoss,OnlineContrastiveLoss +from utils import AllTripletSelector,HardestNegativeTripletSelector, RandomNegativeTripletSelector, SemihardNegativeTripletSelector # Strategies for selecting triplets within a minibatch +from utils import BatchHardTripletSelector,AllPositivePairSelector, HardNegativePairSelector # Strategies for selecting pairs within a minibatch +from trainer import fit + +import torch +from torch.optim import lr_scheduler +import torch.optim as optim + + +import numpy as np +import matplotlib +import matplotlib.pyplot as plt +import argparse +import os +def str2bool(v): + """Convert string to Boolean + + Args: + v: True or False but in string + + Returns: + True or False in Boolean + + Raises: + TyreError + """ + + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') + + +parser = argparse.ArgumentParser(description='Triplet For MNIST') +parser.add_argument('--dataset_name',default='covid19', + help='Choose dataset [...]') +parser.add_argument('--rescale',default=False,type=str2bool, + help='rescale dataset') +parser.add_argument('--iterNo',default=1,type=int, + help='Use for choosing fold validation') +parser.add_argument('--cuda_device',default=0,type=int, + help='Choose cuda_device:(0,1,2,3,4,5,6,7)') +parser.add_argument('--EmbeddingMode',default=False,type = str2bool , + help='True for tripletsLoss(embedding) / False for EntropyLoss(classfication)') +parser.add_argument('--dim',default=128,type=int, + help='The dimension of embedding(type int)') +parser.add_argument('--n_classes',default=3,type=int, + help='The number of classes (type int)') +parser.add_argument('--margin',default=0.5,type=float, + help='Margin used in triplet loss (type float)') +parser.add_argument('--logdir',default='result', + help='Path to log tensorboard, pick a UNIQUE name to log') +parser.add_argument('--start_epoch',default=0,type=int + ,help='Start epoch (int)') +parser.add_argument('--n_epoch',default=200,type=int, + help='End_epoch (int)') +parser.add_argument('--batch_size',default=16,type=int, + help='Batch size (int)') +parser.add_argument('--n_sample_classes',default=3,type=int, + help='For a batch sampler to work comine #samples_per_class') +parser.add_argument('--n_samples_per_class',default=10,type=int, + help='For a batch sampler to work comine #n_sample_classes') +parser.add_argument('--TripletSelector',default='SemihardNegativeTripletSelector', + help='Triplet selector chosen in ({},{},{},{},{})' + .format('AllTripletSelector', + 'HardestNegativeTripletSelector', + 'RandomNegativeTripletSelector', + 'SemihardNegativeTripletSelector', + 'BatchHardTripletSelector')) +args = parser.parse_args() + + + +def extract_embeddings(dataloader, model, dimension): + with torch.no_grad(): + model.eval() + embeddings = np.zeros((len(dataloader.dataset), dimension))#num_of_dim + labels = np.zeros(len(dataloader.dataset)) + k = 0 + for images, target in dataloader: + if cuda: + images = images.cuda() + embeddings[k:k+len(images)] = model.get_embedding(images).data.cpu().numpy() + labels[k:k+len(images)] = target.numpy() + k += len(images) + return embeddings, labels + + +if __name__ == '__main__': + print(args) + + torch.cuda.set_device(args.cuda_device) + logdir = args.logdir + + dataset_name = args.dataset_name + + Attr_Dict = { + 'covid19':{'in_channel':1, + 'n_classes':3, + 'train_dataset' : CovidDataset(iterNo=args.iterNo,train=True), + 'test_dataset' : CovidDataset(iterNo=args.iterNo,train=False), + 'resDir':'./covid19Res/iterNo{}'.format(args.iterNo) + } + } + + num_of_dim = args.dim + n_classes = Attr_Dict[dataset_name]['n_classes'] + train_dataset = Attr_Dict[dataset_name]['train_dataset'] + test_dataset = Attr_Dict[dataset_name]['test_dataset'] + + n_sample_classes = args.n_sample_classes + n_samples_per_class = args.n_samples_per_class + train_batch_sampler = BalancedBatchSampler(train_dataset, n_classes=n_sample_classes, n_samples=n_samples_per_class) + test_batch_sampler = BalancedBatchSampler(test_dataset, n_classes=n_sample_classes, n_samples=n_samples_per_class) + + cuda = torch.cuda.is_available() + + kwargs = {'num_workers': 40, 'pin_memory': True} if cuda else {} + batch_size = args.batch_size + train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, **kwargs) + test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, **kwargs) + + online_train_loader = torch.utils.data.DataLoader(train_dataset, batch_sampler=train_batch_sampler, **kwargs) + online_test_loader = torch.utils.data.DataLoader(test_dataset, batch_sampler=test_batch_sampler, **kwargs) + + + start_epoch = args.start_epoch + n_epochs = args.n_epoch + log_interval = 50 + margin = args.margin + + Selector = { + 'AllTripletSelector':AllTripletSelector(), + 'HardestNegativeTripletSelector':HardestNegativeTripletSelector(margin), + 'RandomNegativeTripletSelector':RandomNegativeTripletSelector(margin), + 'SemihardNegativeTripletSelector':SemihardNegativeTripletSelector(margin), + 'BatchHardTripletSelector':BatchHardTripletSelector(margin) + } + + embedding_net = ResNetEmbeddingNet(dataset_name,num_of_dim) + classification_net = ClassificationNet(embedding_net, dimension = num_of_dim ,n_classes = n_classes) + + + if args.EmbeddingMode: + loader1 = online_train_loader + loader2 = online_test_loader + model = embedding_net + loss_fn = OnlineTripletLoss(margin, Selector[args.TripletSelector]) + lr = 1e-4 + # optimizer = optim.Adam(model.parameters(), lr=lr) + optimizer = optim.Adam( + model.parameters(), + lr=lr, + betas=(0.9, 0.99), + eps=1e-8, + amsgrad=True) + scheduler = lr_scheduler.StepLR(optimizer, 50, gamma=0.1, last_epoch=-1) + metrics = [AverageNonzeroTripletsMetric()] + logName = 'margin{}_{}d-embedding_{}'.format(margin,num_of_dim,args.TripletSelector) + logName = os.path.join(Attr_Dict[dataset_name]['resDir'],logName) + EmbeddingArgs = (num_of_dim,train_loader,test_loader) + + else: + loader1 = online_train_loader + loader2 = test_loader + model = classification_net + loss_fn = torch.nn.NLLLoss() + # if dataset_name == 'skin': + # loss_fn = torch.nn.NLLLoss(torch.FloatTensor([0.036,0.002,0.084,0.134,0.037,0.391,0.316]).cuda()) + lr = 1e-4 + # optimizer = optim.Adam(model.parameters(), lr=lr) + optimizer = optim.Adam( + model.parameters(), + lr=lr, + betas=(0.9, 0.99), + eps=1e-8, + amsgrad=True) + scheduler = lr_scheduler.StepLR(optimizer, 100, gamma=0.1, last_epoch=-1) + metrics = [AccumulatedAccuracyMetric()] + logName = '{}d-oversampling-softmax_cf'.format(num_of_dim) + logName = os.path.join(Attr_Dict[dataset_name]['resDir'],logName) + EmbeddingArgs = () + + + if cuda: + model.cuda() + + logfile = os.path.join(logdir,logName) + fit(dataset_name, + logfile, + loader1, + loader2, + model, + loss_fn, + optimizer, + scheduler, + n_epochs, + cuda, + log_interval, + metrics, + start_epoch, + *EmbeddingArgs) + + diff --git a/skinDatasetFolder.py b/skinDatasetFolder.py new file mode 100644 index 0000000..1eecf57 --- /dev/null +++ b/skinDatasetFolder.py @@ -0,0 +1,128 @@ +import os +import pandas as pd +import torch.utils.data as data +import torchvision.transforms as transforms +import torch +from PIL import Image + + +def make_dataset(iterNo, data_dir): + train_csv = 'split_data/split_data_{}_fold_train.csv'.format(iterNo) + fn = os.path.join(data_dir, train_csv) + print('Loading train from {}'.format(fn)) + + csvfile = pd.read_csv(fn, index_col=0) + raw_train_data = csvfile.values + + train_data = [] + for x, y in raw_train_data: + train_data.append((x, y)) + + + test_csv = 'split_data/split_data_{}_fold_test.csv'.format(iterNo) + fn = os.path.join(data_dir, test_csv) + print('Loading test from {}'.format(fn)) + + csvfile = pd.read_csv(fn, index_col=0) + raw_test_data = csvfile.values + + test_data = [] + for x, y in raw_test_data: + test_data.append((x, y)) + + return train_data, test_data + + +class skinDatasetFolder(data.Dataset): + def __init__(self, train=True, iterNo=1, data_dir='../data'): + + self.train_data, self.test_data = make_dataset(iterNo,data_dir) + self.train = train + if self.train: + self.train_labels = torch.LongTensor([dta[1] for dta in self.train_data]) + else: + self.test_labels = torch.LongTensor([dta[1] for dta in self.test_data]) + + mean , std = get_mean_std(iterNo) + resize_img = 300 + img_size = 224 + normalized = transforms.Normalize(mean=mean,std=std) + transform_train = transforms.Compose([ + transforms.Resize(resize_img), + # transforms.RandomHorizontalFlip(), + # transforms.RandomVerticalFlip(), + # transforms.ColorJitter(0.02, 0.02, 0.02, 0.01), + # transforms.RandomRotation([-180, 180]), + # transforms.RandomAffine([-180, 180], translate=[0.1, 0.1], scale=[0.7, 1.3]), + transforms.RandomCrop(img_size), + transforms.ToTensor(), + normalized + ]) + transform_test = transforms.Compose([ + transforms.Resize(resize_img), + transforms.CenterCrop(img_size), + transforms.ToTensor(), + normalized + ]) + self.transform = transform_train if self.train else transform_test + + raw_train_data = 'ISIC2018_Task3_Training_Input' + self.train_data_dir = os.path.join(data_dir, raw_train_data) + + + def __getitem__(self, index): + """ + Args: + index (int): Index + Returns: + tuple: (sample, target) where target is class_index of the target class. + """ + + if self.train: + path, target = self.train_data[index] + else: + path, target = self.test_data[index] + + path = os.path.join(self.train_data_dir, path) + imagedata = default_loader(path) + + if self.transform is not None: + imagedata = self.transform(imagedata) + + return imagedata, target + + + def __len__(self): + if self.train: + return len(self.train_data) + else: + return len(self.test_data) + + +def accimage_loader(path): + try: + import accimage + return accimage.Image(path) + except ImportError: + return pil_loader(path) + + +def pil_loader(path): + with open(path, 'rb') as f: + img = Image.open(f) + return img.convert('RGB') + + +def default_loader(path): + from torchvision import get_image_backend + + if get_image_backend() == 'accimage': + return accimage_loader(path) + else: + return pil_loader(path) + +def get_mean_std(iterNo): + filename = '../data/split_data/mean_std_shuffle.csv' + csvfile = pd.read_csv(filename).values[int(iterNo)-1] + print(csvfile) + return csvfile[0:3], csvfile[3:] \ No newline at end of file diff --git a/stat.py b/stat.py new file mode 100644 index 0000000..87b185b --- /dev/null +++ b/stat.py @@ -0,0 +1,46 @@ +import numpy as np +import pandas as pd +import os + + +# 'run/result/skinimage/iter1/triplet.../result.csv' +skin7_columns = ['MEL_p', 'MEL_r', 'MEL_f1', 'NV_p', 'NV_r', 'NV_f1', 'BCC_p', 'BCC_r', + 'BCC_f1', 'AKIEC_p', 'AKIEC_r', 'AKIEC_f1', 'BKL_p', 'BKL_r', 'BKL_f1', + 'DF_p', 'DF_r', 'DF_f1', 'VASC_p', 'VASC_r', 'VASC_f1', 'mean_p', + 'mean_r', 'mean_f1'] + +sd198_columns = ['small_mcp','small_mcr','small_mf1','mean_p','mean_r','mean_f1'] +xray13_columns = ['Normal_p','Normal_r','Normal_f1','Lung Opacity_p','Lung Opacity_r','Lung Opacity_f1','‘No Lung Opacity/Not Normal_p','‘No Lung Opacity/Not Normal_r','‘No Lung Opacity/Not Normal_f1','mean_p','mean_r','mean_f1'] +retina_columns = ['0_p','0_r','0_f1','1_p','1_r','1_f1','2_p','2_r','2_f1','3_p','3_r','3_f1','4_p','4_r','4_f1','mean_p','mean_r','mean_f1'] +method2path = { + 'CE' : ['run/result/xray3image_result/iterNo{}/128d-softmax_cf'.format(i+1) for i in range(5)], + 'WCE' : ['run/result/xray3image_result/iterNo{}/128d-CE-no_class_weights'.format(i+1) for i in range(5)], + 'OCE' : ['run/result/xray3image_result/iterNo{}/128d-oversampling-softmax_cf'.format(i+1) for i in range(5)], + 'Focal' : ['run/result/xray3image_result/iterNo{}/128d-focalloss'.format(i+1) for i in range(5)], + 'W-Focal' : ['run/result/xray3image_result/iterNo{}/128d-weight-focalloss'.format(i+1) for i in range(5)], + 'Triplet' : ['run/result/xray3image_result/iterNo{}/margin0.5_128d-embedding_RandomNegativeTripletSelector'.format(i+1) for i in range(5)], + 'T-Center' : ['run/xray3/iter{}/Centerloss_margin0.5_128d'.format(i+1) for i in range(5)] +} + + +if __name__ == '__main__': + Result = [] + for method in method2path: + index = ['{}|iter{}'.format(method,i) for i in range(5)] + columns = xray13_columns + fileList = [os.path.join(iterName,'result.csv') for iterName in method2path[method]] + dfList = [pd.read_csv(file) for file in fileList] + resList = [df[-20:].to_numpy().mean(axis=0) for df in dfList] + methodStat = pd.DataFrame(resList,index=index,columns=columns) + methodStat = methodStat.append( + pd.DataFrame([methodStat.mean().to_numpy()], + index=['{}|mean'.format(method)], + columns=xray13_columns) + ) + Result.append(methodStat) + Result = pd.concat(Result) + Result.to_csv('stat_20.csv') + + + + diff --git a/trainer.py b/trainer.py new file mode 100644 index 0000000..2ce6310 --- /dev/null +++ b/trainer.py @@ -0,0 +1,344 @@ +import torch +import numpy as np +import os +import sys +from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score +from sklearn.preprocessing import StandardScaler +from sklearn.neighbors import KNeighborsClassifier +from sklearn.model_selection import GridSearchCV +from tensorboardX import SummaryWriter +from sklearn.metrics import confusion_matrix +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +import itertools +import csv +Name_dict = { + 'MNIST' : ['0','1','2','3','4','5','6','7','8','9'], + 'skin' : ['MEL', 'NV', 'BCC', 'AKIEC', 'BKL', 'DF', 'VASC'], + 'Retina' : ['0','1','2','3','4'], + 'Xray14': ['Atelectasis','Cardiomegaly','Consolidation','Edema','Effusion','Emphysema','Fibrosis','Hernia','Infiltration','Mass', + 'No_Finding','Nodule','Pleural_Thickening','Pneumonia','Pneumothorax'], + 'xray3' : ['Normal', 'Lung Opacity', '‘No Lung Opacity/Not Normal'], + 'covid19' : ['Normal','covid19','Others'] +} + +param_grid =[ + { + 'weights':['uniform'], + 'n_neighbors': [1,3,5], + 'p': [2] + } +] + +knn_clf = KNeighborsClassifier() +grid_search = GridSearchCV(knn_clf,param_grid,cv=4,scoring='f1_macro') + +cuda = torch.cuda.is_available() + +def extract_embeddings(dataloader, model, dimension): + with torch.no_grad(): + model.eval() + embeddings = np.zeros((len(dataloader.dataset), dimension))#num_of_dim + labels = np.zeros(len(dataloader.dataset)) + k = 0 + for images, target in dataloader: + if cuda: + images = images.cuda() + embeddings[k:k+len(images)] = model.get_embedding(images).data.cpu().numpy() + labels[k:k+len(images)] = target.numpy() + k += len(images) + return embeddings, labels + +def fit(dataset_name, logName, train_loader, val_loader, model, loss_fn, optimizer, scheduler, n_epochs, cuda, log_interval, metrics=[], + start_epoch=0, *EmbeddingArgs): + """ + Loaders, model, loss function and metrics should work together for a given task, + i.e. The model should be able to process data output of loaders, + loss function should process target output of loaders and outputs from the model + + Examples: Classification: batch loader, classification model, NLL loss, accuracy metric + Siamese network: Siamese loader, siamese model, contrastive loss + Online triplet learning: batch loader, embedding model, online triplet loss + """ + if len(EmbeddingArgs) != 0: + assert len(EmbeddingArgs)==3 + Embedding_Mode = True + n_dim = EmbeddingArgs[0] + all_train_loader = EmbeddingArgs[1] + all_test_loader = EmbeddingArgs[2] + train_Args = (n_dim,all_train_loader) + test_Args = (n_dim,all_test_loader) + else: + Embedding_Mode = False + train_Args = () + test_Args = () + + output_writer_path = os.path.join('./run', logName) + checkpoint_path = output_writer_path + csvfileName = os.path.join(output_writer_path,'result.csv') + writer = SummaryWriter(output_writer_path) + + Best_f1 = 0.0 + with open(csvfileName,'w',newline='') as csvfile: + csvwriter = csv.writer(csvfile) + Metrics = ['_p','_r','_f1'] + firstRow = [] + for name in Name_dict[dataset_name]: + for m in Metrics: + firstRow.append(name+m) + firstRow.extend(['mean_p','mean_r','mean_f1']) + csvwriter.writerow(firstRow) + for epoch in range(0, start_epoch): + scheduler.step() + + for epoch in range(start_epoch, n_epochs): + scheduler.step() + + # Train stage + train_loss, metrics = train_epoch(dataset_name,epoch,writer,train_loader, model, loss_fn, optimizer, cuda, log_interval, metrics,*train_Args) + writer.add_scalar('train/loss', train_loss, epoch) + message = 'Epoch: {}/{}. Train set: Average loss: {:.4f}'.format(epoch + 1, n_epochs, train_loss) + for metric in metrics: + message += '\t{}: {}'.format(metric.name(), metric.value()) + + val_loss, metrics, test_f1 , test_mca , test_mcr = test_epoch(dataset_name,epoch,csvwriter,writer,val_loader, model, loss_fn, cuda, metrics,*test_Args) + if test_f1 > Best_f1: + Best_f1 = test_f1 + torch.save(model.state_dict(),checkpoint_path+'/Mf1-{:.4f}'.format(Best_f1)+'.pth') + print('*************** Best_f1 Log ***************\nMf1-{:.4f}\tMCA-{:.4f}\tMCR-{:.4f}'.format(test_f1 , test_mca , test_mcr)) + + if epoch+1 == n_epochs: + print('*************** End_epoch Log ***************\nMf1-{:.4f}\tMCA-{:.4f}\tMCR-{:.4f}'.format(test_f1 , test_mca , test_mcr)) + + val_loss /= len(val_loader) + #######summary_writer######### + #necessary for test_loss{classification or triplet loss} + writer.add_scalar('test/loss', train_loss, epoch) + #### + message += '\nEpoch: {}/{}. Validation set: Average loss: {:.4f}'.format(epoch + 1, n_epochs, + val_loss) + for metric in metrics: + message += '\t{}: {}'.format(metric.name(), metric.value()) + + print(message) + + +def train_epoch(dataset_name,epoch,writer,train_loader,model,loss_fn,optimizer,cuda,log_interval,metrics,*EmbeddingArgs): + if len(EmbeddingArgs) != 0: + assert len(EmbeddingArgs)==2 + Mode = True + n_dim = EmbeddingArgs[0] + all_train_loader = EmbeddingArgs[1] + else: + Mode = False + + for metric in metrics: + metric.reset() + + model.train() + losses = [] + total_loss = 0 + + correct = [] + predicted = [] + + for batch_idx, (data, target) in enumerate(train_loader): + target = target if len(target) > 0 else None + if not type(data) in (tuple, list): + data = (data,) + if cuda: + data = tuple(d.cuda() for d in data) + if target is not None: + target = target.cuda() + + + optimizer.zero_grad() + outputs = model(*data) + + if type(outputs) not in (tuple, list): + outputs = (outputs,) + + loss_inputs = outputs + if target is not None: + target = (target,) + loss_inputs += target + + loss_outputs = loss_fn(*loss_inputs) + loss = loss_outputs[0] if type(loss_outputs) in (tuple, list) else loss_outputs + losses.append(loss.item()) + total_loss += loss.item() + loss.backward() + optimizer.step() + + if Mode==False: + _, pred = torch.max(outputs[0].data, 1) + correct.extend(target[0].cpu().numpy()) + predicted.extend(pred.cpu().numpy()) + + for metric in metrics: + metric(outputs, target, loss_outputs) + + if batch_idx % log_interval == 0: + message = 'Train: [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( + batch_idx * len(data[0]), len(train_loader.dataset), + 100. * batch_idx / len(train_loader), np.mean(losses)) + for metric in metrics: + message += '\t{}: {}'.format(metric.name(), metric.value()) + + print(message) + losses = [] + if Mode: + ###To do correct = [] , predicted = [] + train_data_embeddings , correct = extract_embeddings(all_train_loader,model,n_dim) + grid_search.fit(train_data_embeddings,correct) + predicted = grid_search.best_estimator_.predict(train_data_embeddings) + ### + acc, precision, recall, f1, mca, mcr, m_f1 = getMetrics(dataset_name,correct, predicted) + print('--Train ==> ACC:{}\tMCA:{}\tMCR:{}\tMeanF1:{}'.format(acc,mca,mcr,m_f1)) + ####summary_writer TO DO + writer.add_scalar('train/mca', mca, epoch) + writer.add_scalar('train/acc', acc, epoch) + writer.add_scalar('train/mcr', mcr, epoch) + writer.add_scalar('train/mean_f1',m_f1,epoch) + for lbl,precision_ in enumerate(precision): + writer.add_scalar('train/class_precision_of_CLASS{}'.format(Name_dict[dataset_name][lbl]),precision_,epoch) + for lbl,recall_ in enumerate(recall): + writer.add_scalar('train/class_recall_of_CLASS{}'.format(Name_dict[dataset_name][lbl]),recall_,epoch) + for lbl,f1_ in enumerate(f1): + writer.add_scalar('train/class_f1_of_CLASS{}'.format(Name_dict[dataset_name][lbl]),f1_,epoch) + # + #### + total_loss /= (batch_idx + 1) + return total_loss, metrics + + +def test_epoch(dataset_name,epoch,csvwriter,writer,val_loader,model, loss_fn, cuda, metrics,*EmbeddingArgs): + if len(EmbeddingArgs) != 0: + assert len(EmbeddingArgs)==2 + Mode = True + n_dim = EmbeddingArgs[0] + all_test_loader = EmbeddingArgs[1] + else: + Mode = False + + with torch.no_grad(): + for metric in metrics: + metric.reset() + model.eval() + val_loss = 0 + correct = [] + predicted = [] + + for batch_idx, (data, target) in enumerate(val_loader): + target = target if len(target) > 0 else None + if not type(data) in (tuple, list): + data = (data,) + if cuda: + data = tuple(d.cuda() for d in data) + if target is not None: + target = target.cuda() + + outputs = model(*data) + + if type(outputs) not in (tuple, list): + outputs = (outputs,) + loss_inputs = outputs + if target is not None: + target = (target,) + loss_inputs += target + + loss_outputs = loss_fn(*loss_inputs) + loss = loss_outputs[0] if type(loss_outputs) in (tuple, list) else loss_outputs + val_loss += loss.item() + + if Mode==False: + _, pred = torch.max(outputs[0].data, 1) + correct.extend(target[0].cpu().numpy()) + predicted.extend(pred.cpu().numpy()) + + for metric in metrics: + metric(outputs, target, loss_outputs) + + if Mode: + test_data_embeddings , correct = extract_embeddings(all_test_loader,model,n_dim) + predicted = grid_search.best_estimator_.predict(test_data_embeddings) + if epoch%5==0: + writer.add_embedding(test_data_embeddings, + metadata = correct, + global_step = epoch + ) + + acc, precision, recall, f1, mca, mcr, m_f1 = getMetrics(dataset_name, correct, predicted) + epochRow = [] + for i in range(len(Name_dict[dataset_name])): + epochRow.extend([precision[i],recall[i],f1[i]]) + epochRow.extend([mca,mcr,m_f1]) + csvwriter.writerow(epochRow) + + print('--Test ==> ACC:{}\tMCA:{}\tMCR:{}\tMeanF1:{}'.format(acc,mca,mcr,m_f1)) + + if epoch%5==0: + fig, title = plot_confusion_matrix(dataset_name,correct, predicted,False) + plt.close() + writer.add_figure(title, fig, epoch) + writer.add_scalar('test/mca', mca, epoch) + writer.add_scalar('test/acc', acc, epoch) + writer.add_scalar('test/mcr', mcr, epoch) + writer.add_scalar('test/mean_f1',m_f1,epoch) + for lbl,precision_ in enumerate(precision): + writer.add_scalar('test/class_precision_of_CLASS{}'.format(Name_dict[dataset_name][lbl]),precision_,epoch) + for lbl,recall_ in enumerate(recall): + writer.add_scalar('test/class_recall_of_CLASS{}'.format(Name_dict[dataset_name][lbl]),recall_,epoch) + for lbl,f1_ in enumerate(f1): + writer.add_scalar('test/class_f1_of_CLASS{}'.format(Name_dict[dataset_name][lbl]),f1_,epoch) + + return val_loss, metrics , m_f1 , mca , mcr + + +def getMetrics(name,correct, predicted): + acc = accuracy_score(correct,predicted) + precision = precision_score(correct,predicted,average=None) + recall = recall_score(correct,predicted,average=None) + f1 = f1_score(correct,predicted,average=None) + mca = precision.mean() + mcr = recall.mean() + m_f1 = f1.mean() + return acc, precision, recall, f1, mca, mcr, m_f1 + +def plot_confusion_matrix(name, y_true, y_pred, normalized=True): + classes = Name_dict[name] + n_classes = len(classes) + cm = confusion_matrix(y_true,y_pred) + title = 'confusion matrix {}' + if normalized == True: + cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] + title.format('Normalize') + else: + title.format('Not Normalize') + + np.set_printoptions(precision=2) + fig = plt.figure(figsize=(n_classes, n_classes), dpi=320, facecolor='w', edgecolor='k') + + ax = fig.add_subplot(1, 1, 1) + ax.imshow(cm, cmap='Oranges') + + + tick_marks = np.arange(len(classes)) + ax.set_xlabel('Predicted', fontsize=7) + ax.set_xticks(tick_marks) + ax.set_xticklabels(classes, fontsize=4, rotation=-90, ha='center') + ax.xaxis.set_label_position('bottom') + ax.xaxis.tick_bottom() + + ax.set_ylabel('True Label', fontsize=7) + ax.set_yticks(tick_marks) + ax.set_yticklabels(classes, fontsize=4, va ='center') + ax.yaxis.set_label_position('left') + ax.yaxis.tick_left() + + for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])): + ax.text(j, i, format(cm[i, j], 'f') if cm[i,j]!=0 else '.', horizontalalignment="center", fontsize=6, verticalalignment='center', color= "black") + fig.set_tight_layout(True) + + return fig, title \ No newline at end of file diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..7a7e90a --- /dev/null +++ b/utils.py @@ -0,0 +1,238 @@ +from itertools import combinations + +import numpy as np +import torch + + +def pdist(vectors): + distance_matrix = -2 * vectors.mm(torch.t(vectors)) + vectors.pow(2).sum(dim=1).view(1, -1) + vectors.pow(2).sum( + dim=1).view(-1, 1) + return distance_matrix + + +class PairSelector: + """ + Implementation should return indices of positive pairs and negative pairs that will be passed to compute + Contrastive Loss + return positive_pairs, negative_pairs + """ + + def __init__(self): + pass + + def get_pairs(self, embeddings, labels): + raise NotImplementedError + + +class AllPositivePairSelector(PairSelector): + """ + Discards embeddings and generates all possible pairs given labels. + If balance is True, negative pairs are a random sample to match the number of positive samples + """ + def __init__(self, balance=True): + super(AllPositivePairSelector, self).__init__() + self.balance = balance + + def get_pairs(self, embeddings, labels): + labels = labels.cpu().data.numpy() + all_pairs = np.array(list(combinations(range(len(labels)), 2))) + all_pairs = torch.LongTensor(all_pairs) + positive_pairs = all_pairs[(labels[all_pairs[:, 0]] == labels[all_pairs[:, 1]]).nonzero()] + negative_pairs = all_pairs[(labels[all_pairs[:, 0]] != labels[all_pairs[:, 1]]).nonzero()] + if self.balance: + negative_pairs = negative_pairs[torch.randperm(len(negative_pairs))[:len(positive_pairs)]] + + return positive_pairs, negative_pairs + + +class HardNegativePairSelector(PairSelector): + """ + Creates all possible positive pairs. For negative pairs, pairs with smallest distance are taken into consideration, + matching the number of positive pairs. + """ + + def __init__(self, cpu=True): + super(HardNegativePairSelector, self).__init__() + self.cpu = cpu + + def get_pairs(self, embeddings, labels): + if self.cpu: + embeddings = embeddings.cpu() + distance_matrix = pdist(embeddings) + + labels = labels.cpu().data.numpy() + all_pairs = np.array(list(combinations(range(len(labels)), 2))) + all_pairs = torch.LongTensor(all_pairs) + positive_pairs = all_pairs[(labels[all_pairs[:, 0]] == labels[all_pairs[:, 1]]).nonzero()] + negative_pairs = all_pairs[(labels[all_pairs[:, 0]] != labels[all_pairs[:, 1]]).nonzero()] + + negative_distances = distance_matrix[negative_pairs[:, 0], negative_pairs[:, 1]] + negative_distances = negative_distances.cpu().data.numpy() + top_negatives = np.argpartition(negative_distances, len(positive_pairs))[:len(positive_pairs)] + top_negative_pairs = negative_pairs[torch.LongTensor(top_negatives)] + + return positive_pairs, top_negative_pairs + + +class TripletSelector: + """ + Implementation should return indices of anchors, positive and negative samples + return np array of shape [N_triplets x 3] + """ + + def __init__(self): + pass + + def get_pairs(self, embeddings, labels): + raise NotImplementedError + + +class AllTripletSelector(TripletSelector): + """ + Returns all possible triplets + May be impractical in most cases + """ + + def __init__(self): + super(AllTripletSelector, self).__init__() + + def get_triplets(self, embeddings, labels): + labels = labels.cpu().data.numpy() + triplets = [] + for label in set(labels): + label_mask = (labels == label) + label_indices = np.where(label_mask)[0] + if len(label_indices) < 2: + continue + negative_indices = np.where(np.logical_not(label_mask))[0] + anchor_positives = list(combinations(label_indices, 2)) # All anchor-positive pairs + + # Add all negatives for all positive pairs + temp_triplets = [[anchor_positive[0], anchor_positive[1], neg_ind] for anchor_positive in anchor_positives + for neg_ind in negative_indices] + triplets += temp_triplets + + return torch.LongTensor(np.array(triplets)) + + +def hardest_negative(loss_values): + hard_negative = np.argmax(loss_values) + return hard_negative if loss_values[hard_negative] > 0 else None + + +def random_hard_negative(loss_values): + hard_negatives = np.where(loss_values > 0)[0] + return np.random.choice(hard_negatives) if len(hard_negatives) > 0 else None + + +def semihard_negative(loss_values, margin): + semihard_negatives = np.where(np.logical_and(loss_values < margin, loss_values > 0))[0] + return np.random.choice(semihard_negatives) if len(semihard_negatives) > 0 else None + +class BatchHardTripletSelector(TripletSelector): + def __init__(self,margin,cpu=True): + super(BatchHardTripletSelector,self).__init__() + self.cpu = cpu + self.margin = margin + + def get_triplets(self,embeddings,labels): + if self.cpu: + embeddings = embeddings.cpu() + distance_matrix = pdist(embeddings) + distance_matrix = distance_matrix.cpu() + + labels = labels.cpu().data.numpy() + triplets = [] + + for anchor,label in enumerate(labels): + label_mask = np.where(labels==label)[0] + negative_indices = np.where(labels!=label)[0] + positive_indices = label_mask[label_mask!=anchor] + + ap_distances = distance_matrix[torch.LongTensor(np.array([anchor])),torch.LongTensor(positive_indices)] + an_distances = distance_matrix[torch.LongTensor(np.array([anchor])),torch.LongTensor(negative_indices)] + + ap_distances = ap_distances.data.cpu().numpy() + an_distances = an_distances.data.cpu().numpy() + + nidx = np.argmin(an_distances) + pidx = np.argmax(ap_distances) + + positive = positive_indices[pidx] + negative = negative_indices[nidx] + if ap_distances[pidx] - an_distances[nidx] + self.margin > 0: + triplets.append([anchor,positive,negative]) + + if len(triplets) == 0: + triplets.append([anchor,positive,negative]) + + triplets = np.array(triplets) + + return torch.LongTensor(triplets) + + + + +class FunctionNegativeTripletSelector(TripletSelector): + """ + For each positive pair, takes the hardest negative sample (with the greatest triplet loss value) to create a triplet + Margin should match the margin used in triplet loss. + negative_selection_fn should take array of loss_values for a given anchor-positive pair and all negative samples + and return a negative index for that pair + """ + + def __init__(self, margin, negative_selection_fn, cpu=True): + super(FunctionNegativeTripletSelector, self).__init__() + self.cpu = cpu + self.margin = margin + self.negative_selection_fn = negative_selection_fn + + def get_triplets(self, embeddings, labels): + if self.cpu: + embeddings = embeddings.cpu() + distance_matrix = pdist(embeddings) + distance_matrix = distance_matrix.cpu() + + labels = labels.cpu().data.numpy() + triplets = [] + + for label in set(labels): + label_mask = (labels == label) + label_indices = np.where(label_mask)[0] + if len(label_indices) < 2: + continue + negative_indices = np.where(np.logical_not(label_mask))[0] + anchor_positives = list(combinations(label_indices, 2)) # All anchor-positive pairs + anchor_positives.extend(list(combinations(label_indices[-1::-1],2))) + anchor_positives = np.array(anchor_positives) + + ap_distances = distance_matrix[anchor_positives[:, 0], anchor_positives[:, 1]] + for anchor_positive, ap_distance in zip(anchor_positives, ap_distances): + loss_values = ap_distance - distance_matrix[torch.LongTensor(np.array([anchor_positive[0]])), torch.LongTensor(negative_indices)] + self.margin + loss_values = loss_values.data.cpu().numpy() + hard_negative = self.negative_selection_fn(loss_values) + if hard_negative is not None: + hard_negative = negative_indices[hard_negative] + triplets.append([anchor_positive[0], anchor_positive[1], hard_negative]) + + if len(triplets) == 0: + triplets.append([anchor_positive[0], anchor_positive[1], negative_indices[0]]) + + triplets = np.array(triplets) + + return torch.LongTensor(triplets) + + +def HardestNegativeTripletSelector(margin, cpu=False): return FunctionNegativeTripletSelector(margin=margin, + negative_selection_fn=hardest_negative, + cpu=cpu) + + +def RandomNegativeTripletSelector(margin, cpu=False): return FunctionNegativeTripletSelector(margin=margin, + negative_selection_fn=random_hard_negative, + cpu=cpu) + + +def SemihardNegativeTripletSelector(margin, cpu=False): return FunctionNegativeTripletSelector(margin=margin, + negative_selection_fn=lambda x: semihard_negative(x, margin), + cpu=cpu)