Regularization causes error using mult gpu

[CeadeS]

Hi, when using kernel_regularizer=regularizers.l2(0.00004), in conv2D layer i get „AttributeError: 'Model' object has no attribute '_losses'„ caused by outputs = model(inputs) that merges the outputs of the different splits in one model.
The problem is that the regularizer waits for the loss but it is split over the sifferent models. Is it possible or even good to regularize batch wise?

[kuza55]

This seems like a bug in Keras, there's really no reason this shouldn't work.

[CeadeS]

Do you observe the same behavior or is it just my code, version etc.?

[kuza55]

Don't really have a setup to replicate this atm. I would try to create a reduced test case and file a bug against Keras; copying models the way I do should be fully supported. It sounds like a flaw in the conv2D regularizer.

[CeadeS]

Hi,
I have still no clue how to fix the problem and there is no response on my issues on github either. Using only one gpu is to slow for my problem. Maybe i change the Framework.

[CeadeS]

Maybe you can try the following code, there is only one dense layer and the slice lambda layer. This works even with single gpu,

# -*- coding: utf-8 -*-
import os
import cv2
import numpy as np
import tensorflow as tf
from sklearn.metrics import log_loss
from tensorflow.contrib.keras.api.keras import regularizers
from tensorflow.contrib.keras.api.keras import backend as K
from tensorflow.contrib.keras.api.keras.optimizers import SGD
from tensorflow.contrib.keras.api.keras.datasets import cifar10
from tensorflow.contrib.keras.api.keras import utils as np_utils
from tensorflow.contrib.keras.api.keras.models import Sequential,Model
from tensorflow.contrib.keras.api.keras.layers import Lambda,ZeroPadding2D, Dropout, Dense, Flatten, MaxPooling2D, Convolution2D, Concatenate

num_gpus = 2 ## number of gpus to utilize
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" ## bus order
os.environ["CUDA_VISIBLE_DEVICES"] = "2,3"  ## setting my free gpus
nb_train_samples = 3000 # 3000 training samples
nb_valid_samples = 100 # 100 validation samples
num_classes = 10 # number of classes

def get_slice(x, idx, parts, output_shape):
    shape = K.shape(x)
    size = K.concatenate([shape[:1] // parts, shape[1:]], axis=0)
    stride = K.concatenate([shape[:1] // parts, shape[1:] * 0], axis=0)
    start = stride * idx
    return K.reshape(tf.slice(x, start, size), (-1, output_shape[0], output_shape[1], output_shape[2]))



def load_cifar10_data(img_rows, img_cols):
    # Load cifar10 training and validation sets
    (X_train, Y_train), (X_valid, Y_valid) = cifar10.load_data()
    # Resize trainging images

    if K.image_data_format() == "channels_first":
        X_train = np.array([cv2.resize(img.transpose(1,2,0), (img_rows,img_cols)).transpose(2,0,1) for img in X_train[:nb_train_samples,:,:,:]])
        X_valid = np.array([cv2.resize(img.transpose(1,2,0), (img_rows,img_cols)).transpose(2,0,1) for img in X_valid[:nb_valid_samples,:,:,:]])
    else:
        X_train = np.array([cv2.resize(img, (img_rows,img_cols)) for img in X_train[:nb_train_samples,:,:,:]])
        X_valid = np.array([cv2.resize(img, (img_rows,img_cols)) for img in X_valid[:nb_valid_samples,:,:,:]])

    # Transform targets to keras compatible format
    Y_train = np_utils.to_categorical(Y_train[:nb_train_samples], num_classes)
    Y_valid = np_utils.to_categorical(Y_valid[:nb_valid_samples], num_classes)

    return X_train, Y_train, X_valid, Y_valid

def err_model(img_rows, img_cols, channel=1, num_classes=None):
    model = Sequential()
    model.add(Dense(units=1000, input_shape=(channel, img_rows, img_cols), activation='relu',
                    kernel_regularizer=regularizers.l2(0.00004)))

    inputs = []
    outputs_all = []
    for i in range(len(model.outputs)):
        outputs_all.append([])
    # Slice each input into a piece for processing on this GPU
    for x in model.inputs:
        input_shape = tuple(x.get_shape().as_list())[1:]
        slice_n = Lambda(get_slice, arguments={'idx': i, 'parts': 1, 'output_shape': input_shape})(
            x)
        inputs.append(slice_n)
    outputs = model(inputs)

    if not isinstance(outputs, list):
        outputs = [outputs]

    # Save all the outputs for merging back together later
    for l in range(len(outputs)):
        outputs_all[l].append(outputs[l])
    merged = []
    for outputs in outputs_all:
        merged.append(Concatenate(axis=0)(outputs))
    model = Model(inputs=model.inputs, outputs=merged)

    # Learning rate is changed to 0.001
    sgd = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True)
    model.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy'])
    return model

if __name__ == '__main__':
    print(tf.__version__)
    # Example to fine-tune on 3000 samples from Cifar10
    img_rows, img_cols = 224, 224 # Resolution of inputs
    channel = 3
    num_classes = 10 
    batch_size = 50
    nb_epoch = 10

    # Load Cifar10 data. Please implement your own load_data() module for your own dataset
    K.set_image_data_format('channels_first')
    X_train, Y_train, X_valid, Y_valid = load_cifar10_data(img_rows, img_cols)
    print(X_train.shape)
    # Load our model
    model = err_model(img_rows, img_cols, channel, num_classes)

    # Start Fine-tuning
    model.fit(X_train, Y_train,
              batch_size=batch_size*num_gpus,
              epochs=nb_epoch,
              shuffle=True,
              verbose=1,
              validation_data=(X_valid, Y_valid),
              )

    # Make predictions
    predictions_valid = model.predict(X_valid, batch_size=batch_size, verbose=1)

    # Cross-entropy loss score
    score = log_loss(Y_valid, predictions_valid)

[look4pritam]

Is it solved?