実験の再現性が取れていない

[magnusbarata]

######==========        Determinism test           ==========######
import numpy as np
import tensorflow as tf
import random
from models import find_model

import utils
seed = 7171
utils.set_seed(seed)

assert seed == 7171, 'Seed must be 7171'
assert str(np.sum(np.random.rand(100, 100, 100))) == '499878.65104700223', 'Numpy is not deterministic.'
assert str(sum([random.random() for _ in range(1000)])) == '499.1124110048268', 'Random is not deterministic.'

class Dataset(tf.keras.utils.Sequence):
    def __init__(self, n_samples=12, batch_size=4, vol_size=(256,256,64)):
        self.n_samples = n_samples
        self.batch_size = batch_size
        self.vol_size = vol_size
    
    def __len__(self):
        return np.ceil(self.n_samples / self.batch_size).astype(int)
    
    def __getitem__(self, index):
        X = tf.random.normal((self.batch_size, *self.vol_size, 1))
        y = np.random.randint(0, 2, self.batch_size)
        return X, tf.keras.utils.to_categorical(y)

data = Dataset()
model = find_model('baseline')((*data.vol_size, 1), 2)
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
assert str(sum(map(lambda x: x.sum(), model.get_weights()))) == '1055.7722613811493', '[Before fit] Tensorflow is not deterministic.'
model.fit(Dataset(), epochs=3, verbose=1)
assert str(sum(map(lambda x: x.sum(), model.get_weights()))) == '1066.2186067344155', '[After fit] Tensorflow is not deterministic.'

上記のプログラムを実行すると再現性は取れる．しかし，実際のtrain.pyでは再現性が取れていない．考えられる原因：

• datagen: skimage.transform.resizeの再現性の問題
• set_seed(): os.environ['TF_CUDNN_DETERMINISTIC'] = '1'が設定されていない

実験環境：

• GPU: TITAN RTX
• CUDA: Docker