Ongoing changes 2

plato/examples/demo_prediction_example.py

@@ -22,7 +22,7 @@ def compare_example_predictors(
         minibatch_size = 10,
     ):
     """
-    This demo shows how we can compare different online predictors.  The demo trains both predictors on the dataset,
+    This demo shows how we can compare_learning_curves different online predictors.  The demo trains both predictors on the dataset,
     returning an object that contains the results.
 
     :param test_mode: Set this to True to just run the demo quicky (but not to completion) to see that it doesn't break.
@@ -37,7 +37,7 @@ def compare_example_predictors(
         n_epochs = 1
         n_tests = 3
 
-    # Here we compare three predictors on MNIST - an MLP, a Perceptron, and a Random Forest.
+    # Here we compare_learning_curves three predictors on MNIST - an MLP, a Perceptron, and a Random Forest.
     # - The MLP is defined using Plato's interfaces - we create a Symbolic Predictor (GradientBasedPredictor) and
     #   then compile it into an IPredictor object
     # - The Perceptron directly implements the IPredictor interface.

plato/interfaces/helpers.py

@@ -1,16 +1,16 @@
 import numpy as np
-from plato.core import symbolic_simple, add_update, create_shared_variable, symbolic
-from plato.interfaces.interfaces import IParameterized
 import theano
-from theano.compile.sharedvalue import SharedVariable
+import theano.tensor as tt
+from theano.gof.graph import Variable
 from theano.ifelse import ifelse
-from theano.sandbox.cuda.rng_curand import CURAND_RandomStreams
 from theano.sandbox.rng_mrg import MRG_RandomStreams
 from theano.tensor.shared_randomstreams import RandomStreams
-import theano.tensor as tt
 from theano.tensor.sharedvar import TensorSharedVariable
 from theano.tensor.var import TensorVariable
 
+from plato.core import symbolic_simple, add_update, create_shared_variable, symbolic
+from plato.interfaces.interfaces import IParameterized
+
 __author__ = 'peter'
 
 
@@ -57,21 +57,25 @@ def get_theano_rng(seed, rngtype = 'mrg'):
     :return:
     """
 
+    def load_cuda_rng():
+        from theano.sandbox.cuda.rng_curand import CURAND_RandomStreams
+        return CURAND_RandomStreams
+
     stream_types = {
-        'mrg': MRG_RandomStreams_ext,
-        'mrg-old': MRG_RandomStreams,
-        'default': RandomStreams,
-        'cuda': CURAND_RandomStreams
+        'mrg': lambda: MRG_RandomStreams_ext,
+        'mrg-old': lambda: MRG_RandomStreams,
+        'default': lambda: RandomStreams,
+        'cuda': load_cuda_rng
     }
-    rng_con = stream_types[rngtype]
+    rng_con = stream_types[rngtype]()
 
     if isinstance(seed, np.random.RandomState):
         return rng_con(seed.randint(1e9))
     elif isinstance(seed, int):
         return rng_con(seed)
     elif seed is None:
         return rng_con(np.random.randint(1e9))
-    elif isinstance(seed, tuple(stream_types.values())):
+    elif isinstance(seed, tuple(v() for v in stream_types.values())):
         return seed
     else:
         raise Exception("Can't initialize a random number generator with %s" % (seed, ))
@@ -99,6 +103,7 @@ def identity(x):
     'softmax': softmax,
     'sigm': tt.nnet.sigmoid,
     'sig': tt.nnet.sigmoid,
+    'clip': lambda x: tt.clip(x, 0, 1),
     'd_sigm': lambda x: tt.nnet.sigmoid(x)-tt.nnet.sigmoid(-x),
     'tanh': tt.tanh,
     'sech2': lambda x: (4*tt.cosh(x)**2)/(tt.cosh(2*x)+1)**2,
@@ -195,7 +200,7 @@ def __call__(self, x):
         return x - running_mean
 
 
-def batchify_function(fcn, batch_size):
+def batchify_function(fcn, batch_size, **scan_kwargs):
     """
     Given a symbolic function, transform it so that computes its input in a sequence of minibatches, instead of in
     one go.  This can be useful when:
@@ -214,10 +219,17 @@ def batchify_function(fcn, batch_size):
     def batch_function(*args):
         start_ixs = tt.arange(0, args[0].shape[0], batch_size)
         @symbolic
-        def process_batch(start_ix, end_ix):
+        def process_batch(start_ix, end_ix, *args):
             return fcn(*[arg[start_ix:end_ix] for arg in args])
-        out = process_batch.scan(sequences = [start_ixs, start_ixs+batch_size])
-        return out.reshape((-1, )+tuple(out.shape[i] for i in xrange(2, out.ndim)), ndim=out.ndim-1)
+
+        out = process_batch.scan(sequences = [start_ixs, start_ixs+batch_size], non_sequences = args, **scan_kwargs)
+        # out = theano.scan(process_batch, sequences = [start_ixs, start_ixs+batch_size])
+        if out is None:
+            return None
+        elif isinstance(out, Variable):
+            return out.reshape((-1, )+tuple(out.shape[i] for i in xrange(2, out.ndim)), ndim=out.ndim-1)
+        else:
+            return out.__class__(o.reshape((-1, )+tuple(o.shape[i] for i in xrange(2, p.ndim)), ndim=o.ndim-1) for o in out)
     return batch_function
 
 

plato/interfaces/interfaces.py

@@ -20,6 +20,14 @@ def set_parameter_states(self, states):
             p.set_value(s)
 
 
+def get_parameters(obj):
+
+    if isinstance(obj, IParameterized) or hasattr(obj, 'parameters'):
+        return obj.parameters
+    else:
+        return []
+
+
 class IFreeEnergy(object):
 
     __metaclass__ = ABCMeta

plato/interfaces/test_helpers.py

@@ -43,6 +43,21 @@ def add_them(a, b):
     assert np.allclose(out, arr_a+arr_b)
 
 
+def test_batch_without_return():
+
+    state = create_shared_variable(np.zeros(2))
+
+    @symbolic
+    def do_something_internal(a, b):
+        new_state = state+ a*b
+        add_update(state, new_state)
+        # return new_state
+
+    out = batchify_function(do_something_internal, batch_size=2).compile()(np.arange(6).astype(float), np.arange(1,7).astype(float))
+    assert out is None
+    assert np.array_equal(state.get_value(), [0*1+2*3+4*5, 1*2+3*4+5*6])
+
+
 def test_compute_in_with_state():
 
     @symbolic
@@ -105,3 +120,4 @@ def accumulate(x):
     test_compute_in_with_state()
     test_on_first_pass()
     test_reshaping_shared_variable()
+    test_batch_without_return()

plato/test_core.py

@@ -1,8 +1,8 @@
 from abc import abstractmethod
 
 from artemis.general.hashing import compute_fixed_hash, fixed_hash_eq
-from plato.interfaces.helpers import create_shared_variable
-from plato.tools.common.config import float_precision
+from plato.interfaces.helpers import create_shared_variable, shared_like
+from plato.tools.common.config import hold_float_precision
 from pytest import raises
 from plato.core import symbolic_simple, symbolic_updater, SymbolicFormatError, \
     tdb_trace, get_tdb_traces, symbolic, set_enable_omniscence, EnableOmniscence, clear_tdb_traces, add_update, \
@@ -516,7 +516,7 @@ def do_some_ops(x):
 
 def test_arbitrary_structures():
 
-    with float_precision(64):
+    with hold_float_precision(64):
         @symbolic
         def my_func(inp):
             """
@@ -607,6 +607,50 @@ def my_cumsum(x):
     assert np.array_equal(get_tdb_traces()['x_in_loop_catch_all'], np.arange(4)**3)
 
 
+def test_easy_scan_syntax():
+
+    @symbolic
+    def accumulator(v, shape):
+        accum = create_shared_variable(np.zeros(shape))
+        new_accum = accum + v
+        add_update(accum, new_accum)
+        return new_accum
+
+    x = np.random.randn(5, 3)
+    f = accumulator.partial(shape=x.shape[1:]).scan.compile()
+
+    assert np.allclose(f(x), np.cumsum(x, axis=0))
+
+
+def test_scan_no_return():
+
+    state = create_shared_variable(np.zeros(()))
+
+    @symbolic
+    def do_something_internal(a, b):
+        new_state = state+ a*b
+        add_update(state, new_state)
+
+    out = do_something_internal.scan.compile()(np.arange(6).astype(float), np.arange(1,7).astype(float))
+
+    assert out is None
+    assert np.array_equal(state.get_value(), np.arange(6).dot(np.arange(1, 7)))
+
+
+def test_none_inputs_and_outputs():
+
+    @symbolic
+    def double_if_not_none(params):
+        return [p*2 if p is not None else None for p in params]
+
+    f = double_if_not_none.compile()
+    assert f([1, 2, 3, None, 4]) == [2, 4, 6, None, 8]
+    assert f([1, 2, 3, None, 4]) == [2, 4, 6, None, 8]
+
+    with pytest.raises(TypeError):  # Warns that you're not calling in consistent way.
+        f([1, 2, 3, 3, 4])
+
+
 if __name__ == '__main__':
     test_ival_ishape()
     test_catch_sneaky_updates()
@@ -630,3 +674,6 @@ def my_cumsum(x):
     test_shared_input()
     test_function_reset()
     test_trace_var_in_scan()
+    test_easy_scan_syntax()
+    test_scan_no_return()
+    test_none_inputs_and_outputs()

plato/tools/common/basic.py

@@ -81,7 +81,7 @@ def running_mean_and_variance(data, decay = None, shape = None, elementwise=True
         var_new = s_new
     add_update(mean_last, mean_new)
     add_update(s_last, s_new)
-    return var_new
+    return mean_new, var_new
 
 
 @symbolic
@@ -93,4 +93,4 @@ def running_variance(data, decay=None, shape = None, elementwise=True, initial_v
     :param shape:
     :return:
     """
-    return running_mean_and_variance(data=data, decay=decay, shape=shape, elementwise=elementwise, initial_var = initial_value)
+    return running_mean_and_variance(data=data, decay=decay, shape=shape, elementwise=elementwise, initial_var = initial_value)[1]

plato/tools/common/config.py

@@ -5,7 +5,7 @@
 
 
 @contextmanager
-def float_precision(value):
+def hold_float_precision(value):
     """
     Change the theano float precesion variable (theano.config.floatX) for all code in a context.  Temporarily overrides
     the value defined in .theanorc.
@@ -25,3 +25,16 @@ def float_precision(value):
     theano.config.floatX = value
     yield
     theano.config.floatX = old_precision
+
+
+float_precision = hold_float_precision # Back-compatibility
+
+
+@contextmanager
+def hold_theano_optimizer(value):
+    if value is None:
+        value = 'None'
+    old_val = theano.config.optimizer
+    theano.config.optimizer = value
+    yield
+    theano.config.optimizer = old_val

plato/tools/common/online_predictors.py

@@ -1,4 +1,6 @@
 from abc import ABCMeta, abstractmethod
+from contextlib import contextmanager
+
 from plato.interfaces.decorators import symbolic_simple, symbolic_updater
 from plato.interfaces.interfaces import IParameterized
 from plato.tools.optimization.cost import get_named_cost_function
@@ -80,13 +82,39 @@ def predict(self, inputs):
     def train(self, inputs, labels):
         feedforward_module = self._function if isinstance(self._function, FeedForwardModule) else ParametrizedFeedForwardModule(self._function)
         feedforward_module.train(x=inputs, y=labels, optimizer=self._optimizer, assert_all_params_optimized = self.assert_all_params_optimized, cost_fcn=self._cost_function, regularization_cost=self._regularization_cost)
+        # if isinstance(self._function, FeedForwardModule):  # A bit ugly but oh well
+        # else:
+        #     outputs = self._function.train_call(inputs) if isinstance(self._function, FeedForwardModule) else self._function(inputs)
+        #     cost = self._cost_function(outputs, labels)
+        #     if self._regularization_cost is not None:
+        #         cost += self._regularization_cost(self._function.parameters)
+        #     self._optimizer.update_parameters(cost = cost, parameters = self._function.parameters)
 
     @property
     def parameters(self):
         opt_params = self._optimizer.parameters if isinstance(self._optimizer, IParameterized) else []
         return self._function.parameters + opt_params
 
 
+_LOCAL_LOSSES = None
+
+
+def declare_local_loss(loss):
+    if _LOCAL_LOSSES is not None:
+        _LOCAL_LOSSES.append(loss)
+
+
+@contextmanager
+def capture_local_losses():
+    global  _LOCAL_LOSSES
+    assert _LOCAL_LOSSES is None, "Local loss book already open"
+    _LOCAL_LOSSES = []
+    try:
+        yield _LOCAL_LOSSES
+    finally:
+        _LOCAL_LOSSES = None
+
+
 class CompiledSymbolicPredictor(IPredictor, IParameterized):
     """
     A Predictor containing the compiled methods for a SymbolicPredictor.
@@ -125,7 +153,12 @@ def __call__(self, x):
         raise NotImplementedError()
 
     def train(self, x, y, cost_fcn, optimizer, assert_all_params_optimized=False, regularization_cost = None):
-        cost = cost_fcn(self.train_call(x), y)
+        with capture_local_losses() as local_losses:
+            cost = cost_fcn(self.train_call(x), y)
+
+        if len(local_losses)>0:
+            cost = cost + sum(local_losses)
+
         if regularization_cost is not None:
             cost = cost + regularization_cost(self.parameters)
         if isinstance(optimizer, dict):

plato/tools/convnet/conv_specifiers.py

@@ -1,6 +1,8 @@
 from artemis.fileman.primitive_specifiers import PrimativeSpecifier
+from artemis.general.numpy_helpers import get_rng
 from artemis.general.should_be_builtins import bad_value
-
+from artemis.ml.tools.neuralnets import initialize_weight_matrix, initialize_conv_kernel
+import numpy as np
 __author__ = 'peter'
 
 
@@ -46,11 +48,18 @@ def __init__(self, w, b, mode):
         """
         assert w.ndim==4
         assert b is False or (b.ndim==1 and w.shape[0] == len(b)), "Number of output maps must match"
-        assert isinstance(mode, int) or mode in ('same', 'valid', 'full'), 'Mode "%s" not allowed' % (mode, )
+        assert isinstance(mode, int) or mode in ('same', 'valid', 'full', 'half'), 'Mode "%s" not allowed' % (mode, )
         self.w=w
         self.b=b
         self.mode = mode
 
+    @staticmethod
+    def from_init(k_shape, mode, mag='xavier', use_biases=True, rng=None):
+        n_out_maps, n_in_maps, k_size_y, k_size_x = k_shape
+        w = initialize_conv_kernel(kernel_shape=k_shape, mag=mag, rng=rng)
+        b = np.zeros(n_out_maps) if use_biases else False
+        return ConvolverSpec(w, b, mode)
+
     def shape_transfer(self, (n_samples, n_maps, size_y, size_x)):
         return (n_samples, self.w.shape[0])+{
             'same': (size_y, size_x),
@@ -112,5 +121,28 @@ def shape_transfer(self, input_shape):
             n_samples, input_dims = input_shape
             return n_samples, self.w.shape[1]
 
+    @classmethod
+    def from_init(cls, n_in, n_out, mag = 'xavier', rng=None):
+        w = initialize_weight_matrix(n_in, n_out, mag=mag, rng=rng)
+        b = np.zeros(n_out)
+        return FullyConnectedSpec(w=w, b=b)
+
+
+class ConvNetSpec(PrimativeSpecifier):
+
+    def __init__(self, layer_ordered_dict):
+        self.layer_ordered_dict = layer_ordered_dict
+
+    def shape_transfer(self):
+        raise NotImplementedError()
+
+
+def compute_feature_shape(input_shape, specs):
+
+    shape = input_shape
+    for spec in specs:
+        shape = spec.shape_transfer(shape)
+
+    return shape
 
 # class ConvNetSpec