Time to merge all this stuff!

.gitignore

@@ -71,4 +71,4 @@ venv/
 # Files
 /Data
 /docs/build
-
+.pytest_cache

artemis/experiments/decorators.py

@@ -44,7 +44,7 @@ class ExperimentFunction(object):
     This is the most general decorator.  You can use this to add details on the experiment.
     """
 
-    def __init__(self, show = show_record, compare = compare_experiment_records, display_function=None, comparison_function=None, one_liner_function=sensible_str, is_root=False):
+    def __init__(self, show = None, compare = compare_experiment_records, display_function=None, comparison_function=None, one_liner_function=None, result_parser = None, is_root=False, name=None):
         """
         :param show:  A function that is called when you "show" an experiment record in the UI.  It takes an experiment
             record as an argument.
@@ -55,16 +55,17 @@ def __init__(self, show = show_record, compare = compare_experiment_records, dis
             You can use call this via the UI with the compare_experiment_results command.
         :param one_liner_function: A function that takes your results and returns a 1 line string summarizing them.
         :param is_root: True to make this a root experiment - so that it is not listed to be run itself.
+        :param name: Custom name (if None, experiment will be named after decorated function)
         """
         self.show = show
         self.compare = compare
 
         if display_function is not None:
-            assert show is show_record, "You can't set both display function and show.  (display_function is deprecated)"
+            assert show is None, "You can't set both display function and show.  (display_function is deprecated)"
             show = lambda rec: display_function(rec.get_result())
 
         if comparison_function is not None:
-            assert compare is compare_experiment_records, "You can't set both display function and show.  (display_function is deprecated)"
+            assert compare is None, "You can't set both display function and show.  (display_function is deprecated)"
 
             def compare(records):
                 record_experiment_ids_uniquified = uniquify_duplicates(rec.get_experiment_id() for rec in records)
@@ -74,15 +75,23 @@ def compare(records):
         self.compare = compare
         self.is_root = is_root
         self.one_liner_function = one_liner_function
+        self.result_parser = result_parser
+        self.name = name
 
     def __call__(self, f):
+        """
+        :param Callable f: The function you decorated
+        :return Experiment: An Experiment object (It still behaves as the original function when you call it, but now
+            has additional methods attached to it associated with the experiment).
+        """
         f.is_base_experiment = True
         ex = Experiment(
-            name=f.__name__,
+            name=f.__name__ if self.name is None else self.name,
             function=f,
             show=self.show,
             compare = self.compare,
             one_liner_function=self.one_liner_function,
-            is_root=self.is_root
+            is_root=self.is_root,
+            result_parser=self.result_parser,
         )
         return ex

artemis/experiments/experiment_management.py

@@ -64,7 +64,7 @@ def pull_experiment_records(user, ip, experiment_names, include_variants=True, n
         # +["--include='**/*-{exp_name}{variants}/*'".format(exp_name=exp_name, variants = '*' if include_variants else '') for exp_name in experiment_names]  # This was the old line, but it could be too long for many experiments.
 
     if not need_pass:
-        output = subprocess.check_output(command)
+        output = subprocess.check_output(' '.join(command), shell=True)
     else:
         # This one works if you need a password
         password = getpass.getpass("Enter password for {}@{}:".format(user, ip))
@@ -126,7 +126,11 @@ def select_experiments(user_range, exp_record_dict, return_dict=False):
 
 def _filter_experiments(user_range, exp_record_dict, return_is_in = False):
 
-    if user_range.startswith('~'):
+    if '|' in user_range:
+        is_in = [any(xs) for xs in zip(*(_filter_experiments(subrange, exp_record_dict, return_is_in=True) for subrange in user_range.split('|')))]
+    elif '&' in user_range:
+        is_in = [all(xs) for xs in zip(*(_filter_experiments(subrange, exp_record_dict, return_is_in=True) for subrange in user_range.split('&')))]
+    elif user_range.startswith('~'):
         is_in = _filter_experiments(user_range=user_range[1:], exp_record_dict=exp_record_dict, return_is_in=True)
         is_in = [not r for r in is_in]
     else:
@@ -141,6 +145,9 @@ def _filter_experiments(user_range, exp_record_dict, return_is_in = False):
             elif user_range.startswith('has:'):
                 phrase = user_range[len('has:'):]
                 is_in = [phrase in exp_id for exp_id in exp_record_dict]
+            elif user_range.startswith('tag:'):
+                tag = user_range[len('tag:'):]
+                is_in = [tag in load_experiment(exp_id).get_tags() for exp_id in exp_record_dict]
             elif user_range.startswith('1diff:'):
                 base_range = user_range[len('1diff:'):]
                 base_range_exps = select_experiments(base_range, exp_record_dict) # list<experiment_id>
@@ -308,7 +315,7 @@ def _filter_records(user_range, exp_record_dict):
         try:
             sign = user_range[3]
             assert sign in ('<', '>')
-            filter_func = (lambda a, b: a<b) if sign == '<' else (lambda a, b: a>b)
+            filter_func = (lambda a, b: (a is not None and b is not None) and a<b) if sign == '<' else (lambda a, b: (a is not None and b is not None) and a>b)
             time_delta = parse_time(user_range[4:])
         except:
             if user_range.startswith('dur'):
@@ -323,6 +330,10 @@ def _filter_records(user_range, exp_record_dict):
             current_time = datetime.now()
             for exp_id, _ in base.items():
                 base[exp_id] = [filter_func(current_time - load_experiment_record(rec_id).get_datetime(), time_delta) for rec_id in exp_record_dict[exp_id]]
+    elif user_range.startswith('has:'):
+        phrase = user_range[len('has:'):]
+        for exp_id, records in base.items():
+            base[exp_id] = [True]*len(records) if phrase in exp_id else [False]*len(records)
     else:
         raise RecordSelectionError("Don't know how to interpret subset '{}'.  Possible subsets: {}".format(user_range, list(_named_record_filters.keys())))
     return base
@@ -534,6 +545,34 @@ def run_multiple_experiments(experiments, prefixes = None, parallel = False, dis
         return [ex.run(raise_exceptions=raise_exceptions, display_results=display_results, notes=notes, **run_args) for ex in experiments]
 
 
+def get_multiple_records(experiment, n, only_completed=True, if_not_enough='run'):
+    """
+    Get n records from a single experiment.
+    :param Experiment experiment: The experiment
+    :param int n: Number of records to get
+    :param only_completed: True if you only want completed records
+    :param if_not_enough: What to do if there are not enough records ready.
+        'run': Run more
+        'cut': Just return the number that are already calculated
+        'err': Raise an excepetion
+    :return:
+    """
+    if isinstance(experiment, str):
+        experiment = load_experiment(experiment)
+    assert if_not_enough in ('run', 'cut', 'err')
+    records = experiment.get_records(only_completed=only_completed)
+    if if_not_enough == 'err':
+        assert len(records) >= n, "You asked for {} records, but only {} were available".format(n, len(records))
+        return records[-n:]
+    elif if_not_enough=='run':
+        for k in range(n-len(records)):
+            record = experiment.run()
+            records.append(record)
+        return records[-n:]
+    else:
+        return records
+
+
 def remove_common_results_prefix(results_dict):
     """
     Remove the common prefix for the results you are comparing.

artemis/experiments/experiment_record.py

@@ -13,6 +13,7 @@
 from contextlib import contextmanager
 from getpass import getuser
 from pickle import PicklingError
+import itertools
 
 from datetime import datetime, timedelta
 from uuid import getnode
@@ -23,7 +24,7 @@
 from artemis.general.display import CaptureStdOut
 from artemis.general.functional import get_partial_chain, get_defined_and_undefined_args
 from artemis.general.hashing import compute_fixed_hash
-from artemis.general.should_be_builtins import nested
+from artemis.general.should_be_builtins import nested, natural_keys
 from artemis.general.test_mode import is_test_mode
 from artemis.general.test_mode import set_test_mode
 from artemis._version import __version__ as ARTEMIS_VERSION
@@ -240,6 +241,7 @@ def get_figure_locs(self, include_directory=True):
         :return: A list of string file paths.
         """
         locs = [f for f in os.listdir(self._experiment_directory) if f.startswith('fig-')]
+        locs = sorted(locs, key=natural_keys)
         if include_directory:
             return [os.path.join(self._experiment_directory, f) for f in locs]
         else:
@@ -289,7 +291,7 @@ def get_experiment(self):
         """
         Load the experiment associated with this record.
         Note that this will raise an ExperimentNotFoundError if the experiment has not been imported.
-        :return: An Experiment object
+        :return Experiment: An Experiment object
         """
         from artemis.experiments.experiments import load_experiment
         return load_experiment(self.get_experiment_id())
@@ -310,7 +312,10 @@ def get_runtime(self):
         """
         :return datetime.timedelta: A timedelta object
         """
-        return timedelta(seconds=self.info.get_field(ExpInfoFields.RUNTIME))
+        try:
+            return timedelta(seconds=self.info.get_field(ExpInfoFields.RUNTIME))
+        except KeyError:  # Which will happen if the experiment is still running or was killed without due process
+            return None
 
     def get_dir(self):
         """
@@ -506,6 +511,7 @@ def get_current_record_id():
 def get_current_record_dir(default_if_none = True):
     """
     The directory in which the results of the current experiment are recorded.
+    :param default_if_none: True to put records in the "default" dir if no experiment is running.
     """
     if _CURRENT_EXPERIMENT_RECORD is None and default_if_none:
         return get_artemis_data_path('experiments/default/', make_local_dir=True)
@@ -657,7 +663,34 @@ def clear_experiment_records(ids):
         ExperimentRecord(exp_path).delete()
 
 
-def save_figure_in_record(name, fig=None, default_ext='.pkl'):
+#
+#
+# def save_figure_in_current_experiment_directory(name='fig-{}.pkl', figure = None):
+#
+#     if figure is None:
+#         figure = plt.gcf()
+#
+#     current_dir = get_current_record_dir()
+#     start_ix = _figure_ixs[current_dir] if current_dir in _figure_ixs else 0
+#     for ix in count(start_ix):
+#         full_path = os.path.join(current_dir, name).format(ix)
+#         if not os.path.exists(_figure_ixs[current_dir]):
+#             save_figure(figure, path = full_path)
+#             _figure_ixs[current_dir] = ix+1
+#             return full_path
+_figure_ixs = {}
+
+
+def _get_next_figure_name(name_pattern, directory):
+    start_ix = _figure_ixs[directory] if directory in _figure_ixs else 0
+    for ix in itertools.count(start_ix):
+        full_path = os.path.join(directory, name_pattern).format(ix)
+        if not os.path.exists(full_path):
+            _figure_ixs[directory] = ix+1
+            return full_path
+
+
+def save_figure_in_record(name=None, fig=None, default_ext='.pkl'):
     '''
     Saves the given figure in the experiment directory. If no figure is passed, plt.gcf() is saved instead.
     :param name: The name of the figure to be saved
@@ -667,11 +700,17 @@ def save_figure_in_record(name, fig=None, default_ext='.pkl'):
     '''
     import matplotlib.pyplot as plt
     from artemis.plotting.saving_plots import save_figure
+
     if fig is None:
         fig = plt.gcf()
-    save_path = os.path.join(get_current_record_dir(), name)
-    save_figure(fig, path=save_path, default_ext=default_ext)
-    return save_path
+
+    current_dir = get_current_record_dir()
+    if name is None:
+        path = _get_next_figure_name(name_pattern='fig-{}.pkl', directory=current_dir)
+    else:
+        path = os.path.join(current_dir, name)
+    save_figure(fig, path=path, default_ext=default_ext)
+    return path
 
 
 def get_serialized_args(argdict):