An experimental generic runner

.github/workflows/main.yml

@@ -15,18 +15,8 @@ on:
 
 jobs:
   main:
-    name: test-py-${{ matrix.python }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-20.04]
-        python: ["3.10"]
-        include:
-          - os: ubuntu-20.04
-            python: 3.10
-            # the following has no effect with manual trigger
-            # where the ramp-workflow is specified anyway
-            ramp_workflow_version: master
+    name: test-py-3.10
+    runs-on: ubuntu-latest
 
 
     services:
@@ -49,22 +39,23 @@ jobs:
         with:
           update-conda: true
           activate-conda: false
-          python-version: ${{ matrix.python }}
+          python-version: 3.10
           conda-channels: anaconda
 
       - name: Create envs
         run: |
           conda install --yes mamba -n base -c conda-forge
+          rm -f /usr/share/miniconda/pkgs/cache/*.json # workaround for mamba-org/mamba#488
           mamba create --yes -n testenv python=$PYTHON_VERSION
           mamba env update -n testenv -f environment.yml
           mamba env create -f ci_tools/environment_iris_kit.yml
         env:
-          PYTHON_VERSION: ${{ matrix.python }}
+          PYTHON_VERSION: 3.10
 
       - name: Install ramp-board
         run: |
           source activate testenv
-          if [ "$PYTHON_VERSION" == "3.8" ]; then
+          if [ "$PYTHON_VERSION" == "3.10" ]; then
               python -m pip install "dask==2021.4.1" "distributed==2021.4.1"
           fi
           if [ "${{ matrix.ramp_workflow_version }}" == "master" ]; then
@@ -124,13 +115,10 @@ jobs:
       - uses: actions/setup-python@v2
         name: Install Python
         with:
-          python-version: '3.7'
+          python-version: '3.10'
 
       - name: Install dependencies
         run: pip install flake8 black==22.3.0
 
-      - name: Run flake8
-        run: flake8 ramp-*
-
       - name: Run black
         run: black --check .

.gitignore

@@ -28,3 +28,4 @@ db_engine.yml
 prof
 dask-worker-space/
 coverage.xml
+*.swp

doc/workers.rst

@@ -551,7 +551,11 @@ Create an event config.yml (see :ref:`deploy-ramp-event`) and update the
   console, 'Instances' tab on the left, under 'availability zone'.
 * ``ami_image_name``: name you gave to the image you prepared (see
   :ref:`prepare_instance`). This can be found in the EC2 console, under
-  'Images' -> 'AMI' tab.
+  'Images' -> 'AMI' tab. Note: you don't have to put the entire image name
+  and if you indicate the generic name you chose, it will automatically take
+  the latest version of the image created running the pipeline (e.g. 
+  'challenge-iris' will point to 'challenge-iris 2022-04-19T17-19-18.405Z'
+  if it's the latest one)
 * ``ami_user_name``: user name you used to ssh into your instance.
   Commonly 'ec2-user' or 'ubuntu'.
 * ``instance_type``: found in the EC2 console, 'Instances' tab, 'Description'

ramp-database/ramp_database/tools/event.py

@@ -287,11 +287,11 @@ def add_event(
     session.add(event)
     session.commit()
 
-    X_train, y_train = event.problem.get_train_data()
-    cv = event.problem.module.get_cv(X_train, y_train)
-    for train_indices, test_indices in cv:
-        cv_fold = CVFold(event=event, train_is=train_indices, test_is=test_indices)
-        session.add(cv_fold)
+    # X_train, y_train = event.problem.get_train_data()
+    # cv = event.problem.module.get_cv(X_train, y_train)
+    # for train_indices, test_indices in cv:
+    #     cv_fold = CVFold(event=event, train_is=train_indices, test_is=test_indices)
+    #     session.add(cv_fold)
 
     score_types = event.problem.module.score_types
     for score_type in score_types:

ramp-database/ramp_database/tools/leaderboard.py

@@ -46,80 +46,20 @@ def _compute_leaderboard(
     """
     record_score = []
     event = session.query(Event).filter_by(name=event_name).one()
-    map_score_precision = {
-        score_type.name: score_type.precision for score_type in event.score_types
-    }
     for sub in submissions:
         # take only max n bag
-        df_scores_bag = get_bagged_scores(session, sub.id)
-        highest_level = df_scores_bag.index.get_level_values("n_bag").max()
-        df_scores_bag = df_scores_bag.loc[(slice(None), highest_level), :]
-        df_scores_bag.index = df_scores_bag.index.droplevel("n_bag")
-        df_scores_bag = df_scores_bag.round(map_score_precision)
-
-        df_scores = get_scores(session, sub.id)
-        df_scores = df_scores.round(map_score_precision)
-
-        df_time = get_time(session, sub.id)
-        df_time = df_time.stack().to_frame()
-        df_time.index = df_time.index.set_names(["fold", "step"])
-        df_time = df_time.rename(columns={0: "time"})
-        df_time = df_time.groupby(level="step").sum().T
-
-        df_scores_mean = df_scores.groupby("step").mean()
-        df_scores_std = df_scores.groupby("step").std()
-        df_scores_std.fillna(0, inplace=True)
-
-        # select only the validation and testing steps and rename them to
-        # public and private
-        map_renaming = {"valid": "public", "test": "private"}
-        df_scores_mean = (
-            df_scores_mean.loc[list(map_renaming.keys())]
-            .rename(index=map_renaming)
-            .stack()
+        df = (
+            get_bagged_scores(session, sub.id)
+            .reset_index(drop=True)
+            .max(axis=0)
             .to_frame()
             .T
         )
-        df_scores_std = (
-            df_scores_std.loc[list(map_renaming.keys())]
-            .rename(index=map_renaming)
-            .stack()
-            .to_frame()
-            .T
-        )
-        df_scores_bag = df_scores_bag.rename(index=map_renaming).stack().to_frame().T
-
-        df = pd.concat(
-            [df_scores_bag, df_scores_mean, df_scores_std],
-            axis=1,
-            keys=["bag", "mean", "std"],
-        )
-
-        df.columns = df.columns.set_names(["stat", "set", "score"])
-
-        # change the multi-index into a stacked index
-        df.columns = df.columns.map(lambda x: " ".join(x))
-
-        # add the aggregated time information
-        df_time.index = df.index
-        df_time = df_time.rename(
-            columns={
-                "train": "train time [s]",
-                "valid": "validation time [s]",
-                "test": "test time [s]",
-            }
-        )
-        df = pd.concat([df, df_time], axis=1)
 
         if leaderboard_type == "private":
             df["submission ID"] = sub.basename.replace("submission_", "")
         df["team"] = sub.team.name
         df["submission"] = sub.name_with_link if with_links else sub.name
-        df["contributivity"] = int(round(100 * sub.contributivity))
-        df["historical contributivity"] = int(
-            round(100 * sub.historical_contributivity)
-        )
-        df["max RAM [MB]"] = get_submission_max_ram(session, sub.id)
         df["submitted at (UTC)"] = pd.Timestamp(sub.submission_timestamp)
         record_score.append(df)
 
@@ -128,53 +68,9 @@ def _compute_leaderboard(
 
     # keep only second precision for the time stamp
     df["submitted at (UTC)"] = df["submitted at (UTC)"].astype("datetime64[s]")
+    df.columns.name = None
 
-    # reordered the column
-    stats_order = ["bag", "mean", "std"] if leaderboard_type == "private" else ["bag"]
-    dataset_order = (
-        ["public", "private"] if leaderboard_type == "private" else ["public"]
-    )
-    score_order = [event.official_score_name] + [
-        score_type.name
-        for score_type in event.score_types
-        if score_type.name != event.official_score_name
-    ]
-    score_list = [
-        "{} {} {}".format(stat, dataset, score)
-        for dataset, score, stat in product(dataset_order, score_order, stats_order)
-    ]
-    # Only display train and validation time for the public leaderboard
-    time_list = (
-        ["train time [s]", "validation time [s]", "test time [s]"]
-        if leaderboard_type == "private"
-        else ["train time [s]", "validation time [s]"]
-    )
-    col_ordered = (
-        ["team", "submission"]
-        + score_list
-        + ["contributivity", "historical contributivity"]
-        + time_list
-        + ["max RAM [MB]", "submitted at (UTC)"]
-    )
-    if leaderboard_type == "private":
-        col_ordered = ["submission ID"] + col_ordered
-    df = df[col_ordered]
-
-    # check if the contributivity columns are null
-    contrib_columns = ["contributivity", "historical contributivity"]
-    if (df[contrib_columns] == 0).all(axis=0).all():
-        df = df.drop(columns=contrib_columns)
-
-    df = df.sort_values(
-        "bag {} {}".format(leaderboard_type, event.official_score_name),
-        ascending=event.get_official_score_type(session).is_lower_the_better,
-    )
-
-    # rename the column name for the public leaderboard
-    if leaderboard_type == "public":
-        df = df.rename(
-            columns={key: value for key, value in zip(score_list, score_order)}
-        )
+    df = df.sort_values(by="submitted at (UTC)", ascending=False)
     return df
 
 
@@ -207,105 +103,18 @@ def _compute_competition_leaderboard(
         session, submissions, "private", event_name, with_links=False
     )
 
-    time_list = (
-        ["train time [s]", "validation time [s]", "test time [s]"]
-        if leaderboard_type == "private"
-        else ["train time [s]", "validation time [s]"]
-    )
-
-    col_selected_private = (
-        ["team", "submission"]
-        + ["bag private " + score_name, "bag public " + score_name]
-        + time_list
-        + ["submitted at (UTC)"]
-    )
-    leaderboard_df = private_leaderboard[col_selected_private]
-    leaderboard_df = leaderboard_df.rename(
-        columns={
-            "bag private " + score_name: "private " + score_name,
-            "bag public " + score_name: "public " + score_name,
-        }
-    )
+    def _select_best_submission(df):
+        df = df.sort_values('Total cost')
+        # Take lowest score
+        del df['team']
+        return df.iloc[0]
 
     # select best submission for each team
-    best_df = (
-        leaderboard_df.groupby("team").min()
-        if score_type.is_lower_the_better
-        else leaderboard_df.groupby("team").max()
-    )
-    best_df = best_df[["public " + score_name]].reset_index()
-    best_df["best"] = True
-
-    # merge to get a best indicator column then select best
-    leaderboard_df = pd.merge(
-        leaderboard_df,
-        best_df,
-        how="left",
-        left_on=["team", "public " + score_name],
-        right_on=["team", "public " + score_name],
-    )
-    leaderboard_df = leaderboard_df.fillna(False)
-    leaderboard_df = leaderboard_df[leaderboard_df["best"]]
-    leaderboard_df = leaderboard_df.drop(columns="best")
-
-    # dealing with ties: we need the lowest timestamp
-    best_df = leaderboard_df.groupby("team").min()
-    best_df = best_df[["submitted at (UTC)"]].reset_index()
-    best_df["best"] = True
-    leaderboard_df = pd.merge(
-        leaderboard_df,
-        best_df,
-        how="left",
-        left_on=["team", "submitted at (UTC)"],
-        right_on=["team", "submitted at (UTC)"],
-    )
-    leaderboard_df = leaderboard_df.fillna(False)
-    leaderboard_df = leaderboard_df[leaderboard_df["best"]]
-    leaderboard_df = leaderboard_df.drop(columns="best")
-
-    # sort by public score then by submission timestamp, compute rank
-    leaderboard_df = leaderboard_df.sort_values(
-        by=["public " + score_name, "submitted at (UTC)"],
-        ascending=[score_type.is_lower_the_better, True],
-    )
-    leaderboard_df["public rank"] = np.arange(len(leaderboard_df)) + 1
+    best_df = private_leaderboard.groupby("team").apply(_select_best_submission).reset_index()
+    best_df = best_df.sort_values(by="Total cost")
+    best_df.insert(0, 'rank', np.arange(1, best_df.shape[0]+1, dtype=np.int32))
 
-    # sort by private score then by submission timestamp, compute rank
-    leaderboard_df = leaderboard_df.sort_values(
-        by=["private " + score_name, "submitted at (UTC)"],
-        ascending=[score_type.is_lower_the_better, True],
-    )
-    leaderboard_df["private rank"] = np.arange(len(leaderboard_df)) + 1
-
-    leaderboard_df["move"] = (
-        leaderboard_df["public rank"] - leaderboard_df["private rank"]
-    )
-    leaderboard_df["move"] = [
-        "{:+d}".format(m) if m != 0 else "-" for m in leaderboard_df["move"]
-    ]
-
-    col_selected = (
-        [
-            leaderboard_type + " rank",
-            "team",
-            "submission",
-            leaderboard_type + " " + score_name,
-        ]
-        + time_list
-        + ["submitted at (UTC)"]
-    )
-    if leaderboard_type == "private":
-        col_selected.insert(1, "move")
-
-    df = leaderboard_df[col_selected]
-    df = df.rename(
-        columns={
-            leaderboard_type + " " + score_name: score_name,
-            leaderboard_type + " rank": "rank",
-        }
-    )
-    df = df.sort_values(by="rank")
-    return df
+    return best_df
 
 
 def get_leaderboard_all_info(session, event_name):
@@ -429,7 +238,7 @@ def get_leaderboard(
                 "submission",
                 "submitted at (UTC)",
                 "state",
-                "wating list",
+                "waiting list",
             ]
         else:
             columns = ["team", "submission", "submitted at (UTC)", "error"]
@@ -447,7 +256,7 @@ def get_leaderboard(
                         pd.Timestamp(sub.submission_timestamp),
                         (
                             sub.state_with_link
-                            if leaderboard_type == "error"
+                            if leaderboard_type == "failed"
                             else sub.state
                         ),
                         (