predictor_robojudge.ipynb

{
 "cells": [
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# Copyright 2020 (c) Cognizant Digital Business, Evolutionary AI. All rights reserved. Issued under the Apache 2.0 License."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "LATEST_DATA_URL = \"https://raw.githubusercontent.com/OxCGRT/covid-policy-tracker-legacy/main/legacy_data_202207/OxCGRT_latest.csv\"\n",
    "GEO_FILE = \"countries_regions.csv\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from covid_xprize.scoring.predictor_scoring import load_dataset\n",
    "\n",
    "latest_df = load_dataset(LATEST_DATA_URL, GEO_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "latest_df.sample(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Expecting 180 countries\n",
    "len(latest_df.CountryName.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Expecting 56 regions, plus the empty one '' = 57\n",
    "len(latest_df.RegionName.unique())"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_date_str = \"2020-12-01\"\n",
    "end_date_str = \"2020-12-31\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_date = pd.to_datetime(start_date_str, format='%Y-%m-%d')\n",
    "end_date = pd.to_datetime(end_date_str, format='%Y-%m-%d')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get actual cases between these dates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "NUM_PREV_DAYS_TO_INCLUDE = 6\n",
    "WINDOW_SIZE = 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_actual_cases(df, start_date, end_date):\n",
    "    # 1 day earlier to compute the daily diff\n",
    "    start_date_for_diff = start_date - pd.offsets.Day(WINDOW_SIZE)\n",
    "    actual_df = df[[\"CountryName\", \"RegionName\", \"Date\", \"ConfirmedCases\"]]\n",
    "    # Filter out the data set to include all the data needed to compute the diff\n",
    "    actual_df = actual_df[(actual_df.Date >= start_date_for_diff) & (actual_df.Date <= end_date)]\n",
    "    # Add GeoID column that combines CountryName and RegionName for easier manipulation of data\n",
    "    # np.where usage: if A then B else C\n",
    "    actual_df[\"GeoID\"] = np.where(actual_df[\"RegionName\"] == '',\n",
    "                                  actual_df[\"CountryName\"],\n",
    "                                  actual_df[\"CountryName\"] + ' / ' + actual_df[\"RegionName\"])\n",
    "    actual_df.sort_values(by=[\"GeoID\",\"Date\"], inplace=True)\n",
    "    # Compute the diff\n",
    "    actual_df[\"ActualDailyNewCases\"] = actual_df.groupby(\"GeoID\", group_keys=False)[\"ConfirmedCases\"].diff()\n",
    "    # Compute the 7 day moving average\n",
    "    actual_df[\"ActualDailyNewCases7DMA\"] = actual_df.groupby(\n",
    "        \"GeoID\", group_keys=False)['ActualDailyNewCases'].rolling(\n",
    "        WINDOW_SIZE, center=False).mean().reset_index(0, drop=True)\n",
    "    return actual_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "actual_df = get_actual_cases(latest_df, start_date, end_date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from covid_xprize.oxford_data import add_population_column\n",
    "\n",
    "actual_df = add_population_column(actual_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "actual_df.head(12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ID_COLS = ['CountryName',\n",
    "           'RegionName',\n",
    "           'Date']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add empty rows until end date, if needed, to simplify merge with teams predictions\n",
    "new_rows = []\n",
    "for g in actual_df.GeoID.unique():\n",
    "    gdf = actual_df[actual_df.GeoID == g]\n",
    "    last_known_date = gdf.Date.max()\n",
    "    current_date = last_known_date + np.timedelta64(1, 'D')\n",
    "    while current_date <= end_date:\n",
    "        new_row = [gdf.CountryName.unique()[0],\n",
    "                   gdf.RegionName.unique()[0],\n",
    "                   current_date,\n",
    "                   np.nan,\n",
    "                   g,\n",
    "                   np.nan,\n",
    "                   np.nan,\n",
    "                   gdf.Population.unique()[0]\n",
    "                  ]\n",
    "        new_rows.append(new_row)\n",
    "        # Move to next day\n",
    "        current_date = current_date + np.timedelta64(1, 'D')\n",
    "# Add the new rows\n",
    "if new_rows:\n",
    "    new_rows_df = pd.DataFrame(new_rows, columns=actual_df.columns)\n",
    "    # Append the new rows\n",
    "    actual_df = pd.concat([actual_df, new_rows_df])\n",
    "# Sort\n",
    "actual_df.sort_values(by=ID_COLS, inplace=True, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "actual_df.head(40)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get historical data for 7 days moving average calculation\n",
    "In order to compute the 7 days moving average, we need to get the historical true new cases for the last 7 days before start date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ma_df = actual_df[actual_df[\"Date\"] < start_date]\n",
    "ma_df = ma_df[[\"CountryName\", \"RegionName\", \"Date\", \"ActualDailyNewCases\"]]\n",
    "ma_df = ma_df.rename(columns={\"ActualDailyNewCases\": \"PredictedDailyNewCases\"})\n",
    "ma_df.head()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run the predictions\n",
    "Evaluate some example submissions.  \n",
    "__NOTE: Please run the corresponding example notebooks first in order to train the models that are used in this section.__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "IP_FILE = \"predictions/robojudge_test_scenario.csv\"\n",
    "if not os.path.exists('predictions'):\n",
    "    os.mkdir('predictions')\n",
    "predictions = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from covid_xprize.validation.scenario_generator import generate_scenario\n",
    "\n",
    "countries = None\n",
    "scenario_df = generate_scenario(start_date_str, end_date_str, latest_df, countries, scenario=\"Freeze\")\n",
    "scenario_df.to_csv(IP_FILE, index=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Linear"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check a model has been trained\n",
    "if not os.path.isfile(\"covid_xprize/examples/predictors/linear/models/model.pkl\"):\n",
    "    print(\"ERROR: Please run the notebook in 'covid_xprize/examples/predictors/linear' in order to train a model!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "linear_output_file = \"covid_xprize/examples/predictors/linear/predictions/robojudge_test.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python3 -m covid_xprize.examples.predictors.linear.predict -s {start_date_str} -e {end_date_str} -ip {IP_FILE} -o {linear_output_file}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions[\"Linear\"] = linear_output_file"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check a model has been trained\n",
    "if not os.path.isfile(\"covid_xprize/examples/predictors/lstm/models/trained_model_weights.h5\"):\n",
    "    print(\"ERROR: Please run the notebook in 'covid_xprize/examples/predictors/lstm' in order to train a model!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm_output_file = \"covid_xprize/examples/predictors/lstm/predictions/robojudge_test.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python3 -m covid_xprize.examples.predictors.lstm.predict -s {start_date_str} -e {end_date_str} -ip {IP_FILE} -o {lstm_output_file}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions[\"LSTM\"] = lstm_output_file"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Conditional LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check a model has been trained\n",
    "if not os.path.isfile(\"covid_xprize/examples/predictors/conditional_lstm/models/trained_model_weights.h5\"):\n",
    "    print(\"ERROR: Please run the notebook in 'covid_xprize/examples/predictors/conditional_lstm' in order to train a model!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "conditional_lstm_output_file = \"covid_xprize/examples/predictors/conditional_lstm/predictions/robojudge_test.csv\"\n",
    "!rm {conditional_lstm_output_file}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python3 -m covid_xprize.examples.predictors.conditional_lstm.predict -s {start_date_str} -e {end_date_str} -ip {IP_FILE} -o {conditional_lstm_output_file}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions[\"Conditional_LSTM\"] = conditional_lstm_output_file"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get predictions from submissions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_predictions_from_file(predictor_name, predictions_file, ma_df):\n",
    "    preds_df = pd.read_csv(predictions_file,\n",
    "                           parse_dates=['Date'],\n",
    "                           encoding=\"ISO-8859-1\",\n",
    "                           on_bad_lines='skip')\n",
    "    preds_df[\"RegionName\"] = preds_df[\"RegionName\"].fillna(\"\")\n",
    "    preds_df[\"PredictorName\"] = predictor_name\n",
    "    preds_df[\"Prediction\"] = True\n",
    "    \n",
    "    # Append the true number of cases before start date\n",
    "    ma_df[\"PredictorName\"] = predictor_name\n",
    "    ma_df[\"Prediction\"] = False\n",
    "    preds_df = pd.concat([ma_df, preds_df], ignore_index=True)\n",
    "\n",
    "    # Add GeoID column that combines CountryName and RegionName for easier manipulation of data\n",
    "    # np.where usage: if A then B else C\n",
    "    preds_df[\"GeoID\"] = np.where(preds_df[\"RegionName\"] == '',\n",
    "                                 preds_df[\"CountryName\"],\n",
    "                                 preds_df[\"CountryName\"] + ' / ' + preds_df[\"RegionName\"])\n",
    "    # Sort\n",
    "    preds_df.sort_values(by=[\"GeoID\",\"Date\"], inplace=True)\n",
    "    # Compute the 7 days moving average for PredictedDailyNewCases\n",
    "    preds_df[\"PredictedDailyNewCases7DMA\"] = preds_df.groupby(\n",
    "        \"GeoID\", group_keys=False)['PredictedDailyNewCases'].rolling(\n",
    "        WINDOW_SIZE, center=False).mean().reset_index(0, drop=True)\n",
    "\n",
    "    # Put PredictorName first\n",
    "    preds_df = preds_df[[\"PredictorName\"] + [col for col in preds_df.columns if col != \"PredictorName\"]]\n",
    "    return preds_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_predictor_name = \"Linear\"\n",
    "temp_df = get_predictions_from_file(test_predictor_name, predictions[test_predictor_name], ma_df.copy())\n",
    "temp_df.head(12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "actual_df.head(8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from covid_xprize.validation.predictor_validation import validate_submission\n",
    "\n",
    "ranking_df = pd.DataFrame()\n",
    "for predictor_name, predictions_file in predictions.items():\n",
    "    print(f\"Getting {predictor_name}'s predictions from: {predictions_file}\")\n",
    "    errors = validate_submission(start_date_str, end_date_str, IP_FILE, predictions_file)\n",
    "    if not errors:\n",
    "        preds_df = get_predictions_from_file(predictor_name, predictions_file, ma_df)\n",
    "        merged_df = actual_df.merge(preds_df, on=['CountryName', 'RegionName', 'Date', 'GeoID'], how='left')\n",
    "        ranking_df = pd.concat([ranking_df, merged_df])\n",
    "    else:\n",
    "        print(f\"Predictor {predictor_name} did not submit valid predictions! Please check its errors:\")\n",
    "        print(errors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Keep only predictions (either Prediction == True) or on or after start_date\n",
    "ranking_df = ranking_df[ranking_df[\"Date\"] >= start_date]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from covid_xprize.scoring.predictor_scoring import add_predictor_performance_columns\n",
    "\n",
    "# Compute performance measures\n",
    "ranking_df = add_predictor_performance_columns(ranking_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ranking_df.head(4*2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ranking_df[(ranking_df.CountryName == \"United States\") &\n",
    "           (ranking_df.Date == '2020-12-02')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save to file\n",
    "# ranking_df.to_csv(\"/Users/m_754337/workspace/esp-demo/xprize/tests/fixtures/ranking.csv\", index=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ranking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter to get the metrics for the final day\n",
    "last_known_date = ranking_df[ranking_df[\"Cumul-7DMA-MAE-per-100K\"].notnull()].Date.max()\n",
    "leaderboard_df = ranking_df[ranking_df.Date == last_known_date][[\n",
    "        \"CountryName\",\n",
    "        \"RegionName\",\n",
    "        \"PredictorName\",\n",
    "        \"Cumul-7DMA-MAE-per-100K\",\n",
    "        \"PredictorRank\"]]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Global"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "leaderboard_df.groupby([\"PredictorName\"], group_keys=False).mean() \\\n",
    "    .sort_values(by=[\"Cumul-7DMA-MAE-per-100K\"]).reset_index()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Countries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "leaderboard_df.sort_values(by=['CountryName', 'RegionName'])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Specific country"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "leaderboard_df[(leaderboard_df.CountryName == \"Italy\") &\n",
    "               (leaderboard_df.RegionName == \"\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ranking_df[ranking_df.CountryName == \"Italy\"]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Specific region"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "leaderboard_df[(leaderboard_df.CountryName == \"United States\") &\n",
    "               (leaderboard_df.RegionName == \"California\")]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Continent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "NORTH_AMERICA = [\"Canada\", \"United States\", \"Mexico\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "leaderboard_df[(leaderboard_df.CountryName.isin(NORTH_AMERICA)) &\n",
    "               (leaderboard_df.RegionName == \"\")].groupby('PredictorName', group_keys=False).mean() \\\n",
    "    .sort_values(by=[\"Cumul-7DMA-MAE-per-100K\"]).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "leaderboard_df[(leaderboard_df.CountryName.isin(NORTH_AMERICA)) &\n",
    "               (leaderboard_df.RegionName == \"\")]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ALL_GEO = \"Overall\"\n",
    "DEFAULT_GEO = ALL_GEO"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prediction vs actual"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictor_names = list(ranking_df.PredictorName.dropna().unique())\n",
    "geoid_names = list(ranking_df.GeoID.unique())"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filter by country"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_df = ranking_df.groupby([\"PredictorName\", \"Date\"], group_keys=False)[[\"GeoID\", \"PredictorName\", \"PredictedDailyNewCases7DMA\"]].sum(). \\\n",
    "    sort_values(by=[\"PredictorName\", \"Date\"]).reset_index()\n",
    "all_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import plotly.graph_objects as go\n",
    "\n",
    "fig = go.Figure(layout=dict(title=dict(text=f\"{DEFAULT_GEO} Daily New Cases 7-day Average \",\n",
    "                                       y=0.9,\n",
    "                                       x=0.5,\n",
    "                                       xanchor='center',\n",
    "                                       yanchor='top'\n",
    "                                       ),\n",
    "                             plot_bgcolor='#f2f2f2',\n",
    "                             xaxis_title=\"Date\",\n",
    "                             yaxis_title=\"Daily new cases 7-day average\"\n",
    "                             ))\n",
    "\n",
    "# Keep track of trace visibility by geo ID name\n",
    "geoid_plot_names = []\n",
    "\n",
    "all_df = ranking_df.groupby([\"PredictorName\", \"Date\"], group_keys=False)[[\"GeoID\", \"PredictorName\", \"PredictedDailyNewCases7DMA\"]].sum(). \\\n",
    "    sort_values(by=[\"PredictorName\", \"Date\"]).reset_index()\n",
    "\n",
    "# Add 1 trace per predictor, for all geos\n",
    "for predictor_name in predictor_names:\n",
    "    all_geo_df = all_df[all_df.PredictorName == predictor_name]\n",
    "    fig.add_trace(go.Scatter(x=all_geo_df.Date,\n",
    "                             y=all_geo_df.PredictedDailyNewCases7DMA,\n",
    "                             name=predictor_name,\n",
    "                             visible=(ALL_GEO == DEFAULT_GEO))\n",
    "                 )\n",
    "    geoid_plot_names.append(ALL_GEO)\n",
    "\n",
    "# Add 1 trace per predictor, per geo id\n",
    "for predictor_name in predictor_names:\n",
    "    for geoid_name in geoid_names:\n",
    "        pred_geoid_df = ranking_df[(ranking_df.GeoID == geoid_name) &\n",
    "                                   (ranking_df.PredictorName == predictor_name)]\n",
    "        fig.add_trace(go.Scatter(x=pred_geoid_df.Date,\n",
    "                                 y=pred_geoid_df.PredictedDailyNewCases7DMA,\n",
    "                                 name=predictor_name,\n",
    "                                 visible=(geoid_name == DEFAULT_GEO))\n",
    "                     )\n",
    "        geoid_plot_names.append(geoid_name)\n",
    "\n",
    "# For each geo\n",
    "# Add 1 trace for the true number of cases\n",
    "for geoid_name in geoid_names:\n",
    "    # Only plot data that is known, for ground truth\n",
    "    last_known_date = actual_df[actual_df.ActualDailyNewCases7DMA.notnull()].Date.max()\n",
    "    geo_actual_df = actual_df[(actual_df.GeoID == geoid_name) &\n",
    "                                  (actual_df.Date >= start_date)]\n",
    "    fig.add_trace(go.Scatter(x=geo_actual_df[actual_df.Date <= last_known_date].Date,\n",
    "                             y=geo_actual_df[actual_df.Date <= last_known_date].ActualDailyNewCases7DMA,\n",
    "                             name=\"Ground Truth\",\n",
    "                             visible= (geoid_name == DEFAULT_GEO),\n",
    "                             line=dict(color='orange', width=4, dash='dash'))\n",
    "                  )\n",
    "    geoid_plot_names.append(geoid_name)\n",
    "    \n",
    "# Add 1 trace for the overall ground truth\n",
    "overall_actual_df = actual_df[actual_df.Date >= start_date].groupby([\"Date\"], group_keys=False)[[\"GeoID\", \"ActualDailyNewCases7DMA\"]].sum(). \\\n",
    "    sort_values(by=[\"Date\"]).reset_index()\n",
    "overall_last_known_date = overall_actual_df[overall_actual_df.ActualDailyNewCases7DMA > 0].Date.max()\n",
    "fig.add_trace(go.Scatter(x=overall_actual_df[overall_actual_df.Date <= overall_last_known_date].Date,\n",
    "                         y=overall_actual_df[overall_actual_df.Date <= overall_last_known_date].ActualDailyNewCases7DMA,\n",
    "                         name=\"Ground Truth\",\n",
    "                         visible= (ALL_GEO == DEFAULT_GEO),\n",
    "                         line=dict(color='orange', width=4, dash='dash'))\n",
    "                  )\n",
    "geoid_plot_names.append(geoid_name)\n",
    "\n",
    "# Format x axis\n",
    "fig.update_xaxes(\n",
    "dtick=\"D1\",  # Means 1 day\n",
    "tickformat=\"%d\\n%b\")\n",
    "\n",
    "# Filter\n",
    "buttons=[]\n",
    "for geoid_name in ([ALL_GEO] + geoid_names):\n",
    "    buttons.append(dict(method='update',\n",
    "                        label=geoid_name,\n",
    "                        args = [{'visible': [geoid_name==r for r in geoid_plot_names]},\n",
    "                                {'title': f\"{geoid_name} Daily New Cases 7-day Average \"}]))\n",
    "fig.update_layout(showlegend=True,\n",
    "                  updatemenus=[{\"buttons\": buttons,\n",
    "                                \"direction\": \"down\",\n",
    "                                \"active\": ([ALL_GEO] + geoid_names).index(DEFAULT_GEO),\n",
    "                                \"showactive\": True,\n",
    "                                \"x\": 0.1,\n",
    "                                \"y\": 1.15}])\n",
    "\n",
    "fig.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rankings: by cumulative 7DMA error per 100K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ranking_fig = go.Figure(layout=dict(title=dict(text=f'{DEFAULT_GEO} submission rankings',\n",
    "                                               y=0.9,\n",
    "                                               x=0.5,\n",
    "                                               xanchor='center',\n",
    "                                               yanchor='top'\n",
    "                                               ),\n",
    "                                    plot_bgcolor='#f2f2f2',\n",
    "                                    xaxis_title=\"Date\",\n",
    "                                    yaxis_title=\"MAE per 100K\"\n",
    "                                    ))\n",
    "\n",
    "# Keep track of trace visibility by geo name\n",
    "ranking_geoid_plot_names = []\n",
    "\n",
    "all_df = ranking_df.groupby([\"PredictorName\", \"Date\"], group_keys=False)[[\"GeoID\", \"PredictorName\", \"Cumul-7DMA-MAE-per-100K\"]].mean(). \\\n",
    "    sort_values(by=[\"PredictorName\", \"Date\"]).reset_index()\n",
    "\n",
    "# Add 1 trace per predictor, for all geos\n",
    "for predictor_name in predictor_names:\n",
    "    ranking_geoid_df = all_df[all_df.PredictorName == predictor_name]\n",
    "    ranking_fig.add_trace(go.Scatter(x=ranking_geoid_df.Date,\n",
    "                             y=ranking_geoid_df['Cumul-7DMA-MAE-per-100K'],\n",
    "                             name=predictor_name,\n",
    "                             visible=(ALL_GEO == DEFAULT_GEO))\n",
    "                 )\n",
    "    ranking_geoid_plot_names.append(ALL_GEO)\n",
    "\n",
    "\n",
    "# Add 1 trace per predictor, per country\n",
    "for predictor_name in predictor_names:\n",
    "    for geoid_name in geoid_names:\n",
    "        ranking_geoid_df = ranking_df[(ranking_df.GeoID == geoid_name) &\n",
    "                                        (ranking_df.PredictorName == predictor_name)]\n",
    "        ranking_fig.add_trace(go.Scatter(x=ranking_geoid_df.Date,\n",
    "                                 y=ranking_geoid_df['Cumul-7DMA-MAE-per-100K'],\n",
    "                                 name=predictor_name,\n",
    "                                 visible= (geoid_name == DEFAULT_GEO))\n",
    "                     )\n",
    "        ranking_geoid_plot_names.append(geoid_name)\n",
    "\n",
    "# Format x axis\n",
    "ranking_fig.update_xaxes(\n",
    "dtick=\"D1\",  # Means 1 day\n",
    "tickformat=\"%d\\n%b\")\n",
    "\n",
    "# Filter\n",
    "buttons=[]\n",
    "for geoid_name in ([ALL_GEO] + geoid_names):\n",
    "    buttons.append(dict(method='update',\n",
    "                        label=geoid_name,\n",
    "                        args = [{'visible': [geoid_name==r for r in ranking_geoid_plot_names]},\n",
    "                                {'title': f'{geoid_name} submission rankings'}]))\n",
    "ranking_fig.update_layout(showlegend=True,\n",
    "                          updatemenus=[{\"buttons\": buttons,\n",
    "                                        \"direction\": \"down\",\n",
    "                                        \"active\": ([ALL_GEO] + geoid_names).index(DEFAULT_GEO),\n",
    "                                        \"showactive\": True,\n",
    "                                        \"x\": 0.1,\n",
    "                                        \"y\": 1.15}])\n",
    "\n",
    "ranking_fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": [
     "# Copyright 2020 (c) Cognizant Digital Business, Evolutionary AI. All rights reserved. Issued under the Apache 2.0 License.#%%\n",
     "\n",
     "import pandas as pd\n",
     "import numpy as np\n"
    ]
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}