submit_nn_cv_ensemble.ipynb

{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib as mpl\nimport seaborn as sns\nimport datetime\nimport tqdm\nimport re\nimport time\nfrom string import punctuation\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.preprocessing import MinMaxScaler\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\n\nimport torch, torch.nn as nn\nimport torch.nn.functional as F\n\nsns.set_style('darkgrid')\nmpl.rcParams['figure.figsize'] = [15,10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from kaggle.competitions import nflrush","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom string import punctuation\nimport datetime\nimport re\nimport torch\nimport math\nimport tqdm\n\n\ndef crps(y_true, y_pred):\n    return np.mean(np.square(y_true - y_pred), axis=1)\n\n\ndef yard_to_cdf(yard):\n    yard = np.round(yard).astype(int)\n    indices = yard + 99\n    cdfs = np.zeros((yard.shape[0], 199))\n    for i in range(len(cdfs)):\n        cdfs[i, indices[i]:] = 1\n    return cdfs\n\n\ndef cdf_to_yard(cdf):\n    yard_index = (cdf == 1).argmax(axis=1)\n    yard = yard_index - 99\n    return yard\n\n\ndef cdf_to_yard_torch(cdf):\n    yard_index = torch.sum((torch.as_tensor(cdf) <= 0), dim=1)\n    yard = yard_index - 99\n    return yard\n\n\ndef crps_torch(y_true, y_pred):\n    y_true = torch.as_tensor(y_true)\n    y_pred = torch.as_tensor(y_pred)\n    return torch.mean((y_true - y_pred).pow(2), dim=1)\n\n\ndef crps_loss(y_true, y_pred_pdf):\n    y_pred_cdf = torch.cumsum(torch.as_tensor(y_pred_pdf), dim=1)\n    return crps_torch(y_true, y_pred_cdf).mean()\n\n\ndef crps_loss_cdf(y_true, y_pred_cdf):\n    return crps_torch(y_true, y_pred_cdf).mean()\n\n\ndef clean_StadiumType(txt):\n    if pd.isna(txt):\n        return np.nan\n    txt = txt.lower()\n    txt = ''.join([c for c in txt if c not in punctuation])\n    txt = re.sub(' +', ' ', txt)\n    txt = txt.strip()\n    txt = txt.replace('outside', 'outdoor')\n    txt = txt.replace('outdor', 'outdoor')\n    txt = txt.replace('outddors', 'outdoor')\n    txt = txt.replace('outdoors', 'outdoor')\n    txt = txt.replace('oudoor', 'outdoor')\n    txt = txt.replace('indoors', 'indoor')\n    txt = txt.replace('ourdoor', 'outdoor')\n    txt = txt.replace('retractable', 'rtr.')\n    return txt\n\n\ndef transform_StadiumType(txt):\n    if pd.isna(txt):\n        return np.nan\n    if 'outdoor' in txt or 'open' in txt:\n        return 1\n    if 'indoor' in txt or 'closed' in txt:\n        return 0\n    return np.nan\n\n\ndef str_to_seconds(txt):\n    txt = txt.split(':')\n    ans = int(txt[0]) * 60 + int(txt[1]) + int(txt[2]) / 60\n    return ans\n\n\ndef str_to_float(txt):\n    try:\n        return float(txt)\n    except Exception as e:\n        return np.NaN\n\n\ndef map_weather(txt):\n    ans = 1\n    if pd.isna(txt):\n        return 0\n    if 'partly' in txt:\n        ans *= 0.5\n    if 'climate controlled' in txt or 'indoor' in txt:\n        return ans * 3\n    if 'sunny' in txt or 'sun' in txt:\n        return ans * 2\n    if 'clear' in txt:\n        return ans\n    if 'cloudy' in txt:\n        return -ans\n    if 'rain' in txt or 'rainy' in txt:\n        return -2 * ans\n    if 'snow' in txt:\n        return -3 * ans\n    return 0\n\n\ndef final_drop_cols(df):\n    \"\"\"\n        Drop columns not used in X\n    \"\"\"\n    cols_to_drop = [\n        'Yards',\n        'PlayDirection',\n        'TeamOnOffense', 'NflId', 'NflIdRusher',\n        'TimeHandoff', 'TimeSnap', 'PlayerBirthDate',\n        'FieldPosition',\n        'DisplayName',\n        'PossessionTeam',\n        'PlayerCollegeName',\n        'Position',\n        'HomeTeamAbbr',\n        'VisitorTeamAbbr',\n        'Stadium',\n        'Location',\n        'GameId', 'PlayId', 'Team',\n        'IsRusher', 'IsQB']\n    cols_to_drop = list(set(cols_to_drop).intersection(set(list(df.columns))))\n    df = df.drop(cols_to_drop, axis=1)\n    return df\n\n\ndef standartize_orientations(df):\n    \"\"\"\n        https://www.kaggle.com/cpmpml/initial-wrangling-voronoi-areas-in-python\n\n        Make sure the offensive team is always moving left to right.\n    \"\"\"\n    df['ToLeft'] = df.PlayDirection == \"left\"\n    df['TeamOnOffense'] = \"home\"\n    df.loc[df.PossessionTeam != df.HomeTeamAbbr, 'TeamOnOffense'] = \"away\"\n    df['IsOnOffense'] = df.Team == df.TeamOnOffense  # Is player on offense?\n    df['HomeOnOffense'] = (df['TeamOnOffense'] == 'home').astype(int)\n\n    df['YardLine_std'] = 100 - df.YardLine\n    df.loc[df.FieldPosition.fillna('') == df.PossessionTeam,\n           'YardLine_std'\n    ] = df.loc[df.FieldPosition.fillna('') == df.PossessionTeam,\n               'YardLine']\n    df['X_std'] = df.X\n    df.loc[df.ToLeft, 'X_std'] = 120 - df.loc[df.ToLeft, 'X']\n    df['Y_std'] = df.Y\n    df.loc[df.ToLeft, 'Y_std'] = 160 / 3 - df.loc[df.ToLeft, 'Y']\n\n    df['Dir_rad'] = np.mod(90 - df.Dir, 360) * math.pi / 180.0\n    df['Dir_std'] = df.Dir_rad\n    df.loc[df.ToLeft, 'Dir_std'] = np.mod(np.pi + df.loc[df.ToLeft, 'Dir_rad'],\n                                          2 * np.pi)\n\n    df['Orientation_rad'] = np.mod(df.Orientation, 360) * math.pi / 180.0\n\n    df.loc[df.Season >= 2018, 'Orientation_rad'\n    ] = np.mod(df.loc[df.Season >= 2018, 'Orientation'] - 90,\n               360) * math.pi / 180.0\n\n    df['Orientation_rad'] = np.mod(df.Orientation, 360) * math.pi / 180.0\n    df.loc[df.Season >= 2018, 'Orientation_rad'\n    ] = np.mod(df.loc[df.Season >= 2018, 'Orientation'] - 90,\n               360) * math.pi / 180.0\n    df['Orientation_std'] = df.Orientation_rad\n    df.loc[df.ToLeft, 'Orientation_std'] = np.mod(\n        math.pi + df.loc[df.ToLeft, 'Orientation_rad'], 2 * math.pi)\n\n    replace_cols = ['YardLine', 'X', 'Y', 'Dir', 'Orientation']\n    for col in replace_cols:\n        df[col] = df[col + '_std']\n        df.drop([col + '_std'], axis=1, inplace=True)\n\n    drop_cols = ['Dir_rad', 'Orientation_rad']\n    for col in drop_cols:\n        df.drop([col], axis=1, inplace=True)\n\n    return df\n\n\ndef clean_turf(df):\n    Turf = {'Field Turf': 'Artificial', 'A-Turf Titan': 'Artificial',\n            'Grass': 'Natural', 'UBU Sports Speed S5-M': 'Artificial',\n            'Artificial': 'Artificial', 'DD GrassMaster': 'Artificial',\n            'Natural Grass': 'Natural',\n            'UBU Speed Series-S5-M': 'Artificial', 'FieldTurf': 'Artificial',\n            'FieldTurf 360': 'Artificial', 'Natural grass': 'Natural',\n            'grass': 'Natural',\n            'Natural': 'Natural', 'Artifical': 'Artificial',\n            'FieldTurf360': 'Artificial', 'Naturall Grass': 'Natural',\n            'Field turf': 'Artificial',\n            'SISGrass': 'Artificial', 'Twenty-Four/Seven Turf': 'Artificial',\n            'natural grass': 'Natural'}\n\n    turf_type = df['Turf'].map(Turf)\n    df['TurfIsNatural'] = (turf_type == 'Natural')\n    df = df.drop(['Turf'], axis=1)\n    return df\n\n\ndef clean_abbrs(df):\n    # CAREFUL. What if a new team appears?\n    map_abbr = {'ARI': 'ARZ', 'BAL': 'BLT', 'CLE': 'CLV', 'HOU': 'HST'}\n    for abb in df['PossessionTeam'].unique():\n        map_abbr[abb] = abb\n\n    def safe_map(val):\n        if map_abbr.get(val):\n            return map_abbr[val]\n        else:\n            return val\n\n    df['PossessionTeam'] = df['PossessionTeam'].apply(safe_map)\n    df['HomeTeamAbbr'] = df['HomeTeamAbbr'].apply(safe_map)\n    df['VisitorTeamAbbr'] = df['VisitorTeamAbbr'].apply(safe_map)\n    df['FieldPosition'] = df['FieldPosition'].apply(safe_map)\n    return df\n\n\ndef encode_formations(df):\n    # Formation columns\n    df = pd.concat([df.drop(['OffenseFormation'], axis=1),\n                    pd.get_dummies(df['OffenseFormation'],\n                                   prefix='OffenseFormation')], axis=1)\n    # Filling missing dummy columns at test stage\n    expected_columns = ['OffenseFormation_ACE',\n                        'OffenseFormation_EMPTY',\n                        'OffenseFormation_JUMBO',\n                        'OffenseFormation_PISTOL',\n                        'OffenseFormation_SHOTGUN',\n                        'OffenseFormation_SINGLEBACK',\n                        'OffenseFormation_WILDCAT',\n                        'OffenseFormation_I_FORM']\n    for col in expected_columns:\n        if not col in df.columns:\n            df[col] = 0\n    return df\n\n\ndef clean_weather(df):\n    df['WindSpeed'] = df['WindSpeed'].apply(\n        lambda x: x.lower().replace('mph', '').strip() if not pd.isna(x) else x)\n    df['WindSpeed'] = df['WindSpeed'].apply(lambda x: (int(x.split('-')[0]) + int(\n        x.split('-')[1])) / 2 if not pd.isna(x) and '-' in x else x)\n    df['WindSpeed'] = df['WindSpeed'].apply(\n        lambda x: (int(x.split()[0]) + int(x.split()[-1])) / 2 if not pd.isna(\n            x) and type(x) != float and 'gusts up to' in x else x)\n    df['WindSpeed'] = df['WindSpeed'].apply(str_to_float)\n\n    df = df.drop(['WindDirection'], axis=1)\n\n    df['GameWeather'] = df['GameWeather'].str.lower()\n    indoor = \"indoor\"\n    df['GameWeather'] = df['GameWeather'].apply(\n        lambda x: indoor if not pd.isna(x) and indoor in x else x)\n    df['GameWeather'] = df['GameWeather'].apply(\n        lambda x: x.replace('coudy', 'cloudy').replace('clouidy',\n                                                       'cloudy').replace('party',\n                                                                         'partly') if not pd.isna(\n            x) else x)\n    df['GameWeather'] = df['GameWeather'].apply(\n        lambda x: x.replace('clear and sunny', 'sunny and clear') if not pd.isna(\n            x) else x)\n    df['GameWeather'] = df['GameWeather'].apply(\n        lambda x: x.replace('skies', '').replace(\"mostly\",\n                                                 \"\").strip() if not pd.isna(\n            x) else x)\n    df['GameWeather'] = df['GameWeather'].apply(map_weather)\n    return df\n\n\ndef encode_personell(df):\n    # DefensePersonnel\n    counts = []\n    for i, val in df['DefensePersonnel'].str.split(',').iteritems():\n        row = {'OL': 0, 'RB': 0, 'TE': 0, 'WR': 0, 'DL': 0, 'DB': 0, 'LB': 0,\n               'QB': 0}\n        if val is np.NaN:\n            counts.append({})\n            continue\n        for item in val:\n            name, number = item.strip().split(' ')[::-1]\n            row[name] = int(number)\n        counts.append(row)\n    defense_presonell_df = pd.DataFrame(counts)\n    defense_presonell_df.columns = ['defense_' + x for x in\n                                    defense_presonell_df.columns]\n    defense_presonell_df = defense_presonell_df.fillna(0).astype(int)\n    defense_presonell_df.index = df.index\n    df = pd.concat([df.drop(['DefensePersonnel'], axis=1), defense_presonell_df],\n                   axis=1)\n\n    # OffensePersonnel\n    counts = []\n    for i, val in df['OffensePersonnel'].str.split(',').iteritems():\n        row = {'OL': 0, 'RB': 0, 'TE': 0, 'WR': 0, 'DL': 0, 'DB': 0, 'LB': 0,\n               'QB': 0}\n        if val is np.NaN:\n            counts.append({})\n            continue\n        for item in val:\n            name, number = item.strip().split(' ')[::-1]\n            row[name] = int(number)\n        counts.append(row)\n    offense_personnel_df = pd.DataFrame(counts)\n    offense_personnel_df.columns = ['offense_' + x for x in\n                                    offense_personnel_df.columns]\n    offense_personnel_df = offense_personnel_df.fillna(0).astype(int)\n    offense_personnel_df.index = df.index\n    df = pd.concat([df.drop(['OffensePersonnel'], axis=1), offense_personnel_df],\n                   axis=1)\n    return df\n\n\n# Feature engineering\n\ndef add_play_phys_features(play_id, df):\n    play_index = df['PlayId'] == play_id\n    play_df = df[play_index].copy()\n    offense_index = (play_index & df.IsOnOffense)\n    defense_index = (play_index & ~df.IsOnOffense)\n    offense_df = play_df[offense_index]\n    defense_df = play_df[defense_index]\n\n    offense_centroid_x = offense_df.X.mean()\n    offense_centroid_y = offense_df.Y.mean()\n    offense_x_std = offense_df.X.std()\n    offense_y_std = offense_df.Y.std()\n    offense_mean_force = offense_df.Force.mean()\n\n    defense_centroid_x = defense_df.X.mean()\n    defense_x_std = defense_df.X.std()\n    defense_centroid_y = defense_df.Y.mean()\n    defense_y_std = defense_df.Y.std()\n    defense_mean_force = defense_df.Force.mean()\n\n    rusher_row = offense_df[offense_df.IsRusher].iloc[0]\n\n    qb_row = offense_df[offense_df.Position == 'QB']\n    if not qb_row.empty:\n        qb_row = qb_row.iloc[0]\n\n    offense_centroid_pos = np.array([offense_centroid_x, offense_centroid_y])\n    defense_centroid_pos = np.array([defense_centroid_x, defense_centroid_y])\n    rusher_pos = rusher_row[['X', 'Y']].values[0]\n    qb_pos = qb_row[['X', 'Y']].values[0] if not qb_row.empty else None\n\n    rusher_dist_to_qb = np.linalg.norm(\n        rusher_pos - qb_pos) if not qb_row.empty else None\n    rusher_dist_to_offense_centroid = np.linalg.norm(\n        rusher_pos - offense_centroid_pos)\n    rusher_dist_to_defence_centroid = np.linalg.norm(\n        rusher_pos - defense_centroid_pos)\n\n    # Defender to rusher distances\n    defense_distances_to_runner = []\n    for row in defense_df.itertuples():\n        pos = np.array([row.X, row.Y])\n        defense_distances_to_runner.append(np.linalg.norm(rusher_pos - pos))\n    dist_to_rusher = defense_distances_to_runner = np.array(\n        defense_distances_to_runner)\n    \n    time_to_rusher = defense_df['S'] / dist_to_rusher\n    \n    defender_dist_to_runner_min = defense_distances_to_runner.min()\n    defender_dist_to_runner_mean = defense_distances_to_runner.mean()\n    defender_dist_to_runner_std = defense_distances_to_runner.std()\n    \n    defender_time_to_runner_min = time_to_rusher.min()\n    defender_time_to_runner_mean = time_to_rusher.mean()\n    \n    # closest defenders\n    closest_defender = \\\n    defense_df[dist_to_rusher == defender_dist_to_runner_min].iloc[0]\n    closest_bytime_defender = \\\n    defense_df[time_to_rusher == defender_time_to_runner_min].iloc[0]\n    \n    closest_defender_force_div_rusher_force = (\n            closest_defender.Force / rusher_row.Force)\n    closest_bytime_defender_force_div_rusher_force = (\n            closest_bytime_defender.Force / rusher_row.Force)\n    \n    closest_bytime_defender_speed_div_rusher_speed = closest_bytime_defender.S / rusher_row.S\n    closest_bytime_defender_acceleration_div_rusher_acceleration = closest_bytime_defender.A / rusher_row.A\n\n    # Add play features\n    new_play_features = pd.DataFrame({\n        # 'offense_spread_x': offense_spread_x,\n        # 'offense_spread_y': offense_spread_y,\n        # 'offense_centroid_x': offense_centroid_x,\n        # 'offense_centroid_y': offense_centroid_y,\n        'offense_x_std': offense_x_std,\n        'offense_y_std': offense_y_std,\n        'offense_mean_force': offense_mean_force,\n        # 'offense_mean_dir': offense_mean_dir,\n        # 'defense_spread_x': defense_spread_x,\n        # 'defense_centroid_x': defense_centroid_x,\n        'defense_x_std': defense_x_std,\n        # 'defense_spread_y': defense_spread_y,\n        # 'defense_centroid_y': defense_centroid_y,\n        'defense_y_std': defense_y_std,\n        'defense_mean_force': defense_mean_force,\n        # 'defense_mean_dir': defense_mean_dir,\n\n        'rusher_dist_to_qb': rusher_dist_to_qb,\n        'rusher_dist_to_offense_centroid': rusher_dist_to_offense_centroid,\n        'rusher_dist_to_defence_centroid': rusher_dist_to_defence_centroid,\n\n        'defender_dist_to_runner_min': defender_dist_to_runner_min,\n        'defender_dist_to_runner_mean': defender_dist_to_runner_mean,\n        'defender_dist_to_runner_std': defender_dist_to_runner_std,\n        \n        'defender_time_to_runner_min': defender_time_to_runner_min,\n        'defender_time_to_runner_mean': defender_time_to_runner_mean,\n        \n#         'closest_defender_force_div_rusher_force': closest_defender_force_div_rusher_force,\n#         'closest_bytime_defender_force_div_rusher_force': closest_bytime_defender_force_div_rusher_force,\n        \n#         'closest_bytime_defender_speed_div_rusher_speed': closest_bytime_defender_speed_div_rusher_speed,\n#         'closest_bytime_defender_acceleration_div_rusher_acceleration': closest_bytime_defender_acceleration_div_rusher_acceleration,\n\n    }, index=play_df.index)\n\n    return new_play_features\n\n\ndef add_phys_features(df):\n    df['Force'] = df['A'] * df['PlayerWeight']\n\n    feature_dfs = []\n    for play_id in tqdm.tqdm(df.PlayId.unique()):\n        feature_dfs.append(add_play_phys_features(play_id, df))\n\n    feature_df = pd.concat(feature_dfs)\n\n    df_new = pd.concat([df, feature_df], join='inner', axis=1)\n    assert df_new.shape[0] == df.shape[0]\n    return df_new\n\n\ndef engineer_features(df):\n    df['DefendersInTheBox_vs_Distance'] = (\n            df['DefendersInTheBox'] / df['Distance'])\n\n    df = add_phys_features(df)\n    return df\n\n\ndef preprocess_features(df):\n    \"\"\"Accepts df like train data, returns cleaned, standartized and enriched df\"\"\"\n\n    df = clean_abbrs(df)\n\n    df = standartize_orientations(df)\n\n    df = clean_turf(df)\n\n    df['StadiumType'] = df['StadiumType'].apply(clean_StadiumType)\n    df['StadiumTypeShort'] = df['StadiumType'].apply(transform_StadiumType)\n    df = df.drop(['StadiumType'], axis=1)\n\n    df['HomeField'] = df['FieldPosition'] == df['HomeTeamAbbr']\n    df['Field_eq_Possession'] = df['FieldPosition'] == df['PossessionTeam']\n    df['HomePossesion'] = (df['PossessionTeam'] == df['HomeTeamAbbr'])\n\n    df = encode_formations(df)\n\n    df['GameClock'] = df['GameClock'].apply(str_to_seconds)\n\n    df['PlayerHeight'] = df['PlayerHeight'].apply(\n        lambda x: 12 * int(x.split('-')[0]) + int(x.split('-')[1]))\n    df['PlayerBMI'] = 703 * (df['PlayerWeight'] / (df['PlayerHeight']) ** 2)\n\n    df['TimeHandoff'] = pd.to_datetime(df['TimeHandoff'], utc=True)\n    df['TimeSnap'] = pd.to_datetime(df['TimeSnap'], utc=True)\n    df['TimeDelta'] = (df['TimeHandoff'] - df['TimeSnap']).apply(\n        lambda x: x.total_seconds())\n    df['PlayerBirthDate'] = df['PlayerBirthDate'].apply(\n        lambda x: datetime.datetime.strptime(x, \"%m/%d/%Y\"))\n    df['PlayerBirthDate'] = pd.to_datetime(df['PlayerBirthDate'], utc=True)\n\n    seconds_in_year = 60 * 60 * 24 * 365.25\n    df['PlayerAge'] = (df['TimeHandoff'] - df['PlayerBirthDate']).apply(\n        lambda x: x.total_seconds()) / seconds_in_year\n    # df = df.drop([], axis=1)\n\n    df = clean_weather(df)\n\n    df['IsRusher'] = df['NflId'] == df['NflIdRusher']\n    df['IsQB'] = df['Position'] == 'QB'\n\n    df = encode_personell(df)\n\n    # Feature engineering\n    df = engineer_features(df)\n\n    df = sort_df(df)\n\n    return df\n\n\ndef sort_df(df):\n    df.sort_values(by=['PlayId', 'IsOnOffense', 'IsRusher', 'IsQB'], inplace=True)\n    return df\n\n\ndef make_x(df, fillna=True):\n    source_play_id = df['PlayId']\n\n    df = final_drop_cols(df)\n\n    if fillna:\n        df.fillna(-999, inplace=True)\n\n    # Player features\n    cols_player = ['X',\n                   'Y',\n                   'S',\n                   'A',\n                   'Dis',\n                   'Orientation',\n                   'Dir',\n                   'JerseyNumber',\n                   'PlayerHeight',\n                   'PlayerWeight',\n                   'PlayerBMI',\n                   'PlayerAge',\n                   'Force']\n\n    all_cols_player = np.array([[f'pl{num}_' + x for x in cols_player] for num in\n                                range(1, 23)]).flatten()\n    # print('cols player', all_cols_player)\n\n    X = np.array(df[cols_player]).reshape(-1, len(cols_player) * 22)\n\n    play_id_index = source_play_id[::22]\n    X_df = pd.DataFrame(X, columns=all_cols_player, index=play_id_index)\n\n    assert df[cols_player].shape[0] == X_df.shape[0] * 22\n    assert df[cols_player].shape[1] == X_df.shape[1] / 22\n\n    # Play features\n    # print()\n    cols_play = list(df.drop(cols_player, axis=1).columns)\n\n    X_play_col = np.zeros(shape=(X.shape[0], len(cols_play)))\n    for i, col in enumerate(cols_play):\n        X_play_col[:, i] = df[col][::22]\n\n    X_play_col_df = pd.DataFrame(X_play_col, columns=cols_play,\n                                 index=play_id_index)\n    assert X_df.shape[0] == X_play_col_df.shape[0]\n    X_df = pd.concat([X_df, X_play_col_df], axis=1)\n\n    assert X_df.shape[0] == source_play_id.drop_duplicates().count()\n    return X_df\n\n\ndef make_y(X, df):\n    y = np.zeros(shape=(X.shape[0], 199))\n    for i, yard in enumerate(df['Yards'][::22]):\n        y[i, yard + 99:] = np.ones(shape=(1, 100 - yard))\n    return y\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2020/train.csv', dtype={'WindSpeed': 'object'})\n# df_train = pd.read_csv('data/train.csv', dtype={'WindSpeed': 'object'})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_preprocessed = preprocess_features(df_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = make_x(df_train_preprocessed, fillna=True)\ny_train = make_y(X_train, df_train_preprocessed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.shape, y_train.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# NN"},{"metadata":{"trusted":true},"cell_type":"code","source":"N_FEATURES = X_train.shape[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def iterate_minibatches(X, y, batchsize):\n    X = torch.as_tensor(X)\n    y = torch.as_tensor(y)\n    indices = np.random.permutation(np.arange(len(X)))\n    for start in range(0, len(indices), batchsize):\n        ix = indices[start: start + batchsize]\n        yield X[ix], y[ix]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE = 'cpu'\nBATCH_SIZE = 100\nLEARNING_RATE = 0.0001\nNUM_EPOCHS = 100\nEARLY_STOP_AFTER = 20\nWEIGHT_DECAY = 0.0002\nSCHEDULER_PATIENCE = 5\nL1_LOSS_WEIGHT = 0.001","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class NeuralNet(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.seq = nn.Sequential(\n            nn.Linear(N_FEATURES, N_FEATURES),\n            nn.ReLU(),\n            nn.Dropout(0.1),\n            nn.BatchNorm1d(N_FEATURES),\n            nn.Linear(N_FEATURES, N_FEATURES),\n            nn.ReLU(),\n            nn.BatchNorm1d(N_FEATURES),\n            nn.Linear(N_FEATURES, N_FEATURES),\n            nn.ReLU(),\n            nn.Dropout(0.1),\n            nn.BatchNorm1d(N_FEATURES),\n        )\n        self.head_classifier = nn.Sequential(\n            nn.Linear(N_FEATURES, 199),\n            nn.Softmax()\n        )\n        self.head_regressor = nn.Sequential(\n            nn.Linear(N_FEATURES, 1),\n        )\n\n    def forward(self, x_batch):\n        latent_out = self.seq.forward(x_batch)\n        return self.head_classifier(latent_out), self.head_regressor(latent_out)\n\n    def loss(self, x_batch, y_batch):\n        y_batch = torch.as_tensor(y_batch).double().to(device=DEVICE)\n        y_pdf, y_yard = self(x_batch)\n        loss_crps = crps_loss(y_batch, y_pdf)\n        \n        y_yard = y_yard.flatten().double()\n        y_batch_yard = cdf_to_yard_torch(y_batch).double()\n        loss_mae = torch.mean(torch.abs(y_batch_yard-y_yard))\n        return loss_crps + loss_mae*L1_LOSS_WEIGHT\n\n    def predict_pdf(self, x_batch):\n        y_pdf, y_yard = self(x_batch)\n        return y_pdf\n    \n    def predict_cdf(self, x_batch):\n        y_pred_pdf = self.predict_pdf(x_batch)\n        y_pred_cdf = torch.cumsum(y_pred_pdf, dim=1)\n        y_pred_cdf = torch.clamp(y_pred_cdf, 0, 1)\n        return y_pred_cdf\n\nmodel = NeuralNet()\nmodel = model.double().to(device=DEVICE)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(model, opt, scheduler, X_train, y_train, X_val, y_val, early_stop_patience, batch_size, n_epochs):\n    train_loss = []\n    train_loss_stds = []\n    val_accuracy = []\n    val_accuracy_stds = []\n\n    best_val_acc = None\n    best_model = None\n    patience = 0\n\n    for epoch in range(n_epochs):\n        try:\n            opt.zero_grad()\n\n            start_time = time.time()\n            model.train(True)\n            epoch_loss = []\n\n            for X_batch, y_batch in iter(iterate_minibatches(X_train, y_train, batch_size)):\n                loss = model.loss(X_batch, y_batch)\n                epoch_loss.append(float(loss.item()))\n                loss.backward()\n                opt.step()\n                opt.zero_grad()\n\n            train_loss.append(np.mean(epoch_loss))\n            train_loss_stds.append(np.std(epoch_loss))\n            model.train(False)\n\n            epoch_val_acc = []\n            for X_batch, y_batch in iter(iterate_minibatches(X_val, y_val, batch_size)): \n                y_val_pred_cdf = model.predict_cdf(X_batch).to(device=DEVICE)\n                val_loss = crps_loss_cdf(y_batch, y_val_pred_cdf)\n                epoch_val_acc.append(val_loss.item())\n            val_acc = np.mean(epoch_val_acc)\n            val_accuracy.append(val_acc)\n            val_accuracy_stds.append(np.std(epoch_val_acc))\n\n\n            scheduler.step(val_acc)\n\n            # Then we print the results for this epoch:\n            print(\"Epoch {} of {} took {:.3f}s\".format(\n                epoch + 1, n_epochs, time.time() - start_time))\n            print(\"  training loss: \\t{:.6f}\".format(train_loss[-1]))\n            print(\"  validation score: \\t\\t\\t{:.6f}\".format(val_accuracy[-1]))\n\n            if best_val_acc is None or val_acc < best_val_acc:\n                best_val_acc = val_acc\n                patience = 0\n                torch.save(model.state_dict(), 'best_model')\n            else:\n                patience += 1\n                print(f'Validation score has not improved for {patience} epochs.')\n            if patience >= early_stop_patience:\n                print('Early stopping.')\n                break\n        except KeyboardInterrupt:\n            break\n    model.load_state_dict(torch.load('best_model'))\n    model.train(False)\n\n    history = {\n        'train_loss': train_loss,\n        'train_loss_stds': train_loss_stds,\n        'val_accuracy': val_accuracy,\n        'val_accuracy_stds': val_accuracy_stds,\n    }\n    \n    return model, history\n\ndef train_nn_pipeline(model, X, y, early_stop_after=EARLY_STOP_AFTER, batch_size=BATCH_SIZE, n_epochs=NUM_EPOCHS):\n    scaler = MinMaxScaler()\n    X = scaler.fit_transform(X)\n    X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.1)\n    \n    opt = torch.optim.Adamax(model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(opt, mode='min', patience=SCHEDULER_PATIENCE, verbose=True, threshold=1e-6, eps=1e-9)\n\n    model, train_history = train(model, opt, scheduler, X_train, y_train, X_val, y_val, early_stop_after, batch_size, n_epochs)\n    \n    return model, scaler, train_history\n\ndef nn_pipeline_predict(model, scaler, X):\n    X = torch.as_tensor(scaler.transform(X)).to(device=DEVICE)\n    y_pred = model.predict_cdf(X)\n    return y_pred","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CV"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import RepeatedKFold\n\ncv = RepeatedKFold(n_splits=5, n_repeats=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_cv(X, y, cv, model_class, train_function):\n    models, scalers, losses = [], [], []\n    i = 0\n    X = X.values\n    for train_index, test_index in cv.split(X):\n        print(f'Training model {i}')\n        try:\n            X_train, X_test = X[train_index], X[test_index]\n            y_train, y_test = y[train_index], y[test_index]\n            model = model_class().double().to(device=DEVICE)\n            model, scaler, history = train_nn_pipeline(model, X_train, y_train)\n            y_pred_cdf_test = nn_pipeline_predict(model, scaler, X_test)\n            y_test = torch.as_tensor(y_test).to(device=DEVICE)\n            test_loss = crps_torch(y_test, y_pred_cdf_test).mean()\n            losses.append(test_loss.cpu().data.item())\n            models.append(model)\n            scalers.append(scaler)\n            i+=1\n        except KeyboardInterrupt:\n            break\n    return models, scalers, losses\n\ndef cv_predict(models, scalers, X):\n    X = X.values\n    predicted_cdfs = []\n    for model, scaler in zip(models, scalers):\n        X_prep = torch.as_tensor(scaler.transform(X)).to(device=DEVICE)\n        predicted_cdfs.append(model.predict_cdf(X_prep).cpu().data.numpy())\n    return np.mean(predicted_cdfs, axis=0) \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models, scalers, losses = run_cv(X_train, y_train, cv, NeuralNet, train_nn_pipeline)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# SUBMIT"},{"metadata":{"trusted":true},"cell_type":"code","source":"env = nflrush.make_env()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"def get_predictions(df_test, y_test, models, scalers):\n    X = make_x(preprocess_features(df_test), fillna=True)\n    y_pred = cv_predict(models, scalers, X)\n    return y_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"iter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for df_test, y_cdf_test in tqdm.tqdm(iter_test):\n    y_pred = get_predictions(df_test, y_cdf_test, models, scalers)\n    env.predict(pd.DataFrame(data=y_pred,columns=y_cdf_test.columns))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"env.write_submission_file()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.7"}},"nbformat":4,"nbformat_minor":1}