Coronavirus_Music_Functions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functions used in the Descriptive Statistics, LGBM, Regression, Factor Analysis, and Individual Difference Scripts.\n",
    "Lindsay Warrenburg  \n",
    "lindsay.a.warrenburg@gmail.com  \n",
    "Final version: December 20, 2020"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### **Cleaning / Pre-Processing**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_columns(DATA, missing_threshold):\n",
    "    missing = pd.DataFrame(DATA.isnull().sum()/DATA.shape[0])*100\n",
    "    missing = missing[missing[0] > 0]\n",
    "    missing = missing.sort_values(by=[0], ascending=False)\n",
    "\n",
    "    to_delete = missing[missing[0] > missing_threshold].index\n",
    "\n",
    "    if missing.shape[0] == 0:\n",
    "        print(\"No missing values!\")\n",
    "\n",
    "    else:\n",
    "        percent_missing = missing.loc[to_delete,:].shape[0]/missing.shape[0]*100\n",
    "        DATA.drop(to_delete, axis=1, inplace = True)\n",
    "        DATA.reset_index(inplace = True, drop = True)\n",
    "        print(round(percent_missing, 1), \"percent of cols with >\", missing_threshold, \"% missing data\")\n",
    "    return DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "def drop_text_and_unused_cols(data):\n",
    "    drop_cols = ['Unnamed: 0', \n",
    "             'C1_School closing', 'C2_Workplace closing', 'C3_Cancel public events',\n",
    "             'C4_Restrictions on gatherings', 'C5_Close public transport', 'C6_Stay at home requirements',\n",
    "             'C7_Restrictions on internal movement', 'C8_International travel controls',\n",
    "             'ConfirmedCases', 'ConfirmedDeaths', 'ContainmentHealthIndex', 'ContainmentHealthIndexForDisplay',\n",
    "             'CountryName', 'Date',\n",
    "             'Demographics_COVID_Current State', # only NYC\n",
    "             'Demographics_COVID_Profession', # free text\n",
    "             'Demographics_General_Years of Post-School', 'Demographics_General_Years of School', # combined in the 'Demographics_General_Education column'\n",
    "             'E1_Income support', 'E2_Debt/contract relief', 'E3_Fiscal measures', 'E4_International support',\n",
    "             'ESM_ID', 'ESM_Self-Generated Code', \n",
    "             'EconomicSupportIndex', 'EconomicSupportIndexForDisplay', \n",
    "             'GovernmentResponseIndex', 'GovernmentResponseIndexForDisplay',\n",
    "             'H1_Public information campaigns', 'H2_Testing policy', \n",
    "             'H3_Contact tracing', 'H4_Emergency investment in healthcare', 'H5_Investment in vaccines',\n",
    "             'ResponseId', 'StartDate', \n",
    "             'StringencyIndex', 'StringencyIndexForDisplay', 'StringencyLegacyIndexForDisplay',\n",
    "             'Survey Info_Duration', 'Survey Info_EndDate', 'Survey Info_Group', \n",
    "             'Survey Info_Progress', 'Survey Info_RecordedDate', 'Survey Info_Source', \n",
    "             'date', 'to_cut_actvities'\n",
    "            ]\n",
    "    data = data.drop(columns = drop_cols, axis = 1)\n",
    "    data.reset_index(inplace = True, drop = True) \n",
    "    return data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Descriptive Statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def temp_df(group_col, col_list, df):\n",
    "    df = df.loc[:,[group_col]+col_list].reset_index(drop=True)\n",
    "    row_dict = dict(zip(df.index, df[group_col]))\n",
    "    \n",
    "    # find instances of where \"1\" is in a cell in the dataframe\n",
    "    row, col = np.where(df.values == 1)\n",
    "    \n",
    "    # replace the column index with the column name\n",
    "    colname = df.columns[col]\n",
    "    \n",
    "    # create dataframe of row number (corresponding to person) and which column they responded '1' to\n",
    "    vals = pd.DataFrame()\n",
    "    vals['row'] = row\n",
    "    vals['cols'] = colname\n",
    "    vals['cols'] = vals['cols'].replace(r'^.*-', '', regex=True)\n",
    "    \n",
    "    vals = vals.set_index('row') # row is the row # (index #) for the \"1\" location in the original df\n",
    "\n",
    "    vals['group'] = vals.index.map(row_dict)\n",
    "    \n",
    "    return vals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_grouped_df(group_col, df, tmp):\n",
    "    # create dictionary of the number of participants from each group in the\n",
    "    # group_col (e.g., if group_col = country, how many participants are in each country?)\n",
    "    val_counts = pd.DataFrame(df[group_col].value_counts(dropna=False))\n",
    "    keys = val_counts.index.tolist()\n",
    "    values = val_counts[group_col].tolist()\n",
    "    group_dict = dict(zip(keys, values))\n",
    "\n",
    "    # create dataframe of proportion of people who responded to each question based on the group_dict values\n",
    "    grouped_n = tmp.groupby(['group', 'cols']).size().unstack(fill_value=0)\n",
    "    grouped_n['n'] = grouped_n.index.map(group_dict)\n",
    "    grouped_n.loc['TOTAL']= grouped_n.sum(numeric_only=True, axis=0)\n",
    "    grouped_perc = grouped_n.iloc[:,:-1].div(grouped_n.iloc[:,-1], axis=0).round(4)*100\n",
    "    \n",
    "    return grouped_n, grouped_perc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def group_numeric(data, col_list, group_col, scale_flag):\n",
    "    means = pd.DataFrame(data.groupby([group_col])[col_list].mean())\n",
    "    means.loc['TOTAL'] = data[col_list].mean()\n",
    "\n",
    "    sds = pd.DataFrame(data.groupby([group_col])[col_list].std())\n",
    "    sds.loc['TOTAL'] = data[col_list].std()\n",
    "    \n",
    "    # only scale if the column is a 7-pt Likert scale\n",
    "    if scale_flag:\n",
    "        # center the means (0 = no change now).\n",
    "        perc_change = (means - 4)\n",
    "\n",
    "        # the minimum value is -3 and the maximum value is +3 \n",
    "        # we want to scale that between -1 and + 1\n",
    "        perc_change = perc_change/3\n",
    "\n",
    "        # change to percent -- this is now the \"percent change from 0\"\n",
    "        # i.e., if there's a score of \"-10\" it means the score is 10% LESS than before coronavirus\n",
    "        perc_change = perc_change * 100\n",
    "        perc_change.columns = perc_change.columns + ['_change']\n",
    "\n",
    "        new = means.merge(sds, left_index=True, right_index=True, suffixes=('_mean', '_sd')).merge(perc_change, left_index=True, right_index=True)\n",
    "        new = new.reindex(sorted(new.columns), axis=1)\n",
    "    \n",
    "    else: \n",
    "        new = means.merge(sds, left_index=True, right_index=True, suffixes=('_mean', '_sd'))\n",
    "        \n",
    "    return new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mean_differences(data, index_list, col_list):\n",
    "    tmp = pd.DataFrame(columns=['index']+col_list)\n",
    "    tmp['index']= index_list\n",
    "    tmp = tmp.set_index('index')\n",
    "    \n",
    "    yes_minus_no = tmp.copy()\n",
    "    perc_diff = tmp.copy()\n",
    "    \n",
    "    # note -- this will throw an error if there are not positive AND negative classes\n",
    "    for i,c in list(itertools.product(index_list, col_list)):\n",
    "        pos_class = data.groupby([i])[c].mean()[1]\n",
    "        neg_class = data.groupby([i])[c].mean()[0]\n",
    "                \n",
    "        yes_minus_no.loc[i,c] = pos_class - neg_class\n",
    "        perc_diff.loc[i,c] = (pos_class - neg_class)/7*100\n",
    "        \n",
    "    return yes_minus_no, perc_diff"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### **Describing  High and Low Scorers on the 5 Factors**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "def high_low(DATA, colList):\n",
    "    means_df = pd.DataFrame({\"Score\":['High','Low']}).set_index('Score')\n",
    "    for k in colList:\n",
    "        means_df.loc['High',k] = DATA.loc[DATA.type=='high',k].mean()\n",
    "        means_df.loc['Low',k] = DATA.loc[DATA.type=='low',k].mean()\n",
    "    means_df = means_df.T\n",
    "    means_df['Difference'] = means_df['High'] - means_df['Low'] \n",
    "    means_df.columns=[s+'_Mean' for s in means_df.columns]\n",
    "\n",
    "    sds_df = pd.DataFrame({\"Score\":['High','Low']}).set_index('Score')\n",
    "    for k in colList:\n",
    "        sds_df.loc['High',k] = DATA.loc[DATA.type=='high',k].std()\n",
    "        sds_df.loc['Low',k] = DATA.loc[DATA.type=='low',k].std()\n",
    "    sds_df = sds_df.T\n",
    "    sds_df['Difference'] = sds_df['High'] - sds_df['Low'] \n",
    "    sds_df.columns=[s+'_SD' for s in sds_df.columns]\n",
    "\n",
    "    summary_df = means_df.merge(sds_df, left_index=True, right_index=True)\n",
    "    summary_df = summary_df.reindex(sorted(summary_df.columns), axis=1)\n",
    "    \n",
    "    return summary_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_into_high_low(df, col, low_cutoff, high_cutoff):\n",
    "    high = df[df[col] > np.percentile(df[col], high_cutoff)]\n",
    "    high['type'] = 'high'\n",
    "\n",
    "    low = df[df[col] < np.percentile(df[col], low_cutoff)]\n",
    "    low['type'] = 'low'\n",
    "\n",
    "    new_df = high.append(low, ignore_index=True)\n",
    "    \n",
    "    return new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def factor_summary(DATA):\n",
    "    act_list = [col for col in df.columns if col.startswith('Activities_')]\n",
    "    sit_list = [col for col in df.columns if col.startswith('Situations_')]\n",
    "    fun_list = [col for col in df.columns if col.startswith('Functions_')]\n",
    "    hyp_list = [col for col in df.columns if col.startswith('Hypotheses_')]\n",
    "    eng_list = [col for col in df.columns if col.startswith('Music Engagement_')]\n",
    "    \n",
    "    act_df = high_low(DATA, act_list)\n",
    "    sit_df = high_low(DATA, sit_list)\n",
    "    fun_df = high_low(DATA, fun_list)\n",
    "    hyp_df = high_low(DATA, hyp_list)\n",
    "    eng_df = high_low(DATA, eng_list)\n",
    "    \n",
    "    summary_df = act_df.append(sit_df, ignore_index=False).append(fun_df, ignore_index=False).append(hyp_df, ignore_index=False).append(eng_df, ignore_index=False)\n",
    "    \n",
    "    return summary_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### **LGBM**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lgbmr_model(dv, data): \n",
    "    y = dv\n",
    "    X = data\n",
    "    \n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=123)\n",
    "    \n",
    "    cor = X_train.corr().abs()\n",
    "    cor = cor.unstack()\n",
    "    cor = cor.sort_values(kind=\"quicksort\", ascending = False)\n",
    "    cor = pd.DataFrame(cor)\n",
    "    cor.columns = ['cor']\n",
    "\n",
    "    highcor = cor.query('0.90 < cor < 1')\n",
    "    print((len(highcor)/len(cor))*100, \"% of correlations > 0.90\\n\")\n",
    "    \n",
    "    # grid search with 3-fold CV\n",
    "    estimator = lgb.LGBMRegressor()\n",
    "    parameter_grid = {\n",
    "        'learning_rate': [0.05, 0.1,0.16],\n",
    "        'n_estimators': [50,100,200,300,500],\n",
    "        'num_leaves':[40,50],\n",
    "        'max_depth':[10,30,50], \n",
    "        'subsample_for_bin':[500,1000], \n",
    "        'min_child_samples':[10],\n",
    "        'random_state':[123]\n",
    "    }\n",
    "    model = GridSearchCV(estimator, parameter_grid, cv = 3)\n",
    "    model.fit(X_train, y_train)\n",
    "    \n",
    "    lgbmr = model.best_estimator_\n",
    "    print('Best parameters found by grid search are:', lgbmr)\n",
    "    print(\"\\n=================================\")\n",
    "    \n",
    "    # model\n",
    "    lgbmr.fit(X_train, y_train)\n",
    "    y_pred = lgbmr.predict(X_test)\n",
    "    \n",
    "    print(f'LightGBM Regression_r2: {r2_score(y_test, y_pred).round(4)}')\n",
    "    print(f'LightGBM Regression_Mean_squared_error: {mean_squared_error(y_test, y_pred).round(4)}')\n",
    "    print(f'LightGBM Regression_Mean_absolute_error: {mean_absolute_error(y_test, y_pred).round(4)}')\n",
    "    \n",
    "    return lgbmr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}