LLM_results.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre>✔️ 197 ms (2024-12-16T10:10:44/2024-12-16T10:10:44)</pre>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%reload_ext autotime\n",
    "import pandas as pd\n",
    "import plotly.io as pio\n",
    "import plotly.express as px # Plotting\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas()\n",
    "import statsmodels.formula.api as smf\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "pio.renderers.default = 'notebook'\n",
    "pd.options.plotting.backend = \"plotly\"\n",
    "pd.set_option(\"display.max_columns\", None)\n",
    "pd.set_option(\"display.max_colwidth\", 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre>✔️ 5.83 s (2024-12-16T10:02:36/2024-12-16T10:02:42)</pre>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Index</th>\n",
       "      <th>title</th>\n",
       "      <th>pitch</th>\n",
       "      <th>hero</th>\n",
       "      <th>uri</th>\n",
       "      <th>amountRaised</th>\n",
       "      <th>goal</th>\n",
       "      <th>timeLeft</th>\n",
       "      <th>action</th>\n",
       "      <th>actionUri</th>\n",
       "      <th>start</th>\n",
       "      <th>end</th>\n",
       "      <th>n_questions</th>\n",
       "      <th>location</th>\n",
       "      <th>n_donors</th>\n",
       "      <th>description</th>\n",
       "      <th>use_of_funds</th>\n",
       "      <th>whos_involved</th>\n",
       "      <th>moderated</th>\n",
       "      <th>n_updates</th>\n",
       "      <th>updates</th>\n",
       "      <th>_21</th>\n",
       "      <th>text</th>\n",
       "      <th>condition</th>\n",
       "      <th>ICD10</th>\n",
       "      <th>name</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>ethnicity</th>\n",
       "      <th>urgency</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>truth</th>\n",
       "      <th>notes</th>\n",
       "      <th>smiling</th>\n",
       "      <th>guess_ethnicity</th>\n",
       "      <th>guess-ethnicity</th>\n",
       "      <th>guess</th>\n",
       "      <th>Sentiment</th>\n",
       "      <th>guess ethnicity</th>\n",
       "      <th>Target_completion</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>488</th>\n",
       "      <td>495</td>\n",
       "      <td>Help me support my sick father</td>\n",
       "      <td>Raising money to see my sick father</td>\n",
       "      <td>https://static.givealittle.co.nz/assets/hero/cfa715d1-e709-4367-a625-b20f01636ccb-320:False</td>\n",
       "      <td>https://givealittle.co.nz/cause/help-me-support-my-sick-father</td>\n",
       "      <td>1555.0</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>closed</td>\n",
       "      <td>Read more</td>\n",
       "      <td>https://givealittle.co.nz/cause/help-me-support-my-sick-father</td>\n",
       "      <td>20 Oct 2024</td>\n",
       "      <td>27 Oct 2024</td>\n",
       "      <td>0</td>\n",
       "      <td>Greenhithe, Auckland</td>\n",
       "      <td>39.0</td>\n",
       "      <td>Hey guys. I'm raising money in order to travel down to Nelson to see my father who has been diag...</td>\n",
       "      <td>Travel funds to get from Auckland to Nelson and back.</td>\n",
       "      <td>Created by, and paying to a verified bank account of,_x000D_\\nJoshua Westley</td>\n",
       "      <td>Page Moderated</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Help me support my sick father Raising money to see my sick father Hey guys. I'm raising money i...</td>\n",
       "      <td>Alzheimer's</td>\n",
       "      <td>F03</td>\n",
       "      <td>Dad</td>\n",
       "      <td>Male</td>\n",
       "      <td>Not specified</td>\n",
       "      <td>European</td>\n",
       "      <td>90.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>The text does not mention the person's age or ethnicity. Based on the image, the person appears ...</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>77.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>489</th>\n",
       "      <td>496</td>\n",
       "      <td>My Cambodian Sister Fighting the Big C</td>\n",
       "      <td>Please help my Cambodian sister fight the big C.</td>\n",
       "      <td>https://static.givealittle.co.nz/assets/hero/e17703d2-5687-45ab-9276-b20b0085e2e1-320:False</td>\n",
       "      <td>https://givealittle.co.nz/cause/my-cambodian-sister-fighting-the-big-c</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>closed</td>\n",
       "      <td>Read more</td>\n",
       "      <td>https://givealittle.co.nz/cause/my-cambodian-sister-fighting-the-big-c</td>\n",
       "      <td>16 Oct 2024</td>\n",
       "      <td>29 Oct 2024</td>\n",
       "      <td>0</td>\n",
       "      <td>Nationwide</td>\n",
       "      <td>26.0</td>\n",
       "      <td>Last year, Channa was my tour guide in Cambodia. Having grown up in similar political systems an...</td>\n",
       "      <td>rent, groceries, transport, and other daily necessities</td>\n",
       "      <td>Created by, and paying to a verified bank account of,_x000D_\\n      _x000D_\\n        Antje Decke...</td>\n",
       "      <td>Page Moderated</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Thank you!!!\\n1 November 2024\\nThank you for opening your heart and wallet for Channa. I spoke t...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>My Cambodian Sister Fighting the Big C Please help my Cambodian sister fight the big C. Last yea...</td>\n",
       "      <td>Breast Cancer</td>\n",
       "      <td>C50</td>\n",
       "      <td>Channa</td>\n",
       "      <td>Female</td>\n",
       "      <td>Adult</td>\n",
       "      <td>Asian</td>\n",
       "      <td>100.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>The text is a personal and emotional appeal from the campaign creator to help a friend in need. ...</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>66.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>491</th>\n",
       "      <td>498</td>\n",
       "      <td>Flights needed NZ-London to help 83 year old mum through a mastectomy</td>\n",
       "      <td>Raising funds to send a caring daughter to London to look after her 83 year old mum following a ...</td>\n",
       "      <td>https://static.givealittle.co.nz/assets/hero/03a34b2c-1401-4460-8ca3-b20800b6f41e-320:False</td>\n",
       "      <td>https://givealittle.co.nz/cause/flights-needed-nz-london-to-help-83-year-old-mum</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3340.0</td>\n",
       "      <td>closed</td>\n",
       "      <td>Read more</td>\n",
       "      <td>https://givealittle.co.nz/cause/flights-needed-nz-london-to-help-83-year-old-mum</td>\n",
       "      <td>13 Oct 2024</td>\n",
       "      <td>16 Oct 2024</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>My mother has breast cancer and at 83 is having a mastectomy on 24th October. She lives alone an...</td>\n",
       "      <td>Just the flights</td>\n",
       "      <td>Created by, and paying to a verified bank account of,_x000D_\\n      _x000D_\\n        Sian Jones</td>\n",
       "      <td>Page Moderated</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Flights needed NZ-London to help 83 year old mum through a mastectomy Raising funds to send a ca...</td>\n",
       "      <td>breast cancer</td>\n",
       "      <td>C50</td>\n",
       "      <td>unknown</td>\n",
       "      <td>unknown</td>\n",
       "      <td>83</td>\n",
       "      <td>white</td>\n",
       "      <td>90.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>The ethnicity is not mentioned in the text, but based on the image, I guess the person is white....</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>493</th>\n",
       "      <td>500</td>\n",
       "      <td>Tom’s recovery from traumatic brain injury.</td>\n",
       "      <td>Raising funds for Tom’s arduous recovery journey</td>\n",
       "      <td>https://static.givealittle.co.nz/assets/hero/b96d62a6-aaf0-47f1-9db5-b20700b7b7ac-320:False</td>\n",
       "      <td>https://givealittle.co.nz/cause/toms-recovery-from-traumatic-brain-injury</td>\n",
       "      <td>2680.0</td>\n",
       "      <td>5000.0</td>\n",
       "      <td>closed</td>\n",
       "      <td>Read more</td>\n",
       "      <td>https://givealittle.co.nz/cause/toms-recovery-from-traumatic-brain-injury</td>\n",
       "      <td>12 Oct 2024</td>\n",
       "      <td>24 Oct 2024</td>\n",
       "      <td>0</td>\n",
       "      <td>Nelson / Tasman</td>\n",
       "      <td>22.0</td>\n",
       "      <td>On Monday 7th October 2024 Tom was on his way to work on his quad bike when he came across a her...</td>\n",
       "      <td>We have started this page to help our parents with accomodation, travel and food costs while the...</td>\n",
       "      <td>Created by_x000D_\\nToni Redward  \\n\\n\\n\\n_x000D_\\n      Paying to a verified bank account of_x00...</td>\n",
       "      <td>Page Moderated</td>\n",
       "      <td>2.0</td>\n",
       "      <td>Update\\n25 October 2024\\nWe would like to thank each and every person for their donations throug...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Tom’s recovery from traumatic brain injury. Raising funds for Tom’s arduous recovery journey On ...</td>\n",
       "      <td>traumatic brain injury</td>\n",
       "      <td>S06.0X0A</td>\n",
       "      <td>Tom</td>\n",
       "      <td>male</td>\n",
       "      <td>adult</td>\n",
       "      <td>likely Caucasian</td>\n",
       "      <td>80.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>ICD10 code is an approximation</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>53.600000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>494</th>\n",
       "      <td>501</td>\n",
       "      <td>Give the gift of change. We would appreciate any help we can get!</td>\n",
       "      <td>We are seeking your help to help us pay hospital and medical costs for Pilamita. Your donation w...</td>\n",
       "      <td>https://static.givealittle.co.nz/assets/hero/c30e1e40-0bfc-4975-9fcc-b205008d84a5-320:False</td>\n",
       "      <td>https://givealittle.co.nz/cause/give-the-gift-of-change-your-help-is-appreciated</td>\n",
       "      <td>165.0</td>\n",
       "      <td>50000.0</td>\n",
       "      <td>closed</td>\n",
       "      <td>Read more</td>\n",
       "      <td>https://givealittle.co.nz/cause/give-the-gift-of-change-your-help-is-appreciated</td>\n",
       "      <td>10 Oct 2024</td>\n",
       "      <td>31 Oct 2024</td>\n",
       "      <td>0</td>\n",
       "      <td>Manukau, Auckland</td>\n",
       "      <td>6.0</td>\n",
       "      <td>My brother in-law - Pilamita Faingaa suffered from a huge stroke that has left his left side par...</td>\n",
       "      <td>Any money raised from here will go directly to pay for hospital and medical costs for Pilamita. ...</td>\n",
       "      <td>Created by, and paying to a verified bank account of,_x000D_\\nTaiana Faingaa      on behalf of P...</td>\n",
       "      <td>Page Moderated</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Lunch with Pilamita\\n10 October 2024\\nToday we helped the care staff by offering to feed Pila hi...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Give the gift of change. We would appreciate any help we can get! We are seeking your help to he...</td>\n",
       "      <td>Stroke</td>\n",
       "      <td>I61</td>\n",
       "      <td>Pilamita Faingaa</td>\n",
       "      <td>Male</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Pacific Islander</td>\n",
       "      <td>80.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>The ethnicity of the person is not explicitly mentioned in the text, but based on the image, it ...</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.330000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10244</th>\n",
       "      <td>10397</td>\n",
       "      <td>New Zealand MindBody Trust web-based 'whole person' resources for healthworkers and their patien...</td>\n",
       "      <td>The NZ MindBody Trust web-based project to support and educate clinicians and patients in a 'who...</td>\n",
       "      <td>https://static.givealittle.co.nz/assets/hero/158d0e80-2c47-449a-a05e-a603014e786e-320:False</td>\n",
       "      <td>https://givealittle.co.nz/cause/mindbodyresourcesforpatientsandhealthworkers</td>\n",
       "      <td>6700.0</td>\n",
       "      <td>9000.0</td>\n",
       "      <td>closed</td>\n",
       "      <td>Read more</td>\n",
       "      <td>https://givealittle.co.nz/cause/mindbodyresourcesforpatientsandhealthworkers</td>\n",
       "      <td>9 May 2016</td>\n",
       "      <td>15 Jul 2016</td>\n",
       "      <td>0</td>\n",
       "      <td>Auckland</td>\n",
       "      <td>36.0</td>\n",
       "      <td>The NZ MindBody Trust (2004) is concerned with 'whole person' and person-centred approaches to h...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Created by, and paying to a verified bank account of,_x000D_\\n      _x000D_\\n        New Zealand...</td>\n",
       "      <td>Page Moderated</td>\n",
       "      <td>2.0</td>\n",
       "      <td>The Give-A-Little Campaign Result\\n12 July 2016\\nThe MindBody Trustees want to thank you all for...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>New Zealand MindBody Trust web-based 'whole person' resources for healthworkers and their patien...</td>\n",
       "      <td>chronic illness</td>\n",
       "      <td>not provided</td>\n",
       "      <td>Brian Broom</td>\n",
       "      <td>male</td>\n",
       "      <td>not provided</td>\n",
       "      <td>not provided</td>\n",
       "      <td>0.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>90.0</td>\n",
       "      <td>The text does not provide any specific information about the condition, age, or ethnicity of the...</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>74.444444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10246</th>\n",
       "      <td>10399</td>\n",
       "      <td>Diane's cyst removal to continue my dream job</td>\n",
       "      <td>Large  Radial Ganglion Cyst Removal</td>\n",
       "      <td>https://static.givealittle.co.nz/assets/hero/06af4a19-e239-44bb-a5a8-a60400ab4982-320:False</td>\n",
       "      <td>https://givealittle.co.nz/cause/wristoperation</td>\n",
       "      <td>730.0</td>\n",
       "      <td>5000.0</td>\n",
       "      <td>closed</td>\n",
       "      <td>Read more</td>\n",
       "      <td>https://givealittle.co.nz/cause/wristoperation</td>\n",
       "      <td>9 May 2016</td>\n",
       "      <td>3 Jun 2016</td>\n",
       "      <td>0</td>\n",
       "      <td>Northland</td>\n",
       "      <td>11.0</td>\n",
       "      <td>I am in need of a large ganglion cyst removed from my left wrist. It has entwined itself with 2 ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Created by, and paying to a verified bank account of,_x000D_\\n      _x000D_\\n        Diane Crocker</td>\n",
       "      <td>Page Moderated</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Diane's cyst removal to continue my dream job Large  Radial Ganglion Cyst Removal I am in need o...</td>\n",
       "      <td>Ganglion cyst</td>\n",
       "      <td>M79.61</td>\n",
       "      <td>Diane</td>\n",
       "      <td>Female</td>\n",
       "      <td>Not specified</td>\n",
       "      <td>Caucasian</td>\n",
       "      <td>80.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>The ethnicity is guessed based on the image, as the ethnicity is not mentioned in the text. The ...</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>14.600000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10254</th>\n",
       "      <td>10407</td>\n",
       "      <td>A thank you to Waikato NICU for helping so many little babies and their families</td>\n",
       "      <td>Raising funds for Waikato NICU, and a portable cardiac monitor to support the miracle babies.</td>\n",
       "      <td>https://static.givealittle.co.nz/assets/hero/75b48fa1-0752-44b4-8484-a5fb015ecab2-320:False</td>\n",
       "      <td>https://givealittle.co.nz/cause/charlieandoliver</td>\n",
       "      <td>8912.7</td>\n",
       "      <td>35000.0</td>\n",
       "      <td>closed</td>\n",
       "      <td>Read more</td>\n",
       "      <td>https://givealittle.co.nz/cause/charlieandoliver</td>\n",
       "      <td>3 May 2016</td>\n",
       "      <td>30 Jun 2017</td>\n",
       "      <td>0</td>\n",
       "      <td>Waikato</td>\n",
       "      <td>100.0</td>\n",
       "      <td>This give-a-little page aims to raise funds to allow the NICU at Waikato Hospital to purchase a ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Created by_x000D_\\n      _x000D_\\n        Rebecca Clews_x000D_\\n      \\n\\n\\n\\n\\n\\n\\n_x000D_\\n   ...</td>\n",
       "      <td>Page Moderated</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Quiz Night!\\n7 November 2016\\nGot no plans on the 17th of November? Come on down to the Speights...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>A thank you to Waikato NICU for helping so many little babies and their families Raising funds f...</td>\n",
       "      <td>premature birth</td>\n",
       "      <td>P01.0</td>\n",
       "      <td>Charlie</td>\n",
       "      <td>male</td>\n",
       "      <td>24 weeks gestation</td>\n",
       "      <td>Caucasian</td>\n",
       "      <td>80.0</td>\n",
       "      <td>90.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>The ethnicity was guessed based on the image, which shows a Caucasian woman. There is no informa...</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>25.464857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10255</th>\n",
       "      <td>10408</td>\n",
       "      <td>Help restore David’s smile after being assaulted</td>\n",
       "      <td>Talented professional musician and entertainer David Shanhun needs your help to restore his teet...</td>\n",
       "      <td>https://static.givealittle.co.nz/assets/hero/ee0d17f1-b6a5-4d41-85e9-a5fa0166b741-320:False</td>\n",
       "      <td>https://givealittle.co.nz/cause/help-restore-davids-teeth-and-smile</td>\n",
       "      <td>8048.0</td>\n",
       "      <td>54500.0</td>\n",
       "      <td>closed</td>\n",
       "      <td>Read more</td>\n",
       "      <td>https://givealittle.co.nz/cause/help-restore-davids-teeth-and-smile</td>\n",
       "      <td>3 May 2016</td>\n",
       "      <td>31 Jul 2016</td>\n",
       "      <td>0</td>\n",
       "      <td>Auckland</td>\n",
       "      <td>115.0</td>\n",
       "      <td>David was assaulted on the 16th of April 2016. He had 2 teeth knocked out completely, and 10 tee...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Created by_x000D_\\n      _x000D_\\n        Fiona Shanhun_x000D_\\n      \\n\\n\\n\\n\\n\\n\\n_x000D_\\n   ...</td>\n",
       "      <td>Page Moderated</td>\n",
       "      <td>2.0</td>\n",
       "      <td>Health/Teeth/Face Update\\n24 May 2016\\n***Message from David:***\\nThank you all so much for your...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Help restore David’s smile after being assaulted Talented professional musician and entertainer ...</td>\n",
       "      <td>Assault</td>\n",
       "      <td>S02</td>\n",
       "      <td>David Shanhun</td>\n",
       "      <td>Male</td>\n",
       "      <td>Not mentioned</td>\n",
       "      <td>Caucasian</td>\n",
       "      <td>75.0</td>\n",
       "      <td>90.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>No warnings or errors</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>14.766972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10263</th>\n",
       "      <td>10417</td>\n",
       "      <td>TRUS Biopsy Machine for Taranaki</td>\n",
       "      <td>The men of Taranaki need this TRUS Biopsy Machine to diagnose Prostate Cancer.  Reduce waiting t...</td>\n",
       "      <td>https://static.givealittle.co.nz/assets/hero/c9169845-63b5-4c1e-bb50-a5fc00e5abd0-320:False</td>\n",
       "      <td>https://givealittle.co.nz/cause/prostatebiopsymachinenewplymouth</td>\n",
       "      <td>1910.0</td>\n",
       "      <td>20000.0</td>\n",
       "      <td>closed</td>\n",
       "      <td>Read more</td>\n",
       "      <td>https://givealittle.co.nz/cause/prostatebiopsymachinenewplymouth</td>\n",
       "      <td>29 Apr 2016</td>\n",
       "      <td>31 Aug 2016</td>\n",
       "      <td>0</td>\n",
       "      <td>Taranaki</td>\n",
       "      <td>15.0</td>\n",
       "      <td>Currently the TRUS (Trans Rectal Ultrasound) machine for diagnosing prostate cancer is only avai...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Created by, and paying to a verified bank account of,_x000D_\\n      _x000D_\\n        Rotary Club...</td>\n",
       "      <td>Page Moderated</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TRUS Biopsy Machine for Taranaki The men of Taranaki need this TRUS Biopsy Machine to diagnose P...</td>\n",
       "      <td>Prostate Cancer</td>\n",
       "      <td>C61</td>\n",
       "      <td>Not specified</td>\n",
       "      <td>Male</td>\n",
       "      <td>Not specified</td>\n",
       "      <td>Not specified</td>\n",
       "      <td>80.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>The ethnicity is not specified in the text, but based on the image, it is likely that the person...</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.550000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4067 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Index  \\\n",
       "488      495   \n",
       "489      496   \n",
       "491      498   \n",
       "493      500   \n",
       "494      501   \n",
       "...      ...   \n",
       "10244  10397   \n",
       "10246  10399   \n",
       "10254  10407   \n",
       "10255  10408   \n",
       "10263  10417   \n",
       "\n",
       "                                                                                                     title  \\\n",
       "488                                                                         Help me support my sick father   \n",
       "489                                                                 My Cambodian Sister Fighting the Big C   \n",
       "491                                  Flights needed NZ-London to help 83 year old mum through a mastectomy   \n",
       "493                                                            Tom’s recovery from traumatic brain injury.   \n",
       "494                                      Give the gift of change. We would appreciate any help we can get!   \n",
       "...                                                                                                    ...   \n",
       "10244  New Zealand MindBody Trust web-based 'whole person' resources for healthworkers and their patien...   \n",
       "10246                                                        Diane's cyst removal to continue my dream job   \n",
       "10254                     A thank you to Waikato NICU for helping so many little babies and their families   \n",
       "10255                                                     Help restore David’s smile after being assaulted   \n",
       "10263                                                                     TRUS Biopsy Machine for Taranaki   \n",
       "\n",
       "                                                                                                     pitch  \\\n",
       "488                                                                    Raising money to see my sick father   \n",
       "489                                                       Please help my Cambodian sister fight the big C.   \n",
       "491    Raising funds to send a caring daughter to London to look after her 83 year old mum following a ...   \n",
       "493                                                       Raising funds for Tom’s arduous recovery journey   \n",
       "494    We are seeking your help to help us pay hospital and medical costs for Pilamita. Your donation w...   \n",
       "...                                                                                                    ...   \n",
       "10244  The NZ MindBody Trust web-based project to support and educate clinicians and patients in a 'who...   \n",
       "10246                                                                  Large  Radial Ganglion Cyst Removal   \n",
       "10254        Raising funds for Waikato NICU, and a portable cardiac monitor to support the miracle babies.   \n",
       "10255  Talented professional musician and entertainer David Shanhun needs your help to restore his teet...   \n",
       "10263  The men of Taranaki need this TRUS Biopsy Machine to diagnose Prostate Cancer.  Reduce waiting t...   \n",
       "\n",
       "                                                                                              hero  \\\n",
       "488    https://static.givealittle.co.nz/assets/hero/cfa715d1-e709-4367-a625-b20f01636ccb-320:False   \n",
       "489    https://static.givealittle.co.nz/assets/hero/e17703d2-5687-45ab-9276-b20b0085e2e1-320:False   \n",
       "491    https://static.givealittle.co.nz/assets/hero/03a34b2c-1401-4460-8ca3-b20800b6f41e-320:False   \n",
       "493    https://static.givealittle.co.nz/assets/hero/b96d62a6-aaf0-47f1-9db5-b20700b7b7ac-320:False   \n",
       "494    https://static.givealittle.co.nz/assets/hero/c30e1e40-0bfc-4975-9fcc-b205008d84a5-320:False   \n",
       "...                                                                                            ...   \n",
       "10244  https://static.givealittle.co.nz/assets/hero/158d0e80-2c47-449a-a05e-a603014e786e-320:False   \n",
       "10246  https://static.givealittle.co.nz/assets/hero/06af4a19-e239-44bb-a5a8-a60400ab4982-320:False   \n",
       "10254  https://static.givealittle.co.nz/assets/hero/75b48fa1-0752-44b4-8484-a5fb015ecab2-320:False   \n",
       "10255  https://static.givealittle.co.nz/assets/hero/ee0d17f1-b6a5-4d41-85e9-a5fa0166b741-320:False   \n",
       "10263  https://static.givealittle.co.nz/assets/hero/c9169845-63b5-4c1e-bb50-a5fc00e5abd0-320:False   \n",
       "\n",
       "                                                                                    uri  \\\n",
       "488                      https://givealittle.co.nz/cause/help-me-support-my-sick-father   \n",
       "489              https://givealittle.co.nz/cause/my-cambodian-sister-fighting-the-big-c   \n",
       "491    https://givealittle.co.nz/cause/flights-needed-nz-london-to-help-83-year-old-mum   \n",
       "493           https://givealittle.co.nz/cause/toms-recovery-from-traumatic-brain-injury   \n",
       "494    https://givealittle.co.nz/cause/give-the-gift-of-change-your-help-is-appreciated   \n",
       "...                                                                                 ...   \n",
       "10244      https://givealittle.co.nz/cause/mindbodyresourcesforpatientsandhealthworkers   \n",
       "10246                                    https://givealittle.co.nz/cause/wristoperation   \n",
       "10254                                  https://givealittle.co.nz/cause/charlieandoliver   \n",
       "10255               https://givealittle.co.nz/cause/help-restore-davids-teeth-and-smile   \n",
       "10263                  https://givealittle.co.nz/cause/prostatebiopsymachinenewplymouth   \n",
       "\n",
       "       amountRaised     goal timeLeft     action  \\\n",
       "488          1555.0   2000.0   closed  Read more   \n",
       "489          2000.0   3000.0   closed  Read more   \n",
       "491             0.0   3340.0   closed  Read more   \n",
       "493          2680.0   5000.0   closed  Read more   \n",
       "494           165.0  50000.0   closed  Read more   \n",
       "...             ...      ...      ...        ...   \n",
       "10244        6700.0   9000.0   closed  Read more   \n",
       "10246         730.0   5000.0   closed  Read more   \n",
       "10254        8912.7  35000.0   closed  Read more   \n",
       "10255        8048.0  54500.0   closed  Read more   \n",
       "10263        1910.0  20000.0   closed  Read more   \n",
       "\n",
       "                                                                              actionUri  \\\n",
       "488                      https://givealittle.co.nz/cause/help-me-support-my-sick-father   \n",
       "489              https://givealittle.co.nz/cause/my-cambodian-sister-fighting-the-big-c   \n",
       "491    https://givealittle.co.nz/cause/flights-needed-nz-london-to-help-83-year-old-mum   \n",
       "493           https://givealittle.co.nz/cause/toms-recovery-from-traumatic-brain-injury   \n",
       "494    https://givealittle.co.nz/cause/give-the-gift-of-change-your-help-is-appreciated   \n",
       "...                                                                                 ...   \n",
       "10244      https://givealittle.co.nz/cause/mindbodyresourcesforpatientsandhealthworkers   \n",
       "10246                                    https://givealittle.co.nz/cause/wristoperation   \n",
       "10254                                  https://givealittle.co.nz/cause/charlieandoliver   \n",
       "10255               https://givealittle.co.nz/cause/help-restore-davids-teeth-and-smile   \n",
       "10263                  https://givealittle.co.nz/cause/prostatebiopsymachinenewplymouth   \n",
       "\n",
       "             start          end  n_questions              location  n_donors  \\\n",
       "488    20 Oct 2024  27 Oct 2024            0  Greenhithe, Auckland      39.0   \n",
       "489    16 Oct 2024  29 Oct 2024            0            Nationwide      26.0   \n",
       "491    13 Oct 2024  16 Oct 2024            0                   NaN       NaN   \n",
       "493    12 Oct 2024  24 Oct 2024            0       Nelson / Tasman      22.0   \n",
       "494    10 Oct 2024  31 Oct 2024            0     Manukau, Auckland       6.0   \n",
       "...            ...          ...          ...                   ...       ...   \n",
       "10244   9 May 2016  15 Jul 2016            0              Auckland      36.0   \n",
       "10246   9 May 2016   3 Jun 2016            0             Northland      11.0   \n",
       "10254   3 May 2016  30 Jun 2017            0               Waikato     100.0   \n",
       "10255   3 May 2016  31 Jul 2016            0              Auckland     115.0   \n",
       "10263  29 Apr 2016  31 Aug 2016            0              Taranaki      15.0   \n",
       "\n",
       "                                                                                               description  \\\n",
       "488    Hey guys. I'm raising money in order to travel down to Nelson to see my father who has been diag...   \n",
       "489    Last year, Channa was my tour guide in Cambodia. Having grown up in similar political systems an...   \n",
       "491    My mother has breast cancer and at 83 is having a mastectomy on 24th October. She lives alone an...   \n",
       "493    On Monday 7th October 2024 Tom was on his way to work on his quad bike when he came across a her...   \n",
       "494    My brother in-law - Pilamita Faingaa suffered from a huge stroke that has left his left side par...   \n",
       "...                                                                                                    ...   \n",
       "10244  The NZ MindBody Trust (2004) is concerned with 'whole person' and person-centred approaches to h...   \n",
       "10246  I am in need of a large ganglion cyst removed from my left wrist. It has entwined itself with 2 ...   \n",
       "10254  This give-a-little page aims to raise funds to allow the NICU at Waikato Hospital to purchase a ...   \n",
       "10255  David was assaulted on the 16th of April 2016. He had 2 teeth knocked out completely, and 10 tee...   \n",
       "10263  Currently the TRUS (Trans Rectal Ultrasound) machine for diagnosing prostate cancer is only avai...   \n",
       "\n",
       "                                                                                              use_of_funds  \\\n",
       "488                                                  Travel funds to get from Auckland to Nelson and back.   \n",
       "489                                                rent, groceries, transport, and other daily necessities   \n",
       "491                                                                                       Just the flights   \n",
       "493    We have started this page to help our parents with accomodation, travel and food costs while the...   \n",
       "494    Any money raised from here will go directly to pay for hospital and medical costs for Pilamita. ...   \n",
       "...                                                                                                    ...   \n",
       "10244                                                                                                  NaN   \n",
       "10246                                                                                                  NaN   \n",
       "10254                                                                                                  NaN   \n",
       "10255                                                                                                  NaN   \n",
       "10263                                                                                                  NaN   \n",
       "\n",
       "                                                                                             whos_involved  \\\n",
       "488                           Created by, and paying to a verified bank account of,_x000D_\\nJoshua Westley   \n",
       "489    Created by, and paying to a verified bank account of,_x000D_\\n      _x000D_\\n        Antje Decke...   \n",
       "491        Created by, and paying to a verified bank account of,_x000D_\\n      _x000D_\\n        Sian Jones   \n",
       "493    Created by_x000D_\\nToni Redward  \\n\\n\\n\\n_x000D_\\n      Paying to a verified bank account of_x00...   \n",
       "494    Created by, and paying to a verified bank account of,_x000D_\\nTaiana Faingaa      on behalf of P...   \n",
       "...                                                                                                    ...   \n",
       "10244  Created by, and paying to a verified bank account of,_x000D_\\n      _x000D_\\n        New Zealand...   \n",
       "10246   Created by, and paying to a verified bank account of,_x000D_\\n      _x000D_\\n        Diane Crocker   \n",
       "10254  Created by_x000D_\\n      _x000D_\\n        Rebecca Clews_x000D_\\n      \\n\\n\\n\\n\\n\\n\\n_x000D_\\n   ...   \n",
       "10255  Created by_x000D_\\n      _x000D_\\n        Fiona Shanhun_x000D_\\n      \\n\\n\\n\\n\\n\\n\\n_x000D_\\n   ...   \n",
       "10263  Created by, and paying to a verified bank account of,_x000D_\\n      _x000D_\\n        Rotary Club...   \n",
       "\n",
       "            moderated  n_updates  \\\n",
       "488    Page Moderated        NaN   \n",
       "489    Page Moderated        1.0   \n",
       "491    Page Moderated        NaN   \n",
       "493    Page Moderated        2.0   \n",
       "494    Page Moderated        1.0   \n",
       "...               ...        ...   \n",
       "10244  Page Moderated        2.0   \n",
       "10246  Page Moderated        NaN   \n",
       "10254  Page Moderated        4.0   \n",
       "10255  Page Moderated        2.0   \n",
       "10263  Page Moderated        NaN   \n",
       "\n",
       "                                                                                                   updates  \\\n",
       "488                                                                                                    NaN   \n",
       "489    Thank you!!!\\n1 November 2024\\nThank you for opening your heart and wallet for Channa. I spoke t...   \n",
       "491                                                                                                    NaN   \n",
       "493    Update\\n25 October 2024\\nWe would like to thank each and every person for their donations throug...   \n",
       "494    Lunch with Pilamita\\n10 October 2024\\nToday we helped the care staff by offering to feed Pila hi...   \n",
       "...                                                                                                    ...   \n",
       "10244  The Give-A-Little Campaign Result\\n12 July 2016\\nThe MindBody Trustees want to thank you all for...   \n",
       "10246                                                                                                  NaN   \n",
       "10254  Quiz Night!\\n7 November 2016\\nGot no plans on the 17th of November? Come on down to the Speights...   \n",
       "10255  Health/Teeth/Face Update\\n24 May 2016\\n***Message from David:***\\nThank you all so much for your...   \n",
       "10263                                                                                                  NaN   \n",
       "\n",
       "       _21  \\\n",
       "488    NaN   \n",
       "489    NaN   \n",
       "491    NaN   \n",
       "493    NaN   \n",
       "494    NaN   \n",
       "...    ...   \n",
       "10244  NaN   \n",
       "10246  NaN   \n",
       "10254  NaN   \n",
       "10255  NaN   \n",
       "10263  NaN   \n",
       "\n",
       "                                                                                                      text  \\\n",
       "488    Help me support my sick father Raising money to see my sick father Hey guys. I'm raising money i...   \n",
       "489    My Cambodian Sister Fighting the Big C Please help my Cambodian sister fight the big C. Last yea...   \n",
       "491    Flights needed NZ-London to help 83 year old mum through a mastectomy Raising funds to send a ca...   \n",
       "493    Tom’s recovery from traumatic brain injury. Raising funds for Tom’s arduous recovery journey On ...   \n",
       "494    Give the gift of change. We would appreciate any help we can get! We are seeking your help to he...   \n",
       "...                                                                                                    ...   \n",
       "10244  New Zealand MindBody Trust web-based 'whole person' resources for healthworkers and their patien...   \n",
       "10246  Diane's cyst removal to continue my dream job Large  Radial Ganglion Cyst Removal I am in need o...   \n",
       "10254  A thank you to Waikato NICU for helping so many little babies and their families Raising funds f...   \n",
       "10255  Help restore David’s smile after being assaulted Talented professional musician and entertainer ...   \n",
       "10263  TRUS Biopsy Machine for Taranaki The men of Taranaki need this TRUS Biopsy Machine to diagnose P...   \n",
       "\n",
       "                    condition         ICD10              name   gender  \\\n",
       "488               Alzheimer's           F03               Dad     Male   \n",
       "489             Breast Cancer           C50            Channa   Female   \n",
       "491             breast cancer           C50           unknown  unknown   \n",
       "493    traumatic brain injury      S06.0X0A               Tom     male   \n",
       "494                    Stroke           I61  Pilamita Faingaa     Male   \n",
       "...                       ...           ...               ...      ...   \n",
       "10244         chronic illness  not provided       Brian Broom     male   \n",
       "10246           Ganglion cyst        M79.61             Diane   Female   \n",
       "10254         premature birth         P01.0           Charlie     male   \n",
       "10255                 Assault           S02     David Shanhun     Male   \n",
       "10263         Prostate Cancer           C61     Not specified     Male   \n",
       "\n",
       "                      age         ethnicity  urgency  sentiment  truth  \\\n",
       "488         Not specified          European     90.0      100.0   80.0   \n",
       "489                 Adult             Asian    100.0      100.0   80.0   \n",
       "491                    83             white     90.0       80.0   80.0   \n",
       "493                 adult  likely Caucasian     80.0       50.0   80.0   \n",
       "494               Unknown  Pacific Islander     80.0       80.0   60.0   \n",
       "...                   ...               ...      ...        ...    ...   \n",
       "10244        not provided      not provided      0.0       80.0   90.0   \n",
       "10246       Not specified         Caucasian     80.0       20.0   60.0   \n",
       "10254  24 weeks gestation         Caucasian     80.0       90.0   80.0   \n",
       "10255       Not mentioned         Caucasian     75.0       90.0   80.0   \n",
       "10263       Not specified     Not specified     80.0       80.0   80.0   \n",
       "\n",
       "                                                                                                     notes  \\\n",
       "488    The text does not mention the person's age or ethnicity. Based on the image, the person appears ...   \n",
       "489    The text is a personal and emotional appeal from the campaign creator to help a friend in need. ...   \n",
       "491    The ethnicity is not mentioned in the text, but based on the image, I guess the person is white....   \n",
       "493                                                                         ICD10 code is an approximation   \n",
       "494    The ethnicity of the person is not explicitly mentioned in the text, but based on the image, it ...   \n",
       "...                                                                                                    ...   \n",
       "10244  The text does not provide any specific information about the condition, age, or ethnicity of the...   \n",
       "10246  The ethnicity is guessed based on the image, as the ethnicity is not mentioned in the text. The ...   \n",
       "10254  The ethnicity was guessed based on the image, which shows a Caucasian woman. There is no informa...   \n",
       "10255                                                                                No warnings or errors   \n",
       "10263  The ethnicity is not specified in the text, but based on the image, it is likely that the person...   \n",
       "\n",
       "      smiling guess_ethnicity guess-ethnicity guess  Sentiment  \\\n",
       "488      True             NaN             NaN   NaN        NaN   \n",
       "489      True             NaN             NaN   NaN        NaN   \n",
       "491      True             NaN             NaN   NaN        NaN   \n",
       "493     False             NaN             NaN   NaN        NaN   \n",
       "494     False             NaN             NaN   NaN        NaN   \n",
       "...       ...             ...             ...   ...        ...   \n",
       "10244   False             NaN             NaN   NaN        NaN   \n",
       "10246   False             NaN             NaN   NaN        NaN   \n",
       "10254    True             NaN             NaN   NaN        NaN   \n",
       "10255    True             NaN             NaN   NaN        NaN   \n",
       "10263   False             NaN             NaN   NaN        NaN   \n",
       "\n",
       "      guess ethnicity  Target_completion  \n",
       "488               NaN          77.750000  \n",
       "489               NaN          66.666667  \n",
       "491               NaN           0.000000  \n",
       "493               NaN          53.600000  \n",
       "494               NaN           0.330000  \n",
       "...               ...                ...  \n",
       "10244             NaN          74.444444  \n",
       "10246             NaN          14.600000  \n",
       "10254             NaN          25.464857  \n",
       "10255             NaN          14.766972  \n",
       "10263             NaN           9.550000  \n",
       "\n",
       "[4067 rows x 40 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_excel(\"LLM_results.xlsx\")\n",
    "df = df[(df.timeLeft == \"closed\") & (df.goal > 0)]\n",
    "df.sentiment = df.sentiment.astype(float)\n",
    "df.truth = df.truth.astype(float)\n",
    "df.urgency = df.urgency.astype(float)\n",
    "df[\"Target_completion\"] = df.amountRaised / df.goal * 100\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre>✔️ 10.5 ms (2024-12-16T10:02:42/2024-12-16T10:02:42)</pre>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df.smiling = df.smiling.str.lower().replace({\n",
    "    \"true\": True,\n",
    "    \"false\": False,\n",
    "    \"yes\": True,\n",
    "    \"no\": False,\n",
    "    \"not applicable\": None,\n",
    "    \"unknown\": None,\n",
    "    \"undefined\": None\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre>✔️ 9.15 ms (2024-12-16T10:02:42/2024-12-16T10:02:42)</pre>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "gender\n",
       "female    2074\n",
       "male      1546\n",
       "other      447\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.gender = df.gender.str.lower().where(df.gender.str.lower().isin(['male', 'female']), 'other')\n",
    "df.gender.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre>✔️ 11.6 ms (2024-12-16T10:02:42/2024-12-16T10:02:42)</pre>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df.ethnicity = df.ethnicity.str.lower().replace({\n",
    "    \"caucasian\": \"white\",\n",
    "    \"likely caucasian\": \"white\",\n",
    "    \"likely white\": \"white\",\n",
    "    \"likely of european descent\": \"white\",\n",
    "    \"likely caucasian based on the image\": \"white\",\n",
    "    \"likely european\": \"white\",\n",
    "    \"european\": \"white\",\n",
    "    \"maori\": \"māori\",\n",
    "    \"not mentioned\": None,\n",
    "    \"not specified\": None,\n",
    "    \"unspecified\": None,\n",
    "    \"not found\": None,\n",
    "    \"not provided\": None,\n",
    "    \"unknown\": None,\n",
    "})\n",
    "top_10 = df.ethnicity.value_counts().head(10)\n",
    "df.ethnicity = df.ethnicity.where(df.ethnicity.isin(top_10.index), \"other\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre>✔️ 11.2 ms (2024-12-16T10:04:57/2024-12-16T10:04:57)</pre>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "condition\n",
       "cancer                225\n",
       "breast cancer         173\n",
       "gender dysphoria      109\n",
       "stroke                 65\n",
       "bowel cancer           62\n",
       "multiple sclerosis     59\n",
       "lung cancer            55\n",
       "melanoma               40\n",
       "mental health          37\n",
       "obesity                37\n",
       "endometriosis          34\n",
       "cerebral palsy         34\n",
       "kidney failure         31\n",
       "type 1 diabetes        30\n",
       "heart attack           28\n",
       "infertility            28\n",
       "ovarian cancer         25\n",
       "unknown                25\n",
       "terminal cancer        23\n",
       "cardiac arrest         22\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.condition.str.lower().value_counts().head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre>✔️ 49.8 ms (2024-12-16T10:16:21/2024-12-16T10:16:21)</pre>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>OLS Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>    <td>Target_completion</td> <th>  R-squared:         </th> <td>   0.028</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>                   <td>OLS</td>        <th>  Adj. R-squared:    </th> <td>   0.017</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>             <td>Least Squares</td>   <th>  F-statistic:       </th> <td>   2.431</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>             <td>Mon, 16 Dec 2024</td>  <th>  Prob (F-statistic):</th>  <td>0.0344</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>                 <td>10:16:21</td>      <th>  Log-Likelihood:    </th> <td> -2277.8</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>No. Observations:</th>      <td>   421</td>       <th>  AIC:               </th> <td>   4568.</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Residuals:</th>          <td>   415</td>       <th>  BIC:               </th> <td>   4592.</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Model:</th>              <td>     5</td>       <th>                     </th>     <td> </td>   \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Covariance Type:</th>      <td>nonrobust</td>     <th>                     </th>     <td> </td>   \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "         <td></td>            <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th>       <td>   25.2501</td> <td>   17.544</td> <td>    1.439</td> <td> 0.151</td> <td>   -9.236</td> <td>   59.737</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>smiling[T.True]</th> <td>   11.7268</td> <td>    5.606</td> <td>    2.092</td> <td> 0.037</td> <td>    0.707</td> <td>   22.746</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>gender[T.male]</th>  <td>    9.4733</td> <td>    5.590</td> <td>    1.695</td> <td> 0.091</td> <td>   -1.514</td> <td>   20.461</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>gender[T.other]</th> <td>    2.2159</td> <td>   11.663</td> <td>    0.190</td> <td> 0.849</td> <td>  -20.711</td> <td>   25.142</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>sentiment</th>       <td>   -0.3565</td> <td>    0.139</td> <td>   -2.565</td> <td> 0.011</td> <td>   -0.630</td> <td>   -0.083</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>truth</th>           <td>    0.3923</td> <td>    0.225</td> <td>    1.743</td> <td> 0.082</td> <td>   -0.050</td> <td>    0.835</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "  <th>Omnibus:</th>       <td>447.003</td> <th>  Durbin-Watson:     </th> <td>   2.107</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Prob(Omnibus):</th> <td> 0.000</td>  <th>  Jarque-Bera (JB):  </th> <td>28466.226</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Skew:</th>          <td> 4.583</td>  <th>  Prob(JB):          </th> <td>    0.00</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Kurtosis:</th>      <td>42.227</td>  <th>  Cond. No.          </th> <td>    756.</td> \n",
       "</tr>\n",
       "</table><br/><br/>Notes:<br/>[1] Standard Errors assume that the covariance matrix of the errors is correctly specified."
      ],
      "text/latex": [
       "\\begin{center}\n",
       "\\begin{tabular}{lclc}\n",
       "\\toprule\n",
       "\\textbf{Dep. Variable:}    & Target\\_completion & \\textbf{  R-squared:         } &     0.028   \\\\\n",
       "\\textbf{Model:}            &        OLS         & \\textbf{  Adj. R-squared:    } &     0.017   \\\\\n",
       "\\textbf{Method:}           &   Least Squares    & \\textbf{  F-statistic:       } &     2.431   \\\\\n",
       "\\textbf{Date:}             &  Mon, 16 Dec 2024  & \\textbf{  Prob (F-statistic):} &   0.0344    \\\\\n",
       "\\textbf{Time:}             &      10:16:21      & \\textbf{  Log-Likelihood:    } &   -2277.8   \\\\\n",
       "\\textbf{No. Observations:} &          421       & \\textbf{  AIC:               } &     4568.   \\\\\n",
       "\\textbf{Df Residuals:}     &          415       & \\textbf{  BIC:               } &     4592.   \\\\\n",
       "\\textbf{Df Model:}         &            5       & \\textbf{                     } &             \\\\\n",
       "\\textbf{Covariance Type:}  &     nonrobust      & \\textbf{                     } &             \\\\\n",
       "\\bottomrule\n",
       "\\end{tabular}\n",
       "\\begin{tabular}{lcccccc}\n",
       "                         & \\textbf{coef} & \\textbf{std err} & \\textbf{t} & \\textbf{P$> |$t$|$} & \\textbf{[0.025} & \\textbf{0.975]}  \\\\\n",
       "\\midrule\n",
       "\\textbf{Intercept}       &      25.2501  &       17.544     &     1.439  &         0.151        &       -9.236    &       59.737     \\\\\n",
       "\\textbf{smiling[T.True]} &      11.7268  &        5.606     &     2.092  &         0.037        &        0.707    &       22.746     \\\\\n",
       "\\textbf{gender[T.male]}  &       9.4733  &        5.590     &     1.695  &         0.091        &       -1.514    &       20.461     \\\\\n",
       "\\textbf{gender[T.other]} &       2.2159  &       11.663     &     0.190  &         0.849        &      -20.711    &       25.142     \\\\\n",
       "\\textbf{sentiment}       &      -0.3565  &        0.139     &    -2.565  &         0.011        &       -0.630    &       -0.083     \\\\\n",
       "\\textbf{truth}           &       0.3923  &        0.225     &     1.743  &         0.082        &       -0.050    &        0.835     \\\\\n",
       "\\bottomrule\n",
       "\\end{tabular}\n",
       "\\begin{tabular}{lclc}\n",
       "\\textbf{Omnibus:}       & 447.003 & \\textbf{  Durbin-Watson:     } &     2.107  \\\\\n",
       "\\textbf{Prob(Omnibus):} &   0.000 & \\textbf{  Jarque-Bera (JB):  } & 28466.226  \\\\\n",
       "\\textbf{Skew:}          &   4.583 & \\textbf{  Prob(JB):          } &      0.00  \\\\\n",
       "\\textbf{Kurtosis:}      &  42.227 & \\textbf{  Cond. No.          } &      756.  \\\\\n",
       "\\bottomrule\n",
       "\\end{tabular}\n",
       "%\\caption{OLS Regression Results}\n",
       "\\end{center}\n",
       "\n",
       "Notes: \\newline\n",
       " [1] Standard Errors assume that the covariance matrix of the errors is correctly specified."
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                            OLS Regression Results                            \n",
       "==============================================================================\n",
       "Dep. Variable:      Target_completion   R-squared:                       0.028\n",
       "Model:                            OLS   Adj. R-squared:                  0.017\n",
       "Method:                 Least Squares   F-statistic:                     2.431\n",
       "Date:                Mon, 16 Dec 2024   Prob (F-statistic):             0.0344\n",
       "Time:                        10:16:21   Log-Likelihood:                -2277.8\n",
       "No. Observations:                 421   AIC:                             4568.\n",
       "Df Residuals:                     415   BIC:                             4592.\n",
       "Df Model:                           5                                         \n",
       "Covariance Type:            nonrobust                                         \n",
       "===================================================================================\n",
       "                      coef    std err          t      P>|t|      [0.025      0.975]\n",
       "-----------------------------------------------------------------------------------\n",
       "Intercept          25.2501     17.544      1.439      0.151      -9.236      59.737\n",
       "smiling[T.True]    11.7268      5.606      2.092      0.037       0.707      22.746\n",
       "gender[T.male]      9.4733      5.590      1.695      0.091      -1.514      20.461\n",
       "gender[T.other]     2.2159     11.663      0.190      0.849     -20.711      25.142\n",
       "sentiment          -0.3565      0.139     -2.565      0.011      -0.630      -0.083\n",
       "truth               0.3923      0.225      1.743      0.082      -0.050       0.835\n",
       "==============================================================================\n",
       "Omnibus:                      447.003   Durbin-Watson:                   2.107\n",
       "Prob(Omnibus):                  0.000   Jarque-Bera (JB):            28466.226\n",
       "Skew:                           4.583   Prob(JB):                         0.00\n",
       "Kurtosis:                      42.227   Cond. No.                         756.\n",
       "==============================================================================\n",
       "\n",
       "Notes:\n",
       "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
       "\"\"\""
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "smf.ols(formula=\"Target_completion ~ sentiment + smiling + gender + truth\", data=df).fit().summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre></pre>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>OLS Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>    <td>Target_completion</td> <th>  R-squared:         </th> <td>   0.056</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>                   <td>OLS</td>        <th>  Adj. R-squared:    </th> <td>   0.023</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>             <td>Least Squares</td>   <th>  F-statistic:       </th> <td>   1.717</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>             <td>Mon, 16 Dec 2024</td>  <th>  Prob (F-statistic):</th>  <td>0.0498</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>                 <td>10:02:42</td>      <th>  Log-Likelihood:    </th> <td> -2271.8</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>No. Observations:</th>      <td>   421</td>       <th>  AIC:               </th> <td>   4574.</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Residuals:</th>          <td>   406</td>       <th>  BIC:               </th> <td>   4634.</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Model:</th>              <td>    14</td>       <th>                     </th>     <td> </td>   \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Covariance Type:</th>      <td>nonrobust</td>     <th>                     </th>     <td> </td>   \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "                <td></td>                   <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th>                     <td>   26.4122</td> <td>   17.882</td> <td>    1.477</td> <td> 0.140</td> <td>   -8.741</td> <td>   61.565</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>smiling[T.True]</th>               <td>    6.0377</td> <td>    5.931</td> <td>    1.018</td> <td> 0.309</td> <td>   -5.622</td> <td>   17.697</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>gender[T.male]</th>                <td>   11.5677</td> <td>    5.695</td> <td>    2.031</td> <td> 0.043</td> <td>    0.373</td> <td>   22.762</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>gender[T.other]</th>               <td>   -0.8649</td> <td>   12.253</td> <td>   -0.071</td> <td> 0.944</td> <td>  -24.952</td> <td>   23.222</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>ethnicity[T.asian]</th>            <td>    6.1609</td> <td>   13.648</td> <td>    0.451</td> <td> 0.652</td> <td>  -20.668</td> <td>   32.990</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>ethnicity[T.black]</th>            <td>  -11.2939</td> <td>   35.120</td> <td>   -0.322</td> <td> 0.748</td> <td>  -80.333</td> <td>   57.746</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>ethnicity[T.fijian]</th>           <td>  -21.1519</td> <td>   35.067</td> <td>   -0.603</td> <td> 0.547</td> <td>  -90.087</td> <td>   47.783</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>ethnicity[T.filipino]</th>         <td>   -7.9847</td> <td>   17.862</td> <td>   -0.447</td> <td> 0.655</td> <td>  -43.098</td> <td>   27.129</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>ethnicity[T.indian]</th>           <td>   -7.8303</td> <td>   12.386</td> <td>   -0.632</td> <td> 0.528</td> <td>  -32.178</td> <td>   16.518</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>ethnicity[T.māori]</th>            <td>   19.4902</td> <td>   15.153</td> <td>    1.286</td> <td> 0.199</td> <td>  -10.298</td> <td>   49.279</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>ethnicity[T.other]</th>            <td>    8.7906</td> <td>    8.971</td> <td>    0.980</td> <td> 0.328</td> <td>   -8.845</td> <td>   26.426</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>ethnicity[T.pacific islander]</th> <td>   24.9362</td> <td>   12.777</td> <td>    1.952</td> <td> 0.052</td> <td>   -0.181</td> <td>   50.053</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>ethnicity[T.south asian]</th>      <td> -1.21e-15</td> <td> 2.09e-15</td> <td>   -0.579</td> <td> 0.563</td> <td>-5.32e-15</td> <td>  2.9e-15</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>ethnicity[T.white]</th>            <td>   15.2950</td> <td>    7.480</td> <td>    2.045</td> <td> 0.042</td> <td>    0.590</td> <td>   30.000</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>sentiment</th>                     <td>   -0.2408</td> <td>    0.149</td> <td>   -1.613</td> <td> 0.108</td> <td>   -0.534</td> <td>    0.053</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>truth</th>                         <td>    0.3866</td> <td>    0.231</td> <td>    1.675</td> <td> 0.095</td> <td>   -0.067</td> <td>    0.840</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>urgency</th>                       <td>   -0.2515</td> <td>    0.146</td> <td>   -1.721</td> <td> 0.086</td> <td>   -0.539</td> <td>    0.036</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "  <th>Omnibus:</th>       <td>455.582</td> <th>  Durbin-Watson:     </th> <td>   2.077</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Prob(Omnibus):</th> <td> 0.000</td>  <th>  Jarque-Bera (JB):  </th> <td>31394.200</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Skew:</th>          <td> 4.702</td>  <th>  Prob(JB):          </th> <td>    0.00</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Kurtosis:</th>      <td>44.246</td>  <th>  Cond. No.          </th> <td>3.83e+19</td> \n",
       "</tr>\n",
       "</table><br/><br/>Notes:<br/>[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.<br/>[2] The smallest eigenvalue is 5.31e-33. This might indicate that there are<br/>strong multicollinearity problems or that the design matrix is singular."
      ],
      "text/latex": [
       "\\begin{center}\n",
       "\\begin{tabular}{lclc}\n",
       "\\toprule\n",
       "\\textbf{Dep. Variable:}                & Target\\_completion & \\textbf{  R-squared:         } &     0.056   \\\\\n",
       "\\textbf{Model:}                        &        OLS         & \\textbf{  Adj. R-squared:    } &     0.023   \\\\\n",
       "\\textbf{Method:}                       &   Least Squares    & \\textbf{  F-statistic:       } &     1.717   \\\\\n",
       "\\textbf{Date:}                         &  Mon, 16 Dec 2024  & \\textbf{  Prob (F-statistic):} &   0.0498    \\\\\n",
       "\\textbf{Time:}                         &      10:02:42      & \\textbf{  Log-Likelihood:    } &   -2271.8   \\\\\n",
       "\\textbf{No. Observations:}             &          421       & \\textbf{  AIC:               } &     4574.   \\\\\n",
       "\\textbf{Df Residuals:}                 &          406       & \\textbf{  BIC:               } &     4634.   \\\\\n",
       "\\textbf{Df Model:}                     &           14       & \\textbf{                     } &             \\\\\n",
       "\\textbf{Covariance Type:}              &     nonrobust      & \\textbf{                     } &             \\\\\n",
       "\\bottomrule\n",
       "\\end{tabular}\n",
       "\\begin{tabular}{lcccccc}\n",
       "                                       & \\textbf{coef} & \\textbf{std err} & \\textbf{t} & \\textbf{P$> |$t$|$} & \\textbf{[0.025} & \\textbf{0.975]}  \\\\\n",
       "\\midrule\n",
       "\\textbf{Intercept}                     &      26.4122  &       17.882     &     1.477  &         0.140        &       -8.741    &       61.565     \\\\\n",
       "\\textbf{smiling[T.True]}               &       6.0377  &        5.931     &     1.018  &         0.309        &       -5.622    &       17.697     \\\\\n",
       "\\textbf{gender[T.male]}                &      11.5677  &        5.695     &     2.031  &         0.043        &        0.373    &       22.762     \\\\\n",
       "\\textbf{gender[T.other]}               &      -0.8649  &       12.253     &    -0.071  &         0.944        &      -24.952    &       23.222     \\\\\n",
       "\\textbf{ethnicity[T.asian]}            &       6.1609  &       13.648     &     0.451  &         0.652        &      -20.668    &       32.990     \\\\\n",
       "\\textbf{ethnicity[T.black]}            &     -11.2939  &       35.120     &    -0.322  &         0.748        &      -80.333    &       57.746     \\\\\n",
       "\\textbf{ethnicity[T.fijian]}           &     -21.1519  &       35.067     &    -0.603  &         0.547        &      -90.087    &       47.783     \\\\\n",
       "\\textbf{ethnicity[T.filipino]}         &      -7.9847  &       17.862     &    -0.447  &         0.655        &      -43.098    &       27.129     \\\\\n",
       "\\textbf{ethnicity[T.indian]}           &      -7.8303  &       12.386     &    -0.632  &         0.528        &      -32.178    &       16.518     \\\\\n",
       "\\textbf{ethnicity[T.māori]}            &      19.4902  &       15.153     &     1.286  &         0.199        &      -10.298    &       49.279     \\\\\n",
       "\\textbf{ethnicity[T.other]}            &       8.7906  &        8.971     &     0.980  &         0.328        &       -8.845    &       26.426     \\\\\n",
       "\\textbf{ethnicity[T.pacific islander]} &      24.9362  &       12.777     &     1.952  &         0.052        &       -0.181    &       50.053     \\\\\n",
       "\\textbf{ethnicity[T.south asian]}      &    -1.21e-15  &     2.09e-15     &    -0.579  &         0.563        &    -5.32e-15    &      2.9e-15     \\\\\n",
       "\\textbf{ethnicity[T.white]}            &      15.2950  &        7.480     &     2.045  &         0.042        &        0.590    &       30.000     \\\\\n",
       "\\textbf{sentiment}                     &      -0.2408  &        0.149     &    -1.613  &         0.108        &       -0.534    &        0.053     \\\\\n",
       "\\textbf{truth}                         &       0.3866  &        0.231     &     1.675  &         0.095        &       -0.067    &        0.840     \\\\\n",
       "\\textbf{urgency}                       &      -0.2515  &        0.146     &    -1.721  &         0.086        &       -0.539    &        0.036     \\\\\n",
       "\\bottomrule\n",
       "\\end{tabular}\n",
       "\\begin{tabular}{lclc}\n",
       "\\textbf{Omnibus:}       & 455.582 & \\textbf{  Durbin-Watson:     } &     2.077  \\\\\n",
       "\\textbf{Prob(Omnibus):} &   0.000 & \\textbf{  Jarque-Bera (JB):  } & 31394.200  \\\\\n",
       "\\textbf{Skew:}          &   4.702 & \\textbf{  Prob(JB):          } &      0.00  \\\\\n",
       "\\textbf{Kurtosis:}      &  44.246 & \\textbf{  Cond. No.          } &  3.83e+19  \\\\\n",
       "\\bottomrule\n",
       "\\end{tabular}\n",
       "%\\caption{OLS Regression Results}\n",
       "\\end{center}\n",
       "\n",
       "Notes: \\newline\n",
       " [1] Standard Errors assume that the covariance matrix of the errors is correctly specified. \\newline\n",
       " [2] The smallest eigenvalue is 5.31e-33. This might indicate that there are \\newline\n",
       " strong multicollinearity problems or that the design matrix is singular."
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                            OLS Regression Results                            \n",
       "==============================================================================\n",
       "Dep. Variable:      Target_completion   R-squared:                       0.056\n",
       "Model:                            OLS   Adj. R-squared:                  0.023\n",
       "Method:                 Least Squares   F-statistic:                     1.717\n",
       "Date:                Mon, 16 Dec 2024   Prob (F-statistic):             0.0498\n",
       "Time:                        10:02:42   Log-Likelihood:                -2271.8\n",
       "No. Observations:                 421   AIC:                             4574.\n",
       "Df Residuals:                     406   BIC:                             4634.\n",
       "Df Model:                          14                                         \n",
       "Covariance Type:            nonrobust                                         \n",
       "=================================================================================================\n",
       "                                    coef    std err          t      P>|t|      [0.025      0.975]\n",
       "-------------------------------------------------------------------------------------------------\n",
       "Intercept                        26.4122     17.882      1.477      0.140      -8.741      61.565\n",
       "smiling[T.True]                   6.0377      5.931      1.018      0.309      -5.622      17.697\n",
       "gender[T.male]                   11.5677      5.695      2.031      0.043       0.373      22.762\n",
       "gender[T.other]                  -0.8649     12.253     -0.071      0.944     -24.952      23.222\n",
       "ethnicity[T.asian]                6.1609     13.648      0.451      0.652     -20.668      32.990\n",
       "ethnicity[T.black]              -11.2939     35.120     -0.322      0.748     -80.333      57.746\n",
       "ethnicity[T.fijian]             -21.1519     35.067     -0.603      0.547     -90.087      47.783\n",
       "ethnicity[T.filipino]            -7.9847     17.862     -0.447      0.655     -43.098      27.129\n",
       "ethnicity[T.indian]              -7.8303     12.386     -0.632      0.528     -32.178      16.518\n",
       "ethnicity[T.māori]               19.4902     15.153      1.286      0.199     -10.298      49.279\n",
       "ethnicity[T.other]                8.7906      8.971      0.980      0.328      -8.845      26.426\n",
       "ethnicity[T.pacific islander]    24.9362     12.777      1.952      0.052      -0.181      50.053\n",
       "ethnicity[T.south asian]       -1.21e-15   2.09e-15     -0.579      0.563   -5.32e-15     2.9e-15\n",
       "ethnicity[T.white]               15.2950      7.480      2.045      0.042       0.590      30.000\n",
       "sentiment                        -0.2408      0.149     -1.613      0.108      -0.534       0.053\n",
       "truth                             0.3866      0.231      1.675      0.095      -0.067       0.840\n",
       "urgency                          -0.2515      0.146     -1.721      0.086      -0.539       0.036\n",
       "==============================================================================\n",
       "Omnibus:                      455.582   Durbin-Watson:                   2.077\n",
       "Prob(Omnibus):                  0.000   Jarque-Bera (JB):            31394.200\n",
       "Skew:                           4.702   Prob(JB):                         0.00\n",
       "Kurtosis:                      44.246   Cond. No.                     3.83e+19\n",
       "==============================================================================\n",
       "\n",
       "Notes:\n",
       "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
       "[2] The smallest eigenvalue is 5.31e-33. This might indicate that there are\n",
       "strong multicollinearity problems or that the design matrix is singular.\n",
       "\"\"\""
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "smf.ols(formula=\"Target_completion ~ sentiment + smiling + gender + truth + urgency + ethnicity\", data=df).fit().summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre>✔️ 581 ms (2024-12-16T10:14:49/2024-12-16T10:14:50)</pre>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-3 {\n",
       "  /* Definition of color scheme common for light and dark mode */\n",
       "  --sklearn-color-text: black;\n",
       "  --sklearn-color-line: gray;\n",
       "  /* Definition of color scheme for unfitted estimators */\n",
       "  --sklearn-color-unfitted-level-0: #fff5e6;\n",
       "  --sklearn-color-unfitted-level-1: #f6e4d2;\n",
       "  --sklearn-color-unfitted-level-2: #ffe0b3;\n",
       "  --sklearn-color-unfitted-level-3: chocolate;\n",
       "  /* Definition of color scheme for fitted estimators */\n",
       "  --sklearn-color-fitted-level-0: #f0f8ff;\n",
       "  --sklearn-color-fitted-level-1: #d4ebff;\n",
       "  --sklearn-color-fitted-level-2: #b3dbfd;\n",
       "  --sklearn-color-fitted-level-3: cornflowerblue;\n",
       "\n",
       "  /* Specific color for light theme */\n",
       "  --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, white)));\n",
       "  --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-icon: #696969;\n",
       "\n",
       "  @media (prefers-color-scheme: dark) {\n",
       "    /* Redefinition of color scheme for dark theme */\n",
       "    --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, #111)));\n",
       "    --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-icon: #878787;\n",
       "  }\n",
       "}\n",
       "\n",
       "#sk-container-id-3 {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "#sk-container-id-3 pre {\n",
       "  padding: 0;\n",
       "}\n",
       "\n",
       "#sk-container-id-3 input.sk-hidden--visually {\n",
       "  border: 0;\n",
       "  clip: rect(1px 1px 1px 1px);\n",
       "  clip: rect(1px, 1px, 1px, 1px);\n",
       "  height: 1px;\n",
       "  margin: -1px;\n",
       "  overflow: hidden;\n",
       "  padding: 0;\n",
       "  position: absolute;\n",
       "  width: 1px;\n",
       "}\n",
       "\n",
       "#sk-container-id-3 div.sk-dashed-wrapped {\n",
       "  border: 1px dashed var(--sklearn-color-line);\n",
       "  margin: 0 0.4em 0.5em 0.4em;\n",
       "  box-sizing: border-box;\n",
       "  padding-bottom: 0.4em;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "#sk-container-id-3 div.sk-container {\n",
       "  /* jupyter's `normalize.less` sets `[hidden] { display: none; }`\n",
       "     but bootstrap.min.css set `[hidden] { display: none !important; }`\n",
       "     so we also need the `!important` here to be able to override the\n",
       "     default hidden behavior on the sphinx rendered scikit-learn.org.\n",
       "     See: https://github.com/scikit-learn/scikit-learn/issues/21755 */\n",
       "  display: inline-block !important;\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-3 div.sk-text-repr-fallback {\n",
       "  display: none;\n",
       "}\n",
       "\n",
       "div.sk-parallel-item,\n",
       "div.sk-serial,\n",
       "div.sk-item {\n",
       "  /* draw centered vertical line to link estimators */\n",
       "  background-image: linear-gradient(var(--sklearn-color-text-on-default-background), var(--sklearn-color-text-on-default-background));\n",
       "  background-size: 2px 100%;\n",
       "  background-repeat: no-repeat;\n",
       "  background-position: center center;\n",
       "}\n",
       "\n",
       "/* Parallel-specific style estimator block */\n",
       "\n",
       "#sk-container-id-3 div.sk-parallel-item::after {\n",
       "  content: \"\";\n",
       "  width: 100%;\n",
       "  border-bottom: 2px solid var(--sklearn-color-text-on-default-background);\n",
       "  flex-grow: 1;\n",
       "}\n",
       "\n",
       "#sk-container-id-3 div.sk-parallel {\n",
       "  display: flex;\n",
       "  align-items: stretch;\n",
       "  justify-content: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-3 div.sk-parallel-item {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       "\n",
       "#sk-container-id-3 div.sk-parallel-item:first-child::after {\n",
       "  align-self: flex-end;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-3 div.sk-parallel-item:last-child::after {\n",
       "  align-self: flex-start;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-3 div.sk-parallel-item:only-child::after {\n",
       "  width: 0;\n",
       "}\n",
       "\n",
       "/* Serial-specific style estimator block */\n",
       "\n",
       "#sk-container-id-3 div.sk-serial {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  align-items: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  padding-right: 1em;\n",
       "  padding-left: 1em;\n",
       "}\n",
       "\n",
       "\n",
       "/* Toggleable style: style used for estimator/Pipeline/ColumnTransformer box that is\n",
       "clickable and can be expanded/collapsed.\n",
       "- Pipeline and ColumnTransformer use this feature and define the default style\n",
       "- Estimators will overwrite some part of the style using the `sk-estimator` class\n",
       "*/\n",
       "\n",
       "/* Pipeline and ColumnTransformer style (default) */\n",
       "\n",
       "#sk-container-id-3 div.sk-toggleable {\n",
       "  /* Default theme specific background. It is overwritten whether we have a\n",
       "  specific estimator or a Pipeline/ColumnTransformer */\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "/* Toggleable label */\n",
       "#sk-container-id-3 label.sk-toggleable__label {\n",
       "  cursor: pointer;\n",
       "  display: block;\n",
       "  width: 100%;\n",
       "  margin-bottom: 0;\n",
       "  padding: 0.5em;\n",
       "  box-sizing: border-box;\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "#sk-container-id-3 label.sk-toggleable__label-arrow:before {\n",
       "  /* Arrow on the left of the label */\n",
       "  content: \"▸\";\n",
       "  float: left;\n",
       "  margin-right: 0.25em;\n",
       "  color: var(--sklearn-color-icon);\n",
       "}\n",
       "\n",
       "#sk-container-id-3 label.sk-toggleable__label-arrow:hover:before {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "/* Toggleable content - dropdown */\n",
       "\n",
       "#sk-container-id-3 div.sk-toggleable__content {\n",
       "  max-height: 0;\n",
       "  max-width: 0;\n",
       "  overflow: hidden;\n",
       "  text-align: left;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-3 div.sk-toggleable__content.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-3 div.sk-toggleable__content pre {\n",
       "  margin: 0.2em;\n",
       "  border-radius: 0.25em;\n",
       "  color: var(--sklearn-color-text);\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-3 div.sk-toggleable__content.fitted pre {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-3 input.sk-toggleable__control:checked~div.sk-toggleable__content {\n",
       "  /* Expand drop-down */\n",
       "  max-height: 200px;\n",
       "  max-width: 100%;\n",
       "  overflow: auto;\n",
       "}\n",
       "\n",
       "#sk-container-id-3 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {\n",
       "  content: \"▾\";\n",
       "}\n",
       "\n",
       "/* Pipeline/ColumnTransformer-specific style */\n",
       "\n",
       "#sk-container-id-3 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-3 div.sk-label.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator-specific style */\n",
       "\n",
       "/* Colorize estimator box */\n",
       "#sk-container-id-3 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-3 div.sk-estimator.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-3 div.sk-label label.sk-toggleable__label,\n",
       "#sk-container-id-3 div.sk-label label {\n",
       "  /* The background is the default theme color */\n",
       "  color: var(--sklearn-color-text-on-default-background);\n",
       "}\n",
       "\n",
       "/* On hover, darken the color of the background */\n",
       "#sk-container-id-3 div.sk-label:hover label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "/* Label box, darken color on hover, fitted */\n",
       "#sk-container-id-3 div.sk-label.fitted:hover label.sk-toggleable__label.fitted {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator label */\n",
       "\n",
       "#sk-container-id-3 div.sk-label label {\n",
       "  font-family: monospace;\n",
       "  font-weight: bold;\n",
       "  display: inline-block;\n",
       "  line-height: 1.2em;\n",
       "}\n",
       "\n",
       "#sk-container-id-3 div.sk-label-container {\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "/* Estimator-specific */\n",
       "#sk-container-id-3 div.sk-estimator {\n",
       "  font-family: monospace;\n",
       "  border: 1px dotted var(--sklearn-color-border-box);\n",
       "  border-radius: 0.25em;\n",
       "  box-sizing: border-box;\n",
       "  margin-bottom: 0.5em;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-3 div.sk-estimator.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "/* on hover */\n",
       "#sk-container-id-3 div.sk-estimator:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-3 div.sk-estimator.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Specification for estimator info (e.g. \"i\" and \"?\") */\n",
       "\n",
       "/* Common style for \"i\" and \"?\" */\n",
       "\n",
       ".sk-estimator-doc-link,\n",
       "a:link.sk-estimator-doc-link,\n",
       "a:visited.sk-estimator-doc-link {\n",
       "  float: right;\n",
       "  font-size: smaller;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1em;\n",
       "  height: 1em;\n",
       "  width: 1em;\n",
       "  text-decoration: none !important;\n",
       "  margin-left: 1ex;\n",
       "  /* unfitted */\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted,\n",
       "a:link.sk-estimator-doc-link.fitted,\n",
       "a:visited.sk-estimator-doc-link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "div.sk-estimator:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "div.sk-estimator.fitted:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "/* Span, style for the box shown on hovering the info icon */\n",
       ".sk-estimator-doc-link span {\n",
       "  display: none;\n",
       "  z-index: 9999;\n",
       "  position: relative;\n",
       "  font-weight: normal;\n",
       "  right: .2ex;\n",
       "  padding: .5ex;\n",
       "  margin: .5ex;\n",
       "  width: min-content;\n",
       "  min-width: 20ex;\n",
       "  max-width: 50ex;\n",
       "  color: var(--sklearn-color-text);\n",
       "  box-shadow: 2pt 2pt 4pt #999;\n",
       "  /* unfitted */\n",
       "  background: var(--sklearn-color-unfitted-level-0);\n",
       "  border: .5pt solid var(--sklearn-color-unfitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted span {\n",
       "  /* fitted */\n",
       "  background: var(--sklearn-color-fitted-level-0);\n",
       "  border: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link:hover span {\n",
       "  display: block;\n",
       "}\n",
       "\n",
       "/* \"?\"-specific style due to the `<a>` HTML tag */\n",
       "\n",
       "#sk-container-id-3 a.estimator_doc_link {\n",
       "  float: right;\n",
       "  font-size: 1rem;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1rem;\n",
       "  height: 1rem;\n",
       "  width: 1rem;\n",
       "  text-decoration: none;\n",
       "  /* unfitted */\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "}\n",
       "\n",
       "#sk-container-id-3 a.estimator_doc_link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "#sk-container-id-3 a.estimator_doc_link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "#sk-container-id-3 a.estimator_doc_link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "</style><div id=\"sk-container-id-3\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>RandomForestRegressor()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" checked><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow fitted\">&nbsp;&nbsp;RandomForestRegressor<a class=\"sk-estimator-doc-link fitted\" rel=\"noreferrer\" target=\"_blank\" href=\"https://scikit-learn.org/1.4/modules/generated/sklearn.ensemble.RandomForestRegressor.html\">?<span>Documentation for RandomForestRegressor</span></a><span class=\"sk-estimator-doc-link fitted\">i<span>Fitted</span></span></label><div class=\"sk-toggleable__content fitted\"><pre>RandomForestRegressor()</pre></div> </div></div></div></div>"
      ],
      "text/plain": [
       "RandomForestRegressor()"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = RandomForestRegressor()\n",
    "cols = \"sentiment + smiling + gender + truth + urgency + ethnicity\".split(\" + \")\n",
    "X = pd.get_dummies(df[cols])\n",
    "y = df[\"Target_completion\"]\n",
    "model.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre>✔️ 13.4 ms (2024-12-16T10:15:01/2024-12-16T10:15:01)</pre>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[('sentiment', 0.29360972579137923),\n",
       " ('smiling', 0.1918739363914363),\n",
       " ('gender', 0.21766417627244206),\n",
       " ('truth', 0.03354045864811954),\n",
       " ('urgency', 0.00725001053236176),\n",
       " ('ethnicity', 0.0019828296882488)]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(zip(cols, model.feature_importances_))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}