rq3-analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "414082ba",
   "metadata": {},
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4248a89",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "from sklearn.cluster import KMeans\n",
    "import heapq\n",
    "import scipy.stats as stats\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_selection import chi2\n",
    "import string\n",
    "from nltk.util import ngrams\n",
    "from nltk.corpus import stopwords\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from itertools import combinations\n",
    "\n",
    "from statsmodels.stats.multitest import multipletests\n",
    "\n",
    "english_stopwords = stopwords.words('english')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "331929fd",
   "metadata": {},
   "source": [
    "## Read in processed data from R"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c09ede1",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('d_analysis.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92f030a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['outcome_next_bin'] = df['outcome'].map(lambda s: 0 if pd.isna(s) else 1 if s=='CORRECT' else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f390ce6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[['utterance_combined', 'outcome_next_bin']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4ebc397",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_with_ngrams(s, n=2):\n",
    "    if pd.isna(s):\n",
    "        return np.nan\n",
    "    \n",
    "    translator = str.maketrans('', '', string.punctuation)\n",
    "    s = s.lower()\n",
    "    s = s.replace('/', ' ')\n",
    "    s = s.translate(translator)  # Remove punctuation\n",
    "    \n",
    "    tokens = s.split()\n",
    "    filtered_tokens = [word for word in tokens if word not in english_stopwords and not word.isnumeric()]\n",
    "    \n",
    "    ngram_list = list(ngrams(filtered_tokens, n))\n",
    "    \n",
    "    return filtered_tokens, ngram_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e23556e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['unigrams'] = df.utterance_combined.map(lambda s: tokenize_with_ngrams(s, n=2)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d8082813",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_words_correct = [word for sublist in df[df['outcome_next_bin']==1]['unigrams'] for word in sublist]\n",
    "all_words_incorrect = [word for sublist in df[df['outcome_next_bin']==0]['unigrams'] for word in sublist]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e48d2b55",
   "metadata": {},
   "source": [
    "## Perform ChiSq filtering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32884a5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def identity_tokenizer(text):\n",
    "    return [text]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f838797",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Combine the lists into a corpus\n",
    "corpus = all_words_incorrect + all_words_correct\n",
    "\n",
    "# Create the CountVectorizer to convert text into a matrix of word counts\n",
    "vectorizer = CountVectorizer(tokenizer=identity_tokenizer, lowercase=False, preprocessor=None)\n",
    "\n",
    "# Fit and transform the corpus using CountVectorizer\n",
    "X = vectorizer.fit_transform(corpus)\n",
    "\n",
    "# Determine the length of each individual list (corpus size for each category)\n",
    "n_incorrect = len(all_words_incorrect)\n",
    "n_correct = len(all_words_correct)\n",
    "\n",
    "# Create labels for the corpora (0 for dlfb, 1 for dlweb)\n",
    "labels = np.array([0] * n_incorrect + [1] * n_correct)\n",
    "\n",
    "# Perform chi-square test on the word counts and labels\n",
    "chi2_scores, p_values = chi2(X, labels)\n",
    "\n",
    "# Create a list of (word, chi-square score) tuples\n",
    "word_scores = list(zip(vectorizer.get_feature_names(), chi2_scores, p_values))\n",
    "\n",
    "# Sort the list based on the chi-square scores in descending order\n",
    "word_scores = sorted(word_scores, key=lambda x: abs(x[1]), reverse=True)\n",
    "\n",
    "# Get the top 10 distinctive words based on chi-square scores\n",
    "top_words_chi2 = word_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c7e832c",
   "metadata": {},
   "outputs": [],
   "source": [
    "count_correct = Counter(all_words_correct)\n",
    "count_incorrect = Counter(all_words_incorrect)\n",
    "dat = [(word, count_correct.get(word), count_incorrect.get(word), chi, p) for word, chi, p in top_words_chi2]\n",
    " \n",
    "# Define column names for the DataFrame\n",
    "columns = ['word', 'count_correct', 'count_incorrect', 'chisq', 'p']\n",
    "\n",
    "# Create the DataFrame\n",
    "ans = pd.DataFrame(dat, columns=columns).fillna(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81a5a7e7",
   "metadata": {},
   "source": [
    "## Top unigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d39ee560",
   "metadata": {},
   "outputs": [],
   "source": [
    "ans[ans['count_correct']>=ans['count_incorrect']].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a68451e",
   "metadata": {},
   "outputs": [],
   "source": [
    "ans[ans['count_correct']<=ans['count_incorrect']].head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "899aa2dc",
   "metadata": {},
   "source": [
    "## P-Adjustment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2622d700",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Apply p-adjustment\n",
    "corrections = multipletests(ans['p'], alpha=0.1, method='fdr_bh')\n",
    "ans['p_corrected'] = corrections[1]\n",
    "ans['significant'] = corrections[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62d150bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Show any significant p-values\n",
    "ans[ans['significant']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54158855",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export results\n",
    "ans.to_csv('ans_sorted.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}