Analise_ASCO2021_Breast.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c881b31-db3f-4b2e-9428-3edbf99738fb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pymongo import MongoClient\n",
    "from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator\n",
    "from textblob import TextBlob\n",
    "import re\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "from nltk.sentiment import SentimentIntensityAnalyzer\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer, LancasterStemmer\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk import pos_tag\n",
    "import string\n",
    "import spacy\n",
    "import os\n",
    "from textblob import TextBlob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ccf3c4e-7e70-4942-b45e-e1485b0156c9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Importando os stopwords\n",
    "stopwords = nltk.corpus.stopwords.words('english')\n",
    "\n",
    "# Adicionar a palavra ASCO como stopwords\n",
    "stopwords.append('asco')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1fa5cd9-1522-4627-9229-5a1e3ec68bc3",
   "metadata": {},
   "source": [
    "**<h1>Análise de todos os tweets com breast inclusive os RTs</h1>**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "588f8312-50b5-4889-ab49-13f9027aba4b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"tweets_limpos.csv\", parse_dates = [\"created_at\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "526627a1-b093-4914-b516-2cf70446542c",
   "metadata": {},
   "source": [
    "**Verificando tweets que contem breast no texto**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e75d663-0fa2-4c17-9ad4-f95d6cf1a350",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "breast = df[df.text.str.contains(r'breast', re.I)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1dc127ee-7150-4d42-897b-7a0dc1ed73b0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "breast"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ab39575-b14d-4b04-a768-1bff4861adf0",
   "metadata": {},
   "source": [
    "**Limpar novamente os tweets**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef4095e3-0142-4f05-9b6e-add7b7aad0de",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Função para limpar o texto: substitui todos os caracteres ($,/,@)\n",
    "\n",
    "def cleanUpText(txt):\n",
    "    # Remove caracteres\n",
    "    txt = re.sub(r\"\\``\", '', txt)\n",
    "    txt = re.sub(r'\\$', '', txt)\n",
    "    txt = re.sub(r'\\/', '', txt)\n",
    "    txt = re.sub(r'\\@', '', txt)\n",
    "    return txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a066d45c-cdcb-4f44-b17e-3c9376a81e4f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Aplicar a função de limpar os tweets\n",
    "breast['text'] = breast['text'].apply(cleanUpText)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f13551f6-fd96-4aca-adde-98a9fc4b2911",
   "metadata": {},
   "source": [
    "**Criar lista de tweets**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba13a45f-5bd1-4085-9c0e-5f788dce8597",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Criar lista de palavras de todos os tweets\n",
    "tweets = \"\" \n",
    "\n",
    "# Separar os inisghts em palavras e adicionando na lista \n",
    "for i in breast[\"text\"]: \n",
    "      \n",
    "    # Transforar em string \n",
    "    i = str(i) \n",
    "  \n",
    "    # Separar as palavras \n",
    "    tokens = i.split()\n",
    "        \n",
    "    # Converter em lowercase \n",
    "    for i in range(len(tokens)): \n",
    "        tokens[i] = tokens[i].lower() \n",
    "        \n",
    "     # Adicionar a lista com espaco entre as palavras\n",
    "    tweets += \" \".join(tokens)+\" \""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3663a9b-b6fe-4047-960a-f4eb6bd0370d",
   "metadata": {},
   "source": [
    "**Tokenization**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ab54f58-468e-4c1d-8764-8cd7a895ded5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Separar em palavras (tokenization)\n",
    "tweets = [word for word in nltk.word_tokenize(tweets)]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9bec8a6-5287-4423-9889-5e8909824a66",
   "metadata": {},
   "source": [
    "**Pos_tag**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e265c00-e521-4e1e-a7e3-38285aaf8b65",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Verificar os pos_tag\n",
    "data_tagset = nltk.pos_tag(tweets) \n",
    "df_tagset = pd.DataFrame(data_tagset, columns=['Word', 'Tag'])\n",
    "df_tagset.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e3647c7-7961-4a6d-b43b-0cd0acc522fe",
   "metadata": {},
   "source": [
    "**Lemmatization de acordo com o pos_tag**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f5abe3c-6d8c-4109-abf1-dc9459033c47",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Criar o Lemma de cada palavra e criar df_full\n",
    "lemmatize_text = []\n",
    "\n",
    "for tweet in tweets:\n",
    "    output = [tweet, WordNetLemmatizer().lemmatize(tweet, pos='n'), WordNetLemmatizer().lemmatize(tweet, pos='a'),WordNetLemmatizer().lemmatize(tweet, pos='v')]\n",
    "    lemmatize_text.append(output)\n",
    "    \n",
    "# Adicioanr ao df_full com a palavra original e os lemma\n",
    "df_full = pd.DataFrame(lemmatize_text, columns =['Word', 'Lemmatized Noun', 'Lemmatized Adjective', 'Lemmatized Verb']) \n",
    "\n",
    "df_full['Tag'] = df_tagset['Tag']\n",
    "\n",
    "# Selecionar a palavra lemmatizada de acordo com a pos_tag\n",
    "\n",
    "df_full['Tempt Lemmatized Word']=df_full['Lemmatized Noun'] + ' | ' + df_full['Lemmatized Adjective']+ ' | ' + df_full['Lemmatized Verb']\n",
    "lemma_word = df_full['Tempt Lemmatized Word']\n",
    "tag = df_full['Tag']\n",
    "i = 0\n",
    "new_word = []\n",
    "while i < len(tag):\n",
    "    words = lemma_word[i].split('|')\n",
    "    if tag[i] == 'NN' or tag[i] == 'NNS' or tag[i] == 'NNP' or tag[i] == 'NNPS':        \n",
    "        word = words[0]\n",
    "    elif tag[i] == 'JJ' or tag[i] == 'JJR' or tag[i] == 'JJS':\n",
    "        word = words[1]\n",
    "    elif tag[i] == 'VBG' or tag[i] == 'VBP' or tag[i] == 'VB' or tag[i] == 'VBD' or tag[i] == 'VBN' or tag[i] == 'VBZ':\n",
    "        word = words[2]\n",
    "    else:\n",
    "        word = words[0]\n",
    "    new_word.append(word)\n",
    "    i += 1\n",
    "    \n",
    "df_full['Lemmatized Word']=new_word\n",
    "df_full.drop('Tempt Lemmatized Word', axis = 1, inplace=True)\n",
    "df_full['Lemmatized Word'] = [re.sub(' ', '', x) for x in df_full['Lemmatized Word']]\n",
    "#\n",
    "df_full.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e7ff2773-e77d-4b56-9f17-829091280491",
   "metadata": {},
   "source": [
    "**Excluir os stopwords**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c074bde-f3be-496c-9422-1d0dd873a710",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Tirar os stopwords dos tweets lemmatizados e limpos                \n",
    "lemma_word = df_full['Lemmatized Word']\n",
    "clean_list_lemma = pd.DataFrame([word for word in lemma_word if not word in stopwords], columns=[\"Lemmatized Word\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5db8d4a-04d7-4dff-9178-8c4756389da7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Tirar os stopwords dos tweets como vieram e limpos\n",
    "text_word = df_full['Word']\n",
    "clean_list_word = pd.DataFrame([word for word in text_word if not word in stopwords], columns=[\"Word\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "16f53688-93fe-43ed-929c-f04642bc7381",
   "metadata": {},
   "source": [
    "**Stemmazation**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62becace-f6b0-4319-864e-d6770ba6fce7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tweets_lancaster = [LancasterStemmer().stem(word) for word in tweets]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "adb2e35c-d9af-474e-9492-1e8ec57bb644",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tweets_porter = [PorterStemmer().stem(word) for word in tweets]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4fa32c01-3cec-4798-acfc-fe16265bfc0d",
   "metadata": {},
   "source": [
    "**Collocations - bigramas e trigramas**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8bdf7459-c205-4151-bc59-5c65a0ad8f31",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clean_list_word = [str(x) for x in clean_list_lemma['Lemmatized Word']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee05f735-d865-42f7-ac7c-92dc3bb372de",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "clean_list_word_text = [str(x) for x in clean_list_word['Word']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0abfaeb-3b75-436f-afac-bfbc869617a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Criar um buscador de bigramas nos tokens\n",
    "buscaGramas = nltk.collocations.FreqDist(clean_list_word)\n",
    "\n",
    "# Contar quantas vezes cada bigrama aparece nos tokens dos comentários\n",
    "grama_freq = buscaGramas.most_common(100)\n",
    "\n",
    "# Converter o dicionário anterior em uma tabela de frequência no formato do Pandas para os bigramas\n",
    "FreqTabGramas = pd.DataFrame(list(grama_freq), \n",
    "                               columns = ['Grama', 'Freq']).sort_values(by = 'Freq', ascending = False)\n",
    "\n",
    "FreqTabGramas.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58694fb1-d5e7-47f5-9fba-ec2609d7f498",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Criar um buscador de bigramas nos tokens\n",
    "buscaBigramas = nltk.collocations.BigramCollocationFinder.from_words(clean_list_word)\n",
    "\n",
    "# Contar quantas vezes cada bigrama aparece nos tokens dos comentários\n",
    "bigrama_freq = buscaBigramas.ngram_fd.items()\n",
    "\n",
    "# Converter o dicionário anterior em uma tabela de frequência no formato do Pandas para os bigramas\n",
    "FreqTabBigramas = pd.DataFrame(list(bigrama_freq), \n",
    "                               columns = ['Bigrama', 'Freq']).sort_values(by = 'Freq', ascending = False)\n",
    "\n",
    "FreqTabBigramas.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90c3f639-fc08-4b14-b2c5-dd31fd11b846",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Fazemos o mesmo com trigramas. Fique atento aos métodos que estão sendo usados\n",
    "buscaTrigramas = nltk.collocations.TrigramCollocationFinder.from_words(clean_list_word)\n",
    "\n",
    "# Vamos contar quantas vezes cada trigrama aparece nos tokens dos comentários\n",
    "trigrama_freq = buscaTrigramas.ngram_fd.items()\n",
    "\n",
    "# Tabela de frequência no formato do Pandas para os trigramas\n",
    "FreqTabTrigramas = pd.DataFrame(list(trigrama_freq), \n",
    "                                columns = ['Trigrama','Freq']).sort_values(by = 'Freq', ascending = False)\n",
    "\n",
    "FreqTabTrigramas.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "faa8392b-4bfd-499e-8b44-7244d15f363e",
   "metadata": {},
   "source": [
    "**Remover os substantivos e pronomes**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8bd3900-59fe-4a9a-ba16-d3fb985d49a2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Função para filtrar bigramas ADJ/NN e remover stopwords\n",
    "def filtra_tipo_token_bigrama(ngram):\n",
    "    \n",
    "    # Verifica se é pronome\n",
    "    if '-pron-' in ngram or 't' in ngram:\n",
    "        return False\n",
    "    \n",
    "    # Loop nos ngramas para verificar se é stopword\n",
    "    for word in ngram:\n",
    "        if word in stopwords or word.isspace():\n",
    "            return False\n",
    "        \n",
    "    # Tipos de tokens aceitáveis\n",
    "    acceptable_types = ('JJ', 'JJR', 'JJS', 'NN', 'NNS', 'NNP', 'NNPS')\n",
    "    \n",
    "    # Subtipos\n",
    "    second_type = ('NN', 'NNS', 'NNP', 'NNPS')\n",
    "    \n",
    "    # Tags\n",
    "    tags = nltk.pos_tag(ngram)\n",
    "    \n",
    "    # Retorna o que queremos, ADJ/NN\n",
    "    if tags[0][1] in acceptable_types and tags[1][1] in second_type:\n",
    "        return True\n",
    "    else:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "545fa0fe-e0f6-4b2d-8a6d-43026ade7055",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Agora filtramos os bigramas\n",
    "bigramas_filtrados = FreqTabBigramas[FreqTabBigramas.Bigrama.map(lambda x: filtra_tipo_token_bigrama(x))]\n",
    "# Visualiza a tabela\n",
    "bigramas_filtrados.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d410e3b-0f4f-4637-8e55-31938e0541af",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Função para filtrar trigramas ADJ/NN e remover stopwords\n",
    "def filtra_tipo_token_trigrama(ngram):\n",
    "    \n",
    "    # Verifica se é pronome\n",
    "    if '-pron-' in ngram or 't' in ngram:\n",
    "        return False\n",
    "    \n",
    "    # Loop nos ngramas para verificar se é stopword\n",
    "    for word in ngram:\n",
    "        if word in stopwords or word.isspace():\n",
    "            return False\n",
    "        \n",
    "    # Tipos de tokens aceitáveis\n",
    "    first_type = ('JJ', 'JJR', 'JJS', 'NN', 'NNS', 'NNP', 'NNPS')\n",
    "    \n",
    "    # Subtipos\n",
    "    second_type = ('JJ', 'JJR', 'JJS', 'NN', 'NNS', 'NNP', 'NNPS')\n",
    "    \n",
    "    # Tags\n",
    "    tags = nltk.pos_tag(ngram)\n",
    "    \n",
    "    # Retorna o que queremos, ADJ/NN\n",
    "    if tags[0][1] in first_type and tags[2][1] in second_type:\n",
    "        return True\n",
    "    else:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1dfac55e-3576-4114-a31f-150fbc3a19c2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Agora filtramos os trigramas\n",
    "trigramas_filtrados = FreqTabTrigramas[FreqTabTrigramas.Trigrama.map(lambda x: filtra_tipo_token_trigrama(x))]\n",
    "# Visualiza a tabela\n",
    "trigramas_filtrados.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "046b63c1-42a3-4bdb-98d3-8226f77814bc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Métricas de associação de bigramas (esse objeto possui diversos atributos, como freq, pmi, teste t, etc...)\n",
    "bigramas = nltk.collocations.BigramAssocMeasures()\n",
    "\n",
    "# Criamos a tabela\n",
    "PMITabBigramas = pd.DataFrame(list(buscaBigramas.score_ngrams(bigramas.pmi)), \n",
    "                              columns = ['Bigrama', 'PMI']).sort_values(by = 'PMI', ascending = False)\n",
    "# Visualiza a tabela\n",
    "PMITabBigramas.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8d1d666-c435-4020-a88d-d5a16213fd15",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Métricas de associação de trigramas\n",
    "trigramas = nltk.collocations.TrigramAssocMeasures()\n",
    "\n",
    "# Criamos a tabela\n",
    "PMITabTrigramas = pd.DataFrame(list(buscaTrigramas.score_ngrams(trigramas.pmi)), \n",
    "                               columns = ['Trigrama', 'PMI']).sort_values(by = 'PMI', ascending = False)\n",
    "# Visualiza a tabela\n",
    "PMITabTrigramas.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ce37906-77bb-4364-becb-1d18884fee0d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Criamos a tabela para os bigramas\n",
    "# Observe como o resultado do teste t é obtido: buscaBigramas.score_ngrams(bigramas.student_t)\n",
    "TestetTabBigramas = pd.DataFrame(list(buscaBigramas.score_ngrams(bigramas.student_t)), \n",
    "                             columns = ['Bigrama', 'Teste-t']).sort_values(by = 'Teste-t', ascending = False)\n",
    "# Vamos aplicar o filtro pelo tipo de token conforme aplicamos no método 1\n",
    "bigramas_t_filtrados = TestetTabBigramas[TestetTabBigramas.Bigrama.map(lambda x: filtra_tipo_token_bigrama(x))]\n",
    "# Visualiza a tabela\n",
    "bigramas_t_filtrados.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9096955-f75d-49d7-9543-2e33ea20ca23",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Criamos a tabela para os trigramas\n",
    "TestetTabTrigramas = pd.DataFrame(list(buscaTrigramas.score_ngrams(trigramas.student_t)), \n",
    "                                  columns = ['Trigrama', 'Teste-t']).sort_values(by = 'Teste-t', ascending = False)\n",
    "# Vamos aplicar o filtro pelo tipo de token conforme aplicamos no método 1\n",
    "trigramas_t_filtrados = TestetTabTrigramas[TestetTabTrigramas.Trigrama.map(lambda x: filtra_tipo_token_trigrama(x))]\n",
    "# Visualiza a tabela\n",
    "trigramas_t_filtrados.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b19a1047-2698-4b3b-a0f8-e8fa2f79269e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Prepara a tabela\n",
    "# Observe como estamos coletando a estatística qui-quadrado: buscaBigramas.score_ngrams(bigramas.chi_sq)\n",
    "QuiTabBigramas = pd.DataFrame(list(buscaBigramas.score_ngrams(bigramas.chi_sq)), \n",
    "                              columns = ['Bigrama', 'Qui']).sort_values(by = 'Qui', ascending = False)\n",
    "# Visualiza a tabela\n",
    "QuiTabBigramas.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5cbf590f-a93c-4391-bf74-c945b5e69cf9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Prepara a tabela\n",
    "QuiTabTrigramas = pd.DataFrame(list(buscaTrigramas.score_ngrams(trigramas.chi_sq)), \n",
    "                               columns = ['Trigrama', 'Qui']).sort_values(by = 'Qui', ascending = False)\n",
    "# Visualiza a tabela\n",
    "QuiTabTrigramas.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e9774b89-c402-49e6-829d-6ad0502f4695",
   "metadata": {},
   "source": [
    "**Tabelar os resultados**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52d2e140-4097-45c5-91a1-faebbd9376d6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Vamos extrair os 10 Collocations bigramas mais relevantes de acordo com cada um dos 4 métodos usados\n",
    "# Lembre-se que aplicamos filtros para remover as stopwords e devemos usar a tabela filtrada\n",
    "metodo1_bigrama = bigramas_filtrados[:10].Bigrama.values\n",
    "metodo2_bigrama = PMITabBigramas[:10].Bigrama.values\n",
    "metodo3_bigrama = bigramas_t_filtrados[:10].Bigrama.values\n",
    "metodo4_bigrama = QuiTabBigramas[:10].Bigrama.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "329c4680-6583-4e17-95a8-56871a7b9603",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Vamos extrair os 10 Collocations trigramas mais relevantes de acordo com cada um dos 4 métodos usados\n",
    "# Lembre-se que aplicamos filtros para remover as stopwords e devemos usar a tabela filtrada\n",
    "metodo1_trigrama = trigramas_filtrados[:10].Trigrama.values\n",
    "metodo2_trigrama = PMITabTrigramas[:10].Trigrama.values\n",
    "metodo3_trigrama = trigramas_t_filtrados[:10].Trigrama.values\n",
    "metodo4_trigrama = QuiTabTrigramas[:10].Trigrama.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "416c2b56-a9ae-4174-af68-1d2f026543bd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Vamos criar um super dataframe com todos os resultados para bigramas\n",
    "comparaBigramas = pd.DataFrame([metodo1_bigrama, metodo2_bigrama, metodo3_bigrama, metodo4_bigrama]).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68097b88-8374-437f-9b78-70edfec4995f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Nossa tabela precisa de nomes para as colunas\n",
    "comparaBigramas.columns = ['Frequência', \n",
    "                           'PMI', \n",
    "                           'Teste-t', \n",
    "                           'Teste Qui-quadrado']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "749aa646-369a-4ce2-8c7a-fa689692073d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Visualiza a tabela - Padrão CSS\n",
    "comparaBigramas.style.set_properties(**{'background-color': 'green', \n",
    "                                        'color': 'white', \n",
    "                                        'border-color': 'white'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9205aeb1-8e3a-41df-97f8-a411cac418c8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Vamos criar um super dataframe com todos os resultados para trigramas\n",
    "comparaTrigramas = pd.DataFrame([metodo1_trigrama, metodo2_trigrama, metodo3_trigrama, metodo4_trigrama]).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7d22642-d2d1-40e9-863f-fd73805f9f22",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Nossa tabela precisa de nomes para as colunas\n",
    "comparaTrigramas.columns = ['Frequência', \n",
    "                            'PMI', \n",
    "                            'Teste-t', \n",
    "                            'Teste Qui-quadrado']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d35523a9-fa07-4155-a0ea-a4338ffec78e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Visualiza a tabela\n",
    "comparaTrigramas.style.set_properties(**{'background-color': 'blue', \n",
    "                                         'color': 'white', \n",
    "                                         'border-color': 'white'})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ed196f0-0f15-484e-b4bd-c1912118114d",
   "metadata": {},
   "source": [
    "**Wordcloud**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c10e140d-80eb-409a-9a6e-e0fe2fdd3fc3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "wordcloud_text = str(clean_list_word)\n",
    "wordcloud_text = re.sub(\"'\", '', wordcloud_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e97bb12d-c271-4c57-ad52-1d8698a0ea92",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Setando a wordcloud por palavras\n",
    "wordcloud = WordCloud(width = 3000, height = 2000, \n",
    "                background_color ='black', \n",
    "                collocations=True,\n",
    "                stopwords=stopwords,\n",
    "                colormap='Set2',\n",
    "                min_font_size = 10).generate(wordcloud_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7167fe48-1199-46f2-9791-e2f6ab20da9a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# plotando a wordcloud\n",
    "plt.figure(figsize = (8, 8), facecolor = None) \n",
    "plt.imshow(wordcloud) \n",
    "plt.axis(\"off\") \n",
    "plt.tight_layout(pad = 0) \n",
    "\n",
    "plt.show() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3bc4c56a-9379-43b9-812a-1c65d51106b9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Setando a wordcloud por palavras\n",
    "wordcloud = WordCloud(width = 3000, height = 2000, \n",
    "                background_color ='white', \n",
    "                collocations=False,\n",
    "                stopwords=stopwords,\n",
    "                colormap='Set2',\n",
    "                min_font_size = 10).generate(wordcloud_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "864512b5-b17c-4ce2-80a1-8a30cbee6be1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# plotando a wordcloud\n",
    "plt.figure(figsize = (8, 8), facecolor = None) \n",
    "plt.imshow(wordcloud) \n",
    "plt.axis(\"off\") \n",
    "plt.tight_layout(pad = 0) \n",
    "\n",
    "plt.show() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea625685-417e-41d3-b8a1-a6d346a4cc6a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}