NLP(Phase1).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Untitled5.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "YO-sWLIiTu4o",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7CgpmdW5jdGlvbiBfdXBsb2FkRmlsZXMoaW5wdXRJZCwgb3V0cHV0SWQpIHsKICBjb25zdCBzdGVwcyA9IHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCk7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICAvLyBDYWNoZSBzdGVwcyBvbiB0aGUgb3V0cHV0RWxlbWVudCB0byBtYWtlIGl0IGF2YWlsYWJsZSBmb3IgdGhlIG5leHQgY2FsbAogIC8vIHRvIHVwbG9hZEZpbGVzQ29udGludWUgZnJvbSBQeXRob24uCiAgb3V0cHV0RWxlbWVudC5zdGVwcyA9IHN0ZXBzOwoKICByZXR1cm4gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpOwp9CgovLyBUaGlzIGlzIHJvdWdobHkgYW4gYXN5bmMgZ2VuZXJhdG9yIChub3Qgc3VwcG9ydGVkIGluIHRoZSBicm93c2VyIHlldCksCi8vIHdoZXJlIHRoZXJlIGFyZSBtdWx0aXBsZSBhc3luY2hyb25vdXMgc3RlcHMgYW5kIHRoZSBQeXRob24gc2lkZSBpcyBnb2luZwovLyB0byBwb2xsIGZvciBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcC4KLy8gVGhpcyB1c2VzIGEgUHJvbWlzZSB0byBibG9jayB0aGUgcHl0aG9uIHNpZGUgb24gY29tcGxldGlvbiBvZiBlYWNoIHN0ZXAsCi8vIHRoZW4gcGFzc2VzIHRoZSByZXN1bHQgb2YgdGhlIHByZXZpb3VzIHN0ZXAgYXMgdGhlIGlucHV0IHRvIHRoZSBuZXh0IHN0ZXAuCmZ1bmN0aW9uIF91cGxvYWRGaWxlc0NvbnRpbnVlKG91dHB1dElkKSB7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICBjb25zdCBzdGVwcyA9IG91dHB1dEVsZW1lbnQuc3RlcHM7CgogIGNvbnN0IG5leHQgPSBzdGVwcy5uZXh0KG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSk7CiAgcmV0dXJuIFByb21pc2UucmVzb2x2ZShuZXh0LnZhbHVlLnByb21pc2UpLnRoZW4oKHZhbHVlKSA9PiB7CiAgICAvLyBDYWNoZSB0aGUgbGFzdCBwcm9taXNlIHZhbHVlIHRvIG1ha2UgaXQgYXZhaWxhYmxlIHRvIHRoZSBuZXh0CiAgICAvLyBzdGVwIG9mIHRoZSBnZW5lcmF0b3IuCiAgICBvdXRwdXRFbGVtZW50Lmxhc3RQcm9taXNlVmFsdWUgPSB2YWx1ZTsKICAgIHJldHVybiBuZXh0LnZhbHVlLnJlc3BvbnNlOwogIH0pOwp9CgovKioKICogR2VuZXJhdG9yIGZ1bmN0aW9uIHdoaWNoIGlzIGNhbGxlZCBiZXR3ZWVuIGVhY2ggYXN5bmMgc3RlcCBvZiB0aGUgdXBsb2FkCiAqIHByb2Nlc3MuCiAqIEBwYXJhbSB7c3RyaW5nfSBpbnB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIGlucHV0IGZpbGUgcGlja2VyIGVsZW1lbnQuCiAqIEBwYXJhbSB7c3RyaW5nfSBvdXRwdXRJZCBFbGVtZW50IElEIG9mIHRoZSBvdXRwdXQgZGlzcGxheS4KICogQHJldHVybiB7IUl0ZXJhYmxlPCFPYmplY3Q+fSBJdGVyYWJsZSBvZiBuZXh0IHN0ZXBzLgogKi8KZnVuY3Rpb24qIHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IGlucHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKGlucHV0SWQpOwogIGlucHV0RWxlbWVudC5kaXNhYmxlZCA9IGZhbHNlOwoKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIG91dHB1dEVsZW1lbnQuaW5uZXJIVE1MID0gJyc7CgogIGNvbnN0IHBpY2tlZFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgaW5wdXRFbGVtZW50LmFkZEV2ZW50TGlzdGVuZXIoJ2NoYW5nZScsIChlKSA9PiB7CiAgICAgIHJlc29sdmUoZS50YXJnZXQuZmlsZXMpOwogICAgfSk7CiAgfSk7CgogIGNvbnN0IGNhbmNlbCA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2J1dHRvbicpOwogIGlucHV0RWxlbWVudC5wYXJlbnRFbGVtZW50LmFwcGVuZENoaWxkKGNhbmNlbCk7CiAgY2FuY2VsLnRleHRDb250ZW50ID0gJ0NhbmNlbCB1cGxvYWQnOwogIGNvbnN0IGNhbmNlbFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgY2FuY2VsLm9uY2xpY2sgPSAoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9OwogIH0pOwoKICAvLyBXYWl0IGZvciB0aGUgdXNlciB0byBwaWNrIHRoZSBmaWxlcy4KICBjb25zdCBmaWxlcyA9IHlpZWxkIHsKICAgIHByb21pc2U6IFByb21pc2UucmFjZShbcGlja2VkUHJvbWlzZSwgY2FuY2VsUHJvbWlzZV0pLAogICAgcmVzcG9uc2U6IHsKICAgICAgYWN0aW9uOiAnc3RhcnRpbmcnLAogICAgfQogIH07CgogIGNhbmNlbC5yZW1vdmUoKTsKCiAgLy8gRGlzYWJsZSB0aGUgaW5wdXQgZWxlbWVudCBzaW5jZSBmdXJ0aGVyIHBpY2tzIGFyZSBub3QgYWxsb3dlZC4KICBpbnB1dEVsZW1lbnQuZGlzYWJsZWQgPSB0cnVlOwoKICBpZiAoIWZpbGVzKSB7CiAgICByZXR1cm4gewogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgICAgfQogICAgfTsKICB9CgogIGZvciAoY29uc3QgZmlsZSBvZiBmaWxlcykgewogICAgY29uc3QgbGkgPSBkb2N1bWVudC5jcmVhdGVFbGVtZW50KCdsaScpOwogICAgbGkuYXBwZW5kKHNwYW4oZmlsZS5uYW1lLCB7Zm9udFdlaWdodDogJ2JvbGQnfSkpOwogICAgbGkuYXBwZW5kKHNwYW4oCiAgICAgICAgYCgke2ZpbGUudHlwZSB8fCAnbi9hJ30pIC0gJHtmaWxlLnNpemV9IGJ5dGVzLCBgICsKICAgICAgICBgbGFzdCBtb2RpZmllZDogJHsKICAgICAgICAgICAgZmlsZS5sYXN0TW9kaWZpZWREYXRlID8gZmlsZS5sYXN0TW9kaWZpZWREYXRlLnRvTG9jYWxlRGF0ZVN0cmluZygpIDoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ24vYSd9IC0gYCkpOwogICAgY29uc3QgcGVyY2VudCA9IHNwYW4oJzAlIGRvbmUnKTsKICAgIGxpLmFwcGVuZENoaWxkKHBlcmNlbnQpOwoKICAgIG91dHB1dEVsZW1lbnQuYXBwZW5kQ2hpbGQobGkpOwoKICAgIGNvbnN0IGZpbGVEYXRhUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICAgIGNvbnN0IHJlYWRlciA9IG5ldyBGaWxlUmVhZGVyKCk7CiAgICAgIHJlYWRlci5vbmxvYWQgPSAoZSkgPT4gewogICAgICAgIHJlc29sdmUoZS50YXJnZXQucmVzdWx0KTsKICAgICAgfTsKICAgICAgcmVhZGVyLnJlYWRBc0FycmF5QnVmZmVyKGZpbGUpOwogICAgfSk7CiAgICAvLyBXYWl0IGZvciB0aGUgZGF0YSB0byBiZSByZWFkeS4KICAgIGxldCBmaWxlRGF0YSA9IHlpZWxkIHsKICAgICAgcHJvbWlzZTogZmlsZURhdGFQcm9taXNlLAogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbnRpbnVlJywKICAgICAgfQogICAgfTsKCiAgICAvLyBVc2UgYSBjaHVua2VkIHNlbmRpbmcgdG8gYXZvaWQgbWVzc2FnZSBzaXplIGxpbWl0cy4gU2VlIGIvNjIxMTU2NjAuCiAgICBsZXQgcG9zaXRpb24gPSAwOwogICAgZG8gewogICAgICBjb25zdCBsZW5ndGggPSBNYXRoLm1pbihmaWxlRGF0YS5ieXRlTGVuZ3RoIC0gcG9zaXRpb24sIE1BWF9QQVlMT0FEX1NJWkUpOwogICAgICBjb25zdCBjaHVuayA9IG5ldyBVaW50OEFycmF5KGZpbGVEYXRhLCBwb3NpdGlvbiwgbGVuZ3RoKTsKICAgICAgcG9zaXRpb24gKz0gbGVuZ3RoOwoKICAgICAgY29uc3QgYmFzZTY0ID0gYnRvYShTdHJpbmcuZnJvbUNoYXJDb2RlLmFwcGx5KG51bGwsIGNodW5rKSk7CiAgICAgIHlpZWxkIHsKICAgICAgICByZXNwb25zZTogewogICAgICAgICAgYWN0aW9uOiAnYXBwZW5kJywKICAgICAgICAgIGZpbGU6IGZpbGUubmFtZSwKICAgICAgICAgIGRhdGE6IGJhc2U2NCwKICAgICAgICB9LAogICAgICB9OwoKICAgICAgbGV0IHBlcmNlbnREb25lID0gZmlsZURhdGEuYnl0ZUxlbmd0aCA9PT0gMCA/CiAgICAgICAgICAxMDAgOgogICAgICAgICAgTWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCk7CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPSBgJHtwZXJjZW50RG9uZX0lIGRvbmVgOwoKICAgIH0gd2hpbGUgKHBvc2l0aW9uIDwgZmlsZURhdGEuYnl0ZUxlbmd0aCk7CiAgfQoKICAvLyBBbGwgZG9uZS4KICB5aWVsZCB7CiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICB9CiAgfTsKfQoKc2NvcGUuZ29vZ2xlID0gc2NvcGUuZ29vZ2xlIHx8IHt9OwpzY29wZS5nb29nbGUuY29sYWIgPSBzY29wZS5nb29nbGUuY29sYWIgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYi5fZmlsZXMgPSB7CiAgX3VwbG9hZEZpbGVzLAogIF91cGxvYWRGaWxlc0NvbnRpbnVlLAp9Owp9KShzZWxmKTsK",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "outputId": "107a795d-bc89-4f11-8f22-ff88e10b5c07"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-cd837ae2-79f2-43b0-84ce-d609952e1c5b\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-cd837ae2-79f2-43b0-84ce-d609952e1c5b\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving ar.dic to ar.dic\n",
            "Saving ar.aff to ar.aff\n"
          ]
        }
      ],
      "source": [
        "from google.colab import files\n",
        "uploaded=files.upload()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "file_name=\"ar.aff\"\n",
        "uploaded[file_name].decode(\"utf-8\")\n",
        "file_name2=\"ar.dic\"\n",
        "uploaded[file_name2].decode(\"utf-8\")"
      ],
      "metadata": {
        "id": "EaGMv-BNsX46"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "import pandas as pd\n",
        "import urllib.request\n",
        "import operator\n",
        "!sudo apt install hunspell-ar\n",
        "!sudo apt install libhunspell-dev\n",
        "!sudo apt install aspell\n",
        "!sudo apt install libaspell-dev\n",
        "!sudo apt install aspell-ar\n",
        "!sudo apt install aspell-ar-large\n",
        "!pip install hunspell\n",
        "!pip install aspell-python-py3\n",
        "import hunspell\n",
        "from google.colab import files\n",
        "import pandas as pd\n",
        "import re\n",
        "import gensim\n",
        "from gensim.utils import simple_preprocess\n",
        "from gensim.parsing.preprocessing import STOPWORDS\n",
        "from nltk.stem import WordNetLemmatizer, SnowballStemmer\n",
        "from nltk.stem.porter import *\n",
        "import numpy as np\n",
        "np.random.seed(400)\n",
        "from nltk.corpus import stopwords\n",
        "import nltk\n",
        "nltk.download(\"stopwords\")\n",
        "!pip install farasapy\n",
        "from farasa.pos import FarasaPOSTagger\n",
        "from farasa.ner import FarasaNamedEntityRecognizer\n",
        "from farasa.diacratizer import FarasaDiacritizer\n",
        "from farasa.segmenter import FarasaSegmenter\n",
        "from farasa.stemmer import FarasaStemmer"
      ],
      "metadata": {
        "id": "gJbym6j0sVC9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Books = []\n",
        "Category= []\n",
        "BookID = 1\n",
        "for PageNum in range(200):\n",
        "  PageNum+=1\n",
        "  url = 'https://www.arab-books.com//page/'\n",
        "  url = url+str(PageNum)\n",
        "  response = requests.get(url)\n",
        "  Data = BeautifulSoup(response.text, 'html.parser')\n",
        "\n",
        "  for row in Data.findAll('div',attrs={'class':'excerpt-book'} ):\n",
        "    Book = {}\n",
        "    Book['ID'] = BookID\n",
        "    BookID+=1\n",
        "    Book['Name']=row.text.replace(\" PDF\",\"\").replace('\\n','')\n",
        "    Book['link']=row.a['href']+\"?view=read\"\n",
        "    url = row.a['href']\n",
        "    print(url)\n",
        "\n",
        "    try:\n",
        "\n",
        "      BookResponse = requests.get(url)\n",
        "      BookData = BeautifulSoup(BookResponse.text, 'html.parser')\n",
        "      Mol5sData=BeautifulSoup(BookResponse.text, 'html.parser')\n",
        "   \n",
        "    except requests.exceptions.RequestException as e:\n",
        "      continue\n",
        "   # Descrip #\n",
        "    Mol5sData=Mol5sData.findAll('div', attrs={'class': 'entry-content entry clearfix'})[0]\n",
        "    count=0\n",
        "    s=\"\"\n",
        "    for P in Mol5sData:\n",
        "      if re.findall('<h[0-9]>تحميل.*</h[0-9]>',str(P)):\n",
        "        continue\n",
        "      elif re.findall('<h[0-9]>',str(P)):\n",
        "        count+=1\n",
        "      elif re.findall('<p>',str(P)) and count!=1:\n",
        "        s+=str(P)\n",
        "      if count==3:\n",
        "        break;\n",
        "    H=\"\"\n",
        "    Descc=(re.findall(\"[ا-ي]+\",s))\n",
        "    for i in range(len(Descc)):\n",
        "        H += \" \" + Descc[i]\n",
        "    Book['Description']=H\n",
        "    #print(url)\n",
        "    BookData=BookData.findAll('div', attrs={'class':'book-info'})[0]\n",
        "    print(BookID)\n",
        "    Auther = BookData.findAll('li')[0].text.split('\\n')\n",
        "    Book['Auther']=Auther[1].replace('كتب ','')\n",
        "\n",
        "    Category= []\n",
        "    for fTest in BookData.findAll('li')[1].text.split('\\n'):\n",
        "      if fTest.find('تحميل كتب'):\n",
        "        continue\n",
        "      Category.append(fTest.split('تحميل كتب ')[1])\n",
        "    Book['Category']=Category\n",
        "    Book['Language']=BookData.findAll('li')[2].text.split(': ')[1]\n",
        "    Book['Num Of Pages']=BookData.findAll('li')[3].text.split(': ')[1].replace(' صفحة','')\n",
        "    Book['Dar Al Nashr']=BookData.findAll('li')[4].text.split(': ')[1]\n",
        "    Book['Book Size']=BookData.findAll('li')[5].text.split(': ')[1]\n",
        "    Book['Book Type']=BookData.findAll('li')[6].text.split(': ')[1]\n",
        "    \n",
        "    Books.append(Book)"
      ],
      "metadata": {
        "id": "XrwumlKAT1G2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Bookspd= pd.DataFrame(Books)\n"
      ],
      "metadata": {
        "id": "PlCy9oZnnhiC"
      },
      "execution_count": 212,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import files\n",
        "Bookspd.to_csv(\"booksfinal2.csv\")\n",
        "files.download(\"booksfinal2.csv\")"
      ],
      "metadata": {
        "id": "T8P_EWxDIA_0",
        "outputId": "810f69e9-357b-42e5-b52a-74b8745cd7bf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        }
      },
      "execution_count": 186,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "download(\"download_3f41c9ab-c95b-4e31-a1cf-18e337011b48\", \"booksfinal2.csv\", 6942266)"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "Bookspd[\"Dar Al Nashr\"].replace({\"-\":\"غير معرف\"}, inplace=True)\n",
        "Bookspd[\"Dar Al Nashr\"].replace({\"\":\"غير معرف\"}, inplace=True)\n",
        "Bookspd=  Bookspd.drop(['ID'], axis = 1)\n",
        "Bookspd=  Bookspd.drop(['Language'], axis = 1)\n",
        "Bookspd=  Bookspd.drop(['Num Of Pages'], axis = 1)\n",
        "Bookspd=  Bookspd.drop(['Book Size'], axis = 1)\n",
        "Bookspd=  Bookspd.drop(['Book Type'], axis = 1)\n"
      ],
      "metadata": {
        "id": "OgrucNFy7jug"
      },
      "execution_count": 213,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "word=[]\n",
        "listofdrops=[]\n",
        "for j in range(len(Bookspd[\"Name\"])):\n",
        "    word=(re.findall(\"^[ \\u0621-\\u064A ]+$\",Bookspd[\"Name\"][j]))\n",
        "    if len(word) == 0 :\n",
        "      Bookspd.drop(j, inplace = True)\n",
        "      listofdrops.append(j)"
      ],
      "metadata": {
        "id": "TF9Ak7_nTV2m"
      },
      "execution_count": 214,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "Name=[]\n",
        "matched_word=\"\"\n",
        "for j in Bookspd['Name']:\n",
        " matched_words=re.findall(\"الا[\\u0621-\\u064A]+\",j)\n",
        " if(len(matched_words) > 0 ):\n",
        "   for j in range(len(matched_words)):\n",
        "    Name.append(matched_words[j])\n",
        "Name.remove('الاحوال')"
      ],
      "metadata": {
        "id": "KonqVVPei9Al"
      },
      "execution_count": 215,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "corrected_words1={}\n",
        "if __name__ == '__main__':\n",
        "    hobj = hunspell.HunSpell(file_name2,file_name)\n",
        "    for word in Name:\n",
        "        if not hobj.spell(word):\n",
        "          if(len(hobj.suggest(word))>0):\n",
        "           corrected_words1[word]=hobj.suggest(word)[0]\n",
        "          else:\n",
        "             corrected_words1[word]=word\n",
        "        else:\n",
        "          corrected_words1[word]=word\n",
        "\n",
        "    for word in corrected_words1:\n",
        "      Bookspd['Name']=Bookspd['Name'].str.replace(word,corrected_words1[word])\n"
      ],
      "metadata": {
        "id": "WcVNCRK3i9DR"
      },
      "execution_count": 216,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Cat=[]\n",
        "Cat_list=[]\n",
        "Catt=[]\n",
        "matched_w=\"\"\n",
        "word=\"\"\n",
        "for j in range(len(Bookspd['Category'])):\n",
        "  if j not in listofdrops:\n",
        "    for i in range(len(Bookspd['Category'][j])):\n",
        "        word=(Bookspd['Category'][j][i])\n",
        "        matched_ws=re.findall(\"ا[\\u0621-\\u064A^\\S]+\",word)\n",
        "        if(len(matched_ws)>0):\n",
        "          for k in matched_ws: \n",
        "            if(len(k) >4):     \n",
        "              Cat.append(k)\n",
        "for j in range(len(Cat)):\n",
        "    Cat_list.append(Cat[j])"
      ],
      "metadata": {
        "id": "AnjLkTDdi9Fg"
      },
      "execution_count": 217,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Catt=list(set(Cat_list))\n",
        "corrected_words={}\n",
        "if __name__ == '__main__':\n",
        "    hobj = hunspell.HunSpell(file_name2,file_name)\n",
        "    errors_count = dict()\n",
        "    for word in Catt:\n",
        "      if not hobj.spell(word):\n",
        "         if(len(hobj.suggest(word))>0):\n",
        "             corrected_words[word]=hobj.suggest(word)[0]\n",
        "         else:\n",
        "               corrected_words[word]=word\n",
        "      else:\n",
        "          corrected_words[word]=word\n",
        "    for word in corrected_words:\n",
        "      for j in range(len(Bookspd['Category'])):\n",
        "        if j not in listofdrops:\n",
        "          for i in range(len(Bookspd['Category'][j])):\n",
        "           Bookspd['Category'][j][i]=Bookspd['Category'][j][i].replace(word,corrected_words[word])\n",
        "\n",
        "for j in range(len(Bookspd['Category'])):\n",
        "  if j not in listofdrops:\n",
        "        for i in range(len(Bookspd['Category'][j])): \n",
        "            Bookspd['Category'][j][i]=Bookspd['Category'][j][i].replace(\"أسلامية\",\"إسلامية\")\n"
      ],
      "metadata": {
        "id": "5TpBBNWOpG0e"
      },
      "execution_count": 218,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for j in range(len(Bookspd[\"Category\"])):\n",
        "  if j not in listofdrops:\n",
        "    if len(Bookspd[\"Category\"][j]) == 0 or len(Bookspd[\"Description\"][j]) == 0:\n",
        "        Bookspd.drop(j, inplace = True)"
      ],
      "metadata": {
        "id": "8llqgDptNu84"
      },
      "execution_count": 219,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import files\n",
        "Bookspd.to_csv(\"booksfinal2.csv\")\n",
        "files.download(\"booksfinal2.csv\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "BFnEOPNjpHd7",
        "outputId": "24a5153d-44b8-4339-fb7b-f6a0ee8cdb28"
      },
      "execution_count": 220,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "download(\"download_108014f8-cbb2-4867-a1a7-2f5ba807248d\", \"booksfinal2.csv\", 3701423)"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "upload = files.upload()"
      ],
      "metadata": {
        "id": "5BDFLML-aDzk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data = pd.read_csv('booksfinal2AfterCleaning.csv', error_bad_lines=False)"
      ],
      "metadata": {
        "id": "CsE2YnOgO89t"
      },
      "execution_count": 223,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data.head()"
      ],
      "metadata": {
        "id": "SNr3vl5RPEHu",
        "outputId": "4df381c7-3597-460f-ab22-5c755b09df75",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 310
        }
      },
      "execution_count": 224,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>Name</th>\n",
              "      <th>link</th>\n",
              "      <th>Description</th>\n",
              "      <th>Auther</th>\n",
              "      <th>Category</th>\n",
              "      <th>Dar Al Nashr</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>4611</td>\n",
              "      <td>رواية السعادة ليس لها تاريخ</td>\n",
              "      <td>https://www.arab-books.com/books/%d8%b1%d9%88%...</td>\n",
              "      <td>احسان محمد عبد القدوس واحد من اهم كتاب والروا...</td>\n",
              "      <td>احسان عبدالقدوس</td>\n",
              "      <td>['روايات عربية']</td>\n",
              "      <td>اخبار اليوم</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>4633</td>\n",
              "      <td>رواية بئر الحرمان</td>\n",
              "      <td>https://www.arab-books.com/books/%d8%b1%d9%88%...</td>\n",
              "      <td>احسان محمد عبد القدوس واحد من اهم كتاب والروا...</td>\n",
              "      <td>احسان عبدالقدوس</td>\n",
              "      <td>['روايات عربية']</td>\n",
              "      <td>مكتبة إحسان عبدالقدوس</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>5840</td>\n",
              "      <td>كتاب صور من حياة التابعين</td>\n",
              "      <td>https://www.arab-books.com/books/%d9%83%d8%aa%...</td>\n",
              "      <td>استمع و قر كتاب الرجل النبيل محمد رسول الله</td>\n",
              "      <td>الكاتب عبد الرحمن رأفت الباشا</td>\n",
              "      <td>['اسلامية']</td>\n",
              "      <td>غير معرف</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>749</td>\n",
              "      <td>كتاب اشكالية فصل الدين عن السياسة</td>\n",
              "      <td>https://www.arab-books.com/books/%d9%83%d8%aa%...</td>\n",
              "      <td>اشكالية فصل الدين عن السياسة يفان سترينسكي تر...</td>\n",
              "      <td>إيفان سترينسكي</td>\n",
              "      <td>['السياسة']</td>\n",
              "      <td>مؤسسة هنداوي</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5820</td>\n",
              "      <td>كتاب قصة التتار من البداية الى عين جالوت</td>\n",
              "      <td>https://www.arab-books.com/books/%d9%83%d8%aa%...</td>\n",
              "      <td>اعتمد الكاتب البارع راغب السرجاني في كتاب قصة...</td>\n",
              "      <td>الدكتور راغب السرجاني</td>\n",
              "      <td>['التاريخ الإسلامي']</td>\n",
              "      <td>مؤسسة اقرا للنشر والتوزيع والترجمة</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0  ...                        Dar Al Nashr\n",
              "0        4611  ...                         اخبار اليوم\n",
              "1        4633  ...               مكتبة إحسان عبدالقدوس\n",
              "2        5840  ...                            غير معرف\n",
              "3         749  ...                        مؤسسة هنداوي\n",
              "4        5820  ...  مؤسسة اقرا للنشر والتوزيع والترجمة\n",
              "\n",
              "[5 rows x 7 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 224
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data.drop('Unnamed: 0',axis=1,inplace=True)\n",
        "#data.drop('Unnamed: 0.1',axis=1,inplace=True)\n",
        "#data.drop('ID',axis=1,inplace=True)"
      ],
      "metadata": {
        "id": "CWXl0f9kPFw7"
      },
      "execution_count": 225,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data.head(100)"
      ],
      "metadata": {
        "id": "MgcAlFMFPJdE",
        "outputId": "3e5723ad-81e9-4770-c438-14993d912ce2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 597
        }
      },
      "execution_count": 226,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Name</th>\n",
              "      <th>link</th>\n",
              "      <th>Description</th>\n",
              "      <th>Auther</th>\n",
              "      <th>Category</th>\n",
              "      <th>Dar Al Nashr</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>رواية السعادة ليس لها تاريخ</td>\n",
              "      <td>https://www.arab-books.com/books/%d8%b1%d9%88%...</td>\n",
              "      <td>احسان محمد عبد القدوس واحد من اهم كتاب والروا...</td>\n",
              "      <td>احسان عبدالقدوس</td>\n",
              "      <td>['روايات عربية']</td>\n",
              "      <td>اخبار اليوم</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>رواية بئر الحرمان</td>\n",
              "      <td>https://www.arab-books.com/books/%d8%b1%d9%88%...</td>\n",
              "      <td>احسان محمد عبد القدوس واحد من اهم كتاب والروا...</td>\n",
              "      <td>احسان عبدالقدوس</td>\n",
              "      <td>['روايات عربية']</td>\n",
              "      <td>مكتبة إحسان عبدالقدوس</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>كتاب صور من حياة التابعين</td>\n",
              "      <td>https://www.arab-books.com/books/%d9%83%d8%aa%...</td>\n",
              "      <td>استمع و قر كتاب الرجل النبيل محمد رسول الله</td>\n",
              "      <td>الكاتب عبد الرحمن رأفت الباشا</td>\n",
              "      <td>['اسلامية']</td>\n",
              "      <td>غير معرف</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>كتاب اشكالية فصل الدين عن السياسة</td>\n",
              "      <td>https://www.arab-books.com/books/%d9%83%d8%aa%...</td>\n",
              "      <td>اشكالية فصل الدين عن السياسة يفان سترينسكي تر...</td>\n",
              "      <td>إيفان سترينسكي</td>\n",
              "      <td>['السياسة']</td>\n",
              "      <td>مؤسسة هنداوي</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>كتاب قصة التتار من البداية الى عين جالوت</td>\n",
              "      <td>https://www.arab-books.com/books/%d9%83%d8%aa%...</td>\n",
              "      <td>اعتمد الكاتب البارع راغب السرجاني في كتاب قصة...</td>\n",
              "      <td>الدكتور راغب السرجاني</td>\n",
              "      <td>['التاريخ الإسلامي']</td>\n",
              "      <td>مؤسسة اقرا للنشر والتوزيع والترجمة</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>95</th>\n",
              "      <td>كتاب الموظف غير المرئي</td>\n",
              "      <td>https://www.arab-books.com/books/%d9%83%d8%aa%...</td>\n",
              "      <td>الموظف غير المر ي استخدام الدافع لاكتشاف مكان...</td>\n",
              "      <td>أدريان جوستيك</td>\n",
              "      <td>['إدارة الأعمال']</td>\n",
              "      <td>مؤسسة هنداوي</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>96</th>\n",
              "      <td>كتاب الواسطة في احوال مالطة</td>\n",
              "      <td>https://www.arab-books.com/books/%d9%83%d8%aa%...</td>\n",
              "      <td>الواسطة في احوال مالطة احمد فارس الشدياق تقع ...</td>\n",
              "      <td>أحمد الشدياق</td>\n",
              "      <td>['أدب الرحلات']</td>\n",
              "      <td>مؤسسة هنداوي</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>97</th>\n",
              "      <td>رواية انتقام للكاتب روبرت بار</td>\n",
              "      <td>https://www.arab-books.com/books/%d8%b1%d9%88%...</td>\n",
              "      <td>انتقام روبرت بار ترجمه نبيل العادلي وراجع مصط...</td>\n",
              "      <td>الكاتب روبرت بار</td>\n",
              "      <td>['الروايات العالمية المترجمة']</td>\n",
              "      <td>مؤسسة هنداوي</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>98</th>\n",
              "      <td>كتاب تاريخ آل رومانوف</td>\n",
              "      <td>https://www.arab-books.com/books/%d9%83%d8%aa%...</td>\n",
              "      <td>انتهت سلالة عا لة رومانوف رومانوف ذات الثلاثم...</td>\n",
              "      <td>سليم قبعين</td>\n",
              "      <td>['التاريخ الحديث']</td>\n",
              "      <td>هندواي</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>99</th>\n",
              "      <td>كتاب ادعية ليلة القدر</td>\n",
              "      <td>https://www.arab-books.com/books/%d9%83%d8%aa%...</td>\n",
              "      <td>اولا ما هي ليلة القدر نها ليلة خاصة تتكرر كل ...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>['إسلامية']</td>\n",
              "      <td>المكتبة العربية</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>100 rows × 6 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                        Name  ...                        Dar Al Nashr\n",
              "0                رواية السعادة ليس لها تاريخ  ...                         اخبار اليوم\n",
              "1                          رواية بئر الحرمان  ...               مكتبة إحسان عبدالقدوس\n",
              "2                  كتاب صور من حياة التابعين  ...                            غير معرف\n",
              "3          كتاب اشكالية فصل الدين عن السياسة  ...                        مؤسسة هنداوي\n",
              "4   كتاب قصة التتار من البداية الى عين جالوت  ...  مؤسسة اقرا للنشر والتوزيع والترجمة\n",
              "..                                       ...  ...                                 ...\n",
              "95                    كتاب الموظف غير المرئي  ...                        مؤسسة هنداوي\n",
              "96               كتاب الواسطة في احوال مالطة  ...                        مؤسسة هنداوي\n",
              "97             رواية انتقام للكاتب روبرت بار  ...                        مؤسسة هنداوي\n",
              "98                     كتاب تاريخ آل رومانوف  ...                              هندواي\n",
              "99                     كتاب ادعية ليلة القدر  ...                     المكتبة العربية\n",
              "\n",
              "[100 rows x 6 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 226
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data.head()"
      ],
      "metadata": {
        "id": "AHmoug7qPR_e"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data_text = data[['Description']];\n",
        "data_text['index'] = data_text.index\n",
        "documents = data_text\n",
        "data_text.head()"
      ],
      "metadata": {
        "id": "efM_nCF-PeP1",
        "outputId": "5abfe045-6d0d-47fc-8200-d3eec538d1fe",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 330
        }
      },
      "execution_count": 227,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  \n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Description</th>\n",
              "      <th>index</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>احسان محمد عبد القدوس واحد من اهم كتاب والروا...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>احسان محمد عبد القدوس واحد من اهم كتاب والروا...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>استمع و قر كتاب الرجل النبيل محمد رسول الله</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>اشكالية فصل الدين عن السياسة يفان سترينسكي تر...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>اعتمد الكاتب البارع راغب السرجاني في كتاب قصة...</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                         Description  index\n",
              "0   احسان محمد عبد القدوس واحد من اهم كتاب والروا...      0\n",
              "1   احسان محمد عبد القدوس واحد من اهم كتاب والروا...      1\n",
              "2        استمع و قر كتاب الرجل النبيل محمد رسول الله      2\n",
              "3   اشكالية فصل الدين عن السياسة يفان سترينسكي تر...      3\n",
              "4   اعتمد الكاتب البارع راغب السرجاني في كتاب قصة...      4"
            ]
          },
          "metadata": {},
          "execution_count": 227
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "stopwords_list = stopwords.words(\"arabic\")"
      ],
      "metadata": {
        "id": "ggtxz8G2Pesi"
      },
      "execution_count": 228,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "stemmer = FarasaStemmer()\n",
        "sample = \"\"\"قاتل المسلمون الكفار في غزوة احد وانتصر الكفار لعصيان المسلمون لاوامر الرسول صلى الله عليه وسلم\"\"\"\n",
        "stemmed_text = stemmer.stem(sample) \n",
        "print(stemmed_text)"
      ],
      "metadata": {
        "id": "tiD4AI4GPgrr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def lemmatize_stemming(text):\n",
        "    return stemmer.stem(text)\n",
        "\n",
        "# Tokenize and lemmatize\n",
        "def preprocess(text):\n",
        "    result=[]\n",
        "    for token in gensim.utils.simple_preprocess(text) :\n",
        "        if token not in stopwords_list and len(token) > 2:\n",
        "            # TODO: Apply lemmatize_stemming() on the token, then add to the results list\n",
        "            result.append(lemmatize_stemming(token))\n",
        "            \n",
        "    return result"
      ],
      "metadata": {
        "id": "isC1AWoGPkVI"
      },
      "execution_count": 230,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "document_num = 2791\n",
        "doc_sample = documents[documents['index'] == document_num].values[0][0]\n",
        "\n",
        "doc_sample\n",
        "print(\"Original document: \")\n",
        "words = []\n",
        "for word in doc_sample.split(' '):\n",
        "    words.append(word)\n",
        "print(words)\n",
        "print(\"\\n\\nTokenized and lemmatized document: \")\n",
        "print(preprocess(doc_sample))"
      ],
      "metadata": {
        "id": "3gn_TMdiPlGb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "processed_docs = documents['Description'].map(lambda x: preprocess(x))\n",
        "processed_docs[:10]"
      ],
      "metadata": {
        "id": "_jTh4mzbPoNf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dictionary = gensim.corpora.Dictionary(processed_docs)\n",
        "len(dictionary)\n",
        "dictionary.filter_extremes(no_below=20, no_above=0.3, keep_n=100000)\n",
        "len(dictionary)"
      ],
      "metadata": {
        "id": "MzyNvj6pPprg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "bow_corpus = [dictionary.doc2bow(doc) for doc in processed_docs]\n",
        "bow_corpus[document_num]"
      ],
      "metadata": {
        "id": "e5e6vMEcPrLc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "bow_doc_4310 = bow_corpus[document_num]\n",
        "\n",
        "for i in range(len(bow_doc_4310)):\n",
        "    print(\"Word {} (\\\"{}\\\") appears {} time.\".format(bow_doc_4310[i][0], \n",
        "                                                     dictionary[bow_doc_4310[i][0]], \n",
        "                                                     bow_doc_4310[i][1]))"
      ],
      "metadata": {
        "id": "J4lt0ue-Pt9e"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from gensim import corpora, models\n",
        "\n",
        "# TODO\n",
        "tfidf = models.TfidfModel(bow_corpus)\n",
        "corpus_tfidf = tfidf[bow_corpus]\n",
        "from pprint import pprint\n",
        "for doc in corpus_tfidf:\n",
        "    pprint(doc)\n",
        "    break"
      ],
      "metadata": {
        "id": "CDRKODZYPwXE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "lda_model = gensim.models.LdaMulticore(bow_corpus, num_topics = 10, id2word = dictionary, passes = 50)\n",
        "for idx, topic in lda_model.print_topics(-1):\n",
        "    print(\"Topic: {} \\nWords: {}\".format(idx, topic))\n",
        "    print(\"\\n\")"
      ],
      "metadata": {
        "id": "XdYfKRahPx3K"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "lda_model_tfidf = gensim.models.LdaMulticore(corpus_tfidf, num_topics = 10, id2word = dictionary, passes = 50)\n",
        "for idx, topic in lda_model_tfidf.print_topics(-1):\n",
        "    print(\"Topic: {} \\nWord: {}\".format(idx, topic))\n",
        "    print(\"\\n\")"
      ],
      "metadata": {
        "id": "X2Ya2umtPynw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "processed_docs[2791]\n",
        "document_num = 2791\n",
        "for index, score in sorted(lda_model[bow_corpus[document_num]], key=lambda tup: -1*tup[1]):\n",
        "    print(\"\\nScore: {}\\t \\nTopic: {}\".format(score, lda_model.print_topic(index, 10)))"
      ],
      "metadata": {
        "id": "3vKkt_gOP1sS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for index, score in sorted(lda_model_tfidf[bow_corpus[document_num]], key=lambda tup: -1*tup[1]):\n",
        "    print(\"\\nScore: {}\\t \\nTopic: {}\".format(score, lda_model_tfidf.print_topic(index, 10)))"
      ],
      "metadata": {
        "id": "ZUzsriRoP3RJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "unseen_document = \"العلم والايمان سلاحا المتعلم\"\n",
        "\n",
        "# Data preprocessing step for the unseen document\n",
        "bow_vector = dictionary.doc2bow(preprocess(unseen_document))\n",
        "threshold = 0.1\n",
        "for index, score in sorted(lda_model[bow_vector], key=lambda tup: -1*tup[1]):\n",
        "    if score < threshold: break\n",
        "    print(\"Score: {}\\n Topic: {}\".format(score, lda_model.print_topic(index, 5)))\n",
        "    print()"
      ],
      "metadata": {
        "id": "aJj54mdYP5bj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        ""
      ],
      "metadata": {
        "id": "rOSp_PdkZRdV"
      }
    }
  ]
}