Web_Scraping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import Libraries\n",
    "import requests as rq\n",
    "from bs4 import BeautifulSoup\n",
    "import html5lib\n",
    "import lxml\n",
    "import selenium\n",
    "import time\n",
    "from selenium import webdriver\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font size=4>Use Selenium to scrape data from the web</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Function to get every date from a range\n",
    "def Print_every_day(start,end):\n",
    "    date=[]\n",
    "\n",
    "    month31=[1,3,5,7,8,10,12]\n",
    "    month30=[4,6,9,11]\n",
    "\n",
    "    year=int(start[:4])\n",
    "    month=int(start[4:6])\n",
    "    day=int(start[6:8])\n",
    "    \n",
    "    year_end=int(end[:4])\n",
    "    moth_end=int(end[4:6])\n",
    "    day_end=int(end[6:8])\n",
    "\n",
    "    while (year<year_end) or (year==year_end and month<moth_end) or (year==year_end and month==moth_end and day<day_end):\n",
    "\n",
    "\n",
    "        if year%4==0:#Check if it is a leap year\n",
    "\n",
    "            if month<12:#Check if I'm at the end of the month\n",
    "\n",
    "                if (month in month31): \n",
    "                    if day<31:\n",
    "                        day+=1\n",
    "                    else:\n",
    "                        month+=1\n",
    "                        day=1\n",
    "\n",
    "                elif (month in month30): \n",
    "                    if day<30:\n",
    "                        day+=1\n",
    "                    else:\n",
    "                        month+=1\n",
    "                        day=1\n",
    "\n",
    "                elif day<29: #February\n",
    "                    day+=1\n",
    "                else:\n",
    "                    month+=1\n",
    "                    day=1\n",
    "            else:\n",
    "                if day<31:\n",
    "                    day+=1\n",
    "                else:\n",
    "                    year+=1\n",
    "                    month=1\n",
    "                    day=1\n",
    "        else:\n",
    "            if month<12:\n",
    "\n",
    "                if (month in month31): \n",
    "                    if day<31:\n",
    "                        day+=1\n",
    "                    else:\n",
    "                        month+=1\n",
    "                        day=1\n",
    "\n",
    "                elif (month in month30): \n",
    "                    if day<30:\n",
    "                        day+=1\n",
    "                    else:\n",
    "                        month+=1\n",
    "                        day=1\n",
    "\n",
    "                elif day<28: \n",
    "                    day+=1\n",
    "                else:\n",
    "                    month+=1\n",
    "                    day=1\n",
    "            else:\n",
    "                if day<31:\n",
    "                    day+=1\n",
    "                else:\n",
    "                    year+=1\n",
    "                    month=1\n",
    "                    day=1\n",
    "\n",
    "\n",
    "        if day<10: #Check if I need to add a 0 in front of the number\n",
    "            day=str(day)\n",
    "            day=\"0\"+day\n",
    "        else:\n",
    "            day=str(day)\n",
    "\n",
    "        if month<10: #Check if I need to add a 0 in front of the month\n",
    "            month=str(month)\n",
    "            month=\"0\"+month\n",
    "        else:\n",
    "            month=str(month)\n",
    "        \n",
    "        year=str(year)\n",
    "\n",
    "        date.append(year+month+day) #Add my date to the date list\n",
    "        #Reconvert in int so that i can do operation on them\n",
    "        year=int(year)\n",
    "        month=int(month)\n",
    "        day=int(day)\n",
    "    \n",
    "    return date"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font size=5>Final Code</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "inizio=\"20160719\" #I have to start one day earlier to the one i want to actual start from\n",
    "fine=\"20160720\"\n",
    "\n",
    "sleep_for=1 \n",
    "sleep=1\n",
    "Date=Print_every_day(inizio,fine) \n",
    "\n",
    "#Cycle every page untill i get to the end\n",
    "for j in range(len(Date)):\n",
    "    \n",
    "    url=f\"https://coinmarketcap.com/historical/{Date[j]}/\"\n",
    "    driver=webdriver.Chrome()\n",
    "    driver.get(url)\n",
    "    time.sleep(sleep_for)\n",
    "    scroll_height = driver.execute_script(\"return document.body.scrollHeight;\")\n",
    "    screen_height = driver.execute_script(\"return window.screen.height;\") \n",
    "\n",
    "    \n",
    "    i=1\n",
    "    while True:\n",
    "        \n",
    "        step=screen_height * i\n",
    "        driver.execute_script(f\"window.scrollTo(0, {step});\")\n",
    "        i += 1\n",
    "        time.sleep(sleep)\n",
    "\n",
    "        \n",
    "        if step> scroll_height:\n",
    "            break\n",
    "\n",
    "    #Extract data from the html\n",
    "    html = driver.page_source \n",
    "    driver.quit()\n",
    "    soup=BeautifulSoup(html,\"html.parser\")\n",
    "    righe=soup.find_all(\"tr\",class_=\"cmc-table-row\")#Load all row of the table\n",
    "\n",
    "\n",
    "    Rank=[]\n",
    "    Ticket=[]\n",
    "    Market_Cap=[]\n",
    "    Vol=[]\n",
    "    Price=[]\n",
    "    Supply=[]\n",
    "    Change1h=[]\n",
    "    Change24h=[]\n",
    "    Change7d=[]\n",
    "    Historical_Data_1w=pd.DataFrame()\n",
    "\n",
    "\n",
    "\n",
    "    #Take all data that i need\n",
    "    for x in righe:\n",
    "        num=x.find(\"td\",class_=\"cmc-table__cell cmc-table__cell--sticky cmc-table__cell--sortable cmc-table__cell--left cmc-table__cell--sort-by__rank\")\n",
    "        Symbol=x.find(\"td\",class_=\"cmc-table__cell cmc-table__cell--sortable cmc-table__cell--left cmc-table__cell--sort-by__symbol\")\n",
    "        MC=x.find(\"td\",class_=\"cmc-table__cell cmc-table__cell--sortable cmc-table__cell--right cmc-table__cell--sort-by__market-cap\")\n",
    "        Volume=x.find(\"td\",class_=\"cmc-table__cell cmc-table__cell--sortable cmc-table__cell--right cmc-table__cell--sort-by__volume-24-h\")\n",
    "        PriceData=x.find(\"td\",class_=\"cmc-table__cell cmc-table__cell--sortable cmc-table__cell--right cmc-table__cell--sort-by__price\")\n",
    "        Circulating_Supply=x.find(\"td\",class_=\"cmc-table__cell cmc-table__cell--sortable cmc-table__cell--right cmc-table__cell--sort-by__circulating-supply\")\n",
    "        Change_1h=x.find(\"td\",class_=\"cmc-table__cell cmc-table__cell--sortable cmc-table__cell--right cmc-table__cell--sort-by__percent-change-1-h\")\n",
    "        Change_24h=x.find(\"td\",class_=\"cmc-table__cell cmc-table__cell--sortable cmc-table__cell--right cmc-table__cell--sort-by__percent-change-24-h\")\n",
    "        Change_7d=x.find(\"td\",class_=\"cmc-table__cell cmc-table__cell--sortable cmc-table__cell--right cmc-table__cell--sort-by__percent-change-7-d\")\n",
    "\n",
    "        num=num.text\n",
    "        if \"--\" in num:\n",
    "            num=num.replace(\"--\",\"\")\n",
    "            Rank.append(int(num))\n",
    "        else:\n",
    "            Rank.append(int(num))\n",
    "        \n",
    "        Symbol=Symbol.text\n",
    "        if \"--\" in Symbol:\n",
    "            Symbol=Symbol.replace(\"--\",\"\")\n",
    "            Ticket.append(Symbol)\n",
    "        else:\n",
    "            Ticket.append(Symbol)\n",
    "\n",
    "        MC=MC.text\n",
    "        MC = MC.replace(',', '')\n",
    "        MC = MC.replace('$', '')\n",
    "        if \"--\" in MC:\n",
    "            MC=MC.replace(\"--\",\"\")\n",
    "            Market_Cap.append(MC)\n",
    "        else:\n",
    "            Market_Cap.append(float(MC))\n",
    "        \n",
    "        if Volume!=None:\n",
    "            Volume=Volume.text\n",
    "            Volume = Volume.replace(',', '')\n",
    "            Volume = Volume.replace('$', '')\n",
    "            if \"<\" in Volume:\n",
    "                Volume = Volume.replace('<', '')\n",
    "            if \">\" in Volume:\n",
    "                Volume = Volume.replace('>', '')\n",
    "            if \"--\" in Volume:\n",
    "                Volume=Volume.replace(\"--\",\"\")\n",
    "                Vol.append(Volume)\n",
    "            else:\n",
    "                Vol.append(float(Volume))\n",
    "        else:\n",
    "            Vol.append(None)\n",
    "\n",
    "        PriceData=PriceData.text\n",
    "        PriceData = PriceData.replace(',', '')\n",
    "        PriceData = PriceData.replace('$', '')\n",
    "        if \"--\" in PriceData:\n",
    "            PriceData=PriceData.replace(\"--\",\"\")\n",
    "            Price.append(PriceData)\n",
    "        else:\n",
    "            Price.append(float(PriceData))\n",
    "\n",
    "        Circulating_Supply=Circulating_Supply.text\n",
    "        Circulating_Supply = Circulating_Supply.replace('.', '')\n",
    "        \n",
    "        Circulating_Supply = Circulating_Supply.replace('*', '')\n",
    "        Circulating_Supply = Circulating_Supply.replace(Symbol, '')\n",
    "        if \"--\" in Circulating_Supply:\n",
    "            Circulating_Supply=Circulating_Supply.replace(\"--\",\"\")\n",
    "            Supply.append(Circulating_Supply)\n",
    "        else:\n",
    "            Supply.append(float(Circulating_Supply))\n",
    "\n",
    "\n",
    "        Change_1h=Change_1h.text\n",
    "        Change_1h = Change_1h.replace(',', '.')\n",
    "        Change_1h = Change_1h.replace('%', '')\n",
    "        if \"<\" in Change_1h:\n",
    "            Change_1h = Change_1h.replace('<', '')\n",
    "        if \">\" in Change_1h:\n",
    "            Change_1h = Change_1h.replace('>', '')\n",
    "        if \"--\" in Change_1h:\n",
    "            Change_1h=Change_1h.replace(\"--\",\"\")\n",
    "            Change1h.append(Change_1h)\n",
    "        else:\n",
    "            if len(Change_1h)>=8:#To account for changes 1.046,36% so that i can format them\n",
    "                Change_1h=Change_1h.replace(\".\",\"\")\n",
    "                Change_1h=(float(Change_1h))/100 #Now it would be something like 104636, so i divide by 100 to get the original value\n",
    "                Change1h.append(Change_1h)\n",
    "            else:\n",
    "                Change1h.append(float(Change_1h))\n",
    "\n",
    "        Change_24h=Change_24h.text\n",
    "        Change_24h = Change_24h.replace(',', '.')\n",
    "        Change_24h = Change_24h.replace('%', \"\")\n",
    "        if \"<\" in Change_24h:\n",
    "            Change_24h = Change_24h.replace('<', '')\n",
    "        if \">\" in Change_24h:\n",
    "            Change_24h = Change_24h.replace('>', '')\n",
    "        if \"--\" in Change_24h:\n",
    "            Change_24h = Change_24h.replace('--', \"\")\n",
    "            Change24h.append(Change_24h)\n",
    "        else:\n",
    "            if len(Change_24h)>=8:\n",
    "                Change_24h=Change_24h.replace(\".\",\"\")\n",
    "                Change_24h=(float(Change_24h))/100\n",
    "                Change24h.append(Change_24h)\n",
    "            else:\n",
    "                Change24h.append(float(Change_24h))\n",
    "  \n",
    "        Change_7d=Change_7d.text\n",
    "        Change_7d = Change_7d.replace(',', '.')\n",
    "        Change_7d = Change_7d.replace('%', '')\n",
    "        if \"<\" in Change_7d:\n",
    "            Change_7d = Change_7d.replace('<', \"\")\n",
    "        if \">\" in Change_7d:\n",
    "            Change_7d = Change_7d.replace('>', '')\n",
    "        if \"--\" in Change_7d:\n",
    "            Change_7d = Change_7d.replace('--', \"\")\n",
    "            Change7d.append(Change_7d)\n",
    "        else:\n",
    "            if len(Change_7d)>=8:\n",
    "                Change_7d=Change_7d.replace(\".\",\"\")\n",
    "                Change_7d=(float(Change_7d))/100 \n",
    "                Change7d.append(Change_7d)\n",
    "            else:\n",
    "                Change7d.append(float(Change_7d))\n",
    "            \n",
    "        #Put everything inside a dataframe, and then inside a csv\n",
    "        Historical_Data_1w=pd.DataFrame({\"Rank\":Rank,\"Symbol\":Ticket,\"Market Cap\":Market_Cap,\"Volume\":Vol,\"Price\":Price,  \n",
    "                                 \"Circulating\":Supply,\"1h Variation\":Change1h,\"24h Variation\":Change24h,\"7d Variation\":Change7d})\n",
    "        Historical_Data_1w.to_csv(f\"Hystorical_Data_CoinMarketCap/{Date[j]}.csv\") \n",
    "        \n",
    "        \n",
    "        \n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font size=5>Take the list of the Stable Coin and Asset Backed Crypto, so that i can remove them when i calculate the ALT season index</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Take the ticket of the stable Coin and asset backed tocken\n",
    "urls=[\"https://coinmarketcap.com/view/stablecoin/\",\"https://www.coingecko.com/en/categories/asset-backed-tokens\"]\n",
    "\n",
    "#Cycle every page untill the end date\n",
    "url=\"https://coinmarketcap.com/view/stablecoin/\"\n",
    "Information=rq.get(url)\n",
    "\n",
    "\n",
    "\n",
    "soup=BeautifulSoup(Information.content,\"html.parser\")\n",
    "\n",
    "#Load all row of the table\n",
    "r1=soup.find_all(\"tr\",style=\"cursor:pointer\")\n",
    "r2=soup.find_all(\"tr\",class_=\"sc-240ce903-0 iPeVVh\")\n",
    "\n",
    "Ticket=[]\n",
    "\n",
    "\n",
    "for x in r1:\n",
    "    Symbol=x.find(\"div\",class_=\"sc-1c5f2868-3 hHWqgz\")\n",
    "    \n",
    "    Ticket.append(Symbol.text)\n",
    "\n",
    "for x in r2:\n",
    "    \n",
    "    \n",
    "    Symbol=x.find(\"span\",class_=\"crypto-symbol\") #the class_ might change \n",
    "    Ticket.append(Symbol.text)\n",
    "\n",
    "Stable_Coin=pd.Series(Ticket)\n",
    "Stable_Coin.to_csv(\"Stable_Coin_and_Asset_Backed_Token/Stable Coin List.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Take the Ticket of the asset backed token\n",
    "url=\"https://www.coingecko.com/en/categories/asset-backed-tokens\"\n",
    "\n",
    "driver=webdriver.Chrome()\n",
    "driver.get(url)\n",
    "\n",
    "scroll_height = driver.execute_script(\"return document.body.scrollHeight;\")\n",
    "screen_height = driver.execute_script(\"return window.screen.height;\") \n",
    "\n",
    "\n",
    "i=1\n",
    "while True:\n",
    "    \n",
    "    step=screen_height * i\n",
    "    driver.execute_script(f\"window.scrollTo(0, {step});\")\n",
    "    i += 1\n",
    "    time.sleep(sleep)\n",
    "\n",
    "    \n",
    "    if step> scroll_height:\n",
    "        break\n",
    "\n",
    "    \n",
    "\n",
    "html = driver.page_source \n",
    "driver.quit()\n",
    "\n",
    "\n",
    "soup=BeautifulSoup(html,\"html.parser\")\n",
    "\n",
    "\n",
    "r3=soup.find_all(\"tr\",class_=\"hover:tw-bg-gray-50 tw-bg-white dark:tw-bg-moon-900 hover:dark:tw-bg-moon-800 tw-text-sm\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Ticket=[]\n",
    "\n",
    "#Take every data i need from each row of the table\n",
    "for x in r3:\n",
    "    Symbol=x.find(\"div\",class_=\"tw-block 2lg:tw-inline tw-text-xs tw-leading-4 tw-text-gray-500 dark:tw-text-moon-200 tw-font-medium\")\n",
    "    Symbol=Symbol.text\n",
    "\n",
    "    MarketCap=x.find(\"td\",class_=\"tw-text-end tw-px-1 tw-py-2.5 2lg:tw-p-2.5 tw-bg-inherit tw-text-gray-900 dark:tw-text-moon-50\")\n",
    "    \n",
    "\n",
    "    Final_Symbol=\"\"\n",
    "    for ln in range(len(Symbol)):\n",
    "        \n",
    "        dummy=\"\"\n",
    "\n",
    "        if ln>2 and ln<range(len(Symbol))[-1] and Symbol[ln]!=\"\":\n",
    "            dummy = Symbol[ln]\n",
    "\n",
    "        Final_Symbol+=dummy\n",
    "    \n",
    "    Ticket.append(Final_Symbol)\n",
    "\n",
    "Ticket=Ticket[:32] #Take only the first 32, since the rest dont even have a Market Cap\n",
    "print(Ticket)\n",
    "\n",
    "\n",
    "Assets_Backed_Token=pd.Series(Ticket)\n",
    "Assets_Backed_Token.to_csv(\"Crypto Type List/Asset Backed Token List.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".VirtualEnviromentCrypto",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}