anadolu_agency_news_api.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import  requests \n",
    "import pandas as pd\n",
    "\n",
    "from urllib.error import URLError, HTTPError\n",
    "from urllib3.exceptions import SSLError\n",
    "\n",
    "from requests.exceptions import HTTPError\n",
    "from bs4 import BeautifulSoup\n",
    "from urllib.request import urlopen, Request\n",
    "from socket import timeout\n",
    "\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support.expected_conditions import presence_of_element_located\n",
    "import time\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.aa.com.tr/en/search/?s=akbank\n"
     ]
    }
   ],
   "source": [
    "\n",
    "link = 'https://www.aa.com.tr/en/search/?s='\n",
    "# keyword = 'Turkish Airlines'\n",
    "keyword = 'akbank'\n",
    "\n",
    "keyword2 = '+'.join(keyword.split())\n",
    "keyword3 = '_'.join(keyword.split())\n",
    "\n",
    "link += keyword2\n",
    "print(link)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "chrome_driver_path = 'selenium_p/chromedriver'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def getArticle(link):\n",
    "    text = ''\n",
    "    \n",
    "#     print('-')\n",
    "    try: \n",
    "        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3'}\n",
    "        reg_url = link\n",
    "        req = Request(url=reg_url, headers=headers) \n",
    "        html = urlopen(req).read()  \n",
    "\n",
    "        soup = BeautifulSoup(html, \"html.parser\")\n",
    "\n",
    "        date = soup.find('span',{'class':'tarih'}).text\n",
    "        data = soup.find('div',{'class':'detay-icerik'}).findAll('p')\n",
    "        for p in data:\n",
    "            text +=p.text\n",
    "            \n",
    "        return [date,text]\n",
    "        \n",
    "    except HTTPError as http_err:\n",
    "        print(f'HTTP error occurred: {http_err}')  \n",
    "        pass\n",
    "    except Exception as err:\n",
    "        print(f'Other1 error occurred: {err}')  \n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "21\n",
      "41\n",
      "61\n",
      "81\n",
      "101\n",
      "121\n",
      "141\n",
      "161\n",
      "181\n",
      "201\n",
      "221\n",
      "241\n"
     ]
    }
   ],
   "source": [
    "driver = webdriver.Chrome(executable_path=chrome_driver_path)\n",
    "driver.set_page_load_timeout('10')\n",
    "driver.get(link)\n",
    "\n",
    "result = []\n",
    "\n",
    "news_count = int(driver.find_element_by_id('spnNewsCount').get_attribute('innerHTML'))\n",
    "for i in range(1,news_count,20):\n",
    "    driver.find_element_by_link_text('Continue').send_keys(Keys.ENTER)\n",
    "    time.sleep(2)\n",
    "    print(i)\n",
    "    \n",
    "data = driver.find_element_by_id('myTabContent').find_elements_by_class_name('row')\n",
    "for news in data:\n",
    "    \n",
    "    link = news.find_element_by_tag_name('a').get_attribute('href')\n",
    "    headline = news.find_element_by_tag_name('h4').get_attribute('innerHTML')\n",
    "    description = news.find_element_by_tag_name('p').get_attribute('innerHTML')\n",
    "    date,text = getArticle(link)\n",
    "\n",
    "    result.append([headline,description,text,date,link])\n",
    "\n",
    "\n",
    "time.sleep(4)\n",
    "driver.quit()\n",
    "\n",
    "\n",
    "   \n",
    "df = pd.DataFrame(result, columns =['headline','description','article','date','article_link'])\n",
    "df.to_csv(f'apis_datasets/{keyword3}_AnadoluAgency.csv', encoding='utf-8', index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>headline</th>\n",
       "      <th>description</th>\n",
       "      <th>article</th>\n",
       "      <th>date</th>\n",
       "      <th>article_link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>European bank loans $47M to Turkish leasing co...</td>\n",
       "      <td>New funds to enable company to increase availa...</td>\n",
       "      <td>ANKARAThe European Bank for Reconstruction and...</td>\n",
       "      <td>28.07.2020</td>\n",
       "      <td>https://www.aa.com.tr/en/economy/european-bank...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>European bank’s investments in Turkey near $1B...</td>\n",
       "      <td>European Bank for Reconstruction and Developme...</td>\n",
       "      <td>ISTANBUL The European Bank for Reconstruction ...</td>\n",
       "      <td>22.07.2020</td>\n",
       "      <td>https://www.aa.com.tr/en/economy/european-bank...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Turkey's Akbank realizes first eurobond issuance</td>\n",
       "      <td>Bank says it issued first eurobond in Turkish ...</td>\n",
       "      <td>ANKARATurkey-based private lender Akbank annou...</td>\n",
       "      <td>01.07.2020</td>\n",
       "      <td>https://www.aa.com.tr/en/economy/turkeys-akban...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>European bank loans $75M to Turkey’s Akbank</td>\n",
       "      <td>EBRD says funds will boost Akbank's lending to...</td>\n",
       "      <td>ANKARAThe European Bank for Reconstruction and...</td>\n",
       "      <td>12.06.2020</td>\n",
       "      <td>https://www.aa.com.tr/en/economy/european-bank...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Turkish Airlines remains top brand in Turkey</td>\n",
       "      <td>Flag carrier has nearly $2B brand value in 202...</td>\n",
       "      <td>ISTANBULTurkey's flag carrier Turkish Airlines...</td>\n",
       "      <td>09.06.2020</td>\n",
       "      <td>https://www.aa.com.tr/en/turkey/turkish-airlin...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            headline  \\\n",
       "0  European bank loans $47M to Turkish leasing co...   \n",
       "1  European bank’s investments in Turkey near $1B...   \n",
       "2   Turkey's Akbank realizes first eurobond issuance   \n",
       "3        European bank loans $75M to Turkey’s Akbank   \n",
       "4       Turkish Airlines remains top brand in Turkey   \n",
       "\n",
       "                                         description  \\\n",
       "0  New funds to enable company to increase availa...   \n",
       "1  European Bank for Reconstruction and Developme...   \n",
       "2  Bank says it issued first eurobond in Turkish ...   \n",
       "3  EBRD says funds will boost Akbank's lending to...   \n",
       "4  Flag carrier has nearly $2B brand value in 202...   \n",
       "\n",
       "                                             article        date  \\\n",
       "0  ANKARAThe European Bank for Reconstruction and...  28.07.2020   \n",
       "1  ISTANBUL The European Bank for Reconstruction ...  22.07.2020   \n",
       "2  ANKARATurkey-based private lender Akbank annou...  01.07.2020   \n",
       "3  ANKARAThe European Bank for Reconstruction and...  12.06.2020   \n",
       "4  ISTANBULTurkey's flag carrier Turkish Airlines...  09.06.2020   \n",
       "\n",
       "                                        article_link  \n",
       "0  https://www.aa.com.tr/en/economy/european-bank...  \n",
       "1  https://www.aa.com.tr/en/economy/european-bank...  \n",
       "2  https://www.aa.com.tr/en/economy/turkeys-akban...  \n",
       "3  https://www.aa.com.tr/en/economy/european-bank...  \n",
       "4  https://www.aa.com.tr/en/turkey/turkish-airlin...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}