scrape_spiegel.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bf77d639-d2db-4f9e-b67f-b580c759bf7d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\transformers\\utils\\generic.py:441: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n",
      "  _torch_pytree._register_pytree_node(\n",
      "c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\transformers\\utils\\generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n",
      "  _torch_pytree._register_pytree_node(\n",
      "c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\transformers\\utils\\generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n",
      "  _torch_pytree._register_pytree_node(\n"
     ]
    }
   ],
   "source": [
    " from database_handling.DataDownload import DataDownloader\n",
    "# from database_handling.DataHandleAndOtherHelpers import DataHandler\n",
    " from database_handling.DataUpload import DataUploader\n",
    "# from database_handling.DataDelete import DataDeleter\n",
    "from database_handling.KeycloakLogin import KeycloakLogin\n",
    "from scrapers.SpiegelScraper import SpiegelScraper\n",
    "from text_analysis.NEExtractor import NEExtractor\n",
    "from text_analysis.Summarizer import Summarizer\n",
    "from text_analysis.TopicExtractor import TopicExtractor\n",
    "from text_analysis.Vectorizers import Vectorizer\n",
    "\n",
    "import transformers\n",
    "import json\n",
    "import gc\n",
    "import torch\n",
    "import logging\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "11d07d6a-6fb0-46b2-9d50-1b1ed4ea78c2",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/bin/bash: docker: command not found\n"
     ]
    }
   ],
   "source": [
    "!docker --version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1666dcda-f316-4e79-b89f-ec587ab21d19",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Suppress all logging below CRITICAL level\n",
    "logging.basicConfig(level=logging.DEBUG)\n",
    "\n",
    "scraper = SpiegelScraper(headless=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c6c50d65-7f92-476d-9c9d-e865cd3dcbc9",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "--- Logging error ---\n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\logging\\__init__.py\", line 1083, in emit\n",
      "    msg = self.format(record)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\logging\\__init__.py\", line 927, in format\n",
      "    return fmt.format(record)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\logging\\__init__.py\", line 663, in format\n",
      "    record.message = record.getMessage()\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\logging\\__init__.py\", line 367, in getMessage\n",
      "    msg = msg % self.args\n",
      "TypeError: not all arguments converted during string formatting\n",
      "Call stack:\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\runpy.py\", line 197, in _run_module_as_main\n",
      "    return _run_code(code, main_globals, None,\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\runpy.py\", line 87, in _run_code\n",
      "    exec(code, run_globals)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel_launcher.py\", line 18, in <module>\n",
      "    app.launch_new_instance()\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\traitlets\\config\\application.py\", line 992, in launch_instance\n",
      "    app.start()\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 739, in start\n",
      "    self.io_loop.start()\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\tornado\\platform\\asyncio.py\", line 195, in start\n",
      "    self.asyncio_loop.run_forever()\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\asyncio\\base_events.py\", line 601, in run_forever\n",
      "    self._run_once()\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\asyncio\\base_events.py\", line 1905, in _run_once\n",
      "    handle._run()\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\asyncio\\events.py\", line 80, in _run\n",
      "    self._context.run(self._callback, *self._args)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 545, in dispatch_queue\n",
      "    await self.process_one()\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 534, in process_one\n",
      "    await dispatch(*args)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 437, in dispatch_shell\n",
      "    await result\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 362, in execute_request\n",
      "    await super().execute_request(stream, ident, parent)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 778, in execute_request\n",
      "    reply_content = await reply_content\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 449, in do_execute\n",
      "    res = shell.run_cell(\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 549, in run_cell\n",
      "    return super().run_cell(*args, **kwargs)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3024, in run_cell\n",
      "    result = self._run_cell(\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3079, in _run_cell\n",
      "    result = runner(coro)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\IPython\\core\\async_helpers.py\", line 129, in _pseudo_sync_runner\n",
      "    coro.send(None)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3284, in run_cell_async\n",
      "    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3466, in run_ast_nodes\n",
      "    if await self.run_code(code, result, async_=asy):\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3526, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"C:\\Users\\ra76lax\\AppData\\Local\\Temp\\ipykernel_14100\\3361478615.py\", line 2, in <module>\n",
      "    scraper.start_browser()\n",
      "  File \"c:\\Users\\ra76lax\\Documents\\GitHub\\m3-scraper\\scrapers\\BaseScraper.py\", line 66, in start_browser\n",
      "    logger.debug(f\"Geckodriver path:\", geckodriver_path)\n",
      "Message: 'Geckodriver path:'\n",
      "Arguments: (WindowsPath('geckodriver-folder/geckodriver'),)\n",
      "--- Logging error ---\n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\logging\\__init__.py\", line 1083, in emit\n",
      "    msg = self.format(record)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\logging\\__init__.py\", line 927, in format\n",
      "    return fmt.format(record)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\logging\\__init__.py\", line 663, in format\n",
      "    record.message = record.getMessage()\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\logging\\__init__.py\", line 367, in getMessage\n",
      "    msg = msg % self.args\n",
      "TypeError: not all arguments converted during string formatting\n",
      "Call stack:\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\runpy.py\", line 197, in _run_module_as_main\n",
      "    return _run_code(code, main_globals, None,\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\runpy.py\", line 87, in _run_code\n",
      "    exec(code, run_globals)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel_launcher.py\", line 18, in <module>\n",
      "    app.launch_new_instance()\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\traitlets\\config\\application.py\", line 992, in launch_instance\n",
      "    app.start()\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 739, in start\n",
      "    self.io_loop.start()\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\tornado\\platform\\asyncio.py\", line 195, in start\n",
      "    self.asyncio_loop.run_forever()\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\asyncio\\base_events.py\", line 601, in run_forever\n",
      "    self._run_once()\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\asyncio\\base_events.py\", line 1905, in _run_once\n",
      "    handle._run()\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\asyncio\\events.py\", line 80, in _run\n",
      "    self._context.run(self._callback, *self._args)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 545, in dispatch_queue\n",
      "    await self.process_one()\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 534, in process_one\n",
      "    await dispatch(*args)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 437, in dispatch_shell\n",
      "    await result\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 362, in execute_request\n",
      "    await super().execute_request(stream, ident, parent)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 778, in execute_request\n",
      "    reply_content = await reply_content\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 449, in do_execute\n",
      "    res = shell.run_cell(\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 549, in run_cell\n",
      "    return super().run_cell(*args, **kwargs)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3024, in run_cell\n",
      "    result = self._run_cell(\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3079, in _run_cell\n",
      "    result = runner(coro)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\IPython\\core\\async_helpers.py\", line 129, in _pseudo_sync_runner\n",
      "    coro.send(None)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3284, in run_cell_async\n",
      "    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3466, in run_ast_nodes\n",
      "    if await self.run_code(code, result, async_=asy):\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3526, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"C:\\Users\\ra76lax\\AppData\\Local\\Temp\\ipykernel_14100\\3361478615.py\", line 2, in <module>\n",
      "    scraper.start_browser()\n",
      "  File \"c:\\Users\\ra76lax\\Documents\\GitHub\\m3-scraper\\scrapers\\BaseScraper.py\", line 67, in start_browser\n",
      "    logger.debug(f\"Geckodriver path:\", firefox_binary_path)\n",
      "Message: 'Geckodriver path:'\n",
      "Arguments: (WindowsPath('firefox'),)\n",
      "--- Logging error ---\n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\logging\\__init__.py\", line 1083, in emit\n",
      "    msg = self.format(record)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\logging\\__init__.py\", line 927, in format\n",
      "    return fmt.format(record)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\logging\\__init__.py\", line 663, in format\n",
      "    record.message = record.getMessage()\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\logging\\__init__.py\", line 367, in getMessage\n",
      "    msg = msg % self.args\n",
      "TypeError: not all arguments converted during string formatting\n",
      "Call stack:\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\runpy.py\", line 197, in _run_module_as_main\n",
      "    return _run_code(code, main_globals, None,\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\runpy.py\", line 87, in _run_code\n",
      "    exec(code, run_globals)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel_launcher.py\", line 18, in <module>\n",
      "    app.launch_new_instance()\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\traitlets\\config\\application.py\", line 992, in launch_instance\n",
      "    app.start()\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 739, in start\n",
      "    self.io_loop.start()\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\tornado\\platform\\asyncio.py\", line 195, in start\n",
      "    self.asyncio_loop.run_forever()\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\asyncio\\base_events.py\", line 601, in run_forever\n",
      "    self._run_once()\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\asyncio\\base_events.py\", line 1905, in _run_once\n",
      "    handle._run()\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\asyncio\\events.py\", line 80, in _run\n",
      "    self._context.run(self._callback, *self._args)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 545, in dispatch_queue\n",
      "    await self.process_one()\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 534, in process_one\n",
      "    await dispatch(*args)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 437, in dispatch_shell\n",
      "    await result\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 362, in execute_request\n",
      "    await super().execute_request(stream, ident, parent)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 778, in execute_request\n",
      "    reply_content = await reply_content\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 449, in do_execute\n",
      "    res = shell.run_cell(\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 549, in run_cell\n",
      "    return super().run_cell(*args, **kwargs)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3024, in run_cell\n",
      "    result = self._run_cell(\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3079, in _run_cell\n",
      "    result = runner(coro)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\IPython\\core\\async_helpers.py\", line 129, in _pseudo_sync_runner\n",
      "    coro.send(None)\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3284, in run_cell_async\n",
      "    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3466, in run_ast_nodes\n",
      "    if await self.run_code(code, result, async_=asy):\n",
      "  File \"c:\\Users\\ra76lax\\AppData\\Local\\anaconda3\\envs\\python_m3_scrapers\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3526, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"C:\\Users\\ra76lax\\AppData\\Local\\Temp\\ipykernel_14100\\3361478615.py\", line 2, in <module>\n",
      "    scraper.start_browser()\n",
      "  File \"c:\\Users\\ra76lax\\Documents\\GitHub\\m3-scraper\\scrapers\\BaseScraper.py\", line 68, in start_browser\n",
      "    logger.debug(f\"Current working directory:\", Path.cwd())\n",
      "Message: 'Current working directory:'\n",
      "Arguments: (WindowsPath('c:/Users/ra76lax/Documents/GitHub/m3-scraper'),)\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "__init__() got an unexpected keyword argument 'service'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[3], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m scraper \u001b[38;5;241m=\u001b[39m SpiegelScraper(headless\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m----> 2\u001b[0m \u001b[43mscraper\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstart_browser\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m      3\u001b[0m scraper\u001b[38;5;241m.\u001b[39mlogin()\n",
      "File \u001b[1;32mc:\\Users\\ra76lax\\Documents\\GitHub\\m3-scraper\\scrapers\\BaseScraper.py:80\u001b[0m, in \u001b[0;36mBaseScraper.start_browser\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     78\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m     79\u001b[0m     service \u001b[38;5;241m=\u001b[39m Service(geckodriver_path)\n\u001b[1;32m---> 80\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdriver \u001b[38;5;241m=\u001b[39m \u001b[43mwebdriver\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mFirefox\u001b[49m\u001b[43m(\u001b[49m\u001b[43mservice\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mservice\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfirefox_options\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     81\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mwait \u001b[38;5;241m=\u001b[39m WebDriverWait(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdriver, \u001b[38;5;241m3\u001b[39m)\n\u001b[0;32m     82\u001b[0m     logger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mBrowser started successfully\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "\u001b[1;31mTypeError\u001b[0m: __init__() got an unexpected keyword argument 'service'"
     ]
    }
   ],
   "source": [
    "scraper = SpiegelScraper(headless=True)\n",
    "scraper.start_browser()\n",
    "scraper.login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "63e6c7c0-38fd-41c5-9192-183010e8a596",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#scraper.driver.save_screenshot(\"screenshot.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c12fd4e7-8825-45f7-8142-1466396dea64",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/\n",
      "INFO:scrapers.BaseScraper:Found 170 unique article URLs on the current page: https://www.spiegel.de/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/fuermich/\n",
      "INFO:scrapers.BaseScraper:Found 5 unique article URLs on the current page: https://www.spiegel.de/fuermich/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/schlagzeilen/\n",
      "INFO:scrapers.BaseScraper:Found 310 unique article URLs on the current page: https://www.spiegel.de/schlagzeilen/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/spiegel/\n",
      "INFO:scrapers.BaseScraper:Found 5 unique article URLs on the current page: https://www.spiegel.de/spiegel/print/index-2024-39.html\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/audio/\n",
      "INFO:scrapers.BaseScraper:Found 22 unique article URLs on the current page: https://www.spiegel.de/audio/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/plus/\n",
      "INFO:scrapers.BaseScraper:Found 32 unique article URLs on the current page: https://www.spiegel.de/plus/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/ausland/\n",
      "INFO:scrapers.BaseScraper:Found 29 unique article URLs on the current page: https://www.spiegel.de/ausland/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/panorama/\n",
      "INFO:scrapers.BaseScraper:Found 29 unique article URLs on the current page: https://www.spiegel.de/panorama/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/sport/\n",
      "INFO:scrapers.BaseScraper:Found 30 unique article URLs on the current page: https://www.spiegel.de/sport/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/wirtschaft/\n",
      "INFO:scrapers.BaseScraper:Found 29 unique article URLs on the current page: https://www.spiegel.de/wirtschaft/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/wissenschaft/\n",
      "INFO:scrapers.BaseScraper:Found 25 unique article URLs on the current page: https://www.spiegel.de/wissenschaft/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/netzwelt/\n",
      "INFO:scrapers.BaseScraper:Found 25 unique article URLs on the current page: https://www.spiegel.de/netzwelt/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/kultur/\n",
      "INFO:scrapers.BaseScraper:Found 25 unique article URLs on the current page: https://www.spiegel.de/kultur/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/karriere/\n",
      "INFO:scrapers.BaseScraper:Found 25 unique article URLs on the current page: https://www.spiegel.de/karriere/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/start/\n",
      "INFO:scrapers.BaseScraper:Found 44 unique article URLs on the current page: https://www.spiegel.de/start/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/tests/\n",
      "INFO:scrapers.BaseScraper:Found 96 unique article URLs on the current page: https://www.spiegel.de/tests/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/geschichte/\n",
      "INFO:scrapers.BaseScraper:Found 83 unique article URLs on the current page: https://www.spiegel.de/geschichte/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/auto/\n",
      "INFO:scrapers.BaseScraper:Found 25 unique article URLs on the current page: https://www.spiegel.de/auto/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/video/\n",
      "INFO:scrapers.BaseScraper:Found 197 unique article URLs on the current page: https://www.spiegel.de/video/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/debatten/\n",
      "INFO:scrapers.BaseScraper:Found 6 unique article URLs on the current page: https://www.spiegel.de/debatten/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/deinspiegel/\n",
      "INFO:scrapers.BaseScraper:Found 35 unique article URLs on the current page: https://www.spiegel.de/deinspiegel/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/international/\n",
      "INFO:scrapers.BaseScraper:Found 35 unique article URLs on the current page: https://www.spiegel.de/international/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/politik/\n",
      "INFO:scrapers.BaseScraper:Found 25 unique article URLs on the current page: https://www.spiegel.de/politik/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/gesundheit/\n",
      "INFO:scrapers.BaseScraper:Found 25 unique article URLs on the current page: https://www.spiegel.de/gesundheit/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/psychologie/\n",
      "INFO:scrapers.BaseScraper:Found 25 unique article URLs on the current page: https://www.spiegel.de/psychologie/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/familie/\n",
      "INFO:scrapers.BaseScraper:Found 25 unique article URLs on the current page: https://www.spiegel.de/familie/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/partnerschaft/\n",
      "INFO:scrapers.BaseScraper:Found 25 unique article URLs on the current page: https://www.spiegel.de/partnerschaft/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/reise/\n",
      "INFO:scrapers.BaseScraper:Found 25 unique article URLs on the current page: https://www.spiegel.de/reise/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/stil/\n",
      "INFO:scrapers.BaseScraper:Found 25 unique article URLs on the current page: https://www.spiegel.de/stil/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/services/\n",
      "INFO:scrapers.BaseScraper:Found 10 unique article URLs on the current page: https://www.spiegel.de/services/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/backstage/\n",
      "INFO:scrapers.BaseScraper:Found 28 unique article URLs on the current page: https://www.spiegel.de/backstage/\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/nachrichtenarchiv/\n",
      "INFO:scrapers.BaseScraper:Found 65 unique article URLs on the current page: https://www.spiegel.de/nachrichtenarchiv/\n"
     ]
    }
   ],
   "source": [
    "all_found_urls = scraper.get_article_urls()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2b9550d1-3b18-4e85-aaf5-ba33365a56d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "993"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_found_urls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ba1598a1-4348-4369-adf5-d0ab800ffb6e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Loading credentials from file: credentials_keycloak.txt\n",
      "INFO:root:Getting token\n",
      "INFO:root:Token is None, refreshing token\n",
      "INFO:root:Refreshing token\n",
      "INFO:root:Initializing KeycloakOpenID client\n",
      "INFO:root:Token refreshed successfully. Token expires at: 2024-09-24 12:06:59.240133\n"
     ]
    }
   ],
   "source": [
    "# get the token for the database\n",
    "keycloak_login = KeycloakLogin()\n",
    "token = keycloak_login.get_token()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1c2efb52-4f26-472e-8abf-27a3762ec9b1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data_downloader = DataDownloader(token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a11b0525-0980-4c6b-8a57-fc0da1dc5266",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed batch 0\n",
      "Processed batch 30\n",
      "Processed batch 60\n",
      "Processed batch 90\n",
      "Processed batch 120\n",
      "Processed batch 150\n",
      "Processed batch 180\n",
      "Processed batch 210\n",
      "Processed batch 240\n",
      "Processed batch 270\n",
      "Processed batch 300\n",
      "Processed batch 330\n",
      "Processed batch 360\n",
      "Processed batch 390\n",
      "Processed batch 420\n",
      "Processed batch 450\n",
      "Processed batch 480\n",
      "Processed batch 510\n",
      "Processed batch 540\n",
      "Processed batch 570\n",
      "Processed batch 600\n",
      "Processed batch 630\n",
      "Processed batch 660\n",
      "Processed batch 690\n",
      "Processed batch 720\n",
      "Processed batch 750\n",
      "Processed batch 780\n",
      "Processed batch 810\n",
      "Processed batch 840\n",
      "Processed batch 870\n",
      "Processed batch 900\n",
      "Processed batch 930\n",
      "Processed batch 960\n",
      "Processed batch 990\n",
      "Total URLs already in DB: 292\n"
     ]
    }
   ],
   "source": [
    "batch_size = 30\n",
    "all_urls_already_in_db = []\n",
    "# Loop through all the URLs in chunks of 50\n",
    "for i in range(0, len(all_found_urls), batch_size):\n",
    "    # Get the current batch of URLs (from i to i + batch_size or up to the end of the list)\n",
    "    current_batch = all_found_urls[i:i + batch_size]\n",
    "    \n",
    "    # Fetch data for the current batch\n",
    "    response = data_downloader.get_content_rehydrate(url=current_batch)\n",
    "    \n",
    "    # Extract URLs from the 'items' in the response and store them in the list\n",
    "    batch_urls = [item['url'] for item in response.get('items', [])]\n",
    "    \n",
    "    # Extend the combined list with the URLs from this batch\n",
    "    all_urls_already_in_db.extend(batch_urls)\n",
    "    print(f\"Processed batch\", i)\n",
    "# At the end of the loop, all_urls_already_in_db contains all the URLs\n",
    "print(f\"Total URLs already in DB: {len(all_urls_already_in_db)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1e19512f-9b2e-4e70-bc2c-259e4001c9ea",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "292"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_urls_already_in_db)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "511e5cd8-c6a7-4872-b95d-a6f4d1038ace",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Refreshing token\n",
      "INFO:root:Token refreshed successfully. Token expires at: 2024-09-24 12:08:25.621406\n"
     ]
    }
   ],
   "source": [
    "keycloak_login._refresh_token()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f19fb7a4-49c5-4815-b305-9503d0ccc1fc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data_uploader = DataUploader(token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b37fb440-be61-4193-b10f-d4a3b27244ae",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "responses_for_last_online_verification_date_patch = data_uploader.patch_last_online_verification_date(all_urls_already_in_db)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "eb0a6e97-c381-4a59-aa38-5265ea0884bd",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "292"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(responses_for_last_online_verification_date_patch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "435973f2-3df2-4d81-a12e-a5d258ab61a7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "responses_for_last_online_verification_date_patch[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "563c2888-ec6f-4bdd-ac52-a372f41cbabb",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Refreshing token\n",
      "INFO:root:Token refreshed successfully. Token expires at: 2024-09-24 12:09:53.346373\n"
     ]
    }
   ],
   "source": [
    "keycloak_login._refresh_token()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f52ff8d9-3af8-4e8c-a09b-9748745a7571",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter the dictionaries based on the 'url' key\n",
    "articles_list_for_new_scraping = [url for url in all_found_urls if url not in all_urls_already_in_db]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "8f939d20-f729-49ab-94bf-c6be7f017e5b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/wirtschaft/unternehmen/deutschen-boerse-geraet-ins-visier-von-eu-kartellwaechtern-a-3d667256-d500-4dc0-8c07-b56fa1c4ba8b\n",
      "INFO:scrapers.BaseScraper:Extracted content from https://www.spiegel.de/wirtschaft/unternehmen/deutschen-boerse-geraet-ins-visier-von-eu-kartellwaechtern-a-3d667256-d500-4dc0-8c07-b56fa1c4ba8b\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/sport/fussball/em-2024-toni-kroos-mit-ironischer-reaktion-auf-uefa-bericht-zu-marc-cucurellas-handspiel-a-2ff34607-54c7-432e-b565-1844946e4102\n",
      "INFO:scrapers.BaseScraper:Extracted content from https://www.spiegel.de/sport/fussball/em-2024-toni-kroos-mit-ironischer-reaktion-auf-uefa-bericht-zu-marc-cucurellas-handspiel-a-2ff34607-54c7-432e-b565-1844946e4102\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/reise/europa/couchsurfing-in-der-ukraine-meine-reise-durch-ein-land-im-krieg-a-15963734-b1d0-49ee-9921-30f1fbd4a475\n",
      "INFO:scrapers.BaseScraper:Extracted content from https://www.spiegel.de/reise/europa/couchsurfing-in-der-ukraine-meine-reise-durch-ein-land-im-krieg-a-15963734-b1d0-49ee-9921-30f1fbd4a475\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/netzwelt/netzpolitik/keine-trainingsdaten-keine-ki-meta-macht-stimmung-gegen-eu-datenschuetzer-a-e3eba384-0293-43f8-aa00-e0d02701e4e9\n",
      "INFO:scrapers.BaseScraper:Extracted content from https://www.spiegel.de/netzwelt/netzpolitik/keine-trainingsdaten-keine-ki-meta-macht-stimmung-gegen-eu-datenschuetzer-a-e3eba384-0293-43f8-aa00-e0d02701e4e9\n",
      "INFO:scrapers.BaseScraper:Navigated to https://www.spiegel.de/wirtschaft/klimakonferenz-2024-live-robert-habeck-markus-soeder-maja-goepel-und-maria-furtwaengler-in-berlin-a-d54c1fab-7320-4777-a516-15dcd6420871\n",
      "INFO:scrapers.BaseScraper:Extracted content from https://www.spiegel.de/wirtschaft/klimakonferenz-2024-live-robert-habeck-markus-soeder-maja-goepel-und-maria-furtwaengler-in-berlin-a-d54c1fab-7320-4777-a516-15dcd6420871\n",
      "INFO:scrapers.BaseScraper:Browser closed successfully\n"
     ]
    }
   ],
   "source": [
    "articles = scraper.scrape(articles_list_for_new_scraping[0:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "06664754-2dee-4a4f-b234-e33082a25c28",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def clear_gpu_memory():\n",
    "    \"\"\"Clears GPU memory and forces garbage collection.\"\"\"\n",
    "    torch.cuda.empty_cache()\n",
    "    gc.collect()\n",
    "\n",
    "def process_articles_in_batches(text_analysis_class, method_name, articles, batch_size):\n",
    "    \"\"\"Process articles in batches using the specified text analysis class and method.\"\"\"\n",
    "    processor = text_analysis_class()\n",
    "    method = getattr(processor, method_name)\n",
    "\n",
    "    for i in range(0, len(articles), batch_size):\n",
    "        batch = articles[i:i + batch_size]\n",
    "        batch = method(batch)\n",
    "\n",
    "        # Reassign the processed batch back to the main list\n",
    "        articles[i:i + batch_size] = batch\n",
    "\n",
    "        clear_gpu_memory()  # Clear memory after each batch\n",
    "\n",
    "    del processor  # Delete the processor instance to free up GPU memory\n",
    "    clear_gpu_memory()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "5e20fdd1-6e0a-4416-820d-70f23703dc38",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:text_analysis.NEExtractor:Initializing NEExtractor with model: flair/ner-german-large\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting entity extraction...\n",
      "2024-09-24 12:08:22,267 SequenceTagger predicts: Dictionary with 20 tags: <unk>, O, B-PER, E-PER, S-LOC, B-MISC, I-MISC, E-MISC, S-PER, B-ORG, E-ORG, S-ORG, I-ORG, B-LOC, E-LOC, S-MISC, I-PER, I-LOC, <START>, <STOP>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:text_analysis.NEExtractor:Tagger instantiated successfully\n",
      "INFO:text_analysis.NEExtractor:Starting entity extraction for 5 articles\n",
      "INFO:text_analysis.NEExtractor:Applying entity extraction to all articles\n",
      "Map: 100%|██████████| 5/5 [00:02<00:00,  2.32 examples/s]\n",
      "INFO:text_analysis.NEExtractor:Inferring geographical focus\n",
      "INFO:text_analysis.NEExtractor:Starting geographical focus inference for 5 articles\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1]]\n",
      "NULL\n",
      "\n",
      "[[2]]\n",
      "NULL\n",
      "\n",
      "[[3]]\n",
      "NULL\n",
      "\n",
      "[[4]]\n",
      "NULL\n",
      "\n",
      "[[5]]\n",
      "NULL\n",
      "\n",
      "[[1]]\n",
      "[1] \"newsmap\"   \"stats\"     \"graphics\"  \"grDevices\" \"utils\"     \"datasets\" \n",
      "[7] \"methods\"   \"base\"     \n",
      "\n",
      "[[2]]\n",
      "[1] \"quanteda\"  \"newsmap\"   \"stats\"     \"graphics\"  \"grDevices\" \"utils\"    \n",
      "[7] \"datasets\"  \"methods\"   \"base\"     \n",
      "\n",
      "[[3]]\n",
      " [1] \"dplyr\"     \"quanteda\"  \"newsmap\"   \"stats\"     \"graphics\"  \"grDevices\"\n",
      " [7] \"utils\"     \"datasets\"  \"methods\"   \"base\"     \n",
      "\n",
      "[[4]]\n",
      " [1] \"countrycode\" \"dplyr\"       \"quanteda\"    \"newsmap\"     \"stats\"      \n",
      " [6] \"graphics\"    \"grDevices\"   \"utils\"       \"datasets\"    \"methods\"    \n",
      "[11] \"base\"       \n",
      "\n",
      "[[5]]\n",
      " [1] \"jsonlite\"    \"countrycode\" \"dplyr\"       \"quanteda\"    \"newsmap\"    \n",
      " [6] \"stats\"       \"graphics\"    \"grDevices\"   \"utils\"       \"datasets\"   \n",
      "[11] \"methods\"     \"base\"       \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Package version: 4.1.0\n",
      "Unicode version: 13.0\n",
      "ICU version: 66.1\n",
      "Parallel computing: disabled\n",
      "See https://quanteda.io for tutorials and examples.\n",
      "\n",
      "Attaching package: ‘dplyr’\n",
      "\n",
      "The following objects are masked from ‘package:stats’:\n",
      "\n",
      "    filter, lag\n",
      "\n",
      "The following objects are masked from ‘package:base’:\n",
      "\n",
      "    intersect, setdiff, setequal, union\n",
      "\n",
      "INFO:text_analysis.NEExtractor:Geographical focus inference completed successfully\n",
      "INFO:text_analysis.NEExtractor:Entity extraction and geographical focus inference complete\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] TRUE\n",
      "Entity extraction completed.\n",
      "Starting topic extraction...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Map: 100%|██████████| 5/5 [00:01<00:00,  2.88 examples/s]\n",
      "INFO:sentence_transformers.SentenceTransformer:Use pytorch device_name: cuda\n",
      "INFO:sentence_transformers.SentenceTransformer:Load pretrained SentenceTransformer: deepset/gbert-base\n",
      "WARNING:sentence_transformers.SentenceTransformer:No sentence-transformers model found with name deepset/gbert-base. Creating a new one with mean pooling.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic extraction completed.\n",
      "Starting vectorization...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jovyan/work/Patrick/m3-scraper/patrick_myenv/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
      "  warnings.warn(\n",
      "INFO:sentence_transformers.SentenceTransformer:Use pytorch device_name: cuda\n",
      "INFO:sentence_transformers.SentenceTransformer:Load pretrained SentenceTransformer: T-Systems-onsite/german-roberta-sentence-transformer-v2\n",
      "WARNING:sentence_transformers.SentenceTransformer:No sentence-transformers model found with name T-Systems-onsite/german-roberta-sentence-transformer-v2. Creating a new one with mean pooling.\n",
      "INFO:sentence_transformers.SentenceTransformer:Use pytorch device_name: cuda\n",
      "INFO:sentence_transformers.SentenceTransformer:Load pretrained SentenceTransformer: deutsche-telekom/gbert-large-paraphrase-euclidean\n",
      "INFO:sentence_transformers.SentenceTransformer:Use pytorch device_name: cuda\n",
      "INFO:sentence_transformers.SentenceTransformer:Load pretrained SentenceTransformer: xlm-roberta-large\n",
      "WARNING:sentence_transformers.SentenceTransformer:No sentence-transformers model found with name xlm-roberta-large. Creating a new one with mean pooling.\n",
      "INFO:sentence_transformers.SentenceTransformer:Use pytorch device_name: cuda\n",
      "INFO:sentence_transformers.SentenceTransformer:Load pretrained SentenceTransformer: google/bigbird-roberta-large\n",
      "WARNING:sentence_transformers.SentenceTransformer:No sentence-transformers model found with name google/bigbird-roberta-large. Creating a new one with mean pooling.\n",
      "INFO:sentence_transformers.SentenceTransformer:Use pytorch device_name: cuda\n",
      "INFO:sentence_transformers.SentenceTransformer:Load pretrained SentenceTransformer: severinsimmler/xlm-roberta-longformer-large-16384\n",
      "WARNING:sentence_transformers.SentenceTransformer:No sentence-transformers model found with name severinsimmler/xlm-roberta-longformer-large-16384. Creating a new one with mean pooling.\n",
      "Map:   0%|          | 0/5 [00:00<?, ? examples/s]\n",
      "Batches:   0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00,  3.08it/s]\u001b[A\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 48.41it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 73.63it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 77.11it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 36.47it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 32.37it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 32.93it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 25.37it/s]\n",
      "\n",
      "Batches:   0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00,  2.33it/s]\u001b[A\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 15.79it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 11.14it/s]\n",
      "\n",
      "Batches:   0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00,  1.92it/s]\u001b[A\n",
      "Map:  20%|██        | 1/5 [00:01<00:07,  1.77s/ examples]\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 58.28it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 41.50it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 61.01it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 35.01it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 46.02it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 14.06it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 32.14it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 13.37it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 32.08it/s]\n",
      "\n",
      "Batches:   0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00,  7.64it/s]\u001b[A\n",
      "\n",
      "Batches:   0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00,  7.98it/s]\u001b[A\n",
      "\n",
      "Batches:   0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00,  2.53it/s]\u001b[A\n",
      "Map:  40%|████      | 2/5 [00:02<00:04,  1.39s/ examples]\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 58.13it/s]\n",
      "\n",
      "Batches: 100%|██████████| 3/3 [00:00<00:00, 38.35it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 51.91it/s]\n",
      "\n",
      "Batches: 100%|██████████| 3/3 [00:00<00:00, 32.38it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 58.81it/s]\n",
      "\n",
      "Batches:   0%|          | 0/3 [00:00<?, ?it/s]\u001b[A\n",
      "Batches: 100%|██████████| 3/3 [00:00<00:00, 12.56it/s]\u001b[A\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 31.57it/s]\n",
      "\n",
      "Batches:   0%|          | 0/3 [00:00<?, ?it/s]\u001b[A\n",
      "Batches:  33%|███▎      | 1/3 [00:00<00:00,  5.65it/s]\u001b[A\n",
      "Batches: 100%|██████████| 3/3 [00:00<00:00, 10.70it/s]\u001b[A\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 39.58it/s]\n",
      "\n",
      "Batches:   0%|          | 0/3 [00:00<?, ?it/s]\u001b[A\n",
      "Batches:  33%|███▎      | 1/3 [00:00<00:00,  4.05it/s]\u001b[A\n",
      "Batches: 100%|██████████| 3/3 [00:00<00:00,  7.66it/s]\u001b[A\n",
      "\n",
      "Batches:   0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00,  5.84it/s]\u001b[A\n",
      "\n",
      "Batches:   0%|          | 0/3 [00:00<?, ?it/s]\u001b[A\n",
      "Batches:  33%|███▎      | 1/3 [00:01<00:02,  1.29s/it]\u001b[A\n",
      "Batches:  67%|██████▋   | 2/3 [00:02<00:01,  1.28s/it]\u001b[A\n",
      "Batches: 100%|██████████| 3/3 [00:03<00:00,  1.18s/it]\u001b[A\n",
      "Map:  60%|██████    | 3/5 [00:07<00:06,  3.07s/ examples]\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 72.48it/s]\n",
      "\n",
      "Batches: 100%|██████████| 2/2 [00:00<00:00, 31.91it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 51.17it/s]\n",
      "\n",
      "Batches: 100%|██████████| 2/2 [00:00<00:00, 32.22it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 92.77it/s]\n",
      "\n",
      "Batches:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Batches: 100%|██████████| 2/2 [00:00<00:00, 11.54it/s]\u001b[A\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 36.88it/s]\n",
      "\n",
      "Batches:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Batches: 100%|██████████| 2/2 [00:00<00:00, 10.20it/s]\u001b[A\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 40.31it/s]\n",
      "\n",
      "Batches:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Batches: 100%|██████████| 2/2 [00:00<00:00,  5.83it/s]\u001b[A\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 10.91it/s]\n",
      "\n",
      "Batches:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Batches:  50%|█████     | 1/2 [00:01<00:01,  1.28s/it]\u001b[A\n",
      "Batches: 100%|██████████| 2/2 [00:02<00:00,  1.05s/it]\u001b[A\n",
      "Map:  80%|████████  | 4/5 [00:11<00:03,  3.16s/ examples]\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 107.65it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 41.30it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 55.52it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 31.81it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 37.71it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 13.62it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 74.51it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 12.07it/s]\n",
      "\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 59.21it/s]\n",
      "\n",
      "Batches:   0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00,  8.05it/s]\u001b[A\n",
      "\n",
      "Batches:   0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00,  7.76it/s]\u001b[A\n",
      "\n",
      "Batches:   0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
      "Batches: 100%|██████████| 1/1 [00:00<00:00,  3.40it/s]\u001b[A\n",
      "Map: 100%|██████████| 5/5 [00:12<00:00,  2.46s/ examples]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vectorization completed.\n",
      "Starting summarization...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Map: 100%|██████████| 5/5 [00:27<00:00,  5.44s/ examples]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Summarization completed.\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import gc\n",
    "import json\n",
    "    \n",
    "# Define your batch size\n",
    "batch_size = len(articles)  # Adjust this based on your GPU capacity\n",
    "\n",
    "# Process with Entity Extractor\n",
    "print(\"Starting entity extraction...\")\n",
    "process_articles_in_batches(NEExtractor, 'extract_entities', articles, batch_size)\n",
    "print(\"Entity extraction completed.\")\n",
    "\n",
    "# Process with Topic Extractor\n",
    "print(\"Starting topic extraction...\")\n",
    "process_articles_in_batches(TopicExtractor, 'extract_topics', articles, batch_size)\n",
    "print(\"Topic extraction completed.\")\n",
    "\n",
    "# Process with Vectorizer\n",
    "print(\"Starting vectorization...\")\n",
    "process_articles_in_batches(Vectorizer, 'vectorize', articles, batch_size)\n",
    "print(\"Vectorization completed.\")\n",
    "\n",
    "# Process with Vectorizer\n",
    "print(\"Starting summarization...\")\n",
    "process_articles_in_batches(Summarizer, 'summarize', articles, batch_size)\n",
    "print(\"Summarization completed.\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "ad6dfb28-ad2e-42fb-b7e0-e2b1becb8d91",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Remove main_text and lead_text from articles to save space before uploading\n",
    "for article in articles:\n",
    "    article.pop('main_text', None)\n",
    "    article.pop('lead_text', None)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "ac863e4f-70c6-4499-a1ae-fbaa18b7a53d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Loading credentials from file: credentials_keycloak.txt\n",
      "INFO:root:Getting token\n",
      "INFO:root:Token is None, refreshing token\n",
      "INFO:root:Refreshing token\n",
      "INFO:root:Initializing KeycloakOpenID client\n",
      "INFO:root:Token refreshed successfully. Token expires at: 2024-09-24 12:19:18.127733\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Beginning article upload...\n"
     ]
    }
   ],
   "source": [
    "# Upload each article to the database\n",
    "print(\"Beginning article upload...\")\n",
    "responses = []\n",
    "keycloak_login = KeycloakLogin()\n",
    "token = keycloak_login.get_token()\n",
    "data_uploader = DataUploader(token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "a2b86dd3-c2c6-4ad8-a55e-44cf82dc5da5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for article in articles:\n",
    "    response = data_uploader.post_content(article)\n",
    "    responses.append(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "f5f8aeac-8d56-4196-ad6f-a8ffdd6f154b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Article upload completed.\n"
     ]
    }
   ],
   "source": [
    "# Save the responses to a JSON file\n",
    "with open('responses.json', 'w') as f:\n",
    "    json.dump(responses, f)\n",
    "\n",
    "print(\"Article upload completed.\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}