scraper.py

import datetime
import os
import re

import requests
import lxml.html as html # Converts a HTML file to XPath file


# Constants
HOME_URL = 'https://www.larepublica.co'
XPATH_LINK_TO_ARTICLE = '//text-fill/a/@href'
XPATH_TITLE = '//div[@class="mb-auto"]/text-fill/span/text()'
XPATH_SUMMARY = '//div[@class="lead"]/p/text()'
XPATH_BODY = '//div[@class="html-content"]/p/text()'


def parse_notice(link: str, today: str):
    """_summary_

    Args:
        link (str): _description_
        today (str): _description_
    """
    try:

        response = requests.get(link)

        if response.status_code == 200:
            notice = response.content.decode('utf-8')
            parsed = html.fromstring(notice)

            try:
                title : str = parsed.xpath(XPATH_TITLE)[0]
                title = re.sub(r'[^\w ]', '', title)
                summary : str = parsed.xpath(XPATH_SUMMARY)[0]
                body : str = parsed.xpath(XPATH_BODY)
            except IndexError:
                return

            with open(f'{today}/{title}.txt', 'w', encoding='utf-8') as f:
                f.write(title)
                f.write('\n\n')
                f.write(summary)
                f.write('\n\n')
                for p in body:
                    f.write(p)
                    f.write('\n')

        else:
            raise ValueError(f'Error: {response.status_code}')

    except ValueError as ve:
        print(ve)


def parse_home():
    """Parse the home page to xpath

        Raises:
        ValueError: _description_
    """
    try:

        response = requests.get(HOME_URL)

        if response.status_code == 200:

            home = response.content.decode('utf-8')
            parsed = html.fromstring(home)
            links_to_notices = parsed.xpath(XPATH_LINK_TO_ARTICLE)
            today = datetime.date.today().strftime('%d-%m-%Y')
            
            if not os.path.isdir(today):
                os.mkdir(today)

            for link in links_to_notices:
                parse_notice(link, today)

        else:
            raise ValueError(f'Error: {response.status_code}')

    except ValueError as ve:
        print(ve)


def run():
    parse_home()


if __name__ == '__main__':
    run()