From 1658a881fac9e3c6661edf08c29b40ae982d3a64 Mon Sep 17 00:00:00 2001 From: Nikhil Raj Date: Thu, 16 May 2024 17:48:31 +0530 Subject: [PATCH] Made minor changes --- src/scrape_up/ambitionBox/company.py | 41 ++-- src/scrape_up/codechef/codechef.py | 301 ++------------------------- 2 files changed, 43 insertions(+), 299 deletions(-) diff --git a/src/scrape_up/ambitionBox/company.py b/src/scrape_up/ambitionBox/company.py index 49f30251..f05f0629 100644 --- a/src/scrape_up/ambitionBox/company.py +++ b/src/scrape_up/ambitionBox/company.py @@ -2,19 +2,16 @@ from bs4 import BeautifulSoup -class Comapiens: - def __init__(self,num_pages: int=1): +class Comapiens: + def __init__(self, num_pages: int = 1): self.num_pages = num_pages def write_sorted_list(self, file, company_list): - company_list.sort(key=lambda x: x[1], reverse=True) for company_name, rating in company_list: file.write(f"{company_name.strip()} {rating}\n") def scrape_companies(self): - - headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36" } @@ -26,15 +23,19 @@ def scrape_companies(self): response = requests.get(url, headers=headers) if response.status_code == 200: - soup = BeautifulSoup(response.text, 'lxml') + soup = BeautifulSoup(response.text, "lxml") - companies = soup.find_all('div', class_="companyCardWrapper") + companies = soup.find_all("div", class_="companyCardWrapper") company_ratings = [] for company in companies: - company_name = company.find('h2', class_="companyCardWrapper__companyName").text.strip() - company_star = company.find('span', class_="companyCardWrapper__companyRatingValue") + company_name = company.find( + "h2", class_="companyCardWrapper__companyName" + ).text.strip() + company_star = company.find( + "span", class_="companyCardWrapper__companyRatingValue" + ) if company_name and company_star: try: @@ -46,24 +47,34 @@ def scrape_companies(self): with open("src/scrape_up/ambitionBox/company_ratings.txt", "a") as f: f.write(f"\nPAGE: {url}\n") f.write("COMPANY UNDER 5 STAR\n") - self.write_sorted_list(f, [r for r in company_ratings if 4 < r[1] <= 5]) + self.write_sorted_list( + f, [r for r in company_ratings if 4 < r[1] <= 5] + ) f.write("\nCOMPANY UNDER 4 STAR\n") - self.write_sorted_list(f, [r for r in company_ratings if 3 < r[1] <= 4]) + self.write_sorted_list( + f, [r for r in company_ratings if 3 < r[1] <= 4] + ) # Corrected indentation for following lines f.write("\nCOMPANY UNDER 3 STAR\n") - self.write_sorted_list(f, [r for r in company_ratings if 2 < r[1] <= 3]) + self.write_sorted_list( + f, [r for r in company_ratings if 2 < r[1] <= 3] + ) f.write("\nCOMPANY UNDER 2 STAR\n") - self.write_sorted_list(f, [r for r in company_ratings if 1 < r[1] <= 2]) + self.write_sorted_list( + f, [r for r in company_ratings if 1 < r[1] <= 2] + ) f.write("\nCOMPANY UNDER 1 STAR\n") - self.write_sorted_list(f, [r for r in company_ratings if 0 < r[1] <= 1]) + self.write_sorted_list( + f, [r for r in company_ratings if 0 < r[1] <= 1] + ) else: print(f"Error scraping page {page}: {response.status_code}") if __name__ == "__main__": - c = Comapiens(10) + c = Comapiens(10) c.scrape_companies() diff --git a/src/scrape_up/codechef/codechef.py b/src/scrape_up/codechef/codechef.py index 118ced39..7e1ddfdb 100644 --- a/src/scrape_up/codechef/codechef.py +++ b/src/scrape_up/codechef/codechef.py @@ -13,8 +13,6 @@ class User: | `get_profile()` | Returns name, username, profile_image_link, rating, details etc. | | `get_contests()`| Returns future_contests , past_contests , skill_tests etc in json format. | - - """ def __init__(self, id): @@ -22,7 +20,7 @@ def __init__(self, id): def get_profile(self): """ - Create an object of the 'User' class\n + Create an object of the 'User' class ```python user1 = User(id="heltion") user1.get_profile() @@ -145,8 +143,8 @@ def get_profile(self): except: return None - def get_contests(): - ''' + def get_contests(self): + """ get_contests output { "status": "success", @@ -163,289 +161,24 @@ def get_contests(): "contest_duration": "120", "distinct_users": 0 }, - { - "contest_code": "START135B", - "contest_name": "Starters 135", - "contest_start_date": "22 May 2024 20:00:00", - "contest_end_date": "22 May 2024 22:00:00", - "contest_start_date_iso": "2024-05-22T20:00:00+05:30", - "contest_end_date_iso": "2024-05-22T22:00:00+05:30", - "contest_duration": "120", - "distinct_users": 0 - } + ... ], "practice_contests": [], "past_contests": [ { - "contest_code": "START133", - "contest_name": "Starters 133 (Rated till 6-Star)", - "contest_start_date": "08 May 2024 20:00:00", - "contest_end_date": "08 May 2024 22:00:00", - "contest_start_date_iso": "2024-05-08T20:00:00+05:30", - "contest_end_date_iso": "2024-05-08T22:00:00+05:30", - "contest_duration": "120", - "distinct_users": 20041 - }, - { - "contest_code": "START132", - "contest_name": "Starters 132 (Rated till 6 Stars)", - "contest_start_date": "01 May 2024 20:00:00", - "contest_end_date": "01 May 2024 22:00:00", - "contest_start_date_iso": "2024-05-01T20:00:00+05:30", - "contest_end_date_iso": "2024-05-01T22:00:00+05:30", - "contest_duration": "120", - "distinct_users": 22930 - }, - { - "contest_code": "START131", - "contest_name": "Starters 131 (Rated till 5 Stars)", - "contest_start_date": "24 Apr 2024 20:00:00", - "contest_end_date": "24 Apr 2024 22:00:00", - "contest_start_date_iso": "2024-04-24T20:00:00+05:30", - "contest_end_date_iso": "2024-04-24T22:00:00+05:30", - "contest_duration": "120", - "distinct_users": 27566 - }, - { - "contest_code": "START130", - "contest_name": "Starters 130 (Rated till 6-Stars)", - "contest_start_date": "17 Apr 2024 20:00:00", - "contest_end_date": "17 Apr 2024 22:00:00", - "contest_start_date_iso": "2024-04-17T20:00:00+05:30", - "contest_end_date_iso": "2024-04-17T22:00:00+05:30", - "contest_duration": "120", - "distinct_users": 28434 - }, - { - "contest_code": "START129", - "contest_name": "Starters 129 (Rated till 5-Stars)", - "contest_start_date": "10 Apr 2024 20:00:00", - "contest_end_date": "10 Apr 2024 22:00:00", - "contest_start_date_iso": "2024-04-10T20:00:00+05:30", - "contest_end_date_iso": "2024-04-10T22:00:00+05:30", - "contest_duration": "120", - "distinct_users": 32181 - }, - { - "contest_code": "START128", - "contest_name": "Starters 128 (Rated till 6-Stars)", - "contest_start_date": "03 Apr 2024 20:00:00", - "contest_end_date": "03 Apr 2024 22:00:00", - "contest_start_date_iso": "2024-04-03T20:00:00+05:30", - "contest_end_date_iso": "2024-04-03T22:00:00+05:30", - "contest_duration": "120", - "distinct_users": 34876 - }, - { - "contest_code": "START127", - "contest_name": "Starters 127 (Rated till 5 Stars)", - "contest_start_date": "27 Mar 2024 20:00:00", - "contest_end_date": "27 Mar 2024 22:00:00", - "contest_start_date_iso": "2024-03-27T20:00:00+05:30", - "contest_end_date_iso": "2024-03-27T22:00:00+05:30", - "contest_duration": "120", - "distinct_users": 37656 - }, - { - "contest_code": "START126", - "contest_name": "Starters 126 (Rated till 6 Stars)", - "contest_start_date": "20 Mar 2024 20:00:00", - "contest_end_date": "20 Mar 2024 22:00:00", - "contest_start_date_iso": "2024-03-20T20:00:00+05:30", - "contest_end_date_iso": "2024-03-20T22:00:00+05:30", - "contest_duration": "120", - "distinct_users": 39861 - }, - { - "contest_code": "START125", - "contest_name": "Starters 125 (Rated till 6-Stars)", - "contest_start_date": "13 Mar 2024 20:00:00", - "contest_end_date": "13 Mar 2024 22:00:00", - "contest_start_date_iso": "2024-03-13T20:00:00+05:30", - "contest_end_date_iso": "2024-03-13T22:00:00+05:30", - "contest_duration": "120", - "distinct_users": 40646 - }, - { - "contest_code": "START124", - "contest_name": "Starters 124 (Rated till 5-Stars)", - "contest_start_date": "06 Mar 2024 20:00:00", - "contest_end_date": "06 Mar 2024 22:00:00", - "contest_start_date_iso": "2024-03-06T20:00:00+05:30", - "contest_end_date_iso": "2024-03-06T22:00:00+05:30", - "contest_duration": "120", - "distinct_users": 42655 - }, - { - "contest_code": "START123", - "contest_name": "Starters 123 (Rated till 6-Stars)", - "contest_start_date": "28 Feb 2024 20:00:00", - "contest_end_date": "28 Feb 2024 22:30:00", - "contest_start_date_iso": "2024-02-28T20:00:00+05:30", - "contest_end_date_iso": "2024-02-28T22:30:00+05:30", - "contest_duration": "150", - "distinct_users": 42005 - }, - { - "contest_code": "START122", - "contest_name": "Starters 122 (Rated till 6-Stars)", - "contest_start_date": "21 Feb 2024 20:00:00", - "contest_end_date": "21 Feb 2024 22:00:00", - "contest_start_date_iso": "2024-02-21T20:00:00+05:30", - "contest_end_date_iso": "2024-02-21T22:00:00+05:30", - "contest_duration": "120", - "distinct_users": 39677 - }, - { - "contest_code": "START121", - "contest_name": "Starters 121 (Rated till 6-Stars)", - "contest_start_date": "14 Feb 2024 20:00:00", - "contest_end_date": "14 Feb 2024 22:00:00", - "contest_start_date_iso": "2024-02-14T20:00:00+05:30", - "contest_end_date_iso": "2024-02-14T22:00:00+05:30", - "contest_duration": "120", - "distinct_users": 34442 - }, - { - "contest_code": "START120", - "contest_name": "Starters 120 (Rated till 6-Stars)", - "contest_start_date": "07 Feb 2024 20:00:00", - "contest_end_date": "07 Feb 2024 22:00:00", - "contest_start_date_iso": "2024-02-07T20:00:00+05:30", - "contest_end_date_iso": "2024-02-07T22:00:00+05:30", - "contest_duration": "120", - "distinct_users": 37332 - }, - { - "contest_code": "START119", - "contest_name": "Starters 119 (Rated till 6-Stars)", - "contest_start_date": "31 Jan 2024 20:00:00", - "contest_end_date": "31 Jan 2024 22:00:00", - "contest_start_date_iso": "2024-01-31T20:00:00+05:30", - "contest_end_date_iso": "2024-01-31T22:00:00+05:30", - "contest_duration": "120", - "distinct_users": 33112 - }, - { - "contest_code": "START118", - "contest_name": "Starters 118 (Rated till 5-Stars)", - "contest_start_date": "24 Jan 2024 20:00:00", - "contest_end_date": "24 Jan 2024 22:00:00", - "contest_start_date_iso": "2024-01-24T20:00:00+05:30", - "contest_end_date_iso": "2024-01-24T22:00:00+05:30", - "contest_duration": "120", - "distinct_users": 27683 - }, - { - "contest_code": "START117", - "contest_name": "Starters 117 (Rated till 5-Stars)", - "contest_start_date": "17 Jan 2024 20:00:00", - "contest_end_date": "17 Jan 2024 22:00:00", - "contest_start_date_iso": "2024-01-17T20:00:00+05:30", - "contest_end_date_iso": "2024-01-17T22:00:00+05:30", - "contest_duration": "120", - "distinct_users": 22710 - }, - { - "contest_code": "START116", - "contest_name": "Starters 116 (Rated till 6-Stars)", - "contest_start_date": "10 Jan 2024 20:00:00", - "contest_end_date": "10 Jan 2024 22:15:00", - "contest_start_date_iso": "2024-01-10T20:00:00+05:30", - "contest_end_date_iso": "2024-01-10T22:15:00+05:30", - "contest_duration": "135", - "distinct_users": 26898 - }, - { - "contest_code": "START115", - "contest_name": "Starters 115 (Rated for All)", - "contest_start_date": "03 Jan 2024 20:00:00", - "contest_end_date": "03 Jan 2024 22:30:00", - "contest_start_date_iso": "2024-01-03T20:00:00+05:30", - "contest_end_date_iso": "2024-01-03T22:30:00+05:30", - "contest_duration": "150", - "distinct_users": 26446 - }, - { - "contest_code": "START114", - "contest_name": "Starters 114 (Rated till 6 Stars)", - "contest_start_date": "27 Dec 2023 20:00:00", - "contest_end_date": "27 Dec 2023 22:15:00", - "contest_start_date_iso": "2023-12-27T20:00:00+05:30", - "contest_end_date_iso": "2023-12-27T22:15:00+05:30", - "contest_duration": "135", - "distinct_users": 23396 - } - ], - "skill_tests": [ - { - "contest_code": "LPYAS", - "contest_name": "Python Skill test", - "contest_start_date": "27 Mar 2024 15:00:00", - "contest_end_date": "01 Jan 2027 01:30:00", - "contest_start_date_iso": "2024-03-27T15:00:00+05:30", - "contest_end_date_iso": "2027-01-01T01:30:00+05:30", - "contest_duration": "90", - "problem_count": 30, - "distinct_users": 0 - }, - { - "contest_code": "LCPPAS", - "contest_name": "C++ Skill test", - "contest_start_date": "28 Mar 2024 00:00:00", - "contest_end_date": "01 Jan 2027 01:30:00", - "contest_start_date_iso": "2024-03-28T00:00:00+05:30", - "contest_end_date_iso": "2027-01-01T01:30:00+05:30", - "contest_duration": "90", - "problem_count": 30, - "distinct_users": 0 - }, - { - "contest_code": "LCAS", - "contest_name": "C language skill test", - "contest_start_date": "28 Mar 2024 00:00:00", - "contest_end_date": "01 Jan 2027 01:30:00", - "contest_start_date_iso": "2024-03-28T00:00:00+05:30", - "contest_end_date_iso": "2027-01-01T01:30:00+05:30", - "contest_duration": "90", - "problem_count": 30, - "distinct_users": 0 - }, - { - "contest_code": "LJJAAS", - "contest_name": "Java Skill test", - "contest_start_date": "28 Mar 2024 00:00:00", - "contest_end_date": "01 Jan 2027 01:30:00", - "contest_start_date_iso": "2024-03-28T00:00:00+05:30", - "contest_end_date_iso": "2027-01-01T01:30:00+05:30", - "contest_duration": "90", - "problem_count": 30, - "distinct_users": 0 - }, - { - "contest_code": "BDSAJAS01", - "contest_name": "Data structures and Algorithms in Java test", - "contest_start_date": "17 Apr 2024 17:00:00", - "contest_end_date": "01 Jan 2027 02:00:00", - "contest_start_date_iso": "2024-04-17T17:00:00+05:30", - "contest_end_date_iso": "2027-01-01T02:00:00+05:30", - "contest_duration": "120", - "problem_count": 28, - "distinct_users": 0 - } - ], - "banners": [ - { - "image": "1715586444.png", - "link": "https://www.codechef.com/START134" - }, - { - "image": "1704448967.png", - "link": "https://www.codechef.com/roadmap/become-5-star" - } - ] + "contest_code": "START133", + "contest_name": "Starters 133 (Rated till 6-Star)", + "contest_start_date": "08 May 2024 20:00:00", + "contest_end_date": "08 May 2024 22:00:00", + "contest_start_date_iso": "2024-05-08T20:00:00+05:30", + "contest_end_date_iso": "2024-05-08T22:00:00+05:30", + "contest_duration": "120", + "distinct_users": 20041 + }, + ... + ] } - ''' + """ try: url = "https://www.codechef.com/api/list/contests/all?sort_by=START&sorting_order=asc&offset=0&mode=all" headers = { @@ -454,4 +187,4 @@ def get_contests(): response = requests.get(url, headers=headers).text return response except: - return None \ No newline at end of file + return None