diff --git a/dev-documentation.md b/dev-documentation.md index fb43277f..c1e0a1de 100644 --- a/dev-documentation.md +++ b/dev-documentation.md @@ -1173,9 +1173,10 @@ user = Codechef(id="username") ``` -| Methods | Details | -| --------------- | ---------------------------------------------------------------- | -| `get_profile()` | Returns name, username, profile_image_link, rating, details etc. | +| Methods | Details | +| --------------- | ------------------------------------------------------------------------- | +| `get_profile()` | Returns name, username, profile_image_link, rating, details etc. | +| `get_contests()`| Returns future_contests , past_contests , skill_tests etc in json format. | --- diff --git a/src/scrape_up/ambitionBox/company.py b/src/scrape_up/ambitionBox/company.py index 49f30251..f05f0629 100644 --- a/src/scrape_up/ambitionBox/company.py +++ b/src/scrape_up/ambitionBox/company.py @@ -2,19 +2,16 @@ from bs4 import BeautifulSoup -class Comapiens: - def __init__(self,num_pages: int=1): +class Comapiens: + def __init__(self, num_pages: int = 1): self.num_pages = num_pages def write_sorted_list(self, file, company_list): - company_list.sort(key=lambda x: x[1], reverse=True) for company_name, rating in company_list: file.write(f"{company_name.strip()} {rating}\n") def scrape_companies(self): - - headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36" } @@ -26,15 +23,19 @@ def scrape_companies(self): response = requests.get(url, headers=headers) if response.status_code == 200: - soup = BeautifulSoup(response.text, 'lxml') + soup = BeautifulSoup(response.text, "lxml") - companies = soup.find_all('div', class_="companyCardWrapper") + companies = soup.find_all("div", class_="companyCardWrapper") company_ratings = [] for company in companies: - company_name = company.find('h2', class_="companyCardWrapper__companyName").text.strip() - company_star = company.find('span', class_="companyCardWrapper__companyRatingValue") + company_name = company.find( + "h2", class_="companyCardWrapper__companyName" + ).text.strip() + company_star = company.find( + "span", class_="companyCardWrapper__companyRatingValue" + ) if company_name and company_star: try: @@ -46,24 +47,34 @@ def scrape_companies(self): with open("src/scrape_up/ambitionBox/company_ratings.txt", "a") as f: f.write(f"\nPAGE: {url}\n") f.write("COMPANY UNDER 5 STAR\n") - self.write_sorted_list(f, [r for r in company_ratings if 4 < r[1] <= 5]) + self.write_sorted_list( + f, [r for r in company_ratings if 4 < r[1] <= 5] + ) f.write("\nCOMPANY UNDER 4 STAR\n") - self.write_sorted_list(f, [r for r in company_ratings if 3 < r[1] <= 4]) + self.write_sorted_list( + f, [r for r in company_ratings if 3 < r[1] <= 4] + ) # Corrected indentation for following lines f.write("\nCOMPANY UNDER 3 STAR\n") - self.write_sorted_list(f, [r for r in company_ratings if 2 < r[1] <= 3]) + self.write_sorted_list( + f, [r for r in company_ratings if 2 < r[1] <= 3] + ) f.write("\nCOMPANY UNDER 2 STAR\n") - self.write_sorted_list(f, [r for r in company_ratings if 1 < r[1] <= 2]) + self.write_sorted_list( + f, [r for r in company_ratings if 1 < r[1] <= 2] + ) f.write("\nCOMPANY UNDER 1 STAR\n") - self.write_sorted_list(f, [r for r in company_ratings if 0 < r[1] <= 1]) + self.write_sorted_list( + f, [r for r in company_ratings if 0 < r[1] <= 1] + ) else: print(f"Error scraping page {page}: {response.status_code}") if __name__ == "__main__": - c = Comapiens(10) + c = Comapiens(10) c.scrape_companies() diff --git a/src/scrape_up/codechef/codechef.py b/src/scrape_up/codechef/codechef.py index 42c153b1..7e1ddfdb 100644 --- a/src/scrape_up/codechef/codechef.py +++ b/src/scrape_up/codechef/codechef.py @@ -8,9 +8,11 @@ class User: ```python user1 = User(id="heltion") ``` - | Methods | Details | - | --------------- | ---------------------------------------------------------------- | - | `get_profile()` | Returns name, username, profile_image_link, rating, details etc. | + | Methods | Details | + | --------------- | ------------------------------------------------------------------------- | + | `get_profile()` | Returns name, username, profile_image_link, rating, details etc. | + | `get_contests()`| Returns future_contests , past_contests , skill_tests etc in json format. | + """ def __init__(self, id): @@ -18,7 +20,7 @@ def __init__(self, id): def get_profile(self): """ - Create an object of the 'User' class\n + Create an object of the 'User' class ```python user1 = User(id="heltion") user1.get_profile() @@ -140,3 +142,49 @@ def get_profile(self): return profile_data except: return None + + def get_contests(self): + """ + get_contests output + { + "status": "success", + "message": "All contests list", + "present_contests": [], + "future_contests": [ + { + "contest_code": "START134B", + "contest_name": "Starters 134", + "contest_start_date": "15 May 2024 20:00:00", + "contest_end_date": "15 May 2024 22:00:00", + "contest_start_date_iso": "2024-05-15T20:00:00+05:30", + "contest_end_date_iso": "2024-05-15T22:00:00+05:30", + "contest_duration": "120", + "distinct_users": 0 + }, + ... + ], + "practice_contests": [], + "past_contests": [ + { + "contest_code": "START133", + "contest_name": "Starters 133 (Rated till 6-Star)", + "contest_start_date": "08 May 2024 20:00:00", + "contest_end_date": "08 May 2024 22:00:00", + "contest_start_date_iso": "2024-05-08T20:00:00+05:30", + "contest_end_date_iso": "2024-05-08T22:00:00+05:30", + "contest_duration": "120", + "distinct_users": 20041 + }, + ... + ] + } + """ + try: + url = "https://www.codechef.com/api/list/contests/all?sort_by=START&sorting_order=asc&offset=0&mode=all" + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.162 Safari/537.36" + } + response = requests.get(url, headers=headers).text + return response + except: + return None