diff --git a/.dockerignore b/.dockerignore index 2eea525..0311a10 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,2 @@ -.env \ No newline at end of file +.env +!.git diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..3e0df8d --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "shared_migrations"] + path = shared_migrations + url = https://github.com/Code4GovTech/shared-models-migrations.git diff --git a/Dockerfile b/Dockerfile index 7424686..843acbc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,16 @@ FROM python:3.9-slim-buster WORKDIR /app COPY ./requirements.txt /app RUN pip install -r requirements.txt + +RUN apt-get update && \ + apt-get install -y --no-install-recommends git openssh-client && \ + rm -rf /var/lib/apt/lists/* + + COPY . . +RUN git init +RUN --mount=type=ssh git submodule update --init --recursive + ARG REPOSITORY_MONITOR_APP_PK_PEM RUN echo $REPOSITORY_MONITOR_APP_PK_PEM > /app/utils/repository_monitor_app_pk.pem diff --git a/app.py b/app.py index 836ff6e..072fdad 100644 --- a/app.py +++ b/app.py @@ -3,10 +3,13 @@ from io import BytesIO import aiohttp, asyncio import dotenv, os, json, urllib, sys, dateutil, datetime, sys + +from githubdatapipeline.issues.processor import get_url from utils.github_adapter import GithubAdapter from utils.dispatcher import dispatch_event +from utils.link_pr_issue import AddIssueId from utils.webhook_auth import verify_github_webhook -from utils.db import SupabaseInterface,PostgresORM +from shared_migrations.db.server import ServerQueries from events.ticketEventHandler import TicketEventHandler from events.ticketFeedbackHandler import TicketFeedbackHandler from githubdatapipeline.pull_request.scraper import getNewPRs @@ -22,6 +25,7 @@ from datetime import datetime from quart_cors import cors from utils.migrate_tickets import MigrateTickets +from utils.migrate_users import MigrateContributors scheduler = AsyncIOScheduler() @@ -43,7 +47,7 @@ async def get_github_data(code, discord_id): async with aiohttp.ClientSession() as session: github_response = await GithubAdapter.get_github_data(code) auth_token = (github_response)["access_token"] - + headers = { "Accept": "application/json", "Authorization": f"Bearer {auth_token}" @@ -67,15 +71,16 @@ async def get_github_data(code, discord_id): "discord_id": int(discord_id), "github_id": github_id, "github_url": f"https://github.com/{github_username}", - "email": ','.join(private_emails) + "email": ','.join(private_emails), + "joined_at": datetime.now() } return user_data - + async def comment_cleaner(): while True: await asyncio.sleep(5) - comments = await PostgresORM().readAll("app_comments") + comments = await ServerQueries().readAll("app_comments") for comment in comments: utc_now = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc) update_time = dateutil.parser.parse(comment["updated_at"]) @@ -87,7 +92,7 @@ async def comment_cleaner(): issue_id = comment["issue_id"] comment = await TicketFeedbackHandler().deleteComment(owner, repo, comment_id) print(f"Print Delete Task,{comment}", file=sys.stderr) - print(await PostgresORM().deleteComment(issue_id,"app_comments")) + print(await ServerQueries().deleteComment(issue_id,"app_comments")) async def fetch_github_issues_from_repo(owner, repo): try: @@ -96,11 +101,11 @@ async def fetch_github_issues_from_repo(owner, repo): return response else: print(f"Failed to get issues: {response}") - + except Exception as e: logger.info(e) raise Exception - + repositories_list = [ "KDwevedi/c4gt-docs", "KDwevedi/testing_for_github_app" @@ -159,7 +164,7 @@ async def verify(githubUsername): @app.route("/misc_actions") async def addIssues(): - tickets = await PostgresORM().readAll("ccbp_tickets") + tickets = await ServerQueries().readAll("ccbp_tickets") count =1 for ticket in tickets: print(f'{count}/{len(tickets)}') @@ -167,15 +172,15 @@ async def addIssues(): if ticket["status"] == "closed": # if ticket["api_endpoint_url"] in ["https://api.github.com/repos/glific/glific/issues/2824"]: await TicketEventHandler().onTicketClose({"issue":await get_url(ticket["api_endpoint_url"])}) - - return '' + + return '' @app.route("/update_profile", methods=["POST"]) async def updateGithubStats(): webhook_data = await request.json - data = await PostgresORM().read("github_profile_data", filters={"dpg_points": ("gt", 0)}) + data = await ServerQueries().read("github_profile_data", filters={"dpg_points": ("gt", 0)}) GithubProfileDisplay().update(data) return 'Done' @@ -186,7 +191,7 @@ async def do_update(): while True: print("Starting Update") await asyncio.sleep(21600) - data = await PostgresORM().read("github_profile_data", filters={"dpg_points": ("gt", 0)}) + data = await ServerQueries().read("github_profile_data", filters={"dpg_points": ("gt", 0)}) GithubProfileDisplay().update(data) @@ -216,7 +221,7 @@ async def test(): @app.route("/register/") async def register(discord_userdata): print("🛠️SUCCESSFULLY REDIECTED FROM GITHUB TO SERVER", locals(), file=sys.stderr) - postgres_client = PostgresORM() + postgres_client = ServerQueries() discord_id = discord_userdata print("🛠️SUCCESFULLY DEFINED FUNCTION TO POST TO SUPABASE", locals(), file=sys.stderr) @@ -233,7 +238,7 @@ async def register(discord_userdata): print("🛠️PUSHED USER DETAILS TO SUPABASE") except Exception as e: print("🛠️ENCOUNTERED EXCEPTION PUSHING TO SUPABASE",e, file=sys.stderr) - + print("🛠️FLOW COMPLETED SUCCESSFULLY, REDIRECTING TO DISCORD", file=sys.stderr) return await render_template('success.html'), {"Refresh": f'1; url=https://discord.com/channels/{os.getenv("DISCORD_SERVER_ID")}'} @@ -243,14 +248,14 @@ async def event_handler(): try: data = await request.json print('data is ', data) - secret_key = os.getenv("WEBHOOK_SECRET") + secret_key = os.getenv("WEBHOOK_SECRET") verification_result, error_message = await verify_github_webhook(request,secret_key) - - postgres_client = PostgresORM.get_instance() + + postgres_client = ServerQueries() event_type = request.headers.get("X-GitHub-Event") await dispatch_event(event_type, data, postgres_client) - + return data except Exception as e: logger.info(e) @@ -268,11 +273,11 @@ async def discord_metrics(): data = { "product_name" : product_name, "mentor_messages" : value['mentor_messages'], - "contributor_messages": value['contributor_messages'] + "contributor_messages": value['contributor_messages'] } discord_data.append(data) - data = await PostgresORM().add_discord_metrics(discord_data) + data = await ServerQueries().add_discord_metrics(discord_data) return data @app.route("/metrics/github", methods = ['POST']) @@ -287,16 +292,16 @@ async def github_metrics(): "closed_prs": value['closed_prs'], "open_issues": value['open_issues'], "closed_issues": value['closed_issues'], - "number_of_commits": value['number_of_commits'], + "number_of_commits": value['number_of_commits'], } github_data.append(data) - data = await PostgresORM().add_github_metrics(github_data) + data = await ServerQueries().add_github_metrics(github_data) return data @app.route("/role-master") async def get_role_master(): - role_masters = await PostgresORM().readAll("role_master") + role_masters = await ServerQueries().readAll("role_master") print('role master ', role_masters) return role_masters.data @@ -308,7 +313,7 @@ async def get_program_tickets_user(): filter = '' if request_data: filter = json.loads(request_data.decode('utf-8')) - postgres_client = PostgresORM.get_instance() + postgres_client = ServerQueries() all_issues = await postgres_client.fetch_filtered_issues(filter) print('length of all issue ', len(all_issues)) @@ -334,10 +339,10 @@ async def get_program_tickets_user(): contributors_data = issue["contributors_registration"] if contributors_data: - contributors_name = contributors_data["name"] + contributors_name = contributors_data["name"] if contributors_name: pass - else: + else: contributors_url = contributors_data["github_url"].split('/') contributors_name = contributors_url[-1] if contributors_url else None @@ -377,5 +382,32 @@ async def migrate_tickets(): print('exception occured ', e) return 'failed' + +@app.route('/migrate-contributors') +async def migrate_contributors(): + try: + migrator = MigrateContributors() # Create an instance + + asyncio.create_task(migrator.migration()) + return 'migration started' + except Exception as e: + print('exception occured ', e) + return 'failed' + + +@app.route('/add-issue-id-pr') +async def add_issue_id_pr(): + try: + migrator = AddIssueId() # Create an instance + + asyncio.create_task(migrator.process_prs()) + + # return await migrator.process_prs() + return 'migration started' + except Exception as e: + print('exception occured ', e) + return 'failed' + + if __name__ == '__main__': app.run() \ No newline at end of file diff --git a/events/ticketEventHandler.py b/events/ticketEventHandler.py index afe24e5..3c392b0 100644 --- a/events/ticketEventHandler.py +++ b/events/ticketEventHandler.py @@ -3,7 +3,7 @@ import aiohttp import os, sys, datetime, json -from utils.db import PostgresORM +from shared_migrations.db.server import ServerQueries from utils.markdown_handler import MarkdownHeaders from utils.github_api import GithubAPI from utils.jwt_generator import GenerateJWT @@ -80,8 +80,8 @@ def matchProduct(enteredProductName): async def send_message(ticket_data): - discord_channels = await PostgresORM().readAll("discord_channels") - products = await PostgresORM().readAll("product") + discord_channels = await ServerQueries().readAll("discord_channels") + products = await ServerQueries().readAll("product") url = None # for product in products: @@ -139,7 +139,7 @@ async def get_pull_request(owner, repo, number): class TicketEventHandler: def __init__(self): - self.postgres_client = PostgresORM() + self.postgres_client = ServerQueries() self.ticket_points = { "hard":30, "easy":10, @@ -255,11 +255,11 @@ async def onTicketCreate(self, eventData): repo = url_components[-3] owner = url_components[-4] try: - await PostgresORM().add_data({"issue_id":issue["id"],"updated_at": datetime.utcnow().isoformat()},"app_comments") + await self.postgres_client.add_data({"issue_id":issue["id"],"updated_at": datetime.utcnow().isoformat()},"app_comments") comment = await TicketFeedbackHandler().createComment(owner, repo, issue_number, markdown_contents) if comment: - await PostgresORM().update_data({ + await self.postgres_client.update_data({ "api_url":comment["url"], "comment_id":comment["id"], "issue_id":issue["id"], @@ -343,17 +343,17 @@ async def onTicketEdit(self, eventData): if added_contributor: print('contributors data added') - if await PostgresORM().check_record_exists("app_comments","issue_id",issue["id"]) and ticketType=="ccbp": + if await self.postgres_client.check_record_exists("app_comments","issue_id",issue["id"]) and ticketType=="ccbp": url_components = issue["url"].split('/') repo = url_components[-3] owner = url_components[-4] - comments = await PostgresORM().get_data("issue_id","app_comments",issue["id"],None) + comments = await self.postgres_client.get_data("issue_id","app_comments",issue["id"],None) comment_id = comments[0]["comment_id"] if TicketFeedbackHandler().evaluateDict(markdown_contents): comment = await TicketFeedbackHandler().updateComment(owner, repo, comment_id, markdown_contents) if comment: - await PostgresORM.get_instance().update_data({ + await self.postgres_client.update_data({ "updated_at": datetime.utcnow().isoformat(), "issue_id": issue["id"] },"issue_id","app_comments") @@ -361,7 +361,7 @@ async def onTicketEdit(self, eventData): try: comment = await TicketFeedbackHandler().deleteComment(owner, repo, comment_id) print(f"Print Delete Task,{comment}", file=sys.stderr) - print(await PostgresORM.get_instance().deleteComment(issue["id"],"app_comments")) + print(await self.postgres_client.deleteComment(issue["id"],"app_comments")) except: print("Error in deletion") elif ticketType=="ccbp": @@ -373,14 +373,14 @@ async def onTicketEdit(self, eventData): try: - await PostgresORM().add_data({ + await self.postgres_client.add_data({ "issue_id":issue["id"], "updated_at": datetime.utcnow().isoformat() },"app_comments") comment = await TicketFeedbackHandler().createComment(owner, repo, issue_number, markdown_contents) if comment: - await PostgresORM().update_data({ + await self.postgres_client.update_data({ "api_url":comment["url"], "comment_id":comment["id"], "issue_id":issue["id"], diff --git a/githubdatapipeline/issues/destination.py b/githubdatapipeline/issues/destination.py index 17b8cdf..112fcc7 100644 --- a/githubdatapipeline/issues/destination.py +++ b/githubdatapipeline/issues/destination.py @@ -1,4 +1,4 @@ -from utils.db import SupabaseInterface, PostgresORM +from shared_migrations.db.server import ServerQueries import sys def hasCommunityLabel(labels): if any([label["name"].lower() == "c4gt community" for label in labels]): @@ -7,7 +7,7 @@ def hasCommunityLabel(labels): async def recordIssue(issue): # currentTickets = SupabaseInterface.get_instance().readAll(table="community_program_tickets") - currentTickets = await PostgresORM().readAll(table="community_program_tickets") + currentTickets = await ServerQueries().readAll(table="community_program_tickets") iss = { "url": issue["url"] if issue.get("url") else None, "repository_url": issue["repository_url"] if issue.get("repository_url") else None, @@ -32,11 +32,11 @@ async def recordIssue(issue): if iss["id"] in [ticket["id"] for ticket in currentTickets]: # SupabaseInterface.get_instance().update(table="community_program_tickets", update=iss, query_key="id", query_value=iss["id"]) - await PostgresORM().update_data(data=iss, col_name="id", table="community_program_tickets") + await ServerQueries().update_data(data=iss, col_name="id", table="community_program_tickets") print("updated", file = sys.stderr) else: # SupabaseInterface.get_instance().insert(table="community_program_tickets", data=iss) - await PostgresORM().add_data(data=iss, table="community_program_tickets") + await ServerQueries().add_data(data=iss, table="community_program_tickets") print("created", file = sys.stderr) diff --git a/githubdatapipeline/pull_request/scraper.py b/githubdatapipeline/pull_request/scraper.py index 549ee56..c8cc608 100644 --- a/githubdatapipeline/pull_request/scraper.py +++ b/githubdatapipeline/pull_request/scraper.py @@ -1,5 +1,5 @@ import aiohttp, os, sys -from utils.db import PostgresORM +from shared_migrations.db.server import ServerQueries from aiographql.client import GraphQLClient, GraphQLRequest import asyncio @@ -413,7 +413,7 @@ async def get_pull_request(owner, repo, number): async def get_closed_tickets(): - tickets = await PostgresORM().readAll("ccbp_tickets") + tickets = await ServerQueries().readAll("ccbp_tickets") if tickets is None: print("No tickets found.") return [] diff --git a/handlers/issue_comment_handler.py b/handlers/issue_comment_handler.py index 488ce7c..395794c 100644 --- a/handlers/issue_comment_handler.py +++ b/handlers/issue_comment_handler.py @@ -3,8 +3,11 @@ from utils.logging_file import logger import logging from utils.user_activity import UserActivity +from shared_migrations.db.server import ServerQueries class Issue_commentHandler(EventHandler): + def __init__(self): + self.postgres_client = ServerQueries() async def handle_event(self, data, postgres_client): try: @@ -16,7 +19,7 @@ async def handle_event(self, data, postgres_client): if next((l for l in labels if l['name'].lower() == 'c4gt community'), None): handler_method = getattr(self, f'handle_issue_comment_{module_name}', None) if handler_method: - await handler_method(data, postgres_client) + await handler_method(data) await UserActivity.log_user_activity(data, 'comment') else: logging.info(f"No handler found for module: {module_name}") @@ -28,7 +31,7 @@ async def handle_event(self, data, postgres_client): logging.info(e) raise Exception - async def handle_issue_comment_created(self, data, postgres_client): + async def handle_issue_comment_created(self, data): try: #generate sample dict for ticket comment table print(f'creating comment with {data["issue"]}') @@ -52,7 +55,7 @@ async def handle_issue_comment_created(self, data, postgres_client): print('comments data ', comment_data) - save_data = await postgres_client.add_data(comment_data,"ticket_comments") + save_data = await self.postgres_client.add_data(comment_data,"ticket_comments") print('saved data in comments created ', save_data) if save_data == None: logger.info(f"{datetime.now()}--- Failed to save data in ticket_comments") @@ -62,7 +65,7 @@ async def handle_issue_comment_created(self, data, postgres_client): raise Exception - async def handle_issue_comment_edited(self, data, postgres_client): + async def handle_issue_comment_edited(self, data): try: #generate sample dict for ticket comment table print(f'editing comment with {data["issue"]}') @@ -72,7 +75,7 @@ async def handle_issue_comment_edited(self, data, postgres_client): 'updated_at':str(datetime.now()) } - save_data = await postgres_client.update_data(comment_data, "id", "ticket_comments") + save_data = await self.postgres_client.update_data(comment_data, "id", "ticket_comments") print('saved data in comments edited ', save_data) if save_data == None: logger.info(f"{datetime.now()}--- Failed to save data in ticket_comments") @@ -81,12 +84,12 @@ async def handle_issue_comment_edited(self, data, postgres_client): logger.info(f"{datetime.now()}---{e}") raise Exception - async def handle_issue_comment_deleted(self, data, postgres_client): + async def handle_issue_comment_deleted(self, data): try: print(f'deleting comment with {data["issue"]}') comment_id = data['comment']['id'] # data = await postgres_client.deleteIssueComment(comment_id) - await postgres_client.delete("ticket_comments","id", comment_id) + await self.postgres_client.delete("ticket_comments","id", comment_id) print('data in comment deleted', data) except Exception as e: print('Exception occured ', e) diff --git a/handlers/issues_handler.py b/handlers/issues_handler.py index 93b89c0..7008a51 100644 --- a/handlers/issues_handler.py +++ b/handlers/issues_handler.py @@ -2,10 +2,13 @@ import json from handlers.EventHandler import EventHandler from events.ticketEventHandler import TicketEventHandler -from utils.db import PostgresORM +from shared_migrations.db.server import ServerQueries from utils.user_activity import UserActivity class IssuesHandler(EventHandler): + def __init__(self): + self.postgres_client = ServerQueries() + async def handle_event(self, data, postgres_client): # Implement your logic for handling issue events here try: @@ -40,12 +43,12 @@ async def handle_event(self, data, postgres_client): async def handle_issue_created(self, data): # Implement your logic for handling issue events here try: - postgres_client = PostgresORM.get_instance() + if data.get("issue"): issue = data["issue"] print('inside issue created with', issue) - if await postgres_client.get_issue_from_issue_id(issue["id"]): - await postgres_client.delete("issues", "issue_id", issue["id"]) + if await self.postgres_client.get_issue_from_issue_id(issue["id"]): + await self.postgres_client.delete("issues", "issue_id", issue["id"]) await TicketEventHandler().onTicketCreate(data) except Exception as e: @@ -56,7 +59,7 @@ async def handle_issue_created(self, data): async def handle_issue_opened(self, data): # Implement your logic for handling issue events here try: - postgres_client = PostgresORM.get_instance() + if data.get("issue"): issue = data["issue"] print('inside issue opened with', issue) @@ -69,10 +72,10 @@ async def handle_issue_opened(self, data): async def handle_issue_labeled(self, data): try: print(json.dumps(data, indent=4)) - postgres_client = PostgresORM.get_instance() + issue = data["issue"] print('inside issue labeled with', issue) - db_issue = await postgres_client.get_data('id', 'issues', issue["id"]) + db_issue = await self.postgres_client.get_data('id', 'issues', issue["id"]) if not db_issue: await self.handle_issue_opened(data) labels = issue["labels"] @@ -80,7 +83,7 @@ async def handle_issue_labeled(self, data): if labels: label_names = [l['name'] for l in labels] db_issue["labels"] = label_names - await postgres_client.update_data(db_issue, 'id', 'issues') + await self.postgres_client.update_data(db_issue, 'id', 'issues') return "success" except Exception as e: @@ -90,17 +93,17 @@ async def handle_issue_labeled(self, data): async def handle_issue_unlabeled(self, data): try: - postgres_client = PostgresORM.get_instance() + if data["action"] == "unlabeled": issue = data["issue"] - db_issue = await postgres_client.get_issue_from_issue_id(issue["id"]) + db_issue = await self.postgres_client.get_issue_from_issue_id(issue["id"]) print('db issue in unlabeled is ', db_issue) if db_issue: print('inside of if for unlabeled ') - await postgres_client.delete("issue_contributors","issue_id",db_issue[0]["id"]) - await postgres_client.delete("issue_mentors","issue_id",db_issue[0]["id"]) + await self.postgres_client.delete("issue_contributors","issue_id",db_issue[0]["id"]) + await self.postgres_client.delete("issue_mentors","issue_id",db_issue[0]["id"]) # Delete Ticket - await postgres_client.delete("issues","id",db_issue[0]["id"]) + await self.postgres_client.delete("issues","id",db_issue[0]["id"]) print('issue removed') else: print('issue could not be removed') @@ -112,13 +115,13 @@ async def handle_issue_unlabeled(self, data): async def handle_issue_edited(self, data): try: - postgres_client = PostgresORM.get_instance() + print(json.dumps(data, indent=4)) issue = data["issue"] print('inside issue edited with', issue) - db_issue = await postgres_client.get_data('issue_id', 'issues', issue["id"], "*") + db_issue = await self.postgres_client.get_data('issue_id', 'issues', issue["id"], "*") if not db_issue: - await self.handle_issue_opened(data, postgres_client) + await self.handle_issue_opened(data, self.postgres_client) body = issue["body"] print(body) @@ -134,10 +137,10 @@ async def handle_issue_edited(self, data): async def handle_issue_closed(self, data): try: - postgres_client = PostgresORM.get_instance() + issue = data["issue"] print('inside issue closed with', issue) - issue_exist = await postgres_client.get_data('issue_id', 'issues', issue["id"]) + issue_exist = await self.postgres_client.get_data('issue_id', 'issues', issue["id"], "*") if issue_exist: await TicketEventHandler().onTicketClose(issue) return "success" @@ -148,7 +151,7 @@ async def handle_issue_closed(self, data): async def handle_issue_assigned(self, data): try: - postgres_client = PostgresORM.get_instance() + issue = data["issue"] print('inside issue closed with', issue) @@ -160,12 +163,12 @@ async def handle_issue_assigned(self, data): async def handle_issue_unassigned(self, data): try: - postgres_client = PostgresORM.get_instance() + issue = data["issue"] - db_issue = await postgres_client.get_issue_from_issue_id(issue["id"]) + db_issue = await self.postgres_client.get_issue_from_issue_id(issue["id"]) print('db issue in unlabeled is ', db_issue) if db_issue: - await postgres_client.delete("issue_contributors","issue_id",db_issue[0]["id"]) + await self.postgres_client.delete("issue_contributors","issue_id",db_issue[0]["id"]) return "success" except Exception as e: print('exception occured while removing an assignee to a ticket ', e) @@ -173,16 +176,16 @@ async def handle_issue_unassigned(self, data): async def log_user_activity(self, data): try: - postgres_client = PostgresORM.get_instance() + issue = data["issue"] print('inside user activity', issue) - issue = await postgres_client.get_data('issue_id', 'issues', issue["id"]) + issue = await self.postgres_client.get_data('issue_id', 'issues', issue["id"]) user_id = data['issue']['user']['id'] - contributor = await postgres_client.get_data('github_id', 'contributors_registration', user_id, '*') + contributor = await self.postgres_client.get_data('github_id', 'contributors_registration', user_id, '*') contributor_id = contributor[0]["id"] - mentor = await postgres_client.get_data('issue_id', 'issue_mentors',issue[0]["id"]) + mentor = await self.postgres_client.get_data('issue_id', 'issue_mentors',issue[0]["id"]) activity_data = { "issue_id": issue[0]["id"], "activity": f"issue_{data['action']}", @@ -191,7 +194,7 @@ async def log_user_activity(self, data): "contributor_id": contributor_id, "mentor_id": mentor[0]["angel_mentor_id"] if mentor else None } - saved_activity_data = await postgres_client.add_data(activity_data,"user_activity") + saved_activity_data = await self.postgres_client.add_data(activity_data,"user_activity") return saved_activity_data except Exception as e: diff --git a/handlers/pull_request_handler.py b/handlers/pull_request_handler.py index a33959b..3c6b559 100644 --- a/handlers/pull_request_handler.py +++ b/handlers/pull_request_handler.py @@ -1,17 +1,54 @@ -import logging +import logging, httpx, os, re, aiohttp from handlers.EventHandler import EventHandler from datetime import datetime from utils.user_activity import UserActivity +from shared_migrations.db.server import ServerQueries class Pull_requestHandler(EventHandler): def convert_to_datetime(self, date_str): return datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%SZ') + + def extract_issue_number(self, title): + match = re.search(r'#(\d+)', title) + if match: + return int(match.group(1)) + return None + + async def get_issue_data(self, owner, repo, issue_number): + try: + GITHUB_TOKEN = os.getenv('GITHUB_TOKEN') + headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {GITHUB_TOKEN}", + "X-GitHub-Api-Version": "2022-11-28" + } + + GITHUB_ISSUE_URL = "https://api.github.com/repos/{owner}/{repo}/issues/{issue_number}" + + description_url = GITHUB_ISSUE_URL.format( + owner=owner, repo=repo, issue_number=issue_number) + async with httpx.AsyncClient() as client: + issue_response = await client.get(description_url, headers=headers) + if issue_response.status_code == 200: + + issue_details = issue_response.json() + issue_id = issue_details.id + return issue_id + + return None + except Exception as e: + print('Exception occured while getting issue data ', e) + return None + - async def handle_event(self, data, postgres_client): + async def handle_event(self, data, dummy_ps_client): # Implement your logic for handling issue events here try: print('inside pull request handler ', data) + + postgres_client = ServerQueries() + merged_by = data['pull_request']['merged_by']['id'] if data['pull_request']['merged_by'] else None merged_at = data['pull_request']['merged_at'] merged_by_username = data['pull_request']['merged_by']['login'] if data['pull_request']['merged_by'] else None @@ -19,6 +56,23 @@ async def handle_event(self, data, postgres_client): raised_at = self.convert_to_datetime(data['pull_request']['updated_at']) if merged_at: merged_at = self.convert_to_datetime(merged_at) + + api_url = data['pull_request']["url"] + + issue_id = None + + async with aiohttp.ClientSession() as session: + async with session.get(api_url) as response: + pr_data = await response.json() + if pr_data: + pr_title = pr_data["title"] + issue_number = self.extract_issue_number(pr_title) + if issue_number: + url_parts = api_url.split('/') + owner = url_parts[4] + repo = url_parts[5] + issue_id = await self.get_issue_data(owner, repo, issue_number) + pr_data = { "created_at": created_at, "api_url":data['pull_request']['url'], @@ -33,6 +87,8 @@ async def handle_event(self, data, postgres_client): "merged_by_username": merged_by_username, "pr_id": data['pull_request']['id'], "ticket_url": data['pull_request']['issue_url'], + "title": data['pull_request']['title'], + "issue_id": issue_id if issue_id else None, "ticket_complexity": None } diff --git a/models/models.py b/models/models.py index 06f5e83..7eecee3 100644 --- a/models/models.py +++ b/models/models.py @@ -228,6 +228,9 @@ class ContributorsDiscord(Base): field_name = Column(Text, nullable=True, name='name') # Adjusted field name chapter = Column(Text, nullable=True, comment="the chapter they're associated with") gender = Column(Text, nullable=True) + country = Column(Text, nullable=True) + city = Column(Text, nullable=True) + experience = Column(Text, nullable=True) is_active = Column(Boolean, nullable=False) def __repr__(self): @@ -1090,6 +1093,8 @@ class PrHistory(Base): pr_id = Column(BigInteger, nullable=False) ticket_url = Column(Text, nullable=False) ticket_complexity = Column(Text, nullable=True) + title = Column(Text, nullable=True) + issue_id = Column(BigInteger, nullable=True) def __repr__(self): return f"" @@ -1110,7 +1115,9 @@ def to_dict(self): 'merged_by_username': self.merged_by_username, 'pr_id': self.pr_id, 'ticket_url': self.ticket_url, - 'ticket_complexity': self.ticket_complexity + 'ticket_complexity': self.ticket_complexity, + 'title': self.title, + 'issue_id': self.issue_id } class PrStaging(Base): diff --git a/shared_migrations b/shared_migrations new file mode 160000 index 0000000..ec465a2 --- /dev/null +++ b/shared_migrations @@ -0,0 +1 @@ +Subproject commit ec465a29357edf78b2727d31e50a6f6d3b4d57dc diff --git a/utils/link_pr_issue.py b/utils/link_pr_issue.py new file mode 100644 index 0000000..f1702bb --- /dev/null +++ b/utils/link_pr_issue.py @@ -0,0 +1,88 @@ +import aiohttp +import os, re, httpx, time +from datetime import datetime +from shared_migrations.db.server import ServerQueries + +class AddIssueId: + def __init__(self): + self.postgres_client = ServerQueries() + return + + def convert_to_datetime(self, date_str): + return datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%SZ') + + def extract_issue_number(self, title): + match = re.search(r'#(\d+)', title) + if match: + return int(match.group(1)) + return None + + async def get_issue_data(self, owner, repo, issue_number): + try: + GITHUB_TOKEN = os.getenv('API_TOKEN') + headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {GITHUB_TOKEN}", + "X-GitHub-Api-Version": "2022-11-28" + } + + GITHUB_ISSUE_URL = "https://api.github.com/repos/{owner}/{repo}/issues/{issue_number}" + + description_url = GITHUB_ISSUE_URL.format( + owner=owner, repo=repo, issue_number=issue_number) + async with httpx.AsyncClient() as client: + issue_response = await client.get(description_url, headers=headers) + if issue_response.status_code == 200: + + issue_details = issue_response.json() + issue_id = issue_details["id"] + return issue_id + + return None + except Exception as e: + print('Exception occured while getting issue data ', e) + return None + + async def process_prs(self): + try: + all_prs = await self.postgres_client.readAll('pr_history') + print('length of all prs is ', len(all_prs)) + + GITHUB_TOKEN = os.getenv('API_TOKEN') + headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {GITHUB_TOKEN}", + "X-GitHub-Api-Version": "2022-11-28" + } + + for pr in all_prs: + time.sleep(2) + print('processing pr id #', pr["id"]) + api_url = pr["api_url"] + + pr_data = None + async with aiohttp.ClientSession() as session: + async with session.get(api_url, headers=headers) as response: + if response.status == 200: + pr_data = await response.json() + if pr_data: + pr_title = pr_data["title"] + issue_number = self.extract_issue_number(pr_title) + if issue_number: + url_parts = api_url.split('/') + owner = url_parts[4] + repo = url_parts[5] + issue_id = await self.get_issue_data(owner, repo, issue_number) + + if issue_id: + pr["issue_id"] = issue_id + pr["title"] = pr_title + pr["created_at"] = pr["created_at"].replace(tzinfo=None) + pr["raised_at"] = pr["raised_at"].replace(tzinfo=None) + await self.postgres_client.update_data(pr, 'id', 'pr_history') + + return 'updated' + + except Exception as e: + print('exception occured while processing PRs ', e) + return 'failed' \ No newline at end of file diff --git a/utils/migrate_tickets.py b/utils/migrate_tickets.py index 0b925e0..7dbd1e2 100644 --- a/utils/migrate_tickets.py +++ b/utils/migrate_tickets.py @@ -1,11 +1,11 @@ import logging -from utils.db import PostgresORM +from shared_migrations.db.server import ServerQueries import aiohttp from datetime import datetime class MigrateTickets: def __init__(self): - self.postgres_client = PostgresORM.get_instance() + self.postgres_client = ServerQueries() return diff --git a/utils/migrate_users.py b/utils/migrate_users.py new file mode 100644 index 0000000..5f548a9 --- /dev/null +++ b/utils/migrate_users.py @@ -0,0 +1,113 @@ +from utils.db import PostgresORM, SupabaseInterface +from datetime import datetime +from shared_migrations.db.server import ServerQueries + +class MigrateContributors: + def __init__(self): + self.postgres_client = ServerQueries() + self.supabase_client = SupabaseInterface.get_instance() + return + + async def migration(self): + try: + #migrate contributors discord + await self.migrate_contributors_discord() + + #migrate contributors registraiton + await self.migrate_contributors_registration() + + return 'success' + + except Exception as e: + print('Exception occured while migrating users ', e) + return 'failed' + + def convert_to_datetime(self, date_str): + dt = datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%S.%f%z') + return dt.replace(tzinfo=None) + + async def migrate_contributors_discord(self): + try: + supabase_contributors_discord = self.supabase_client.readAll("contributors_discord") + postgres_contributors_discord = await self.postgres_client.readAll("contributors_discord") + print('length of supabase_contributors_discord ', len(supabase_contributors_discord)) + print('length of postgres_contributors_discord ', len(postgres_contributors_discord)) + + if len(supabase_contributors_discord) == len(postgres_contributors_discord): + print('skipping migration to contributors discord as both lens are equal') + return + + for contributor in supabase_contributors_discord: + # Ensure the correct key is used to compare 'discord_id' values. + discord_id = contributor['discord_id'] + present = next((pcd for pcd in postgres_contributors_discord if pcd['discord_id'] == discord_id), None) + + if present: + continue + else: + #insert the record in postgres + # format = '' + print('inserting contributors discord for ', contributor['discord_id']) + + contributors_data = { + 'github_url': contributor['github_url'], + 'discord_id': contributor['discord_id'], + 'discord_username': contributor['discord_username'], + 'joined_at': self.convert_to_datetime(contributor['joined_at']), + 'email': contributor['email'], + 'chapter': contributor['chapter'], + 'gender': contributor['gender'], + 'is_active': contributor['is_active'], + 'country': contributor['country'], + 'city': contributor['city'], + 'experience': contributor['experience'], + 'field_name': contributor['name'] + } + await self.postgres_client.add_data(contributors_data, "contributors_discord") + + return 'success' + + except Exception as e: + print('Exception occured while migrate contributors discord ', e) + return 'failed' + + + + async def migrate_contributors_registration(self): + try: + supabase_contributors_registration = self.supabase_client.readAll("contributors_registration") + postgres_contributors_registration = await self.postgres_client.readAll("contributors_registration") + + print('length of supabase_contributors_registration ', len(supabase_contributors_registration)) + print('length of postgres_contributors_registration ', len(postgres_contributors_registration)) + + if len(supabase_contributors_registration) == len(postgres_contributors_registration): + print('skipping migration to contributors discord as both lens are equal') + return + + for contributor in supabase_contributors_registration: + # Ensure the correct key is used to compare 'discord_id' values. + discord_id = contributor['discord_id'] + present = next((pcd for pcd in postgres_contributors_registration if pcd['discord_id'] == discord_id), None) + + if present: + continue + else: + #insert the record in postgres + print('inserting contributors discord for ', contributor['discord_id']) + contributors_data = { + 'discord_id': contributor['discord_id'], + 'github_id': contributor['github_id'], + 'github_url': contributor['github_url'], + 'discord_username': contributor['discord_username'], + 'joined_at': self.convert_to_datetime(contributor['joined_at']), + 'email': contributor['email'], + 'name': contributor['name'] + } + await self.postgres_client.add_data(contributors_data, "contributors_registration") + + return 'success' + + except Exception as e: + print('Exception occured while migrate contributors discord ', e) + return 'failed' \ No newline at end of file diff --git a/utils/user_activity.py b/utils/user_activity.py index bdc5907..3f463ac 100644 --- a/utils/user_activity.py +++ b/utils/user_activity.py @@ -1,10 +1,11 @@ import logging -from utils.db import PostgresORM +from shared_migrations.db.server import ServerQueries class UserActivity: - async def log_user_activity(data, activity): + + async def log_user_activity(self, data, activity): try: - postgres_client = PostgresORM.get_instance() + postgres_client = ServerQueries() issue = data["issue"] print('inside user activity', issue) issue = await postgres_client.get_data('issue_id', 'issues', issue["id"])