From 0941ac8eabcff5d7d43f447f0d0e8a3d9a36e9ab Mon Sep 17 00:00:00 2001 From: Lee sang Yeop Date: Sun, 4 Feb 2024 21:14:00 +0900 Subject: [PATCH] =?UTF-8?q?fix(noticeHandler):=20=ED=95=99=EA=B3=BC=20?= =?UTF-8?q?=EA=B3=B5=EC=A7=80=EC=82=AC=ED=95=AD=20=ED=81=AC=EB=A1=A4?= =?UTF-8?q?=EB=A7=81=EC=9D=84=20=EB=B9=84=EB=8F=99=EA=B8=B0=EB=A1=9C=20?= =?UTF-8?q?=EC=B2=98=EB=A6=AC=ED=95=98=EB=8F=84=EB=A1=9D=20=EB=B3=80?= =?UTF-8?q?=EA=B2=BD=20=EB=B9=84=EB=8F=99=EA=B8=B0=EC=B2=98=EB=A6=AC=20?= =?UTF-8?q?=EC=9D=B4=EC=99=B8=EC=97=90=EB=8F=84=20=EA=B8=B0=EC=A1=B4?= =?UTF-8?q?=EC=97=90=20=ED=95=A8=EC=88=98=EB=8B=A8=EC=9C=84=EC=9D=98=20?= =?UTF-8?q?=ED=8A=B8=EB=9E=9C=EC=9E=AD=EC=85=98=EC=9D=84=20=ED=95=99?= =?UTF-8?q?=EA=B3=BC=20=EB=8B=A8=EC=9C=84=EC=9D=98=20=ED=8A=B8=EB=9E=9C?= =?UTF-8?q?=EC=9E=AD=EC=85=98=EC=9C=BC=EB=A1=9C=20=EC=B6=95=EC=86=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/db/data/noticeHandler.ts | 131 ++++++++++++++++++++++------------- 1 file changed, 83 insertions(+), 48 deletions(-) diff --git a/src/db/data/noticeHandler.ts b/src/db/data/noticeHandler.ts index 1a26e14..b6ef790 100644 --- a/src/db/data/noticeHandler.ts +++ b/src/db/data/noticeHandler.ts @@ -17,6 +17,7 @@ export interface PushNoti { interface NotiLink { link: string; + rep_yn: boolean; } export const saveDepartmentToDB = async (college: College[]): Promise => { @@ -74,79 +75,113 @@ const convertAllNoticeToNormalNotice = async ( } }; -const convertSpecificNoticeToPinnedNotice = async ( +const convertSpecificNoticePinned = async ( tableName: string, noticeLink: string, + isPinned: boolean, + connection?: PoolConnection, ): Promise => { - const query = `UPDATE ${tableName} SET rep_yn = true WHERE link = '${noticeLink}';`; + const query = `UPDATE ${tableName} SET rep_yn = ${isPinned} WHERE link = '${noticeLink}';`; try { - await db.execute(query); + if (connection) await connection.execute(query); + else await db.execute(query); } catch (error) { console.log(error.message + '고정 공지로 변경 실패'); // notificationToSlack(error.message + '\n 고정 공지로 변경 실패'); } }; -export const saveMajorNoticeToDB = async ( - connection?: PoolConnection, -): Promise => { - await convertAllNoticeToNormalNotice('major_notices', connection); +export const saveMajorNoticeToDB = async (): Promise => { + // await convertAllNoticeToNormalNotice('major_notices', connection); const query = 'SELECT * FROM departments;'; - const colleges = await selectQuery(query, connection); - - const getNotiLinkQuery = `SELECT link FROM major_notices;`; - const noticeLinksInDB = ( - await selectQuery(getNotiLinkQuery, connection) - ).map((noticeLink) => noticeLink.link); + const colleges = await selectQuery(query); - const savePromises: Promise[] = []; const newNoticeMajor: PushNoti = {}; + const failedMajor: number[] = []; - for (const college of colleges) { - console.log(college.id); - const noticeLink = await noticeCrawling(college); - const noticeLists = await noticeListCrawling(noticeLink); - - const normalNotices = noticeLists.normalNotice; - const pinnedNotices = noticeLists.pinnedNotice; - - if (normalNotices.length === 0) { - notificationToSlack(`${noticeLink} 크롤링 실패`); - continue; - } - - for (const notice of normalNotices) { - const result = await noticeContentCrawling(notice); - if (result.link === '') { - // notificationToSlack(`${notice} 콘텐츠 크롤링 실패`); - continue; + const savePromises = colleges.map(async (college) => { + const connection = await db.getConnection(); + await connection.beginTransaction(); + try { + console.log(college.id); + const noticeLink = await noticeCrawling(college); + const noticeLists = await noticeListCrawling(noticeLink); + const normalNotices = noticeLists.normalNotice; + const pinnedNotices = noticeLists.pinnedNotice; + const getNotiLinkQuery = `SELECT link, rep_yn FROM major_notices WHERE department_id = '${college.id}'`; + const noticeDataInDB = await selectQuery( + getNotiLinkQuery, + connection, + ); + const noticeLinksInDB = noticeDataInDB.map((noti) => noti.link); + const pinnedNoticeLinksInDB = noticeDataInDB + .filter((noti) => noti.rep_yn === true) + .map((noti) => noti.link); + + if (normalNotices.length === 0) { + notificationToSlack(`${noticeLink} 크롤링 실패`); + connection.release(); + return; } - if (noticeLinksInDB.includes(result.link)) continue; - if (!newNoticeMajor[college.id]) newNoticeMajor[college.id] = []; - newNoticeMajor[college.id].push(result.title); - savePromises.push(saveMajorNotice(result, college.id, false, connection)); - } - - if (pinnedNotices) { - for (const notice of pinnedNotices) { + for (const notice of normalNotices) { const result = await noticeContentCrawling(notice); if (result.link === '') { - notificationToSlack(`${notice} 콘텐츠 크롤링 실패`); + // notificationToSlack(`${notice} 콘텐츠 크롤링 실패`); continue; } - if (!noticeLinksInDB.includes(result.link)) { - savePromises.push( - saveMajorNotice(result, college.id, true, connection), + if (noticeLinksInDB.includes(result.link)) return; + if (!newNoticeMajor[college.id]) newNoticeMajor[college.id] = []; + newNoticeMajor[college.id].push(result.title); + saveMajorNotice(result, college.id, false, connection); + noticeLinksInDB.push(result.link); + } + + if (pinnedNotices) { + await pinnedNoticeLinksInDB + .filter((noti) => !pinnedNotices.includes(noti)) + .map( + async (noti) => + await convertSpecificNoticePinned( + 'major_notices', + noti, + false, + connection, + ), ); - continue; + for (const notice of pinnedNotices) { + const result = await noticeContentCrawling(notice); + if (result.link === '') { + notificationToSlack(`${notice} 콘텐츠 크롤링 실패`); + continue; + } + + if (!noticeLinksInDB.includes(result.link)) { + saveMajorNotice(result, college.id, true, connection); + continue; + } + + if (!pinnedNoticeLinksInDB.includes(result.link)) + convertSpecificNoticePinned( + 'major_notices', + result.link, + true, + connection, + ); } - convertSpecificNoticeToPinnedNotice('major_notices', result.link); } + + connection.commit(); + } catch (error) { + notificationToSlack(college.id + '크롤링 실패' + error.message); + failedMajor.push(college.id); + connection.rollback(); + } finally { + connection.release(); } - } + }); await Promise.all(savePromises); return newNoticeMajor; @@ -188,7 +223,7 @@ export const saveSchoolNoticeToDB = async (): Promise => { for (const noticeLink of pinnedNotices) { if (schoolNoticeLinksInDB.includes(noticeLink)) { - await convertSpecificNoticeToPinnedNotice('notices', noticeLink); + await convertSpecificNoticePinned('notices', noticeLink, true); continue; }