From 8d7179a47a886abe0281e2c29dfb8458551df9ae Mon Sep 17 00:00:00 2001 From: Marina Date: Fri, 9 Jun 2023 13:30:31 +0300 Subject: [PATCH 01/12] first try of check_if_github with mistake --- app/main/check_packs/pack_config.py | 3 +- app/main/checks/__init__.py | 3 +- .../checks/presentation_checks/__init__.py | 1 + .../checks/presentation_checks/if_github.py | 46 +++++++++++++++++++ requirements.txt | 6 ++- 5 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 app/main/checks/presentation_checks/if_github.py diff --git a/app/main/check_packs/pack_config.py b/app/main/check_packs/pack_config.py index cf0b85b2..0b46f3b5 100644 --- a/app/main/check_packs/pack_config.py +++ b/app/main/check_packs/pack_config.py @@ -14,7 +14,8 @@ ['pres_right_words'], ['pres_image_share'], ['future_dev'], - ['pres_banned_words_check'] + ['pres_banned_words_check'], + ['if_github'], ] BASE_REPORT_CRITERION = [ ["simple_check"], diff --git a/app/main/checks/__init__.py b/app/main/checks/__init__.py index 8f65c507..8807d3c8 100644 --- a/app/main/checks/__init__.py +++ b/app/main/checks/__init__.py @@ -17,7 +17,8 @@ PresRightWordsCheck.id: PresRightWordsCheck, PresImageShareCheck.id: PresImageShareCheck, FurtherDev.id: FurtherDev, - PresBannedWordsCheck.id: PresBannedWordsCheck + PresBannedWordsCheck.id: PresBannedWordsCheck, + PresIfRefGitHubWork.id: PresIfRefGitHubWork }, 'report': { ReportSimpleCheck.id: ReportSimpleCheck, diff --git a/app/main/checks/presentation_checks/__init__.py b/app/main/checks/presentation_checks/__init__.py index 81ec9cbd..62fd6c30 100644 --- a/app/main/checks/presentation_checks/__init__.py +++ b/app/main/checks/presentation_checks/__init__.py @@ -10,3 +10,4 @@ from .pres_right_words import PresRightWordsCheck from .image_share import PresImageShareCheck from .banned_words import PresBannedWordsCheck +from .if_github import PresIfRefGitHubWork diff --git a/app/main/checks/presentation_checks/if_github.py b/app/main/checks/presentation_checks/if_github.py new file mode 100644 index 00000000..631f9df2 --- /dev/null +++ b/app/main/checks/presentation_checks/if_github.py @@ -0,0 +1,46 @@ +import re + +import lxml +import requests +from lxml import html + +from ..base_check import BasePresCriterion, answer +from app.utils.parse_for_html import format_header + +class PresIfRefGitHubWork(BasePresCriterion): + description = "Проверка действительности ссылки на github" + id = 'if_github' + + def check(self): + wrong_repo_ref = [] + empty_repo_ref = [] + string_from_text = '' + text_from_slide = [] + string_result = 'Непройдена! ' + for page, slide in enumerate(self.file.get_text_from_slides(), 1): + text_from_slide.append(slide) + string_from_text = ''.join(text_from_slide) + result = re.findall(r'(((((http(s)?://)?(github|gitlab|bitbucket)+)+(.com|.org)+)+/\w+)+/\w+)+', string_from_text) + if not result: + return answer(True, 'Нечего проверять!') + else: + for i in result: + try: + page = requests.get(i[0]) + if page.status_code != 200: + raise requests.exceptions.ConnectionError + tree = lxml.html.fromstring(page.content) + file_and_folders = tree.xpath('//*[@class="js-navigation-open"]') + if not file_and_folders: + empty_repo_ref.append(i[0]) + except (requests.exceptions.SSLError, requests.exceptions.ConnectionError): + wrong_repo_ref.append(i[0]) + if wrong_repo_ref: + string_result += f"Найдены некорректные ссылки на репозитории! {wrong_repo_ref}" + if empty_repo_ref: + string_result += f", Также Найдены пустые репозитории! {empty_repo_ref}" + elif empty_repo_ref: + string_result += f"Найдены пустые репозитории! {empty_repo_ref}" + else: + string_result = 'Пройдена!' + return answer(True, string_result) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 9acf11fc..d689642e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ werkzeug==2.0.0 Flask==2.0.3 jinja2==3.0.0 -requests==2.24.0 +requests~=2.31.0 python-pptx==0.6.18 odfpy==1.4.1 pymongo==3.11.1 @@ -25,3 +25,7 @@ docx2python~=2.0.4 oauthlib~=3.1.0 pdfplumber==0.6.1 pytest~=7.1.2 + +lxml~=4.9.2 +urllib3~=2.0.3 +beautifulsoup4~=4.12.2 \ No newline at end of file From c27b2dc5573c6676461153a709575adf27dacb3d Mon Sep 17 00:00:00 2001 From: Marina Date: Sun, 11 Jun 2023 16:49:22 +0300 Subject: [PATCH 02/12] try to fix parsing --- .../checks/presentation_checks/if_github.py | 39 +++++++++++-------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/app/main/checks/presentation_checks/if_github.py b/app/main/checks/presentation_checks/if_github.py index 631f9df2..60ee3d15 100644 --- a/app/main/checks/presentation_checks/if_github.py +++ b/app/main/checks/presentation_checks/if_github.py @@ -1,11 +1,12 @@ import re -import lxml -import requests from lxml import html +import requests + from ..base_check import BasePresCriterion, answer -from app.utils.parse_for_html import format_header +# from app.utils.parse_for_html import format_header + class PresIfRefGitHubWork(BasePresCriterion): description = "Проверка действительности ссылки на github" @@ -20,27 +21,33 @@ def check(self): for page, slide in enumerate(self.file.get_text_from_slides(), 1): text_from_slide.append(slide) string_from_text = ''.join(text_from_slide) - result = re.findall(r'(((((http(s)?://)?(github|gitlab|bitbucket)+)+(.com|.org)+)+/\w+)+/\w+)+', string_from_text) + result = re.findall( + r'((((((http(s)?://)?(github|gitlab|bitbucket)+)+(.com|.org)+)+/[a-zA-Z0-9_-]+)+/[a-zA-Z0-9_-]+)+/*)+', + string_from_text) if not result: return answer(True, 'Нечего проверять!') else: for i in result: try: - page = requests.get(i[0]) - if page.status_code != 200: + link = requests.get(i[0]) + if link.status_code != 200: raise requests.exceptions.ConnectionError - tree = lxml.html.fromstring(page.content) - file_and_folders = tree.xpath('//*[@class="js-navigation-open"]') - if not file_and_folders: + tree = html.fromstring(link.content) + if not tree.xpath("//a[contains(concat(' ',normalize-space(@class),' '),' tree-item-link ')][contains(@href,'/tree/master/')]/text()"): + # @ class ='js-navigation-open Link--primary' or empty_repo_ref.append(i[0]) except (requests.exceptions.SSLError, requests.exceptions.ConnectionError): wrong_repo_ref.append(i[0]) - if wrong_repo_ref: - string_result += f"Найдены некорректные ссылки на репозитории! {wrong_repo_ref}" - if empty_repo_ref: - string_result += f", Также Найдены пустые репозитории! {empty_repo_ref}" - elif empty_repo_ref: - string_result += f"Найдены пустые репозитории! {empty_repo_ref}" + if wrong_repo_ref and not empty_repo_ref: + string_result += f"Найдены некорректные ссылки на репозитории! {wrong_repo_ref} ВСЕГО {result}" + check_result = False + elif empty_repo_ref and not wrong_repo_ref: + string_result += f"Найдены пустые репозитории! {empty_repo_ref} ВСЕГО {result}" + check_result = False + elif empty_repo_ref and wrong_repo_ref: + string_result += f"Найдены пустые репозитории! {empty_repo_ref} Также Найдены некорректные репозитории! {wrong_repo_ref} ВСЕГО {result}" + check_result = False else: string_result = 'Пройдена!' - return answer(True, string_result) \ No newline at end of file + check_result = True + return answer(check_result, string_result) From f2542dc56c707c77ba296e2f4fddfc95f66b94fe Mon Sep 17 00:00:00 2001 From: Marina Date: Tue, 13 Jun 2023 15:54:50 +0300 Subject: [PATCH 03/12] add check for empty repo GitHub --- app/main/checks/presentation_checks/if_github.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/app/main/checks/presentation_checks/if_github.py b/app/main/checks/presentation_checks/if_github.py index 60ee3d15..819278ff 100644 --- a/app/main/checks/presentation_checks/if_github.py +++ b/app/main/checks/presentation_checks/if_github.py @@ -17,7 +17,7 @@ def check(self): empty_repo_ref = [] string_from_text = '' text_from_slide = [] - string_result = 'Непройдена! ' + string_result = 'Не пройдена! ' for page, slide in enumerate(self.file.get_text_from_slides(), 1): text_from_slide.append(slide) string_from_text = ''.join(text_from_slide) @@ -33,19 +33,18 @@ def check(self): if link.status_code != 200: raise requests.exceptions.ConnectionError tree = html.fromstring(link.content) - if not tree.xpath("//a[contains(concat(' ',normalize-space(@class),' '),' tree-item-link ')][contains(@href,'/tree/master/')]/text()"): - # @ class ='js-navigation-open Link--primary' or + if not tree.xpath("//a[@class ='js-navigation-open Link--primary']"): empty_repo_ref.append(i[0]) except (requests.exceptions.SSLError, requests.exceptions.ConnectionError): wrong_repo_ref.append(i[0]) if wrong_repo_ref and not empty_repo_ref: - string_result += f"Найдены некорректные ссылки на репозитории! {wrong_repo_ref} ВСЕГО {result}" + string_result += f"Найдены некорректные ссылки на репозитории: {wrong_repo_ref} " check_result = False elif empty_repo_ref and not wrong_repo_ref: - string_result += f"Найдены пустые репозитории! {empty_repo_ref} ВСЕГО {result}" + string_result += f"Найдены пустые репозитории: {empty_repo_ref} " check_result = False elif empty_repo_ref and wrong_repo_ref: - string_result += f"Найдены пустые репозитории! {empty_repo_ref} Также Найдены некорректные репозитории! {wrong_repo_ref} ВСЕГО {result}" + string_result += f"Найдены пустые репозитории: {empty_repo_ref} Также Найдены некорректные репозитории: {wrong_repo_ref}" check_result = False else: string_result = 'Пройдена!' From 177fedbdd25d1f92d74e29c246e596be13e0ef85 Mon Sep 17 00:00:00 2001 From: Marina Date: Thu, 15 Jun 2023 21:23:48 +0300 Subject: [PATCH 04/12] fix space-problem --- .../presentation_checks/find_def_sld.py | 9 ++- .../checks/presentation_checks/if_github.py | 77 ++++++++++++++----- 2 files changed, 62 insertions(+), 24 deletions(-) diff --git a/app/main/checks/presentation_checks/find_def_sld.py b/app/main/checks/presentation_checks/find_def_sld.py index 46c64b0f..33a66ff5 100644 --- a/app/main/checks/presentation_checks/find_def_sld.py +++ b/app/main/checks/presentation_checks/find_def_sld.py @@ -8,18 +8,19 @@ class FindDefSld(BasePresCriterion): def __init__(self, file_info, key_slide): super().__init__(file_info) self.type_of_slide = key_slide + self.found_idxs = [] def check(self): - found_slides, found_idxs = [], [] + found_slides = [] for i, title in enumerate(self.file.get_titles(), 1): if str(title).lower().find(str(self.type_of_slide).lower()) != -1: found_slides.append(self.file.get_text_from_slides()[i - 1]) - found_idxs.append(i) + self.found_idxs.append(i) if len(found_slides) == 0: return answer(False, 'Слайд не найден') else: - found_idxs = self.format_page_link(found_idxs) - return answer(True, 'Найден под номером: {}'.format(', '.join(map(str, found_idxs)))) + found_idxs_link = self.format_page_link(self.found_idxs) + return answer(True, 'Найден под номером: {}'.format(', '.join(map(str, found_idxs_link)))) @property def name(self): diff --git a/app/main/checks/presentation_checks/if_github.py b/app/main/checks/presentation_checks/if_github.py index 819278ff..e88cc452 100644 --- a/app/main/checks/presentation_checks/if_github.py +++ b/app/main/checks/presentation_checks/if_github.py @@ -1,8 +1,11 @@ import re +import os +import subprocess from lxml import html import requests +from .find_def_sld import FindDefSld from ..base_check import BasePresCriterion, answer # from app.utils.parse_for_html import format_header @@ -12,41 +15,75 @@ class PresIfRefGitHubWork(BasePresCriterion): description = "Проверка действительности ссылки на github" id = 'if_github' + def __init__(self, file_info): + super().__init__(file_info) + self.check_apr = FindDefSld(file_info=file_info, key_slide="Апробация") + + + def check(self): wrong_repo_ref = [] empty_repo_ref = [] string_from_text = '' text_from_slide = [] + text_from_slide_apr = [] string_result = 'Не пройдена! ' + string_from_text_aprob = '' + page_ap = '' + + found_slides_result = self.check_apr.check() + found_slides_result_2 = self.check_apr.__getattribute__("found_idxs") + for item in found_slides_result_2: + page_ap += str(item) + for page, slide in enumerate(self.file.get_text_from_slides(), 1): + slide.replace(" ", '') text_from_slide.append(slide) - string_from_text = ''.join(text_from_slide) - result = re.findall( - r'((((((http(s)?://)?(github|gitlab|bitbucket)+)+(.com|.org)+)+/[a-zA-Z0-9_-]+)+/[a-zA-Z0-9_-]+)+/*)+', - string_from_text) + string_from_text = ' '.join(text_from_slide) + + if str(page) == page_ap: + text_from_slide_apr.append(slide.replace(" ", '')) + string_from_text_aprob = ' '.join(text_from_slide_apr) + + result = re.findall(r'((((((http(s)?://)?(github|gitlab|bitbucket)+)+(.com|.org)+)+/[a-zA-Z0-9_-]+)+/[a-zA-Z0-9_-]+)+/*)+', string_from_text) + result_2 = re.findall( + r'\(github\.com\)|\(gitlab\.com\)|\(bitbucket\.org\)|\(github\)|\(gitlab\)|\(bitbucket\)', + string_from_text_aprob) + + # return answer(True, result, result_2) if not result: return answer(True, 'Нечего проверять!') else: + if result_2: + string_result += f" Вместо выражений {result_2} следует привести ссылки вида 'https//github.com/...'" for i in result: try: link = requests.get(i[0]) if link.status_code != 200: raise requests.exceptions.ConnectionError - tree = html.fromstring(link.content) - if not tree.xpath("//a[@class ='js-navigation-open Link--primary']"): - empty_repo_ref.append(i[0]) + git_link = i[0]+'.git' + local_dir_name = '~/src/2/git_l' + subprocess.run(["git", "clone", git_link, local_dir_name]) + if os.path.exists(local_dir_name): + contents = os.listdir(local_dir_name) + if len(contents) == 0: + empty_repo_ref.append(i[0]) + + # tree = html.fromstring(link.content) + # if not tree.xpath("//a[@class ='js-navigation-open Link--primary']"): + # empty_repo_ref.append(i[0]) except (requests.exceptions.SSLError, requests.exceptions.ConnectionError): wrong_repo_ref.append(i[0]) - if wrong_repo_ref and not empty_repo_ref: - string_result += f"Найдены некорректные ссылки на репозитории: {wrong_repo_ref} " - check_result = False - elif empty_repo_ref and not wrong_repo_ref: - string_result += f"Найдены пустые репозитории: {empty_repo_ref} " - check_result = False - elif empty_repo_ref and wrong_repo_ref: - string_result += f"Найдены пустые репозитории: {empty_repo_ref} Также Найдены некорректные репозитории: {wrong_repo_ref}" - check_result = False - else: - string_result = 'Пройдена!' - check_result = True - return answer(check_result, string_result) + if wrong_repo_ref and not empty_repo_ref: + string_result += f"Найдены нерабочие ссылки на репозитории: {wrong_repo_ref}" + check_result = False + elif empty_repo_ref and not wrong_repo_ref: + string_result += f"Найдены пустые репозитории: {empty_repo_ref}" + check_result = False + elif empty_repo_ref and wrong_repo_ref: + string_result += f"Найдены пустые репозитории: {empty_repo_ref} Также найдены нерабочие репозитории: {wrong_repo_ref}" + check_result = False + else: + string_result = 'Пройдена!' + check_result = True + return answer(check_result, string_result) From eb82e1a6f440d969afdd9f1f867801c88d61f158 Mon Sep 17 00:00:00 2001 From: Marina Date: Sat, 17 Jun 2023 21:10:46 +0300 Subject: [PATCH 05/12] full check git-links 1.0 --- app/main/check_packs/pack_config.py | 2 +- app/main/checks/__init__.py | 2 +- .../checks/presentation_checks/__init__.py | 2 +- .../checks/presentation_checks/if_github.py | 89 ------------------ .../presentation_checks/verify_git_link.py | 90 +++++++++++++++++++ requirements.txt | 1 - 6 files changed, 93 insertions(+), 93 deletions(-) delete mode 100644 app/main/checks/presentation_checks/if_github.py create mode 100644 app/main/checks/presentation_checks/verify_git_link.py diff --git a/app/main/check_packs/pack_config.py b/app/main/check_packs/pack_config.py index 0b46f3b5..370438e6 100644 --- a/app/main/check_packs/pack_config.py +++ b/app/main/check_packs/pack_config.py @@ -15,7 +15,7 @@ ['pres_image_share'], ['future_dev'], ['pres_banned_words_check'], - ['if_github'], + ['verify_git_link'], ] BASE_REPORT_CRITERION = [ ["simple_check"], diff --git a/app/main/checks/__init__.py b/app/main/checks/__init__.py index 8807d3c8..9b73523e 100644 --- a/app/main/checks/__init__.py +++ b/app/main/checks/__init__.py @@ -18,7 +18,7 @@ PresImageShareCheck.id: PresImageShareCheck, FurtherDev.id: FurtherDev, PresBannedWordsCheck.id: PresBannedWordsCheck, - PresIfRefGitHubWork.id: PresIfRefGitHubWork + PresVerifyGitLinkCheck.id: PresVerifyGitLinkCheck }, 'report': { ReportSimpleCheck.id: ReportSimpleCheck, diff --git a/app/main/checks/presentation_checks/__init__.py b/app/main/checks/presentation_checks/__init__.py index 62fd6c30..cf365e18 100644 --- a/app/main/checks/presentation_checks/__init__.py +++ b/app/main/checks/presentation_checks/__init__.py @@ -10,4 +10,4 @@ from .pres_right_words import PresRightWordsCheck from .image_share import PresImageShareCheck from .banned_words import PresBannedWordsCheck -from .if_github import PresIfRefGitHubWork +from .verify_git_link import PresVerifyGitLinkCheck diff --git a/app/main/checks/presentation_checks/if_github.py b/app/main/checks/presentation_checks/if_github.py deleted file mode 100644 index e88cc452..00000000 --- a/app/main/checks/presentation_checks/if_github.py +++ /dev/null @@ -1,89 +0,0 @@ -import re -import os -import subprocess - -from lxml import html -import requests - -from .find_def_sld import FindDefSld - -from ..base_check import BasePresCriterion, answer -# from app.utils.parse_for_html import format_header - - -class PresIfRefGitHubWork(BasePresCriterion): - description = "Проверка действительности ссылки на github" - id = 'if_github' - - def __init__(self, file_info): - super().__init__(file_info) - self.check_apr = FindDefSld(file_info=file_info, key_slide="Апробация") - - - - def check(self): - wrong_repo_ref = [] - empty_repo_ref = [] - string_from_text = '' - text_from_slide = [] - text_from_slide_apr = [] - string_result = 'Не пройдена! ' - string_from_text_aprob = '' - page_ap = '' - - found_slides_result = self.check_apr.check() - found_slides_result_2 = self.check_apr.__getattribute__("found_idxs") - for item in found_slides_result_2: - page_ap += str(item) - - for page, slide in enumerate(self.file.get_text_from_slides(), 1): - slide.replace(" ", '') - text_from_slide.append(slide) - string_from_text = ' '.join(text_from_slide) - - if str(page) == page_ap: - text_from_slide_apr.append(slide.replace(" ", '')) - string_from_text_aprob = ' '.join(text_from_slide_apr) - - result = re.findall(r'((((((http(s)?://)?(github|gitlab|bitbucket)+)+(.com|.org)+)+/[a-zA-Z0-9_-]+)+/[a-zA-Z0-9_-]+)+/*)+', string_from_text) - result_2 = re.findall( - r'\(github\.com\)|\(gitlab\.com\)|\(bitbucket\.org\)|\(github\)|\(gitlab\)|\(bitbucket\)', - string_from_text_aprob) - - # return answer(True, result, result_2) - if not result: - return answer(True, 'Нечего проверять!') - else: - if result_2: - string_result += f" Вместо выражений {result_2} следует привести ссылки вида 'https//github.com/...'" - for i in result: - try: - link = requests.get(i[0]) - if link.status_code != 200: - raise requests.exceptions.ConnectionError - git_link = i[0]+'.git' - local_dir_name = '~/src/2/git_l' - subprocess.run(["git", "clone", git_link, local_dir_name]) - if os.path.exists(local_dir_name): - contents = os.listdir(local_dir_name) - if len(contents) == 0: - empty_repo_ref.append(i[0]) - - # tree = html.fromstring(link.content) - # if not tree.xpath("//a[@class ='js-navigation-open Link--primary']"): - # empty_repo_ref.append(i[0]) - except (requests.exceptions.SSLError, requests.exceptions.ConnectionError): - wrong_repo_ref.append(i[0]) - if wrong_repo_ref and not empty_repo_ref: - string_result += f"Найдены нерабочие ссылки на репозитории: {wrong_repo_ref}" - check_result = False - elif empty_repo_ref and not wrong_repo_ref: - string_result += f"Найдены пустые репозитории: {empty_repo_ref}" - check_result = False - elif empty_repo_ref and wrong_repo_ref: - string_result += f"Найдены пустые репозитории: {empty_repo_ref} Также найдены нерабочие репозитории: {wrong_repo_ref}" - check_result = False - else: - string_result = 'Пройдена!' - check_result = True - return answer(check_result, string_result) diff --git a/app/main/checks/presentation_checks/verify_git_link.py b/app/main/checks/presentation_checks/verify_git_link.py new file mode 100644 index 00000000..4b082233 --- /dev/null +++ b/app/main/checks/presentation_checks/verify_git_link.py @@ -0,0 +1,90 @@ + +import re +import requests +from lxml import html +from urllib.parse import quote + +from .find_def_sld import FindDefSld +from ..base_check import BasePresCriterion, answer + +# for check if gitlab-repository is closed: +GITLAB_URL = 'https://gitlab.com/api/v4' +PRIVATE_TOKEN = 'glpat-JeZApxShRgB1nsGrMsst' + + +class PresVerifyGitLinkCheck(BasePresCriterion): + description = "Проверка действительности ссылки на github" + id = 'verify_git_link' + + def __init__(self, file_info): + super().__init__(file_info) + self.check_aprb = FindDefSld(file_info=file_info, key_slide="Апробация") + + def check(self): + wrong_repo_ref = [] + empty_repo_ref = [] + string_result = 'Не пройдена!' + + self.check_aprb.check() + page_aprb = ''.join((str(item) for item in self.check_aprb.__getattribute__("found_idxs"))) + + text_from_slide = [slide for page, slide in enumerate(self.file.get_text_from_slides(), 1)] + text_from_slide_aprb = [ + slide.replace(" ", '') for page, slide in enumerate(self.file.get_text_from_slides(), 1) + if str(page) == page_aprb] + + string_from_text = ' '.join(text_from_slide) + string_from_text_aprb = ' '.join(text_from_slide_aprb) + + found_repo = re.findall( + r'((((((http(s)?://)?(github|gitlab|bitbucket)+)+(.com|.org)+)+/[a-zA-Z0-9_-]+)+/[a-zA-Z0-9_-]+)+/*)+', + string_from_text) + found_repo_aprb = re.findall( + r'((((((http(s)?://)?(github|gitlab|bitbucket)+)+(.com|.org)+)+/[a-zA-Z0-9_-]+)+/[a-zA-Z0-9_-]+)+/*)+', + string_from_text_aprb) + found_repo_aprb_incorrect = re.findall( + r'\(github\.com\)|\(gitlab\.com\)|\(bitbucket\.org\)|\(github\)|\(gitlab\)|\(bitbucket\)', + string_from_text_aprb) + + if not found_repo: + return answer(True, 'Нечего проверять!') + else: + if found_repo_aprb_incorrect: + string_result += f"
В слайде 'Апробация' вместо выражений {', '.join([repr(repo) for repo in found_repo_aprb_incorrect])}" \ + f" следует привести ссылки вида 'https//github.com/...'" + if not found_repo_aprb and not found_repo_aprb_incorrect and re.findall( + r'репозиторий|репозитория|репозиторию|репозиториев|репозиториям|', string_from_text_aprb): + string_result += f'
В слайде "Апробация" есть упоминания репозиториев,' \ + f'однако ссылки на них либо некорректны, либо отсутствуют.' + for i in found_repo: + try: + link = requests.get(i[0]) + if link.status_code != 200: + raise requests.exceptions.ConnectionError + if re.findall(r'github', i[0]): + tree = html.fromstring(link.content) + if not tree.xpath("//a[@class ='js-navigation-open Link--primary']"): + empty_repo_ref.append(i[0]) + if re.findall(r'gitlab', i[0]): + project_id = quote(i[0].replace('https://gitlab.com/', ''), safe='') + url = f'{GITLAB_URL}/projects/{project_id}?private_token={PRIVATE_TOKEN}' + response = requests.get(url) + project_info = response.json() + if project_info['visibility'] == 'private': + wrong_repo_ref.append(i[0]) + except (requests.exceptions.SSLError, requests.exceptions.ConnectionError): + wrong_repo_ref.append(i[0]) + if wrong_repo_ref and not empty_repo_ref: + string_result += f"
Найдены несуществующие или закрытые репозитории: {', '.join([repr(repo) for repo in wrong_repo_ref])}" + check_result = False + elif empty_repo_ref and not wrong_repo_ref: + string_result += f"
Найдены пустые репозитории: {', '.join([repr(repo) for repo in empty_repo_ref])}" + check_result = False + elif empty_repo_ref and wrong_repo_ref: + string_result += f"
Найдены пустые репозитории: {', '.join([repr(repo) for repo in empty_repo_ref])}" \ + f"
Также найдены несуществующие или закрытые репозитории: {', '.join([repr(repo) for repo in wrong_repo_ref])}" + check_result = False + else: + string_result = 'Пройдена!' + check_result = True + return answer(check_result, string_result) diff --git a/requirements.txt b/requirements.txt index d689642e..2ddae6b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,4 +28,3 @@ pytest~=7.1.2 lxml~=4.9.2 urllib3~=2.0.3 -beautifulsoup4~=4.12.2 \ No newline at end of file From 85991d0ca385a8fe1a873fb88830d3a8a8350e0a Mon Sep 17 00:00:00 2001 From: Marina Date: Fri, 7 Jul 2023 13:31:16 +0300 Subject: [PATCH 06/12] add deep_check --- .../presentation_checks/verify_git_link.py | 74 ++++++++++--------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/app/main/checks/presentation_checks/verify_git_link.py b/app/main/checks/presentation_checks/verify_git_link.py index 4b082233..2ec4efdc 100644 --- a/app/main/checks/presentation_checks/verify_git_link.py +++ b/app/main/checks/presentation_checks/verify_git_link.py @@ -8,21 +8,26 @@ from ..base_check import BasePresCriterion, answer # for check if gitlab-repository is closed: -GITLAB_URL = 'https://gitlab.com/api/v4' -PRIVATE_TOKEN = 'glpat-JeZApxShRgB1nsGrMsst' +# GITLAB_URL = 'https://gitlab.com/api/v4' +# PRIVATE_TOKEN = 'glpat-JeZApxShRgB1nsGrMsst' class PresVerifyGitLinkCheck(BasePresCriterion): description = "Проверка действительности ссылки на github" id = 'verify_git_link' - def __init__(self, file_info): + def __init__(self, file_info, deep_check=False): super().__init__(file_info) + self.deep_check = deep_check + self.wrong_repo_ref = [] + self.empty_repo_ref = [] + self.check_aprb = FindDefSld(file_info=file_info, key_slide="Апробация") + self.pattern_for_repo = r'((((((http(s)?://)?(github|gitlab|bitbucket)+)+(.com|.org)+)+/[a-zA-Z0-9_-]+)+/[a-zA-Z0-9_-]+)+/*)+' + self.pattern_for_repo_incorrect = r'\(github\.com\)|\(gitlab\.com\)|\(bitbucket\.org\)|\(github\)|\(gitlab\)|\(bitbucket\)' + self.pattern_repo_mention = r'репозиторий|репозитория|репозиторию|репозиториев|репозиториям|' def check(self): - wrong_repo_ref = [] - empty_repo_ref = [] string_result = 'Не пройдена!' self.check_aprb.check() @@ -36,15 +41,9 @@ def check(self): string_from_text = ' '.join(text_from_slide) string_from_text_aprb = ' '.join(text_from_slide_aprb) - found_repo = re.findall( - r'((((((http(s)?://)?(github|gitlab|bitbucket)+)+(.com|.org)+)+/[a-zA-Z0-9_-]+)+/[a-zA-Z0-9_-]+)+/*)+', - string_from_text) - found_repo_aprb = re.findall( - r'((((((http(s)?://)?(github|gitlab|bitbucket)+)+(.com|.org)+)+/[a-zA-Z0-9_-]+)+/[a-zA-Z0-9_-]+)+/*)+', - string_from_text_aprb) - found_repo_aprb_incorrect = re.findall( - r'\(github\.com\)|\(gitlab\.com\)|\(bitbucket\.org\)|\(github\)|\(gitlab\)|\(bitbucket\)', - string_from_text_aprb) + found_repo = re.findall(self.pattern_for_repo, string_from_text) + found_repo_aprb = re.findall(self.pattern_for_repo, string_from_text_aprb) + found_repo_aprb_incorrect = re.findall(self.pattern_for_repo_incorrect, string_from_text_aprb) if not found_repo: return answer(True, 'Нечего проверять!') @@ -52,8 +51,7 @@ def check(self): if found_repo_aprb_incorrect: string_result += f"
В слайде 'Апробация' вместо выражений {', '.join([repr(repo) for repo in found_repo_aprb_incorrect])}" \ f" следует привести ссылки вида 'https//github.com/...'" - if not found_repo_aprb and not found_repo_aprb_incorrect and re.findall( - r'репозиторий|репозитория|репозиторию|репозиториев|репозиториям|', string_from_text_aprb): + if not found_repo_aprb and not found_repo_aprb_incorrect and re.findall(self.pattern_repo_mention, string_from_text_aprb): string_result += f'
В слайде "Апробация" есть упоминания репозиториев,' \ f'однако ссылки на них либо некорректны, либо отсутствуют.' for i in found_repo: @@ -61,30 +59,36 @@ def check(self): link = requests.get(i[0]) if link.status_code != 200: raise requests.exceptions.ConnectionError - if re.findall(r'github', i[0]): - tree = html.fromstring(link.content) - if not tree.xpath("//a[@class ='js-navigation-open Link--primary']"): - empty_repo_ref.append(i[0]) - if re.findall(r'gitlab', i[0]): - project_id = quote(i[0].replace('https://gitlab.com/', ''), safe='') - url = f'{GITLAB_URL}/projects/{project_id}?private_token={PRIVATE_TOKEN}' - response = requests.get(url) - project_info = response.json() - if project_info['visibility'] == 'private': - wrong_repo_ref.append(i[0]) + else: + if self.deep_check: + self.deep_check_repo(i, link) except (requests.exceptions.SSLError, requests.exceptions.ConnectionError): - wrong_repo_ref.append(i[0]) - if wrong_repo_ref and not empty_repo_ref: - string_result += f"
Найдены несуществующие или закрытые репозитории: {', '.join([repr(repo) for repo in wrong_repo_ref])}" + self.wrong_repo_ref.append(i[0]) + if self.wrong_repo_ref and not self.empty_repo_ref: + string_result += f"
Найдены несуществующие или закрытые репозитории: {', '.join([repr(repo) for repo in self.wrong_repo_ref])}" check_result = False - elif empty_repo_ref and not wrong_repo_ref: - string_result += f"
Найдены пустые репозитории: {', '.join([repr(repo) for repo in empty_repo_ref])}" + elif self.empty_repo_ref and not self.wrong_repo_ref: + string_result += f"
Найдены пустые репозитории: {', '.join([repr(repo) for repo in self.empty_repo_ref])}" check_result = False - elif empty_repo_ref and wrong_repo_ref: - string_result += f"
Найдены пустые репозитории: {', '.join([repr(repo) for repo in empty_repo_ref])}" \ - f"
Также найдены несуществующие или закрытые репозитории: {', '.join([repr(repo) for repo in wrong_repo_ref])}" + elif self.empty_repo_ref and self.wrong_repo_ref: + string_result += f"
Найдены пустые репозитории: {', '.join([repr(repo) for repo in self.empty_repo_ref])}" \ + f"
Также найдены несуществующие или закрытые репозитории: {', '.join([repr(repo) for repo in self.wrong_repo_ref])}" check_result = False else: string_result = 'Пройдена!' check_result = True return answer(check_result, string_result) + + def deep_check_repo(self, repo, link): + if re.findall(r'github', repo[0]): + tree = html.fromstring(link.content) + if not tree.xpath("//a[@class ='js-navigation-open Link--primary']"): + self.empty_repo_ref.append(repo[0]) + + # if re.findall(r'gitlab', i[0]): + # project_id = quote(i[0].replace('https://gitlab.com/', ''), safe='') + # url = f'{GITLAB_URL}/projects/{project_id}?private_token={PRIVATE_TOKEN}' + # response = requests.get(url) + # project_info = response.json() + # if project_info['visibility'] == 'private': + # wrong_repo_ref.append(i[0]) From e82b3171815f2e2d02ba495b46c7806a91d80d46 Mon Sep 17 00:00:00 2001 From: Marina Date: Mon, 10 Jul 2023 17:53:48 +0300 Subject: [PATCH 07/12] question --- app/main/checks/presentation_checks/verify_git_link.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/main/checks/presentation_checks/verify_git_link.py b/app/main/checks/presentation_checks/verify_git_link.py index 2ec4efdc..9acb4d61 100644 --- a/app/main/checks/presentation_checks/verify_git_link.py +++ b/app/main/checks/presentation_checks/verify_git_link.py @@ -30,7 +30,7 @@ def __init__(self, file_info, deep_check=False): def check(self): string_result = 'Не пройдена!' - self.check_aprb.check() + a = self.check_aprb.check() page_aprb = ''.join((str(item) for item in self.check_aprb.__getattribute__("found_idxs"))) text_from_slide = [slide for page, slide in enumerate(self.file.get_text_from_slides(), 1)] @@ -44,6 +44,7 @@ def check(self): found_repo = re.findall(self.pattern_for_repo, string_from_text) found_repo_aprb = re.findall(self.pattern_for_repo, string_from_text_aprb) found_repo_aprb_incorrect = re.findall(self.pattern_for_repo_incorrect, string_from_text_aprb) + print(f'page:{a}, {text_from_slide_aprb}') if not found_repo: return answer(True, 'Нечего проверять!') From 03275ce91dd95870d10e1ac5d938bc06b5980cf6 Mon Sep 17 00:00:00 2001 From: Marina Date: Mon, 10 Jul 2023 17:54:26 +0300 Subject: [PATCH 08/12] req --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index fb97d986..0a3a20de 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,11 +25,11 @@ docx2python~=2.0.4 oauthlib~=3.1.0 pdfplumber==0.6.1 pytest~=7.1.2 -<<<<<<< HEAD + lxml~=4.9.2 urllib3~=2.0.3 -======= + filetype==1.2.0 language-tool-python==2.7.1 ->>>>>>> master + From 7397a03e06bb43251132fe31191a24e729d39426 Mon Sep 17 00:00:00 2001 From: Marina Date: Mon, 10 Jul 2023 19:03:33 +0300 Subject: [PATCH 09/12] changes --- .../presentation_checks/verify_git_link.py | 81 ++++++++++--------- app/server.py | 1 + 2 files changed, 42 insertions(+), 40 deletions(-) diff --git a/app/main/checks/presentation_checks/verify_git_link.py b/app/main/checks/presentation_checks/verify_git_link.py index 9acb4d61..12a94572 100644 --- a/app/main/checks/presentation_checks/verify_git_link.py +++ b/app/main/checks/presentation_checks/verify_git_link.py @@ -44,47 +44,48 @@ def check(self): found_repo = re.findall(self.pattern_for_repo, string_from_text) found_repo_aprb = re.findall(self.pattern_for_repo, string_from_text_aprb) found_repo_aprb_incorrect = re.findall(self.pattern_for_repo_incorrect, string_from_text_aprb) - print(f'page:{a}, {text_from_slide_aprb}') + print(f'page:{a}, {page_aprb} !!!!!!!!!{text_from_slide_aprb}') + return answer(False, found_repo_aprb_incorrect, found_repo_aprb) - if not found_repo: - return answer(True, 'Нечего проверять!') - else: - if found_repo_aprb_incorrect: - string_result += f"
В слайде 'Апробация' вместо выражений {', '.join([repr(repo) for repo in found_repo_aprb_incorrect])}" \ - f" следует привести ссылки вида 'https//github.com/...'" - if not found_repo_aprb and not found_repo_aprb_incorrect and re.findall(self.pattern_repo_mention, string_from_text_aprb): - string_result += f'
В слайде "Апробация" есть упоминания репозиториев,' \ - f'однако ссылки на них либо некорректны, либо отсутствуют.' - for i in found_repo: - try: - link = requests.get(i[0]) - if link.status_code != 200: - raise requests.exceptions.ConnectionError - else: - if self.deep_check: - self.deep_check_repo(i, link) - except (requests.exceptions.SSLError, requests.exceptions.ConnectionError): - self.wrong_repo_ref.append(i[0]) - if self.wrong_repo_ref and not self.empty_repo_ref: - string_result += f"
Найдены несуществующие или закрытые репозитории: {', '.join([repr(repo) for repo in self.wrong_repo_ref])}" - check_result = False - elif self.empty_repo_ref and not self.wrong_repo_ref: - string_result += f"
Найдены пустые репозитории: {', '.join([repr(repo) for repo in self.empty_repo_ref])}" - check_result = False - elif self.empty_repo_ref and self.wrong_repo_ref: - string_result += f"
Найдены пустые репозитории: {', '.join([repr(repo) for repo in self.empty_repo_ref])}" \ - f"
Также найдены несуществующие или закрытые репозитории: {', '.join([repr(repo) for repo in self.wrong_repo_ref])}" - check_result = False - else: - string_result = 'Пройдена!' - check_result = True - return answer(check_result, string_result) - - def deep_check_repo(self, repo, link): - if re.findall(r'github', repo[0]): - tree = html.fromstring(link.content) - if not tree.xpath("//a[@class ='js-navigation-open Link--primary']"): - self.empty_repo_ref.append(repo[0]) + # if not found_repo: + # return answer(True, 'Нечего проверять!') + # else: + # # if found_repo_aprb_incorrect: + # # string_result += f"
В слайде 'Апробация' вместо выражений {', '.join([repr(repo) for repo in found_repo_aprb_incorrect])}" \ + # # f" следует привести ссылки вида 'https//github.com/...'" + # # if not found_repo_aprb and not found_repo_aprb_incorrect and re.findall(self.pattern_repo_mention, string_from_text_aprb): + # # string_result += f'
В слайде "Апробация" есть упоминания репозиториев,' \ + # # f'однако ссылки на них либо некорректны, либо отсутствуют.' + # for i in found_repo: + # try: + # link = requests.get(i[0]) + # if link.status_code != 200: + # raise requests.exceptions.ConnectionError + # else: + # if self.deep_check: + # self.deep_check_repo(i, link) + # except (requests.exceptions.SSLError, requests.exceptions.ConnectionError): + # self.wrong_repo_ref.append(i[0]) + # if self.wrong_repo_ref and not self.empty_repo_ref: + # string_result += f"
Найдены несуществующие или закрытые репозитории: {', '.join([repr(repo) for repo in self.wrong_repo_ref])}" + # check_result = False + # elif self.empty_repo_ref and not self.wrong_repo_ref: + # string_result += f"
Найдены пустые репозитории: {', '.join([repr(repo) for repo in self.empty_repo_ref])}" + # check_result = False + # elif self.empty_repo_ref and self.wrong_repo_ref: + # string_result += f"
Найдены пустые репозитории: {', '.join([repr(repo) for repo in self.empty_repo_ref])}" \ + # f"
Также найдены несуществующие или закрытые репозитории: {', '.join([repr(repo) for repo in self.wrong_repo_ref])}" + # check_result = False + # else: + # string_result = 'Пройдена!' + # check_result = True + # return answer(check_result, string_result) + # + # def deep_check_repo(self, repo, link): + # if re.findall(r'github', repo[0]): + # tree = html.fromstring(link.content) + # if not tree.xpath("//a[@class ='js-navigation-open Link--primary']"): + # self.empty_repo_ref.append(repo[0]) # if re.findall(r'gitlab', i[0]): # project_id = quote(i[0].replace('https://gitlab.com/', ''), safe='') diff --git a/app/server.py b/app/server.py index 1bfe9127..7fc4bd58 100644 --- a/app/server.py +++ b/app/server.py @@ -275,6 +275,7 @@ def get_status(task_id): 'pres_image_share': 'Проверка доли объема презентации, приходящейся на изображения', 'pres_banned_words_check': 'Проверка наличия запретных слов в презентации', 'conclusion_actual': 'Соответствие заключения задачам', + 'verify_git_link': 'Проверка действительности ссылки на github', 'conclusion_along': 'Наличие направлений дальнейшего развития', 'simple_check': 'Простейшая проверка отчёта', 'banned_words_in_literature': 'Наличие запрещенных слов в списке литературы', From f15190e6465f8372d236f96d5b0da1088c721553 Mon Sep 17 00:00:00 2001 From: Marina Date: Thu, 13 Jul 2023 17:10:41 +0300 Subject: [PATCH 10/12] optimize code --- .../presentation_checks/find_def_sld.py | 4 + .../presentation_checks/verify_git_link.py | 103 +++++++++--------- .../presentations/pptx/presentation_pptx.py | 1 + 3 files changed, 55 insertions(+), 53 deletions(-) diff --git a/app/main/checks/presentation_checks/find_def_sld.py b/app/main/checks/presentation_checks/find_def_sld.py index 33a66ff5..af16ce03 100644 --- a/app/main/checks/presentation_checks/find_def_sld.py +++ b/app/main/checks/presentation_checks/find_def_sld.py @@ -19,6 +19,10 @@ def check(self): if len(found_slides) == 0: return answer(False, 'Слайд не найден') else: + if self.type_of_slide == 'Апробация': + self.file.found_index['Апробация'] = ''.join(str(item) for item in self.found_idxs) + else: + self.file.found_index['Апробация'] = None found_idxs_link = self.format_page_link(self.found_idxs) return answer(True, 'Найден под номером: {}'.format(', '.join(map(str, found_idxs_link)))) diff --git a/app/main/checks/presentation_checks/verify_git_link.py b/app/main/checks/presentation_checks/verify_git_link.py index 12a94572..a9f25ddc 100644 --- a/app/main/checks/presentation_checks/verify_git_link.py +++ b/app/main/checks/presentation_checks/verify_git_link.py @@ -22,70 +22,67 @@ def __init__(self, file_info, deep_check=False): self.wrong_repo_ref = [] self.empty_repo_ref = [] - self.check_aprb = FindDefSld(file_info=file_info, key_slide="Апробация") + # self.check_aprb = FindDefSld(file_info=file_info, key_slide="Апробация") self.pattern_for_repo = r'((((((http(s)?://)?(github|gitlab|bitbucket)+)+(.com|.org)+)+/[a-zA-Z0-9_-]+)+/[a-zA-Z0-9_-]+)+/*)+' self.pattern_for_repo_incorrect = r'\(github\.com\)|\(gitlab\.com\)|\(bitbucket\.org\)|\(github\)|\(gitlab\)|\(bitbucket\)' self.pattern_repo_mention = r'репозиторий|репозитория|репозиторию|репозиториев|репозиториям|' def check(self): string_result = 'Не пройдена!' + text_from_slide = [slide for page, slide in enumerate(self.file.get_text_from_slides(), 1)] + string_from_text = ' '.join(text_from_slide) + found_repo = re.findall(self.pattern_for_repo, string_from_text) - a = self.check_aprb.check() - page_aprb = ''.join((str(item) for item in self.check_aprb.__getattribute__("found_idxs"))) + if not found_repo: + return answer(True, 'Нечего проверять!') - text_from_slide = [slide for page, slide in enumerate(self.file.get_text_from_slides(), 1)] - text_from_slide_aprb = [ - slide.replace(" ", '') for page, slide in enumerate(self.file.get_text_from_slides(), 1) - if str(page) == page_aprb] + else: + if self.file.found_index['Апробация'] is not None: + page_aprb = self.file.found_index['Апробация'] + text_from_slide_aprb = [ + slide.replace(" ", '') for page, slide in enumerate(self.file.get_text_from_slides(), 1) + if str(page) == page_aprb] - string_from_text = ' '.join(text_from_slide) - string_from_text_aprb = ' '.join(text_from_slide_aprb) + string_from_text_aprb = ' '.join(text_from_slide_aprb) + found_repo_aprb = re.findall(self.pattern_for_repo, string_from_text_aprb) + found_repo_aprb_incorrect = re.findall(self.pattern_for_repo_incorrect, string_from_text_aprb) + if found_repo_aprb_incorrect: + string_result += f"
В слайде 'Апробация' вместо выражений {', '.join([repr(repo) for repo in found_repo_aprb_incorrect])}" \ + f" следует привести ссылки вида 'https//github.com/...'" + if not found_repo_aprb and not found_repo_aprb_incorrect and re.findall(self.pattern_repo_mention, string_from_text_aprb): + string_result += f'
В слайде "Апробация" есть упоминания репозиториев,' \ + f'однако ссылки на них либо некорректны, либо отсутствуют.' - found_repo = re.findall(self.pattern_for_repo, string_from_text) - found_repo_aprb = re.findall(self.pattern_for_repo, string_from_text_aprb) - found_repo_aprb_incorrect = re.findall(self.pattern_for_repo_incorrect, string_from_text_aprb) - print(f'page:{a}, {page_aprb} !!!!!!!!!{text_from_slide_aprb}') - return answer(False, found_repo_aprb_incorrect, found_repo_aprb) + for i in found_repo: + try: + link = requests.get(i[0]) + if link.status_code != 200: + raise requests.exceptions.ConnectionError + else: + if self.deep_check: + self.deep_check_repo(i, link) + except (requests.exceptions.SSLError, requests.exceptions.ConnectionError): + self.wrong_repo_ref.append(i[0]) + if self.wrong_repo_ref and not self.empty_repo_ref: + string_result += f"
Найдены несуществующие или закрытые репозитории: {', '.join([repr(repo) for repo in self.wrong_repo_ref])}" + check_result = False + elif self.empty_repo_ref and not self.wrong_repo_ref: + string_result += f"
Найдены пустые репозитории: {', '.join([repr(repo) for repo in self.empty_repo_ref])}" + check_result = False + elif self.empty_repo_ref and self.wrong_repo_ref: + string_result += f"
Найдены пустые репозитории: {', '.join([repr(repo) for repo in self.empty_repo_ref])}" \ + f"
Также найдены несуществующие или закрытые репозитории: {', '.join([repr(repo) for repo in self.wrong_repo_ref])}" + check_result = False + else: + string_result = 'Пройдена!' + check_result = True + return answer(check_result, string_result) - # if not found_repo: - # return answer(True, 'Нечего проверять!') - # else: - # # if found_repo_aprb_incorrect: - # # string_result += f"
В слайде 'Апробация' вместо выражений {', '.join([repr(repo) for repo in found_repo_aprb_incorrect])}" \ - # # f" следует привести ссылки вида 'https//github.com/...'" - # # if not found_repo_aprb and not found_repo_aprb_incorrect and re.findall(self.pattern_repo_mention, string_from_text_aprb): - # # string_result += f'
В слайде "Апробация" есть упоминания репозиториев,' \ - # # f'однако ссылки на них либо некорректны, либо отсутствуют.' - # for i in found_repo: - # try: - # link = requests.get(i[0]) - # if link.status_code != 200: - # raise requests.exceptions.ConnectionError - # else: - # if self.deep_check: - # self.deep_check_repo(i, link) - # except (requests.exceptions.SSLError, requests.exceptions.ConnectionError): - # self.wrong_repo_ref.append(i[0]) - # if self.wrong_repo_ref and not self.empty_repo_ref: - # string_result += f"
Найдены несуществующие или закрытые репозитории: {', '.join([repr(repo) for repo in self.wrong_repo_ref])}" - # check_result = False - # elif self.empty_repo_ref and not self.wrong_repo_ref: - # string_result += f"
Найдены пустые репозитории: {', '.join([repr(repo) for repo in self.empty_repo_ref])}" - # check_result = False - # elif self.empty_repo_ref and self.wrong_repo_ref: - # string_result += f"
Найдены пустые репозитории: {', '.join([repr(repo) for repo in self.empty_repo_ref])}" \ - # f"
Также найдены несуществующие или закрытые репозитории: {', '.join([repr(repo) for repo in self.wrong_repo_ref])}" - # check_result = False - # else: - # string_result = 'Пройдена!' - # check_result = True - # return answer(check_result, string_result) - # - # def deep_check_repo(self, repo, link): - # if re.findall(r'github', repo[0]): - # tree = html.fromstring(link.content) - # if not tree.xpath("//a[@class ='js-navigation-open Link--primary']"): - # self.empty_repo_ref.append(repo[0]) + def deep_check_repo(self, repo, link): + if re.findall(r'github', repo[0]): + tree = html.fromstring(link.content) + if not tree.xpath("//a[@class ='js-navigation-open Link--primary']"): + self.empty_repo_ref.append(repo[0]) # if re.findall(r'gitlab', i[0]): # project_id = quote(i[0].replace('https://gitlab.com/', ''), safe='') diff --git a/app/main/presentations/pptx/presentation_pptx.py b/app/main/presentations/pptx/presentation_pptx.py index 869846e0..dd909f8c 100644 --- a/app/main/presentations/pptx/presentation_pptx.py +++ b/app/main/presentations/pptx/presentation_pptx.py @@ -9,6 +9,7 @@ def __init__(self, presentation_name): PresentationBasic.__init__(self, presentation_name) self.prs = Presentation(presentation_name) self.add_slides() + self.found_index = {} def add_slides(self): for index, slide in enumerate(self.prs.slides, 1): From 6289ec5cefce52500cad9610518adcebcefebbed Mon Sep 17 00:00:00 2001 From: Marina Date: Wed, 8 Nov 2023 12:21:15 +0300 Subject: [PATCH 11/12] fix conflicts --- app/main/checks/__init__.py | 5 +---- app/main/checks/presentation_checks/__init__.py | 3 --- requirements.txt | 7 ------- 3 files changed, 1 insertion(+), 14 deletions(-) diff --git a/app/main/checks/__init__.py b/app/main/checks/__init__.py index cf090732..8ddc897e 100644 --- a/app/main/checks/__init__.py +++ b/app/main/checks/__init__.py @@ -15,11 +15,8 @@ PresImageShareCheck.id: PresImageShareCheck, FurtherDev.id: FurtherDev, PresBannedWordsCheck.id: PresBannedWordsCheck, -<<<<<<< HEAD - PresVerifyGitLinkCheck.id: PresVerifyGitLinkCheck -======= + PresVerifyGitLinkCheck.id: PresVerifyGitLinkCheck, PresEmptySlideCheck.id: PresEmptySlideCheck, ->>>>>>> master }, 'report': { ReportSimpleCheck.id: ReportSimpleCheck, diff --git a/app/main/checks/presentation_checks/__init__.py b/app/main/checks/presentation_checks/__init__.py index 7369cd04..095bf658 100644 --- a/app/main/checks/presentation_checks/__init__.py +++ b/app/main/checks/presentation_checks/__init__.py @@ -10,8 +10,5 @@ from .pres_right_words import PresRightWordsCheck from .image_share import PresImageShareCheck from .banned_words import PresBannedWordsCheck -<<<<<<< HEAD from .verify_git_link import PresVerifyGitLinkCheck -======= from .empty_slide_check import PresEmptySlideCheck ->>>>>>> master diff --git a/requirements.txt b/requirements.txt index 5e894bf1..47560892 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,15 +25,8 @@ docx2python~=2.0.4 oauthlib~=3.1.0 pdfplumber==0.6.1 pytest~=7.1.2 - - lxml~=4.9.2 urllib3~=2.0.3 - filetype==1.2.0 language-tool-python==2.7.1 -<<<<<<< HEAD - -======= markdown==3.4.4 ->>>>>>> master From fd9dab009b31fcb8b3cfb4ac60e7e3557ccb622e Mon Sep 17 00:00:00 2001 From: Marina Date: Mon, 13 Nov 2023 19:38:03 +0300 Subject: [PATCH 12/12] fix aprob_mistake --- app/main/checks/presentation_checks/find_def_sld.py | 6 ++---- app/main/checks/presentation_checks/verify_git_link.py | 10 +++------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/app/main/checks/presentation_checks/find_def_sld.py b/app/main/checks/presentation_checks/find_def_sld.py index af16ce03..baa6f031 100644 --- a/app/main/checks/presentation_checks/find_def_sld.py +++ b/app/main/checks/presentation_checks/find_def_sld.py @@ -17,12 +17,10 @@ def check(self): found_slides.append(self.file.get_text_from_slides()[i - 1]) self.found_idxs.append(i) if len(found_slides) == 0: + self.file.found_index[str(self.type_of_slide)] = None return answer(False, 'Слайд не найден') else: - if self.type_of_slide == 'Апробация': - self.file.found_index['Апробация'] = ''.join(str(item) for item in self.found_idxs) - else: - self.file.found_index['Апробация'] = None + self.file.found_index[str(self.type_of_slide)] = ''.join(str(item) for item in self.found_idxs) found_idxs_link = self.format_page_link(self.found_idxs) return answer(True, 'Найден под номером: {}'.format(', '.join(map(str, found_idxs_link)))) diff --git a/app/main/checks/presentation_checks/verify_git_link.py b/app/main/checks/presentation_checks/verify_git_link.py index a9f25ddc..64135954 100644 --- a/app/main/checks/presentation_checks/verify_git_link.py +++ b/app/main/checks/presentation_checks/verify_git_link.py @@ -16,7 +16,7 @@ class PresVerifyGitLinkCheck(BasePresCriterion): description = "Проверка действительности ссылки на github" id = 'verify_git_link' - def __init__(self, file_info, deep_check=False): + def __init__(self, file_info, deep_check=True): super().__init__(file_info) self.deep_check = deep_check self.wrong_repo_ref = [] @@ -63,16 +63,12 @@ def check(self): self.deep_check_repo(i, link) except (requests.exceptions.SSLError, requests.exceptions.ConnectionError): self.wrong_repo_ref.append(i[0]) - if self.wrong_repo_ref and not self.empty_repo_ref: + if self.wrong_repo_ref: string_result += f"
Найдены несуществующие или закрытые репозитории: {', '.join([repr(repo) for repo in self.wrong_repo_ref])}" check_result = False - elif self.empty_repo_ref and not self.wrong_repo_ref: + if self.empty_repo_ref: string_result += f"
Найдены пустые репозитории: {', '.join([repr(repo) for repo in self.empty_repo_ref])}" check_result = False - elif self.empty_repo_ref and self.wrong_repo_ref: - string_result += f"
Найдены пустые репозитории: {', '.join([repr(repo) for repo in self.empty_repo_ref])}" \ - f"
Также найдены несуществующие или закрытые репозитории: {', '.join([repr(repo) for repo in self.wrong_repo_ref])}" - check_result = False else: string_result = 'Пройдена!' check_result = True