From a01c91000ac8d4b8a710e4fea882e1f03dc25d1d Mon Sep 17 00:00:00 2001 From: Marina Date: Thu, 11 Apr 2024 09:28:53 +0300 Subject: [PATCH 01/69] start for /users --- app/db/db_methods.py | 2 ++ app/routes/users.py | 29 +++++++++++++++++++++++++++++ app/server.py | 2 ++ app/templates/user_list.html | 16 ++++++++++++++++ 4 files changed, 49 insertions(+) create mode 100644 app/routes/users.py create mode 100644 app/templates/user_list.html diff --git a/app/db/db_methods.py b/app/db/db_methods.py index b372d607..7bd67455 100644 --- a/app/db/db_methods.py +++ b/app/db/db_methods.py @@ -59,6 +59,8 @@ def get_user(username): else: return None +def get_all_users(): + return users_collection.find() # Returns True if user was found and updated and false if not (username can not be changed!) def edit_user(user): diff --git a/app/routes/users.py b/app/routes/users.py new file mode 100644 index 00000000..fe05b355 --- /dev/null +++ b/app/routes/users.py @@ -0,0 +1,29 @@ +from flask import abort, Blueprint, render_template +from flask_login import current_user +from functools import wraps +from app.db.db_methods import get_all_users, get_user + +users = Blueprint('users', __name__, template_folder='templates', static_folder='static') + + +def admin_required(route_func): + @wraps(route_func) + def my_wrapper(*args, **kwargs): + if current_user and current_user.is_admin: + return route_func(*args, **kwargs) + abort(403) + return my_wrapper + + +@users.route('/', methods=["GET"]) +@admin_required +def index(): + users = list(get_all_users()) + usernames = [(user['name'], user['username']) for user in users] + return render_template('user_list.html', usernames=usernames) + +@users.route('/', methods=["GET"]) +@admin_required +def user_info(username): + user_info = get_user(username) + return f"{user_info}" diff --git a/app/server.py b/app/server.py index 20434a16..2e833d9f 100644 --- a/app/server.py +++ b/app/server.py @@ -31,6 +31,7 @@ from utils import checklist_filter, decorator_assertion, get_file_len, format_check from app.main.checks import CRITERIA_INFO from routes.admin import admin +from routes.users import users logger = get_root_logger('web') UPLOAD_FOLDER = '/usr/src/project/files' @@ -52,6 +53,7 @@ app.config['CELERY_BROKER_URL'] = os.environ.get("CELERY_BROKER_URL", "redis://localhost:6379") app.register_blueprint(admin, url_prefix='/admin') +app.register_blueprint(users, url_prefix='/users') app.logger.addHandler(get_logging_stdout_handler()) diff --git a/app/templates/user_list.html b/app/templates/user_list.html new file mode 100644 index 00000000..d2d80673 --- /dev/null +++ b/app/templates/user_list.html @@ -0,0 +1,16 @@ + + + + + + User List + + +

Список пользователей

+ + + From ba96d8037c7126dc786284dbef93d0d772983bcf Mon Sep 17 00:00:00 2001 From: Marina Date: Thu, 11 Apr 2024 19:36:38 +0300 Subject: [PATCH 02/69] first try of yser's pages --- app/routes/users.py | 2 +- app/templates/one_user_info.html | 51 ++++++++++++++++++++++++++++++++ app/templates/user_list.html | 51 ++++++++++++++++++++++---------- 3 files changed, 87 insertions(+), 17 deletions(-) create mode 100644 app/templates/one_user_info.html diff --git a/app/routes/users.py b/app/routes/users.py index fe05b355..4957ab14 100644 --- a/app/routes/users.py +++ b/app/routes/users.py @@ -26,4 +26,4 @@ def index(): @admin_required def user_info(username): user_info = get_user(username) - return f"{user_info}" + return render_template('one_user_info.html', user_info=user_info) diff --git a/app/templates/one_user_info.html b/app/templates/one_user_info.html new file mode 100644 index 00000000..f0949036 --- /dev/null +++ b/app/templates/one_user_info.html @@ -0,0 +1,51 @@ +{# Accepts: header dependicies, results, id, filename #} + +{% extends "root.html" %} + +{% block title %}Информация о пользователе{% endblock %} + +{% block main %} + +
{% include "header.html" %}
+
+

+ Информация о пользователе {{ user_info.username }}: +

+ + + + + + + + + + + + + + + + + + + + + + + + +
UsernameNameFormatscriteria
+ + {{ user_info.username }}{{ user_info.name }}{{ user_info.criteria }}{{ user_info.formats }}
+
+ Список всех загрузок пользователя +
+
+ + +
+ +{% endblock %} diff --git a/app/templates/user_list.html b/app/templates/user_list.html index d2d80673..c3a9d3aa 100644 --- a/app/templates/user_list.html +++ b/app/templates/user_list.html @@ -1,16 +1,35 @@ - - - - - - User List - - -

Список пользователей

-
    - {% for name, username in usernames %} -

    {{ name }}

    - {% endfor %} -
- - +{# Accepts: header dependicies, results, id, filename #} + + +{% extends "root.html" %} + +{% block title %}Информация о пользователях{% endblock %} + +{% block main %} + +
{% include "header.html" %}
+
+
+

+ Список пользователей: +

+ + + + + + + + + + {% for name, username in usernames %} + + + {% endfor %} + + +
UsernameName
{{ name }}{{ username }}
+
+
+ +{% endblock %} From a771fc14111fe444cabb16512c5e3d758c5d8f84 Mon Sep 17 00:00:00 2001 From: Marina Date: Thu, 11 Apr 2024 19:38:40 +0300 Subject: [PATCH 03/69] format document --- app/templates/one_user_info.html | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/app/templates/one_user_info.html b/app/templates/one_user_info.html index f0949036..8e2783b8 100644 --- a/app/templates/one_user_info.html +++ b/app/templates/one_user_info.html @@ -11,7 +11,6 @@

Информация о пользователе {{ user_info.username }}:

- @@ -37,15 +36,13 @@

Список всех загрузок пользователя + class="col text-center link">Список всех загрузок пользователя
- - {% endblock %} From fe354e387096e240a101242c8a7d98a1ae7e8360 Mon Sep 17 00:00:00 2001 From: Marina Date: Thu, 25 Apr 2024 11:20:57 +0300 Subject: [PATCH 04/69] add count_of_check --- app/routes/users.py | 2 +- app/templates/one_user_info.html | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/routes/users.py b/app/routes/users.py index 4957ab14..324d7ef1 100644 --- a/app/routes/users.py +++ b/app/routes/users.py @@ -26,4 +26,4 @@ def index(): @admin_required def user_info(username): user_info = get_user(username) - return render_template('one_user_info.html', user_info=user_info) + return render_template('one_user_info.html', user_info=user_info, check_counts = len(user_info.presentations)) diff --git a/app/templates/one_user_info.html b/app/templates/one_user_info.html index 8e2783b8..23c52ce1 100644 --- a/app/templates/one_user_info.html +++ b/app/templates/one_user_info.html @@ -20,6 +20,7 @@

Name Formats criteria + count of check @@ -31,9 +32,10 @@

{{ user_info.name }} {{ user_info.criteria }} {{ user_info.formats }} + {{ check_counts}} - +
Список всех загрузок пользователя From 27ba19d9f039240090734e1a3f9ff0c241fbc98e Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Sat, 27 Apr 2024 17:35:42 +0300 Subject: [PATCH 05/69] Update error criterion feedback on check --- app/main/check_packs/base_criterion_pack.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/main/check_packs/base_criterion_pack.py b/app/main/check_packs/base_criterion_pack.py index c478fd6b..9ab687c2 100644 --- a/app/main/check_packs/base_criterion_pack.py +++ b/app/main/check_packs/base_criterion_pack.py @@ -26,8 +26,9 @@ def check(self): try: criterion_check_result = criterion.check() except Exception as e: - logger.error(f'{criterion.id}: oшибка во время проверки: {e}') - criterion_check_result = {'score': 0, 'verdict': [UNEXPECTED_CHECK_FAIL_MSG]} + err_msg = f'{criterion.id}: oшибка во время проверки: {e}' + logger.error(err_msg) + criterion_check_result = {'score': 0, 'verdict': [UNEXPECTED_CHECK_FAIL_MSG, f"Информация об ошибке для администратора: {err_msg}"]} if criterion.priority and not criterion_check_result['score']: failed_priority_check = True criterion_check_result['verdict'] = [PRIORITY_CHECK_FAILED_MSG] + list(criterion_check_result['verdict']) From a1150853568466560247338d1d1b73e7fda24a5f Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Mon, 29 Apr 2024 17:02:41 +0300 Subject: [PATCH 06/69] =?UTF-8?q?add=20form=20of=20"=D1=81=D1=82=D1=83?= =?UTF-8?q?=D0=B4=D0=B5=D0=BD=D1=82"=20for=20theme=20search?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/main/checks/report_checks/find_theme_in_report.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/app/main/checks/report_checks/find_theme_in_report.py b/app/main/checks/report_checks/find_theme_in_report.py index 56dd9a00..9602e44f 100644 --- a/app/main/checks/report_checks/find_theme_in_report.py +++ b/app/main/checks/report_checks/find_theme_in_report.py @@ -69,9 +69,13 @@ def find_theme(self): if key == 1: lower_text = text_on_page.lower() text_without_punct = lower_text.translate(str.maketrans('', '', string.punctuation)) - list_full = text_without_punct.split() - start = list_full.index('тема') + 1 - end = list_full.index('студент') + list_full = tuple(text_without_punct.split()) + start, end = 0, len(list_full) + for index, value in enumerate(list_full): + if value == "тема": + start = index + 1 + elif value in {"студент", "студентка"}: + end = index list_theme = list_full[start:end] lemma_theme = {MORPH_ANALYZER.parse(word)[0].normal_form for word in list_theme if word not in stop_words} From 49e17961dd3ccab0701dffe00618c2db186e8866 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Mon, 29 Apr 2024 17:28:27 +0300 Subject: [PATCH 07/69] update conclusion_actual to exclude future dev text --- .../checks/presentation_checks/sld_similarity.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/app/main/checks/presentation_checks/sld_similarity.py b/app/main/checks/presentation_checks/sld_similarity.py index 2d9d1db1..47f189ad 100644 --- a/app/main/checks/presentation_checks/sld_similarity.py +++ b/app/main/checks/presentation_checks/sld_similarity.py @@ -1,6 +1,7 @@ from nlp.similarity_of_texts import check_similarity -from utils import get_text_from_slides, tasks_conclusions_feedback +from utils import get_text_from_slides, tasks_conclusions_feedback +from app.nlp.stemming import Stemming from ..base_check import BasePresCriterion, answer @@ -21,6 +22,13 @@ def check(self): if goals == "" or conclusions == "": return answer(False, 'Задач или заключения не существует') - results = check_similarity(goals, conclusions) + stemming = Stemming() + stemming.parse_text(conclusions, False) + try: + dev_index = conclusions.index(stemming.further_dev_sentence) + except ValueError: + dev_index = len(conclusions) + + results = check_similarity(goals, conclusions[:dev_index]) return answer(results[0] >= self.actual_number, *tasks_conclusions_feedback(results)) From e0fca05cad0646391db3186ec39e50275b22d5e1 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Mon, 29 Apr 2024 17:42:39 +0300 Subject: [PATCH 08/69] update check_similarity for using recognized_conclusions --- app/nlp/similarity_of_texts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/nlp/similarity_of_texts.py b/app/nlp/similarity_of_texts.py index e537bc60..87472aab 100644 --- a/app/nlp/similarity_of_texts.py +++ b/app/nlp/similarity_of_texts.py @@ -10,8 +10,8 @@ def check_similarity(string1, string2): base_conclusions = stemming.get_sentences(string2, False) ignore = re.compile('[0-9]+[.]?|Заключение|‹#›') clear_conclusions = [ch for ch in base_conclusions if not re.fullmatch(ignore, ch)] - recognized_conclusions = [s for s in clear_conclusions if s != further_dev.get('dev_sentence')] + recognized_conclusions = "\n".join(s for s in clear_conclusions if s != further_dev.get('dev_sentence')) - percentage_of_similarity = int(compare_sentences(string1, string2) * 100) + percentage_of_similarity = int(compare_sentences(string1, recognized_conclusions) * 100) return percentage_of_similarity, further_dev, recognized_conclusions From 4fa7e9ea563f69f72814616dc804517f36557495 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Mon, 29 Apr 2024 17:46:32 +0300 Subject: [PATCH 09/69] update check_similarity for using recognized_conclusions --- app/nlp/similarity_of_texts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/nlp/similarity_of_texts.py b/app/nlp/similarity_of_texts.py index 87472aab..99d6990a 100644 --- a/app/nlp/similarity_of_texts.py +++ b/app/nlp/similarity_of_texts.py @@ -10,7 +10,7 @@ def check_similarity(string1, string2): base_conclusions = stemming.get_sentences(string2, False) ignore = re.compile('[0-9]+[.]?|Заключение|‹#›') clear_conclusions = [ch for ch in base_conclusions if not re.fullmatch(ignore, ch)] - recognized_conclusions = "\n".join(s for s in clear_conclusions if s != further_dev.get('dev_sentence')) + recognized_conclusions = [s for s in clear_conclusions if s != further_dev.get('dev_sentence')] percentage_of_similarity = int(compare_sentences(string1, recognized_conclusions) * 100) From e76bd0588d13aa39e40c59eb0550ff1e33787cc2 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Mon, 29 Apr 2024 17:49:27 +0300 Subject: [PATCH 10/69] update check_similarity for using recognized_conclusions --- app/nlp/similarity_of_texts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/nlp/similarity_of_texts.py b/app/nlp/similarity_of_texts.py index 99d6990a..7621ff5b 100644 --- a/app/nlp/similarity_of_texts.py +++ b/app/nlp/similarity_of_texts.py @@ -10,7 +10,7 @@ def check_similarity(string1, string2): base_conclusions = stemming.get_sentences(string2, False) ignore = re.compile('[0-9]+[.]?|Заключение|‹#›') clear_conclusions = [ch for ch in base_conclusions if not re.fullmatch(ignore, ch)] - recognized_conclusions = [s for s in clear_conclusions if s != further_dev.get('dev_sentence')] + recognized_conclusions = "".join(s for s in clear_conclusions if s != further_dev.get('dev_sentence')) percentage_of_similarity = int(compare_sentences(string1, recognized_conclusions) * 100) From 1086492f8a4bb35e87191483dab93db51d381719 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Mon, 29 Apr 2024 19:10:49 +0300 Subject: [PATCH 11/69] improving parse text for check_similarity --- app/nlp/similarity_of_texts.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/app/nlp/similarity_of_texts.py b/app/nlp/similarity_of_texts.py index 7621ff5b..3dc9cf26 100644 --- a/app/nlp/similarity_of_texts.py +++ b/app/nlp/similarity_of_texts.py @@ -6,12 +6,18 @@ def check_similarity(string1, string2): stemming = Stemming() + + stemming.parse_text(string2, False) further_dev = stemming.further_dev() - base_conclusions = stemming.get_sentences(string2, False) + base_conclusions = stemming.sentences ignore = re.compile('[0-9]+[.]?|Заключение|‹#›') - clear_conclusions = [ch for ch in base_conclusions if not re.fullmatch(ignore, ch)] - recognized_conclusions = "".join(s for s in clear_conclusions if s != further_dev.get('dev_sentence')) + conclusions = [ch for ch in base_conclusions if not re.fullmatch(ignore, ch)] + cleaned_conclusions = "\n".join(s for s in conclusions if s != further_dev.get('dev_sentence')) + + tasks = stemming.get_sentences(string1, True) + ignore = re.compile('[0-9][.]?|Задачи:|‹#›') # [:]? + cleaned_tasks = "\n".join(task for task in tasks if not re.fullmatch(ignore, task)) - percentage_of_similarity = int(compare_sentences(string1, recognized_conclusions) * 100) + percentage_of_similarity = int(compare_sentences(cleaned_tasks, cleaned_conclusions) * 100) - return percentage_of_similarity, further_dev, recognized_conclusions + return percentage_of_similarity, further_dev, conclusions From ddbb5a1a47a290070890193da7369e0e76a4228f Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Mon, 29 Apr 2024 19:11:12 +0300 Subject: [PATCH 12/69] update compare_sentences for tasks/conclusion --- app/nlp/find_tasks_on_slides.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/nlp/find_tasks_on_slides.py b/app/nlp/find_tasks_on_slides.py index 02571718..ef027d48 100644 --- a/app/nlp/find_tasks_on_slides.py +++ b/app/nlp/find_tasks_on_slides.py @@ -12,7 +12,7 @@ def compare_sentences(sentence_1, sentence_2): stemming = Stemming() set_1 = stemming.get_filtered_docs(sentence_1, False) set_2 = stemming.get_filtered_docs(sentence_2, False) - rvector = set_1.union(set_2) + rvector = set_1 #.union(set_2) vector_1 = [w in set_1 for w in rvector] vector_2 = [w in set_2 for w in rvector] cosine_similarity = 1 - distance.cosine(vector_1, vector_2) From b64de77f4ec6631fbf99bbe2e7901474ee6f39b3 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Mon, 29 Apr 2024 19:11:37 +0300 Subject: [PATCH 13/69] uodate SldSimilarity --- .../checks/presentation_checks/sld_similarity.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/app/main/checks/presentation_checks/sld_similarity.py b/app/main/checks/presentation_checks/sld_similarity.py index 47f189ad..de8917ea 100644 --- a/app/main/checks/presentation_checks/sld_similarity.py +++ b/app/main/checks/presentation_checks/sld_similarity.py @@ -1,6 +1,5 @@ -from nlp.similarity_of_texts import check_similarity - from utils import get_text_from_slides, tasks_conclusions_feedback +from app.nlp.similarity_of_texts import check_similarity from app.nlp.stemming import Stemming from ..base_check import BasePresCriterion, answer @@ -19,16 +18,7 @@ def __init__(self, file_info, goals='Цель и задачи', conclusion='За def check(self): goals = get_text_from_slides(self.file, self.goals) conclusions = get_text_from_slides(self.file, self.conclusion) - if goals == "" or conclusions == "": - return answer(False, 'Задач или заключения не существует') - - stemming = Stemming() - stemming.parse_text(conclusions, False) - try: - dev_index = conclusions.index(stemming.further_dev_sentence) - except ValueError: - dev_index = len(conclusions) - results = check_similarity(goals, conclusions[:dev_index]) + results = check_similarity(goals, conclusions) return answer(results[0] >= self.actual_number, *tasks_conclusions_feedback(results)) From 6c304a0ecc8dca62fd51fb4fd1c7e0d9291c5f38 Mon Sep 17 00:00:00 2001 From: Marina Date: Thu, 2 May 2024 12:18:48 +0300 Subject: [PATCH 14/69] base for bootstrap-table --- app/routes/users.py | 43 +++++- app/templates/user_list.html | 49 +++++-- assets/scripts/check_list.js | 2 +- assets/scripts/main.js | 1 + assets/scripts/user_list.js | 256 +++++++++++++++++++++++++++++++++++ 5 files changed, 337 insertions(+), 14 deletions(-) create mode 100644 assets/scripts/user_list.js diff --git a/app/routes/users.py b/app/routes/users.py index 324d7ef1..1ee1704b 100644 --- a/app/routes/users.py +++ b/app/routes/users.py @@ -1,7 +1,9 @@ -from flask import abort, Blueprint, render_template +from flask import abort, Blueprint, render_template, request, jsonify from flask_login import current_user from functools import wraps from app.db.db_methods import get_all_users, get_user +from utils import checklist_filter, format_check_for_table +from db import db_methods users = Blueprint('users', __name__, template_folder='templates', static_folder='static') @@ -15,6 +17,45 @@ def my_wrapper(*args, **kwargs): return my_wrapper + +@users.route("/data") +@admin_required +def user_list_data(): + data = request.args.copy() + filter_query = checklist_filter(data) + # parse and validate rest query + limit = data.get("limit", '') + limit = int(limit) if limit.isdigit() else 10 + + offset = data.get("offset", '') + offset = int(offset) if offset.isdigit() else 0 + + sort = data.get("sort") + sort = 'upload-date' if not sort else sort + + order = data.get("order") + order = 'desc' if not order else order + + sort = "_id" if sort == "upload-date" else sort + + query = dict(filter=filter_query, limit=limit, offset=offset, sort=sort, order=order) + + if data.get("latest"): + rows, count = db_methods.get_latest_check_cursor(**query) + else: + # get data and records count + rows, count = db_methods.get_checks_cursor(**query) + + # construct response + response = { + "total": count, + "rows": [format_check_for_table(item) for item in rows] + } + + # return json data + return jsonify(response) + + @users.route('/', methods=["GET"]) @admin_required def index(): diff --git a/app/templates/user_list.html b/app/templates/user_list.html index c3a9d3aa..7e73756d 100644 --- a/app/templates/user_list.html +++ b/app/templates/user_list.html @@ -6,28 +6,53 @@ {% block title %}Информация о пользователях{% endblock %} {% block main %} +
{% include "header.html" %}
-

+

Список пользователей:

- +
- - + + - - - {% for name, username in usernames %} - - - {% endfor %} - -
UsernameNameUsernameName
{{ name }}{{ username }}
diff --git a/assets/scripts/check_list.js b/assets/scripts/check_list.js index ade99bf7..1480cabf 100644 --- a/assets/scripts/check_list.js +++ b/assets/scripts/check_list.js @@ -253,4 +253,4 @@ function downdloadBlob(blob, filename) { document.body.appendChild(a); a.click(); a.remove(); -} \ No newline at end of file +} diff --git a/assets/scripts/main.js b/assets/scripts/main.js index 13ae8b9c..2ae1ce2e 100644 --- a/assets/scripts/main.js +++ b/assets/scripts/main.js @@ -28,6 +28,7 @@ import './version'; import './check_list'; import './logs'; import './admin_criterions'; +import './user_list'; import '../favicon.ico'; import '../styles/404.css'; diff --git a/assets/scripts/user_list.js b/assets/scripts/user_list.js new file mode 100644 index 00000000..f69a5906 --- /dev/null +++ b/assets/scripts/user_list.js @@ -0,0 +1,256 @@ +import { debounce, isFloat, resetTable, ajaxRequest, onPopState } from "./utils" + +let $table; +const AJAX_URL = "/users/data"; +const filter_prefix = 'filter_'; +let is_latest = false; +let debounceInterval = 500; + + +String.prototype.insert = function (index, string) { + if (index > 0) { + return this.substring(0, index) + string + this.substr(index) + } + return string + this +} + + +$(() => { + initTable() + window.onpopstate = onPopState + + const $dataFilter = $(".bootstrap-table-filter-control-result") + $dataFilter.on("keypress", (e) => { + const val = $dataFilter.val() + const numbers = val.split("-") + + if (e.key === ".") { + const carret = $dataFilter[0].selectionStart + let expectedStr + if (carret <= numbers[0].length) { + expectedStr = numbers[0].insert(carret, ".") + } else { + expectedStr = numbers[1].insert(carret - numbers[0].length - 1, ".") + } + + if (isFloat(expectedStr)) { + return + } + } + + if (e.key >= "0" && e.key <= "9") { + return + } + + if (e.key === "-") { + if (numbers.length === 1) { + return + } + } + + e.preventDefault() + }) +}) + + +function extract_filters(params){ + var filters = {} + for (const [key, value] of Object.entries(params)) { + const index = key.indexOf(filter_prefix) !== -1 + if (index !== -1) { + filters[key.substring(filter_prefix.length)] = value + } + } + return filters +} + + +function initTable() { + $table = $("#user-list-table"); + + // get query string + const queryString = window.location.search; + console.log(queryString) + // parse query search to js object + const params = Object.fromEntries(new URLSearchParams(decodeURIComponent(queryString)).entries()) + // configure filter + params.filter = extract_filters(params) + console.log(params) + + // check correct order query + if (params.order !== "asc" && params.order !== "desc" && params.order !== "") { + params.order = "" + } + + // check correct sort query + if (params.sort !== "") { + let match = false + $table.find("th[data-sortable='true']").each(function () { + if ($(this).data("field") === params.sort) { + match = true + return false + } + }) + + if (match === false) { + params.sort = "" + } + } + + + // check pair of sort and order + if ([params.sort, params.order].includes("")) { + params.sort = "" + params.order = "" + } + + // Fill filters + $table.on("created-controls.bs.table", function () { + if (params.filter) { + console.log(params.filter) + for (const [key, value] of Object.entries(params.filter)) { + const $input = $(`.bootstrap-table-filter-control-${key}`) + $input.val(value) + } + } + }) + + // activate bs table + $table.bootstrapTable({ + pageNumber: parseInt(params.page) || 1, + pageSize: parseInt(params.size) || 10, + sortName: params.sort, + sortOrder: params.order, + buttons: buttons, + + queryParams: queryParams, + ajax: debouncedAjaxRequest, + + columns: [{ + field: "_id", + formatter: idFormatter + }] + }) +} + + +// debounced ajax calls. +const debouncedAjaxRequest = debounce(function(params) {ajaxRequest(AJAX_URL, params)}, debounceInterval); + + +function queryParams(params) { + let filters = {} + $('.filter-control').each(function () { + const name = $(this).parents("th").data("field") + const val = this.querySelector("input").value + if (val) { + filters[name] = val + } + }) + + const query = { + limit: params.limit, + offset: params.offset, + sort: params.sort, + order: params.order, + latest: params.latest + } + + if (!$.isEmptyObject(filters)) { + for (const [key, value] of Object.entries(filters)){ + query[`${filter_prefix}${key}`] = value + } + } + + return query +} + + +function idFormatter(value, row, index, field) { + return `${value.slice(0, 5)}-${value.slice(-5)}` +} + + +function timeStamp() { + var now = new Date(); + var date = [now.getMonth() + 1, now.getDate(), now.getFullYear()]; + var time = [now.getHours(), now.getMinutes(), now.getSeconds()]; + var suffix = (time[0] < 12) ? "AM" : "PM"; + + time[0] = (time[0] < 12) ? time[0] : time[0] - 12; + time[0] = time[0] || 12; + for (var i = 1; i < 3; i++) { + if (time[i] < 10) { + time[i] = "0" + time[i]; + } + } + return '[' + date.join(".") + "_" + time.join(".") + suffix + ']'; +} + + +function buttons() { + let buttonsObj = {}; + + buttonsObj["ResetTable"] = { + text: 'Reset', + event: function() { resetTable($table, queryParams) } + }; + + if (is_admin) { + buttonsObj["FetchCSV"] = { + text: 'CSV', + event: function () { + //const queryString = window.location.search + const params = window.location.search + $("[name=FetchCSV]")[0].innerHTML = " Exporting..." + fetch('get_csv' + '?' + params) + .then(response => response.blob()) + .then(blob => { + $("[name=FetchCSV]")[0].textContent = "CSV" + downdloadBlob(blob, `Статистика.csv`) + }); + } + }; + + buttonsObj["FetchZip"] = { + text: 'Скачать архив', + event: function () { + const params = window.location.search + $("[name=FetchZip]")[0].innerHTML = " Архивирование..." + fetch('get_zip' + '?' + params) + .then(response => response.ok ? response.blob() : false) + .then(blob => { + $("[name=FetchZip]")[0].textContent = "Скачать архив" + if (blob) + downdloadBlob(blob, `Статистика_и_файлы.zip`) + else + alert("Error during file download") + }); + } + }; + + buttonsObj["LatestChecks"] = { + text: 'Latest', + event: function () { + is_latest = !is_latest; + let query = {} + if (is_latest === true){ + query = { query: { latest: is_latest } } + } + $("#user-list-table").bootstrapTable('refresh', query); + } + }; + } + return buttonsObj; +} + + +function downdloadBlob(blob, filename) { + var url = window.URL.createObjectURL(blob); + var a = document.createElement('a'); + a.href = url; + a.download = filename; + document.body.appendChild(a); + a.click(); + a.remove(); +} From 6c765c138b08ca856dd79b7db07daa7c434607e4 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 2 May 2024 18:20:44 +0300 Subject: [PATCH 15/69] update ImageReferences (get images from page) --- app/main/checks/report_checks/image_references.py | 10 ++++++++-- app/main/reports/pdf_document/pdf_document_manager.py | 5 +++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/app/main/checks/report_checks/image_references.py b/app/main/checks/report_checks/image_references.py index 72a1a284..3c378a28 100644 --- a/app/main/checks/report_checks/image_references.py +++ b/app/main/checks/report_checks/image_references.py @@ -26,12 +26,18 @@ def check(self): if not len(self.headers): return answer(False, "Не найдено ни одного заголовка.

Проверьте корректность использования стилей.") number_of_images, all_numbers = self.count_images_vkr() - if not number_of_images: + count_file_image_object = self.file.pdf_file.get_image_num + if not count_file_image_object and not number_of_images: + return answer(False, f'В отчёте найдено {count_file_image_object} рисунков (), но не найдено ни одной подписи рисунка.

Если в вашей работе присутствуют рисунки, убедитесь, что для их подписи был ' + f'использован стиль {self.image_style}, и формат: ' + f'"Рисунок <Номер рисунка> — <Название рисунка>".') + elif not number_of_images: return answer(True, f'Не найдено ни одного рисунка.

Если в вашей работе присутствуют рисунки, убедитесь, что для их подписи был ' f'использован стиль {self.image_style}, и формат: ' - f'"Рисунок <Номер рисунка> -- <Название рисунка>".') + f'"Рисунок <Номер рисунка> — <Название рисунка>".') else: return answer(False, 'Во время обработки произошла критическая ошибка') + references = self.search_references() if len(references.symmetric_difference(all_numbers)) == 0: return answer(True, f"Пройдена!") diff --git a/app/main/reports/pdf_document/pdf_document_manager.py b/app/main/reports/pdf_document/pdf_document_manager.py index 9c72dc3a..27ff7e9e 100644 --- a/app/main/reports/pdf_document/pdf_document_manager.py +++ b/app/main/reports/pdf_document/pdf_document_manager.py @@ -27,6 +27,11 @@ def get_text_on_page(self): # def get_text_on_page(self): # return {page + 1: self.pages[page].extract_text() for page in range(self.page_count_all)} + def get_image_num(self, page_without_pril=None): + if not page_without_pril: + page_without_pril = self.pdf_file.page_count + return sum(map(lambda page: len(self.pdf_file.get_page_images(page)), self.pdf_file[:page_without_pril])) + def page_images(self, page_without_pril): total_height = 0 for page_num in range(page_without_pril): From a82d7ce20239d0ed6051969ff7f7fd17c9e0e7b4 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 2 May 2024 18:27:39 +0300 Subject: [PATCH 16/69] add using new pdf_file method get_image_num --- app/main/checks/report_checks/image_references.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main/checks/report_checks/image_references.py b/app/main/checks/report_checks/image_references.py index 3c378a28..9d7cd5a7 100644 --- a/app/main/checks/report_checks/image_references.py +++ b/app/main/checks/report_checks/image_references.py @@ -26,7 +26,7 @@ def check(self): if not len(self.headers): return answer(False, "Не найдено ни одного заголовка.

Проверьте корректность использования стилей.") number_of_images, all_numbers = self.count_images_vkr() - count_file_image_object = self.file.pdf_file.get_image_num + count_file_image_object = self.file.pdf_file.get_image_num() if not count_file_image_object and not number_of_images: return answer(False, f'В отчёте найдено {count_file_image_object} рисунков (), но не найдено ни одной подписи рисунка.

Если в вашей работе присутствуют рисунки, убедитесь, что для их подписи был ' f'использован стиль {self.image_style}, и формат: ' From 8d63e3874ea870f7555f18c3212256c1e79aea8f Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 2 May 2024 18:40:09 +0300 Subject: [PATCH 17/69] ImageReferences updates --- app/main/checks/report_checks/image_references.py | 2 +- app/main/reports/pdf_document/pdf_document_manager.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/main/checks/report_checks/image_references.py b/app/main/checks/report_checks/image_references.py index 9d7cd5a7..ee008882 100644 --- a/app/main/checks/report_checks/image_references.py +++ b/app/main/checks/report_checks/image_references.py @@ -26,7 +26,7 @@ def check(self): if not len(self.headers): return answer(False, "Не найдено ни одного заголовка.

Проверьте корректность использования стилей.") number_of_images, all_numbers = self.count_images_vkr() - count_file_image_object = self.file.pdf_file.get_image_num() + count_file_image_object = self.file.pdf_file.get_image_num(page_without_pril=self.file.page_count) if not count_file_image_object and not number_of_images: return answer(False, f'В отчёте найдено {count_file_image_object} рисунков (), но не найдено ни одной подписи рисунка.

Если в вашей работе присутствуют рисунки, убедитесь, что для их подписи был ' f'использован стиль {self.image_style}, и формат: ' diff --git a/app/main/reports/pdf_document/pdf_document_manager.py b/app/main/reports/pdf_document/pdf_document_manager.py index 27ff7e9e..a29f5e82 100644 --- a/app/main/reports/pdf_document/pdf_document_manager.py +++ b/app/main/reports/pdf_document/pdf_document_manager.py @@ -30,7 +30,7 @@ def get_text_on_page(self): def get_image_num(self, page_without_pril=None): if not page_without_pril: page_without_pril = self.pdf_file.page_count - return sum(map(lambda page: len(self.pdf_file.get_page_images(page)), self.pdf_file[:page_without_pril])) + return sum(map(lambda page: len(self.pdf_file.get_page_images(page)), self.pages[:page_without_pril])) def page_images(self, page_without_pril): total_height = 0 From 36896ce88b2a031aaf714ef7520fbb137df11b96 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 2 May 2024 18:52:25 +0300 Subject: [PATCH 18/69] update PdfDocumentManager.get_image_num --- app/main/checks/report_checks/image_references.py | 2 +- app/main/reports/pdf_document/pdf_document_manager.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/app/main/checks/report_checks/image_references.py b/app/main/checks/report_checks/image_references.py index ee008882..f9302d40 100644 --- a/app/main/checks/report_checks/image_references.py +++ b/app/main/checks/report_checks/image_references.py @@ -27,7 +27,7 @@ def check(self): return answer(False, "Не найдено ни одного заголовка.

Проверьте корректность использования стилей.") number_of_images, all_numbers = self.count_images_vkr() count_file_image_object = self.file.pdf_file.get_image_num(page_without_pril=self.file.page_count) - if not count_file_image_object and not number_of_images: + if count_file_image_object and not number_of_images: return answer(False, f'В отчёте найдено {count_file_image_object} рисунков (), но не найдено ни одной подписи рисунка.

Если в вашей работе присутствуют рисунки, убедитесь, что для их подписи был ' f'использован стиль {self.image_style}, и формат: ' f'"Рисунок <Номер рисунка> — <Название рисунка>".') diff --git a/app/main/reports/pdf_document/pdf_document_manager.py b/app/main/reports/pdf_document/pdf_document_manager.py index a29f5e82..5ca3bf85 100644 --- a/app/main/reports/pdf_document/pdf_document_manager.py +++ b/app/main/reports/pdf_document/pdf_document_manager.py @@ -27,10 +27,8 @@ def get_text_on_page(self): # def get_text_on_page(self): # return {page + 1: self.pages[page].extract_text() for page in range(self.page_count_all)} - def get_image_num(self, page_without_pril=None): - if not page_without_pril: - page_without_pril = self.pdf_file.page_count - return sum(map(lambda page: len(self.pdf_file.get_page_images(page)), self.pages[:page_without_pril])) + def get_image_num(self): + return len(self.pdf_file.get_page_images(0)) def page_images(self, page_without_pril): total_height = 0 From 5544dfd879579aea8c21d11c3ec26d36f18d1eb0 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 2 May 2024 18:56:46 +0300 Subject: [PATCH 19/69] add get_image_num usage without args --- app/main/checks/report_checks/image_references.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main/checks/report_checks/image_references.py b/app/main/checks/report_checks/image_references.py index f9302d40..609b1325 100644 --- a/app/main/checks/report_checks/image_references.py +++ b/app/main/checks/report_checks/image_references.py @@ -26,7 +26,7 @@ def check(self): if not len(self.headers): return answer(False, "Не найдено ни одного заголовка.

Проверьте корректность использования стилей.") number_of_images, all_numbers = self.count_images_vkr() - count_file_image_object = self.file.pdf_file.get_image_num(page_without_pril=self.file.page_count) + count_file_image_object = self.file.pdf_file.get_image_num() if count_file_image_object and not number_of_images: return answer(False, f'В отчёте найдено {count_file_image_object} рисунков (), но не найдено ни одной подписи рисунка.

Если в вашей работе присутствуют рисунки, убедитесь, что для их подписи был ' f'использован стиль {self.image_style}, и формат: ' From 26ef255c78d224358a20574a2bba8ada11eb4b51 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 2 May 2024 18:59:07 +0300 Subject: [PATCH 20/69] fix ImageReferences feedback --- .../checks/report_checks/image_references.py | 2 +- app/nlp/stemming.py | 1 + app/servants/pre_luncher.py | 6 +++--- docker-compose.yml | 20 +------------------ 4 files changed, 6 insertions(+), 23 deletions(-) diff --git a/app/main/checks/report_checks/image_references.py b/app/main/checks/report_checks/image_references.py index 609b1325..f9f68e6b 100644 --- a/app/main/checks/report_checks/image_references.py +++ b/app/main/checks/report_checks/image_references.py @@ -28,7 +28,7 @@ def check(self): number_of_images, all_numbers = self.count_images_vkr() count_file_image_object = self.file.pdf_file.get_image_num() if count_file_image_object and not number_of_images: - return answer(False, f'В отчёте найдено {count_file_image_object} рисунков (), но не найдено ни одной подписи рисунка.

Если в вашей работе присутствуют рисунки, убедитесь, что для их подписи был ' + return answer(False, f'В отчёте найдено {count_file_image_object} рисунков, но не найдено ни одной подписи рисунка.

Если в вашей работе присутствуют рисунки, убедитесь, что для их подписи был ' f'использован стиль {self.image_style}, и формат: ' f'"Рисунок <Номер рисунка> — <Название рисунка>".') elif not number_of_images: diff --git a/app/nlp/stemming.py b/app/nlp/stemming.py index 784f3fe9..44491dbc 100644 --- a/app/nlp/stemming.py +++ b/app/nlp/stemming.py @@ -27,6 +27,7 @@ def get_sentences(self, string, flag): return self.sentences def parse_text(self, string, flag): + # flag = True -> find tasks, else -> find future dev ?? # morph = MorphAnalyzer() FURTHER_DEVELOPMENT = MORPH_ANALYZER.parse('дальнейшие'.lower())[0].normal_form FURTHER_IMPROVEMENTS = MORPH_ANALYZER.parse('улучшения'.lower())[0].normal_form diff --git a/app/servants/pre_luncher.py b/app/servants/pre_luncher.py index 9e1b3331..20d19e80 100644 --- a/app/servants/pre_luncher.py +++ b/app/servants/pre_luncher.py @@ -36,9 +36,9 @@ def init(app, debug): user.is_admin = True edit_user(user) - user.file_type = DEFAULT_REPORT_TYPE_INFO - file_type = DEFAULT_REPORT_TYPE_INFO['type'] - user.criteria = BASE_PACKS[file_type].name + user.file_type = {'type': 'pres'} + file_type = user.file_type['type'] + user.criteria ="MscPresentationPack" user.formats = list(ALLOWED_EXTENSIONS.get(file_type)) user.two_files = True diff --git a/docker-compose.yml b/docker-compose.yml index a2518331..ecb70807 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,6 @@ services: - "${WEB_PORT:-8080}:8080" depends_on: - mongodb - - worker env_file: - .env environment: @@ -26,7 +25,7 @@ services: worker: image: document_insight_system_image restart: always - command: celery --app=app.tasks.celery worker -n celery@worker --beat -Q ${CELERY_QUERIES} --loglevel=info + command: celery --app=app.tasks.celery worker -n celery@worker --beat -Q ${CELERY_QUERIES} --loglevel=info -c 2 environment: - CELERY_BROKER_URL=${REDIS_URL} - CELERY_RESULT_BACKEND=${REDIS_URL} @@ -41,23 +40,6 @@ services: cpuset: ${CONTAINER_CPU:-0-1} mem_limit: ${WORKER_MEMORY:-1G} - worker-dashboard: - image: document_insight_system_image - command: celery --broker=${REDIS_URL} flower --port=${FLOWER_PORT} --url_prefix=${FLOWER_PREFIX} --basic_auth=${FLOWER_AUTH} --persistent=True - ports: - - "${FLOWER_PORT}:5555" - environment: - - SECRET_KEY=${FLOWER_SECRET_KEY} - - CELERY_BROKER=${REDIS_URL} - - CELERY_BACKEND=${REDIS_URL} - volumes: - - flower_data:/etc/db/ - - "/etc/timezone:/etc/timezone:ro" - - "/etc/localtime:/etc/localtime:ro" - depends_on: - - worker - - redis - cpuset: ${CONTAINER_CPU:-0-1} redis: image: redis:6-alpine From 26867c81aacc7504e03d8e6d80a8fda070487191 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 2 May 2024 19:00:45 +0300 Subject: [PATCH 21/69] Revert "fix ImageReferences feedback" This reverts commit 26ef255c78d224358a20574a2bba8ada11eb4b51. --- .../checks/report_checks/image_references.py | 2 +- app/nlp/stemming.py | 1 - app/servants/pre_luncher.py | 6 +++--- docker-compose.yml | 20 ++++++++++++++++++- 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/app/main/checks/report_checks/image_references.py b/app/main/checks/report_checks/image_references.py index f9f68e6b..609b1325 100644 --- a/app/main/checks/report_checks/image_references.py +++ b/app/main/checks/report_checks/image_references.py @@ -28,7 +28,7 @@ def check(self): number_of_images, all_numbers = self.count_images_vkr() count_file_image_object = self.file.pdf_file.get_image_num() if count_file_image_object and not number_of_images: - return answer(False, f'В отчёте найдено {count_file_image_object} рисунков, но не найдено ни одной подписи рисунка.

Если в вашей работе присутствуют рисунки, убедитесь, что для их подписи был ' + return answer(False, f'В отчёте найдено {count_file_image_object} рисунков (), но не найдено ни одной подписи рисунка.

Если в вашей работе присутствуют рисунки, убедитесь, что для их подписи был ' f'использован стиль {self.image_style}, и формат: ' f'"Рисунок <Номер рисунка> — <Название рисунка>".') elif not number_of_images: diff --git a/app/nlp/stemming.py b/app/nlp/stemming.py index 44491dbc..784f3fe9 100644 --- a/app/nlp/stemming.py +++ b/app/nlp/stemming.py @@ -27,7 +27,6 @@ def get_sentences(self, string, flag): return self.sentences def parse_text(self, string, flag): - # flag = True -> find tasks, else -> find future dev ?? # morph = MorphAnalyzer() FURTHER_DEVELOPMENT = MORPH_ANALYZER.parse('дальнейшие'.lower())[0].normal_form FURTHER_IMPROVEMENTS = MORPH_ANALYZER.parse('улучшения'.lower())[0].normal_form diff --git a/app/servants/pre_luncher.py b/app/servants/pre_luncher.py index 20d19e80..9e1b3331 100644 --- a/app/servants/pre_luncher.py +++ b/app/servants/pre_luncher.py @@ -36,9 +36,9 @@ def init(app, debug): user.is_admin = True edit_user(user) - user.file_type = {'type': 'pres'} - file_type = user.file_type['type'] - user.criteria ="MscPresentationPack" + user.file_type = DEFAULT_REPORT_TYPE_INFO + file_type = DEFAULT_REPORT_TYPE_INFO['type'] + user.criteria = BASE_PACKS[file_type].name user.formats = list(ALLOWED_EXTENSIONS.get(file_type)) user.two_files = True diff --git a/docker-compose.yml b/docker-compose.yml index ecb70807..a2518331 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,7 @@ services: - "${WEB_PORT:-8080}:8080" depends_on: - mongodb + - worker env_file: - .env environment: @@ -25,7 +26,7 @@ services: worker: image: document_insight_system_image restart: always - command: celery --app=app.tasks.celery worker -n celery@worker --beat -Q ${CELERY_QUERIES} --loglevel=info -c 2 + command: celery --app=app.tasks.celery worker -n celery@worker --beat -Q ${CELERY_QUERIES} --loglevel=info environment: - CELERY_BROKER_URL=${REDIS_URL} - CELERY_RESULT_BACKEND=${REDIS_URL} @@ -40,6 +41,23 @@ services: cpuset: ${CONTAINER_CPU:-0-1} mem_limit: ${WORKER_MEMORY:-1G} + worker-dashboard: + image: document_insight_system_image + command: celery --broker=${REDIS_URL} flower --port=${FLOWER_PORT} --url_prefix=${FLOWER_PREFIX} --basic_auth=${FLOWER_AUTH} --persistent=True + ports: + - "${FLOWER_PORT}:5555" + environment: + - SECRET_KEY=${FLOWER_SECRET_KEY} + - CELERY_BROKER=${REDIS_URL} + - CELERY_BACKEND=${REDIS_URL} + volumes: + - flower_data:/etc/db/ + - "/etc/timezone:/etc/timezone:ro" + - "/etc/localtime:/etc/localtime:ro" + depends_on: + - worker + - redis + cpuset: ${CONTAINER_CPU:-0-1} redis: image: redis:6-alpine From 8cd55b6e9b1ae87e9948e66f686c29d3c773a006 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 2 May 2024 19:02:37 +0300 Subject: [PATCH 22/69] update ImageReferences feedback --- app/main/checks/report_checks/image_references.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main/checks/report_checks/image_references.py b/app/main/checks/report_checks/image_references.py index 609b1325..f9f68e6b 100644 --- a/app/main/checks/report_checks/image_references.py +++ b/app/main/checks/report_checks/image_references.py @@ -28,7 +28,7 @@ def check(self): number_of_images, all_numbers = self.count_images_vkr() count_file_image_object = self.file.pdf_file.get_image_num() if count_file_image_object and not number_of_images: - return answer(False, f'В отчёте найдено {count_file_image_object} рисунков (), но не найдено ни одной подписи рисунка.

Если в вашей работе присутствуют рисунки, убедитесь, что для их подписи был ' + return answer(False, f'В отчёте найдено {count_file_image_object} рисунков, но не найдено ни одной подписи рисунка.

Если в вашей работе присутствуют рисунки, убедитесь, что для их подписи был ' f'использован стиль {self.image_style}, и формат: ' f'"Рисунок <Номер рисунка> — <Название рисунка>".') elif not number_of_images: From 13f9aac71433132fb7814ccf76fc4cbace7de809 Mon Sep 17 00:00:00 2001 From: Marina Date: Sat, 4 May 2024 18:29:52 +0300 Subject: [PATCH 23/69] table with filters for /users --- app/db/db_methods.py | 15 ++++ app/routes/users.py | 52 ++++++++------ app/templates/user_list.html | 6 +- assets/scripts/user_list.js | 132 +++++++---------------------------- 4 files changed, 74 insertions(+), 131 deletions(-) diff --git a/app/db/db_methods.py b/app/db/db_methods.py index 3c249340..d80d92db 100644 --- a/app/db/db_methods.py +++ b/app/db/db_methods.py @@ -308,6 +308,21 @@ def get_logs_cursor(filter={}, limit=10, offset=0, sort=None, order=None): return rows, count +def get_user_cursor(filter={}, limit=10, offset=0, sort=None, order=None): + sort = 'username' if sort == 'username' else sort + + count = users_collection.count_documents(filter) + rows = users_collection.find(filter) + + if sort and order in ("asc, desc"): + rows = rows.sort(sort, pymongo.ASCENDING if order == + "asc" else pymongo.DESCENDING) + + rows = rows.skip(offset) if offset else rows + rows = rows.limit(limit) if limit else rows + + return rows, count + # Get stats for one user, return a list in the form # [check_id, login, time of check_id's creation, result(0/1)] diff --git a/app/routes/users.py b/app/routes/users.py index 1ee1704b..bdf20333 100644 --- a/app/routes/users.py +++ b/app/routes/users.py @@ -1,3 +1,4 @@ +import json from flask import abort, Blueprint, render_template, request, jsonify from flask_login import current_user from functools import wraps @@ -20,39 +21,44 @@ def my_wrapper(*args, **kwargs): @users.route("/data") @admin_required -def user_list_data(): - data = request.args.copy() - filter_query = checklist_filter(data) - # parse and validate rest query - limit = data.get("limit", '') - limit = int(limit) if limit.isdigit() else 10 +def users_data(): + filters = request.args.get("filter", "{}") + try: + filters = json.loads(filters) + filters = filters if filters else {} + except Exception as e: + # logger.warning("Can't parse filters") + # logger.warning(repr(e)) + filters = {} + filter_query = {} + if f_username := filters.get("username", None): + filter_query["username"] = {"$regex": f_username} - offset = data.get("offset", '') - offset = int(offset) if offset.isdigit() else 0 + if f_name := filters.get("name", None): + filter_query["name"] = {"$regex": f_name} - sort = data.get("sort") - sort = 'upload-date' if not sort else sort + + limit = request.args.get("limit", "") + limit = int(limit) if limit.isnumeric() else 10 - order = data.get("order") - order = 'desc' if not order else order + offset = request.args.get("offset", "") + offset = int(offset) if offset.isnumeric() else 0 - sort = "_id" if sort == "upload-date" else sort + sort = request.args.get("sort", "") + sort = 'username' if not sort else sort - query = dict(filter=filter_query, limit=limit, offset=offset, sort=sort, order=order) + order = request.args.get("order", "") + order = 'username' if not order else order - if data.get("latest"): - rows, count = db_methods.get_latest_check_cursor(**query) - else: - # get data and records count - rows, count = db_methods.get_checks_cursor(**query) + rows, count = db_methods.get_user_cursor(filter=filter_query, limit=limit, offset=offset, sort=sort, order=order) - # construct response response = { "total": count, - "rows": [format_check_for_table(item) for item in rows] + "rows": [{ + "username": item["username"], + "name": item["name"], + } for item in rows] } - - # return json data return jsonify(response) diff --git a/app/templates/user_list.html b/app/templates/user_list.html index 7e73756d..aa5a82f2 100644 --- a/app/templates/user_list.html +++ b/app/templates/user_list.html @@ -49,7 +49,7 @@

data-show-button-text="true"> - Username + Username Name @@ -57,4 +57,6 @@

-{% endblock %} +{% endblock main %} + + diff --git a/assets/scripts/user_list.js b/assets/scripts/user_list.js index f69a5906..3826cddd 100644 --- a/assets/scripts/user_list.js +++ b/assets/scripts/user_list.js @@ -1,9 +1,7 @@ import { debounce, isFloat, resetTable, ajaxRequest, onPopState } from "./utils" let $table; -const AJAX_URL = "/users/data"; -const filter_prefix = 'filter_'; -let is_latest = false; +const AJAX_URL = "/users/data" let debounceInterval = 500; @@ -53,29 +51,14 @@ $(() => { }) -function extract_filters(params){ - var filters = {} - for (const [key, value] of Object.entries(params)) { - const index = key.indexOf(filter_prefix) !== -1 - if (index !== -1) { - filters[key.substring(filter_prefix.length)] = value - } - } - return filters -} - - function initTable() { $table = $("#user-list-table"); // get query string const queryString = window.location.search; - console.log(queryString) + // parse query search to js object - const params = Object.fromEntries(new URLSearchParams(decodeURIComponent(queryString)).entries()) - // configure filter - params.filter = extract_filters(params) - console.log(params) + const params = Object.fromEntries(new URLSearchParams(queryString).entries()) // check correct order query if (params.order !== "asc" && params.order !== "desc" && params.order !== "") { @@ -107,7 +90,7 @@ function initTable() { // Fill filters $table.on("created-controls.bs.table", function () { if (params.filter) { - console.log(params.filter) + params.filter = JSON.parse(decodeURI(params.filter)) for (const [key, value] of Object.entries(params.filter)) { const $input = $(`.bootstrap-table-filter-control-${key}`) $input.val(value) @@ -122,17 +105,24 @@ function initTable() { sortName: params.sort, sortOrder: params.order, buttons: buttons, + detailView: true, + detailViewIcon: false, + detailViewByClick: true, + detailFormatter: detailFormatter, queryParams: queryParams, ajax: debouncedAjaxRequest, columns: [{ - field: "_id", - formatter: idFormatter + field: "username", + formatter: usernameFormatter }] }) } - + +function usernameFormatter(value, row, index, field) { + return `${value}` +} // debounced ajax calls. const debouncedAjaxRequest = debounce(function(params) {ajaxRequest(AJAX_URL, params)}, debounceInterval); @@ -153,104 +143,34 @@ function queryParams(params) { offset: params.offset, sort: params.sort, order: params.order, - latest: params.latest } if (!$.isEmptyObject(filters)) { - for (const [key, value] of Object.entries(filters)){ - query[`${filter_prefix}${key}`] = value - } + query.filter = JSON.stringify(filters) } return query } -function idFormatter(value, row, index, field) { - return `${value.slice(0, 5)}-${value.slice(-5)}` -} - - -function timeStamp() { - var now = new Date(); - var date = [now.getMonth() + 1, now.getDate(), now.getFullYear()]; - var time = [now.getHours(), now.getMinutes(), now.getSeconds()]; - var suffix = (time[0] < 12) ? "AM" : "PM"; - - time[0] = (time[0] < 12) ? time[0] : time[0] - 12; - time[0] = time[0] || 12; - for (var i = 1; i < 3; i++) { - if (time[i] < 10) { - time[i] = "0" + time[i]; - } - } - return '[' + date.join(".") + "_" + time.join(".") + suffix + ']'; -} - - function buttons() { - let buttonsObj = {}; + let buttonsObj = {} buttonsObj["ResetTable"] = { text: 'Reset', event: function() { resetTable($table, queryParams) } - }; - - if (is_admin) { - buttonsObj["FetchCSV"] = { - text: 'CSV', - event: function () { - //const queryString = window.location.search - const params = window.location.search - $("[name=FetchCSV]")[0].innerHTML = " Exporting..." - fetch('get_csv' + '?' + params) - .then(response => response.blob()) - .then(blob => { - $("[name=FetchCSV]")[0].textContent = "CSV" - downdloadBlob(blob, `Статистика.csv`) - }); - } - }; - - buttonsObj["FetchZip"] = { - text: 'Скачать архив', - event: function () { - const params = window.location.search - $("[name=FetchZip]")[0].innerHTML = " Архивирование..." - fetch('get_zip' + '?' + params) - .then(response => response.ok ? response.blob() : false) - .then(blob => { - $("[name=FetchZip]")[0].textContent = "Скачать архив" - if (blob) - downdloadBlob(blob, `Статистика_и_файлы.zip`) - else - alert("Error during file download") - }); - } - }; - - buttonsObj["LatestChecks"] = { - text: 'Latest', - event: function () { - is_latest = !is_latest; - let query = {} - if (is_latest === true){ - query = { query: { latest: is_latest } } - } - $("#user-list-table").bootstrapTable('refresh', query); - } - }; } - return buttonsObj; + + return buttonsObj } -function downdloadBlob(blob, filename) { - var url = window.URL.createObjectURL(blob); - var a = document.createElement('a'); - a.href = url; - a.download = filename; - document.body.appendChild(a); - a.click(); - a.remove(); +function detailFormatter(index, row) { + var html = [] + $.each(row, function (key, value) { + if (key === 'message' || key === 'pathname') { + html.push('

' + key + ': ' + row[key] + '

') + } + }) + return html.join('') } From 98168da8c865e7387a0e89a9f0bfb1c6a59ed2d0 Mon Sep 17 00:00:00 2001 From: Marina Date: Sun, 5 May 2024 16:39:12 +0300 Subject: [PATCH 24/69] btstr table is changed --- app/routes/users.py | 29 ++++-- app/templates/one_user_info.html | 83 ++++++++------- app/templates/user_list.html | 3 + assets/scripts/main.js | 1 + assets/scripts/one_user_info.js | 167 +++++++++++++++++++++++++++++++ 5 files changed, 242 insertions(+), 41 deletions(-) create mode 100644 assets/scripts/one_user_info.js diff --git a/app/routes/users.py b/app/routes/users.py index bdf20333..d024063a 100644 --- a/app/routes/users.py +++ b/app/routes/users.py @@ -17,8 +17,6 @@ def my_wrapper(*args, **kwargs): abort(403) return my_wrapper - - @users.route("/data") @admin_required def users_data(): @@ -37,7 +35,22 @@ def users_data(): if f_name := filters.get("name", None): filter_query["name"] = {"$regex": f_name} - + if f_formats := filters.get("all_formats", None): + filter_query["formats"] = {"$regex": f_formats} + + if f_criteria := filters.get("all_criteria", None): + filter_query["criteria"] = {"$regex": f_criteria} + + if f_check_counts := filters.get("check_counts", None): + try: + f_check_counts_value, f_check_counts_cond = int(f_check_counts.split()[1]), f_check_counts.split()[0] + if f_check_counts_cond == '>': + filter_query["$expr"] = {"$gte": [{"$size": "$presentations"}, f_check_counts_value]} + elif f_check_counts_cond == "<": + filter_query["$expr"] = {"$lte": [{"$size": "$presentations"}, f_check_counts_value]} + except ValueError: + pass + limit = request.args.get("limit", "") limit = int(limit) if limit.isnumeric() else 10 @@ -57,6 +70,10 @@ def users_data(): "rows": [{ "username": item["username"], "name": item["name"], + "all_formats": item["formats"], + "all_criteria": item["criteria"], + "check_counts": len(item["presentations"]), + } for item in rows] } return jsonify(response) @@ -65,12 +82,10 @@ def users_data(): @users.route('/', methods=["GET"]) @admin_required def index(): - users = list(get_all_users()) - usernames = [(user['name'], user['username']) for user in users] - return render_template('user_list.html', usernames=usernames) + return render_template('user_list.html') @users.route('/', methods=["GET"]) @admin_required def user_info(username): user_info = get_user(username) - return render_template('one_user_info.html', user_info=user_info, check_counts = len(user_info.presentations)) + return render_template('one_user_info.html', user_info=user_info) diff --git a/app/templates/one_user_info.html b/app/templates/one_user_info.html index 23c52ce1..fae90366 100644 --- a/app/templates/one_user_info.html +++ b/app/templates/one_user_info.html @@ -1,50 +1,65 @@ {# Accepts: header dependicies, results, id, filename #} + {% extends "root.html" %} -{% block title %}Информация о пользователе{% endblock %} +{% block title %}Информация о пользователях{% endblock %} {% block main %} +
{% include "header.html" %}
-
-

- Информация о пользователе {{ user_info.username }}: +
+
+

+ Страница пользователя: {{ user_info.username }}

- + Список всех загрузок пользователя +
- - - - - - + + + + + - - - - - - - - - - - - - -
UsernameNameFormatscriteriacount of checkUsernameNameFormatsCriteriaCount of checks
- - {{ user_info.username }}{{ user_info.name }}{{ user_info.criteria }}{{ user_info.formats }}{{ check_counts}}
-
- Список всех загрузок пользователя -
-
+
-{% endblock %} +{% endblock main %} diff --git a/app/templates/user_list.html b/app/templates/user_list.html index aa5a82f2..d6c0baf1 100644 --- a/app/templates/user_list.html +++ b/app/templates/user_list.html @@ -51,6 +51,9 @@

Username Name + Formats + Criteria + Count of checks diff --git a/assets/scripts/main.js b/assets/scripts/main.js index 2ae1ce2e..18bd5f35 100644 --- a/assets/scripts/main.js +++ b/assets/scripts/main.js @@ -29,6 +29,7 @@ import './check_list'; import './logs'; import './admin_criterions'; import './user_list'; +import './one_user_info' import '../favicon.ico'; import '../styles/404.css'; diff --git a/assets/scripts/one_user_info.js b/assets/scripts/one_user_info.js new file mode 100644 index 00000000..c8c46e39 --- /dev/null +++ b/assets/scripts/one_user_info.js @@ -0,0 +1,167 @@ +import { debounce, isFloat, resetTable, ajaxRequest, onPopState } from "./utils" + +let $table; +const AJAX_URL = "/users/data" +let debounceInterval = 500; + + +String.prototype.insert = function (index, string) { + if (index > 0) { + return this.substring(0, index) + string + this.substr(index) + } + return string + this +} + + +$(() => { + initTable() + window.onpopstate = onPopState + + const $dataFilter = $(".bootstrap-table-filter-control-result") + $dataFilter.on("keypress", (e) => { + const val = $dataFilter.val() + const numbers = val.split("-") + + if (e.key === ".") { + const carret = $dataFilter[0].selectionStart + let expectedStr + if (carret <= numbers[0].length) { + expectedStr = numbers[0].insert(carret, ".") + } else { + expectedStr = numbers[1].insert(carret - numbers[0].length - 1, ".") + } + + if (isFloat(expectedStr)) { + return + } + } + + if (e.key >= "0" && e.key <= "9") { + return + } + + if (e.key === "-") { + if (numbers.length === 1) { + return + } + } + + e.preventDefault() + }) +}) + + +function initTable() { + $table = $("#one-user-table"); + + // get query string + const queryString = window.location.search; + + // parse query search to js object + const params = Object.fromEntries(new URLSearchParams(queryString).entries()) + + // check correct order query + if (params.order !== "asc" && params.order !== "desc" && params.order !== "") { + params.order = "" + } + + // check correct sort query + if (params.sort !== "") { + let match = false + $table.find("th[data-sortable='true']").each(function () { + if ($(this).data("field") === params.sort) { + match = true + return false + } + }) + + if (match === false) { + params.sort = "" + } + } + + + // check pair of sort and order + if ([params.sort, params.order].includes("")) { + params.sort = "" + params.order = "" + } + + // Fill filters + $table.on("created-controls.bs.table", function () { + if (params.filter) { + params.filter = JSON.parse(decodeURI(params.filter)) + for (const [key, value] of Object.entries(params.filter)) { + const $input = $(`.bootstrap-table-filter-control-${key}`) + $input.val(value) + } + } + }) + + // activate bs table + $table.bootstrapTable({ + pageNumber: parseInt(params.page) || 1, + pageSize: parseInt(params.size) || 10, + sortName: params.sort, + sortOrder: params.order, + buttons: buttons, + detailView: true, + detailViewIcon: false, + detailViewByClick: true, + detailFormatter: detailFormatter, + + queryParams: queryParams, + ajax: debouncedAjaxRequest, + }) +} + +// debounced ajax calls. +const debouncedAjaxRequest = debounce(function(params) {ajaxRequest(AJAX_URL, params)}, debounceInterval); + + +function queryParams(params) { + let filters = {} + $('.filter-control').each(function () { + const name = $(this).parents("th").data("field") + const val = this.querySelector("input").value + if (val) { + filters[name] = val + } + }) + + const query = { + limit: params.limit, + offset: params.offset, + sort: params.sort, + order: params.order, + } + + if (!$.isEmptyObject(filters)) { + query.filter = JSON.stringify(filters) + } + + return query +} + + +function buttons() { + let buttonsObj = {} + + buttonsObj["ResetTable"] = { + text: 'Reset', + event: function() { resetTable($table, queryParams) } + } + + return buttonsObj +} + + +function detailFormatter(index, row) { + var html = [] + $.each(row, function (key, value) { + if (key === 'message' || key === 'pathname') { + html.push('

' + key + ': ' + row[key] + '

') + } + }) + return html.join('') +} From b74b3b99df204c1c155ce95470958f1e93e66f37 Mon Sep 17 00:00:00 2001 From: Marina Date: Mon, 6 May 2024 11:55:01 +0300 Subject: [PATCH 25/69] admin_pages_list.html is added --- app/routes/admin.py | 2 +- app/templates/admin_pages_list.html | 16 ++++++++++++++++ app/templates/header.html | 1 + 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 app/templates/admin_pages_list.html diff --git a/app/routes/admin.py b/app/routes/admin.py index 4edef375..e3e3214b 100644 --- a/app/routes/admin.py +++ b/app/routes/admin.py @@ -18,7 +18,7 @@ def my_wrapper(*args, **kwargs): @admin.route('/', methods=["GET"]) @admin_required def index(): - return "There will be a list of all admin pages here" + return render_template('admin_pages_list.html') @admin.route('/criterions', methods=["GET"]) diff --git a/app/templates/admin_pages_list.html b/app/templates/admin_pages_list.html new file mode 100644 index 00000000..69869528 --- /dev/null +++ b/app/templates/admin_pages_list.html @@ -0,0 +1,16 @@ +{% extends "root.html" %} + +{% block title %}Список страниц для администраторов{% endblock %} + +{% block main %} + +
{% include "header.html" %}
+
+

Список страниц для администраторов:

+
+
  • Таблица с информацией о критериях
    • +
  • Таблица с информацией о пользователях
    • +
    +
    + +{% endblock main %} diff --git a/app/templates/header.html b/app/templates/header.html index 5abe3a65..0cab17c0 100644 --- a/app/templates/header.html +++ b/app/templates/header.html @@ -17,6 +17,7 @@ Наборы критериев Логи
    + Список админ-страниц Celery Version {% endif %} From 69e0b03ecbdab498e54ebb0db93bbb71b46a7d81 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Mon, 13 May 2024 18:41:08 +0300 Subject: [PATCH 26/69] update page_images method for pdf_document_manager --- app/main/reports/pdf_document/pdf_document_manager.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/main/reports/pdf_document/pdf_document_manager.py b/app/main/reports/pdf_document/pdf_document_manager.py index 9c72dc3a..b9471179 100644 --- a/app/main/reports/pdf_document/pdf_document_manager.py +++ b/app/main/reports/pdf_document/pdf_document_manager.py @@ -33,9 +33,10 @@ def page_images(self, page_without_pril): page = self.pdf_file[page_num] images = self.pdf_file.get_page_images(page) for image in images: - image_coord = page.get_image_bbox(image[7], transform=0) - total_height += (image_coord[3] - image_coord[1]) - + image_coord = page.get_image_bbox(image[7], transform=0) # might be [1.0, 1.0, -1.0, -1.0] + image_height = image_coord[3] - image_coord[1] + if image_height > 0: + total_height += image_height return total_height def page_height(self, page_without_pril): From 67e35fcb2c72b529326489c381e8103767127373 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Mon, 20 May 2024 08:22:51 +0300 Subject: [PATCH 27/69] add NIR_CONFIG --- .../report_checks/style_check_settings.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/app/main/checks/report_checks/style_check_settings.py b/app/main/checks/report_checks/style_check_settings.py index 4392bfdc..23c6a346 100644 --- a/app/main/checks/report_checks/style_check_settings.py +++ b/app/main/checks/report_checks/style_check_settings.py @@ -115,6 +115,24 @@ class StyleCheckSettings: "regex": HEADER_NUM_REGEX } ] + + NIR_CONFIG = [ + { + "style": HEADER_1_STYLE, + "docx_style": ["heading 2"], + "headers": ["ПОСТАНОВКА ЗАДАЧИ", "РЕЗУЛЬТАТЫ РАБОТЫ В ВЕСЕННЕМ СЕМЕСТРЕ", "ОПИСАНИЕ ПРЕДПОЛАГАЕМОГО МЕТОДА РЕШЕНИЯ", + "ПЛАН РАБОТЫ НА ОСЕННИЙ СЕМЕСТР", "СПИСОК ИСПОЛЬЗОВАННЫХ ИСТОЧНИКОВ"], + "unify_regex": None, + "regex": HEADER_REGEX + }, + { + "style": HEADER_1_NUM_STYLE, + "docx_style": ["heading 3", "heading 4"], + "headers": ["ПЛАН", "РЕЗУЛЬТАТЫ"], + "unify_regex": None, + "regex": HEADER_NUM_REGEX + } + ] MD_CONFIG = [ { @@ -197,5 +215,6 @@ class StyleCheckSettings: 'LR_MAIN_TEXT': LR_MAIN_TEXT_CONFIG, 'VKR_HEADERS': VKR_CONFIG, 'VKR_MAIN_TEXT': VKR_MAIN_TEXT_CONFIG, + 'NIR_HEADERS': NIR_CONFIG, 'MD_HEADERS' : MD_CONFIG } From 933352db30ff284417b224f9d36da5516c7fdc57 Mon Sep 17 00:00:00 2001 From: Marina Date: Mon, 3 Jun 2024 17:22:05 +0300 Subject: [PATCH 28/69] check for sequence is added --- .../report_checks/literature_references.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/app/main/checks/report_checks/literature_references.py b/app/main/checks/report_checks/literature_references.py index e271ba4c..889c9a8c 100644 --- a/app/main/checks/report_checks/literature_references.py +++ b/app/main/checks/report_checks/literature_references.py @@ -45,13 +45,16 @@ def check(self): if not number_of_sources: return answer(False, f'В Списке использованных источников не найдено ни одного источника.

    Проверьте корректность использования нумированного списка.') - references = self.search_references(start_literature_par) + references, ref_sequence = self.search_references(start_literature_par) all_numbers = set() for i in range(1, number_of_sources + 1): all_numbers.add(i) if len(references.symmetric_difference(all_numbers)) == 0: if not self.min_ref <= number_of_sources <= self.max_ref: return answer(False, f'Список источников оформлен верно, однако их количество ({number_of_sources}) не удовлетворяет необходимому критерию.
    Количество источников должно быть от {self.min_ref} до {self.max_ref}.') + elif ref_sequence: + result_str += f"Источники должны нумероваться в порядке упоминания в тексте. Неправильная последовательность: [{'], ['.join(num for num in ref_sequence)}]" + return answer(False, result_str) else: return answer(True, f"Пройдена!") elif len(references.difference(all_numbers)): @@ -76,6 +79,8 @@ def check(self): return answer(False, result_str) def search_references(self, start_par): + prev_ref = 0 + ref_sequence = [] array_of_references = set() for i in range(0, start_par): if isinstance(self.file.paragraphs[i], str): @@ -89,9 +94,16 @@ def search_references(self, start_par): start, end = re.split(r'[ -]+', one_part) for k in range(int(start), int(end) + 1): array_of_references.add(k) + if int(k) - prev_ref != 1: + ref_sequence.append(str(k)) + prev_ref = int(k) elif one_part != '': array_of_references.add(int(one_part)) - return array_of_references + if int(one_part) - prev_ref != 1: + ref_sequence.append(str(one_part)) + prev_ref = int(one_part) + print(array_of_references, ref_sequence) + return array_of_references, ref_sequence def find_start_paragraph(self): start_index = 0 From 08ab6f3b44a4833c2186b6bef75d6c2d897db42e Mon Sep 17 00:00:00 2001 From: Marina Date: Mon, 3 Jun 2024 20:10:30 +0300 Subject: [PATCH 29/69] view of result_string is changed --- app/main/checks/report_checks/literature_references.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app/main/checks/report_checks/literature_references.py b/app/main/checks/report_checks/literature_references.py index 889c9a8c..d5e77c76 100644 --- a/app/main/checks/report_checks/literature_references.py +++ b/app/main/checks/report_checks/literature_references.py @@ -53,7 +53,7 @@ def check(self): if not self.min_ref <= number_of_sources <= self.max_ref: return answer(False, f'Список источников оформлен верно, однако их количество ({number_of_sources}) не удовлетворяет необходимому критерию.
    Количество источников должно быть от {self.min_ref} до {self.max_ref}.') elif ref_sequence: - result_str += f"Источники должны нумероваться в порядке упоминания в тексте. Неправильная последовательность: [{'], ['.join(num for num in ref_sequence)}]" + result_str += f"Источники должны нумероваться в порядке упоминания в тексте. Неправильные последовательности: {'; '.join(num for num in ref_sequence)}" return answer(False, result_str) else: return answer(True, f"Пройдена!") @@ -95,14 +95,15 @@ def search_references(self, start_par): for k in range(int(start), int(end) + 1): array_of_references.add(k) if int(k) - prev_ref != 1: - ref_sequence.append(str(k)) + ref_sequence.append(f'[{prev_ref}], [{k}]') prev_ref = int(k) elif one_part != '': array_of_references.add(int(one_part)) if int(one_part) - prev_ref != 1: - ref_sequence.append(str(one_part)) + ref_sequence.append(f'[{prev_ref}], [{one_part}]') prev_ref = int(one_part) - print(array_of_references, ref_sequence) + if ref_sequence[0][1] == '0': + ref_sequence[0] = ref_sequence[0].replace('[0],', '') return array_of_references, ref_sequence def find_start_paragraph(self): From c6e00a9e7278f7de87bf5ef8b6985125524af104 Mon Sep 17 00:00:00 2001 From: Marina Date: Tue, 4 Jun 2024 14:47:46 +0300 Subject: [PATCH 30/69] logic of lit_sequence is changed: now only first mention of literature references is counts --- .../report_checks/literature_references.py | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/app/main/checks/report_checks/literature_references.py b/app/main/checks/report_checks/literature_references.py index d5e77c76..c439a3bc 100644 --- a/app/main/checks/report_checks/literature_references.py +++ b/app/main/checks/report_checks/literature_references.py @@ -93,17 +93,26 @@ def search_references(self, start_par): if re.match(r'\d+[ \-]+\d+', one_part): start, end = re.split(r'[ -]+', one_part) for k in range(int(start), int(end) + 1): + if k not in array_of_references: + if int(k) - prev_ref != 1: + ref_sequence.append(f'[{prev_ref}], [{k}]') + prev_ref = int(k) + else: + if int(k) - prev_ref == 1: + prev_ref = int(k) array_of_references.add(k) - if int(k) - prev_ref != 1: - ref_sequence.append(f'[{prev_ref}], [{k}]') - prev_ref = int(k) elif one_part != '': + if int(one_part) not in array_of_references: + if int(one_part) - prev_ref != 1: + ref_sequence.append(f'[{prev_ref}], [{one_part}]') + prev_ref = int(one_part) + else: + if int(one_part) - prev_ref == 1: + prev_ref = int(one_part) array_of_references.add(int(one_part)) - if int(one_part) - prev_ref != 1: - ref_sequence.append(f'[{prev_ref}], [{one_part}]') - prev_ref = int(one_part) - if ref_sequence[0][1] == '0': - ref_sequence[0] = ref_sequence[0].replace('[0],', '') + if ref_sequence: + if ref_sequence[0][1] == '0': + ref_sequence[0] = ref_sequence[0].replace('[0],', '') return array_of_references, ref_sequence def find_start_paragraph(self): From ca79a8c8af4a3c0e738031df98e623b90b8f3ac6 Mon Sep 17 00:00:00 2001 From: Marina Date: Tue, 4 Jun 2024 18:10:04 +0300 Subject: [PATCH 31/69] new check added "empty page task check" --- app/main/check_packs/pack_config.py | 1 + app/main/checks/report_checks/__init__.py | 3 ++- .../report_checks/empty_task_page_check.py | 25 +++++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 app/main/checks/report_checks/empty_task_page_check.py diff --git a/app/main/check_packs/pack_config.py b/app/main/check_packs/pack_config.py index 598c3cc2..38879ae8 100644 --- a/app/main/check_packs/pack_config.py +++ b/app/main/check_packs/pack_config.py @@ -43,6 +43,7 @@ ["spelling_check"], ["max_abstract_size_check"], ["theme_in_report_check"], + ["empty_task_page_check"], ] DEFAULT_TYPE = 'pres' diff --git a/app/main/checks/report_checks/__init__.py b/app/main/checks/report_checks/__init__.py index 50729335..532f50b9 100644 --- a/app/main/checks/report_checks/__init__.py +++ b/app/main/checks/report_checks/__init__.py @@ -23,4 +23,5 @@ from .style_check import ReportStyleCheck from .spelling_check import SpellingCheck from .max_abstract_size_check import ReportMaxSizeOfAbstractCheck -from .template_name import ReportTemplateNameCheck \ No newline at end of file +from .template_name import ReportTemplateNameCheck +from .empty_task_page_check import EmptyTaskPageCheck diff --git a/app/main/checks/report_checks/empty_task_page_check.py b/app/main/checks/report_checks/empty_task_page_check.py new file mode 100644 index 00000000..4c4a4be1 --- /dev/null +++ b/app/main/checks/report_checks/empty_task_page_check.py @@ -0,0 +1,25 @@ +from ..base_check import BaseReportCriterion, answer + +PAGE_NAME = "ЗАДАНИЕ НА ВЫПУСКНУЮ КВАЛИФИКАЦИОННУЮ РАБОТУ" +TASK_PATTERN = "заданиенавыпускнуюквалификационнуюработу" + +class EmptyTaskPageCheck(BaseReportCriterion): + label = "Проверка на пустоту страницы с заданием" + description = f'Страница "{PAGE_NAME}" должна содержать текст' + id = 'empty_task_page_check' + + def __init__(self, file_info): + super().__init__(file_info) + + def check(self): + if self.file.page_counter() < 4: + return answer(False, "В отчете недостаточно страниц. Нечего проверять.") + task_page = self.file.pdf_file.text_on_page[2] + task_page_string = task_page.replace('\n', ' ').replace(' ', '').lower() + if TASK_PATTERN not in task_page_string: + return answer(False, f'Страница "{PAGE_NAME}" не найдена. Убедитесь, что она является второй в документе и не содержит ошибок в заголовке.') + task_page_without_head = task_page_string.replace(TASK_PATTERN, '') + if len(task_page_without_head) > 1: #not 0 because of page number + return answer(True, 'Пройдена!') + else: + return answer(False, f'Страница "{PAGE_NAME}" не содержит текста.') From 5a2f0ef86bba581962c2bbdc473f6834d2bb5f16 Mon Sep 17 00:00:00 2001 From: Marina Date: Wed, 5 Jun 2024 10:41:30 +0300 Subject: [PATCH 32/69] rided of dublicate parts in code --- .../report_checks/literature_references.py | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/app/main/checks/report_checks/literature_references.py b/app/main/checks/report_checks/literature_references.py index c439a3bc..6a606884 100644 --- a/app/main/checks/report_checks/literature_references.py +++ b/app/main/checks/report_checks/literature_references.py @@ -93,28 +93,25 @@ def search_references(self, start_par): if re.match(r'\d+[ \-]+\d+', one_part): start, end = re.split(r'[ -]+', one_part) for k in range(int(start), int(end) + 1): - if k not in array_of_references: - if int(k) - prev_ref != 1: - ref_sequence.append(f'[{prev_ref}], [{k}]') - prev_ref = int(k) - else: - if int(k) - prev_ref == 1: - prev_ref = int(k) - array_of_references.add(k) + prev_ref = self.add_references(k, prev_ref, array_of_references, ref_sequence) elif one_part != '': - if int(one_part) not in array_of_references: - if int(one_part) - prev_ref != 1: - ref_sequence.append(f'[{prev_ref}], [{one_part}]') - prev_ref = int(one_part) - else: - if int(one_part) - prev_ref == 1: - prev_ref = int(one_part) - array_of_references.add(int(one_part)) + prev_ref = self.add_references(int(one_part), prev_ref, array_of_references, ref_sequence) if ref_sequence: if ref_sequence[0][1] == '0': ref_sequence[0] = ref_sequence[0].replace('[0],', '') return array_of_references, ref_sequence + def add_references(self, k, prev_ref, array_of_references, ref_sequence): + if k not in array_of_references: + if k - prev_ref != 1: + ref_sequence.append(f'[{prev_ref}], [{k}]') + prev_ref = k + else: + if k - prev_ref == 1: + prev_ref = k + array_of_references.add(k) + return prev_ref + def find_start_paragraph(self): start_index = 0 for i in range(len(self.file.paragraphs)): From bc911b900b28a1a21cc0949c17d4289540d034ef Mon Sep 17 00:00:00 2001 From: Marina Date: Mon, 24 Jun 2024 21:20:39 +0300 Subject: [PATCH 33/69] check is changed (first part) --- .../report_checks/empty_task_page_check.py | 27 +++++++++++++------ .../pdf_document/pdf_document_manager.py | 7 ++++- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/app/main/checks/report_checks/empty_task_page_check.py b/app/main/checks/report_checks/empty_task_page_check.py index 4c4a4be1..deb15947 100644 --- a/app/main/checks/report_checks/empty_task_page_check.py +++ b/app/main/checks/report_checks/empty_task_page_check.py @@ -1,7 +1,7 @@ from ..base_check import BaseReportCriterion, answer PAGE_NAME = "ЗАДАНИЕ НА ВЫПУСКНУЮ КВАЛИФИКАЦИОННУЮ РАБОТУ" -TASK_PATTERN = "заданиенавыпускнуюквалификационнуюработу" + class EmptyTaskPageCheck(BaseReportCriterion): label = "Проверка на пустоту страницы с заданием" @@ -14,12 +14,23 @@ def __init__(self, file_info): def check(self): if self.file.page_counter() < 4: return answer(False, "В отчете недостаточно страниц. Нечего проверять.") - task_page = self.file.pdf_file.text_on_page[2] - task_page_string = task_page.replace('\n', ' ').replace(' ', '').lower() - if TASK_PATTERN not in task_page_string: + rows_text = self.file.pdf_file.page_rows_text(1) + print(rows_text) + if 'ЗАДАНИЕ' not in rows_text[0][4]: return answer(False, f'Страница "{PAGE_NAME}" не найдена. Убедитесь, что она является второй в документе и не содержит ошибок в заголовке.') - task_page_without_head = task_page_string.replace(TASK_PATTERN, '') - if len(task_page_without_head) > 1: #not 0 because of page number - return answer(True, 'Пройдена!') - else: + elif len(rows_text) < 4: return answer(False, f'Страница "{PAGE_NAME}" не содержит текста.') + else: + result = {"студент", 'темаработы', 'руководитель'} + check_words = {"студент", 'темаработы', 'руководитель'} + for k in check_words: + for row in rows_text: + row_string = row[4].replace('\n', '').replace(' ', '').replace('_', '').lower() + if k in row_string: + clear_row_string = row_string.replace(k, '') + if len(clear_row_string) > 2: + result.discard(k) + if not result: + return answer(True, 'Пройдена!') + else: + return answer(False, f'Некоторые необходимые поля пусты или отсутствуют. Проверьте: {result}') diff --git a/app/main/reports/pdf_document/pdf_document_manager.py b/app/main/reports/pdf_document/pdf_document_manager.py index 9c72dc3a..cd741e7f 100644 --- a/app/main/reports/pdf_document/pdf_document_manager.py +++ b/app/main/reports/pdf_document/pdf_document_manager.py @@ -46,7 +46,12 @@ def page_height(self, page_without_pril): available_space = (height - top_margin - bottom_margin)*page_without_pril return available_space - + + def page_rows_text(self, page_num): + page = self.pdf_file.load_page(page_num) + text_blocks = page.get_text("blocks") + return text_blocks + # def get_only_text_on_page(self): # if not self.only_text_on_page: # only_text_on_page = {} From 139b7376cf92262cbaa7d9204d6410f73623dae0 Mon Sep 17 00:00:00 2001 From: Marina Date: Thu, 27 Jun 2024 15:45:39 +0300 Subject: [PATCH 34/69] second part of improvement --- .../report_checks/empty_task_page_check.py | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/app/main/checks/report_checks/empty_task_page_check.py b/app/main/checks/report_checks/empty_task_page_check.py index deb15947..b2b9ed2a 100644 --- a/app/main/checks/report_checks/empty_task_page_check.py +++ b/app/main/checks/report_checks/empty_task_page_check.py @@ -1,3 +1,4 @@ +import re from ..base_check import BaseReportCriterion, answer PAGE_NAME = "ЗАДАНИЕ НА ВЫПУСКНУЮ КВАЛИФИКАЦИОННУЮ РАБОТУ" @@ -15,22 +16,30 @@ def check(self): if self.file.page_counter() < 4: return answer(False, "В отчете недостаточно страниц. Нечего проверять.") rows_text = self.file.pdf_file.page_rows_text(1) - print(rows_text) if 'ЗАДАНИЕ' not in rows_text[0][4]: return answer(False, f'Страница "{PAGE_NAME}" не найдена. Убедитесь, что она является второй в документе и не содержит ошибок в заголовке.') elif len(rows_text) < 4: return answer(False, f'Страница "{PAGE_NAME}" не содержит текста.') else: - result = {"студент", 'темаработы', 'руководитель'} - check_words = {"студент", 'темаработы', 'руководитель'} + result = {'Cтудент, Группа', 'Дата выдачи задания, Дата представления ВКР к защите', 'студент', 'руководитель', 'консультант', 'тема работы'} + check_first_pattern = r'^студент+[а-яА-ЯёЁa-zA-Z]+группа\d+$' + check_date = r'^«\d+»[а-яА-ЯёЁa-zA-Z]+20\d+г«\d+»[а-яА-ЯёЁa-zA-Z]+20\d+г$' + start_string = 0 + for row in rows_text: + row_string = row[4].replace('\n', '').replace('.', '').replace(' ', '').replace('_', '').lower() + if re.match(check_first_pattern, row_string): + result.discard('Cтудент, Группа') + start_string = row[5] + if re.match(check_date, row_string): + result.discard('Дата выдачи задания, Дата представления ВКР к защите') + check_words = {'студент', 'руководитель', 'консультант', 'тема работы'} for k in check_words: - for row in rows_text: + for row in rows_text[start_string+1:]: row_string = row[4].replace('\n', '').replace(' ', '').replace('_', '').lower() - if k in row_string: - clear_row_string = row_string.replace(k, '') - if len(clear_row_string) > 2: + if k.replace(' ', '') in row_string: + if len(row_string) > (len(k)+2): result.discard(k) if not result: return answer(True, 'Пройдена!') else: - return answer(False, f'Некоторые необходимые поля пусты или отсутствуют. Проверьте: {result}') + return answer(False, f'Некоторые необходимые поля пусты или отсутствуют. Проверьте поля: {", ".join(result)}') From 3bd6522eef9db2eb8c6627fb79c3ab76ef287c2d Mon Sep 17 00:00:00 2001 From: Marina Date: Fri, 28 Jun 2024 17:17:38 +0300 Subject: [PATCH 35/69] final improvements --- .../report_checks/empty_task_page_check.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/app/main/checks/report_checks/empty_task_page_check.py b/app/main/checks/report_checks/empty_task_page_check.py index b2b9ed2a..229c7808 100644 --- a/app/main/checks/report_checks/empty_task_page_check.py +++ b/app/main/checks/report_checks/empty_task_page_check.py @@ -11,6 +11,10 @@ class EmptyTaskPageCheck(BaseReportCriterion): def __init__(self, file_info): super().__init__(file_info) + self.check_words = {'студент', 'руководитель', 'тема работы'} + self.check_first_pattern = r'^студент+[а-яА-ЯёЁa-zA-Z]+группа\d+$' + self.check_date_pattern = r'^«\d+»[а-яА-ЯёЁa-zA-Z]+20\d+г«\d+»[а-яА-ЯёЁa-zA-Z]+20\d+г$' + self.result = {'Cтудент, Группа', 'Дата выдачи задания, Дата представления ВКР к защите', 'Студент', 'Руководитель', 'Тема работы'} def check(self): if self.file.page_counter() < 4: @@ -21,25 +25,21 @@ def check(self): elif len(rows_text) < 4: return answer(False, f'Страница "{PAGE_NAME}" не содержит текста.') else: - result = {'Cтудент, Группа', 'Дата выдачи задания, Дата представления ВКР к защите', 'студент', 'руководитель', 'консультант', 'тема работы'} - check_first_pattern = r'^студент+[а-яА-ЯёЁa-zA-Z]+группа\d+$' - check_date = r'^«\d+»[а-яА-ЯёЁa-zA-Z]+20\d+г«\d+»[а-яА-ЯёЁa-zA-Z]+20\d+г$' start_string = 0 for row in rows_text: row_string = row[4].replace('\n', '').replace('.', '').replace(' ', '').replace('_', '').lower() - if re.match(check_first_pattern, row_string): - result.discard('Cтудент, Группа') + if re.match(self.check_first_pattern, row_string): + self.result.discard('Cтудент, Группа') start_string = row[5] - if re.match(check_date, row_string): - result.discard('Дата выдачи задания, Дата представления ВКР к защите') - check_words = {'студент', 'руководитель', 'консультант', 'тема работы'} - for k in check_words: + if re.match(self.check_date_pattern, row_string): + self.result.discard('Дата выдачи задания, Дата представления ВКР к защите') + for k in self.check_words: for row in rows_text[start_string+1:]: row_string = row[4].replace('\n', '').replace(' ', '').replace('_', '').lower() if k.replace(' ', '') in row_string: if len(row_string) > (len(k)+2): - result.discard(k) - if not result: + self.result.discard(k.capitalize()) + if not self.result: return answer(True, 'Пройдена!') else: - return answer(False, f'Некоторые необходимые поля пусты или отсутствуют. Проверьте поля: {", ".join(result)}') + return answer(False, f'Некоторые необходимые поля пусты или отсутствуют. Проверьте поля: «{"», «".join(self.result)}»') From b3c0ca20f75f9d60410e79b5d1d124d37987a02a Mon Sep 17 00:00:00 2001 From: Marina Date: Tue, 30 Jul 2024 12:52:50 +0300 Subject: [PATCH 36/69] new check added --- app/main/check_packs/pack_config.py | 1 + .../checks/presentation_checks/__init__.py | 1 + .../name_of_image_check.py | 25 +++++++++ app/servants/pre_luncher.py | 51 ------------------- 4 files changed, 27 insertions(+), 51 deletions(-) create mode 100644 app/main/checks/presentation_checks/name_of_image_check.py delete mode 100644 app/servants/pre_luncher.py diff --git a/app/main/check_packs/pack_config.py b/app/main/check_packs/pack_config.py index 598c3cc2..47253612 100644 --- a/app/main/check_packs/pack_config.py +++ b/app/main/check_packs/pack_config.py @@ -18,6 +18,7 @@ ['pres_empty_slide'], ['theme_in_pres_check'], ['verify_git_link'], + ['pres_image_name'], ] BASE_REPORT_CRITERION = [ ["simple_check"], diff --git a/app/main/checks/presentation_checks/__init__.py b/app/main/checks/presentation_checks/__init__.py index d605c1d3..f4cbd511 100644 --- a/app/main/checks/presentation_checks/__init__.py +++ b/app/main/checks/presentation_checks/__init__.py @@ -13,3 +13,4 @@ from .find_theme_in_pres import FindThemeInPres from .verify_git_link import PresVerifyGitLinkCheck from .empty_slide_check import PresEmptySlideCheck +from .name_of_image_check import PresImageNameCheck diff --git a/app/main/checks/presentation_checks/name_of_image_check.py b/app/main/checks/presentation_checks/name_of_image_check.py new file mode 100644 index 00000000..c5b42ba6 --- /dev/null +++ b/app/main/checks/presentation_checks/name_of_image_check.py @@ -0,0 +1,25 @@ +from ..base_check import BasePresCriterion, answer + + +class PresImageNameCheck(BasePresCriterion): + label = "Проверка наличия подписи к рисункам" + description = '' + id = 'pres_image_name' + + def __init__(self, file_info): + super().__init__(file_info) + + def check(self): + buf = [] + count = 0 + for slide in self.file.slides: + if len(slide.get_images()) > 0: + count += 1 + buf.append(slide.get_page_number()) + buf = ' '.join(list(map(str,buf))) + if count / len(self.file.slides) > self.limit: + return answer(False, f'Проверка не пройдена! Изображения в презентации занимают около {round(count / len(self.file.slides), 2)} количества всех слайдов, \ + ограничение - {round(self.limit, 2)}') + else: + return answer(True, f'Пройдена!') + return answer(False, 'Во время обработки произошла критическая ошибка') diff --git a/app/servants/pre_luncher.py b/app/servants/pre_luncher.py deleted file mode 100644 index 9e1b3331..00000000 --- a/app/servants/pre_luncher.py +++ /dev/null @@ -1,51 +0,0 @@ -import hashlib -import logging - -from db.db_methods import add_user, get_user, get_client, edit_user, save_criteria_pack -from main.check_packs.pack_config import BASE_PACKS, DEFAULT_REPORT_TYPE_INFO - -from pymongo.errors import ConnectionFailure -from server import ALLOWED_EXTENSIONS - -logger = logging.getLogger('root_logger') - - -def get_hash(password): return hashlib.md5(password.encode()).hexdigest() - - -def update_base_check_pack(): - for pack in BASE_PACKS.values(): - save_criteria_pack(pack.to_json()) - - -def init(app, debug): - try: - get_client().admin.command('ismaster') - logger.info("MongoDB работает!") - except ConnectionFailure: - logger.error("MongoDB не доступна!") - return False - - cred_id = "admin" - cred_pass = app.config['ADMIN_PASSWORD'] - user = get_user(cred_id) - - if user is None: - user = add_user(cred_id, get_hash(cred_pass)) - user.name = cred_id - user.is_admin = True - edit_user(user) - - user.file_type = DEFAULT_REPORT_TYPE_INFO - file_type = DEFAULT_REPORT_TYPE_INFO['type'] - user.criteria = BASE_PACKS[file_type].name - user.formats = list(ALLOWED_EXTENSIONS.get(file_type)) - user.two_files = True - - edit_user(user) - - logger.info(f"Создан администратор по умолчанию: логин: {user.username}, пароль уточняйте у разработчика") - - update_base_check_pack() - - return True From 3064fd76fae2f7780bac21e07b5d225f1c01b643 Mon Sep 17 00:00:00 2001 From: Marina Date: Wed, 31 Jul 2024 16:09:46 +0300 Subject: [PATCH 37/69] + check for slides with >2 images --- .../name_of_image_check.py | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/app/main/checks/presentation_checks/name_of_image_check.py b/app/main/checks/presentation_checks/name_of_image_check.py index c5b42ba6..58c74213 100644 --- a/app/main/checks/presentation_checks/name_of_image_check.py +++ b/app/main/checks/presentation_checks/name_of_image_check.py @@ -10,16 +10,18 @@ def __init__(self, file_info): super().__init__(file_info) def check(self): - buf = [] - count = 0 - for slide in self.file.slides: - if len(slide.get_images()) > 0: - count += 1 - buf.append(slide.get_page_number()) - buf = ' '.join(list(map(str,buf))) - if count / len(self.file.slides) > self.limit: - return answer(False, f'Проверка не пройдена! Изображения в презентации занимают около {round(count / len(self.file.slides), 2)} количества всех слайдов, \ - ограничение - {round(self.limit, 2)}') + wrong_slide = [] + for num, slide in enumerate(self.file.slides, 1): + count_image = len(slide.get_images()) + if count_image > 1: + slide_text = slide.get_text() + if slide_text.count('Рисунок') != count_image: + wrong_slide.append(num) + elif len(slide.get_images()) == 1: + # slide_text = slide.get_text() + # slide_title = slide.get_titles() + pass + if wrong_slide: + return answer(False, f'{wrong_slide}') else: - return answer(True, f'Пройдена!') - return answer(False, 'Во время обработки произошла критическая ошибка') + return answer(True, 'Пройдена!') From 267d053d1abbc0d055340983335542f591fc7816 Mon Sep 17 00:00:00 2001 From: Marina Date: Thu, 1 Aug 2024 14:05:22 +0300 Subject: [PATCH 38/69] try with size of placeholder --- .../name_of_image_check.py | 35 ++++++++++++++----- app/main/presentations/pptx/slide_pptx.py | 18 ++++++++++ app/main/presentations/slide_basic.py | 4 +++ 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/app/main/checks/presentation_checks/name_of_image_check.py b/app/main/checks/presentation_checks/name_of_image_check.py index 58c74213..df6b5ddc 100644 --- a/app/main/checks/presentation_checks/name_of_image_check.py +++ b/app/main/checks/presentation_checks/name_of_image_check.py @@ -12,16 +12,33 @@ def __init__(self, file_info): def check(self): wrong_slide = [] for num, slide in enumerate(self.file.slides, 1): - count_image = len(slide.get_images()) - if count_image > 1: - slide_text = slide.get_text() - if slide_text.count('Рисунок') != count_image: - wrong_slide.append(num) - elif len(slide.get_images()) == 1: - # slide_text = slide.get_text() - # slide_title = slide.get_titles() - pass + captions = slide.get_captions() + # print(captions, 'cappp', num) + for caption in captions: + if 'Рисунок' not in caption: + wrong_slide.append(caption) if wrong_slide: return answer(False, f'{wrong_slide}') else: return answer(True, 'Пройдена!') + + + + + + + # if count_image > 1: + # slide_text = slide.get_text() + + # if slide_text.count('Рисунок') != count_image: + # wrong_slide.append(num) + # elif len(slide.get_images()) == 1: + # slide_title = slide.get_title().lower() + + # slide_text = slide.get_text().lower().replace(slide_title, '') + # # print(slide_text) + + # if wrong_slide: + # return answer(False, f'{wrong_slide}') + # else: + # return answer(True, 'Пройдена!') diff --git a/app/main/presentations/pptx/slide_pptx.py b/app/main/presentations/pptx/slide_pptx.py index 46681cdd..37edf167 100644 --- a/app/main/presentations/pptx/slide_pptx.py +++ b/app/main/presentations/pptx/slide_pptx.py @@ -9,6 +9,8 @@ class SlidePPTX(SlideBasic): def __init__(self, container, w, h, index=-1): SlideBasic.__init__(self, container) self.dimensions = [w, h] + self.size_of_shape = [] + self.captions = [] for p in container.placeholders: if p.is_placeholder and p.placeholder_format.type == PP_PLACEHOLDER.SLIDE_NUMBER: if p.text == '‹#›': @@ -26,8 +28,24 @@ def __init__(self, container, w, h, index=-1): self.images.append(shape) if hasattr(shape, "text"): self.text += "\n" + shape.text + if self.text.strip(): + self.size_of_shape.append((shape.text, shape.top, shape.left, shape.width)) if shape.has_table: self.table.append(shape) + if self.images: + sorted_size_of_shape = sorted(self.size_of_shape, key=lambda x:x[1]) + for image in self.images: + for top in sorted_size_of_shape: + if top[1] > (image.top + image.height): + if top[2] > image.left and (top[3]+top[2]) < (image.left+image.width): + caption = top[0] + self.captions.append(caption) + break + if abs(top[2] - image.left) < 400000 and abs((top[3]+top[2]) - (image.left+image.width)) < 400000: + caption = top[0] + self.captions.append(caption) + break + def __str__(self): return super().__str__() diff --git a/app/main/presentations/slide_basic.py b/app/main/presentations/slide_basic.py index 17114320..20d830b5 100644 --- a/app/main/presentations/slide_basic.py +++ b/app/main/presentations/slide_basic.py @@ -6,6 +6,7 @@ def __init__(self, container): # Extracting only the properties we need! self.dimensions = [-1, -1] self.images = [] self.table = [] + self.captions = [] def get_title(self): return self.title @@ -21,6 +22,9 @@ def get_images(self): def get_table(self): return self.table + + def get_captions(self): + return self.captions def __str__(self): return f"\tTitle: {self.title}.\n\tText: {self.text}.\n\tPage_num: {self.page_number}" From 87ceb6a125a888998c8af64b9cdf682924346900 Mon Sep 17 00:00:00 2001 From: Marina Date: Fri, 2 Aug 2024 16:59:13 +0300 Subject: [PATCH 39/69] full check (need more tests) --- .../name_of_image_check.py | 54 +++++++++---------- app/main/presentations/pptx/slide_pptx.py | 22 ++++---- 2 files changed, 35 insertions(+), 41 deletions(-) diff --git a/app/main/checks/presentation_checks/name_of_image_check.py b/app/main/checks/presentation_checks/name_of_image_check.py index df6b5ddc..79fae440 100644 --- a/app/main/checks/presentation_checks/name_of_image_check.py +++ b/app/main/checks/presentation_checks/name_of_image_check.py @@ -1,5 +1,5 @@ from ..base_check import BasePresCriterion, answer - +from app.utils.parse_for_html import format_header class PresImageNameCheck(BasePresCriterion): label = "Проверка наличия подписи к рисункам" @@ -10,35 +10,31 @@ def __init__(self, file_info): super().__init__(file_info) def check(self): - wrong_slide = [] + slides_without_capture = set() + slide_with_image_only = set() + result_str = 'Не пройдена! ' + all_captions = [] for num, slide in enumerate(self.file.slides, 1): captions = slide.get_captions() - # print(captions, 'cappp', num) - for caption in captions: - if 'Рисунок' not in caption: - wrong_slide.append(caption) - if wrong_slide: - return answer(False, f'{wrong_slide}') + if captions: + for caption in captions: + body_text = slide.get_text().replace(captions[0][0], '').replace(slide.get_title(), '').replace('', '').replace(' ', '') + if body_text.strip() or slide.get_table(): + all_captions.append(caption[0]) + if 'Рисунок' not in caption[0]: + slides_without_capture.add(num) + else: + if caption[0] != slide.get_title(): + slide_with_image_only.add(num) + if slides_without_capture: + result_str += format_header( + 'Подписи к рисункам на следующих слайдах отсутствуют или не содержат слова "Рисунок": {}'.format( + ', '.join(self.format_page_link(sorted(slides_without_capture))))) + if slide_with_image_only: + result_str += format_header( + 'Подписи к рисункам на следующих слайдах без текста необязательны: {}'.format( + ', '.join(self.format_page_link(sorted(slide_with_image_only))))) + if result_str: + return answer(False, result_str + f'Список всех обнаруженных подписей: {", ".join(all_captions)}') else: return answer(True, 'Пройдена!') - - - - - - - # if count_image > 1: - # slide_text = slide.get_text() - - # if slide_text.count('Рисунок') != count_image: - # wrong_slide.append(num) - # elif len(slide.get_images()) == 1: - # slide_title = slide.get_title().lower() - - # slide_text = slide.get_text().lower().replace(slide_title, '') - # # print(slide_text) - - # if wrong_slide: - # return answer(False, f'{wrong_slide}') - # else: - # return answer(True, 'Пройдена!') diff --git a/app/main/presentations/pptx/slide_pptx.py b/app/main/presentations/pptx/slide_pptx.py index 37edf167..e71dff7f 100644 --- a/app/main/presentations/pptx/slide_pptx.py +++ b/app/main/presentations/pptx/slide_pptx.py @@ -28,24 +28,22 @@ def __init__(self, container, w, h, index=-1): self.images.append(shape) if hasattr(shape, "text"): self.text += "\n" + shape.text - if self.text.strip(): + if shape.text.replace(' ', '').replace('', ''): # we replace number of page because it is read as text too self.size_of_shape.append((shape.text, shape.top, shape.left, shape.width)) if shape.has_table: self.table.append(shape) + if self.images: - sorted_size_of_shape = sorted(self.size_of_shape, key=lambda x:x[1]) for image in self.images: - for top in sorted_size_of_shape: - if top[1] > (image.top + image.height): - if top[2] > image.left and (top[3]+top[2]) < (image.left+image.width): - caption = top[0] - self.captions.append(caption) - break - if abs(top[2] - image.left) < 400000 and abs((top[3]+top[2]) - (image.left+image.width)) < 400000: - caption = top[0] - self.captions.append(caption) - break + ''' + The next expression finds the most close text for image. + It is work this way, because a holder for picture and a holder for capture don't strictly correspond in size. + For example, sometimes the capture holder runs over the picture holder, the text width is always different ect + ''' + sorted_size_of_shape = sorted(self.size_of_shape, + key=lambda x:(abs(x[1]-(image.top+image.height))+abs(x[2]-image.left) + abs(x[3]+x[2]-(image.left+image.width)))) + self.captions.append(sorted_size_of_shape[0]) def __str__(self): return super().__str__() From bd25fbe580240da6a5ba7eff25072f87247c04cc Mon Sep 17 00:00:00 2001 From: Marina Date: Fri, 2 Aug 2024 17:04:58 +0300 Subject: [PATCH 40/69] info is added, names are changed --- app/main/check_packs/pack_config.py | 2 +- app/main/checks/presentation_checks/__init__.py | 2 +- app/main/checks/presentation_checks/name_of_image_check.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/main/check_packs/pack_config.py b/app/main/check_packs/pack_config.py index 47253612..ee95f88c 100644 --- a/app/main/check_packs/pack_config.py +++ b/app/main/check_packs/pack_config.py @@ -18,7 +18,7 @@ ['pres_empty_slide'], ['theme_in_pres_check'], ['verify_git_link'], - ['pres_image_name'], + ['pres_image_capture'], ] BASE_REPORT_CRITERION = [ ["simple_check"], diff --git a/app/main/checks/presentation_checks/__init__.py b/app/main/checks/presentation_checks/__init__.py index f4cbd511..cb1f3c25 100644 --- a/app/main/checks/presentation_checks/__init__.py +++ b/app/main/checks/presentation_checks/__init__.py @@ -13,4 +13,4 @@ from .find_theme_in_pres import FindThemeInPres from .verify_git_link import PresVerifyGitLinkCheck from .empty_slide_check import PresEmptySlideCheck -from .name_of_image_check import PresImageNameCheck +from .name_of_image_check import PresImageCaptureCheck diff --git a/app/main/checks/presentation_checks/name_of_image_check.py b/app/main/checks/presentation_checks/name_of_image_check.py index 79fae440..99318b80 100644 --- a/app/main/checks/presentation_checks/name_of_image_check.py +++ b/app/main/checks/presentation_checks/name_of_image_check.py @@ -1,10 +1,10 @@ from ..base_check import BasePresCriterion, answer from app.utils.parse_for_html import format_header -class PresImageNameCheck(BasePresCriterion): +class PresImageCaptureCheck(BasePresCriterion): label = "Проверка наличия подписи к рисункам" - description = '' - id = 'pres_image_name' + description = 'Подписи к рисункам должны содержать слово "Рисунок". Подпись к рисункам на слайдах без текста необязательна' + id = 'pres_image_capture' def __init__(self, file_info): super().__init__(file_info) From 37fb32c61e281bab8b5c117adca04a2305c1f2a6 Mon Sep 17 00:00:00 2001 From: Marina Date: Wed, 14 Aug 2024 17:13:08 +0300 Subject: [PATCH 41/69] new test added --- .github/workflows/selenium_tests.yml | 6 ++--- tests/basic_selenium_test.py | 2 +- tests/main.py | 3 ++- tests/scripts/docker_check_tests.sh | 8 +++--- tests/test_criterion_packs_page.py | 39 ++++++++++++++++++++++++++++ 5 files changed, 49 insertions(+), 9 deletions(-) create mode 100644 tests/test_criterion_packs_page.py diff --git a/.github/workflows/selenium_tests.yml b/.github/workflows/selenium_tests.yml index a31e1253..9c30bb76 100644 --- a/.github/workflows/selenium_tests.yml +++ b/.github/workflows/selenium_tests.yml @@ -7,17 +7,17 @@ jobs: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Build docker-compose with docker-compose-selenium (tests) run: | cp .env_example .env cp app/VERSION_example.json app/VERSION.json - docker-compose -f docker-compose.yml -f docker-compose-selenium.yml build + docker compose -f docker-compose.yml -f docker-compose-selenium.yml build - name: Run docker-compose with docker-compose-selenium (tests) run: | - docker-compose -f docker-compose.yml -f docker-compose-selenium.yml up -d + docker compose -f docker-compose.yml -f docker-compose-selenium.yml up -d chmod +x tests/scripts/docker_check_tests.sh ./tests/scripts/docker_check_tests.sh diff --git a/tests/basic_selenium_test.py b/tests/basic_selenium_test.py index 709da385..ae0b64c3 100644 --- a/tests/basic_selenium_test.py +++ b/tests/basic_selenium_test.py @@ -11,7 +11,7 @@ class BasicSeleniumTest(unittest.TestCase): chrome_options = Options() - chrome_options.add_argument("--headless=new") + # chrome_options.add_argument("--headless=new") chrome_options.add_argument('--disable-gpu') chrome_options.add_argument("--no-sandbox") driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=chrome_options) diff --git a/tests/main.py b/tests/main.py index e9de1e39..5f76412e 100644 --- a/tests/main.py +++ b/tests/main.py @@ -8,6 +8,7 @@ from test_single_card_check import SingleCheckTestSelenium from test_version import VersionTestSelenium from test_file_load import FileLoadTestSelenium +from test_criterion_packs_page import CriterionPacksTestSelenium def parse_arguments(): parser = argparse.ArgumentParser(description='Run Selenium tests with specified data') @@ -26,7 +27,7 @@ def main(): args = parse_arguments() suite = unittest.TestSuite() - tests = (AuthTestSelenium, StatisticTestSelenium, FileLoadTestSelenium, SingleCheckTestSelenium, VersionTestSelenium) + tests = (AuthTestSelenium, StatisticTestSelenium, FileLoadTestSelenium, SingleCheckTestSelenium, VersionTestSelenium, CriterionPacksTestSelenium) param = (args.host, args.login, args.password, args.report, args.report_doc, args.pres) for test in tests: suite.addTest(BasicSeleniumTest.parametrize(test, param=param)) diff --git a/tests/scripts/docker_check_tests.sh b/tests/scripts/docker_check_tests.sh index 67deee05..b09d55f9 100755 --- a/tests/scripts/docker_check_tests.sh +++ b/tests/scripts/docker_check_tests.sh @@ -1,7 +1,7 @@ # !/bin/bash service="selenium-tests" -container_id=$(docker-compose -f docker-compose.yml -f docker-compose-selenium.yml ps -q $service) +container_id=$(docker compose -f docker-compose.yml -f docker-compose-selenium.yml ps -q $service) if [ -z "$container_id" ]; then echo "Контейнер сервиса $service не найден." @@ -17,11 +17,11 @@ echo "tests are finished" EXIT_CODE=$(docker inspect "$container_id" --format='{{.State.ExitCode}}') echo "tests logs:" -docker-compose -f docker-compose.yml -f docker-compose-selenium.yml logs selenium-tests +docker compose -f docker-compose.yml -f docker-compose-selenium.yml logs selenium-tests echo "web logs:" -docker-compose logs web +docker compose logs web echo "worker logs:" -docker-compose logs worker +docker compose logs worker if [ "$EXIT_CODE" -eq 0 ]; then echo "tests finished with code $EXIT_CODE (OK)" diff --git a/tests/test_criterion_packs_page.py b/tests/test_criterion_packs_page.py new file mode 100644 index 00000000..be07f323 --- /dev/null +++ b/tests/test_criterion_packs_page.py @@ -0,0 +1,39 @@ +from basic_selenium_test import BasicSeleniumTest +from selenium.webdriver.common.by import By +from selenium.common.exceptions import NoSuchElementException + +class CriterionPacksTestSelenium(BasicSeleniumTest): + + + def begin(self): + self.authorization() + URL = self.get_url('/criterion_packs') + self.get_driver().get(URL) + self.get_driver().implicitly_wait(30) + + def test_open_criterions_pack_list(self): + self.begin() + string_in_table = self.driver.find_element(By.XPATH, "/html/body/div[1]/div[2]/table/tbody/tr[1]/td[4]/a") + self.assertNotEqual(string_in_table, None) + # except NoSuchElementException: + # empty_table = self.driver.find_element(By.CLASS_NAME, "no-records-found") + # self.assertNotEqual(empty_table, None) + + + def test_open_one_pack(self): + self.begin() + string_in_table = self.driver.find_element(By.XPATH, "/html/body/div[1]/div[2]/table/tbody/tr[1]/td[4]/a") + pack_name = string_in_table.get_attribute("href").split("/")[-1] + URL = self.get_url(f'/criterion_pack/{pack_name}') + self.get_driver().get(URL) + obj = self.get_driver().find_element(By.ID, 'raw_criterions') + self.assertNotEquals(obj, None) + + def test_open_new_pack(self): + self.begin() + part_of_link_text = "создать" + xpath_expression = f"//a[contains(translate(text(), 'ABCDEFGHIJKLMNOPQRSTUVWXYZАБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ', 'abcdefghijklmnopqrstuvwxyzабвгдеёжзийклмнопрстуфхцчшщъыьэюя'), '{part_of_link_text.lower()}')]" + link_element = self.driver.find_element(By.XPATH, xpath_expression) + link_element.click() + expected_url = self.get_url('/criterion_pack') + self.assertEqual(self.driver.current_url, expected_url) From fd97961da98cb248f04046fb2b0ec22489cd34f1 Mon Sep 17 00:00:00 2001 From: Marina Date: Wed, 14 Aug 2024 19:47:38 +0300 Subject: [PATCH 42/69] 1/2 of tests --- tests/README.md | 3 +++ tests/basic_selenium_test.py | 2 +- tests/test_criterion_packs_page.py | 15 ++++++++++----- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/README.md b/tests/README.md index 9f341b08..eb9c4586 100644 --- a/tests/README.md +++ b/tests/README.md @@ -70,4 +70,7 @@ class FileLoadTestSelenium(BasicSeleniumTest) with 3 tests Test check: if reports wit different extensions loads correctly use default documents from "/tests" or your own example +### Test for open criterion_packs_page: +class CriterionPacksTestSelenium +group of checks with '/criterion_packs' diff --git a/tests/basic_selenium_test.py b/tests/basic_selenium_test.py index ae0b64c3..709da385 100644 --- a/tests/basic_selenium_test.py +++ b/tests/basic_selenium_test.py @@ -11,7 +11,7 @@ class BasicSeleniumTest(unittest.TestCase): chrome_options = Options() - # chrome_options.add_argument("--headless=new") + chrome_options.add_argument("--headless=new") chrome_options.add_argument('--disable-gpu') chrome_options.add_argument("--no-sandbox") driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=chrome_options) diff --git a/tests/test_criterion_packs_page.py b/tests/test_criterion_packs_page.py index be07f323..0bf703d5 100644 --- a/tests/test_criterion_packs_page.py +++ b/tests/test_criterion_packs_page.py @@ -1,10 +1,10 @@ from basic_selenium_test import BasicSeleniumTest from selenium.webdriver.common.by import By -from selenium.common.exceptions import NoSuchElementException +from selenium.webdriver.support.ui import Select +# from selenium.common.exceptions import NoSuchElementException class CriterionPacksTestSelenium(BasicSeleniumTest): - def begin(self): self.authorization() URL = self.get_url('/criterion_packs') @@ -22,12 +22,17 @@ def test_open_criterions_pack_list(self): def test_open_one_pack(self): self.begin() - string_in_table = self.driver.find_element(By.XPATH, "/html/body/div[1]/div[2]/table/tbody/tr[1]/td[4]/a") + string_in_table = self.driver.find_element(By.XPATH, "//table[contains(@class, 'table')]//tr[1]/td[4]/a") pack_name = string_in_table.get_attribute("href").split("/")[-1] + pack_type = self.driver.find_element(By.XPATH, "//table[contains(@class, 'table')]//tr[1]/td[2]").text.replace('.', ' ').split(' ')[0] URL = self.get_url(f'/criterion_pack/{pack_name}') self.get_driver().get(URL) - obj = self.get_driver().find_element(By.ID, 'raw_criterions') - self.assertNotEquals(obj, None) + opened_pack_name = self.get_driver().find_element(By.ID, 'pack_name').get_attribute('value') + opened_pack_type = self.get_driver().find_element(By.ID, 'file_type') + select = Select(opened_pack_type) + selected_type_text = select.first_selected_option.text.strip() + self.assertEqual(pack_name, opened_pack_name) + self.assertEqual(pack_type, selected_type_text) def test_open_new_pack(self): self.begin() From 1a5ad057b8eebea1aebd504444f0f9931d9fc664 Mon Sep 17 00:00:00 2001 From: Marina Date: Fri, 16 Aug 2024 14:47:13 +0300 Subject: [PATCH 43/69] fix bug with changing of pres pack --- .../presentation_checks/find_def_sld.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/app/main/checks/presentation_checks/find_def_sld.py b/app/main/checks/presentation_checks/find_def_sld.py index 5b363eae..a40518c9 100644 --- a/app/main/checks/presentation_checks/find_def_sld.py +++ b/app/main/checks/presentation_checks/find_def_sld.py @@ -12,18 +12,19 @@ def __init__(self, file_info, key_slide): self.found_idxs = [] def check(self): - for i, title in enumerate(self.file.get_titles(), 1): - if str(title).lower().find(str(self.type_of_slide).lower()) != -1: - #found_slides.append(self.file.get_text_from_slides()[i - 1]) - self.found_idxs.append(i) + if self.file is not None: + for i, title in enumerate(self.file.get_titles(), 1): + if str(title).lower().find(str(self.type_of_slide).lower()) != -1: + #found_slides.append(self.file.get_text_from_slides()[i - 1]) + self.found_idxs.append(i) # save fot future - self.file.found_index[str(self.type_of_slide)] = self.found_idxs.copy() + self.file.found_index[str(self.type_of_slide)] = self.found_idxs.copy() - if self.found_idxs: - return answer(True, 'Найден под номером: {}'.format(', '.join(map(str, self.format_page_link(self.found_idxs))))) - else: - return answer(False, 'Слайд не найден') + if self.found_idxs: + return answer(True, 'Найден под номером: {}'.format(', '.join(map(str, self.format_page_link(self.found_idxs))))) + else: + return answer(False, 'Слайд не найден') @property def name(self): From 2baa0ad753f13e553ad2807cd0e73bde86c17ab5 Mon Sep 17 00:00:00 2001 From: Marina Date: Fri, 16 Aug 2024 15:41:24 +0300 Subject: [PATCH 44/69] tests for one pack added --- tests/test_criterion_packs_page.py | 44 +++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/tests/test_criterion_packs_page.py b/tests/test_criterion_packs_page.py index 0bf703d5..03f417c5 100644 --- a/tests/test_criterion_packs_page.py +++ b/tests/test_criterion_packs_page.py @@ -1,6 +1,8 @@ from basic_selenium_test import BasicSeleniumTest from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import Select +from selenium.webdriver.support.wait import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC # from selenium.common.exceptions import NoSuchElementException class CriterionPacksTestSelenium(BasicSeleniumTest): @@ -11,6 +13,26 @@ def begin(self): self.get_driver().get(URL) self.get_driver().implicitly_wait(30) + def pack_changing(self): + form = self.get_driver().find_element(By.ID, 'raw_criterions') + text_form = form.get_attribute('value') + form.clear() + form.send_keys(text_form) + save_button = self.get_driver().find_element(By.ID, 'pack_submit_button') + save_button.click() + WebDriverWait(self.driver, 10).until(EC.presence_of_element_located((By.ID, "success-text"))) + success_text = self.get_driver().find_element(By.ID, "success-text") + self.assertNotEqual(success_text, None) + + def pack_wrong_changing(self): + form = self.get_driver().find_element(By.ID, 'raw_criterions') + form.send_keys('some_wrong_text') + save_button = self.get_driver().find_element(By.ID, 'pack_submit_button') + save_button.click() + WebDriverWait(self.driver, 10).until(EC.presence_of_element_located((By.ID, "error-text"))) + success_text = self.get_driver().find_element(By.ID, "error-text") + self.assertNotEqual(success_text, None) + def test_open_criterions_pack_list(self): self.begin() string_in_table = self.driver.find_element(By.XPATH, "/html/body/div[1]/div[2]/table/tbody/tr[1]/td[4]/a") @@ -19,8 +41,17 @@ def test_open_criterions_pack_list(self): # empty_table = self.driver.find_element(By.CLASS_NAME, "no-records-found") # self.assertNotEqual(empty_table, None) + def test_open_new_pack(self): + self.begin() + part_of_link_text = "создать" + xpath_expression = f"//a[contains(translate(text(), 'ABCDEFGHIJKLMNOPQRSTUVWXYZАБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ', 'abcdefghijklmnopqrstuvwxyzабвгдеёжзийклмнопрстуфхцчшщъыьэюя'), '{part_of_link_text.lower()}')]" + link_element = self.driver.find_element(By.XPATH, xpath_expression) + link_element.click() + expected_url = self.get_url('/criterion_pack') + self.assertEqual(self.driver.current_url, expected_url) + - def test_open_one_pack(self): + def test_for_one_pack(self): self.begin() string_in_table = self.driver.find_element(By.XPATH, "//table[contains(@class, 'table')]//tr[1]/td[4]/a") pack_name = string_in_table.get_attribute("href").split("/")[-1] @@ -33,12 +64,5 @@ def test_open_one_pack(self): selected_type_text = select.first_selected_option.text.strip() self.assertEqual(pack_name, opened_pack_name) self.assertEqual(pack_type, selected_type_text) - - def test_open_new_pack(self): - self.begin() - part_of_link_text = "создать" - xpath_expression = f"//a[contains(translate(text(), 'ABCDEFGHIJKLMNOPQRSTUVWXYZАБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ', 'abcdefghijklmnopqrstuvwxyzабвгдеёжзийклмнопрстуфхцчшщъыьэюя'), '{part_of_link_text.lower()}')]" - link_element = self.driver.find_element(By.XPATH, xpath_expression) - link_element.click() - expected_url = self.get_url('/criterion_pack') - self.assertEqual(self.driver.current_url, expected_url) + self.pack_changing() + self.pack_wrong_changing() From d89dff75ec5865dd501b9c5a653f7878d3e95cbc Mon Sep 17 00:00:00 2001 From: Marina Date: Fri, 16 Aug 2024 16:10:12 +0300 Subject: [PATCH 45/69] description_tests added --- tests/test_criterion_packs_page.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/test_criterion_packs_page.py b/tests/test_criterion_packs_page.py index 03f417c5..816062dc 100644 --- a/tests/test_criterion_packs_page.py +++ b/tests/test_criterion_packs_page.py @@ -30,8 +30,8 @@ def pack_wrong_changing(self): save_button = self.get_driver().find_element(By.ID, 'pack_submit_button') save_button.click() WebDriverWait(self.driver, 10).until(EC.presence_of_element_located((By.ID, "error-text"))) - success_text = self.get_driver().find_element(By.ID, "error-text") - self.assertNotEqual(success_text, None) + error_text = self.get_driver().find_element(By.ID, "error-text") + self.assertNotEqual(error_text, None) def test_open_criterions_pack_list(self): self.begin() @@ -66,3 +66,12 @@ def test_for_one_pack(self): self.assertEqual(pack_type, selected_type_text) self.pack_changing() self.pack_wrong_changing() + + + def test_pack_description(self): + self.authorization() + description = self.driver.find_element(By.ID, 'btn_table_info') + WebDriverWait(self.driver, 10).until(EC.presence_of_element_located((By.ID, "table_info"))) + table_info = self.get_driver().find_element(By.ID, "table_info") + rows = table_info.find_elements(By.TAG_NAME, 'li') + self.assertNotEqual(rows, None) From a132e58af8fa23030c0b4dd01eb8c11180d2c10f Mon Sep 17 00:00:00 2001 From: Marina Date: Sun, 18 Aug 2024 19:21:12 +0300 Subject: [PATCH 46/69] criterions_admin_test part 1 --- tests/main.py | 10 +++++++++- tests/test_admin_criterions_page.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 tests/test_admin_criterions_page.py diff --git a/tests/main.py b/tests/main.py index 5f76412e..d8001d53 100644 --- a/tests/main.py +++ b/tests/main.py @@ -9,6 +9,7 @@ from test_version import VersionTestSelenium from test_file_load import FileLoadTestSelenium from test_criterion_packs_page import CriterionPacksTestSelenium +from test_admin_criterions_page import AdminCriterionsTestSelenium def parse_arguments(): parser = argparse.ArgumentParser(description='Run Selenium tests with specified data') @@ -27,7 +28,14 @@ def main(): args = parse_arguments() suite = unittest.TestSuite() - tests = (AuthTestSelenium, StatisticTestSelenium, FileLoadTestSelenium, SingleCheckTestSelenium, VersionTestSelenium, CriterionPacksTestSelenium) + tests = (AuthTestSelenium, + StatisticTestSelenium, + FileLoadTestSelenium, + SingleCheckTestSelenium, + VersionTestSelenium, + CriterionPacksTestSelenium, + AdminCriterionsTestSelenium) + param = (args.host, args.login, args.password, args.report, args.report_doc, args.pres) for test in tests: suite.addTest(BasicSeleniumTest.parametrize(test, param=param)) diff --git a/tests/test_admin_criterions_page.py b/tests/test_admin_criterions_page.py new file mode 100644 index 00000000..aa116825 --- /dev/null +++ b/tests/test_admin_criterions_page.py @@ -0,0 +1,29 @@ +from basic_selenium_test import BasicSeleniumTest +from selenium.webdriver.common.by import By + +class AdminCriterionsTestSelenium(BasicSeleniumTest): + + def begin(self): + self.authorization() + URL = self.get_url('/admin/criterions') + self.get_driver().get(URL) + self.get_driver().implicitly_wait(30) + + def test_open_criterions_list(self): + self.begin() + table = self.driver.find_element(By.ID, 'results_table') + self.assertNotEqual(table, None) + rows = table.find_elements(By.TAG_NAME, 'tr') + self.assertNotEqual(rows, None) + headers = self.driver.find_elements(By.XPATH, "//table//th") + header_id = any("id" in header.text.lower() for header in headers) + header_label = any('label' in header.text.lower() for header in headers) + self.assertTrue(header_id, 'Id header is not found') + self.assertTrue(header_label, 'label header is not found') + + def test_open_description(self): + self.begin() + button = self.driver.find_element(By.XPATH, '//table[contains(@class, "table")]//tbody/tr[1]/td[1]/i') + button.click() + description = self.driver.find_element(By.XPATH, '//table[contains(@class, "table")]//tbody/tr[2]/td').text.strip() + self.assertNotEqual(description, None) From e9618aef0bff6f95ed0255439eb70c84ebbe59a2 Mon Sep 17 00:00:00 2001 From: Marina Date: Sun, 18 Aug 2024 19:31:01 +0300 Subject: [PATCH 47/69] header (criterion_pack_list_tests) added --- tests/test_criterion_packs_page.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_criterion_packs_page.py b/tests/test_criterion_packs_page.py index 816062dc..b94b56ef 100644 --- a/tests/test_criterion_packs_page.py +++ b/tests/test_criterion_packs_page.py @@ -35,8 +35,16 @@ def pack_wrong_changing(self): def test_open_criterions_pack_list(self): self.begin() - string_in_table = self.driver.find_element(By.XPATH, "/html/body/div[1]/div[2]/table/tbody/tr[1]/td[4]/a") + string_in_table = self.driver.find_element(By.XPATH, "//table[contains(@class, 'table')]//tbody/tr[1]/td[4]/a") self.assertNotEqual(string_in_table, None) + headers = self.driver.find_elements(By.XPATH, "//table//th") + header_name = any("название" in header.text.lower() for header in headers) + header_type = any('тип' in header.text.lower() for header in headers) + header_edit = any('редактировать' in header.text.lower() for header in headers) + self.assertTrue(header_name, 'header "название" is not found') + self.assertTrue(header_type, 'header "тип" is not found') + self.assertTrue(header_edit, 'header "редактировать" is not found') + # except NoSuchElementException: # empty_table = self.driver.find_element(By.CLASS_NAME, "no-records-found") # self.assertNotEqual(empty_table, None) From 37c59d11288eef0d6fd60957e3d0ed3d59b92f3e Mon Sep 17 00:00:00 2001 From: Marina Date: Sun, 18 Aug 2024 19:55:13 +0300 Subject: [PATCH 48/69] condition for admin added --- tests/test_admin_criterions_page.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_admin_criterions_page.py b/tests/test_admin_criterions_page.py index aa116825..7048a0ae 100644 --- a/tests/test_admin_criterions_page.py +++ b/tests/test_admin_criterions_page.py @@ -5,9 +5,17 @@ class AdminCriterionsTestSelenium(BasicSeleniumTest): def begin(self): self.authorization() - URL = self.get_url('/admin/criterions') + URL = self.get_url('/admin') self.get_driver().get(URL) self.get_driver().implicitly_wait(30) + page_text = self.driver.find_element(By.TAG_NAME, "body").text + if 'администратор' not in page_text.lower(): + self.skipTest("This test runs inly with admin's login and password") + else: + URL = self.get_url('/admin/criterions') + self.get_driver().get(URL) + self.get_driver().implicitly_wait(30) + def test_open_criterions_list(self): self.begin() From 62aae28e642f399e9627c52e6f2ce46687668244 Mon Sep 17 00:00:00 2001 From: Marina Date: Sat, 24 Aug 2024 12:44:16 +0300 Subject: [PATCH 49/69] remove "-" from docker compose and add v4 --- .github/workflows/selenium_tests.yml | 6 +++--- tests/scripts/docker_check_tests.sh | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/selenium_tests.yml b/.github/workflows/selenium_tests.yml index a31e1253..9c30bb76 100644 --- a/.github/workflows/selenium_tests.yml +++ b/.github/workflows/selenium_tests.yml @@ -7,17 +7,17 @@ jobs: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Build docker-compose with docker-compose-selenium (tests) run: | cp .env_example .env cp app/VERSION_example.json app/VERSION.json - docker-compose -f docker-compose.yml -f docker-compose-selenium.yml build + docker compose -f docker-compose.yml -f docker-compose-selenium.yml build - name: Run docker-compose with docker-compose-selenium (tests) run: | - docker-compose -f docker-compose.yml -f docker-compose-selenium.yml up -d + docker compose -f docker-compose.yml -f docker-compose-selenium.yml up -d chmod +x tests/scripts/docker_check_tests.sh ./tests/scripts/docker_check_tests.sh diff --git a/tests/scripts/docker_check_tests.sh b/tests/scripts/docker_check_tests.sh index 67deee05..b09d55f9 100755 --- a/tests/scripts/docker_check_tests.sh +++ b/tests/scripts/docker_check_tests.sh @@ -1,7 +1,7 @@ # !/bin/bash service="selenium-tests" -container_id=$(docker-compose -f docker-compose.yml -f docker-compose-selenium.yml ps -q $service) +container_id=$(docker compose -f docker-compose.yml -f docker-compose-selenium.yml ps -q $service) if [ -z "$container_id" ]; then echo "Контейнер сервиса $service не найден." @@ -17,11 +17,11 @@ echo "tests are finished" EXIT_CODE=$(docker inspect "$container_id" --format='{{.State.ExitCode}}') echo "tests logs:" -docker-compose -f docker-compose.yml -f docker-compose-selenium.yml logs selenium-tests +docker compose -f docker-compose.yml -f docker-compose-selenium.yml logs selenium-tests echo "web logs:" -docker-compose logs web +docker compose logs web echo "worker logs:" -docker-compose logs worker +docker compose logs worker if [ "$EXIT_CODE" -eq 0 ]; then echo "tests finished with code $EXIT_CODE (OK)" From 7bce448e6b6ea94dcbac069aa574262435ed113b Mon Sep 17 00:00:00 2001 From: Marina Date: Sat, 24 Aug 2024 13:09:34 +0300 Subject: [PATCH 50/69] remove "-" and v4 in build.yml --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0ff659da..0be14874 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Build system images (non-pulling) run: | @@ -16,10 +16,10 @@ jobs: - name: Build docker-compose run: | cp .env_example .env - docker-compose build + docker compose build - name: Run docker-compose run: | - docker-compose up -d + docker compose up -d sleep 10 - name: Run tests run: | From c7a6977ca32508099490addcc65bde88dbfb7ecc Mon Sep 17 00:00:00 2001 From: Marina Date: Sun, 25 Aug 2024 21:04:01 +0300 Subject: [PATCH 51/69] fix actions (remove "-") --- .github/workflows/build.yml | 4 ++-- .github/workflows/selenium_tests.yml | 6 +++--- tests/scripts/docker_check_tests.sh | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0ff659da..dfc884d5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,10 +16,10 @@ jobs: - name: Build docker-compose run: | cp .env_example .env - docker-compose build + docker compose build - name: Run docker-compose run: | - docker-compose up -d + docker compose up -d sleep 10 - name: Run tests run: | diff --git a/.github/workflows/selenium_tests.yml b/.github/workflows/selenium_tests.yml index a31e1253..9c30bb76 100644 --- a/.github/workflows/selenium_tests.yml +++ b/.github/workflows/selenium_tests.yml @@ -7,17 +7,17 @@ jobs: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Build docker-compose with docker-compose-selenium (tests) run: | cp .env_example .env cp app/VERSION_example.json app/VERSION.json - docker-compose -f docker-compose.yml -f docker-compose-selenium.yml build + docker compose -f docker-compose.yml -f docker-compose-selenium.yml build - name: Run docker-compose with docker-compose-selenium (tests) run: | - docker-compose -f docker-compose.yml -f docker-compose-selenium.yml up -d + docker compose -f docker-compose.yml -f docker-compose-selenium.yml up -d chmod +x tests/scripts/docker_check_tests.sh ./tests/scripts/docker_check_tests.sh diff --git a/tests/scripts/docker_check_tests.sh b/tests/scripts/docker_check_tests.sh index 67deee05..b09d55f9 100755 --- a/tests/scripts/docker_check_tests.sh +++ b/tests/scripts/docker_check_tests.sh @@ -1,7 +1,7 @@ # !/bin/bash service="selenium-tests" -container_id=$(docker-compose -f docker-compose.yml -f docker-compose-selenium.yml ps -q $service) +container_id=$(docker compose -f docker-compose.yml -f docker-compose-selenium.yml ps -q $service) if [ -z "$container_id" ]; then echo "Контейнер сервиса $service не найден." @@ -17,11 +17,11 @@ echo "tests are finished" EXIT_CODE=$(docker inspect "$container_id" --format='{{.State.ExitCode}}') echo "tests logs:" -docker-compose -f docker-compose.yml -f docker-compose-selenium.yml logs selenium-tests +docker compose -f docker-compose.yml -f docker-compose-selenium.yml logs selenium-tests echo "web logs:" -docker-compose logs web +docker compose logs web echo "worker logs:" -docker-compose logs worker +docker compose logs worker if [ "$EXIT_CODE" -eq 0 ]; then echo "tests finished with code $EXIT_CODE (OK)" From 51f4d4de688036556cba28e105be5454deec8901 Mon Sep 17 00:00:00 2001 From: Marina Date: Sun, 25 Aug 2024 21:06:40 +0300 Subject: [PATCH 52/69] improving of html view --- .../name_of_image_check.py | 12 ++--- app/servants/pre_luncher.py | 51 +++++++++++++++++++ app/utils/__init__.py | 2 +- app/utils/parse_for_html.py | 6 +++ 4 files changed, 64 insertions(+), 7 deletions(-) create mode 100644 app/servants/pre_luncher.py diff --git a/app/main/checks/presentation_checks/name_of_image_check.py b/app/main/checks/presentation_checks/name_of_image_check.py index 99318b80..faffdf17 100644 --- a/app/main/checks/presentation_checks/name_of_image_check.py +++ b/app/main/checks/presentation_checks/name_of_image_check.py @@ -1,5 +1,5 @@ from ..base_check import BasePresCriterion, answer -from app.utils.parse_for_html import format_header +from utils import name_of_image_check_results class PresImageCaptureCheck(BasePresCriterion): label = "Проверка наличия подписи к рисункам" @@ -27,14 +27,14 @@ def check(self): if caption[0] != slide.get_title(): slide_with_image_only.add(num) if slides_without_capture: - result_str += format_header( + result_str += ( 'Подписи к рисункам на следующих слайдах отсутствуют или не содержат слова "Рисунок": {}'.format( - ', '.join(self.format_page_link(sorted(slides_without_capture))))) + ', '.join(self.format_page_link(sorted(slides_without_capture)))) + '
    ') if slide_with_image_only: - result_str += format_header( + result_str += ( 'Подписи к рисункам на следующих слайдах без текста необязательны: {}'.format( - ', '.join(self.format_page_link(sorted(slide_with_image_only))))) + ', '.join(self.format_page_link(sorted(slide_with_image_only)))) + '
    ') if result_str: - return answer(False, result_str + f'Список всех обнаруженных подписей: {", ".join(all_captions)}') + return answer(False, name_of_image_check_results(result_str, all_captions)) else: return answer(True, 'Пройдена!') diff --git a/app/servants/pre_luncher.py b/app/servants/pre_luncher.py new file mode 100644 index 00000000..9e1b3331 --- /dev/null +++ b/app/servants/pre_luncher.py @@ -0,0 +1,51 @@ +import hashlib +import logging + +from db.db_methods import add_user, get_user, get_client, edit_user, save_criteria_pack +from main.check_packs.pack_config import BASE_PACKS, DEFAULT_REPORT_TYPE_INFO + +from pymongo.errors import ConnectionFailure +from server import ALLOWED_EXTENSIONS + +logger = logging.getLogger('root_logger') + + +def get_hash(password): return hashlib.md5(password.encode()).hexdigest() + + +def update_base_check_pack(): + for pack in BASE_PACKS.values(): + save_criteria_pack(pack.to_json()) + + +def init(app, debug): + try: + get_client().admin.command('ismaster') + logger.info("MongoDB работает!") + except ConnectionFailure: + logger.error("MongoDB не доступна!") + return False + + cred_id = "admin" + cred_pass = app.config['ADMIN_PASSWORD'] + user = get_user(cred_id) + + if user is None: + user = add_user(cred_id, get_hash(cred_pass)) + user.name = cred_id + user.is_admin = True + edit_user(user) + + user.file_type = DEFAULT_REPORT_TYPE_INFO + file_type = DEFAULT_REPORT_TYPE_INFO['type'] + user.criteria = BASE_PACKS[file_type].name + user.formats = list(ALLOWED_EXTENSIONS.get(file_type)) + user.two_files = True + + edit_user(user) + + logger.info(f"Создан администратор по умолчанию: логин: {user.username}, пароль уточняйте у разработчика") + + update_base_check_pack() + + return True diff --git a/app/utils/__init__.py b/app/utils/__init__.py index 9b948c7f..6fe84709 100644 --- a/app/utils/__init__.py +++ b/app/utils/__init__.py @@ -5,7 +5,7 @@ from .get_file_len import get_file_len from .get_text_from_slides import get_text_from_slides from .parse_for_html import format_descriptions, format_header, find_tasks_on_slides_feedback, \ - tasks_conclusions_feedback + tasks_conclusions_feedback, name_of_image_check_results from .repeated_timer import RepeatedTimer from .timezone import timezone_offset from .check_file import check_file diff --git a/app/utils/parse_for_html.py b/app/utils/parse_for_html.py index b03376aa..d118810d 100644 --- a/app/utils/parse_for_html.py +++ b/app/utils/parse_for_html.py @@ -25,3 +25,9 @@ def tasks_conclusions_feedback(results): return format_header('Соответствует на {}%'.format(results[0]), tag.br), \ 'Распознанные заключения: ', \ *format_descriptions(results[2], tag.div_class, tag.br + tag.close_div) + +def name_of_image_check_results(result_str, all_captions): + return format_header(result_str) + \ + f'Список всех обнаруженных подписей:' + \ + ''.join(format_descriptions(all_captions, tag.div_class, tag.br + tag.close_div)) + From a1aff7a6011f979f86891f5e8779b7a6e2b44468 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Mon, 26 Aug 2024 14:26:55 +0300 Subject: [PATCH 53/69] update get_csv/zip routes for token access --- README.md | 1 + app/server.py | 15 +++++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 8abfbaa9..8aa9524e 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ SIGNUP_PAGE_ENABLED=... CONSUMER_KEY=... CONSUMER_SECRET=... +ACCESS_TOKEN=... ``` ## Run diff --git a/app/server.py b/app/server.py index b8dbc5b7..c374224f 100644 --- a/app/server.py +++ b/app/server.py @@ -507,11 +507,19 @@ def get_stats(): return [format_check_for_table(item, set_link=URL_DOMEN) for item in rows] +def check_access_token(access_token): + # if request has access_token, and it's equal to ACCESS_TOKEN from env -> accept, esle - check user + return access_token and (access_token == os.environ.getenv('ACCESS_TOKEN')) + + +def check_export_access(): + return current_user.is_admin or check_access_token(request.args.get('access_token', None)) + + @app.route("/get_csv") -@login_required def get_csv(): from io import StringIO - if not current_user.is_admin: + if not check_export_access(): abort(403) response = get_stats() df = pd.read_json(StringIO(json.dumps(response))) @@ -523,9 +531,8 @@ def get_csv(): @app.route("/get_zip") -@login_required def get_zip(): - if not current_user.is_admin: + if not check_export_access(): abort(403) original_names = request.args.get('original_names', False) == 'true' From 9bdbe2ac684da811dcd25ef147c97f001f863c06 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Mon, 26 Aug 2024 14:32:06 +0300 Subject: [PATCH 54/69] improve check_export_access --- app/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/server.py b/app/server.py index c374224f..1a7bab47 100644 --- a/app/server.py +++ b/app/server.py @@ -513,7 +513,8 @@ def check_access_token(access_token): def check_export_access(): - return current_user.is_admin or check_access_token(request.args.get('access_token', None)) + return check_access_token(request.args.get('access_token', None)) \ + or (current_user.is_authenticated and current_user.is_admin) @app.route("/get_csv") From 95a17262b8ac34585c296d884fd84c530ad10fb9 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Mon, 26 Aug 2024 14:37:02 +0300 Subject: [PATCH 55/69] fix os.environ using --- app/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/server.py b/app/server.py index 1a7bab47..1ee67c8f 100644 --- a/app/server.py +++ b/app/server.py @@ -509,7 +509,7 @@ def get_stats(): def check_access_token(access_token): # if request has access_token, and it's equal to ACCESS_TOKEN from env -> accept, esle - check user - return access_token and (access_token == os.environ.getenv('ACCESS_TOKEN')) + return access_token and (access_token == os.environ.get('ACCESS_TOKEN')) def check_export_access(): From 45a3e4f07580763b1362b5eef5e3285f26b19caf Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Mon, 26 Aug 2024 14:44:07 +0300 Subject: [PATCH 56/69] update checklist_filter logic --- app/server.py | 4 ++-- app/utils/checklist_filter.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/server.py b/app/server.py index 1ee67c8f..366978d9 100644 --- a/app/server.py +++ b/app/server.py @@ -489,8 +489,8 @@ def check_list_data(): def get_query(req): - # query for download csv/zip - filter_query = checklist_filter(req.args) + # query for download csv/zip (only for admins) + filter_query = checklist_filter(req.args, is_admin=True) limit = False offset = False sort = req.args.get("sort", "") diff --git a/app/utils/checklist_filter.py b/app/utils/checklist_filter.py index 795c9d2a..9e809dde 100644 --- a/app/utils/checklist_filter.py +++ b/app/utils/checklist_filter.py @@ -8,7 +8,7 @@ logger = logging.getLogger('root_logger') FILTER_PREFIX = 'filter_' -def checklist_filter(data): +def checklist_filter(data, is_admin=False): from utils import timezone_offset filters = {key[len(FILTER_PREFIX):]: data[key] for key in data if key.startswith(FILTER_PREFIX)} @@ -80,7 +80,7 @@ def checklist_filter(data): logger.warning(repr(e)) # set user filter for current non-admin user - if not current_user.is_admin: + if not (is_admin or current_user.is_admin): filter_query["user"] = current_user.username return filter_query From 2aea76b5b3ba22f95b64c4d842f5519633e3f398 Mon Sep 17 00:00:00 2001 From: Marina Date: Tue, 27 Aug 2024 10:48:22 +0300 Subject: [PATCH 57/69] change structure of style_settings --- .../report_checks/needed_headers_check.py | 16 ++-- .../report_checks/style_check_settings.py | 79 +++++++++++++------ 2 files changed, 67 insertions(+), 28 deletions(-) diff --git a/app/main/checks/report_checks/needed_headers_check.py b/app/main/checks/report_checks/needed_headers_check.py index 14223ebe..dc584de6 100644 --- a/app/main/checks/report_checks/needed_headers_check.py +++ b/app/main/checks/report_checks/needed_headers_check.py @@ -18,18 +18,22 @@ def __init__(self, file_info, main_heading_style="heading 2", headers_map=None): self.config = headers_map else: self.config = 'VKR_HEADERS' if (self.file_type['report_type'] == 'VKR') else 'LR_HEADERS' - self.patterns = StyleCheckSettings.CONFIGS.get(self.config)[0]["headers"] + # self.patterns = StyleCheckSettings.CONFIGS.get(self.config)[0]["headers"] def late_init(self): self.headers = self.file.make_chapters(self.file_type['report_type']) self.headers_page = self.file.find_header_page(self.file_type['report_type']) self.chapters_str = self.file.show_chapters(self.file_type['report_type']) - # TODO: change self.headers_main = self.file.get_main_headers(self.file_type['report_type']) - if self.headers_main == "Задание 1": - self.patterns = StyleCheckSettings.CONFIGS.get(self.config)[0]["headers"] - elif self.headers_main == "Задание 2": - self.patterns = StyleCheckSettings.CONFIGS.get(self.config)[1]["headers"] + print(self.headers_main) + if self.headers_main in StyleCheckSettings.CONFIGS.get(self.config): + self.patterns = StyleCheckSettings.CONFIGS.get(self.config)[self.headers_main]['headers'] + else: + self.patterns = StyleCheckSettings.CONFIGS.get(self.config)['any_header']['headers'] + # if self.headers_main == "Задание 1": + # self.patterns = StyleCheckSettings.CONFIGS.get(self.config)[0]["headers"] + # elif self.headers_main == "Задание 2": + # self.patterns = StyleCheckSettings.CONFIGS.get(self.config)[1]["headers"] def check(self): if self.file.page_counter() < 4: diff --git a/app/main/checks/report_checks/style_check_settings.py b/app/main/checks/report_checks/style_check_settings.py index 23c6a346..5723e5fd 100644 --- a/app/main/checks/report_checks/style_check_settings.py +++ b/app/main/checks/report_checks/style_check_settings.py @@ -82,14 +82,14 @@ class StyleCheckSettings: } # Order of styles may be significant! First level 1, then level 2 and so on. - LR_CONFIG = [ + LR_CONFIG = {'any_header': { "style": HEADER_1_STYLE, "docx_style": ["heading 1"], "headers": ["Исходный код программы"], "unify_regex": APPENDIX_UNIFY_REGEX, "regex": APPENDIX_REGEX, - }, + },'second_header': { "style": HEADER_2_STYLE, "docx_style": ["heading 2"], @@ -97,16 +97,16 @@ class StyleCheckSettings: "unify_regex": None, "regex": HEADER_1_REGEX } - ] + } - VKR_CONFIG = [ + VKR_CONFIG = {'any_header': { "style": HEADER_1_STYLE, "docx_style": ["heading 2"], "headers": ["ВВЕДЕНИЕ", "ЗАКЛЮЧЕНИЕ", "СПИСОК ИСПОЛЬЗОВАННЫХ ИСТОЧНИКОВ"], "unify_regex": None, "regex": HEADER_REGEX - }, + },'second_header': { "style": HEADER_1_NUM_STYLE, "docx_style": ["heading 2", "heading 3", "heading 4"], @@ -114,9 +114,9 @@ class StyleCheckSettings: "unify_regex": None, "regex": HEADER_NUM_REGEX } - ] + } - NIR_CONFIG = [ + NIR_CONFIG = {'any_header': { "style": HEADER_1_STYLE, "docx_style": ["heading 2"], @@ -124,7 +124,7 @@ class StyleCheckSettings: "ПЛАН РАБОТЫ НА ОСЕННИЙ СЕМЕСТР", "СПИСОК ИСПОЛЬЗОВАННЫХ ИСТОЧНИКОВ"], "unify_regex": None, "regex": HEADER_REGEX - }, + }, 'second_header': { "style": HEADER_1_NUM_STYLE, "docx_style": ["heading 3", "heading 4"], @@ -132,9 +132,10 @@ class StyleCheckSettings: "unify_regex": None, "regex": HEADER_NUM_REGEX } - ] + } - MD_CONFIG = [ + MD_CONFIG = { + 'Задание 1': { "style": HEADER_1_STYLE, "docx_style": ["heading 2"], @@ -146,6 +147,7 @@ class StyleCheckSettings: "unify_regex": None, "regex": HEADER_REGEX }, + 'Задание 2': { "style": HEADER_1_STYLE, "docx_style": ["heading 2"], @@ -155,22 +157,54 @@ class StyleCheckSettings: ], "unify_regex": None, "regex": HEADER_REGEX - }, + } + # },'': + # { + # "style": HEADER_1_NUM_STYLE, + # "docx_style": ["heading 2", "heading 3", "heading 4"], + # "headers": [], + # "unify_regex": None, + # "regex": HEADER_NUM_REGEX + # },'': + # { + # "style": "Main_header", + # "docx_style": ["heading 1"], + # "headers": ["Задание"], + # "unify_regex": None, + # "regex": HEADER_NUM_REGEX + # } + } + + OPNP_CONFIG = { + 'Сравнение аналогов': { - "style": HEADER_1_NUM_STYLE, - "docx_style": ["heading 2", "heading 3", "heading 4"], - "headers": [], + "style": HEADER_1_STYLE, + "docx_style": ["heading 2"], + "headers": ["Принцип отбора аналогов", + "Критерии сравнения аналогов", + "Выводы по итогам сравнения", + "Выбор метода решения", + ], "unify_regex": None, - "regex": HEADER_NUM_REGEX + "regex": HEADER_REGEX }, + 'any_header': { - "style": "Main_header", - "docx_style": ["heading 1"], - "headers": ["Задание"], + "style": HEADER_1_STYLE, + "docx_style": ["heading 2"], + "headers": ["Аннотация", + "Введение", + "Обзор предметной области == Сравнение аналогов", + "Выбор метода решения", + "Заключение" + "Список литературы" + ], "unify_regex": None, - "regex": HEADER_NUM_REGEX - } - ] + "regex": HEADER_REGEX + }, + + } + LR_MAIN_TEXT_CONFIG = [ { @@ -216,5 +250,6 @@ class StyleCheckSettings: 'VKR_HEADERS': VKR_CONFIG, 'VKR_MAIN_TEXT': VKR_MAIN_TEXT_CONFIG, 'NIR_HEADERS': NIR_CONFIG, - 'MD_HEADERS' : MD_CONFIG + 'MD_HEADERS' : MD_CONFIG, + 'OPNP_HEADERS' : OPNP_CONFIG, } From 1389104e17acff0b912079ac831449afb2738d79 Mon Sep 17 00:00:00 2001 From: Marina Date: Tue, 27 Aug 2024 21:31:09 +0300 Subject: [PATCH 58/69] checks are changed(because of list --> dict), needed_headers added --- app/main/checks/report_checks/chapters.py | 4 ++-- .../checks/report_checks/needed_headers_check.py | 14 ++++++-------- app/main/checks/report_checks/sections_check.py | 2 +- .../checks/report_checks/short_sections_check.py | 2 +- app/main/checks/report_checks/style_check.py | 2 +- .../checks/report_checks/style_check_settings.py | 2 +- 6 files changed, 12 insertions(+), 14 deletions(-) diff --git a/app/main/checks/report_checks/chapters.py b/app/main/checks/report_checks/chapters.py index ad181135..e822d7fc 100644 --- a/app/main/checks/report_checks/chapters.py +++ b/app/main/checks/report_checks/chapters.py @@ -16,13 +16,13 @@ def __init__(self, file_info): self.target_styles = StyleCheckSettings.VKR_CONFIG self.target_styles = list(map(lambda elem: { "style": self.construct_style_from_description(elem["style"]) - }, self.target_styles)) + }, self.target_styles.values())) self.docx_styles = {} self.style_regex = {} self.config = 'VKR_HEADERS' if (self.file_type['report_type'] == 'VKR') else 'LR_HEADERS' self.presets = StyleCheckSettings.CONFIGS.get(self.config) level = 0 - for format_description in self.presets: + for _, format_description in self.presets.items(): self.docx_styles.update({level: format_description["docx_style"]}) pattern = re.compile(format_description["regex"]) self.style_regex.update({level: pattern}) diff --git a/app/main/checks/report_checks/needed_headers_check.py b/app/main/checks/report_checks/needed_headers_check.py index dc584de6..64c815ef 100644 --- a/app/main/checks/report_checks/needed_headers_check.py +++ b/app/main/checks/report_checks/needed_headers_check.py @@ -25,20 +25,18 @@ def late_init(self): self.headers_page = self.file.find_header_page(self.file_type['report_type']) self.chapters_str = self.file.show_chapters(self.file_type['report_type']) self.headers_main = self.file.get_main_headers(self.file_type['report_type']) - print(self.headers_main) if self.headers_main in StyleCheckSettings.CONFIGS.get(self.config): self.patterns = StyleCheckSettings.CONFIGS.get(self.config)[self.headers_main]['headers'] else: - self.patterns = StyleCheckSettings.CONFIGS.get(self.config)['any_header']['headers'] - # if self.headers_main == "Задание 1": - # self.patterns = StyleCheckSettings.CONFIGS.get(self.config)[0]["headers"] - # elif self.headers_main == "Задание 2": - # self.patterns = StyleCheckSettings.CONFIGS.get(self.config)[1]["headers"] + if 'any_header' in StyleCheckSettings.CONFIGS.get(self.config): + self.patterns = StyleCheckSettings.CONFIGS.get(self.config)['any_header']['headers'] def check(self): if self.file.page_counter() < 4: return answer(False, "В отчете недостаточно страниц. Нечего проверять.") self.late_init() + if not self.patterns: + return answer(False, "Не удалось сформировать требуемые заголовки исходя из названия работы. Проверьте наименование работы.") result_string = '' patterns = [] for pattern in self.patterns: @@ -59,7 +57,7 @@ def check(self): if not result_string: result_str = f'Все необходимые заголовки обнаружены!' result_str += f'

    Ниже представлена иерархия обработанных заголовков, ' \ - f'сравните с Содержанием {self.format_page_link([self.headers_page])}:' + f'сравните с Содержанием {self.format_page_link([self.headers_page])}:' result_str += self.chapters_str result_str += '
    Если список не точный, убедитесь, что для каждого заголовка указан верный стиль.' return answer(True, result_str) @@ -74,7 +72,7 @@ def check(self): ''' result_str += f'

    Ниже представлена иерархия обработанных заголовков, ' \ - f'сравните с Содержанием {self.format_page_link([self.headers_page])}:' + f'сравните с Содержанием {self.format_page_link([self.headers_page])}:' result_str += self.chapters_str result_str += '
    Если список не точный, убедитесь, что для каждого заголовка указан верный стиль.' return answer(False, result_str) diff --git a/app/main/checks/report_checks/sections_check.py b/app/main/checks/report_checks/sections_check.py index 887eabb2..8fc03139 100644 --- a/app/main/checks/report_checks/sections_check.py +++ b/app/main/checks/report_checks/sections_check.py @@ -32,7 +32,7 @@ def check(self): self.file.parse_effective_styles() result = True result_str = "" - for preset in self.presets: + for _, preset in self.presets.items(): full_style = self.construct_style_from_description(preset["style"]) precheck_dict = {key: preset["style"].get(key) for key in self.prechecked_props} precheck_style = self.construct_style_from_description(precheck_dict) diff --git a/app/main/checks/report_checks/short_sections_check.py b/app/main/checks/report_checks/short_sections_check.py index df9d3360..35d76b6e 100644 --- a/app/main/checks/report_checks/short_sections_check.py +++ b/app/main/checks/report_checks/short_sections_check.py @@ -25,7 +25,7 @@ def __init__(self, file_info, min_section_count=5, min_section_len=20, main_head if prechecked_props_lst is None: prechecked_props_lst = StyleCheckSettings.PRECHECKED_PROPS self.styles: List[Style] = [] - for format_description in self.presets: + for _, format_description in self.presets.items(): prechecked_dict = {key: format_description["style"].get(key) for key in prechecked_props_lst} style = Style() style.__dict__.update(prechecked_dict) diff --git a/app/main/checks/report_checks/style_check.py b/app/main/checks/report_checks/style_check.py index 94e06453..e421a828 100644 --- a/app/main/checks/report_checks/style_check.py +++ b/app/main/checks/report_checks/style_check.py @@ -25,7 +25,7 @@ def __init__(self, file_info, header_styles=None, target_styles=None, key_proper self.target_styles)) if header_styles is None: self.header_styles = [] - for style_dict in StyleCheckSettings.LR_CONFIG: + for _, style_dict in StyleCheckSettings.LR_CONFIG.items(): header_style = {key: style_dict["style"].get(key) for key in StyleCheckSettings.PRECHECKED_PROPS} style = Style() style.__dict__.update(header_style) diff --git a/app/main/checks/report_checks/style_check_settings.py b/app/main/checks/report_checks/style_check_settings.py index 5723e5fd..f327d823 100644 --- a/app/main/checks/report_checks/style_check_settings.py +++ b/app/main/checks/report_checks/style_check_settings.py @@ -196,7 +196,7 @@ class StyleCheckSettings: "Введение", "Обзор предметной области == Сравнение аналогов", "Выбор метода решения", - "Заключение" + "Заключение", "Список литературы" ], "unify_regex": None, From 3057424a02d2fcfda1c2c197c59ba9a5fca60875 Mon Sep 17 00:00:00 2001 From: Marina Date: Tue, 27 Aug 2024 21:34:17 +0300 Subject: [PATCH 59/69] fix actions (remove"-" from docker-compose) --- .github/workflows/build.yml | 6 +++--- .github/workflows/selenium_tests.yml | 8 ++++---- tests/scripts/docker_check_tests.sh | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0ff659da..0be14874 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Build system images (non-pulling) run: | @@ -16,10 +16,10 @@ jobs: - name: Build docker-compose run: | cp .env_example .env - docker-compose build + docker compose build - name: Run docker-compose run: | - docker-compose up -d + docker compose up -d sleep 10 - name: Run tests run: | diff --git a/.github/workflows/selenium_tests.yml b/.github/workflows/selenium_tests.yml index a31e1253..aa850300 100644 --- a/.github/workflows/selenium_tests.yml +++ b/.github/workflows/selenium_tests.yml @@ -7,17 +7,17 @@ jobs: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - - name: Build docker-compose with docker-compose-selenium (tests) + - name: Build docker compose with docker-compose-selenium (tests) run: | cp .env_example .env cp app/VERSION_example.json app/VERSION.json - docker-compose -f docker-compose.yml -f docker-compose-selenium.yml build + docker compose -f docker-compose.yml -f docker-compose-selenium.yml build - name: Run docker-compose with docker-compose-selenium (tests) run: | - docker-compose -f docker-compose.yml -f docker-compose-selenium.yml up -d + docker compose -f docker-compose.yml -f docker-compose-selenium.yml up -d chmod +x tests/scripts/docker_check_tests.sh ./tests/scripts/docker_check_tests.sh diff --git a/tests/scripts/docker_check_tests.sh b/tests/scripts/docker_check_tests.sh index 67deee05..b09d55f9 100755 --- a/tests/scripts/docker_check_tests.sh +++ b/tests/scripts/docker_check_tests.sh @@ -1,7 +1,7 @@ # !/bin/bash service="selenium-tests" -container_id=$(docker-compose -f docker-compose.yml -f docker-compose-selenium.yml ps -q $service) +container_id=$(docker compose -f docker-compose.yml -f docker-compose-selenium.yml ps -q $service) if [ -z "$container_id" ]; then echo "Контейнер сервиса $service не найден." @@ -17,11 +17,11 @@ echo "tests are finished" EXIT_CODE=$(docker inspect "$container_id" --format='{{.State.ExitCode}}') echo "tests logs:" -docker-compose -f docker-compose.yml -f docker-compose-selenium.yml logs selenium-tests +docker compose -f docker-compose.yml -f docker-compose-selenium.yml logs selenium-tests echo "web logs:" -docker-compose logs web +docker compose logs web echo "worker logs:" -docker-compose logs worker +docker compose logs worker if [ "$EXIT_CODE" -eq 0 ]; then echo "tests finished with code $EXIT_CODE (OK)" From 3d4d6be1dd0c3c42e021122ac8a789865ea4c636 Mon Sep 17 00:00:00 2001 From: Marina Date: Wed, 28 Aug 2024 11:57:12 +0300 Subject: [PATCH 60/69] impoving of section check --- app/main/checks/report_checks/section_component.py | 8 +++++++- app/main/checks/report_checks/style_check_settings.py | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/app/main/checks/report_checks/section_component.py b/app/main/checks/report_checks/section_component.py index ac049e15..6dba0813 100644 --- a/app/main/checks/report_checks/section_component.py +++ b/app/main/checks/report_checks/section_component.py @@ -11,7 +11,8 @@ def __init__(self, file_info, chapter='Введение', patterns=('цель', super().__init__(file_info) self.intro = {} if headers_map: - self.chapter = StyleCheckSettings.CONFIGS.get(headers_map)[0]["headers"][0] + self.config = headers_map + self.chapter = None patterns = ('цель', 'задач') else: self.chapter = chapter @@ -21,12 +22,17 @@ def __init__(self, file_info, chapter='Введение', patterns=('цель', self.patterns.append({"name": pattern.capitalize(), "text": pattern, "marker": 0}) def late_init(self): + if self.chapter is None: + self.headers_main = self.file.get_main_headers(self.file_type['report_type']) + self.chapter = StyleCheckSettings.CONFIGS.get(self.config)[self.headers_main]["header_for_report_section_component"] self.chapters = self.file.make_chapters(self.file_type['report_type']) def check(self): if self.file.page_counter() < 4: return answer(False, "В отчете недостаточно страниц. Нечего проверять.") self.late_init() + if not self.chapter: + return answer(True, f'Данная проверка не предусмотрена для работы с темой "{self.headers_main}"') result_str = '' for intro in self.chapters: header = intro["text"].lower() diff --git a/app/main/checks/report_checks/style_check_settings.py b/app/main/checks/report_checks/style_check_settings.py index f327d823..a7d9308f 100644 --- a/app/main/checks/report_checks/style_check_settings.py +++ b/app/main/checks/report_checks/style_check_settings.py @@ -144,6 +144,7 @@ class StyleCheckSettings: "Методы обоснования", "Статья", ], + "header_for_report_section_component": "Поставленная цель и задачи", "unify_regex": None, "regex": HEADER_REGEX }, @@ -155,6 +156,7 @@ class StyleCheckSettings: "Характеристика выводов", "Статья", ], + "header_for_report_section_component": "", "unify_regex": None, "regex": HEADER_REGEX } From a699df427911e81d225d984fa3564e9d463f9c60 Mon Sep 17 00:00:00 2001 From: Marina Date: Wed, 28 Aug 2024 12:57:30 +0300 Subject: [PATCH 61/69] add new check^ key_words --- app/main/check_packs/pack_config.py | 1 + app/main/checks/report_checks/__init__.py | 3 ++- .../checks/report_checks/key_words_check.py | 24 +++++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 app/main/checks/report_checks/key_words_check.py diff --git a/app/main/check_packs/pack_config.py b/app/main/check_packs/pack_config.py index 598c3cc2..051cb20a 100644 --- a/app/main/check_packs/pack_config.py +++ b/app/main/check_packs/pack_config.py @@ -43,6 +43,7 @@ ["spelling_check"], ["max_abstract_size_check"], ["theme_in_report_check"], + ['Key_words_report_check'], ] DEFAULT_TYPE = 'pres' diff --git a/app/main/checks/report_checks/__init__.py b/app/main/checks/report_checks/__init__.py index 50729335..c917b31b 100644 --- a/app/main/checks/report_checks/__init__.py +++ b/app/main/checks/report_checks/__init__.py @@ -23,4 +23,5 @@ from .style_check import ReportStyleCheck from .spelling_check import SpellingCheck from .max_abstract_size_check import ReportMaxSizeOfAbstractCheck -from .template_name import ReportTemplateNameCheck \ No newline at end of file +from .template_name import ReportTemplateNameCheck +from .key_words_check import KeyWordsReportCheck diff --git a/app/main/checks/report_checks/key_words_check.py b/app/main/checks/report_checks/key_words_check.py new file mode 100644 index 00000000..999f03ed --- /dev/null +++ b/app/main/checks/report_checks/key_words_check.py @@ -0,0 +1,24 @@ +import re +from ..base_check import BaseReportCriterion, answer + + +class KeyWordsReportCheck(BaseReportCriterion): + label = 'Проверка наличия раздела "Ключевые слова"' + description = 'Раздел идет сразу после названия работы и содержит не менее трех ключевых слов' + id = 'Key_words_report_check' + + def __init__(self, file_info, min_key_words = 3): + super().__init__(file_info) + self.min_key_words = min_key_words + + def check(self): + key_words_chapter = self.file.paragraphs[1].lower() + if 'ключевые слова' not in key_words_chapter: + return answer(False, 'Раздел "Ключевые слова" не найден') + cleaned_str = re.sub(r'<[^>]*>', '', key_words_chapter) + final_str = cleaned_str.replace('ключевые слова', '').replace(':','').replace(' ', '') + key_words = final_str.split(',') + if len(key_words) >= self.min_key_words: + return answer(True, 'Пройдена!') + else: + return answer(False, f'Не пройдена! Количество ключевых слов должно быть не менее {self.min_key_words}') From 392de37c9e9215d628e0a4fab6dba48c533a433f Mon Sep 17 00:00:00 2001 From: Marina Date: Thu, 29 Aug 2024 11:49:21 +0300 Subject: [PATCH 62/69] fix count of paragraphs, lit_ref and section_component --- app/main/checks/report_checks/literature_references.py | 1 + app/main/checks/report_checks/section_component.py | 7 ++++--- app/main/reports/md_uploader/md_uploader.py | 3 +-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app/main/checks/report_checks/literature_references.py b/app/main/checks/report_checks/literature_references.py index e271ba4c..96f71791 100644 --- a/app/main/checks/report_checks/literature_references.py +++ b/app/main/checks/report_checks/literature_references.py @@ -71,6 +71,7 @@ def check(self):
  • Убедитесь, что для ссылки на источник используются квадратные скобки;
  • Убедитесь, что для оформления списка литературы был использован нумированный список;
  • Убедитесь, что после и перед нумированным списком отсутствуют непустые абзацы.
    • +
  • Убедитесь, что один источник не разбит на двае строки клавишей "Enter".
    • ''' return answer(False, result_str) diff --git a/app/main/checks/report_checks/section_component.py b/app/main/checks/report_checks/section_component.py index 6dba0813..68781533 100644 --- a/app/main/checks/report_checks/section_component.py +++ b/app/main/checks/report_checks/section_component.py @@ -12,7 +12,7 @@ def __init__(self, file_info, chapter='Введение', patterns=('цель', self.intro = {} if headers_map: self.config = headers_map - self.chapter = None + self.chapter = '' patterns = ('цель', 'задач') else: self.chapter = chapter @@ -22,9 +22,10 @@ def __init__(self, file_info, chapter='Введение', patterns=('цель', self.patterns.append({"name": pattern.capitalize(), "text": pattern, "marker": 0}) def late_init(self): - if self.chapter is None: + if not self.chapter: self.headers_main = self.file.get_main_headers(self.file_type['report_type']) - self.chapter = StyleCheckSettings.CONFIGS.get(self.config)[self.headers_main]["header_for_report_section_component"] + if self.headers_main in StyleCheckSettings.CONFIGS.get(self.config): + self.chapter = StyleCheckSettings.CONFIGS.get(self.config)[self.headers_main]["header_for_report_section_component"] self.chapters = self.file.make_chapters(self.file_type['report_type']) def check(self): diff --git a/app/main/reports/md_uploader/md_uploader.py b/app/main/reports/md_uploader/md_uploader.py index ce55fb44..ec337830 100644 --- a/app/main/reports/md_uploader/md_uploader.py +++ b/app/main/reports/md_uploader/md_uploader.py @@ -73,7 +73,7 @@ def parse(self, md_text): def make_paragraphs(self, html_text): html_text = html_text.replace("
  • ", "").replace("
    • ", "").replace("", "").replace("
      ", "") - self.paragraphs = html_text.split('\n') + self.paragraphs = [paragraph for paragraph in html_text.split('\n') if paragraph.strip()] return self.paragraphs def page_counter(self): # we need this just to find a last page and make link to the literature in banned_words_in_literature @@ -121,7 +121,6 @@ def parse_effective_styles(self): else: paragraph["runs"].append({"text": par, "style": 'body text'}) self.styled_paragraphs.append(paragraph) - return self.styled_paragraphs def make_chapters(self, work_type): From f0cd5408e30ff8c27c0b6192db02716a9390e4ee Mon Sep 17 00:00:00 2001 From: Marina Date: Thu, 29 Aug 2024 20:11:47 +0300 Subject: [PATCH 63/69] check min and max literature ref is added --- .../checks/report_checks/literature_references.py | 14 +++++++++----- .../checks/report_checks/style_check_settings.py | 2 ++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/app/main/checks/report_checks/literature_references.py b/app/main/checks/report_checks/literature_references.py index 96f71791..4be154ab 100644 --- a/app/main/checks/report_checks/literature_references.py +++ b/app/main/checks/report_checks/literature_references.py @@ -1,5 +1,5 @@ import re - +from .style_check_settings import StyleCheckSettings from ..base_check import BaseReportCriterion, answer @@ -8,13 +8,17 @@ class ReferencesToLiteratureCheck(BaseReportCriterion): description = '' id = 'literature_references' - def __init__(self, file_info, min_ref=1, max_ref=1000): + def __init__(self, file_info, min_ref=1, max_ref=1000, headers_map=None): super().__init__(file_info) self.headers = [] self.literature_header = [] self.name_pattern = r'список[ \t]*(использованных|использованной|)[ \t]*(источников|литературы)' - self.min_ref = min_ref - self.max_ref = max_ref + if headers_map: + self.min_ref = StyleCheckSettings.CONFIGS.get(headers_map)['min_ref_for_literature_references_check'] + self.max_ref = StyleCheckSettings.CONFIGS.get(headers_map)['mах_ref_for_literature_references_check'] + else: + self.min_ref = min_ref + self.max_ref = max_ref def late_init_vkr(self): self.headers = self.file.make_chapters(self.file_type['report_type']) @@ -51,7 +55,7 @@ def check(self): all_numbers.add(i) if len(references.symmetric_difference(all_numbers)) == 0: if not self.min_ref <= number_of_sources <= self.max_ref: - return answer(False, f'Список источников оформлен верно, однако их количество ({number_of_sources}) не удовлетворяет необходимому критерию.
      Количество источников должно быть от {self.min_ref} до {self.max_ref}.') + return answer(False, f'Список источников оформлен верно, однако их количество ({number_of_sources}) не удовлетворяет необходимому критерию.
      Количество источников должно быть не менее {self.min_ref}.') else: return answer(True, f"Пройдена!") elif len(references.difference(all_numbers)): diff --git a/app/main/checks/report_checks/style_check_settings.py b/app/main/checks/report_checks/style_check_settings.py index a7d9308f..0462d950 100644 --- a/app/main/checks/report_checks/style_check_settings.py +++ b/app/main/checks/report_checks/style_check_settings.py @@ -204,6 +204,8 @@ class StyleCheckSettings: "unify_regex": None, "regex": HEADER_REGEX }, + 'min_ref_for_literature_references_check': 5, + 'mах_ref_for_literature_references_check': 1000, #just for future possible edit } From d3cdd872ceaea25133843023b986d9a5d86416ca Mon Sep 17 00:00:00 2001 From: Marina Date: Fri, 30 Aug 2024 13:27:35 +0300 Subject: [PATCH 64/69] banned_words_check is added --- .../report_checks/banned_words_check.py | 27 ++++++-- .../report_checks/literature_references.py | 5 +- .../report_checks/style_check_settings.py | 68 ++++++++++++++----- 3 files changed, 75 insertions(+), 25 deletions(-) diff --git a/app/main/checks/report_checks/banned_words_check.py b/app/main/checks/report_checks/banned_words_check.py index e88dedab..ec18d4a9 100644 --- a/app/main/checks/report_checks/banned_words_check.py +++ b/app/main/checks/report_checks/banned_words_check.py @@ -1,5 +1,5 @@ import re - +from .style_check_settings import StyleCheckSettings from ..base_check import BaseReportCriterion, answer, morph @@ -8,15 +8,32 @@ class ReportBannedWordsCheck(BaseReportCriterion): description = 'Запрещено упоминание слова "мы"' id = 'banned_words_check' - def __init__(self, file_info, words=["мы"], min_count=3, max_count=6): + def __init__(self, file_info, headers_map=None): super().__init__(file_info) - self.words = [morph.normal_forms(word)[0] for word in words] - self.min_count = min_count - self.max_count = max_count + self.words = [] + self.min_count = 0 + self.max_count = 0 + if headers_map: + self.config = headers_map + else: + self.config = 'VKR_HEADERS' if (self.file_type['report_type'] == 'VKR') else 'LR_HEADERS' + + def late_init(self): + self.headers_main = self.file.get_main_headers(self.file_type['report_type']) + if self.headers_main in StyleCheckSettings.CONFIGS.get(self.config): + self.words = [morph.normal_forms(word)[0] for word in StyleCheckSettings.CONFIGS.get(self.config)[self.headers_main]['banned_words']] + self.min_count = StyleCheckSettings.CONFIGS.get(self.config)[self.headers_main]['min_count_for_banned_words_check'] + self.max_count = StyleCheckSettings.CONFIGS.get(self.config)[self.headers_main]['min_count_for_banned_words_check'] + else: + if 'any_header' in StyleCheckSettings.CONFIGS.get(self.config): + self.words = [morph.normal_forms(word)[0] for word in StyleCheckSettings.CONFIGS.get(self.config)['any_header']['banned_words']] + self.min_count = StyleCheckSettings.CONFIGS.get(self.config)['any_header']['min_count_for_banned_words_check'] + self.max_count = StyleCheckSettings.CONFIGS.get(self.config)['any_header']['min_count_for_banned_words_check'] def check(self): if self.file.page_counter() < 4: return answer(False, "В отчете недостаточно страниц. Нечего проверять.") + self.late_init() detected_lines = {} result_str = f'Запрещенные слова: {"; ".join(self.words)}
      ' count = 0 diff --git a/app/main/checks/report_checks/literature_references.py b/app/main/checks/report_checks/literature_references.py index 4be154ab..719346dc 100644 --- a/app/main/checks/report_checks/literature_references.py +++ b/app/main/checks/report_checks/literature_references.py @@ -14,8 +14,9 @@ def __init__(self, file_info, min_ref=1, max_ref=1000, headers_map=None): self.literature_header = [] self.name_pattern = r'список[ \t]*(использованных|использованной|)[ \t]*(источников|литературы)' if headers_map: - self.min_ref = StyleCheckSettings.CONFIGS.get(headers_map)['min_ref_for_literature_references_check'] - self.max_ref = StyleCheckSettings.CONFIGS.get(headers_map)['mах_ref_for_literature_references_check'] + if StyleCheckSettings.CONFIGS.get(headers_map)['limits']: + self.min_ref = int(StyleCheckSettings.CONFIGS.get(headers_map)['any_header']['min_ref_for_literature_references_check']) + self.max_ref = int(StyleCheckSettings.CONFIGS.get(headers_map)['any_header']['mах_ref_for_literature_references_check']) else: self.min_ref = min_ref self.max_ref = max_ref diff --git a/app/main/checks/report_checks/style_check_settings.py b/app/main/checks/report_checks/style_check_settings.py index 0462d950..b4c9a093 100644 --- a/app/main/checks/report_checks/style_check_settings.py +++ b/app/main/checks/report_checks/style_check_settings.py @@ -82,49 +82,61 @@ class StyleCheckSettings: } # Order of styles may be significant! First level 1, then level 2 and so on. - LR_CONFIG = {'any_header': + LR_CONFIG = { + 'any_header': { "style": HEADER_1_STYLE, "docx_style": ["heading 1"], "headers": ["Исходный код программы"], "unify_regex": APPENDIX_UNIFY_REGEX, "regex": APPENDIX_REGEX, - },'second_header': + "banned_words": ['мы'], + 'min_count_for_banned_words_check': 3, + 'max_count_for_banned_words_check': 6, + }, + 'second_header': { "style": HEADER_2_STYLE, "docx_style": ["heading 2"], "headers": ["Цель работы", "Выполнение работы", "Выводы"], "unify_regex": None, - "regex": HEADER_1_REGEX + "regex": HEADER_1_REGEX, } } - VKR_CONFIG = {'any_header': + VKR_CONFIG = { + 'any_header': { "style": HEADER_1_STYLE, "docx_style": ["heading 2"], "headers": ["ВВЕДЕНИЕ", "ЗАКЛЮЧЕНИЕ", "СПИСОК ИСПОЛЬЗОВАННЫХ ИСТОЧНИКОВ"], "unify_regex": None, - "regex": HEADER_REGEX - },'second_header': + "regex": HEADER_REGEX, + "banned_words": ['мы'], + 'min_count_for_banned_words_check': 3, + 'max_count_for_banned_words_check': 6, + }, + 'second_header': { "style": HEADER_1_NUM_STYLE, "docx_style": ["heading 2", "heading 3", "heading 4"], "headers": [], "unify_regex": None, - "regex": HEADER_NUM_REGEX + "regex": HEADER_NUM_REGEX, } } - NIR_CONFIG = {'any_header': + NIR_CONFIG = { + 'any_header': { "style": HEADER_1_STYLE, "docx_style": ["heading 2"], "headers": ["ПОСТАНОВКА ЗАДАЧИ", "РЕЗУЛЬТАТЫ РАБОТЫ В ВЕСЕННЕМ СЕМЕСТРЕ", "ОПИСАНИЕ ПРЕДПОЛАГАЕМОГО МЕТОДА РЕШЕНИЯ", "ПЛАН РАБОТЫ НА ОСЕННИЙ СЕМЕСТР", "СПИСОК ИСПОЛЬЗОВАННЫХ ИСТОЧНИКОВ"], "unify_regex": None, - "regex": HEADER_REGEX - }, 'second_header': + "regex": HEADER_REGEX, + }, + 'second_header': { "style": HEADER_1_NUM_STYLE, "docx_style": ["heading 3", "heading 4"], @@ -146,7 +158,10 @@ class StyleCheckSettings: ], "header_for_report_section_component": "Поставленная цель и задачи", "unify_regex": None, - "regex": HEADER_REGEX + "regex": HEADER_REGEX, + "banned_words": ['мы'], + 'min_count_for_banned_words_check': 3, + 'max_count_for_banned_words_check': 6, }, 'Задание 2': { @@ -158,8 +173,13 @@ class StyleCheckSettings: ], "header_for_report_section_component": "", "unify_regex": None, - "regex": HEADER_REGEX + "regex": HEADER_REGEX, + "banned_words": ['мы'], + 'min_count_for_banned_words_check': 3, + 'max_count_for_banned_words_check': 6, } + } + # },'': # { # "style": HEADER_1_NUM_STYLE, @@ -175,7 +195,7 @@ class StyleCheckSettings: # "unify_regex": None, # "regex": HEADER_NUM_REGEX # } - } + OPNP_CONFIG = { 'Сравнение аналогов': @@ -188,7 +208,16 @@ class StyleCheckSettings: "Выбор метода решения", ], "unify_regex": None, - "regex": HEADER_REGEX + "regex": HEADER_REGEX, + "banned_words": ['аттач', 'билдить', 'бинарник', 'валидный', 'дебаг', 'деплоить', 'десктопное', 'железо', + 'исходники', 'картинка', 'консольное', 'конфиг', 'кусок', 'либа', 'лог', 'мануал', 'машина', + 'отнаследованный', 'парсинг', 'пост', 'распаковать', 'сбоит', 'скачать', 'склонировать', 'скрипт', + 'тестить', 'тул', 'тула', 'тулза', 'фиксить', 'флажок', 'флаг', 'юзкейс', 'продакт', 'продакшн', + 'прод', 'фидбек', 'дедлайн', 'дэдлайн'], + 'min_ref_for_literature_references_check': 5, + 'mах_ref_for_literature_references_check': 1000, #just for future possible edit + 'min_count_for_banned_words_check': 0, + 'max_count_for_banned_words_check': 0 }, 'any_header': { @@ -202,10 +231,13 @@ class StyleCheckSettings: "Список литературы" ], "unify_regex": None, - "regex": HEADER_REGEX - }, - 'min_ref_for_literature_references_check': 5, - 'mах_ref_for_literature_references_check': 1000, #just for future possible edit + "regex": HEADER_REGEX, + "banned_words": ['оптимально', 'оптимальный', 'надежный', 'интуитивный'], + 'min_ref_for_literature_references_check': 5, + 'mах_ref_for_literature_references_check': 1000, #just for future possible edit + 'min_count_for_banned_words_check': 0, + 'max_count_for_banned_words_check': 0 + } } From 0f316c48c5311f66b8f977b020daa8c27625e891 Mon Sep 17 00:00:00 2001 From: Marina Date: Sat, 31 Aug 2024 17:11:37 +0300 Subject: [PATCH 65/69] added searching of key words --- .../checks/report_checks/key_words_check.py | 56 ++++++++++++++++--- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/app/main/checks/report_checks/key_words_check.py b/app/main/checks/report_checks/key_words_check.py index 999f03ed..adaabfc5 100644 --- a/app/main/checks/report_checks/key_words_check.py +++ b/app/main/checks/report_checks/key_words_check.py @@ -1,24 +1,66 @@ import re +import string + +from nltk.tokenize import word_tokenize +from nltk.corpus import stopwords +from pymorphy2 import MorphAnalyzer from ..base_check import BaseReportCriterion, answer +MORPH_ANALYZER = MorphAnalyzer() + class KeyWordsReportCheck(BaseReportCriterion): - label = 'Проверка наличия раздела "Ключевые слова"' - description = 'Раздел идет сразу после названия работы и содержит не менее трех ключевых слов' + label = 'Проверка наличия раздела "Ключевые слова" и упоминание их в тексте' + description = 'Раздел идет сразу после названия работы и содержит не менее трех ключевых слов. Слова упоминаются в тексте' id = 'Key_words_report_check' def __init__(self, file_info, min_key_words = 3): super().__init__(file_info) self.min_key_words = min_key_words + self.chapters = [] + self.text_par = [] + self.lemme_list = [] + + def late_init(self): + self.chapters = self.file.make_chapters(self.file_type['report_type']) def check(self): key_words_chapter = self.file.paragraphs[1].lower() if 'ключевые слова' not in key_words_chapter: return answer(False, 'Раздел "Ключевые слова" не найден') cleaned_str = re.sub(r'<[^>]*>', '', key_words_chapter) - final_str = cleaned_str.replace('ключевые слова', '').replace(':','').replace(' ', '') - key_words = final_str.split(',') - if len(key_words) >= self.min_key_words: - return answer(True, 'Пройдена!') - else: + final_str = cleaned_str.replace('ключевые слова', '').replace(':','') + key_words_result = [word.strip() for word in final_str.split(',')] + if len(key_words_result) < self.min_key_words: return answer(False, f'Не пройдена! Количество ключевых слов должно быть не менее {self.min_key_words}') + stop_words = set(stopwords.words("russian")) + if self.file.page_counter() < 4: + return answer(False, "В отчете недостаточно страниц. Нечего проверять.") + self.late_init() + for intro in self.chapters: + header = intro["text"].lower() + if header not in ['аннотация', "ключевые слова"]: + self.intro = intro + for intro_par in self.intro['child']: + par = intro_par['text'].lower() + self.text_par.append(par) + for phrase in key_words_result: + words = word_tokenize(phrase) + words_lemma = [MORPH_ANALYZER.parse(w)[0].normal_form for w in words if w.lower() not in stop_words] + phrase_lemma = ' '.join(words_lemma) + self.lemme_list.append(phrase) + for text in self.text_par: + cleaned_text = re.sub(r'<[^>]*>', '', text) + translator = str.maketrans('', '', string.punctuation) + text_without_punct = cleaned_text.translate(translator) + word_in_text = word_tokenize(text_without_punct) + lemma_text = [MORPH_ANALYZER.parse(w)[0].normal_form for w in word_in_text if w.lower() not in stop_words] + lemma_text_str = ' '.join(lemma_text) + if phrase_lemma in lemma_text_str: + del self.lemme_list[-1] + break + + if self.lemme_list: + return answer(False, f"Не пройдена! В тексте не найдены следующие ключевые слова: «{'», «'.join(self.lemme_list)}»") + else: + return answer(True, f'Пройдена!') From 1be94cd813c0da5c341186169096ea85e9d59a59 Mon Sep 17 00:00:00 2001 From: Marina Date: Sat, 31 Aug 2024 17:16:29 +0300 Subject: [PATCH 66/69] config improving --- .../report_checks/style_check_settings.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/app/main/checks/report_checks/style_check_settings.py b/app/main/checks/report_checks/style_check_settings.py index b4c9a093..971fd548 100644 --- a/app/main/checks/report_checks/style_check_settings.py +++ b/app/main/checks/report_checks/style_check_settings.py @@ -180,23 +180,6 @@ class StyleCheckSettings: } } - # },'': - # { - # "style": HEADER_1_NUM_STYLE, - # "docx_style": ["heading 2", "heading 3", "heading 4"], - # "headers": [], - # "unify_regex": None, - # "regex": HEADER_NUM_REGEX - # },'': - # { - # "style": "Main_header", - # "docx_style": ["heading 1"], - # "headers": ["Задание"], - # "unify_regex": None, - # "regex": HEADER_NUM_REGEX - # } - - OPNP_CONFIG = { 'Сравнение аналогов': { @@ -225,7 +208,7 @@ class StyleCheckSettings: "docx_style": ["heading 2"], "headers": ["Аннотация", "Введение", - "Обзор предметной области == Сравнение аналогов", + "Обзор предметной области", "Выбор метода решения", "Заключение", "Список литературы" From 62428c73ad87380a3632907eabb3896b096ce0cd Mon Sep 17 00:00:00 2001 From: Marina Date: Wed, 4 Sep 2024 18:19:38 +0300 Subject: [PATCH 67/69] fix mistakes --- app/main/check_packs/pack_config.py | 2 +- .../checks/report_checks/banned_words_check.py | 4 ++-- app/main/checks/report_checks/key_words_check.py | 2 +- .../report_checks/literature_references.py | 16 ++++++++++------ 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/app/main/check_packs/pack_config.py b/app/main/check_packs/pack_config.py index 1ff5d434..c053ce0a 100644 --- a/app/main/check_packs/pack_config.py +++ b/app/main/check_packs/pack_config.py @@ -44,7 +44,7 @@ ["spelling_check"], ["max_abstract_size_check"], ["theme_in_report_check"], - ['Key_words_report_check'], + ['key_words_report_check'], ["empty_task_page_check"], ] diff --git a/app/main/checks/report_checks/banned_words_check.py b/app/main/checks/report_checks/banned_words_check.py index ec18d4a9..351b403e 100644 --- a/app/main/checks/report_checks/banned_words_check.py +++ b/app/main/checks/report_checks/banned_words_check.py @@ -23,12 +23,12 @@ def late_init(self): if self.headers_main in StyleCheckSettings.CONFIGS.get(self.config): self.words = [morph.normal_forms(word)[0] for word in StyleCheckSettings.CONFIGS.get(self.config)[self.headers_main]['banned_words']] self.min_count = StyleCheckSettings.CONFIGS.get(self.config)[self.headers_main]['min_count_for_banned_words_check'] - self.max_count = StyleCheckSettings.CONFIGS.get(self.config)[self.headers_main]['min_count_for_banned_words_check'] + self.max_count = StyleCheckSettings.CONFIGS.get(self.config)[self.headers_main]['max_count_for_banned_words_check'] else: if 'any_header' in StyleCheckSettings.CONFIGS.get(self.config): self.words = [morph.normal_forms(word)[0] for word in StyleCheckSettings.CONFIGS.get(self.config)['any_header']['banned_words']] self.min_count = StyleCheckSettings.CONFIGS.get(self.config)['any_header']['min_count_for_banned_words_check'] - self.max_count = StyleCheckSettings.CONFIGS.get(self.config)['any_header']['min_count_for_banned_words_check'] + self.max_count = StyleCheckSettings.CONFIGS.get(self.config)['any_header']['max_count_for_banned_words_check'] def check(self): if self.file.page_counter() < 4: diff --git a/app/main/checks/report_checks/key_words_check.py b/app/main/checks/report_checks/key_words_check.py index adaabfc5..9a403613 100644 --- a/app/main/checks/report_checks/key_words_check.py +++ b/app/main/checks/report_checks/key_words_check.py @@ -12,7 +12,7 @@ class KeyWordsReportCheck(BaseReportCriterion): label = 'Проверка наличия раздела "Ключевые слова" и упоминание их в тексте' description = 'Раздел идет сразу после названия работы и содержит не менее трех ключевых слов. Слова упоминаются в тексте' - id = 'Key_words_report_check' + id = 'key_words_report_check' def __init__(self, file_info, min_key_words = 3): super().__init__(file_info) diff --git a/app/main/checks/report_checks/literature_references.py b/app/main/checks/report_checks/literature_references.py index ba5482b5..32be66ed 100644 --- a/app/main/checks/report_checks/literature_references.py +++ b/app/main/checks/report_checks/literature_references.py @@ -14,16 +14,20 @@ def __init__(self, file_info, min_ref=1, max_ref=1000, headers_map=None): self.literature_header = [] self.name_pattern = r'список[ \t]*(использованных|использованной|)[ \t]*(источников|литературы)' if headers_map: - if StyleCheckSettings.CONFIGS.get(headers_map)['limits']: - self.min_ref = int(StyleCheckSettings.CONFIGS.get(headers_map)['any_header']['min_ref_for_literature_references_check']) - self.max_ref = int(StyleCheckSettings.CONFIGS.get(headers_map)['any_header']['mах_ref_for_literature_references_check']) + self.config = headers_map else: - self.min_ref = min_ref - self.max_ref = max_ref + self.config = 'VKR_HEADERS' if (self.file_type['report_type'] == 'VKR') else 'LR_HEADERS' def late_init_vkr(self): self.headers = self.file.make_chapters(self.file_type['report_type']) - self.literature_header = self.file.find_literature_vkr(self.file_type['report_type']) + self.headers_main = self.file.get_main_headers(self.file_type['report_type']) + if self.headers_main in StyleCheckSettings.CONFIGS.get(self.config): + self.min_ref = StyleCheckSettings.CONFIGS.get(self.config)[self.headers_main]['min_ref_for_literature_references_check'] + self.max_ref = StyleCheckSettings.CONFIGS.get(self.config)[self.headers_main]['mах_ref_for_literature_references_check'] + else: + if 'any_header' in StyleCheckSettings.CONFIGS.get(self.config): + self.min_ref = StyleCheckSettings.CONFIGS.get(self.config)['any_header']['min_ref_for_literature_references_check'] + self.max_ref = StyleCheckSettings.CONFIGS.get(self.config)['any_header']['mах_ref_for_literature_references_check'] def check(self): if self.file.page_counter() < 4: From 6c78e3a5507ce288bdf60c14141158235f331435 Mon Sep 17 00:00:00 2001 From: Marina Date: Fri, 20 Sep 2024 16:32:27 +0300 Subject: [PATCH 68/69] style setting improving and fix lit_ref for VKR and LR --- .../checks/report_checks/style_check_settings.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/app/main/checks/report_checks/style_check_settings.py b/app/main/checks/report_checks/style_check_settings.py index 971fd548..88e1a7ad 100644 --- a/app/main/checks/report_checks/style_check_settings.py +++ b/app/main/checks/report_checks/style_check_settings.py @@ -10,6 +10,9 @@ class StyleCheckSettings: HEADER_REGEX = "^\\D+.+$" HEADER_1_REGEX = "^()([\\w\\s]+)$" HEADER_2_REGEX = "^()([\\w\\s]+)\\.$" + STD_BANNED_WORDS = ['мы'] + STD_MIN_LIT_REF = 1 + STD_MAX_LIT_REF = 1000 #just in case for future edit HEADER_1_STYLE = { "bold": True, "italic": False, @@ -90,9 +93,11 @@ class StyleCheckSettings: "headers": ["Исходный код программы"], "unify_regex": APPENDIX_UNIFY_REGEX, "regex": APPENDIX_REGEX, - "banned_words": ['мы'], + "banned_words": STD_BANNED_WORDS, 'min_count_for_banned_words_check': 3, 'max_count_for_banned_words_check': 6, + 'min_ref_for_literature_references_check': STD_MIN_LIT_REF, + 'mах_ref_for_literature_references_check': STD_MAX_LIT_REF }, 'second_header': { @@ -112,9 +117,11 @@ class StyleCheckSettings: "headers": ["ВВЕДЕНИЕ", "ЗАКЛЮЧЕНИЕ", "СПИСОК ИСПОЛЬЗОВАННЫХ ИСТОЧНИКОВ"], "unify_regex": None, "regex": HEADER_REGEX, - "banned_words": ['мы'], + "banned_words": STD_BANNED_WORDS, 'min_count_for_banned_words_check': 3, 'max_count_for_banned_words_check': 6, + 'min_ref_for_literature_references_check': STD_MIN_LIT_REF, + 'mах_ref_for_literature_references_check': STD_MAX_LIT_REF }, 'second_header': { @@ -159,7 +166,7 @@ class StyleCheckSettings: "header_for_report_section_component": "Поставленная цель и задачи", "unify_regex": None, "regex": HEADER_REGEX, - "banned_words": ['мы'], + "banned_words": STD_BANNED_WORDS, 'min_count_for_banned_words_check': 3, 'max_count_for_banned_words_check': 6, }, @@ -174,7 +181,7 @@ class StyleCheckSettings: "header_for_report_section_component": "", "unify_regex": None, "regex": HEADER_REGEX, - "banned_words": ['мы'], + "banned_words": STD_BANNED_WORDS, 'min_count_for_banned_words_check': 3, 'max_count_for_banned_words_check': 6, } From 11141814226a6aa06af7e82cc574925a186841a8 Mon Sep 17 00:00:00 2001 From: Marina Date: Fri, 27 Sep 2024 15:20:25 +0300 Subject: [PATCH 69/69] fix LR config --- app/main/checks/report_checks/short_sections_check.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/main/checks/report_checks/short_sections_check.py b/app/main/checks/report_checks/short_sections_check.py index 35d76b6e..53729cfb 100644 --- a/app/main/checks/report_checks/short_sections_check.py +++ b/app/main/checks/report_checks/short_sections_check.py @@ -30,14 +30,13 @@ def __init__(self, file_info, min_section_count=5, min_section_len=20, main_head style = Style() style.__dict__.update(prechecked_dict) self.styles.append(style) - def late_init(self): self.file.parse_effective_styles() try: self.cutoff_line = self.file.pdf_file.get_text_on_page()[2].split("\n")[0] except: self.cutoff_line = None - for preset in self.presets: + for _, preset in self.presets.items(): if preset["unify_regex"] is not None: self.file.unify_multiline_entities(preset["unify_regex"])