From 7581e7ab63e954f97bd37abc90088209faf60ab9 Mon Sep 17 00:00:00 2001 From: Jiri Kyjovsky Date: Tue, 6 Feb 2024 18:12:45 +0100 Subject: [PATCH] frontend: migrate monitor, modules, mock_chroots, webhooks enpoints to restx --- frontend/coprs_frontend/coprs/helpers.py | 6 +- .../coprs/views/apiv3_ns/__init__.py | 20 ++- .../views/apiv3_ns/apiv3_mock_chroots.py | 43 ++++-- .../coprs/views/apiv3_ns/apiv3_modules.py | 80 +++++++---- .../coprs/views/apiv3_ns/apiv3_monitor.py | 133 ++++++++++-------- .../coprs/views/apiv3_ns/apiv3_webhooks.py | 49 ++++--- .../coprs/views/apiv3_ns/schema/fields.py | 13 ++ .../coprs/views/apiv3_ns/schema/schemas.py | 48 +++++++ 8 files changed, 272 insertions(+), 120 deletions(-) diff --git a/frontend/coprs_frontend/coprs/helpers.py b/frontend/coprs_frontend/coprs/helpers.py index 8ba9ff7c6..127d304eb 100644 --- a/frontend/coprs_frontend/coprs/helpers.py +++ b/frontend/coprs_frontend/coprs/helpers.py @@ -6,9 +6,10 @@ import random import string import json -from os.path import normpath import posixpath import re +from http import HTTPStatus +from os.path import normpath from urllib.parse import urlparse, parse_qs, urlunparse, urlencode import html5_parser @@ -829,7 +830,6 @@ def db_column_length(column): return getattr(column, "property").columns[0].type.length -@flask.stream_with_context def streamed_json(stream, start_string=None, stop_string=None): """ Flask response generator for JSON structures (arrays only for now) @@ -867,7 +867,7 @@ def _batched_stream(count=100): def _response(): return app.response_class( - _batched_stream(), + flask.stream_with_context(_batched_stream()), mimetype="application/json", ) diff --git a/frontend/coprs_frontend/coprs/views/apiv3_ns/__init__.py b/frontend/coprs_frontend/coprs/views/apiv3_ns/__init__.py index a6b35e0bf..0b1ac425d 100644 --- a/frontend/coprs_frontend/coprs/views/apiv3_ns/__init__.py +++ b/frontend/coprs_frontend/coprs/views/apiv3_ns/__init__.py @@ -113,14 +113,16 @@ def pagination_wrapper(*args, **kwargs): return pagination_decorator +def _shared_file_upload_wrapper(): + data = json.loads(flask.request.files["json"].read()) or {} + flask.request.form = ImmutableMultiDict(list(data.items())) + def file_upload(): def file_upload_decorator(f): @wraps(f) def file_upload_wrapper(*args, **kwargs): if "json" in flask.request.files: - data = json.loads(flask.request.files["json"].read()) or {} - tuples = [(k, v) for k, v in data.items()] - flask.request.form = ImmutableMultiDict(tuples) + _shared_file_upload_wrapper() return f(*args, **kwargs) return file_upload_wrapper return file_upload_decorator @@ -506,3 +508,15 @@ def create_pagination(self, *args, **kwargs): kwargs = _shared_pagination_wrapper(**kwargs) return endpoint_method(self, *args, **kwargs) return create_pagination + + +def restx_file_upload(endpoint_method): + """ + Allow uploading a file to a form via endpoint by using this function as an endpoint decorator. + """ + @wraps(endpoint_method) + def inner(self, *args, **kwargs): + if "json" in flask.request.files: + _shared_file_upload_wrapper() + return endpoint_method(self, *args, **kwargs) + return inner diff --git a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_mock_chroots.py b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_mock_chroots.py index 5cbd4c470..e9dfed06e 100644 --- a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_mock_chroots.py +++ b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_mock_chroots.py @@ -1,17 +1,36 @@ -import flask +# pylint: disable=missing-class-docstring + + +from http import HTTPStatus + +from flask_restx import Namespace, Resource from html2text import html2text -from coprs.views.apiv3_ns import apiv3_ns + +from coprs.views.apiv3_ns import api from coprs.logic.coprs_logic import MockChrootsLogic -@apiv3_ns.route("/mock-chroots/list") -def list_chroots(): - chroots = MockChrootsLogic.active_names_with_comments() - response = {} - for chroot, comment in chroots: - if comment: - response[chroot] = html2text(comment).strip("\n") - else: - response[chroot] = "" +apiv3_mock_chroots_ns = Namespace("mock-chroots", description="Mock chroots") +api.add_namespace(apiv3_mock_chroots_ns) + + +@apiv3_mock_chroots_ns.route("/list") +class MockChroot(Resource): + # FIXME: we can't have proper model here, - one of REST API rules that flask-restx follows + # is to have keys in JSON constant, we don't do that here. + @apiv3_mock_chroots_ns.response(HTTPStatus.OK.value, "OK, Mock chroot data follows...") + def get(self): + """ + Get list of mock chroots + Get list of all currently active mock chroots with additional comment in format + `mock_chroot_name: additional_comment`. + """ + chroots = MockChrootsLogic.active_names_with_comments() + response = {} + for chroot, comment in chroots: + if comment: + response[chroot] = html2text(comment).strip("\n") + else: + response[chroot] = "" - return flask.jsonify(response) + return response diff --git a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_modules.py b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_modules.py index 3de3b157b..8498e8cdd 100644 --- a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_modules.py +++ b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_modules.py @@ -1,43 +1,67 @@ +# pylint: disable=missing-class-docstring + + +from http import HTTPStatus + import flask import sqlalchemy +from flask_restx import Namespace, Resource from requests.exceptions import RequestException, InvalidSchema from wtforms import ValidationError + from coprs import forms, db_session_scope -from coprs.views.apiv3_ns import apiv3_ns, get_copr, file_upload, POST -from coprs.views.misc import api_login_required +from coprs.views.apiv3_ns import api, get_copr, restx_file_upload +from coprs.views.apiv3_ns.schema.schemas import module_build_model, fullname_params, module_add_input_model +from coprs.views.misc import restx_api_login_required from coprs.exceptions import DuplicateException, BadRequest, InvalidForm from coprs.logic.modules_logic import ModuleProvider, ModuleBuildFacade +apiv3_module_ns = Namespace("module", description="Module") +api.add_namespace(apiv3_module_ns) + + def to_dict(module): return { "nsv": module.nsv, } -@apiv3_ns.route("/module/build//", methods=POST) -@api_login_required -@file_upload() -def build_module(ownername, projectname): - copr = get_copr(ownername, projectname) - form = forms.get_module_build_form(meta={'csrf': False}) - if not form.validate_on_submit(): - raise InvalidForm(form) - - facade = None - try: - mod_info = ModuleProvider.from_input(form.modulemd.data or form.scmurl.data) - facade = ModuleBuildFacade(flask.g.user, copr, mod_info.yaml, - mod_info.filename, form.distgit.data) - with db_session_scope(): - module = facade.submit_build() - return flask.jsonify(to_dict(module)) - - except (ValidationError, RequestException, InvalidSchema, RuntimeError) as ex: - raise BadRequest(str(ex)) from ex - - except sqlalchemy.exc.IntegrityError as err: - raise DuplicateException("Module {}-{}-{} already exists" - .format(facade.modulemd.get_module_name(), - facade.modulemd.get_stream_name(), - facade.modulemd.get_version())) from err +@apiv3_module_ns.route("/build//") +class Module(Resource): + @restx_api_login_required + @restx_file_upload + @apiv3_module_ns.doc(params=fullname_params) + @apiv3_module_ns.expect(module_add_input_model) + @apiv3_module_ns.marshal_with(module_build_model) + @apiv3_module_ns.response(HTTPStatus.OK.value, "Module build successfully submitted") + @apiv3_module_ns.response( + HTTPStatus.BAD_REQUEST.value, HTTPStatus.BAD_REQUEST.description + ) + def post(self, ownername, projectname): + """ + Create a module build + Create a module build for ownername/projectname project. + """ + copr = get_copr(ownername, projectname) + form = forms.get_module_build_form(meta={'csrf': False}) + if not form.validate_on_submit(): + raise InvalidForm(form) + + facade = None + try: + mod_info = ModuleProvider.from_input(form.modulemd.data or form.scmurl.data) + facade = ModuleBuildFacade(flask.g.user, copr, mod_info.yaml, + mod_info.filename, form.distgit.data) + with db_session_scope(): + module = facade.submit_build() + return to_dict(module) + + except (ValidationError, RequestException, InvalidSchema, RuntimeError) as ex: + raise BadRequest(str(ex)) from ex + + except sqlalchemy.exc.IntegrityError as err: + raise DuplicateException("Module {}-{}-{} already exists" + .format(facade.modulemd.get_module_name(), + facade.modulemd.get_stream_name(), + facade.modulemd.get_version())) from err diff --git a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_monitor.py b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_monitor.py index ed848a692..0078727e5 100644 --- a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_monitor.py +++ b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_monitor.py @@ -2,21 +2,26 @@ /api_3/monitor routes """ +# pylint: disable=missing-class-docstring + + import flask +from http import HTTPStatus +from flask_restx import Namespace, Resource +from flask_restx.swagger import ref from coprs.exceptions import BadRequest from coprs.logic.builds_logic import BuildsMonitorLogic from coprs.logic.coprs_logic import CoprDirsLogic -from coprs.views.apiv3_ns import ( - apiv3_ns, - GET, - get_copr, - query_params, - streamed_json_array_response, -) - +from coprs.views.apiv3_ns import api, get_copr, streamed_json_array_response, query_to_parameters +from coprs.views.apiv3_ns.schema.schemas import fullname_params, monitor_model from coprs.measure import checkpoint + +apiv3_monitor_ns = Namespace("monitor", description="Monitor") +api.add_namespace(apiv3_monitor_ns) + + def monitor_generator(copr_dir, additional_fields): """ Continuosly fill-up the package_monitor() buffer. @@ -48,54 +53,66 @@ def monitor_generator(copr_dir, additional_fields): checkpoint("Last package queried") -@apiv3_ns.route("/monitor", methods=GET) -@query_params() -def package_monitor(ownername, projectname, project_dirname=None): - """ - For list of the project packages return list of JSON dictionaries informing - about status of the last chroot builds (status, build log, etc.). - """ - checkpoint("API3 monitor start") - - additional_fields = flask.request.args.getlist("additional_fields[]") - - copr = get_copr(ownername, projectname) - - valid_additional_fields = [ - "url_build_log", - "url_backend_log", - "url_build", - ] - - if additional_fields: - additional_fields = set(additional_fields) - bad_fields = [] - for field in sorted(additional_fields): - if field not in valid_additional_fields: - bad_fields += [field] - if bad_fields: - raise BadRequest( - "Wrong additional_fields argument(s): " + - ", ".join(bad_fields) +@apiv3_monitor_ns.route("") +class Monitor(Resource): + @query_to_parameters + @apiv3_monitor_ns.doc(params=fullname_params) + # marshalling not possible with streaming JSON like this, flask-restx tries + # to serialize it to JSON and fails or returns empty responses + # passing the documentation from marshalling just to response documentation + @apiv3_monitor_ns.response( + HTTPStatus.PARTIAL_CONTENT.value, HTTPStatus.PARTIAL_CONTENT.description, monitor_model + ) + @apiv3_monitor_ns.response( + HTTPStatus.BAD_REQUEST.value, HTTPStatus.BAD_REQUEST.description + ) + def get(self, ownername, projectname, project_dirname=None): + """ + Get info about builds + For list of the project packages return list of JSON dictionaries informing + about status of the last chroot builds (status, build log, etc.). + """ + checkpoint("API3 monitor start") + + additional_fields = flask.request.args.getlist("additional_fields[]") + + copr = get_copr(ownername, projectname) + + valid_additional_fields = [ + "url_build_log", + "url_backend_log", + "url_build", + ] + + if additional_fields: + additional_fields = set(additional_fields) + bad_fields = [] + for field in sorted(additional_fields): + if field not in valid_additional_fields: + bad_fields += [field] + if bad_fields: + raise BadRequest( + "Wrong additional_fields argument(s): " + + ", ".join(bad_fields) + ) + else: + additional_fields = set() + + if project_dirname: + copr_dir = CoprDirsLogic.get_by_copr(copr, project_dirname) + else: + copr_dir = copr.main_dir + + # Preload those to avoid the error sqlalchemy.orm.exc.DetachedInstanceError + # http://sqlalche.me/e/13/bhk3 + _ = copr_dir.copr.active_chroots + _ = copr_dir.copr.group + + try: + return streamed_json_array_response( + monitor_generator(copr_dir, additional_fields), + "Project monitor request successful", + "packages", ) - else: - additional_fields = set() - - if project_dirname: - copr_dir = CoprDirsLogic.get_by_copr(copr, project_dirname) - else: - copr_dir = copr.main_dir - - # Preload those to avoid the error sqlalchemy.orm.exc.DetachedInstanceError - # http://sqlalche.me/e/13/bhk3 - _ = copr_dir.copr.active_chroots - _ = copr_dir.copr.group - - try: - return streamed_json_array_response( - monitor_generator(copr_dir, additional_fields), - "Project monitor request successful", - "packages", - ) - finally: - checkpoint("Streaming prepared") + finally: + checkpoint("Streaming prepared") diff --git a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_webhooks.py b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_webhooks.py index da0bed73c..7cb82f287 100644 --- a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_webhooks.py +++ b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_webhooks.py @@ -2,11 +2,21 @@ APIv3 endpoints related to webhooks """ -import flask +# pylint: disable=missing-class-docstring + + +from http import HTTPStatus + +from flask_restx import Namespace, Resource + from coprs import db -from coprs.views.misc import api_login_required -from coprs.views.apiv3_ns import editable_copr, POST -from coprs.views.apiv3_ns import apiv3_ns +from coprs.views.misc import restx_api_login_required +from coprs.views.apiv3_ns import api, restx_editable_copr +from coprs.views.apiv3_ns.schema.schemas import fullname_params, webhook_secret_model + + +apiv3_webhooks_ns = Namespace("webhook", description="Webhooks") +api.add_namespace(apiv3_webhooks_ns) def to_dict(copr): @@ -23,15 +33,22 @@ def to_dict(copr): } -@apiv3_ns.route("/webhook/generate//", methods=POST) -@api_login_required -@editable_copr -def new_webhook_secret(copr): - """ - Generate a new webhook secret for a given project. - Not an additional secret, though. The previous secret gets lost. - """ - copr.new_webhook_secret() - db.session.add(copr) - db.session.commit() - return flask.jsonify(to_dict(copr)) +@apiv3_webhooks_ns.route("/generate//") +class WebhookSecret(Resource): + @restx_api_login_required + @restx_editable_copr + @apiv3_webhooks_ns.doc(params=fullname_params) + @apiv3_webhooks_ns.marshal_with(webhook_secret_model) + @apiv3_webhooks_ns.response(HTTPStatus.OK.value, "Webhook secret created") + @apiv3_webhooks_ns.response( + HTTPStatus.BAD_REQUEST.value, HTTPStatus.BAD_REQUEST.description + ) + def post(self, copr): + """ + Generate a new webhook secret for a given project. + Not an additional secret, though. The previous secret gets lost. + """ + copr.new_webhook_secret() + db.session.add(copr) + db.session.commit() + return to_dict(copr) diff --git a/frontend/coprs_frontend/coprs/views/apiv3_ns/schema/fields.py b/frontend/coprs_frontend/coprs/views/apiv3_ns/schema/fields.py index fa3d24599..92634ade8 100644 --- a/frontend/coprs_frontend/coprs/views/apiv3_ns/schema/fields.py +++ b/frontend/coprs_frontend/coprs/views/apiv3_ns/schema/fields.py @@ -439,6 +439,13 @@ def __init__(self, example=None, **kwargs): description="Number of seconds we allow the builds to run.", ) +nsv = String( + example="name-stream-version", + description="NSV of the module build in format name-stream-version." +) + +webhook_secret = String(example="really-secret-string-do-not-share") + release = String(example="1.fc39") epoch = Integer(example=3) @@ -447,6 +454,8 @@ def __init__(self, example=None, **kwargs): version = String(example="1.0") +modulemd = Raw(example="YAML file", description="Modulemd YAML file") + # TODO: these needs description chroot_repos = Raw() @@ -459,6 +468,10 @@ def __init__(self, example=None, **kwargs): memory_limit = Integer() +distgit = String() + +scmurl = Url() + # TODO: specify those only in Repo schema? baseurl = Url() diff --git a/frontend/coprs_frontend/coprs/views/apiv3_ns/schema/schemas.py b/frontend/coprs_frontend/coprs/views/apiv3_ns/schema/schemas.py index 48b7b33eb..f90e0bc98 100644 --- a/frontend/coprs_frontend/coprs/views/apiv3_ns/schema/schemas.py +++ b/frontend/coprs_frontend/coprs/views/apiv3_ns/schema/schemas.py @@ -86,6 +86,9 @@ def schema_attrs_from_fields(cls) -> dict[str, Any]: @staticmethod def _convert_schema_class_dict_to_schema(d: dict) -> dict: + """ + Returns the same dictionary that was passed as param, doesn't create copy of it. + """ # if in fields.py file is attribute that has different name # than model, add it to `unicorn_fields` like # "field_name_in_fields.py": "what_you_want_to_name_it" @@ -533,6 +536,47 @@ class NevraPackages(Schema): packages: List = List(Nested(_nevra_model)) +@dataclass +class ModuleBuild(Schema): + nsv: String + + +@dataclass +class WebhookSecret(Schema): + id_field: String + name: String + ownername: String + full_name: String + webhook_secret: String + + +@dataclass +class ModuleAdd(InputSchema): + modulemd: String + distgit: String + scmurl: String + + +@dataclass +class _ModulePackage(Schema): + name: String + # inconsistent keys in chroots dict, impossible with flask-restx to do + chroots: Raw = Raw( + description="Chroots and their states", + example={"fedora-rawhide-i386": {"state": "waiting", "status": 1, "build_id": 1}}, + ) + + +_module_package_model = _ModulePackage.get_cls().model() + + +@dataclass +class Monitor(Schema): + message: String = String(example="Project monitor request successful") + output: String = String(example="ok") + packages: List = List(Nested(_module_package_model)) + + # OUTPUT MODELS project_chroot_model = ProjectChroot.get_cls().model() project_chroot_build_config_model = ProjectChrootBuildConfig.get_cls().model() @@ -543,6 +587,9 @@ class NevraPackages(Schema): build_chroot_model = BuildChroot.get_cls().model() build_chroot_config_model = BuildChrootConfig.get_cls().model() nevra_packages_model = NevraPackages.get_cls().model() +module_build_model = ModuleBuild.get_cls().model() +webhook_secret_model = WebhookSecret.get_cls().model() +monitor_model = Monitor.get_cls().model() pagination_project_model = Pagination(items=List(Nested(project_model))).model() pagination_build_chroot_model = Pagination(items=List(Nested(build_chroot_model))).model() @@ -561,6 +608,7 @@ class NevraPackages(Schema): project_edit_input_model = ProjectEdit.get_cls().input_model() project_fork_input_model = ProjectFork.get_cls().input_model() project_delete_input_model = ProjectDelete.get_cls().input_model() +module_add_input_model = ModuleAdd.get_cls().input_model() # PARAMETER SCHEMAS