From 3e8ce49dbe8091c7d34afdc411c3a012eb68f4d3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Sep 2024 08:03:02 +0200 Subject: [PATCH 01/90] Bump debugpy from 1.8.5 to 1.8.6 in /requirements/partial (#2648) Bumps [debugpy](https://github.com/microsoft/debugpy) from 1.8.5 to 1.8.6. - [Release notes](https://github.com/microsoft/debugpy/releases) - [Commits](https://github.com/microsoft/debugpy/compare/v1.8.5...v1.8.6) --- updated-dependencies: - dependency-name: debugpy dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/partial/requirements_development.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/partial/requirements_development.txt b/requirements/partial/requirements_development.txt index 3d3b9af5f2..66bfd85f31 100644 --- a/requirements/partial/requirements_development.txt +++ b/requirements/partial/requirements_development.txt @@ -1,7 +1,7 @@ aiosmtpd==1.4.6 autoflake==2.3.1 black==24.8.0 -debugpy==1.8.5 +debugpy==1.8.6 flake8==7.1.1 isort==5.13.2 mypy==1.11.2 From 8ac3dd3d96b3d478e567b7b41226eb75cad77762 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20B=C3=B6hlke?= Date: Tue, 24 Sep 2024 15:27:40 +0200 Subject: [PATCH 02/90] change imports to os_authlib --- .../action/actions/user/forget_password_confirm.py | 2 +- openslides_backend/http/views/base_view.py | 2 +- openslides_backend/presenter/presenter.py | 2 +- openslides_backend/services/auth/adapter.py | 11 ++++------- openslides_backend/services/auth/interface.py | 2 +- .../services/shared/authenticated_service.py | 2 +- tests/util.py | 2 +- 7 files changed, 10 insertions(+), 13 deletions(-) diff --git a/openslides_backend/action/actions/user/forget_password_confirm.py b/openslides_backend/action/actions/user/forget_password_confirm.py index d3badc8371..766b3f62aa 100644 --- a/openslides_backend/action/actions/user/forget_password_confirm.py +++ b/openslides_backend/action/actions/user/forget_password_confirm.py @@ -2,7 +2,7 @@ from typing import Any from urllib.parse import unquote -from authlib.exceptions import InvalidCredentialsException +from os_authlib.exceptions import InvalidCredentialsException from openslides_backend.action.util.typing import ActionData diff --git a/openslides_backend/http/views/base_view.py b/openslides_backend/http/views/base_view.py index 9948d1656b..8d0b3c48bc 100644 --- a/openslides_backend/http/views/base_view.py +++ b/openslides_backend/http/views/base_view.py @@ -4,7 +4,7 @@ from re import Pattern from typing import Any, Optional -from authlib import AUTHENTICATION_HEADER, COOKIE_NAME +from os_authlib import AUTHENTICATION_HEADER, COOKIE_NAME from werkzeug.exceptions import BadRequest as WerkzeugBadRequest from ...shared.exceptions import View400Exception diff --git a/openslides_backend/presenter/presenter.py b/openslides_backend/presenter/presenter.py index bb4bd2864a..8d6ff04d87 100644 --- a/openslides_backend/presenter/presenter.py +++ b/openslides_backend/presenter/presenter.py @@ -1,7 +1,7 @@ from collections.abc import Callable import fastjsonschema -from authlib import AUTHENTICATION_HEADER, COOKIE_NAME +from os_authlib import AUTHENTICATION_HEADER, COOKIE_NAME from fastjsonschema import JsonSchemaException from ..http.request import Request diff --git a/openslides_backend/services/auth/adapter.py b/openslides_backend/services/auth/adapter.py index c95dcd598d..191dbc2b89 100644 --- a/openslides_backend/services/auth/adapter.py +++ b/openslides_backend/services/auth/adapter.py @@ -1,6 +1,6 @@ from urllib import parse -from authlib import ( +from os_authlib import ( ANONYMOUS_USER, AUTHORIZATION_HEADER, AuthenticateException, @@ -8,11 +8,10 @@ AuthorizationException, InvalidCredentialsException, ) - +from .interface import AuthenticationService +from ..shared.authenticated_service import AuthenticatedService from ...shared.exceptions import AuthenticationException from ...shared.interfaces.logging import LoggingModule -from ..shared.authenticated_service import AuthenticatedService -from .interface import AuthenticationService class AuthenticationHTTPAdapter(AuthenticationService, AuthenticatedService): @@ -34,9 +33,7 @@ def authenticate(self) -> tuple[int, str | None]: f"Start request to authentication service with the following data: access_token: {self.access_token}, cookie: {self.refresh_id}" ) try: - return self.auth_handler.authenticate( - self.access_token, parse.unquote(self.refresh_id) - ) + return self.auth_handler.authenticate(self.access_token) except (AuthenticateException, InvalidCredentialsException) as e: self.logger.debug(f"Error in auth service: {e.message}") raise AuthenticationException(e.message) diff --git a/openslides_backend/services/auth/interface.py b/openslides_backend/services/auth/interface.py index df16cedd25..eba3404d3b 100644 --- a/openslides_backend/services/auth/interface.py +++ b/openslides_backend/services/auth/interface.py @@ -1,6 +1,6 @@ from typing import Any, Protocol -from authlib import AUTHENTICATION_HEADER, COOKIE_NAME # noqa +from os_authlib import AUTHENTICATION_HEADER, COOKIE_NAME # noqa from ..shared.authenticated_service import AuthenticatedServiceInterface diff --git a/openslides_backend/services/shared/authenticated_service.py b/openslides_backend/services/shared/authenticated_service.py index 7534f689a4..3d225e001b 100644 --- a/openslides_backend/services/shared/authenticated_service.py +++ b/openslides_backend/services/shared/authenticated_service.py @@ -1,7 +1,7 @@ from abc import abstractmethod from typing import Protocol -from authlib import AUTHENTICATION_HEADER, COOKIE_NAME +from os_authlib import AUTHENTICATION_HEADER, COOKIE_NAME class AuthenticatedServiceInterface(Protocol): diff --git a/tests/util.py b/tests/util.py index 01f88f4287..ce7d2ce2dc 100644 --- a/tests/util.py +++ b/tests/util.py @@ -2,7 +2,7 @@ from typing import Any, TypedDict, cast import simplejson as json -from authlib import AUTHENTICATION_HEADER, COOKIE_NAME, AuthenticateException +from os_authlib import AUTHENTICATION_HEADER, COOKIE_NAME, AuthenticateException from werkzeug.test import Client as WerkzeugClient from werkzeug.test import TestResponse from werkzeug.wrappers import Response as BaseResponse From 8b3b0090c26be04d2363934345c765fb8f6220ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20B=C3=B6hlke?= Date: Wed, 25 Sep 2024 14:44:18 +0200 Subject: [PATCH 03/90] Cleanup saml and auth service based code --- .../action/actions/organization/update.py | 25 -- .../action/actions/user/__init__.py | 7 - .../action/actions/user/create.py | 2 - .../action/actions/user/forget_password.py | 119 --------- .../actions/user/forget_password_confirm.py | 63 ----- .../actions/user/generate_new_password.py | 39 --- .../action/actions/user/password_mixins.py | 51 ---- .../actions/user/reset_password_to_default.py | 54 ----- .../action/actions/user/save_saml_account.py | 226 ------------------ .../action/actions/user/set_password.py | 41 ---- .../action/actions/user/set_password_self.py | 57 ----- openslides_backend/services/auth/adapter.py | 7 - 12 files changed, 691 deletions(-) delete mode 100644 openslides_backend/action/actions/user/forget_password.py delete mode 100644 openslides_backend/action/actions/user/forget_password_confirm.py delete mode 100644 openslides_backend/action/actions/user/generate_new_password.py delete mode 100644 openslides_backend/action/actions/user/password_mixins.py delete mode 100644 openslides_backend/action/actions/user/reset_password_to_default.py delete mode 100644 openslides_backend/action/actions/user/save_saml_account.py delete mode 100644 openslides_backend/action/actions/user/set_password.py delete mode 100644 openslides_backend/action/actions/user/set_password_self.py diff --git a/openslides_backend/action/actions/organization/update.py b/openslides_backend/action/actions/organization/update.py index 31eb695776..93ae972a5e 100644 --- a/openslides_backend/action/actions/organization/update.py +++ b/openslides_backend/action/actions/organization/update.py @@ -13,7 +13,6 @@ from ...mixins.send_email_mixin import EmailCheckMixin, EmailSenderCheckMixin from ...util.default_schema import DefaultSchema from ...util.register import register_action -from ..user.save_saml_account import allowed_user_fields from ..user.update import UserUpdate @@ -48,36 +47,12 @@ class OrganizationUpdate( "limit_of_meetings", "limit_of_users", "url", - "saml_enabled", - "saml_login_button_text", - "saml_attr_mapping", - "saml_metadata_idp", - "saml_metadata_sp", - "saml_private_key", ) model = Organization() - saml_props = { - field: {**optional_str_schema, "max_length": 256} - for field in allowed_user_fields - } - saml_props["meeting"] = { - "type": ["object", "null"], - "properties": { - field: {**optional_str_schema, "max_length": 256} - for field in ("external_id", "external_group_id") - }, - "additionalProperties": False, - } schema = DefaultSchema(Organization()).get_update_schema( optional_properties=group_A_fields + group_B_fields, additional_optional_fields={ - "saml_attr_mapping": { - "type": ["object", "null"], - "properties": saml_props, - "required": ["saml_id"], - "additionalProperties": False, - }, }, ) check_email_field = "users_email_replyto" diff --git a/openslides_backend/action/actions/user/__init__.py b/openslides_backend/action/actions/user/__init__.py index 19443902c3..6e49b5178d 100644 --- a/openslides_backend/action/actions/user/__init__.py +++ b/openslides_backend/action/actions/user/__init__.py @@ -4,17 +4,10 @@ assign_meetings, create, delete, - forget_password, - forget_password_confirm, - generate_new_password, merge_together, participant_import, participant_json_upload, - reset_password_to_default, - save_saml_account, send_invitation_email, - set_password, - set_password_self, set_present, toggle_presence_by_number, update, diff --git a/openslides_backend/action/actions/user/create.py b/openslides_backend/action/actions/user/create.py index 090870035e..49c5edbc2c 100644 --- a/openslides_backend/action/actions/user/create.py +++ b/openslides_backend/action/actions/user/create.py @@ -16,7 +16,6 @@ from ...util.typing import ActionResultElement from ..meeting_user.mixin import CheckLockOutPermissionMixin from .create_update_permissions_mixin import CreateUpdatePermissionsMixin -from .password_mixins import SetPasswordMixin from .user_mixins import LimitOfUserMixin, UserMixin, UsernameMixin, check_gender_helper @@ -25,7 +24,6 @@ class UserCreate( EmailCheckMixin, CreateAction, CreateUpdatePermissionsMixin, - SetPasswordMixin, LimitOfUserMixin, UsernameMixin, CheckLockOutPermissionMixin, diff --git a/openslides_backend/action/actions/user/forget_password.py b/openslides_backend/action/actions/user/forget_password.py deleted file mode 100644 index 40de46e003..0000000000 --- a/openslides_backend/action/actions/user/forget_password.py +++ /dev/null @@ -1,119 +0,0 @@ -from collections import defaultdict -from time import time -from typing import Any -from urllib.parse import quote - -from openslides_backend.shared.util import ONE_ORGANIZATION_FQID - -from ....i18n.translator import translate as _ -from ....models.models import User -from ....shared.exceptions import ActionException -from ....shared.filters import FilterOperator -from ...generics.update import UpdateAction -from ...mixins.send_email_mixin import EmailSettings, EmailUtils -from ...util.default_schema import DefaultSchema -from ...util.register import register_action -from ...util.typing import ActionData - - -class format_dict(defaultdict): - def __missing__(self, key: str) -> str: - return f"'{key}'" - - -@register_action("user.forget_password") -class UserForgetPassword(UpdateAction): - """ - Action to send forget password mail(s). - """ - - model = User() - schema = DefaultSchema(User()).get_default_schema( - title="user forget password schema", - additional_required_fields={"email": {"type": "string"}}, - ) - skip_archived_meeting_check = True - - def get_updated_instances(self, action_data: ActionData) -> ActionData: - self.PW_FORGET_EMAIL_TEMPLATE = _( - """You are receiving this email because you have requested a new password for your OpenSlides account. - -Please open the following link and choose a new password: -{url}/login/forget-password-confirm?user_id={user_id}&token={token} - -The link will be valid for 10 minutes.""" - ) - self.PW_FORGET_EMAIL_SUBJECT = _("Reset your OpenSlides password") - for instance in action_data: - email = instance.pop("email") - - # check if email adress is valid - if not EmailUtils.check_email(EmailSettings.default_from_email): - raise ActionException( - "The server was configured improperly. Please contact your administrator." - ) - if not EmailUtils.check_email(email): - raise ActionException(f"'{email}' is not a valid email adress.") - - # search for users with email - filter_ = FilterOperator("email", "=", email) - results = self.datastore.filter( - self.model.collection, filter_, ["id", "username", "saml_id"] - ) - - organization = self.datastore.get( - ONE_ORGANIZATION_FQID, ["url"], lock_result=False - ) - url = organization.get("url", "") - - # try to send the mails. - try: - with EmailUtils.get_mail_connection() as mail_client: - for user in results.values(): - if user.get("saml_id"): - raise ActionException( - f"user {user['saml_id']} is a Single Sign On user and has no local OpenSlides password." - ) - username = user["username"] - ok, errors = EmailUtils.send_email_safe( - mail_client, - self.logger, - EmailSettings.default_from_email, - email, - self.PW_FORGET_EMAIL_SUBJECT + f": {username}", - self.get_email_body( - user["id"], - self.get_token(user["id"], email), - user["username"], - url, - ), - html=False, - ) - if ok: - yield {"id": user["id"], "last_email_sent": round(time())} - except ActionException as e: - self.logger.error(f"send mail action exception: {str(e)}") - raise - except Exception as e: - self.logger.error(f"General send mail exception: {str(e)}") - raise ActionException( - "The server was configured improperly. Please contact your administrator." - ) - - def get_token(self, user_id: int, email: str) -> str: - return quote(self.auth.create_authorization_token(user_id, email)) - - def get_email_body(self, user_id: int, token: str, username: str, url: str) -> str: - body_format = format_dict( - None, - { - "user_id": user_id, - "token": token, - "username": username, - "url": url, - }, - ) - return self.PW_FORGET_EMAIL_TEMPLATE.format_map(body_format) - - def check_permissions(self, instance: dict[str, Any]) -> None: - pass diff --git a/openslides_backend/action/actions/user/forget_password_confirm.py b/openslides_backend/action/actions/user/forget_password_confirm.py deleted file mode 100644 index 766b3f62aa..0000000000 --- a/openslides_backend/action/actions/user/forget_password_confirm.py +++ /dev/null @@ -1,63 +0,0 @@ -from collections.abc import Callable -from typing import Any -from urllib.parse import unquote - -from os_authlib.exceptions import InvalidCredentialsException - -from openslides_backend.action.util.typing import ActionData - -from ....models.models import User -from ....shared.exceptions import ActionException -from ....shared.schema import required_id_schema -from ...generics.update import UpdateAction -from ...util.default_schema import DefaultSchema -from ...util.register import register_action -from .password_mixins import ClearSessionsMixin - - -@register_action("user.forget_password_confirm") -class UserForgetPasswordConfirm(UpdateAction, ClearSessionsMixin): - """ - Action to set a forgotten password. - """ - - model = User() - schema = DefaultSchema(User()).get_default_schema( - title="user forget password confirm schema", - additional_required_fields={ - "new_password": {"type": "string"}, - "user_id": required_id_schema, - "authorization_token": {"type": "string"}, - }, - ) - skip_archived_meeting_check = True - - def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: - user_id = instance.pop("user_id") - user = self.datastore.get(f"user/{user_id}", ["saml_id"], lock_result=False) - new_password = instance.pop("new_password") - if user.get("saml_id"): - raise ActionException( - f"user {user['saml_id']} is a Single Sign On user and has no local OpenSlides password." - ) - token = instance.pop("authorization_token") - self.check_token(user_id, token) - instance["id"] = user_id - instance["password"] = self.auth.hash(new_password) - return instance - - def check_token(self, user_id: int, token: str) -> None: - try: - if not self.auth.verify_authorization_token(user_id, unquote(token)): - raise ActionException("Failed to verify token.") - except InvalidCredentialsException: - raise ActionException("Failed to verify token.") - - def check_permissions(self, instance: dict[str, Any]) -> None: - pass - - def get_on_success(self, action_data: ActionData) -> Callable[[], None] | None: - def on_success() -> None: - self.auth.clear_all_sessions() - - return on_success diff --git a/openslides_backend/action/actions/user/generate_new_password.py b/openslides_backend/action/actions/user/generate_new_password.py deleted file mode 100644 index 5c2117875a..0000000000 --- a/openslides_backend/action/actions/user/generate_new_password.py +++ /dev/null @@ -1,39 +0,0 @@ -from typing import Any - -from ....action.generics.update import UpdateAction -from ....action.mixins.archived_meeting_check_mixin import CheckForArchivedMeetingMixin -from ....models.models import User -from ....permissions.management_levels import OrganizationManagementLevel -from ....permissions.permissions import Permissions -from ....shared.mixins.user_scope_mixin import UserScopeMixin -from ...util.crypto import get_random_password -from ...util.default_schema import DefaultSchema -from ...util.register import register_action -from .password_mixins import ClearSessionsMixin, SetPasswordMixin - - -@register_action("user.generate_new_password") -class UserGenerateNewPassword( - SetPasswordMixin, - CheckForArchivedMeetingMixin, - UserScopeMixin, - ClearSessionsMixin, - UpdateAction, -): - model = User() - schema = DefaultSchema(User()).get_update_schema() - permission = OrganizationManagementLevel.CAN_MANAGE_USERS - - def check_permissions(self, instance: dict[str, Any]) -> None: - self.check_permissions_for_scope( - instance["id"], meeting_permission=Permissions.User.CAN_UPDATE - ) - - def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: - """ - Generates new password and call the super code. - """ - instance["password"] = get_random_password() - instance["set_as_default"] = True - self.set_password(instance) - return instance diff --git a/openslides_backend/action/actions/user/password_mixins.py b/openslides_backend/action/actions/user/password_mixins.py deleted file mode 100644 index 55275463e7..0000000000 --- a/openslides_backend/action/actions/user/password_mixins.py +++ /dev/null @@ -1,51 +0,0 @@ -from collections.abc import Callable -from typing import Any - -from openslides_backend.shared.patterns import fqid_from_collection_and_id - -from ....shared.exceptions import ActionException -from ...action import Action -from ...util.typing import ActionData - - -class SetPasswordMixin(Action): - def reset_password(self, instance: dict[str, Any]) -> None: - instance["password"] = instance["default_password"] - self.set_password(instance) - - def set_password(self, instance: dict[str, Any]) -> None: - """ - Hashes the password given in the instance (which is assumed to be plain text) and sets it as - the default password if `set_as_default` is True in the instance. - """ - if "meta_new" not in instance: - user = self.datastore.get( - fqid_from_collection_and_id("user", instance["id"]), - ["saml_id"], - lock_result=False, - ) - else: - user = instance - if user.get("saml_id"): - raise ActionException( - f"user {user['saml_id']} is a Single Sign On user and has no local OpenSlides password." - ) - - password = instance.pop("password") - instance["password"] = self.auth.hash(password) - if instance.pop("set_as_default", False): - instance["default_password"] = password - - -class ClearSessionsMixin(Action): - """Adds an on_success method to the action that clears all sessions.""" - - def get_on_success(self, action_data: ActionData) -> Callable[[], None] | None: - def on_success() -> None: - self.auth.clear_all_sessions() - - # only clear session if the user changed his own password - if any(instance["id"] == self.user_id for instance in action_data): - return on_success - else: - return None diff --git a/openslides_backend/action/actions/user/reset_password_to_default.py b/openslides_backend/action/actions/user/reset_password_to_default.py deleted file mode 100644 index 04315ab25b..0000000000 --- a/openslides_backend/action/actions/user/reset_password_to_default.py +++ /dev/null @@ -1,54 +0,0 @@ -from typing import Any - -from ....action.mixins.archived_meeting_check_mixin import CheckForArchivedMeetingMixin -from ....models.models import User -from ....permissions.management_levels import OrganizationManagementLevel -from ....permissions.permissions import Permissions -from ....shared.exceptions import ActionException -from ....shared.mixins.user_scope_mixin import UserScopeMixin -from ....shared.patterns import fqid_from_collection_and_id -from ...generics.update import UpdateAction -from ...util.default_schema import DefaultSchema -from ...util.register import register_action -from .password_mixins import ClearSessionsMixin - - -class UserResetPasswordToDefaultMixin( - UpdateAction, CheckForArchivedMeetingMixin, ClearSessionsMixin -): - def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: - """ - Gets the default_password and reset password. - """ - instance = super().update_instance(instance) - user = self.datastore.get( - fqid_from_collection_and_id(self.model.collection, instance["id"]), - ["default_password", "saml_id"], - lock_result=False, - ) - if user.get("saml_id"): - raise ActionException( - f"user {user['saml_id']} is a Single Sign On user and has no local OpenSlides password." - ) - default_password = self.auth.hash(str(user.get("default_password"))) - instance["password"] = default_password - return instance - - -@register_action("user.reset_password_to_default") -class UserResetPasswordToDefaultAction( - UserResetPasswordToDefaultMixin, - UserScopeMixin, -): - """ - Action to reset a password to default of a user. - """ - - model = User() - schema = DefaultSchema(User()).get_update_schema() - permission = OrganizationManagementLevel.CAN_MANAGE_USERS - - def check_permissions(self, instance: dict[str, Any]) -> None: - self.check_permissions_for_scope( - instance["id"], meeting_permission=Permissions.User.CAN_UPDATE - ) diff --git a/openslides_backend/action/actions/user/save_saml_account.py b/openslides_backend/action/actions/user/save_saml_account.py deleted file mode 100644 index c41156b20d..0000000000 --- a/openslides_backend/action/actions/user/save_saml_account.py +++ /dev/null @@ -1,226 +0,0 @@ -from collections.abc import Iterable -from typing import Any, cast - -import fastjsonschema - -from openslides_backend.shared.util import ONE_ORGANIZATION_FQID - -from ....models.models import User -from ....shared.exceptions import ActionException -from ....shared.filters import And, FilterOperator -from ....shared.interfaces.event import Event -from ....shared.schema import schema_version -from ....shared.typing import Schema -from ...mixins.meeting_user_helper import get_meeting_user -from ...mixins.send_email_mixin import EmailCheckMixin -from ...mixins.singular_action_mixin import SingularActionMixin -from ...util.action_type import ActionType -from ...util.register import register_action -from ...util.typing import ActionData, ActionResultElement -from .create import UserCreate -from .update import UserUpdate -from .user_mixins import UsernameMixin - -allowed_user_fields = [ - "saml_id", - "title", - "first_name", - "last_name", - "email", - "gender", - "pronoun", - "is_active", - "is_physical_person", -] - - -@register_action("user.save_saml_account", action_type=ActionType.STACK_INTERNAL) -class UserSaveSamlAccount( - EmailCheckMixin, - UsernameMixin, - SingularActionMixin, -): - """ - Internal action to save (create or update) a saml account. - It should be called from the auth service. - """ - - user: dict[str, Any] = {} - saml_attr_mapping: dict[str, str] - check_email_field = "email" - model = User() - schema: Schema = {} - skip_archived_meeting_check = True - - def validate_instance(self, instance: dict[str, Any]) -> None: - organization = self.datastore.get( - ONE_ORGANIZATION_FQID, - ["saml_enabled", "saml_attr_mapping"], - lock_result=False, - ) - if not organization.get("saml_enabled"): - raise ActionException( - "SingleSignOn is not enabled in OpenSlides configuration" - ) - self.saml_attr_mapping = organization.get("saml_attr_mapping", {}) - if not self.saml_attr_mapping or not isinstance(self.saml_attr_mapping, dict): - raise ActionException( - "SingleSignOn field attributes are not configured in OpenSlides" - ) - self.schema = { - "$schema": schema_version, - "title": "create saml account schema", - "type": "object", - "properties": { - payload_field: { - "oneOf": [ - (type_def := self.model.get_field(model_field).get_schema()), - { - "type": "array", - "items": type_def, - "minItems": 1 if model_field == "saml_id" else 0, - }, - ] - } - for model_field, payload_field in self.saml_attr_mapping.items() - if model_field in allowed_user_fields - }, - "required": [self.saml_attr_mapping["saml_id"]], - "additionalProperties": True, - } - try: - fastjsonschema.validate(self.schema, instance) - except fastjsonschema.JsonSchemaException as exception: - raise ActionException(exception.message) - - def validate_fields(self, instance_old: dict[str, Any]) -> dict[str, Any]: - """ - Transforms the payload fields into model fields, removes the possible array-wrapped format - """ - instance: dict[str, Any] = dict() - for model_field, payload_field in self.saml_attr_mapping.items(): - if ( - isinstance(payload_field, str) - and payload_field in instance_old - and model_field in allowed_user_fields - ): - value = ( - tx[0] - if isinstance((tx := instance_old[payload_field]), list) and len(tx) - else tx - ) - if value not in (None, []): - instance[model_field] = value - - return super().validate_fields(instance) - - def prepare_action_data(self, action_data: ActionData) -> ActionData: - """Necessary to prevent id reservation in CreateAction's prepare_action_data""" - return action_data - - def check_permissions(self, instance: dict[str, Any]) -> None: - pass - - def base_update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: - meeting_id, group_id = self.check_for_group_add() - users = self.datastore.filter( - "user", - FilterOperator("saml_id", "=", instance["saml_id"]), - ["id", *allowed_user_fields], - ) - if len(users) == 1: - self.user = next(iter(users.values())) - instance["id"] = (user_id := cast(int, self.user["id"])) - if meeting_id and group_id: - meeting_user = get_meeting_user( - self.datastore, meeting_id, user_id, ["id", "group_ids"] - ) - if meeting_user: - old_group_ids = meeting_user["group_ids"] - if group_id not in old_group_ids: - instance["meeting_id"] = meeting_id - instance["group_ids"] = old_group_ids + [group_id] - else: - instance["meeting_id"] = meeting_id - instance["group_ids"] = [group_id] - - instance = { - k: v for k, v in instance.items() if k == "id" or v != self.user.get(k) - } - if len(instance) > 1: - self.execute_other_action(UserUpdate, [instance]) - elif len(users) == 0: - instance = self.set_defaults(instance) - if group_id: - instance["meeting_id"] = meeting_id - instance["group_ids"] = [group_id] - self.execute_other_action(UserCreate, [instance]) - else: - ActionException( - f"More than one existing user found in database with saml_id {instance['saml_id']}" - ) - return instance - - def create_events(self, instance: dict[str, Any]) -> Iterable[Event]: - """ - delegated to execute_other_actions - """ - return [] - - def create_action_result_element( - self, instance: dict[str, Any] - ) -> ActionResultElement | None: - return {"user_id": instance["id"]} - - def set_defaults(self, instance: dict[str, Any]) -> dict[str, Any]: - if "is_active" not in instance: - instance["is_active"] = True - if "is_physical_person" not in instance: - instance["is_physical_person"] = True - instance["can_change_own_password"] = False - instance["username"] = self.generate_usernames([instance.get("saml_id", "")])[0] - return instance - - def check_for_group_add(self) -> tuple[int, int] | tuple[None, None]: - NoneResult = (None, None) - if not ( - meeting_info := cast(dict, self.saml_attr_mapping.get("meeting")) - ) or not (external_id := meeting_info.get("external_id")): - return NoneResult - - meetings = self.datastore.filter( - collection="meeting", - filter=FilterOperator("external_id", "=", external_id), - mapped_fields=["id", "default_group_id"], - ) - if len(meetings) == 1: - meeting = next(iter(meetings.values())) - group_id = meeting["default_group_id"] - else: - self.logger.warning( - f"save_saml_account found {len(meetings)} meetings with external_id '{external_id}'" - ) - return NoneResult - if external_group_id := meeting_info.get("external_group_id"): - groups = self.datastore.filter( - collection="group", - filter=And( - [ - FilterOperator("external_id", "=", external_group_id), - FilterOperator("meeting_id", "=", meeting.get("id")), - ] - ), - mapped_fields=["id"], - ) - if len(groups) == 1: - group_id = next(iter(groups.keys())) - else: - self.logger.warning( - f"save_saml_account found no group in meeting '{external_id}' for '{external_group_id}', but use default_group of meeting" - ) - if not group_id: - self.logger.warning( - f"save_saml_account found no group in meeting '{external_id}' for '{external_group_id}'" - ) - return NoneResult - return meeting.get("id"), group_id diff --git a/openslides_backend/action/actions/user/set_password.py b/openslides_backend/action/actions/user/set_password.py deleted file mode 100644 index 6b856c4d6c..0000000000 --- a/openslides_backend/action/actions/user/set_password.py +++ /dev/null @@ -1,41 +0,0 @@ -from typing import Any - -from ....action.generics.update import UpdateAction -from ....action.mixins.archived_meeting_check_mixin import CheckForArchivedMeetingMixin -from ....models.models import User -from ....permissions.management_levels import OrganizationManagementLevel -from ....permissions.permissions import Permissions -from ....shared.mixins.user_scope_mixin import UserScopeMixin -from ...util.default_schema import DefaultSchema -from ...util.register import register_action -from .password_mixins import ClearSessionsMixin, SetPasswordMixin - - -@register_action("user.set_password") -class UserSetPasswordAction( - SetPasswordMixin, - UserScopeMixin, - CheckForArchivedMeetingMixin, - ClearSessionsMixin, - UpdateAction, -): - """ - Action to set the password and default_pasword. - """ - - model = User() - schema = DefaultSchema(User()).get_update_schema( - required_properties=["password"], - additional_optional_fields={"set_as_default": {"type": "boolean"}}, - ) - history_information = "Password changed" - permission = OrganizationManagementLevel.CAN_MANAGE_USERS - - def check_permissions(self, instance: dict[str, Any]) -> None: - self.check_permissions_for_scope( - instance["id"], meeting_permission=Permissions.User.CAN_UPDATE - ) - - def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: - self.set_password(instance) - return instance diff --git a/openslides_backend/action/actions/user/set_password_self.py b/openslides_backend/action/actions/user/set_password_self.py deleted file mode 100644 index fe77327303..0000000000 --- a/openslides_backend/action/actions/user/set_password_self.py +++ /dev/null @@ -1,57 +0,0 @@ -from typing import Any - -from ....action.mixins.archived_meeting_check_mixin import CheckForArchivedMeetingMixin -from ....models.models import User -from ....shared.exceptions import ActionException, PermissionDenied -from ....shared.patterns import fqid_from_collection_and_id -from ...generics.update import UpdateAction -from ...util.default_schema import DefaultSchema -from ...util.register import register_action -from .password_mixins import ClearSessionsMixin - - -@register_action("user.set_password_self") -class UserSetPasswordSelf( - UpdateAction, CheckForArchivedMeetingMixin, ClearSessionsMixin -): - """ - Action to update the own password. - """ - - model = User() - schema = DefaultSchema(User()).get_default_schema( - additional_required_fields={ - "old_password": {"type": "string", "minLength": 1}, - "new_password": {"type": "string", "minLength": 1}, - } - ) - - def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: - old_pw = instance.pop("old_password") - new_pw = instance.pop("new_password") - - db_instance = self.datastore.get( - fqid_from_collection_and_id(self.model.collection, self.user_id), - ["password", "saml_id"], - lock_result=False, - ) - if db_instance.get("saml_id"): - raise ActionException( - f"user {db_instance['saml_id']} is a Single Sign On user and has no local OpenSlides password." - ) - if not self.auth.is_equal(old_pw, db_instance["password"]): - raise ActionException("Wrong password") - - instance["password"] = self.auth.hash(new_pw) - return instance - - def check_permissions(self, instance: dict[str, Any]) -> None: - self.assert_not_anonymous() - instance["id"] = self.user_id - user = self.datastore.get( - fqid_from_collection_and_id(self.model.collection, self.user_id), - ["can_change_own_password"], - lock_result=False, - ) - if not user.get("can_change_own_password"): - raise PermissionDenied("You cannot change your password.") diff --git a/openslides_backend/services/auth/adapter.py b/openslides_backend/services/auth/adapter.py index 191dbc2b89..a1198aff35 100644 --- a/openslides_backend/services/auth/adapter.py +++ b/openslides_backend/services/auth/adapter.py @@ -47,13 +47,6 @@ def is_equal(self, toHash: str, toCompare: str) -> bool: def is_anonymous(self, user_id: int) -> bool: return user_id == ANONYMOUS_USER - def create_authorization_token(self, user_id: int, email: str) -> str: - try: - response = self.auth_handler.create_authorization_token(user_id, email) - except AuthenticateException as e: - raise AuthenticationException(e.message) - return response.headers.get(AUTHORIZATION_HEADER, "") - def verify_authorization_token(self, user_id: int, token: str) -> bool: try: found_user_id, _ = self.auth_handler.verify_authorization_token(token) From 8abec7b95f7ada41c00d4ff80194350ddf3ba51a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20B=C3=B6hlke?= Date: Wed, 25 Sep 2024 15:34:13 +0200 Subject: [PATCH 04/90] cleanup --- docs/Actions-Overview.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/Actions-Overview.md b/docs/Actions-Overview.md index 428e226260..382a6a7291 100644 --- a/docs/Actions-Overview.md +++ b/docs/Actions-Overview.md @@ -231,7 +231,6 @@ A more general format description see in [Action-Service](https://github.com/Ope - [user.toggle_presence_by_number](actions/user.toggle_presence_by_number.md) - [user.update](actions/user.update.md) - [user.update_self](actions/user.update_self.md) -- [user.save_saml_account](actions/user.save_saml_account.md) - [meeting_user.create](actions/meeting_user.create.md) - [meeting_user.update](actions/meeting_user.update.md) - [meeting_user.delete](actions/meeting_user.delete.md) From 54c3b2c9098b8e0cdea558f9c51b57ca615d5cab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20B=C3=B6hlke?= Date: Wed, 25 Sep 2024 15:38:09 +0200 Subject: [PATCH 05/90] Use pip-auth libraries code directly from local source --- Makefile | 3 +++ dev/Dockerfile.dev | 1 + openslides_backend/http/views/action_view.py | 9 +++++++++ requirements/partial/requirements_packaged_services.txt | 2 +- 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 658f265ab5..2513566beb 100644 --- a/Makefile +++ b/Makefile @@ -142,7 +142,10 @@ run-dev-otel run-bash-otel: | start-dev-otel run-dev-attach-otel # Build standalone development container (not usable inside the docker container) build-dev: + rm -rf pip-auth + cp -r ../openslides-auth-service/libraries/pip-auth pip-auth docker build --file=dev/Dockerfile.dev . --tag=openslides-backend-dev + rm -rf pip-auth rebuild-dev: docker build --file=dev/Dockerfile.dev . --tag=openslides-backend-dev --no-cache diff --git a/dev/Dockerfile.dev b/dev/Dockerfile.dev index 47a45a06fa..7f87d8958d 100644 --- a/dev/Dockerfile.dev +++ b/dev/Dockerfile.dev @@ -5,6 +5,7 @@ RUN apt-get update && apt-get install --yes make git curl ncat vim bash-completi WORKDIR /app COPY requirements/ requirements/ +COPY pip-auth /pip-auth ARG REQUIREMENTS_FILE=requirements_development.txt RUN . requirements/export_service_commits.sh && pip install --no-cache-dir --requirement requirements/$REQUIREMENTS_FILE diff --git a/openslides_backend/http/views/action_view.py b/openslides_backend/http/views/action_view.py index ac9de5955b..c33457b50d 100644 --- a/openslides_backend/http/views/action_view.py +++ b/openslides_backend/http/views/action_view.py @@ -2,6 +2,7 @@ from base64 import b64decode from pathlib import Path +from os_authlib.message_bus import MessageBus from ...action.action_handler import ActionHandler from ...action.action_worker import handle_action_in_worker_thread from ...i18n.translator import Translator @@ -27,6 +28,10 @@ class ActionView(BaseView): ActionHandler after retrieving request user id. """ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.message_bus = MessageBus() + @route(["handle_request", "handle_separately"]) def action_route(self, request: Request) -> RouteResponse: self.logger.debug("Start dispatching action request.") @@ -84,6 +89,10 @@ def health_route(self, request: Request) -> RouteResponse: def info_route(self, request: Request) -> RouteResponse: return {"healthinfo": {"actions": dict(ActionHandler.get_health_info())}}, None + @route("logout", method="POST", json=False) + def backchannel_logout(self, request: Request) -> RouteResponse: + return {}, None + @route("version", method="GET", json=False) def version_route(self, _: Request) -> RouteResponse: with open(VERSION_PATH) as file: diff --git a/requirements/partial/requirements_packaged_services.txt b/requirements/partial/requirements_packaged_services.txt index f91652ad62..3a0770d920 100644 --- a/requirements/partial/requirements_packaged_services.txt +++ b/requirements/partial/requirements_packaged_services.txt @@ -1,2 +1,2 @@ git+https://github.com/OpenSlides/openslides-datastore-service.git@${DATASTORE_COMMIT_HASH} -git+https://github.com/OpenSlides/openslides-auth-service.git@${AUTH_COMMIT_HASH}#egg=authlib&subdirectory=auth/libraries/pip-auth +-e /pip-auth From 27594eaca3e05796ca8c0ff64a8b108e6c68f1ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20B=C3=B6hlke?= Date: Fri, 27 Sep 2024 09:20:23 +0200 Subject: [PATCH 06/90] Implement back-channel logout endpoint --- openslides_backend/http/views/action_view.py | 26 +++++++++++++++++++- openslides_backend/http/views/base_view.py | 4 +++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/openslides_backend/http/views/action_view.py b/openslides_backend/http/views/action_view.py index c33457b50d..bb2a568388 100644 --- a/openslides_backend/http/views/action_view.py +++ b/openslides_backend/http/views/action_view.py @@ -1,4 +1,5 @@ import binascii +import json from base64 import b64decode from pathlib import Path @@ -91,7 +92,30 @@ def info_route(self, request: Request) -> RouteResponse: @route("logout", method="POST", json=False) def backchannel_logout(self, request: Request) -> RouteResponse: - return {}, None + # topic '', field 'sessionId', value sessionId + self.logger.debug("Received logout request") + try: + logout_token = request.form.get("logout_token") + if not logout_token: + self.logger.error("Missing logout_token") + raise ServerError("Missing logout_token") + + # Verify and decode the logout token + decoded_token = self.services.authentication().auth_handler.verify_logout_token(logout_token) + if decoded_token is None: + return AuthenticationException("Invalid logout token") + + # Extract the session ID (sid) from the token + session_id = decoded_token.get("sid") + if not session_id: + return AuthenticationException("Missing session ID (sid) in logout token") + + self.logger.debug(f"Session ID to terminate: {session_id}") + self.message_bus.redis.xadd("logout", {"sessionId": session_id}) + + return { "success": True }, None + except json.JSONDecodeError: + return ServerError("Invalid JSON payload", status=400) @route("version", method="GET", json=False) def version_route(self, _: Request) -> RouteResponse: diff --git a/openslides_backend/http/views/base_view.py b/openslides_backend/http/views/base_view.py index 8d0b3c48bc..cfca9ff2dc 100644 --- a/openslides_backend/http/views/base_view.py +++ b/openslides_backend/http/views/base_view.py @@ -109,14 +109,18 @@ def dispatch(self, request: Request) -> RouteResponse: if route_options["json"]: # Check mimetype and parse JSON body. The result is cached in request.json if not request.is_json: + self.logger.debug(f"Wrong media type {request.content_type}. Use 'Content-Type: application/json' instead.") raise View400Exception( "Wrong media type. Use 'Content-Type: application/json' instead." ) try: + self.logger.debug(f"Unpacking JSON.") request_body = request.get_json() except WerkzeugBadRequest as exception: + self.logger.debug(f"Request contains invalid JSON.") raise View400Exception(exception.description) self.logger.debug(f"Request contains JSON: {request_body}.") + self.logger.debug(f"Executing handler.") return func(request) raise NotFound() From 221e2d8ce2720e784283e590ebd5a7774d8316c7 Mon Sep 17 00:00:00 2001 From: Hannes Janott Date: Fri, 27 Sep 2024 16:00:51 +0200 Subject: [PATCH 07/90] 2170 introduce gender model (#2485) * draft create update delete + tests + testupdates + migration * update meta rep * general error removal and improvement for draft * remove meeting import error * update meeting import so that checker removes gender_id * general code improvements delete gender checks for back relation in organiziation * md documentation + pleasing mypy * use defaultdict * use new in memory flag of datastore * fix mypy error by upping datastore version * change the permission from can_manage_organization to can_manage_users * improve documentation * lock result false + no orga check * add test for gender import * make name and org id required * Use gender string for saml and meeting import. Refine gender actions and mixin. General code improvements. * cleanup and documentation improvement * beautify code * add test update gender on user merge. * update docs test for meeting import and saml account (+ new test) create gender for saml accounts * updated meta * meeting import create user with gender. export gender strings. improve tests. * Separate test for gender on meeting import. General code improvements in saml, export, import and migration plus additional test for empty string. * extend test for gender import, plus fixes. docs update * improve user and gender updates/creation * Gender will not be created without being used anymore. Fix error where import without any new genders would result in exception. * Improve gender action documentation * fixed typo in migration gender female * update meta * move to upstream main --------- Co-authored-by: Ralf Peschke Co-authored-by: rrenkert --- docs/Actions-Overview.md | 6 + docs/actions/account.import.md | 2 +- docs/actions/account.json_upload.md | 4 +- docs/actions/gender.create.md | 13 ++ docs/actions/gender.delete.md | 13 ++ docs/actions/gender.update.md | 15 ++ docs/actions/meeting.import.md | 3 +- docs/actions/organization.update.md | 3 - docs/actions/participant.import.md | 2 +- docs/actions/participant.json_upload.md | 4 +- docs/actions/user.create.md | 6 +- docs/actions/user.merge_together.md | 2 +- docs/actions/user.save_saml_account.md | 2 +- docs/actions/user.update.md | 2 +- docs/actions/user.update_self.md | 2 +- docs/presenters/export_meeting.md | 3 +- global/data/example-data.json | 36 ++++- global/data/initial-data.json | 30 +++- global/meta | 2 +- openslides_backend/action/actions/__init__.py | 1 + .../action/actions/gender/__init__.py | 1 + .../action/actions/gender/create.py | 28 ++++ .../action/actions/gender/delete.py | 18 +++ .../action/actions/gender/mixins.py | 38 +++++ .../action/actions/gender/update.py | 22 +++ .../action/actions/meeting/clone.py | 3 +- .../action/actions/meeting/import_.py | 72 ++++++++- .../action/actions/organization/update.py | 25 +-- .../action/actions/user/base_import.py | 10 +- .../action/actions/user/base_json_upload.py | 26 ++- .../action/actions/user/create.py | 6 +- .../user/create_update_permissions_mixin.py | 2 +- .../action/actions/user/merge_together.py | 4 +- .../action/actions/user/participant_import.py | 6 + .../action/actions/user/save_saml_account.py | 43 ++++- .../action/actions/user/update.py | 6 +- .../action/actions/user/update_self.py | 6 +- .../action/actions/user/user_mixins.py | 19 ++- .../migrations/migration_handler.py | 2 +- .../migrations/0057_gender_model.py | 74 +++++++++ openslides_backend/models/models.py | 16 +- openslides_backend/shared/export_helper.py | 4 + tests/system/action/gender/__init__.py | 0 tests/system/action/gender/test_create.py | 117 ++++++++++++++ tests/system/action/gender/test_delete.py | 97 ++++++++++++ tests/system/action/gender/test_update.py | 121 ++++++++++++++ tests/system/action/meeting/test_import.py | 148 +++++++++++++++++- .../system/action/organization/test_update.py | 36 ----- .../system/action/user/test_account_import.py | 38 ++--- .../action/user/test_account_json_upload.py | 31 +++- tests/system/action/user/test_create.py | 21 +-- .../system/action/user/test_merge_together.py | 63 ++++++-- .../action/user/test_participant_import.py | 20 ++- .../user/test_participant_json_upload.py | 24 ++- .../action/user/test_save_saml_account.py | 88 ++++++++--- tests/system/action/user/test_update.py | 43 +++-- tests/system/action/user/test_update_self.py | 5 +- .../migrations/test_0057_gender_model.py | 63 ++++++++ .../presenter/test_check_database_all.py | 18 ++- tests/system/presenter/test_export_meeting.py | 3 + 60 files changed, 1281 insertions(+), 237 deletions(-) create mode 100644 docs/actions/gender.create.md create mode 100644 docs/actions/gender.delete.md create mode 100644 docs/actions/gender.update.md create mode 100644 openslides_backend/action/actions/gender/__init__.py create mode 100644 openslides_backend/action/actions/gender/create.py create mode 100644 openslides_backend/action/actions/gender/delete.py create mode 100644 openslides_backend/action/actions/gender/mixins.py create mode 100644 openslides_backend/action/actions/gender/update.py create mode 100644 openslides_backend/migrations/migrations/0057_gender_model.py create mode 100644 tests/system/action/gender/__init__.py create mode 100644 tests/system/action/gender/test_create.py create mode 100644 tests/system/action/gender/test_delete.py create mode 100644 tests/system/action/gender/test_update.py create mode 100644 tests/system/migrations/test_0057_gender_model.py diff --git a/docs/Actions-Overview.md b/docs/Actions-Overview.md index 428e226260..bed33a0fbf 100644 --- a/docs/Actions-Overview.md +++ b/docs/Actions-Overview.md @@ -207,6 +207,12 @@ A more general format description see in [Action-Service](https://github.com/Ope - [theme.delete](actions/theme.delete.md) - [theme.update](actions/theme.update.md) +## Gender + +- [gender.create](actions/theme.create.md) +- [gender.delete](actions/theme.delete.md) +- [gender.update](actions/theme.update.md) + ## Topics - [topic.create](actions/topic.create.md) diff --git a/docs/actions/account.import.md b/docs/actions/account.import.md index d543aa2b5b..2fb7c9b2db 100644 --- a/docs/actions/account.import.md +++ b/docs/actions/account.import.md @@ -1,5 +1,5 @@ ## Payload -```js +``` { // required id: Id; // action worker id diff --git a/docs/actions/account.json_upload.md b/docs/actions/account.json_upload.md index c02ab33ef4..d4e8ee2582 100644 --- a/docs/actions/account.json_upload.md +++ b/docs/actions/account.json_upload.md @@ -2,7 +2,7 @@ Because the data fields are all converted from CSV import file, **they are all of type `string`**. The types noted below are the internal types after conversion in the backend. See [here](preface_special_imports.md#internal-types) for the representation of the types. -```js +``` { // required data: { @@ -14,7 +14,7 @@ The types noted below are the internal types after conversion in the backend. Se member_number: string, // unique member_number, info: done (used as matching field), new (newly added) or error title: string, pronoun: string, - gender: string, // as defined in organization/genders, info: done or warning + gender: string, // info: done or warning default_password: string, // info: generated, done or warning is_active: boolean, is_physical_person: boolean, diff --git a/docs/actions/gender.create.md b/docs/actions/gender.create.md new file mode 100644 index 0000000000..ac2924478b --- /dev/null +++ b/docs/actions/gender.create.md @@ -0,0 +1,13 @@ +## Payload +``` +{ +// Required + name: string; +} +``` + +## Action +Creates the gender if a gender with the same name doesn't exist. + +## Permissions +The user needs to have the organization management level `can_manage_organization`. diff --git a/docs/actions/gender.delete.md b/docs/actions/gender.delete.md new file mode 100644 index 0000000000..f1feb720c5 --- /dev/null +++ b/docs/actions/gender.delete.md @@ -0,0 +1,13 @@ +## Payload +``` +{ +//Required + id: Id; +} +``` + +## Action +Deletes the gender if it is not one of the four default genders. + +## Permissions +The user needs to have the organization management level `can_manage_organization`. diff --git a/docs/actions/gender.update.md b/docs/actions/gender.update.md new file mode 100644 index 0000000000..00187c97bf --- /dev/null +++ b/docs/actions/gender.update.md @@ -0,0 +1,15 @@ +## Payload +``` +{ +// Required + id: Id; +// Group A + name: string; +} +``` + +## Action +Updates the gender if a gender with that name does not exist and is not one of the four default genders. + +## Permissions +- Group A: The user needs the OML `can_manage_organization` diff --git a/docs/actions/meeting.import.md b/docs/actions/meeting.import.md index 60a61cace7..e00797dedf 100644 --- a/docs/actions/meeting.import.md +++ b/docs/actions/meeting.import.md @@ -16,10 +16,11 @@ Import one meeting from a file. The file must only contain exactly one meeting. - The request user is assigned to the admin group. - meeting.is_active_in_organization_id is set. - It has to be checked, whether the organization.limit_of_meetings is unlimited(=0) or lower than the active meetings in organization.active_meeting_ids, if the new meeting is not archived (`is_active_in_organization_id` is set) -- Search for users and if username, first-name, last-name and email are identical use this user instead creating a duplicate. Keep the data, including password, of the exiating user. +- Search for users and if username, first-name, last-name and email are identical use this existing user instead of creating a duplicate. Keep the data, including password, of the existing user. Relevant relations such as to the meeting will be updated though. - Users, that still have to be duplicated: - Imported usernames will be checked for uniqueness and adjusted in the case of collisions. - All previously set user passwords will be replaced +- Genders will only be updated or imported if a new user needs to be created. Updated users will not have their genders updated. ## Permissions diff --git a/docs/actions/organization.update.md b/docs/actions/organization.update.md index 2c9229d083..0d24439ab3 100644 --- a/docs/actions/organization.update.md +++ b/docs/actions/organization.update.md @@ -17,7 +17,6 @@ users_email_replyto: string; users_email_subject: string; users_email_body: text; - genders: string[]; require_duplicate_from: boolean; // Group B @@ -38,8 +37,6 @@ Updates the organization. It has to be checked that the theme_id has to be one of the theme_ids. -Checks for deleted genders and clean the gender of users which have deleted genders. - ## Permissions - Users with OML of `can_manage_organization` can modify group A - Users with OML of `superadmin` can modify group B diff --git a/docs/actions/participant.import.md b/docs/actions/participant.import.md index e2dbe4c620..59f48d077a 100644 --- a/docs/actions/participant.import.md +++ b/docs/actions/participant.import.md @@ -1,5 +1,5 @@ ## Payload -```js +``` { // required id: Id; // import_preview id diff --git a/docs/actions/participant.json_upload.md b/docs/actions/participant.json_upload.md index 17ee256ca1..5b7777c46c 100644 --- a/docs/actions/participant.json_upload.md +++ b/docs/actions/participant.json_upload.md @@ -1,7 +1,7 @@ ## Payload Because the data fields are all converted from CSV import file, **they are all of type `string`**. The types noted below are the internal types after conversion in the backend. See [here](preface_special_imports.md#internal-types) for the representation of the types. -```js +``` { // required meeting_id: Id, @@ -14,7 +14,7 @@ The types noted below are the internal types after conversion in the backend. Se member_number: string, // unique member_number, info: done, error, new (newly added) or remove (missing field permission) title: string, // info: done or remove (missing field permission) pronoun: string, // info: done or remove (missing field permission) - gender: string, // as defined in organization/genders, info: done, warning (undefined gender) or remove (missing field permission) + gender: string, // info: done, warning (undefined gender) or remove (missing field permission) default_password: string, // info: generated, done, warning or remove (missing field permission) is_active: boolean, // info: done or remove (missing field permission) is_physical_person: boolean, // info: done or remove (missing field permission) diff --git a/docs/actions/user.create.md b/docs/actions/user.create.md index a9a7d4039b..358991d6e6 100644 --- a/docs/actions/user.create.md +++ b/docs/actions/user.create.md @@ -11,7 +11,7 @@ is_active: boolean; is_physical_person: boolean; can_change_own_password: boolean; - gender: string; + gender_id: Id; pronoun: string; email: string; default_vote_weight: decimal(6); @@ -56,8 +56,8 @@ Creates a user. * If no `default_password` is given a random one is generated. The default password is hashed via the auth service and the hash is saved within `password`. A given `default_password`is also stored as hashed password. * If `username` is given, it has to be unique within all users. If there already exists a user with the same username, an error must be returned. If the `username` is not given, 1. the saml_id will be used or 2. it has to be generated (see [user.create#generate-a-username](user.create.md#generate-a-username) below). Also the username may not contain spaces. * The `organization_management_level` as restring can be taken from the enum of this user field. -* Remove starting and trailing spaces from `username`, `first_name` and `last_name` -* The given `gender` must be present in `organization/genders` +* Remove starting and trailing spaces from `username`, `first_name` and `last_name`. +* The given `gender_id` must be present in the database. * If `saml_id` is set in payload, there may be no `password` or `default_password` set or generated and `set_change_own_password` will be set to False. * The `member_number` must be unique within all users. * Will throw an error if the `group_ids` contain the meetings `anonymous_group_id`. diff --git a/docs/actions/user.merge_together.md b/docs/actions/user.merge_together.md index b06b4ca6b1..1219ffa4d1 100644 --- a/docs/actions/user.merge_together.md +++ b/docs/actions/user.merge_together.md @@ -13,7 +13,7 @@ is_active: boolean; is_physical_person: boolean; default_password: string; - gender: string; + gender_id: Id; email: string; default_vote_weight: decimal(6); pronoun: string; diff --git a/docs/actions/user.save_saml_account.md b/docs/actions/user.save_saml_account.md index b8c6013e32..523507efce 100644 --- a/docs/actions/user.save_saml_account.md +++ b/docs/actions/user.save_saml_account.md @@ -16,7 +16,7 @@ ## Action The attributes for the payload are all configured in organization-wide settings. The configuration consists of a list of source attribute - target attribute pairs, where the target attributes are the ones documented in the payload. -Creates or updates the saml-account, depending whether the given `saml_id` exists or not. The `saml_id` is guaranteed to be unique in the whole system. The other fields will be set on creation or update. +Creates or updates the saml-account, depending whether the given `saml_id` exists or not. The `saml_id` is guaranteed to be unique in the whole system. If a gender does not exist in the collection, it will be created. The other fields will be set on creation or update. The action must be `STACK_INTERNAL`. It should be called only from the auth service. Extras to do on creation: diff --git a/docs/actions/user.update.md b/docs/actions/user.update.md index 7bb8c5b8f7..495e32389c 100644 --- a/docs/actions/user.update.md +++ b/docs/actions/user.update.md @@ -14,7 +14,7 @@ is_active: boolean; is_physical_person: boolean; can_change_own_password: boolean; - gender: string; + gender_id: Id; pronoun: string; email: string; default_vote_weight: decimal(6); diff --git a/docs/actions/user.update_self.md b/docs/actions/user.update_self.md index c8a5c01322..3c53a3c58f 100644 --- a/docs/actions/user.update_self.md +++ b/docs/actions/user.update_self.md @@ -4,7 +4,7 @@ // Optional username: string; email: string; - gender: string; + gender_id: Id; pronoun: string; } ``` diff --git a/docs/presenters/export_meeting.md b/docs/presenters/export_meeting.md index 73e61cdac2..02d73abc1b 100644 --- a/docs/presenters/export_meeting.md +++ b/docs/presenters/export_meeting.md @@ -11,10 +11,11 @@ ``` JSON with export ``` +The users genders are exported as their names not ids. # Logic The presenter exports the meeting, the collections which belong to the meeting and users of the meeting. It uses the meeting.user_id for that. And it excludes the organization tags and the committee. -Will raise an exception if the meeting is locked voa the `locked_from_inside` setting. +Will raise an exception if the meeting is locked via the `locked_from_inside` setting. # Permissions The request user must have the `SUPERADMIN` organization management level. diff --git a/global/data/example-data.json b/global/data/example-data.json index 459b34d162..6ffb08e605 100644 --- a/global/data/example-data.json +++ b/global/data/example-data.json @@ -1,5 +1,31 @@ { - "_migration_index": 57, + "_migration_index": 58, + "gender":{ + "1":{ + "id": 1, + "name": "male", + "organization_id": 1, + "user_ids":[1] + }, + "2":{ + "id": 2, + "name": "female", + "organization_id": 1, + "user_ids":[2] + }, + "3":{ + "id": 3, + "name": "diverse", + "organization_id": 1, + "user_ids":[3] + }, + "4":{ + "id": 4, + "name": "non-binary", + "organization_id": 1, + "user_ids":[] + } + }, "organization": { "1": { "id": 1, @@ -7,7 +33,7 @@ "legal_notice": "OpenSlides is a free web based presentation and assembly system for visualizing and controlling agenda, motions and elections of an assembly.", "login_text": "Good Morning!", "default_language": "en", - "genders": ["male", "female", "diverse", "non-binary"], + "gender_ids": [1, 2 ,3 ,4], "enable_electronic_voting": true, "enable_chat": true, "reset_password_verbose_errors": true, @@ -60,7 +86,7 @@ "password": "316af7b2ddc20ead599c38541fbe87e9a9e4e960d4017d6e59de188b41b2758flD5BVZAZ8jLy4nYW9iomHcnkXWkfk3PgBjeiTSxjGG7+fBjMBxsaS1vIiAMxYh+K38l0gDW4wcP+i8tgoc4UBg==", "default_password": "admin", "can_change_own_password": true, - "gender": "male", + "gender_id": 1, "default_vote_weight": "1.000000", "organization_management_level": "superadmin", "is_present_in_meeting_ids": [ @@ -89,7 +115,7 @@ "password": "316af7b2ddc20ead599c38541fbe87e9a9e4e960d4017d6e59de188b41b2758fDB3tv5HcCtPRREt7bPGqerTf1AbmoKXt/fVFkLY4znDRh2Yy0m3ZjXD0nHI8oa6KrGlHH/cvysfvf8i2fWIzmw==", "default_password": "a", "can_change_own_password": true, - "gender": "female", + "gender_id": 2, "default_vote_weight": "1.000000", "committee_ids": [ 1 @@ -110,7 +136,7 @@ "password": "316af7b2ddc20ead599c38541fbe87e9a9e4e960d4017d6e59de188b41b2758fIxDxvpkn6dDLRxT9DxJhZ/f04AL2oK2beICRFobSw53CI93U+dfN+w+NaL7BvrcR4JWuMj9NkH4dVjnnI0YTkg==", "default_password": "jKwSLGCk", "can_change_own_password": true, - "gender": "diverse", + "gender_id": 3, "default_vote_weight": "1.000000", "option_ids": [8, 11], "meeting_user_ids": [3], diff --git a/global/data/initial-data.json b/global/data/initial-data.json index 3251be6248..14aa1328ff 100644 --- a/global/data/initial-data.json +++ b/global/data/initial-data.json @@ -1,12 +1,38 @@ { - "_migration_index": 57, + "_migration_index": 58, + "gender":{ + "1":{ + "id": 1, + "name": "male", + "organization_id": 1, + "user_ids":[] + }, + "2":{ + "id": 2, + "name": "female", + "organization_id": 1, + "user_ids":[] + }, + "3":{ + "id": 3, + "name": "diverse", + "organization_id": 1, + "user_ids":[] + }, + "4":{ + "id": 4, + "name": "non-binary", + "organization_id": 1, + "user_ids":[] + } + }, "organization": { "1": { "id": 1, "name": "[Your organization]", "legal_notice": "OpenSlides is a free web based presentation and assembly system for visualizing and controlling agenda, motions and elections of an assembly. The event organizer is resposible for the content.", "login_text": "Welcome to OpenSlides. Please login.", - "genders": ["male", "female", "diverse", "non-binary"], + "gender_ids": [1, 2, 3, 4], "default_language": "en", "enable_electronic_voting": true, "limit_of_meetings": 0, diff --git a/global/meta b/global/meta index 84bb58e141..f322efd811 160000 --- a/global/meta +++ b/global/meta @@ -1 +1 @@ -Subproject commit 84bb58e141e088f9fd0678214d5b51d040d53034 +Subproject commit f322efd81152a9b5ac864ae7145790c1673f29bf diff --git a/openslides_backend/action/actions/__init__.py b/openslides_backend/action/actions/__init__.py index ba80e746f8..1c2636071f 100644 --- a/openslides_backend/action/actions/__init__.py +++ b/openslides_backend/action/actions/__init__.py @@ -12,6 +12,7 @@ def prepare_actions_map() -> None: chat_group, chat_message, committee, + gender, group, list_of_speakers, mediafile, diff --git a/openslides_backend/action/actions/gender/__init__.py b/openslides_backend/action/actions/gender/__init__.py new file mode 100644 index 0000000000..26e86a804e --- /dev/null +++ b/openslides_backend/action/actions/gender/__init__.py @@ -0,0 +1 @@ +from . import create, delete, update # noqa diff --git a/openslides_backend/action/actions/gender/create.py b/openslides_backend/action/actions/gender/create.py new file mode 100644 index 0000000000..223e788b0d --- /dev/null +++ b/openslides_backend/action/actions/gender/create.py @@ -0,0 +1,28 @@ +from typing import Any + +from ....action.mixins.archived_meeting_check_mixin import CheckForArchivedMeetingMixin +from ....models.models import Gender +from ....permissions.management_levels import OrganizationManagementLevel +from ....shared.util import ONE_ORGANIZATION_ID +from ...generics.create import CreateAction +from ...util.default_schema import DefaultSchema +from ...util.register import register_action +from .mixins import GenderUniqueMixin + + +@register_action("gender.create") +class GenderCreate(CreateAction, CheckForArchivedMeetingMixin, GenderUniqueMixin): + """ + Action to create a gender. + """ + + model = Gender() + schema = DefaultSchema(Gender()).get_create_schema( + required_properties=["name"], + ) + permission = OrganizationManagementLevel.CAN_MANAGE_USERS + + def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: + super().update_instance(instance) + instance["organization_id"] = ONE_ORGANIZATION_ID + return instance diff --git a/openslides_backend/action/actions/gender/delete.py b/openslides_backend/action/actions/gender/delete.py new file mode 100644 index 0000000000..9c952b27af --- /dev/null +++ b/openslides_backend/action/actions/gender/delete.py @@ -0,0 +1,18 @@ +from ....models.models import Gender +from ....permissions.management_levels import OrganizationManagementLevel +from ...generics.delete import DeleteAction +from ...util.default_schema import DefaultSchema +from ...util.register import register_action +from .mixins import GenderPermissionMixin + + +@register_action("gender.delete") +class GenderDeleteAction(DeleteAction, GenderPermissionMixin): + """ + Action to delete a gender. + """ + + model = Gender() + schema = DefaultSchema(Gender()).get_delete_schema() + permission = OrganizationManagementLevel.CAN_MANAGE_USERS + skip_archived_meeting_check = True diff --git a/openslides_backend/action/actions/gender/mixins.py b/openslides_backend/action/actions/gender/mixins.py new file mode 100644 index 0000000000..d2cbf313ba --- /dev/null +++ b/openslides_backend/action/actions/gender/mixins.py @@ -0,0 +1,38 @@ +from typing import Any + +from datastore.shared.util import fqid_from_collection_and_id + +from openslides_backend.action.action import Action +from openslides_backend.shared.exceptions import ActionException, PermissionException + +from ...mixins.check_unique_name_mixin import CheckUniqueInContextMixin + + +class GenderUniqueMixin(CheckUniqueInContextMixin): + def validate_instance(self, instance: dict[str, Any]) -> None: + if instance.get("name") == "": + raise ActionException("Empty gender name not allowed.") + super().validate_instance(instance) + self.check_unique_in_context( + "name", + instance.get("name", ""), + "Gender '" + instance.get("name", "") + "' already exists.", + instance.get("id"), + ) + + +class GenderPermissionMixin(Action): + def check_permissions(self, instance: dict[str, Any]) -> None: + super().check_permissions(instance) + # default genders shall not be mutable + gender_id = instance.get("id", 0) + if 0 < gender_id < 5: + if gender_name := self.datastore.get( + fqid_from_collection_and_id("gender", gender_id), + ["name"], + lock_result=False, + ).get("name"): + msg = f"Cannot delete or update gender '{gender_name}' from default selection." + else: + msg = f"Cannot delete or update gender '{gender_id}' from default selection." + raise PermissionException(msg) diff --git a/openslides_backend/action/actions/gender/update.py b/openslides_backend/action/actions/gender/update.py new file mode 100644 index 0000000000..aa47a9124e --- /dev/null +++ b/openslides_backend/action/actions/gender/update.py @@ -0,0 +1,22 @@ +from ....action.mixins.archived_meeting_check_mixin import CheckForArchivedMeetingMixin +from ....models.models import Gender +from ....permissions.management_levels import OrganizationManagementLevel +from ...generics.update import UpdateAction +from ...util.default_schema import DefaultSchema +from ...util.register import register_action +from .mixins import GenderPermissionMixin, GenderUniqueMixin + + +@register_action("gender.update") +class GenderUpdateAction( + UpdateAction, CheckForArchivedMeetingMixin, GenderPermissionMixin, GenderUniqueMixin +): + """ + Action to update a gender. + """ + + model = Gender() + schema = DefaultSchema(Gender()).get_update_schema( + required_properties=["name"], + ) + permission = OrganizationManagementLevel.CAN_MANAGE_USERS diff --git a/openslides_backend/action/actions/meeting/clone.py b/openslides_backend/action/actions/meeting/clone.py index 8438cda138..27b4fae8a1 100644 --- a/openslides_backend/action/actions/meeting/clone.py +++ b/openslides_backend/action/actions/meeting/clone.py @@ -14,12 +14,13 @@ from openslides_backend.shared.interfaces.event import Event, EventType from openslides_backend.shared.patterns import fqid_from_collection_and_id from openslides_backend.shared.schema import id_list_schema, required_id_schema +from openslides_backend.shared.util import ONE_ORGANIZATION_ID from ....shared.export_helper import export_meeting from ...util.default_schema import DefaultSchema from ...util.register import register_action from ...util.typing import ActionData -from .import_ import ONE_ORGANIZATION_ID, MeetingImport +from .import_ import MeetingImport updatable_fields = [ "committee_id", diff --git a/openslides_backend/action/actions/meeting/import_.py b/openslides_backend/action/actions/meeting/import_.py index f7840d384d..1e50b6782a 100644 --- a/openslides_backend/action/actions/meeting/import_.py +++ b/openslides_backend/action/actions/meeting/import_.py @@ -33,10 +33,14 @@ fqid_from_collection_and_id, ) from openslides_backend.shared.schema import models_map_object -from openslides_backend.shared.util import ONE_ORGANIZATION_FQID from ....shared.interfaces.event import Event, ListFields, ListFieldsDict -from ....shared.util import ALLOWED_HTML_TAGS_STRICT, ONE_ORGANIZATION_ID, validate_html +from ....shared.util import ( + ALLOWED_HTML_TAGS_STRICT, + ONE_ORGANIZATION_FQID, + ONE_ORGANIZATION_ID, + validate_html, +) from ...action import RelationUpdates from ...mixins.singular_action_mixin import SingularActionMixin from ...util.crypto import get_random_password @@ -146,6 +150,7 @@ def prefetch(self, action_data: ActionData) -> None: def preprocess_data(self, instance: dict[str, Any]) -> dict[str, Any]: self.check_one_meeting(instance) self.check_locked(instance) + self.stash_gender_relations(instance) self.remove_not_allowed_fields(instance) self.set_committee_and_orga_relation(instance) instance = self.migrate_data(instance) @@ -160,6 +165,14 @@ def check_locked(self, instance: dict[str, Any]) -> None: if list(instance["meeting"]["meeting"].values())[0].get("locked_from_inside"): raise ActionException("Cannot import a locked meeting.") + def stash_gender_relations(self, instance: dict[str, Any]) -> None: + # The Checker will not allow the gender to have or not to have an orga relation. So we need to circumvent it. + self.user_id_to_gender = {} + users = instance["meeting"].get("user", {}) + for user in users.values(): + if gender := user.pop("gender", None): + self.user_id_to_gender[user["id"]] = gender + def remove_not_allowed_fields(self, instance: dict[str, Any]) -> None: json_data = instance["meeting"] @@ -245,6 +258,7 @@ def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: meeting_json = instance["meeting"] self.update_admin_group(meeting_json) self.upload_mediadata() + self.handle_gender_string(instance) return instance def empty_if_none(self, value: str | None) -> str: @@ -304,6 +318,44 @@ def check_usernames_and_generate_new_ones(self, json_data: dict[str, Any]) -> No for entry, username in zip(user_entries, new_usernames): entry["username"] = username + def handle_gender_string(self, instance: dict[str, Any]) -> None: + for user_id in list(self.user_id_to_gender.keys()): + if user_id in self.merge_user_map: + del self.user_id_to_gender[user_id] + genders = self.datastore.get_all("gender", ["id", "name"], lock_result=True) + gender_dict = {gender.get("name", ""): gender for gender in genders.values()} + new_genders = list( + {value for value in self.user_id_to_gender.values()}.difference(gender_dict) + ) + if new_genders: + new_genders.sort() # fix order for tests + new_gender_ids = self.datastore.reserve_ids("gender", len(new_genders)) + new_gender_dict = { + new_gender: { + "id": new_gender_id, + "name": new_gender, + "meta_new": True, + "organization_id": ONE_ORGANIZATION_ID, + } + for new_gender, new_gender_id in zip(new_genders, new_gender_ids) + } + gender_dict.update(new_gender_dict) + instance["meeting"]["gender"] = { + str(gender["id"]): gender for gender in gender_dict.values() + } + for user_id, gender in self.user_id_to_gender.items(): + gender_id = gender_dict[gender]["id"] + replace_user_id = self.replace_map["user"][user_id] + instance["meeting"]["user"][str(replace_user_id)]["gender_id"] = gender_id + try: + instance["meeting"]["gender"][str(gender_id)]["user_ids"].append( + replace_user_id + ) + except KeyError: + instance["meeting"]["gender"][str(gender_id)]["user_ids"] = [ + replace_user_id + ] + def check_limit_of_meetings( self, text: str = "import", text2: str = "active " ) -> None: @@ -549,11 +601,14 @@ def create_events( meeting_id = meeting["id"] events = [] update_events = [] + add_genders_to_organisation = [] for collection in json_data: for entry in json_data[collection].values(): fqid = fqid_from_collection_and_id(collection, entry["id"]) meta_new = entry.pop("meta_new", None) if meta_new: + if collection == "gender": + add_genders_to_organisation.append(entry["id"]) events.append( self.build_event( EventType.Create, @@ -561,20 +616,19 @@ def create_events( entry, ) ) - elif collection == "user": + elif collection in ["user", "gender"]: list_fields: ListFields = {"add": {}, "remove": {}} - fields: dict[str, Any] = {} for field, value in entry.items(): model_field = model_registry[collection]().try_get_field(field) if isinstance(model_field, RelationListField): list_fields["add"][field] = value - if fields or list_fields["add"]: + if list_fields["add"]: update_events.append( self.build_event( EventType.Update, fqid, - fields=fields if fields else None, - list_fields=list_fields if list_fields["add"] else None, + fields=None, + list_fields=list_fields, ) ) elif collection == "meeting_user": @@ -599,12 +653,14 @@ def create_events( ) ) - # add meetings to organization if set in meeting + # add meetings to organization if set in meeting, also genders if new created adder: ListFieldsDict = {} if meeting.get("is_active_in_organization_id"): adder["active_meeting_ids"] = [meeting_id] if meeting.get("template_for_organization_id"): adder["template_meeting_ids"] = [meeting_id] + if add_genders_to_organisation: + adder["gender_ids"] = add_genders_to_organisation if adder: events.append( diff --git a/openslides_backend/action/actions/organization/update.py b/openslides_backend/action/actions/organization/update.py index 31eb695776..0c67c74ecc 100644 --- a/openslides_backend/action/actions/organization/update.py +++ b/openslides_backend/action/actions/organization/update.py @@ -7,14 +7,13 @@ from ....permissions.management_levels import OrganizationManagementLevel from ....permissions.permission_helper import has_organization_management_level from ....shared.exceptions import ActionException, MissingPermission -from ....shared.filters import FilterOperator, Or +from ....shared.filters import FilterOperator from ....shared.schema import optional_str_schema from ...generics.update import UpdateAction from ...mixins.send_email_mixin import EmailCheckMixin, EmailSenderCheckMixin from ...util.default_schema import DefaultSchema from ...util.register import register_action from ..user.save_saml_account import allowed_user_fields -from ..user.update import UserUpdate @register_action("organization.update") @@ -37,7 +36,6 @@ class OrganizationUpdate( "users_email_replyto", "users_email_subject", "users_email_body", - "genders", "require_duplicate_from", ) @@ -123,25 +121,4 @@ def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: raise ActionException( f"Active users: {count_active_users}. You cannot set the limit lower." ) - if "genders" in instance: - organization = self.datastore.get(ONE_ORGANIZATION_FQID, ["genders"]) - removed_genders = [ - gender - for gender in organization.get("genders", []) - if gender not in instance["genders"] - ] - - if removed_genders: - filter__ = Or( - *[ - FilterOperator("gender", "=", gender) - for gender in removed_genders - ] - ) - users = self.datastore.filter("user", filter__, ["id"]).values() - payload_remove_gender = [ - {"id": entry["id"], "gender": None} for entry in users - ] - if payload_remove_gender: - self.execute_other_action(UserUpdate, payload_remove_gender) return instance diff --git a/openslides_backend/action/actions/user/base_import.py b/openslides_backend/action/actions/user/base_import.py index c053bf3b2a..00eef6c5e1 100644 --- a/openslides_backend/action/actions/user/base_import.py +++ b/openslides_backend/action/actions/user/base_import.py @@ -70,14 +70,12 @@ def handle_remove_and_group_fields(self, entry: dict[str, Any]) -> dict[str, Any ) if field_data or entry.get(field): entry[field] = value - - if ( - isinstance(entry.get("gender"), dict) - and entry["gender"].get("info") == ImportState.WARNING - ): + if isinstance(entry.get("gender"), dict): + if entry["gender"].get("info") != ImportState.WARNING: + entry["gender_id"] = entry["gender"]["id"] entry.pop("gender") - # remove all fields fields marked with "remove"-state + # remove all fields marked with "remove"-state to_remove = [] for k, v in entry.items(): if isinstance(v, dict): diff --git a/openslides_backend/action/actions/user/base_json_upload.py b/openslides_backend/action/actions/user/base_json_upload.py index 3df2781b9c..a4af743b3b 100644 --- a/openslides_backend/action/actions/user/base_json_upload.py +++ b/openslides_backend/action/actions/user/base_json_upload.py @@ -2,7 +2,6 @@ from typing import Any, cast from ....models.models import User -from ....shared.exceptions import ActionException from ....shared.patterns import fqid_from_collection_and_id from ...mixins.import_mixins import ( BaseJsonUploadAction, @@ -14,7 +13,7 @@ from ...mixins.send_email_mixin import EmailUtils from ...util.crypto import get_random_password from ...util.default_schema import DefaultSchema -from .user_mixins import UsernameMixin, check_gender_helper +from .user_mixins import UsernameMixin class BaseUserJsonUpload(UsernameMixin, BaseJsonUploadAction): @@ -61,7 +60,6 @@ def get_schema( "email", "title", "pronoun", - "gender", "default_password", "is_active", "is_physical_person", @@ -69,6 +67,7 @@ def get_schema( "member_number", ), **additional_user_fields, + "gender": {"type": "string"}, }, "additionalProperties": False, }, @@ -355,10 +354,20 @@ def validate_entry(self, entry: dict[str, Any]) -> dict[str, Any]: self.handle_default_password(entry) if gender := entry.get("gender"): - try: - check_gender_helper(self.datastore, entry) - entry["gender"] = {"info": ImportState.DONE, "value": gender} - except ActionException: + if gender_model := next( + ( + model + for model in self.gender_dict.values() + if model["name"] == gender + ), + None, + ): + entry["gender"] = { + "info": ImportState.DONE, + "value": gender, + "id": gender_model["id"], + } + else: entry["gender"] = {"info": ImportState.WARNING, "value": gender} messages.append(f"Gender '{gender}' is not in the allowed gender list.") @@ -489,6 +498,9 @@ def setup_lookups(self, data: list[dict[str, Any]]) -> None: field="member_number", mapped_fields=["username", "member_number", "saml_id"], ) + self.gender_dict = self.datastore.get_all( + "gender", ["id", "name"], lock_result=False + ) self.all_id_mapping: dict[int, list[SearchFieldType]] = defaultdict(list) for lookup in ( diff --git a/openslides_backend/action/actions/user/create.py b/openslides_backend/action/actions/user/create.py index 090870035e..49494ec5f1 100644 --- a/openslides_backend/action/actions/user/create.py +++ b/openslides_backend/action/actions/user/create.py @@ -17,7 +17,7 @@ from ..meeting_user.mixin import CheckLockOutPermissionMixin from .create_update_permissions_mixin import CreateUpdatePermissionsMixin from .password_mixins import SetPasswordMixin -from .user_mixins import LimitOfUserMixin, UserMixin, UsernameMixin, check_gender_helper +from .user_mixins import LimitOfUserMixin, UserMixin, UsernameMixin, check_gender_exists @register_action("user.create") @@ -46,7 +46,7 @@ class UserCreate( "is_physical_person", "default_password", "can_change_own_password", - "gender", + "gender_id", "email", "default_vote_weight", "organization_management_level", @@ -96,7 +96,7 @@ def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: instance["default_password"] = get_random_password() self.reset_password(instance) instance["organization_id"] = ONE_ORGANIZATION_ID - check_gender_helper(self.datastore, instance) + check_gender_exists(self.datastore, instance) return instance def create_action_result_element( diff --git a/openslides_backend/action/actions/user/create_update_permissions_mixin.py b/openslides_backend/action/actions/user/create_update_permissions_mixin.py index ee5270ab9b..4a64e86a96 100644 --- a/openslides_backend/action/actions/user/create_update_permissions_mixin.py +++ b/openslides_backend/action/actions/user/create_update_permissions_mixin.py @@ -184,7 +184,7 @@ class CreateUpdatePermissionsMixin(UserMixin, UserScopeMixin, Action): "is_active", "is_physical_person", "can_change_own_password", - "gender", + "gender_id", "pronoun", "email", "default_vote_weight", diff --git a/openslides_backend/action/actions/user/merge_together.py b/openslides_backend/action/actions/user/merge_together.py index 4e817caa68..2c854305d3 100644 --- a/openslides_backend/action/actions/user/merge_together.py +++ b/openslides_backend/action/actions/user/merge_together.py @@ -57,7 +57,7 @@ class UserMergeTogether( "is_active", "is_physical_person", "default_password", - "gender", + "gender_id", "email", "default_vote_weight", "pronoun", @@ -90,7 +90,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: "title", "first_name", "last_name", - "gender", + "gender_id", "email", "default_vote_weight", ], diff --git a/openslides_backend/action/actions/user/participant_import.py b/openslides_backend/action/actions/user/participant_import.py index a99b9e1af6..fb4e1c1fc3 100644 --- a/openslides_backend/action/actions/user/participant_import.py +++ b/openslides_backend/action/actions/user/participant_import.py @@ -118,6 +118,12 @@ def validate_entry(self, row: ImportRow) -> None: super().validate_entry(row) entry = row["data"] entry["meeting_id"] = self.meeting_id + + if isinstance(entry.get("gender"), dict): + if entry["gender"].get("info") != ImportState.WARNING: + entry["gender_id"] = entry["gender"]["id"] + entry.pop("gender") + if "groups" not in entry: raise ActionException( f"There is no group in the data of user '{self.get_value_from_union_str_object(entry.get('username'))}'. Is there a default group for the meeting?" diff --git a/openslides_backend/action/actions/user/save_saml_account.py b/openslides_backend/action/actions/user/save_saml_account.py index c41156b20d..1989cde148 100644 --- a/openslides_backend/action/actions/user/save_saml_account.py +++ b/openslides_backend/action/actions/user/save_saml_account.py @@ -17,6 +17,7 @@ from ...util.action_type import ActionType from ...util.register import register_action from ...util.typing import ActionData, ActionResultElement +from ..gender.create import GenderCreate from .create import UserCreate from .update import UserUpdate from .user_mixins import UsernameMixin @@ -83,11 +84,26 @@ def validate_instance(self, instance: dict[str, Any]) -> None: ] } for model_field, payload_field in self.saml_attr_mapping.items() - if model_field in allowed_user_fields + # handle only allowed fields. handle gender separately since it needs conversion to id + if model_field in allowed_user_fields and model_field != "gender" }, "required": [self.saml_attr_mapping["saml_id"]], "additionalProperties": True, } + self.schema["properties"].update( + { + "gender": { + "oneOf": [ + {"type": ["string", "null"], "maxLength": 256}, + { + "type": "array", + "items": {"type": ["string", "null"], "maxLength": 256}, + "minItems": 0, + }, + ] + }, + } + ) try: fastjsonschema.validate(self.schema, instance) except fastjsonschema.JsonSchemaException as exception: @@ -126,8 +142,30 @@ def base_update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: users = self.datastore.filter( "user", FilterOperator("saml_id", "=", instance["saml_id"]), - ["id", *allowed_user_fields], + ["id", "gender_id", *allowed_user_fields], ) + + if gender := instance.get("gender"): + if gender == "": + instance["gender_id"] = None + else: + gender_dict = self.datastore.filter( + "gender", + FilterOperator("name", "=", gender), + ["id"], + ) + if gender_dict: + gender_id = next(iter(gender_dict.keys())) + else: + action_result = self.execute_other_action( + GenderCreate, [{"name": gender}] + ) + gender_id = action_result[0].get("id", 0) # type: ignore + instance["gender_id"] = gender_id + del instance["gender"] + elif gender == "": + instance["gender_id"] = None + del instance["gender"] if len(users) == 1: self.user = next(iter(users.values())) instance["id"] = (user_id := cast(int, self.user["id"])) @@ -143,7 +181,6 @@ def base_update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: else: instance["meeting_id"] = meeting_id instance["group_ids"] = [group_id] - instance = { k: v for k, v in instance.items() if k == "id" or v != self.user.get(k) } diff --git a/openslides_backend/action/actions/user/update.py b/openslides_backend/action/actions/user/update.py index 0af07e83ac..a5df918baa 100644 --- a/openslides_backend/action/actions/user/update.py +++ b/openslides_backend/action/actions/user/update.py @@ -23,7 +23,7 @@ LimitOfUserMixin, UpdateHistoryMixin, UserMixin, - check_gender_helper, + check_gender_exists, ) @@ -63,7 +63,7 @@ class UserUpdate( "is_physical_person", "default_password", "can_change_own_password", - "gender", + "gender_id", "email", "default_vote_weight", "organization_management_level", @@ -140,7 +140,7 @@ def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: if instance.get("is_active") and not user.get("is_active"): self.check_limit_of_user(1) - check_gender_helper(self.datastore, instance) + check_gender_exists(self.datastore, instance) return instance @original_instances diff --git a/openslides_backend/action/actions/user/update_self.py b/openslides_backend/action/actions/user/update_self.py index 19bfc105b5..037386abfb 100644 --- a/openslides_backend/action/actions/user/update_self.py +++ b/openslides_backend/action/actions/user/update_self.py @@ -5,7 +5,7 @@ from ...mixins.send_email_mixin import EmailCheckMixin from ...util.default_schema import DefaultSchema from ...util.register import register_action -from .user_mixins import UpdateHistoryMixin, UserMixin, check_gender_helper +from .user_mixins import UpdateHistoryMixin, UserMixin, check_gender_exists @register_action("user.update_self") @@ -16,7 +16,7 @@ class UserUpdateSelf(EmailCheckMixin, UpdateAction, UserMixin, UpdateHistoryMixi model = User() schema = DefaultSchema(User()).get_default_schema( - optional_properties=["username", "pronoun", "gender", "email"] + optional_properties=["username", "pronoun", "gender_id", "email"] ) check_email_field = "email" @@ -26,7 +26,7 @@ def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: """ instance["id"] = self.user_id instance = super().update_instance(instance) - check_gender_helper(self.datastore, instance) + check_gender_exists(self.datastore, instance) return instance def check_permissions(self, instance: dict[str, Any]) -> None: diff --git a/openslides_backend/action/actions/user/user_mixins.py b/openslides_backend/action/actions/user/user_mixins.py index 4b8cfa7c36..f2b3c836d5 100644 --- a/openslides_backend/action/actions/user/user_mixins.py +++ b/openslides_backend/action/actions/user/user_mixins.py @@ -272,14 +272,17 @@ def has_multiple_search_data(self, payload_index: int) -> list[str]: return [] -def check_gender_helper(datastore: DatastoreService, instance: dict[str, Any]) -> None: - if instance.get("gender"): - organization = datastore.get(ONE_ORGANIZATION_FQID, ["genders"]) - if organization.get("genders"): - if not instance["gender"] in organization["genders"]: - raise ActionException( - f"Gender '{instance['gender']}' is not in the allowed gender list." - ) +def check_gender_exists(datastore: DatastoreService, instance: dict[str, Any]) -> None: + """raises ActionException if the gender is non existant""" + if gender_id := instance.get("gender_id"): + if not datastore.get( + fqid_from_collection_and_id("gender", gender_id), + ["id", "name"], + lock_result=False, + ): + raise ActionException( + f"GenderId '{gender_id}' is not in the allowed gender list." + ) class AdminIntegrityCheckMixin(Action): diff --git a/openslides_backend/migrations/migration_handler.py b/openslides_backend/migrations/migration_handler.py index b6e050dfc0..2a5c7985cf 100644 --- a/openslides_backend/migrations/migration_handler.py +++ b/openslides_backend/migrations/migration_handler.py @@ -13,7 +13,7 @@ from . import MigrationWrapper # Amount of time that should be waited for a result from the migrate thread before returning an empty result -THREAD_WAIT_TIME = 0.1 +THREAD_WAIT_TIME = 0.14 class MigrationState(str, Enum): diff --git a/openslides_backend/migrations/migrations/0057_gender_model.py b/openslides_backend/migrations/migrations/0057_gender_model.py new file mode 100644 index 0000000000..30782f09a2 --- /dev/null +++ b/openslides_backend/migrations/migrations/0057_gender_model.py @@ -0,0 +1,74 @@ +from collections import defaultdict + +from datastore.migrations import BaseModelMigration +from datastore.shared.util import fqid_from_collection_and_id +from datastore.writer.core import ( + BaseRequestEvent, + RequestCreateEvent, + RequestUpdateEvent, +) + + +class Migration(BaseModelMigration): + """ + This migration introduces the new gender model which enables custom gender names for non default genders. + This requires to replace all gender strings in organization and user models to be replaced with the corresponding gender id. + If the migration runs in memory then all gender information is left untouched since the import will still handle it as a string. + """ + + target_migration_index = 58 + + def migrate_models(self) -> list[BaseRequestEvent] | None: + events: list[BaseRequestEvent] = [] + if not self.reader.is_in_memory_migration: + users = self.reader.get_all("user", ["gender"]) + default_genders = ["male", "female", "diverse", "non-binary"] + gender_strings = self.reader.get("organization/1", ["genders"]).get( + "genders", "" + ) + gender_strings = default_genders + [ + gender for gender in gender_strings if gender not in default_genders + ] + # update organization + events.append( + RequestUpdateEvent( + fqid_from_collection_and_id("organization", 1), + { + "gender_ids": [n + 1 for n in range(len(gender_strings))], + "genders": None, + }, + ) + ) + userids_for_gender = defaultdict(list) + # update users + for user_id, user in users.items(): + user_gender_string = user.get("gender", "") + if user_gender_string in gender_strings: + gender_id = gender_strings.index(user_gender_string) + 1 + events.append( + RequestUpdateEvent( + fqid_from_collection_and_id("user", user_id), + {"gender_id": gender_id, "gender": None}, + ) + ) + userids_for_gender[gender_id].append(user_id) + else: + events.append( + RequestUpdateEvent( + fqid_from_collection_and_id("user", user_id), + {"gender": None}, + ) + ) + for gender_id, gender in enumerate(gender_strings, start=1): + events.append( + RequestCreateEvent( + fqid_from_collection_and_id("gender", gender_id), + { + "id": gender_id, + "name": gender, + "organization_id": 1, + "user_ids": userids_for_gender.get(gender_id), + }, + ) + ) + return events diff --git a/openslides_backend/models/models.py b/openslides_backend/models/models.py index c9a5db379b..f7ca9200a8 100644 --- a/openslides_backend/models/models.py +++ b/openslides_backend/models/models.py @@ -16,7 +16,7 @@ class Organization(Model): privacy_policy = fields.TextField() login_text = fields.TextField() reset_password_verbose_errors = fields.BooleanField() - genders = fields.CharArrayField(default=["male", "female", "diverse", "non-binary"]) + gender_ids = fields.RelationListField(to={"gender": "organization_id"}) enable_electronic_voting = fields.BooleanField() enable_chat = fields.BooleanField() limit_of_meetings = fields.IntegerField( @@ -98,7 +98,6 @@ class User(Model): password = fields.CharField() default_password = fields.CharField() can_change_own_password = fields.BooleanField(default=True) - gender = fields.CharField() email = fields.CharField() default_vote_weight = fields.DecimalField( default="1.000000", constraints={"minimum": "0.000001"} @@ -106,6 +105,7 @@ class User(Model): last_email_sent = fields.TimestampField() is_demo_user = fields.BooleanField() last_login = fields.TimestampField(read_only=True) + gender_id = fields.RelationField(to={"gender": "user_ids"}) organization_management_level = fields.CharField( constraints={ "description": "Hierarchical permission level for the whole organization.", @@ -202,6 +202,18 @@ class MeetingUser(Model): ) +class Gender(Model): + collection = "gender" + verbose_name = "gender" + + id = fields.IntegerField(constant=True) + name = fields.CharField(required=True, constraints={"description": "unique"}) + organization_id = fields.OrganizationField( + to={"organization": "gender_ids"}, required=True + ) + user_ids = fields.RelationListField(to={"user": "gender_id"}) + + class OrganizationTag(Model): collection = "organization_tag" verbose_name = "organization tag" diff --git a/openslides_backend/shared/export_helper.py b/openslides_backend/shared/export_helper.py index 237b89443a..5c05bec769 100644 --- a/openslides_backend/shared/export_helper.py +++ b/openslides_backend/shared/export_helper.py @@ -157,11 +157,15 @@ def add_users( ) for user in users.values(): + gender_dict = datastore.get_all("gender", ["name"], lock_result=False) user["meeting_ids"] = [meeting_id] if meeting_id in (user.get("is_present_in_meeting_ids") or []): user["is_present_in_meeting_ids"] = [meeting_id] else: user["is_present_in_meeting_ids"] = None + if user.get("gender_id"): + user["gender"] = gender_dict.get(user["gender_id"], {}).get("name") + del user["gender_id"] # limit user fields to exported objects collection_field_tupels = [ ("meeting_user", "meeting_user_ids"), diff --git a/tests/system/action/gender/__init__.py b/tests/system/action/gender/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/system/action/gender/test_create.py b/tests/system/action/gender/test_create.py new file mode 100644 index 0000000000..ea027a7a50 --- /dev/null +++ b/tests/system/action/gender/test_create.py @@ -0,0 +1,117 @@ +from openslides_backend.permissions.management_levels import OrganizationManagementLevel +from openslides_backend.shared.util import ONE_ORGANIZATION_FQID +from tests.system.action.base import BaseActionTestCase + + +class GenderCreateActionTest(BaseActionTestCase): + + def test_create(self) -> None: + self.set_models( + { + ONE_ORGANIZATION_FQID: { + "name": "test_organization1", + "gender_ids": [1], + }, + "user/20": {"username": "test_user20"}, + "gender/1": {"organization_id": 1, "name": "male"}, + } + ) + gender_name = "female" + + response = self.request( + "gender.create", + { + "name": gender_name, + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "gender/1", + { + "name": "male", + "organization_id": 1, + }, + ) + self.assert_model_exists( + "gender/2", + { + "name": gender_name, + "organization_id": 1, + }, + ) + self.assert_model_exists(ONE_ORGANIZATION_FQID, {"gender_ids": [1, 2]}) + + def test_create_wrong_field(self) -> None: + response = self.request( + "gender.create", + { + "name": "test_gender_name", + "wrong_field": "test", + }, + ) + + self.assert_status_code(response, 400) + self.assertIn( + "data must not contain {'wrong_field'} properties", + response.json["message"], + ) + self.assert_model_not_exists("gender/1") + + def test_create_existing_field(self) -> None: + self.set_models({"gender/1": {"name": "exists"}}) + response = self.request( + "gender.create", + { + "name": "exists", + }, + ) + + self.assert_status_code(response, 400) + self.assertIn( + "Gender 'exists' already exists.", + response.json["message"], + ) + self.assert_model_exists("gender/1") + self.assert_model_not_exists("gender/2") + + def test_create_empty_data(self) -> None: + response = self.request("gender.create", {}) + self.assert_status_code(response, 400) + self.assertIn( + "data must contain ['name'] properties", + response.json["message"], + ) + self.assert_model_not_exists("gender/1") + + def test_create_empty_data_list(self) -> None: + response = self.request_multi("gender.create", []) + self.assert_status_code(response, 200) + self.assert_model_not_exists("gender/1") + + def test_create_empty_name(self) -> None: + response = self.request("gender.create", {"name": ""}) + self.assert_status_code(response, 400) + self.assertIn( + "Empty gender name not allowed.", + response.json["message"], + ) + self.assert_model_not_exists("gender/1") + + def test_permission(self) -> None: + self.base_permission_test( + {}, + "gender.create", + { + "name": "test_Xcdfgee", + }, + OrganizationManagementLevel.CAN_MANAGE_USERS, + ) + + def test_no_permission(self) -> None: + self.base_permission_test( + {}, + "gender.create", + { + "name": "test_Xcdfghee", + }, + ) diff --git a/tests/system/action/gender/test_delete.py b/tests/system/action/gender/test_delete.py new file mode 100644 index 0000000000..152786043f --- /dev/null +++ b/tests/system/action/gender/test_delete.py @@ -0,0 +1,97 @@ +from openslides_backend.shared.util import ONE_ORGANIZATION_FQID +from tests.system.action.base import BaseActionTestCase + + +class GenderDeleteActionTest(BaseActionTestCase): + gender_id = 5 + gender_fqid = f"gender/{gender_id}" + gender_name = "fairy" + + def create_data(self) -> None: + self.set_models( + { + ONE_ORGANIZATION_FQID: {"gender_ids": [self.gender_id]}, + "user/20": {"gender_id": 1}, + "user/21": {"gender_id": self.gender_id}, + "gender/1": { + "id": 1, + "name": "male", + "organization_id": 1, + "user_ids": [20], + }, + self.gender_fqid: { + "id": self.gender_id, + "name": self.gender_name, + "organization_id": 1, + "user_ids": [21], + }, + } + ) + + def test_delete_correctly(self) -> None: + self.create_data() + self.set_models( + { + "gender/6": { + "name": "dragon", + "organization_id": 1, + }, + ONE_ORGANIZATION_FQID: {"gender_ids": [1, self.gender_id, 6]}, + } + ) + response = self.request("gender.delete", {"id": self.gender_id}) + + self.assert_status_code(response, 200) + gender1 = self.assert_model_deleted( + self.gender_fqid, + {"organization_id": 1, "name": self.gender_name}, + ) + self.assertCountEqual(gender1["user_ids"], [21]) + + self.assert_model_exists("user/20", {"gender_id": 1}) + self.assert_model_exists("user/21", {"gender_id": None}) + organization1 = self.get_model(ONE_ORGANIZATION_FQID) + self.assertCountEqual(organization1["gender_ids"], [1, 6]) + self.assert_model_exists("gender/1", {"name": "male"}) + self.assert_model_exists("gender/6", {"name": "dragon"}) + + def test_delete_wrong_id(self) -> None: + self.create_data() + response = self.request("gender.delete", {"id": 6}) + self.assert_status_code(response, 400) + self.assertIn("Model 'gender/6' does not exist.", response.json["message"]) + self.assert_model_exists(self.gender_fqid) + + def test_delete_default_gender(self) -> None: + self.create_data() + + response = self.request("gender.delete", {"id": 1}) + + self.assert_status_code(response, 400) + assert ( + "Cannot delete or update gender 'male' from default selection." + in response.json["message"] + ) + self.assert_model_exists("gender/1", {"name": "male"}) + + def test_delete_no_permission(self) -> None: + self.create_data() + self.set_models({"user/1": {"organization_management_level": None}}) + + response = self.request("gender.delete", {"id": self.gender_id}) + self.assert_status_code(response, 403) + assert ( + "Missing OrganizationManagementLevel: can_manage_users" + in response.json["message"] + ) + self.assert_model_exists(self.gender_fqid) + + def test_delete_permission(self) -> None: + self.create_data() + self.set_models( + {"user/1": {"organization_management_level": "can_manage_users"}} + ) + + response = self.request("gender.delete", {"id": self.gender_id}) + self.assert_status_code(response, 200) + self.assert_model_deleted(self.gender_fqid) diff --git a/tests/system/action/gender/test_update.py b/tests/system/action/gender/test_update.py new file mode 100644 index 0000000000..37daf4aac2 --- /dev/null +++ b/tests/system/action/gender/test_update.py @@ -0,0 +1,121 @@ +from openslides_backend.shared.util import ONE_ORGANIZATION_FQID +from tests.system.action.base import BaseActionTestCase + + +class GenderUpdateActionTest(BaseActionTestCase): + gender_id = 5 + gender_fqid = f"gender/{gender_id}" + gender_name = "dragon" + + def create_data(self) -> None: + self.set_models( + { + ONE_ORGANIZATION_FQID: {"name": "test_organization1"}, + self.gender_fqid: { + "name": self.gender_name, + "organization_id": 1, + }, + "gender/2": {"name": "female", "organization_id": 1}, + "user/20": {"username": "test_user20", "gender_id": self.gender_id}, + "user/21": {"username": "test_user21"}, + } + ) + + def test_update_correctly(self) -> None: + self.create_data() + new_name = "gender_testname_updated" + response = self.request( + "gender.update", {"id": self.gender_id, "name": new_name} + ) + self.assert_status_code(response, 200) + model = self.get_model(self.gender_fqid) + self.assertEqual(model.get("name"), new_name) + + def test_update_empty_name(self) -> None: + self.create_data() + new_name = "" + response = self.request( + "gender.update", {"id": self.gender_id, "name": new_name} + ) + self.assert_status_code(response, 400) + self.assertIn("Empty gender name not allowed.", response.json["message"]) + model = self.get_model(self.gender_fqid) + self.assertEqual(model.get("name"), self.gender_name) + + def test_update_empty(self) -> None: + self.create_data() + response = self.request("gender.update", {}) + self.assert_status_code(response, 400) + self.assertIn( + "data must contain ['id', 'name'] properties", response.json["message"] + ) + model = self.get_model(self.gender_fqid) + self.assertEqual(model.get("name"), self.gender_name) + + def test_update_empty_list(self) -> None: + self.create_data() + response = self.request_multi("gender.update", []) + self.assert_status_code(response, 200) + self.assert_model_not_exists("gender/1") + self.assert_model_not_exists("gender/3") + self.assert_model_not_exists("gender/4") + self.assert_model_not_exists("gender/6") + + def test_update_default_gender(self) -> None: + self.create_data() + response = self.request( + "gender.update", + { + "id": 2, + "name": "so wrong to change this gender", + }, + ) + self.assert_status_code(response, 400) + self.assertIn( + "Cannot delete or update gender 'female' from default selection.", + response.json["message"], + ) + + def test_update_wrong_id(self) -> None: + self.create_data() + response = self.request("gender.update", {"id": 200, "name": "xxxxx"}) + self.assert_status_code(response, 400) + model = self.get_model(self.gender_fqid) + self.assertEqual(model.get("name"), self.gender_name) + + def test_update_wrong_field(self) -> None: + self.create_data() + response = self.request("gender.update", {"id": 5, "Mercedes": "xxxxx"}) + self.assert_status_code(response, 400) + model = self.get_model(self.gender_fqid) + self.assertEqual(model.get("name"), self.gender_name) + + def test_update_no_permission(self) -> None: + self.create_data() + self.set_models({"user/1": {"organization_management_level": None}}) + + response = self.request( + "gender.update", {"id": self.gender_id, "name": "testy"} + ) + self.assert_status_code(response, 403) + assert ( + "Missing OrganizationManagementLevel: can_manage_users" + in response.json["message"] + ) + self.assert_model_exists( + self.gender_fqid, + {"id": self.gender_id, "name": self.gender_name}, + ) + + def test_update_permission(self) -> None: + self.create_data() + self.set_models( + {"user/1": {"organization_management_level": "can_manage_users"}} + ) + response = self.request( + "gender.update", {"id": self.gender_id, "name": "testy"} + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + self.gender_fqid, {"id": self.gender_id, "name": "testy"} + ) diff --git a/tests/system/action/meeting/test_import.py b/tests/system/action/meeting/test_import.py index 84c6db7938..36afbcd07e 100644 --- a/tests/system/action/meeting/test_import.py +++ b/tests/system/action/meeting/test_import.py @@ -26,6 +26,7 @@ def setUp(self) -> None: ONE_ORGANIZATION_FQID: { "active_meeting_ids": [1], "committee_ids": [1], + "gender_ids": [1, 4], }, "committee/1": {"organization_id": 1, "meeting_ids": [1]}, "meeting/1": { @@ -40,6 +41,8 @@ def setUp(self) -> None: "sequential_number": 26, "number_value": 31, }, + "gender/1": {"name": "male", "organization_id": 1}, + "gender/4": {"name": "diverse", "organization_id": 1}, } ) @@ -354,7 +357,6 @@ def get_user_data(self, obj_id: int, data: dict[str, Any] = {}) -> dict[str, Any "is_physical_person": True, "default_password": "admin", "can_change_own_password": True, - "gender": "male", "email": "", "default_vote_weight": "1.000000", "last_email_sent": None, @@ -2136,7 +2138,10 @@ def test_check_missing_admin_group_in_meeting(self) -> None: ) def test_with_listfields_from_migration(self) -> None: - """test for listFields in event.data after migration. Uses migration 0035 to create one""" + """ + Test for listFields in event.data after migration. Uses migration 0035 to create one + Additionally adds a gender to user 1 to show that migration 0057 does not interfere with the import. + """ data = self.create_request_data( { "motion": { @@ -2181,6 +2186,7 @@ def test_with_listfields_from_migration(self) -> None: data["meeting"]["meeting"]["1"]["motion_ids"] = [5, 6] data["meeting"]["meeting"]["1"]["list_of_speakers_ids"] = [1, 2] data["meeting"]["motion_state"]["1"]["motion_ids"] = [5, 6] + data["meeting"]["user"]["1"]["gender"] = "male" data["meeting"]["_migration_index"] = 35 assert ( data["meeting"]["motion"]["5"]["referenced_in_motion_state_extension_ids"] @@ -2236,7 +2242,6 @@ def test_all_migrations(self) -> None: "assignment_poll_default_100_percent_base" ] = "YN" data["meeting"]["meeting"]["1"]["poll_default_100_percent_base"] = "YNA" - with CountDatastoreCalls(verbose=True) as counter: response = self.request("meeting.import", data) self.assert_status_code(response, 200) @@ -2389,6 +2394,143 @@ def test_import_new_user_with_vote(self) -> None: }, ) + def test_gender_import(self) -> None: + """ + Each user represents different cases. The user number belongs to the request data, NOT the pre-existing-users. + User 1 shows that a new user will be created with gender_id (also with a new gender). + User 2 shows that a new user will be created with gender_id, but the same like User 1. + User 3 shows that the gender is not being updated on existing user with id=2 (same name etc.). + User 4 shows that a new user with empty gender can be created. + User 5 shows that a new user with a second new gender will be created. + User 6 shows that a new user will be added to an existing gender with filled user_ids. + User 7 shows the same as User 3 but with a new gender which should not be created. + """ + data = self.create_request_data({}) + self.update_model(ONE_ORGANIZATION_FQID, {"user_ids": [1, 2]}) + self.update_model("gender/4", {"user_ids": [2]}) + self.set_models( + { + "user/2": { + "username": "other_user", + "first_name": "other", + "last_name": "user", + "email": "other@us.er", + "organization_id": 1, + "gender_id": 4, + } + } + ) + data["meeting"]["user"]["1"]["gender"] = "needs_to_be_created" + other_users_request_data = { + "2": { + "id": 2, + "username": "newer_user", + "first_name": "newer", + "last_name": "user", + "gender": "needs_to_be_created", + "organization_id": 1, + }, + "3": { + "id": 3, + "username": "other_user", + "first_name": "other", + "last_name": "user", + "email": "other@us.er", + "gender": "male", + "organization_id": 1, + }, + "4": { + "id": 4, + "username": "new_user", + "first_name": "new", + "last_name": "user", + "gender": "", + "organization_id": 1, + }, + "5": { + "id": 5, + "username": "newest_user", + "first_name": "newest", + "last_name": "user", + "gender": "needs_to_be_created_too", + "organization_id": 1, + }, + "6": { + "id": 6, + "username": "ultra_newest_user", + "first_name": "ultra newest", + "last_name": "user", + "gender": "diverse", + "organization_id": 1, + }, + "7": { + "id": 7, + "username": "other_user", + "first_name": "other", + "last_name": "user", + "email": "other@us.er", + "gender": "not_to_be_created", + "organization_id": 1, + }, + } + data["meeting"]["user"].update(other_users_request_data) + response = self.request("meeting.import", data) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/2", + { + "username": "other_user", + "gender": None, + "gender_id": 4, + }, + ) + self.assert_model_exists( + "user/3", {"username": "test", "gender": None, "gender_id": 5} + ) + self.assert_model_exists( + "user/4", + { + "username": "newer_user", + "gender": None, + "gender_id": 5, + }, + ) + self.assert_model_exists( + "user/5", + { + "username": "new_user", + "gender": None, + "gender_id": None, + }, + ) + self.assert_model_exists( + "user/6", + { + "username": "newest_user", + "gender": None, + "gender_id": 6, + }, + ) + self.assert_model_exists( + "user/7", + { + "username": "ultra_newest_user", + "gender": None, + "gender_id": 4, + }, + ) + self.assert_model_exists("gender/4", {"name": "diverse", "user_ids": [2, 7]}) + self.assert_model_exists( + "gender/5", {"name": "needs_to_be_created", "user_ids": [3, 4]} + ) + self.assert_model_exists( + "gender/6", {"name": "needs_to_be_created_too", "user_ids": [6]} + ) + self.assert_model_exists( + "organization/1", + {"user_ids": [1, 2, 3, 4, 5, 6, 7], "gender_ids": [1, 4, 5, 6]}, + ) + def test_import_existing_user_with_vote(self) -> None: self.set_models( { diff --git a/tests/system/action/organization/test_update.py b/tests/system/action/organization/test_update.py index d23bbcd2a9..4d5d866614 100644 --- a/tests/system/action/organization/test_update.py +++ b/tests/system/action/organization/test_update.py @@ -145,7 +145,6 @@ def test_update_some_more_fields(self) -> None: """ ), "saml_private_key": "private key dependency", - "genders": ["male", "female", "rabbit"], }, ) self.assert_status_code(response, 200) @@ -171,7 +170,6 @@ def test_update_some_more_fields(self) -> None: "saml_login_button_text": "Text for SAML login button", "saml_attr_mapping": self.saml_attr_mapping, "saml_private_key": "private key dependency", - "genders": ["male", "female", "rabbit"], }, ) assert ( @@ -317,40 +315,6 @@ def test_update_default_language(self) -> None: self.assert_status_code(response, 200) self.assert_model_exists(ONE_ORGANIZATION_FQID, {"default_language": "it"}) - def test_update_genders_remove(self) -> None: - self.set_models( - { - "organization/1": {"genders": ["male", "female", "test"]}, - "user/6": {"username": "with_test_gender", "gender": "test"}, - "user/7": {"username": "not_changed", "gender": "male"}, - } - ) - response = self.request( - "organization.update", {"id": 1, "genders": ["male", "female"]} - ) - self.assert_status_code(response, 200) - self.assert_model_exists(ONE_ORGANIZATION_FQID, {"genders": ["male", "female"]}) - self.assert_model_exists( - "user/6", {"username": "with_test_gender", "gender": None} - ) - self.assert_model_exists( - "user/7", {"username": "not_changed", "gender": "male"} - ) - - def test_update_genders_empty(self) -> None: - self.set_models( - { - "organization/1": {"genders": ["male", "female", "test"]}, - "user/6": {"gender": "test"}, - "user/7": {"gender": "male"}, - } - ) - response = self.request("organization.update", {"id": 1, "genders": []}) - self.assert_status_code(response, 200) - self.assert_model_exists(ONE_ORGANIZATION_FQID, {"genders": []}) - self.assert_model_exists("user/6", {"gender": None}) - self.assert_model_exists("user/7", {"gender": None}) - def test_update_group_a_no_permissions(self) -> None: self.set_organization_management_level( OrganizationManagementLevel.CAN_MANAGE_USERS diff --git a/tests/system/action/user/test_account_import.py b/tests/system/action/user/test_account_import.py index 675cdd1ca7..d035479e8a 100644 --- a/tests/system/action/user/test_account_import.py +++ b/tests/system/action/user/test_account_import.py @@ -12,9 +12,11 @@ def setUp(self) -> None: super().setUp() self.set_models( { - "organization/1": { - "genders": ["male", "female", "diverse", "non-binary"] - }, + "organization/1": {"gender_ids": [1, 2, 3, 4]}, + "gender/1": {"name": "male"}, + "gender/2": {"name": "female"}, + "gender/3": {"name": "diverse"}, + "gender/4": {"name": "non-binary"}, "import_preview/2": { "state": ImportState.DONE, "name": "account", @@ -53,8 +55,8 @@ def setUp(self) -> None: "value": "email@test.com", "info": ImportState.DONE, }, - "gender": { - "value": "male", + "gender_id": { + "value": 1, "info": ImportState.DONE, }, }, @@ -71,8 +73,8 @@ def setUp(self) -> None: "state": ImportState.ERROR, "messages": ["test"], "data": { - "gender": { - "value": "male", + "gender_id": { + "value": 1, "info": ImportState.DONE, } }, @@ -163,8 +165,8 @@ def test_import_username_and_update(self) -> None: "id": 1, }, "first_name": "Testy", - "gender": { - "value": "non-binary", + "gender_id": { + "value": 4, "info": ImportState.DONE, }, }, @@ -176,9 +178,7 @@ def test_import_username_and_update(self) -> None: ) response = self.request("account.import", {"id": 7, "import": True}) self.assert_status_code(response, 200) - self.assert_model_exists( - "user/1", {"first_name": "Testy", "gender": "non-binary"} - ) + self.assert_model_exists("user/1", {"first_name": "Testy", "gender_id": 4}) def test_ignore_unknown_gender(self) -> None: self.set_models( @@ -194,7 +194,7 @@ def test_ignore_unknown_gender(self) -> None: { "state": ImportState.NEW, "messages": [ - "Gender 'notAGender' is not in the allowed gender list." + "GenderId '5' is not in the allowed gender list." ], "data": { "id": 1, @@ -203,8 +203,8 @@ def test_ignore_unknown_gender(self) -> None: "info": ImportState.DONE, "id": 1, }, - "gender": { - "value": "notAGender", + "gender_id": { + "value": 5, "info": ImportState.WARNING, }, }, @@ -217,7 +217,7 @@ def test_ignore_unknown_gender(self) -> None: response = self.request("account.import", {"id": 7, "import": True}) self.assert_status_code(response, 200) user = self.assert_model_exists("user/1") - assert user.get("gender") is None + assert user.get("gender_id") is None def test_import_names_and_email_and_create(self) -> None: response = self.request("account.import", {"id": 3, "import": True}) @@ -227,7 +227,7 @@ def test_import_names_and_email_and_create(self) -> None: { "username": "TestyTester", "first_name": "Testy", - "gender": "male", + "gender_id": 1, "last_name": "Tester", "email": "email@test.com", }, @@ -854,14 +854,14 @@ def test_json_upload_wrong_gender(self) -> None: response_import = self.request("account.import", {"id": 1, "import": True}) self.assert_status_code(response_import, 200) user = self.assert_model_exists("user/2", {"username": "test"}) - assert "gender" not in user.keys() + assert "gender_id" not in user.keys() def test_json_upload_wrong_gender_2(self) -> None: self.json_upload_wrong_gender_2() response_import = self.request("account.import", {"id": 1, "import": True}) self.assert_status_code(response_import, 200) user = self.assert_model_exists("user/2", {"username": "test"}) - assert "gender" not in user.keys() + assert "gender_id" not in user.keys() def test_json_upload_legacy_username(self) -> None: self.json_upload_legacy_username() diff --git a/tests/system/action/user/test_account_json_upload.py b/tests/system/action/user/test_account_json_upload.py index 4d37f9270f..7ac032d76d 100644 --- a/tests/system/action/user/test_account_json_upload.py +++ b/tests/system/action/user/test_account_json_upload.py @@ -10,14 +10,20 @@ class AccountJsonUpload(BaseActionTestCase): def test_json_upload_simple(self) -> None: start_time = int(time()) + self.set_models( + { + "organization/1": {"gender_ids": [1, 2, 3, 4]}, + "gender/1": {"name": "male"}, + "gender/2": {"name": "female"}, + "gender/3": {"name": "diverse"}, + "gender/4": {"name": "non-binary"}, + } + ) response = self.request( "account.json_upload", { "data": [ { - "organization/1": { - "genders": ["male", "female", "diverse", "non-binary"] - }, "username": "test", "default_password": "secret", "is_active": "1", @@ -40,7 +46,7 @@ def test_json_upload_simple(self) -> None: "is_active": True, "is_physical_person": False, "default_vote_weight": {"value": "1.120000", "info": ImportState.DONE}, - "gender": {"value": "female", "info": ImportState.DONE}, + "gender": {"id": 2, "value": "female", "info": ImportState.DONE}, }, } import_preview_id = response.json["results"][0][0].get("id") @@ -1078,7 +1084,7 @@ def test_json_upload_dont_recognize_empty_name_and_email(self) -> None: "user/5": { "email": "balu@ntvtn.de", "title": "title", - "gender": "non-binary", + "gender_id": 4, "pronoun": "pronoun", "password": "$argon2id$v=19$m=65536,t=3,p=4$iQbqhQ2/XYiFnO6vP6rtGQ$Bv3QuH4l9UQACws9hiuCCUBQepVRnCTqmOn5TkXfnQ8", "username": "balubear", @@ -1393,7 +1399,13 @@ def json_upload_generate_default_password(self) -> None: def json_upload_wrong_gender(self) -> None: self.set_models( - {"organization/1": {"genders": ["male", "female", "diverse", "non-binary"]}} + { + "organization/1": {"gender_ids": [1, 2, 3, 4]}, + "gender/1": {"name": "male"}, + "gender/2": {"name": "female"}, + "gender/3": {"name": "diverse"}, + "gender/4": {"name": "non-binary"}, + } ) response = self.request( "account.json_upload", @@ -1415,7 +1427,12 @@ def json_upload_wrong_gender(self) -> None: def json_upload_wrong_gender_2(self) -> None: self.set_models( - {ONE_ORGANIZATION_FQID: {"genders": ["dragon", "lobster", "snake"]}} + { + ONE_ORGANIZATION_FQID: {"gender_ids": [1, 2, 3]}, + "gender/1": {"name": "dragon"}, + "gender/2": {"name": "lobster"}, + "gender/3": {"name": "snake"}, + } ) response = self.request( "account.json_upload", diff --git a/tests/system/action/user/test_create.py b/tests/system/action/user/test_create.py index a5dc47693d..96e76bf4d3 100644 --- a/tests/system/action/user/test_create.py +++ b/tests/system/action/user/test_create.py @@ -549,7 +549,13 @@ def test_create_permission_group_A_oml_manage_user(self) -> None: OrganizationManagementLevel.CAN_MANAGE_USERS, self.user_id ) self.set_models( - {"organization/1": {"genders": ["male", "female", "diverse", "non-binary"]}} + { + "organization/1": {"gender_ids": [1, 2, 3, 4]}, + "gender/1": {"name": "male"}, + "gender/2": {"name": "female"}, + "gender/3": {"name": "diverse"}, + "gender/4": {"name": "non-binary"}, + } ) response = self.request_json( @@ -565,7 +571,7 @@ def test_create_permission_group_A_oml_manage_user(self) -> None: "is_active": True, "is_physical_person": True, "default_password": "new default_password", - "gender": "female", + "gender_id": 2, "email": "info@openslides.com", "default_vote_weight": "1.234000", "can_change_own_password": False, @@ -598,7 +604,7 @@ def test_create_permission_group_A_oml_manage_user(self) -> None: "is_active": True, "is_physical_person": True, "default_password": "new default_password", - "gender": "female", + "gender_id": 2, "email": "info@openslides.com", "default_vote_weight": "1.234000", "can_change_own_password": False, @@ -1204,19 +1210,16 @@ def test_create_username_with_spaces(self) -> None: assert "Username may not contain spaces" in response.json["message"] def test_create_gender(self) -> None: - self.set_models({"organization/1": {"genders": ["male", "female"]}}) + self.set_models({"organization/1": {"gender_ids": [1, 2]}}) response = self.request( "user.create", { "username": "test_Xcdfgee", - "gender": "test", + "gender_id": 5, }, ) self.assert_status_code(response, 400) - assert ( - "Gender 'test' is not in the allowed gender list." - in response.json["message"] - ) + assert "Model 'gender/5' does not exist." in response.json["message"] def test_exceed_limit_of_users(self) -> None: self.set_models( diff --git a/tests/system/action/user/test_merge_together.py b/tests/system/action/user/test_merge_together.py index a7b42af190..af2159c389 100644 --- a/tests/system/action/user/test_merge_together.py +++ b/tests/system/action/user/test_merge_together.py @@ -73,7 +73,23 @@ def setUp(self) -> None: "enable_electronic_voting": True, "committee_ids": [1, 2, 3], "user_ids": [2, 3, 4, 5, 6], - "genders": ["male", "female", "diverse", "non-binary"], + "gender_ids": [1, 2, 3, 4], + }, + "gender/1": { + "name": "male", + "organization_id": 1, + }, + "gender/2": { + "name": "female", + "organization_id": 1, + }, + "gender/3": { + "name": "diverse", + "organization_id": 1, + }, + "gender/4": { + "name": "non-binary", + "organization_id": 1, }, "committee/1": { "organization_id": 1, @@ -318,9 +334,9 @@ def test_merge_configuration_up_to_date(self) -> None: If this test fails, it is likely because new fields have been added to the collections listed in the AssertionError without considering the necessary changes to the user merge. - This can be fixed by editing the collection_field_groups in the + This can be fixed by editing the _collection_field_groups in the action class if it is the 'user' collection, - or else in the corresponding mixin class. + or else in the corresponding merge mixin class in merge_mixins.py. """ action = actions_map["user.merge_together"] merge_together = action( @@ -519,7 +535,7 @@ def setup_complex_user_fields(self) -> None: "first_name": "Nick", "is_active": False, "can_change_own_password": True, - "gender": "male", + "gender_id": 1, "email": "nick.everything@rob.banks", "last_email_sent": 123456789, "committee_management_ids": [1], @@ -539,7 +555,7 @@ def setup_complex_user_fields(self) -> None: "organization_management_level": OrganizationManagementLevel.SUPERADMIN, "is_active": True, "is_physical_person": False, - "gender": "female", + "gender_id": 2, "last_email_sent": 234567890, "is_present_in_meeting_ids": [2, 3], "member_number": "souperadmin", @@ -613,7 +629,7 @@ def test_merge_with_user_fields(self) -> None: "first_name": "Nick", "is_active": False, "can_change_own_password": True, - "gender": "male", + "gender_id": 1, "email": "nick.everything@rob.banks", "is_present_in_meeting_ids": [3, 4], "committee_management_ids": [1, 3], @@ -725,7 +741,7 @@ def test_with_custom_fields_complex(self) -> None: "pronoun": "for", "member_number": "this", "default_password": "now", - "gender": "female", + "gender_id": 2, "email": "user.in@this.organization", "is_active": False, "is_physical_person": None, @@ -743,13 +759,40 @@ def test_with_custom_fields_complex(self) -> None: "pronoun": "for", "member_number": "this", "default_password": "now", - "gender": "female", + "gender_id": 2, "email": "user.in@this.organization", "is_active": False, "is_physical_person": None, "default_vote_weight": "0.424242", }, ) + self.assert_model_exists( + "gender/2", + {"id": 2, "name": "female", "user_ids": [2], "organization_id": 1}, + ) + + def test_gender_not_changed(self) -> None: + self.setup_complex_user_fields() + response = self.request( + "user.merge_together", + { + "id": 3, + "user_ids": [2, 4, 5, 6], + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/3", + { + "gender_id": None, + }, + ) + self.assert_model_exists( + "gender/1", + { + "user_ids": None, + }, + ) def test_with_custom_fields_simple(self) -> None: response = self.request( @@ -764,7 +807,7 @@ def test_with_custom_fields_simple(self) -> None: "pronoun": "for", "member_number": "this", "default_password": "now", - "gender": "female", + "gender_id": 2, "email": "user.in@this.organization", "is_active": False, "is_physical_person": None, @@ -782,7 +825,7 @@ def test_with_custom_fields_simple(self) -> None: "pronoun": "for", "member_number": "this", "default_password": "now", - "gender": "female", + "gender_id": 2, "email": "user.in@this.organization", "is_active": False, "is_physical_person": None, diff --git a/tests/system/action/user/test_participant_import.py b/tests/system/action/user/test_participant_import.py index c6d7cd17b9..c676b6f04d 100644 --- a/tests/system/action/user/test_participant_import.py +++ b/tests/system/action/user/test_participant_import.py @@ -38,6 +38,7 @@ def setUp(self) -> None: "info": ImportState.DONE, }, "gender": { + "id": 1, "value": "male", "info": ImportState.DONE, }, @@ -49,9 +50,11 @@ def setUp(self) -> None: self.set_models( { - "organization/1": { - "genders": ["male", "female", "diverse", "non-binary"] - }, + "organization/1": {"gender_ids": [1, 2, 3, 4]}, + "gender/1": {"name": "male"}, + "gender/2": {"name": "female"}, + "gender/3": {"name": "diverse"}, + "gender/4": {"name": "non-binary"}, "import_preview/1": self.import_preview1_data, "meeting/1": { "is_active_in_organization_id": 1, @@ -108,7 +111,7 @@ def test_import_names_and_email_and_create(self) -> None: { "username": "jonny", "first_name": "Testy", - "gender": "male", + "gender_id": 1, "last_name": "Tester", "email": "email@test.com", "meeting_ids": [1], @@ -233,7 +236,7 @@ def test_import_gender_warning(self) -> None: user = self.assert_model_exists( "user/2", {"username": "jonny", "first_name": "Testy"} ) - assert user.get("gender") is None + assert user.get("gender_id") is None def test_import_error_state_done_missing_username(self) -> None: self.import_preview1_data["result"]["rows"][0]["data"].pop("username") @@ -563,6 +566,7 @@ def test_json_upload_update_multiple_users_okay(self) -> None: "can_change_own_password": False, "meeting_ids": [1], "meeting_user_ids": [38], + "gender_id": 3, }, ) level_up = self.assert_model_exists("structure_level/1") @@ -640,6 +644,7 @@ def test_json_upload_update_multiple_users_okay(self) -> None: "meeting_user_ids": [35], "is_physical_person": True, "is_active": True, + "gender_id": None, }, ) self.assert_model_exists( @@ -686,6 +691,7 @@ def test_json_upload_update_multiple_users_okay(self) -> None: "meeting_user_ids": [37], "is_physical_person": True, "is_active": True, + "gender_id": 2, }, ) self.assert_model_exists( @@ -728,6 +734,7 @@ def test_json_upload_one_structure_level_newly_created(self) -> None: {"id": created_groups["group4"], "info": "new", "value": "group4"}, ], "structure_level": [{"info": "new", "value": "level up", "id": 2}], + "gender_id": 3, } row = result["rows"][1] @@ -762,6 +769,7 @@ def test_json_upload_one_structure_level_newly_created(self) -> None: assert row["state"] == ImportState.NEW assert row["messages"] == [ "Because this participant is connected with a saml_id: The default_password will be ignored and password will not be changeable in OpenSlides.", + "Gender 'unknown' is not in the allowed gender list.", ] assert row["data"] == { "username": {"info": ImportState.DONE, "value": "new_user5"}, @@ -839,6 +847,7 @@ def test_json_upload_update_multiple_users_all_error(self) -> None: {"info": "new", "value": "group4"}, ], "structure_level": [{"info": "new", "value": "level up"}], + "gender_id": 3, } row = result["rows"][1] @@ -877,6 +886,7 @@ def test_json_upload_update_multiple_users_all_error(self) -> None: assert row["state"] == ImportState.ERROR assert row["messages"] == [ "Because this participant is connected with a saml_id: The default_password will be ignored and password will not be changeable in OpenSlides.", + "Gender 'unknown' is not in the allowed gender list.", "Error: saml_id 'saml5' found in different id (11 instead of None)", ] assert row["data"] == { diff --git a/tests/system/action/user/test_participant_json_upload.py b/tests/system/action/user/test_participant_json_upload.py index 07ac0f3659..08c17f9c9a 100644 --- a/tests/system/action/user/test_participant_json_upload.py +++ b/tests/system/action/user/test_participant_json_upload.py @@ -12,9 +12,7 @@ def setUp(self) -> None: super().setUp() self.set_models( { - "organization/1": { - "genders": ["male", "female", "diverse", "non-binary"] - }, + "organization/1": {"gender_ids": [1, 2, 3, 4]}, "meeting/1": { "name": "test", "group_ids": [1, 7], @@ -380,6 +378,7 @@ def test_json_upload_names_and_email_find_add_meeting_data(self) -> None: "username": "test", }, "group/1": {"default_group_for_meeting_id": 1}, + "gender/1": {"name": "male"}, } ) fix_fields = { @@ -416,7 +415,7 @@ def test_json_upload_names_and_email_find_add_meeting_data(self) -> None: assert row["data"]["username"] == {"value": "test", "info": "done", "id": 34} assert row["data"]["vote_weight"] == {"value": "1.456000", "info": "done"} assert row["data"]["is_present"] == {"value": False, "info": "done"} - assert row["data"]["gender"] == {"value": "male", "info": "done"} + assert row["data"]["gender"] == {"id": 1, "value": "male", "info": "done"} assert row["data"]["groups"] == [ {"value": "testgroup", "info": "generated", "id": 1} ] @@ -1505,12 +1504,18 @@ def json_upload_username_username_and_saml_id_found(self) -> None: def json_upload_multiple_users(self) -> None: self.set_models( { + "organization/1": {"gender_ids": [1, 2, 3, 4]}, + "gender/1": {"name": "male"}, + "gender/2": {"name": "female"}, + "gender/3": {"name": "diverse"}, + "gender/4": {"name": "non-binary"}, "user/2": { "username": "user2", "password": "secret", "default_password": "secret", "can_change_own_password": True, "default_vote_weight": "2.300000", + "gender_id": 1, }, "user/3": { "username": "user3", @@ -1581,6 +1586,7 @@ def json_upload_multiple_users(self) -> None: "saml_id": "test_saml_id2", "groups": ["group3", "group4"], "structure_level": ["level up"], + "gender": "diverse", }, { "saml_id": "saml3", @@ -1597,6 +1603,7 @@ def json_upload_multiple_users(self) -> None: "username": "new_user5", "saml_id": "saml5", "structure_level": ["level up", "no. 5"], + "gender": "unknown", }, {"saml_id": "new_saml6", "groups": ["group4"], "is_present": "1"}, { @@ -1609,6 +1616,7 @@ def json_upload_multiple_users(self) -> None: "unknown", "group7M1", ], + "gender": "female", }, ], }, @@ -1631,6 +1639,7 @@ def json_upload_multiple_users(self) -> None: {"info": "new", "value": "group4"}, ], "structure_level": [{"value": "level up", "info": ImportState.NEW}], + "gender": {"id": 3, "info": ImportState.DONE, "value": "diverse"}, } assert import_preview["result"]["rows"][1]["state"] == ImportState.DONE @@ -1661,7 +1670,8 @@ def json_upload_multiple_users(self) -> None: assert import_preview["result"]["rows"][3]["state"] == ImportState.NEW assert import_preview["result"]["rows"][3]["messages"] == [ - "Because this participant is connected with a saml_id: The default_password will be ignored and password will not be changeable in OpenSlides." + "Because this participant is connected with a saml_id: The default_password will be ignored and password will not be changeable in OpenSlides.", + "Gender 'unknown' is not in the allowed gender list.", ] assert import_preview["result"]["rows"][3]["data"] == { "saml_id": {"info": "new", "value": "saml5"}, @@ -1672,6 +1682,7 @@ def json_upload_multiple_users(self) -> None: {"value": "level up", "info": ImportState.NEW}, {"value": "no. 5", "info": ImportState.NEW}, ], + "gender": {"info": ImportState.WARNING, "value": "unknown"}, } assert import_preview["result"]["rows"][4]["state"] == ImportState.NEW @@ -1706,6 +1717,7 @@ def json_upload_multiple_users(self) -> None: {"info": "new", "value": "unknown"}, {"id": 7, "info": "done", "value": "group7M1"}, ], + "gender": {"id": 2, "info": ImportState.DONE, "value": "female"}, } def json_upload_with_complicated_names(self) -> None: @@ -2272,7 +2284,7 @@ def json_upload_dont_recognize_empty_name_and_email(self) -> None: "user/5": { "email": "balu@ntvtn.de", "title": "title", - "gender": "non-binary", + "gender_id": 4, "pronoun": "pronoun", "password": "$argon2id$v=19$m=65536,t=3,p=4$iQbqhQ2/XYiFnO6vP6rtGQ$Bv3QuH4l9UQACws9hiuCCUBQepVRnCTqmOn5TkXfnQ8", "username": "balubear", diff --git a/tests/system/action/user/test_save_saml_account.py b/tests/system/action/user/test_save_saml_account.py index 9a849391ae..795f948812 100644 --- a/tests/system/action/user/test_save_saml_account.py +++ b/tests/system/action/user/test_save_saml_account.py @@ -6,18 +6,6 @@ class UserBaseSamlAccount(BaseActionTestCase): def setUp(self) -> None: - self.results = { - "saml_id": "111222333", - "title": "Dr.", - "first_name": "Max", - "last_name": "Mustermann", - "email": "test@example.com", - "gender": "male", - "pronoun": "er", - "is_active": True, - "is_physical_person": True, - } - super().setUp() self.set_models( { @@ -34,7 +22,12 @@ def setUp(self) -> None: "is_active": "is_active", "is_physical_person": "is_person", }, - } + "gender_ids": [1, 2, 3, 4], + }, + "gender/1": {"organization_id": 1, "name": "male"}, + "gender/2": {"organization_id": 1, "name": "female"}, + "gender/3": {"organization_id": 1, "name": "diverse"}, + "gender/4": {"organization_id": 1, "name": "non-binary"}, } ) @@ -70,7 +63,7 @@ def test_save_attr_no_saml_id_provided(self) -> None: def test_save_attr_empty_saml_id_provided(self) -> None: response = self.request( - "user.save_saml_account", {"username": [], "lastName": "Cartwright"} + "user.save_saml_account", {"username": "", "lastName": "Cartwright"} ) self.assert_status_code(response, 400) self.assertIn( @@ -117,6 +110,24 @@ def test_suppress_not_allowed_field(self) -> None: self.assert_status_code(response, 200) self.assert_model_exists("user/2", {"username": "Joe", "default_number": None}) + def test_create_new_gender(self) -> None: + response = self.request( + "user.save_saml_account", + { + "username": "111222333", + "gender": "cloud", + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/2", + { + "username": "111222333", + "gender_id": 5, + }, + ) + self.assert_model_exists("gender/5", {"name": "cloud"}) + class UserCreateSamlAccount(UserBaseSamlAccount): def test_create_saml_account_all_fields(self) -> None: @@ -147,7 +158,7 @@ def test_create_saml_account_all_fields(self) -> None: "first_name": "Max", "last_name": "Mustermann", "email": "test@example.com", - "gender": "male", + "gender_id": 1, "pronoun": "er", "is_active": True, "is_physical_person": True, @@ -182,7 +193,7 @@ def test_create_saml_account_all_fields_as_list(self) -> None: "first_name": "Max", "last_name": "Mustermann", "email": "test@example.com", - "gender": "male", + "gender_id": 1, "pronoun": "er", "is_active": True, "is_physical_person": True, @@ -263,7 +274,11 @@ def test_update_saml_account_correct(self) -> None: ) def test_update_saml_account_all_fields(self) -> None: - self.set_models({"user/78": {"username": "Saml", "saml_id": "111222333"}}) + self.set_models( + { + "user/78": {"username": "Saml", "saml_id": "111222333", "gender_id": 4}, + } + ) response = self.request( "user.save_saml_account", { @@ -288,7 +303,7 @@ def test_update_saml_account_all_fields(self) -> None: "first_name": "Max", "last_name": "Mustermann", "email": "test@example.com", - "gender": "male", + "gender_id": 1, "pronoun": "er", "is_active": True, "is_physical_person": True, @@ -303,7 +318,7 @@ def test_update_saml_account_change_nothing(self) -> None: "first_name": "Max", "last_name": "Mustermann", "email": "test@example.com", - "gender": "male", + "gender_id": 1, "pronoun": "er", "is_active": True, "is_physical_person": True, @@ -340,7 +355,7 @@ def test_create_saml_account_all_fields_mixed_changes(self) -> None: "first_name": "Max", "last_name": "Mustermann", "email": "test@example.com", - "gender": "male", + "gender_id": 1, "pronoun": "er", "is_active": True, "is_physical_person": True, @@ -370,13 +385,42 @@ def test_create_saml_account_all_fields_mixed_changes(self) -> None: "first_name": "Maxx", "last_name": "Mustermann", "email": "", - "gender": "male", + "gender_id": 1, "pronoun": "er", "is_active": False, "is_physical_person": True, }, ) + def test_gender_to_none(self) -> None: + self.set_models( + { + "user/78": {"username": "Saml", "saml_id": "111222333", "gender_id": 4}, + } + ) + response = self.request( + "user.save_saml_account", + { + "username": "111222333", + "title": "Dr.", + "firstName": "Max", + "lastName": "Mustermann", + "gender": "", + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/78", + { + "saml_id": "111222333", + "username": "Saml", + "title": "Dr.", + "first_name": "Max", + "last_name": "Mustermann", + "gender_id": None, + }, + ) + class UserSamlAccountBoolean(UserBaseSamlAccount): def test_create_saml_account_boolean_defaults(self) -> None: @@ -396,7 +440,7 @@ def test_create_saml_account_boolean_defaults(self) -> None: "first_name": None, "last_name": None, "email": None, - "gender": None, + "gender_id": None, "pronoun": None, "is_active": True, "is_physical_person": True, diff --git a/tests/system/action/user/test_update.py b/tests/system/action/user/test_update.py index 130bba0012..75405ef691 100644 --- a/tests/system/action/user/test_update.py +++ b/tests/system/action/user/test_update.py @@ -7,6 +7,7 @@ class UserUpdateActionTest(BaseActionTestCase): + def permission_setup(self) -> None: self.create_meeting() self.user_id = self.create_user("test", group_ids=[1]) @@ -712,7 +713,13 @@ def test_perm_group_A_oml_manage_user(self) -> None: ) self.set_user_groups(111, [1, 6]) self.set_models( - {"organization/1": {"genders": ["male", "female", "diverse", "non-binary"]}} + { + "organization/1": {"gender_ids": [1, 2, 3, 4]}, + "gender/1": {"name": "male"}, + "gender/2": {"name": "female"}, + "gender/3": {"name": "diverse"}, + "gender/4": {"name": "non-binary"}, + } ) response = self.request( @@ -726,7 +733,7 @@ def test_perm_group_A_oml_manage_user(self) -> None: "is_active": True, "is_physical_person": True, "default_password": "new default_password", - "gender": "female", + "gender_id": 2, "email": "info@openslides.com ", # space intentionally, will be stripped "default_vote_weight": "1.234000", "can_change_own_password": False, @@ -743,7 +750,7 @@ def test_perm_group_A_oml_manage_user(self) -> None: "is_active": True, "is_physical_person": True, "default_password": "new default_password", - "gender": "female", + "gender_id": 2, "email": "info@openslides.com", "default_vote_weight": "1.234000", "can_change_own_password": False, @@ -936,9 +943,8 @@ def test_perm_group_A_locked_meeting(self) -> None: self.set_user_groups(111, [1, 6]) self.set_models( { - "organization/1": { - "genders": ["male", "female", "diverse", "non-binary"] - }, + "organization/1": {"gender_ids": [2]}, + "gender/2": {"name": "female"}, "meeting/4": {"locked_from_inside": True}, } ) @@ -954,7 +960,7 @@ def test_perm_group_A_locked_meeting(self) -> None: "is_active": True, "is_physical_person": True, "default_password": "new default_password", - "gender": "female", + "gender_id": 2, "email": "info@openslides.com ", # space intentionally, will be stripped "default_vote_weight": "1.234000", "can_change_own_password": False, @@ -1765,22 +1771,25 @@ def test_update_gender(self) -> None: {"username": "username_srtgb123"}, ) self.set_models( - {"organization/1": {"genders": ["male", "female", "diverse", "non-binary"]}} + { + "organization/1": {"gender_ids": [1, 2, 3, 4]}, + "gender/1": {"name": "male"}, + "gender/2": {"name": "female"}, + "gender/3": {"name": "diverse"}, + "gender/4": {"name": "non-binary"}, + } ) - response = self.request("user.update", {"id": 111, "gender": "test"}) + response = self.request("user.update", {"id": 111, "gender_id": 5}) self.assert_status_code(response, 400) - assert ( - "Gender 'test' is not in the allowed gender list." - in response.json["message"] - ) + assert "Model 'gender/5' does not exist." in response.json["message"] - response = self.request("user.update", {"id": 111, "gender": "diverse"}) + response = self.request("user.update", {"id": 111, "gender_id": 3}) self.assert_status_code(response, 200) - self.assert_model_exists("user/111", {"gender": "diverse"}) + self.assert_model_exists("user/111", {"gender_id": 3}) - response = self.request("user.update", {"id": 111, "gender": "non-binary"}) + response = self.request("user.update", {"id": 111, "gender_id": 4}) self.assert_status_code(response, 200) - self.assert_model_exists("user/111", {"gender": "non-binary"}) + self.assert_model_exists("user/111", {"gender_id": 4}) def test_update_not_in_update_is_present_in_meeting_ids(self) -> None: self.create_model( diff --git a/tests/system/action/user/test_update_self.py b/tests/system/action/user/test_update_self.py index 47dd3b632d..2b313d0198 100644 --- a/tests/system/action/user/test_update_self.py +++ b/tests/system/action/user/test_update_self.py @@ -3,6 +3,7 @@ class UserUpdateSelfActionTest(BaseActionTestCase): def test_update_correct(self) -> None: + self.set_models({"gender/1": {"name": "male"}}) self.update_model( "user/1", {"username": "username_srtgb123"}, @@ -13,7 +14,7 @@ def test_update_correct(self) -> None: "username": "username_Xcdfgee", "email": " email1@example.com ", "pronoun": "Test", - "gender": "male", + "gender_id": 1, }, ) self.assert_status_code(response, 200) @@ -21,7 +22,7 @@ def test_update_correct(self) -> None: assert model.get("username") == "username_Xcdfgee" assert model.get("email") == "email1@example.com" assert model.get("pronoun") == "Test" - assert model.get("gender") == "male" + assert model.get("gender_id") == 1 self.assert_history_information("user/1", ["Personal data changed"]) def test_username_already_given(self) -> None: diff --git a/tests/system/migrations/test_0057_gender_model.py b/tests/system/migrations/test_0057_gender_model.py new file mode 100644 index 0000000000..58498d2396 --- /dev/null +++ b/tests/system/migrations/test_0057_gender_model.py @@ -0,0 +1,63 @@ +def create_data(write): + write( + { + "type": "create", + "fqid": "organization/1", + "fields": {"id": 1, "genders": ["male", "female", "diverse", "non-binary"]}, + }, + {"type": "create", "fqid": "user/1", "fields": {"id": 1, "gender": "male"}}, + {"type": "create", "fqid": "user/2", "fields": {"id": 2, "gender": "female"}}, + {"type": "create", "fqid": "user/3", "fields": {"id": 3, "gender": "diverse"}}, + { + "type": "create", + "fqid": "user/4", + "fields": {"id": 4, "gender": "non-binary"}, + }, + { + "type": "create", + "fqid": "user/5", + "fields": {"id": 5, "gender": "non-binary"}, + }, + ) + + +def test_migration_full(write, finalize, assert_model): + create_data(write) + + finalize("0057_gender_model") + + assert_model( + "gender/3", + {"id": 3, "name": "diverse", "organization_id": 1, "user_ids": [3]}, + ) + assert_model( + "gender/4", + {"id": 4, "name": "non-binary", "organization_id": 1, "user_ids": [4, 5]}, + ) + assert_model( + "user/5", + { + "id": 5, + "gender_id": 4, + }, + ) + + +def test_migration_with_empty_gender(write, finalize, assert_model): + create_data(write) + write( + {"type": "create", "fqid": "user/6", "fields": {"id": 6, "gender": None}}, + ) + + finalize("0057_gender_model") + + assert_model( + "gender/4", + {"id": 4, "name": "non-binary", "organization_id": 1, "user_ids": [4, 5]}, + ) + assert_model( + "user/6", + { + "id": 6, + }, + ) diff --git a/tests/system/presenter/test_check_database_all.py b/tests/system/presenter/test_check_database_all.py index 32eb7de009..ea8007fdec 100644 --- a/tests/system/presenter/test_check_database_all.py +++ b/tests/system/presenter/test_check_database_all.py @@ -141,7 +141,7 @@ def test_correct(self) -> None: "limit_of_meetings": 0, "users_email_sender": "test@example.com", "limit_of_users": 0, - "genders": ["male", "female", "diverse", "non-binary"], + "gender_ids": [1, 2, 3, 4], "user_ids": [1], "users_email_body": "ballspamhamfoo", "users_email_subject": "hamfoo", @@ -154,6 +154,10 @@ def test_correct(self) -> None: "saml_enabled": False, "saml_login_button_text": "Login button text", }, + "gender/1": {"name": "male", "organization_id": 1}, + "gender/2": {"name": "female", "organization_id": 1}, + "gender/3": {"name": "diverse", "organization_id": 1}, + "gender/4": {"name": "non-binary", "organization_id": 1}, "theme/1": { "name": "Test Theme", "accent_500": "#000000", @@ -304,7 +308,7 @@ def test_correct_relations(self) -> None: "organization_tag_ids": [1], "template_meeting_ids": [1], "limit_of_meetings": 0, - "genders": ["male", "female", "diverse", "non-binary"], + "gender_ids": [1, 2, 3, 4], "user_ids": [1, 2, 3, 4, 5, 6], "users_email_sender": "test@example.com", "limit_of_users": 0, @@ -321,6 +325,10 @@ def test_correct_relations(self) -> None: "mediafile_ids": [3, 4], "published_mediafile_ids": [3, 4], }, + "gender/1": {"name": "male", "organization_id": 1}, + "gender/2": {"name": "female", "organization_id": 1}, + "gender/3": {"name": "diverse", "organization_id": 1}, + "gender/4": {"name": "non-binary", "organization_id": 1}, "theme/1": { "name": "Test Theme", "accent_500": "#000000", @@ -625,7 +633,7 @@ def test_relation_2(self) -> None: "active_meeting_ids": [1, 2], "organization_tag_ids": [1], "limit_of_meetings": 0, - "genders": ["male", "female", "diverse", "non-binary"], + "gender_ids": [1, 2, 3, 4], "user_ids": [1], "users_email_sender": "test@example.com", "limit_of_users": 0, @@ -640,6 +648,10 @@ def test_relation_2(self) -> None: "saml_enabled": True, "saml_login_button_text": "SAML Login", }, + "gender/1": {"name": "male", "organization_id": 1}, + "gender/2": {"name": "female", "organization_id": 1}, + "gender/3": {"name": "diverse", "organization_id": 1}, + "gender/4": {"name": "non-binary", "organization_id": 1}, "theme/1": { "name": "Test Theme", "accent_500": "#000000", diff --git a/tests/system/presenter/test_export_meeting.py b/tests/system/presenter/test_export_meeting.py index 8dcc519709..ac64eb1425 100644 --- a/tests/system/presenter/test_export_meeting.py +++ b/tests/system/presenter/test_export_meeting.py @@ -118,9 +118,11 @@ def test_add_users(self) -> None: "present_user_ids": [1], "meeting_user_ids": [1], }, + "gender/1": {"name": "male"}, "user/1": { "is_present_in_meeting_ids": [1], "meeting_user_ids": [1], + "gender_id": 1, }, "meeting_user/1": { "meeting_id": 1, @@ -141,6 +143,7 @@ def test_add_users(self) -> None: assert data["user"]["1"]["is_active"] is True assert data["user"]["1"]["meeting_ids"] == [1] assert data["user"]["1"]["is_present_in_meeting_ids"] == [1] + assert data["user"]["1"]["gender"] == "male" assert data["meeting_user"]["1"]["group_ids"] == [11] def test_add_users_in_2_meetings(self) -> None: From 824386fe39f6dbb608a8cfb0ca8f42b2f345ae72 Mon Sep 17 00:00:00 2001 From: Hannes Janott Date: Fri, 27 Sep 2024 16:45:54 +0200 Subject: [PATCH 08/90] add new meeting specific setting hide metadata background for projector (#2647) * add new meeting specific setting hide metadata background for projector * add setting to example-data.json * repair test with missing setting entry * add default value to example-data.json and setting to tests * black --- global/data/example-data.json | 1 + global/meta | 2 +- openslides_backend/action/actions/meeting/update.py | 1 + openslides_backend/models/models.py | 1 + tests/system/action/meeting/test_update.py | 6 +++++- tests/system/presenter/test_check_database_all.py | 3 ++- 6 files changed, 11 insertions(+), 3 deletions(-) diff --git a/global/data/example-data.json b/global/data/example-data.json index 6ffb08e605..ffe64b86d2 100644 --- a/global/data/example-data.json +++ b/global/data/example-data.json @@ -322,6 +322,7 @@ "motions_enable_reason_on_projector": false, "motions_enable_sidebox_on_projector": false, "motions_enable_recommendation_on_projector": true, + "motions_hide_metadata_background": false, "motions_show_referring_motions": true, "motions_show_sequential_number": true, "motions_reason_required": false, diff --git a/global/meta b/global/meta index f322efd811..e5b74fac6e 160000 --- a/global/meta +++ b/global/meta @@ -1 +1 @@ -Subproject commit f322efd81152a9b5ac864ae7145790c1673f29bf +Subproject commit e5b74fac6e9c12255237e173a07fb22c25f05703 diff --git a/openslides_backend/action/actions/meeting/update.py b/openslides_backend/action/actions/meeting/update.py index 4e9a7f3697..afbc9d9d69 100644 --- a/openslides_backend/action/actions/meeting/update.py +++ b/openslides_backend/action/actions/meeting/update.py @@ -102,6 +102,7 @@ "motions_enable_text_on_projector", "motions_enable_reason_on_projector", "motions_enable_sidebox_on_projector", + "motions_hide_metadata_background", "motions_enable_recommendation_on_projector", "motions_show_referring_motions", "motions_show_sequential_number", diff --git a/openslides_backend/models/models.py b/openslides_backend/models/models.py index f7ca9200a8..20c7268a6e 100644 --- a/openslides_backend/models/models.py +++ b/openslides_backend/models/models.py @@ -486,6 +486,7 @@ class Meeting(Model, MeetingModelMixin): motions_enable_reason_on_projector = fields.BooleanField(default=False) motions_enable_sidebox_on_projector = fields.BooleanField(default=False) motions_enable_recommendation_on_projector = fields.BooleanField(default=True) + motions_hide_metadata_background = fields.BooleanField(default=False) motions_show_referring_motions = fields.BooleanField(default=True) motions_show_sequential_number = fields.BooleanField(default=True) motions_recommendations_by = fields.CharField() diff --git a/tests/system/action/meeting/test_update.py b/tests/system/action/meeting/test_update.py index 4c332019ce..a64455b8fb 100644 --- a/tests/system/action/meeting/test_update.py +++ b/tests/system/action/meeting/test_update.py @@ -370,9 +370,13 @@ def test_update_only_one_time_one_removal_from_db(self) -> None: def test_update_new_meeting_setting(self) -> None: meeting, _ = self.basic_test( - {"agenda_show_topic_navigation_on_detail_view": True} + { + "agenda_show_topic_navigation_on_detail_view": True, + "motions_hide_metadata_background": True, + } ) assert meeting.get("agenda_show_topic_navigation_on_detail_view") is True + assert meeting.get("motions_hide_metadata_background") is True def test_update_group_a_no_permissions(self) -> None: self.base_permission_test( diff --git a/tests/system/presenter/test_check_database_all.py b/tests/system/presenter/test_check_database_all.py index ea8007fdec..7f8967df88 100644 --- a/tests/system/presenter/test_check_database_all.py +++ b/tests/system/presenter/test_check_database_all.py @@ -82,7 +82,8 @@ def get_meeting_defaults(self) -> dict[str, Any]: "motions_enable_text_on_projector": False, "motions_enable_reason_on_projector": False, "motions_enable_sidebox_on_projector": False, - "motions_enable_recommendation_on_projector": True, + "motions_enable_recommendation_on_projector": False, + "motions_hide_metadata_background": False, "motions_show_referring_motions": True, "motions_show_sequential_number": True, "motions_recommendation_text_mode": "diff", From ed82f6b981541524d08e00071b82820d3d2d92c9 Mon Sep 17 00:00:00 2001 From: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> Date: Mon, 30 Sep 2024 17:04:42 +0200 Subject: [PATCH 09/90] Allow to edit own delegation via user.update_self (#2632) * Allow to edit own delegation via user.update_self * Remove meeting setting * Update meta --- docs/actions/user.update_self.md | 6 + global/meta | 2 +- .../action/actions/user/update_self.py | 21 ++- openslides_backend/models/models.py | 1 + openslides_backend/permissions/permissions.py | 8 +- tests/system/action/user/test_update_self.py | 146 ++++++++++++++++++ 6 files changed, 180 insertions(+), 4 deletions(-) diff --git a/docs/actions/user.update_self.md b/docs/actions/user.update_self.md index 3c53a3c58f..61d879898c 100644 --- a/docs/actions/user.update_self.md +++ b/docs/actions/user.update_self.md @@ -6,6 +6,8 @@ email: string; gender_id: Id; pronoun: string; + meeting_id: ID; + vote_delegated_to_id: Id; } ``` @@ -14,5 +16,9 @@ Updates the request user. Removes starting and trailing spaces from `username`. The given `gender` must be present in `organization/genders`. +`meeting_id` is only for editing meeting-internal data, and the value will be thrown away afterwards. + ## Permissions The user must not be the anonymous. + +The request user fulfills the conditions for editing his own delegations, if he has the permission user.can_edit_own_delegation. diff --git a/global/meta b/global/meta index e5b74fac6e..72b0730034 160000 --- a/global/meta +++ b/global/meta @@ -1 +1 @@ -Subproject commit e5b74fac6e9c12255237e173a07fb22c25f05703 +Subproject commit 72b07300348f3e53903ea07b3e819884c6df5a65 diff --git a/openslides_backend/action/actions/user/update_self.py b/openslides_backend/action/actions/user/update_self.py index 037386abfb..287851d6d4 100644 --- a/openslides_backend/action/actions/user/update_self.py +++ b/openslides_backend/action/actions/user/update_self.py @@ -1,6 +1,9 @@ from typing import Any -from ....models.models import User +from ....models.models import MeetingUser, User +from ....permissions.permission_helper import has_perm +from ....permissions.permissions import Permissions +from ....shared.exceptions import MissingPermission from ...generics.update import UpdateAction from ...mixins.send_email_mixin import EmailCheckMixin from ...util.default_schema import DefaultSchema @@ -16,7 +19,10 @@ class UserUpdateSelf(EmailCheckMixin, UpdateAction, UserMixin, UpdateHistoryMixi model = User() schema = DefaultSchema(User()).get_default_schema( - optional_properties=["username", "pronoun", "gender_id", "email"] + optional_properties=["username", "pronoun", "gender_id", "email"], + additional_optional_fields={ + **MeetingUser().get_properties("meeting_id", "vote_delegated_to_id") + }, ) check_email_field = "email" @@ -31,3 +37,14 @@ def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: def check_permissions(self, instance: dict[str, Any]) -> None: self.assert_not_anonymous() + if ( + (meeting_id := instance.get("meeting_id")) + and "vote_delegated_to_id" in instance + and not has_perm( + self.datastore, + self.user_id, + Permissions.User.CAN_EDIT_OWN_DELEGATION, + meeting_id, + ) + ): + raise MissingPermission(Permissions.User.CAN_EDIT_OWN_DELEGATION) diff --git a/openslides_backend/models/models.py b/openslides_backend/models/models.py index 20c7268a6e..15487147d8 100644 --- a/openslides_backend/models/models.py +++ b/openslides_backend/models/models.py @@ -999,6 +999,7 @@ class Group(Model): "user.can_see_sensitive_data", "user.can_see", "user.can_update", + "user.can_edit_own_delegation", ] } ) diff --git a/openslides_backend/permissions/permissions.py b/openslides_backend/permissions/permissions.py index 096eb11523..eec5e46b4e 100644 --- a/openslides_backend/permissions/permissions.py +++ b/openslides_backend/permissions/permissions.py @@ -71,6 +71,7 @@ class _Tag(str, Permission, Enum): class _User(str, Permission, Enum): + CAN_EDIT_OWN_DELEGATION = "user.can_edit_own_delegation" CAN_MANAGE = "user.can_manage" CAN_MANAGE_PRESENCE = "user.can_manage_presence" CAN_SEE = "user.can_see" @@ -144,9 +145,14 @@ class Permissions: _Projector.CAN_SEE: [_Projector.CAN_MANAGE], _Projector.CAN_MANAGE: [], _Tag.CAN_MANAGE: [], - _User.CAN_SEE: [_User.CAN_MANAGE_PRESENCE, _User.CAN_SEE_SENSITIVE_DATA], + _User.CAN_SEE: [ + _User.CAN_MANAGE_PRESENCE, + _User.CAN_SEE_SENSITIVE_DATA, + _User.CAN_EDIT_OWN_DELEGATION, + ], _User.CAN_MANAGE_PRESENCE: [_User.CAN_MANAGE], _User.CAN_SEE_SENSITIVE_DATA: [_User.CAN_UPDATE], _User.CAN_UPDATE: [_User.CAN_MANAGE], _User.CAN_MANAGE: [], + _User.CAN_EDIT_OWN_DELEGATION: [], } diff --git a/tests/system/action/user/test_update_self.py b/tests/system/action/user/test_update_self.py index 2b313d0198..35fa780176 100644 --- a/tests/system/action/user/test_update_self.py +++ b/tests/system/action/user/test_update_self.py @@ -1,3 +1,4 @@ +from openslides_backend.permissions.permissions import Permissions from tests.system.action.base import BaseActionTestCase @@ -89,3 +90,148 @@ def test_update_broken_email(self) -> None: ) self.assert_status_code(response, 400) assert "email must be valid email." in response.json["message"] + + def test_update_delegation(self) -> None: + self.create_meeting() + self.set_models( + { + "user/1": {"username": "username_srtgb123", "meeting_user_ids": [11]}, + "meeting_user/11": {"user_id": 1, "meeting_id": 1, "group_ids": []}, + "meeting/1": { + "meeting_user_ids": [11], + }, + } + ) + self.set_user_groups(1, [3]) + self.create_user("mandy", [3]) + response = self.request( + "user.update_self", + {"meeting_id": 1, "vote_delegated_to_id": 12}, + ) + self.assert_status_code(response, 200) + self.assert_model_exists("meeting_user/11", {"vote_delegated_to_id": 12}) + + def test_update_delegation_without_meeting_id(self) -> None: + self.create_meeting() + self.set_models( + { + "user/1": {"username": "username_srtgb123", "meeting_user_ids": [11]}, + "meeting_user/11": {"user_id": 1, "meeting_id": 1, "group_ids": []}, + "meeting/1": { + "meeting_user_ids": [11], + }, + } + ) + self.set_user_groups(1, [3]) + self.create_user("mandy", [3]) + response = self.request( + "user.update_self", + {"vote_delegated_to_id": 12}, + ) + self.assert_status_code(response, 400) + self.assertIn( + "Missing meeting_id in instance, because meeting related fields used", + response.json["message"], + ) + + def test_update_delegation_wrong_meeting(self) -> None: + self.create_meeting() + self.create_meeting(4) + self.set_models( + { + "user/1": {"username": "username_srtgb123", "meeting_user_ids": [11]}, + "meeting_user/11": {"user_id": 1, "meeting_id": 1, "group_ids": []}, + "meeting/1": { + "meeting_user_ids": [11], + }, + } + ) + self.set_user_groups(1, [3]) + self.set_user_groups(1, [5]) + self.create_user("mandy", [3]) + response = self.request( + "user.update_self", + {"meeting_id": 4, "vote_delegated_to_id": 13}, + ) + self.assert_status_code(response, 400) + self.assertIn( + "User 2's delegation id don't belong to meeting 4.", + response.json["message"], + ) + + def test_update_with_meeting_id(self) -> None: + self.create_meeting() + self.set_models( + { + "user/1": {"username": "username_srtgb123", "meeting_user_ids": [11]}, + "meeting_user/11": {"user_id": 1, "meeting_id": 1, "group_ids": []}, + "meeting/1": { + "meeting_user_ids": [11], + }, + } + ) + self.set_user_groups(1, [3]) + self.create_user("mandy", [3]) + response = self.request( + "user.update_self", + { + "meeting_id": 1, + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists("meeting_user/11", {"vote_delegated_to_id": None}) + + def test_update_delegation_self(self) -> None: + self.create_meeting() + self.set_models( + { + "user/1": {"username": "username_srtgb123", "meeting_user_ids": [11]}, + "meeting_user/11": {"user_id": 1, "meeting_id": 1, "group_ids": []}, + "meeting/1": { + "meeting_user_ids": [11], + }, + } + ) + self.set_user_groups(1, [3]) + self.create_user("mandy", [3]) + response = self.request( + "user.update_self", + {"meeting_id": 1, "vote_delegated_to_id": 11}, + ) + self.assert_status_code(response, 400) + self.assertIn( + "User 1 can't delegate the vote to himself.", + response.json["message"], + ) + + def test_update_delegation_permission(self) -> None: + self.base_permission_test( + { + "meeting/1": { + "meeting_user_ids": [11, 12], + }, + "user/1": {"username": "username_srtgb123", "meeting_user_ids": [11]}, + "user/3": {"username": "username_srtgb124", "meeting_user_ids": [12]}, + "meeting_user/12": {"user_id": 3, "meeting_id": 1, "group_ids": [3]}, + "group/3": {"meeting_user_ids": [12]}, + }, + "user.update_self", + {"meeting_id": 1, "vote_delegated_to_id": 12}, + Permissions.User.CAN_EDIT_OWN_DELEGATION, + ) + + def test_update_delegation_permission_denied(self) -> None: + self.base_permission_test( + { + "meeting/1": { + "meeting_user_ids": [11, 12], + }, + "user/1": {"username": "username_srtgb123", "meeting_user_ids": [11]}, + "user/3": {"username": "username_srtgb124", "meeting_user_ids": [12]}, + "meeting_user/12": {"user_id": 3, "meeting_id": 1, "group_ids": [3]}, + "group/3": {"meeting_user_ids": [12]}, + }, + "user.update_self", + {"meeting_id": 1, "vote_delegated_to_id": 12}, + None, + ) From 8d67a306adcbd6f7e5472fbe9830ef09a1be1962 Mon Sep 17 00:00:00 2001 From: "openslides-automation[bot]" <125256978+openslides-automation[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 14:31:38 +0200 Subject: [PATCH 10/90] Update meta repository (#2653) Co-authored-by: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> --- global/meta | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global/meta b/global/meta index 72b0730034..e2e3395d07 160000 --- a/global/meta +++ b/global/meta @@ -1 +1 @@ -Subproject commit 72b07300348f3e53903ea07b3e819884c6df5a65 +Subproject commit e2e3395d07d03ef0089747e2ba94557131a75768 From f9c8be0df5bf498cbc9ca3b19d5878045dbbad6e Mon Sep 17 00:00:00 2001 From: Hannes Janott Date: Tue, 8 Oct 2024 10:10:50 +0200 Subject: [PATCH 11/90] migration to set default motion poll method (#2661) * add migration to set default motion poll method * Add method description and check for deleted models. --- global/data/example-data.json | 2 +- global/data/initial-data.json | 2 +- .../0058_fix_motion_poll_default_method.py | 30 +++++++++++++++ ...est_0058_fix_motion_poll_default_method.py | 37 +++++++++++++++++++ 4 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 openslides_backend/migrations/migrations/0058_fix_motion_poll_default_method.py create mode 100644 tests/system/migrations/test_0058_fix_motion_poll_default_method.py diff --git a/global/data/example-data.json b/global/data/example-data.json index ffe64b86d2..4dbe8bed54 100644 --- a/global/data/example-data.json +++ b/global/data/example-data.json @@ -1,5 +1,5 @@ { - "_migration_index": 58, + "_migration_index": 59, "gender":{ "1":{ "id": 1, diff --git a/global/data/initial-data.json b/global/data/initial-data.json index 14aa1328ff..f684b378e0 100644 --- a/global/data/initial-data.json +++ b/global/data/initial-data.json @@ -1,5 +1,5 @@ { - "_migration_index": 58, + "_migration_index": 59, "gender":{ "1":{ "id": 1, diff --git a/openslides_backend/migrations/migrations/0058_fix_motion_poll_default_method.py b/openslides_backend/migrations/migrations/0058_fix_motion_poll_default_method.py new file mode 100644 index 0000000000..abdcf90791 --- /dev/null +++ b/openslides_backend/migrations/migrations/0058_fix_motion_poll_default_method.py @@ -0,0 +1,30 @@ +from datastore.migrations import BaseModelMigration +from datastore.shared.util import fqid_from_collection_and_id +from datastore.writer.core import BaseRequestEvent, RequestUpdateEvent + +from ...shared.filters import And, FilterOperator + + +class Migration(BaseModelMigration): + """ + This migration fills the field 'motion_poll_default_method' for every meeting with 'YNA' mode if it's not set. + This was nessecary due to being added as a default field. + """ + + target_migration_index = 59 + + def migrate_models(self) -> list[BaseRequestEvent]: + meetings_to_fix = self.reader.filter( + "meeting", + And( + FilterOperator("motion_poll_default_method", "=", None), + FilterOperator("meta_deleted", "!=", True), + ), + ) + return [ + RequestUpdateEvent( + fqid_from_collection_and_id("meeting", meeting_id), + {"motion_poll_default_method": "YNA"}, + ) + for meeting_id, meeting in meetings_to_fix.items() + ] diff --git a/tests/system/migrations/test_0058_fix_motion_poll_default_method.py b/tests/system/migrations/test_0058_fix_motion_poll_default_method.py new file mode 100644 index 0000000000..88e1783953 --- /dev/null +++ b/tests/system/migrations/test_0058_fix_motion_poll_default_method.py @@ -0,0 +1,37 @@ +def test_migration_full(write, finalize, assert_model): + write( + { + "type": "create", + "fqid": "meeting/1", + "fields": {"id": 1, "motion_poll_default_method": None}, + }, + { + "type": "create", + "fqid": "meeting/2", + "fields": {"id": 2, "motion_poll_default_method": "YN"}, + }, + { + "type": "create", + "fqid": "meeting/3", + "fields": {"id": 3, "motion_poll_default_method": None}, + }, + { + "type": "delete", + "fqid": "meeting/3", + }, + ) + + finalize("0058_fix_motion_poll_default_method") + + assert_model( + "meeting/1", + {"id": 1, "motion_poll_default_method": "YNA"}, + ) + assert_model( + "meeting/2", + {"id": 2, "motion_poll_default_method": "YN"}, + ) + assert_model( + "meeting/3", + {"id": 3, "meta_deleted": True}, + ) From f8ab039417176946d598e9b9695e1a307b045cad Mon Sep 17 00:00:00 2001 From: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> Date: Tue, 8 Oct 2024 12:02:20 +0200 Subject: [PATCH 12/90] Global enable anonymous (#2660) * Global enable anonymous * Fix meeting.update --- docs/actions/meeting.update.md | 4 +- docs/actions/organization.update.md | 3 +- global/meta | 2 +- .../action/actions/meeting/update.py | 10 ++++- .../action/actions/organization/update.py | 1 + openslides_backend/models/models.py | 1 + tests/system/action/meeting/test_update.py | 42 +++++++++++++++++++ .../system/action/organization/test_update.py | 2 + 8 files changed, 61 insertions(+), 4 deletions(-) diff --git a/docs/actions/meeting.update.md b/docs/actions/meeting.update.md index afce0266ac..753183ff8b 100644 --- a/docs/actions/meeting.update.md +++ b/docs/actions/meeting.update.md @@ -174,7 +174,7 @@ // Group D external_id: string; - enable_anonymous: boolean; + enable_anonymous: boolean custom_translations: JSON; // Group E @@ -202,6 +202,8 @@ If `enable_anonymous` is set, this action will create an anonymous group for the The meetings `anonymous_group_id` may not be used for the `assignment_poll_default_group_ids`, `topic_poll_default_group_ids` and `motion_poll_default_group_ids` fields. +`enable_anonymous` may only be set to true if `enable_anonymous` is set to true in the organization. + ## Permissions - Users with `meeting.can_manage_settings` can modify group A - Users with `user.can_update` can modify group B diff --git a/docs/actions/organization.update.md b/docs/actions/organization.update.md index 0d24439ab3..c49aec75a4 100644 --- a/docs/actions/organization.update.md +++ b/docs/actions/organization.update.md @@ -21,9 +21,10 @@ // Group B enable_electronic_voting: boolean; + enable_chat: boolean; + enable_anonymous: boolean; reset_password_verbose_errors: boolean; limit_of_meetings: int; - enable_chat: boolean; saml_enabled: boolean; saml_login_button_text: string; saml_attr_mapping: JSON; diff --git a/global/meta b/global/meta index e2e3395d07..886d440faa 160000 --- a/global/meta +++ b/global/meta @@ -1 +1 @@ -Subproject commit e2e3395d07d03ef0089747e2ba94557131a75768 +Subproject commit 886d440faa77f27e72f2da8da40f5b1882beb8f9 diff --git a/openslides_backend/action/actions/meeting/update.py b/openslides_backend/action/actions/meeting/update.py index afbc9d9d69..0d4d95438d 100644 --- a/openslides_backend/action/actions/meeting/update.py +++ b/openslides_backend/action/actions/meeting/update.py @@ -235,8 +235,16 @@ def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: ) self.check_locking(instance, set_as_template) organization = self.datastore.get( - ONE_ORGANIZATION_FQID, ["require_duplicate_from"], lock_result=False + ONE_ORGANIZATION_FQID, + ["require_duplicate_from", "enable_anonymous"], + lock_result=False, ) + if instance.get("enable_anonymous") and not organization.get( + "enable_anonymous" + ): + raise ActionException( + "Anonymous users can not be enabled in this organization." + ) if ( organization.get("require_duplicate_from") and set_as_template is not None diff --git a/openslides_backend/action/actions/organization/update.py b/openslides_backend/action/actions/organization/update.py index 0c67c74ecc..c82a1c90fc 100644 --- a/openslides_backend/action/actions/organization/update.py +++ b/openslides_backend/action/actions/organization/update.py @@ -42,6 +42,7 @@ class OrganizationUpdate( group_B_fields = ( "enable_electronic_voting", "enable_chat", + "enable_anonymous", "reset_password_verbose_errors", "limit_of_meetings", "limit_of_users", diff --git a/openslides_backend/models/models.py b/openslides_backend/models/models.py index 15487147d8..0522122701 100644 --- a/openslides_backend/models/models.py +++ b/openslides_backend/models/models.py @@ -37,6 +37,7 @@ class Organization(Model): required=True, constraints={"enum": ["en", "de", "it", "es", "ru", "cs", "fr"]} ) require_duplicate_from = fields.BooleanField() + enable_anonymous = fields.BooleanField() saml_enabled = fields.BooleanField() saml_login_button_text = fields.CharField(default="SAML login") saml_attr_mapping = fields.JSONField() diff --git a/tests/system/action/meeting/test_update.py b/tests/system/action/meeting/test_update.py index a64455b8fb..fcdda7e4eb 100644 --- a/tests/system/action/meeting/test_update.py +++ b/tests/system/action/meeting/test_update.py @@ -12,6 +12,7 @@ class MeetingUpdateActionTest(BaseActionTestCase): def setUp(self) -> None: super().setUp() self.test_models: dict[str, dict[str, Any]] = { + ONE_ORGANIZATION_FQID: {"enable_anonymous": True}, "committee/1": {"name": "test_committee"}, "meeting/1": { "name": "test_name", @@ -622,6 +623,7 @@ def test_update_with_user(self) -> None: """Also tests if the anonymous group is created""" self.set_models( { + ONE_ORGANIZATION_FQID: {"enable_anonymous": True}, "committee/1": {"meeting_ids": [3]}, "meeting/3": { "is_active_in_organization_id": 1, @@ -673,6 +675,46 @@ def test_update_with_user(self) -> None: }, ) + def test_update_anonymous_if_disabled_in_orga(self) -> None: + self.set_models( + { + "committee/1": {"meeting_ids": [3]}, + "meeting/3": { + "is_active_in_organization_id": 1, + "committee_id": 1, + "group_ids": [11], + "admin_group_id": 11, + }, + "group/11": {"meeting_id": 3, "admin_group_for_meeting_id": 3}, + } + ) + response = self.request_json( + [ + { + "action": "meeting.update", + "data": [ + { + "name": "meeting", + "welcome_title": "title", + "welcome_text": "", + "description": "", + "location": "", + "start_time": 1623016800, + "end_time": 1623016800, + "enable_anonymous": True, + "organization_tag_ids": [], + "id": 3, + } + ], + }, + ] + ) + self.assert_status_code(response, 400) + self.assertIn( + "Anonymous users can not be enabled in this organization.", + response.json["message"], + ) + def test_update_set_as_template_true(self) -> None: self.set_models(self.test_models) response = self.request( diff --git a/tests/system/action/organization/test_update.py b/tests/system/action/organization/test_update.py index 4d5d866614..182db03a65 100644 --- a/tests/system/action/organization/test_update.py +++ b/tests/system/action/organization/test_update.py @@ -48,6 +48,7 @@ def test_update(self) -> None: "name": "testtest", "description": "blablabla", "saml_attr_mapping": self.saml_attr_mapping, + "enable_anonymous": True, }, ) self.assert_status_code(response, 200) @@ -57,6 +58,7 @@ def test_update(self) -> None: "name": "testtest", "description": "blablabla", "saml_attr_mapping": self.saml_attr_mapping, + "enable_anonymous": True, }, ) From d73953d2579cd2f0dfa4f53873941190155b13e6 Mon Sep 17 00:00:00 2001 From: "openslides-automation[bot]" <125256978+openslides-automation[bot]@users.noreply.github.com> Date: Tue, 8 Oct 2024 12:38:11 +0200 Subject: [PATCH 13/90] Update meta repository (#2665) Co-authored-by: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> --- global/meta | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global/meta b/global/meta index 886d440faa..6c33dcc24f 160000 --- a/global/meta +++ b/global/meta @@ -1 +1 @@ -Subproject commit 886d440faa77f27e72f2da8da40f5b1882beb8f9 +Subproject commit 6c33dcc24f93862f978eb3f977470596bd5e3f66 From 318cfbe2ffef7f33902c7d678ff4482b78f70a2a Mon Sep 17 00:00:00 2001 From: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> Date: Tue, 8 Oct 2024 12:58:21 +0200 Subject: [PATCH 14/90] Ensure speaker.delete only resets projector countdown with active speakers (#2657) * Ensure speaker.delete only resets projector countdown with active speakers * Style * Switch values --- docs/actions/speaker.delete.md | 1 + .../action/actions/speaker/delete.py | 3 +- tests/system/action/speaker/test_delete.py | 70 +++++++++++++++++++ 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/docs/actions/speaker.delete.md b/docs/actions/speaker.delete.md index 394875e60e..43f35c3be0 100644 --- a/docs/actions/speaker.delete.md +++ b/docs/actions/speaker.delete.md @@ -5,6 +5,7 @@ ## Action Deletes the given speaker. +Resets the projector_countdown if he was speaking. ## Permissions If the `speaker/meeting_user_id` doesn't belong to the request user, he needs `list_of_speakers.can_manage` diff --git a/openslides_backend/action/actions/speaker/delete.py b/openslides_backend/action/actions/speaker/delete.py index 3ac58bc582..89f17f1929 100644 --- a/openslides_backend/action/actions/speaker/delete.py +++ b/openslides_backend/action/actions/speaker/delete.py @@ -65,5 +65,6 @@ def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: and not speaker.get("pause_time") ): self.decrease_structure_level_countdown(self.end_time, speaker) - self.control_los_countdown(speaker["meeting_id"], CountdownCommand.RESET) + if speaker.get("begin_time") and not speaker.get("end_time"): + self.control_los_countdown(speaker["meeting_id"], CountdownCommand.RESET) return super().update_instance(instance) diff --git a/tests/system/action/speaker/test_delete.py b/tests/system/action/speaker/test_delete.py index e588f84da7..a69abc3354 100644 --- a/tests/system/action/speaker/test_delete.py +++ b/tests/system/action/speaker/test_delete.py @@ -1,3 +1,5 @@ +from math import floor +from time import time from typing import Any from openslides_backend.action.mixins.delegation_based_restriction_mixin import ( @@ -399,3 +401,71 @@ def test_with_paused_structure_level_speaker(self) -> None: }, ) assert sllos["remaining_time"] == 18 + + def add_coupled_countdown(self) -> int: + """Returns the current date that was used to set the countdown_time""" + now = floor(time()) + self.set_models( + { + "meeting/1": { + "list_of_speakers_couple_countdown": True, + "list_of_speakers_countdown_id": 75, + }, + "projector_countdown/75": { + "running": True, + "default_time": 200, + "countdown_time": now + 100, + "meeting_id": 1, + }, + "speaker/890": { + "begin_time": now + 100, + }, + } + ) + return now + + def test_delete_update_countdown(self) -> None: + self.set_models(self.permission_test_models) + self.add_coupled_countdown() + response = self.request("speaker.delete", {"id": 890}) + self.assert_status_code(response, 200) + countdown = self.get_model("projector_countdown/75") + assert countdown.get("running") is False + self.assertAlmostEqual(countdown["countdown_time"], 100, delta=200) + + def test_delete_dont_update_countdown(self) -> None: + self.set_models(self.permission_test_models) + self.set_models( + { + "meeting/1": { + "speaker_ids": [890, 891], + "meeting_user_ids": [7, 8], + }, + "user/8": { + "username": "test_username2", + "meeting_user_ids": [8], + "is_active": True, + "default_password": DEFAULT_PASSWORD, + "password": self.auth.hash(DEFAULT_PASSWORD), + }, + "meeting_user/8": {"meeting_id": 1, "user_id": 8, "speaker_ids": [891]}, + "list_of_speakers/23": {"speaker_ids": [890, 891]}, + "speaker/891": { + "meeting_user_id": 8, + "list_of_speakers_id": 23, + "meeting_id": 1, + }, + } + ) + now = self.add_coupled_countdown() + response = self.request("speaker.delete", {"id": 891}) + self.assert_status_code(response, 200) + countdown = self.assert_model_exists( + "projector_countdown/75", + { + "running": True, + "default_time": 200, + "meeting_id": 1, + }, + ) + self.assertAlmostEqual(countdown["countdown_time"], now, delta=200) From 780eda011a78408810819612c23a9a270d27e15d Mon Sep 17 00:00:00 2001 From: "openslides-automation[bot]" <125256978+openslides-automation[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 13:52:30 +0200 Subject: [PATCH 15/90] Update meta repository (main) (#2670) * Update meta repository * Generate models --------- Co-authored-by: bastianjoel <9317999+bastianjoel@users.noreply.github.com> Co-authored-by: Luisa --- global/meta | 2 +- openslides_backend/models/models.py | 90 ++++++++++++++--------------- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/global/meta b/global/meta index 6c33dcc24f..08d0088ebf 160000 --- a/global/meta +++ b/global/meta @@ -1 +1 @@ -Subproject commit 6c33dcc24f93862f978eb3f977470596bd5e3f66 +Subproject commit 08d0088ebf955ebdfda2fe512d9353ae942944c4 diff --git a/openslides_backend/models/models.py b/openslides_backend/models/models.py index 0522122701..a06c12ed6d 100644 --- a/openslides_backend/models/models.py +++ b/openslides_backend/models/models.py @@ -9,7 +9,7 @@ class Organization(Model): collection = "organization" verbose_name = "organization" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) name = fields.CharField() description = fields.HTMLStrictField() legal_notice = fields.TextField() @@ -81,7 +81,7 @@ class User(Model): collection = "user" verbose_name = "user" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) username = fields.CharField(required=True) member_number = fields.CharField() saml_id = fields.CharField( @@ -207,7 +207,7 @@ class Gender(Model): collection = "gender" verbose_name = "gender" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) name = fields.CharField(required=True, constraints={"description": "unique"}) organization_id = fields.OrganizationField( to={"organization": "gender_ids"}, required=True @@ -219,7 +219,7 @@ class OrganizationTag(Model): collection = "organization_tag" verbose_name = "organization tag" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) name = fields.CharField(required=True) color = fields.ColorField(required=True) tagged_ids = fields.GenericRelationListField( @@ -292,7 +292,7 @@ class Committee(Model): collection = "committee" verbose_name = "committee" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) name = fields.CharField(required=True) description = fields.HTMLStrictField() external_id = fields.CharField(constraints={"description": "unique"}) @@ -327,7 +327,7 @@ class Meeting(Model, MeetingModelMixin): collection = "meeting" verbose_name = "meeting" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) external_id = fields.CharField(constraints={"description": "unique in committee"}) welcome_title = fields.CharField(default="Welcome to OpenSlides") welcome_text = fields.HTMLPermissiveField(default="Space for your welcome text.") @@ -934,7 +934,7 @@ class StructureLevel(Model): collection = "structure_level" verbose_name = "structure level" - id = fields.IntegerField(required=True) + id = fields.IntegerField(required=True, constant=True) name = fields.CharField(required=True) color = fields.ColorField() default_time = fields.IntegerField(constraints={"minimum": 0}) @@ -954,7 +954,7 @@ class Group(Model): collection = "group" verbose_name = "group" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) external_id = fields.CharField(constraints={"description": "unique in meeting"}) name = fields.CharField(required=True) permissions = fields.CharArrayField( @@ -1061,7 +1061,7 @@ class PersonalNote(Model): collection = "personal_note" verbose_name = "personal note" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) note = fields.HTMLStrictField() star = fields.BooleanField() meeting_user_id = fields.RelationField( @@ -1082,7 +1082,7 @@ class Tag(Model): collection = "tag" verbose_name = "tag" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) name = fields.CharField(required=True) tagged_ids = fields.GenericRelationListField( to={"agenda_item": "tag_ids", "assignment": "tag_ids", "motion": "tag_ids"}, @@ -1097,7 +1097,7 @@ class AgendaItem(Model, AgendaItemModelMixin): collection = "agenda_item" verbose_name = "agenda item" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) item_number = fields.CharField() comment = fields.CharField() closed = fields.BooleanField(default=False) @@ -1152,7 +1152,7 @@ class ListOfSpeakers(Model): collection = "list_of_speakers" verbose_name = "list of speakers" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) closed = fields.BooleanField(default=False) sequential_number = fields.IntegerField( required=True, @@ -1198,7 +1198,7 @@ class StructureLevelListOfSpeakers(Model): collection = "structure_level_list_of_speakers" verbose_name = "structure level list of speakers" - id = fields.IntegerField(required=True) + id = fields.IntegerField(required=True, constant=True) structure_level_id = fields.RelationField( to={"structure_level": "structure_level_list_of_speakers_ids"}, required=True, @@ -1244,7 +1244,7 @@ class PointOfOrderCategory(Model): collection = "point_of_order_category" verbose_name = "point of order category" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) text = fields.CharField(required=True) rank = fields.IntegerField(required=True) meeting_id = fields.RelationField( @@ -1259,7 +1259,7 @@ class Speaker(Model): collection = "speaker" verbose_name = "speaker" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) begin_time = fields.TimestampField() end_time = fields.TimestampField() pause_time = fields.TimestampField(read_only=True) @@ -1304,7 +1304,7 @@ class Topic(Model): collection = "topic" verbose_name = "topic" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) title = fields.CharField(required=True) text = fields.HTMLPermissiveField() sequential_number = fields.IntegerField( @@ -1351,7 +1351,7 @@ class Motion(Model): collection = "motion" verbose_name = "motion" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) number = fields.CharField() number_value = fields.IntegerField( read_only=True, @@ -1510,7 +1510,7 @@ class MotionSubmitter(Model): collection = "motion_submitter" verbose_name = "motion submitter" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) weight = fields.IntegerField() meeting_user_id = fields.RelationField( to={"meeting_user": "motion_submitter_ids"}, required=True @@ -1530,7 +1530,7 @@ class MotionEditor(Model): collection = "motion_editor" verbose_name = "motion editor" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) weight = fields.IntegerField() meeting_user_id = fields.RelationField( to={"meeting_user": "motion_editor_ids"}, required=True @@ -1550,7 +1550,7 @@ class MotionWorkingGroupSpeaker(Model): collection = "motion_working_group_speaker" verbose_name = "motion working group speaker" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) weight = fields.IntegerField() meeting_user_id = fields.RelationField( to={"meeting_user": "motion_working_group_speaker_ids"}, required=True @@ -1570,7 +1570,7 @@ class MotionComment(Model): collection = "motion_comment" verbose_name = "motion comment" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) comment = fields.HTMLStrictField() motion_id = fields.RelationField( to={"motion": "comment_ids"}, @@ -1593,7 +1593,7 @@ class MotionCommentSection(Model): collection = "motion_comment_section" verbose_name = "motion comment section" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) name = fields.CharField(required=True) weight = fields.IntegerField(default=10000) sequential_number = fields.IntegerField( @@ -1625,7 +1625,7 @@ class MotionCategory(Model): collection = "motion_category" verbose_name = "motion category" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) name = fields.CharField(required=True) prefix = fields.CharField() weight = fields.IntegerField(default=10000) @@ -1658,7 +1658,7 @@ class MotionBlock(Model): collection = "motion_block" verbose_name = "motion block" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) title = fields.CharField(required=True) internal = fields.BooleanField() sequential_number = fields.IntegerField( @@ -1697,7 +1697,7 @@ class MotionChangeRecommendation(Model): collection = "motion_change_recommendation" verbose_name = "motion change recommendation" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) rejected = fields.BooleanField(default=False) internal = fields.BooleanField(default=False) type = fields.CharField( @@ -1724,7 +1724,7 @@ class MotionState(Model): collection = "motion_state" verbose_name = "motion state" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) name = fields.CharField(required=True) weight = fields.IntegerField(required=True) recommendation_label = fields.CharField() @@ -1798,7 +1798,7 @@ class MotionWorkflow(Model): collection = "motion_workflow" verbose_name = "motion workflow" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) name = fields.CharField(required=True) sequential_number = fields.IntegerField( required=True, @@ -1836,7 +1836,7 @@ class MotionStatuteParagraph(Model): collection = "motion_statute_paragraph" verbose_name = "motion statute paragraph" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) title = fields.CharField(required=True) text = fields.HTMLStrictField() weight = fields.IntegerField(default=10000) @@ -1860,7 +1860,7 @@ class Poll(Model, PollModelMixin): collection = "poll" verbose_name = "poll" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) description = fields.TextField() title = fields.CharField(required=True) type = fields.CharField( @@ -1963,7 +1963,7 @@ class Option(Model): collection = "option" verbose_name = "option" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) weight = fields.IntegerField(default=10000) text = fields.HTMLStrictField() yes = fields.DecimalField() @@ -1998,7 +1998,7 @@ class Vote(Model): collection = "vote" verbose_name = "vote" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) weight = fields.DecimalField(constant=True) value = fields.CharField(constant=True) user_token = fields.CharField(required=True, constant=True) @@ -2019,7 +2019,7 @@ class Assignment(Model): collection = "assignment" verbose_name = "assignment" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) title = fields.CharField(required=True) description = fields.HTMLStrictField() open_posts = fields.IntegerField(default=0, constraints={"minimum": 0}) @@ -2078,7 +2078,7 @@ class AssignmentCandidate(Model): collection = "assignment_candidate" verbose_name = "assignment candidate" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) weight = fields.IntegerField(default=10000) assignment_id = fields.RelationField( to={"assignment": "candidate_ids"}, @@ -2098,7 +2098,7 @@ class PollCandidateList(Model): collection = "poll_candidate_list" verbose_name = "poll candidate list" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) poll_candidate_ids = fields.RelationListField( to={"poll_candidate": "poll_candidate_list_id"}, on_delete=fields.OnDelete.CASCADE, @@ -2119,7 +2119,7 @@ class PollCandidate(Model): collection = "poll_candidate" verbose_name = "poll candidate" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) poll_candidate_list_id = fields.RelationField( to={"poll_candidate_list": "poll_candidate_ids"}, required=True, @@ -2137,7 +2137,7 @@ class Mediafile(Model): collection = "mediafile" verbose_name = "mediafile" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) title = fields.CharField( constraints={"description": "Title and parent_id must be unique."} ) @@ -2178,7 +2178,7 @@ class MeetingMediafile(Model): collection = "meeting_mediafile" verbose_name = "meeting mediafile" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) mediafile_id = fields.RelationField( to={"mediafile": "meeting_mediafile_ids"}, required=True ) @@ -2267,7 +2267,7 @@ class Projector(Model): collection = "projector" verbose_name = "projector" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) name = fields.CharField() is_internal = fields.BooleanField(default=False) scale = fields.IntegerField(default=0) @@ -2369,7 +2369,7 @@ class Projection(Model): collection = "projection" verbose_name = "projection" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) options = fields.JSONField() stable = fields.BooleanField(default=False) weight = fields.IntegerField() @@ -2410,7 +2410,7 @@ class ProjectorMessage(Model): collection = "projector_message" verbose_name = "projector message" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) message = fields.HTMLStrictField() projection_ids = fields.RelationListField( to={"projection": "content_object_id"}, @@ -2426,7 +2426,7 @@ class ProjectorCountdown(Model): collection = "projector_countdown" verbose_name = "projector countdown" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) title = fields.CharField(required=True) description = fields.CharField(default="") default_time = fields.IntegerField() @@ -2452,7 +2452,7 @@ class ChatGroup(Model): collection = "chat_group" verbose_name = "chat group" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) name = fields.CharField(required=True) weight = fields.IntegerField(default=10000) chat_message_ids = fields.RelationListField( @@ -2475,7 +2475,7 @@ class ChatMessage(Model): collection = "chat_message" verbose_name = "chat message" - id = fields.IntegerField(constant=True) + id = fields.IntegerField(required=True, constant=True) content = fields.HTMLStrictField(required=True) created = fields.TimestampField(required=True) meeting_user_id = fields.RelationField( @@ -2493,7 +2493,7 @@ class ActionWorker(Model): collection = "action_worker" verbose_name = "action worker" - id = fields.IntegerField() + id = fields.IntegerField(required=True, constant=True) name = fields.CharField(required=True) state = fields.CharField( required=True, constraints={"enum": ["running", "end", "aborted"]} @@ -2507,7 +2507,7 @@ class ImportPreview(Model): collection = "import_preview" verbose_name = "import preview" - id = fields.IntegerField() + id = fields.IntegerField(required=True, constant=True) name = fields.CharField( required=True, constraints={ From 5a2412da1570feee3b9329e051a700e9aea8888d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 15:30:29 +0200 Subject: [PATCH 16/90] Bump black from 24.8.0 to 24.10.0 in /requirements/partial (#2668) Bumps [black](https://github.com/psf/black) from 24.8.0 to 24.10.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/24.8.0...24.10.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/partial/requirements_development.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/partial/requirements_development.txt b/requirements/partial/requirements_development.txt index 66bfd85f31..251fc97ccd 100644 --- a/requirements/partial/requirements_development.txt +++ b/requirements/partial/requirements_development.txt @@ -1,6 +1,6 @@ aiosmtpd==1.4.6 autoflake==2.3.1 -black==24.8.0 +black==24.10.0 debugpy==1.8.6 flake8==7.1.1 isort==5.13.2 From 48ad18f4998489e64552bacd971389721c8ec06a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20B=C3=B6hlke?= Date: Mon, 14 Oct 2024 15:52:29 +0200 Subject: [PATCH 17/90] Change build scripts so that fullstack feature use-case (wiring own libs locally) is optional --- Makefile | 13 ++++++++----- requirements/export_service_commits.sh | 2 +- .../partial/requirements_packaged_services.txt | 2 +- requirements/requirements_development_fullstack.txt | 4 ++++ 4 files changed, 14 insertions(+), 7 deletions(-) create mode 100644 requirements/requirements_development_fullstack.txt diff --git a/Makefile b/Makefile index 2513566beb..e033536741 100644 --- a/Makefile +++ b/Makefile @@ -142,10 +142,13 @@ run-dev-otel run-bash-otel: | start-dev-otel run-dev-attach-otel # Build standalone development container (not usable inside the docker container) build-dev: - rm -rf pip-auth - cp -r ../openslides-auth-service/libraries/pip-auth pip-auth - docker build --file=dev/Dockerfile.dev . --tag=openslides-backend-dev - rm -rf pip-auth + docker build --file=dev/Dockerfile.dev . --target development --tag=openslides-backend-dev + +build-dev-fullstack: + DOCKER_BUILDKIT=1 docker build --file=dev/Dockerfile.dev . --target development-fullstack \ + --build-context pipauth=../openslides-auth-service/libraries/pip-auth \ + --build-context datastore=../openslides-datastore-service \ + --tag=openslides-backend-dev-fullstack rebuild-dev: - docker build --file=dev/Dockerfile.dev . --tag=openslides-backend-dev --no-cache + docker build --file=dev/Dockerfile.dev . --target development --tag=openslides-backend-dev --no-cache diff --git a/requirements/export_service_commits.sh b/requirements/export_service_commits.sh index 8e62af5325..3cc76e86e0 100755 --- a/requirements/export_service_commits.sh +++ b/requirements/export_service_commits.sh @@ -1,3 +1,3 @@ #!/bin/bash export DATASTORE_COMMIT_HASH=46ee759b96507ed62c7dfe95972cbae4a826b2e2 -export AUTH_COMMIT_HASH=c12b89bd2bcefc6868fcc5cd3d57c23d04bd47b0 +export AUTH_COMMIT_HASH=01940817cf3435a7c20b4bcca076317f15f7fe23 diff --git a/requirements/partial/requirements_packaged_services.txt b/requirements/partial/requirements_packaged_services.txt index 3a0770d920..676825fe60 100644 --- a/requirements/partial/requirements_packaged_services.txt +++ b/requirements/partial/requirements_packaged_services.txt @@ -1,2 +1,2 @@ git+https://github.com/OpenSlides/openslides-datastore-service.git@${DATASTORE_COMMIT_HASH} --e /pip-auth +git+https://github.com/kryptance/openslides-auth-service.git@${AUTH_COMMIT_HASH}#egg=os_authlib&subdirectory=libraries/pip-auth diff --git a/requirements/requirements_development_fullstack.txt b/requirements/requirements_development_fullstack.txt new file mode 100644 index 0000000000..b5f3cd3bd8 --- /dev/null +++ b/requirements/requirements_development_fullstack.txt @@ -0,0 +1,4 @@ +-r partial/requirements_production.txt +-r partial/requirements_development.txt +-e /pip-auth +-e /openslides-datastore-service From ba000be21aa8242af4ede331e498083da6f6fb85 Mon Sep 17 00:00:00 2001 From: "openslides-automation[bot]" <125256978+openslides-automation[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 14:06:38 +0000 Subject: [PATCH 18/90] Update meta repository (#2672) Co-authored-by: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> --- global/meta | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global/meta b/global/meta index 08d0088ebf..0e78306465 160000 --- a/global/meta +++ b/global/meta @@ -1 +1 @@ -Subproject commit 08d0088ebf955ebdfda2fe512d9353ae942944c4 +Subproject commit 0e783064656c8521eb4d69f5c1dfbc8add46431b From 732506529a297d1e71f08b129f17a44f09700b56 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 16:25:58 +0200 Subject: [PATCH 19/90] Bump types-redis in /requirements/partial (#2658) Bumps [types-redis](https://github.com/python/typeshed) from 4.6.0.20240903 to 4.6.0.20241004. - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-redis dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/partial/requirements_local_services.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/partial/requirements_local_services.txt b/requirements/partial/requirements_local_services.txt index 00d228d7c9..e3320d1d2b 100644 --- a/requirements/partial/requirements_local_services.txt +++ b/requirements/partial/requirements_local_services.txt @@ -1 +1 @@ -types-redis==4.6.0.20240903 +types-redis==4.6.0.20241004 From f3741829b6e93e14b48c84ffc59c0bcf6b59c1fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20B=C3=B6hlke?= Date: Mon, 14 Oct 2024 17:06:50 +0200 Subject: [PATCH 20/90] Add stages for normal dev deps and fullstack dep resolution (local) --- dev/Dockerfile.dev | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/dev/Dockerfile.dev b/dev/Dockerfile.dev index 7f87d8958d..3cc613229c 100644 --- a/dev/Dockerfile.dev +++ b/dev/Dockerfile.dev @@ -1,14 +1,9 @@ -FROM python:3.10.13-slim-bookworm +FROM python:3.10.13-slim-bookworm as base RUN apt-get update && apt-get install --yes make git curl ncat vim bash-completion mime-support gcc libpq-dev libmagic1 WORKDIR /app -COPY requirements/ requirements/ -COPY pip-auth /pip-auth -ARG REQUIREMENTS_FILE=requirements_development.txt -RUN . requirements/export_service_commits.sh && pip install --no-cache-dir --requirement requirements/$REQUIREMENTS_FILE - COPY dev/.bashrc . COPY dev/cleanup.sh . @@ -43,3 +38,17 @@ ENV DEFAULT_FROM_EMAIL noreply@example.com STOPSIGNAL SIGKILL ENTRYPOINT ["./entrypoint.sh"] CMD exec python -m debugpy --listen 0.0.0.0:5678 openslides_backend + +FROM base as development + +COPY requirements/ requirements/ +ARG REQUIREMENTS_FILE=requirements_development.txt +RUN . requirements/export_service_commits.sh && pip install --no-cache-dir --requirement requirements/$REQUIREMENTS_FILE + +FROM base as development-fullstack + +COPY --from=pipauth / /pip-auth +COPY --from=datastore / /openslides-datastore-service +COPY requirements/ requirements/ +ARG REQUIREMENTS_FILE=requirements_development_fullstack.txt +RUN pip install --no-cache-dir --requirement requirements/$REQUIREMENTS_FILE From 46914e53a5a832601bcbebb94ccbaa251d417a6c Mon Sep 17 00:00:00 2001 From: "openslides-automation[bot]" <125256978+openslides-automation[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2024 09:33:38 +0200 Subject: [PATCH 21/90] Update meta repository (#2673) Co-authored-by: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> --- global/meta | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global/meta b/global/meta index 0e78306465..2a34d9d4c0 160000 --- a/global/meta +++ b/global/meta @@ -1 +1 @@ -Subproject commit 0e783064656c8521eb4d69f5c1dfbc8add46431b +Subproject commit 2a34d9d4c0698de013854ec6b64fed3dac640568 From 990bf6c4facbf72d1e239079e6d1d6ec6fcec488 Mon Sep 17 00:00:00 2001 From: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> Date: Tue, 15 Oct 2024 12:38:36 +0200 Subject: [PATCH 22/90] Fix assignment_candidate cascade deletion error (#2662) --- docs/actions/assignment_candidate.delete.md | 2 +- .../actions/assignment_candidate/delete.py | 33 ++++++++++--------- tests/system/action/assignment/test_delete.py | 4 +++ .../system/action/user/test_merge_together.py | 29 ++++++++++++++++ 4 files changed, 51 insertions(+), 17 deletions(-) diff --git a/docs/actions/assignment_candidate.delete.md b/docs/actions/assignment_candidate.delete.md index c07333ed25..2f6c794934 100644 --- a/docs/actions/assignment_candidate.delete.md +++ b/docs/actions/assignment_candidate.delete.md @@ -4,7 +4,7 @@ ``` ## Action -Deletes an assignment candidate for the assignment. It is forbidden to remove a candidate from a finished assignment. +Deletes an assignment candidate for the assignment. It is forbidden to remove a candidate from a finished assignment if the action is called externally. ## Permissions If the `assignment_candidate/user_id` is equal to the request user id, the user needs `assignment.can_nominate_self`, else the user needs `assignment.can_nominate_other`. diff --git a/openslides_backend/action/actions/assignment_candidate/delete.py b/openslides_backend/action/actions/assignment_candidate/delete.py index 505b1292eb..3320c364b2 100644 --- a/openslides_backend/action/actions/assignment_candidate/delete.py +++ b/openslides_backend/action/actions/assignment_candidate/delete.py @@ -22,21 +22,22 @@ class AssignmentCandidateDelete(PermissionMixin, DeleteAction): def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: instance = super().update_instance(instance) - assignment_candidate = self.datastore.get( - fqid_from_collection_and_id(self.model.collection, instance["id"]), - mapped_fields=["assignment_id"], - ) - assignment = self.datastore.get( - fqid_from_collection_and_id( - "assignment", assignment_candidate["assignment_id"] - ), - mapped_fields=["phase", "meeting_id"], - lock_result=False, - ) - if assignment.get("phase") == "finished" and not self.is_meeting_deleted( - assignment.get("meeting_id", 0) - ): - raise ActionException( - "It is not permitted to remove a candidate from a finished assignment!" + if not self.internal: + assignment_candidate = self.datastore.get( + fqid_from_collection_and_id(self.model.collection, instance["id"]), + mapped_fields=["assignment_id"], ) + assignment = self.datastore.get( + fqid_from_collection_and_id( + "assignment", assignment_candidate["assignment_id"] + ), + mapped_fields=["phase", "meeting_id"], + lock_result=False, + ) + if assignment.get("phase") == "finished" and not self.is_meeting_deleted( + assignment.get("meeting_id", 0) + ): + raise ActionException( + "It is not permitted to remove a candidate from a finished assignment!" + ) return instance diff --git a/tests/system/action/assignment/test_delete.py b/tests/system/action/assignment/test_delete.py index 0d4bc7c404..1f25066a4d 100644 --- a/tests/system/action/assignment/test_delete.py +++ b/tests/system/action/assignment/test_delete.py @@ -26,6 +26,8 @@ def test_delete_correct_cascading(self) -> None: "agenda_item_id": 333, "projection_ids": [1], "meeting_id": 110, + "phase": "finished", + "candidate_ids": [1111], }, "list_of_speakers/222": { "closed": False, @@ -46,6 +48,7 @@ def test_delete_correct_cascading(self) -> None: "current_projection_ids": [1], "meeting_id": 110, }, + "assignment_candidate/1111": {"assignment_id": 111, "meeting_id": 110}, } ) response = self.request("assignment.delete", {"id": 111}) @@ -54,6 +57,7 @@ def test_delete_correct_cascading(self) -> None: self.assert_model_deleted("agenda_item/333") self.assert_model_deleted("list_of_speakers/222") self.assert_model_deleted("projection/1") + self.assert_model_deleted("assignment_candidate/1111") def test_delete_wrong_id(self) -> None: self.set_models( diff --git a/tests/system/action/user/test_merge_together.py b/tests/system/action/user/test_merge_together.py index af2159c389..1c4685b0fc 100644 --- a/tests/system/action/user/test_merge_together.py +++ b/tests/system/action/user/test_merge_together.py @@ -1431,6 +1431,35 @@ def test_merge_with_assignment_candidates(self) -> None: for id_ in range(2, 10): self.assert_history_information(f"assignment/{id_}", ["Candidates merged"]) + def test_merge_with_assignment_candidates_in_finished_assignment(self) -> None: + self.set_models( + { + "meeting/1": { + "assignment_ids": [11], + "assignment_candidate_ids": [112, 114], + }, + "assignment/11": { + "meeting_id": 1, + "phase": "finished", + "candidate_ids": [112, 114], + }, + "assignment_candidate/112": { + "meeting_id": 1, + "assignment_id": 11, + "meeting_user_id": 12, + }, + "assignment_candidate/114": { + "meeting_id": 1, + "assignment_id": 11, + "meeting_user_id": 14, + }, + "meeting_user/12": {"assignment_candidate_ids": [112]}, + "meeting_user/14": {"assignment_candidate_ids": [114]}, + } + ) + response = self.request("user.merge_together", {"id": 2, "user_ids": [4]}) + self.assert_status_code(response, 200) + def test_merge_with_motion_working_group_speakers(self) -> None: self.base_assignment_or_motion_model_test( "motion", "motion_working_group_speaker" From 4733846313e451adb02e358048e0ac7769362f76 Mon Sep 17 00:00:00 2001 From: Hannes Janott Date: Tue, 15 Oct 2024 16:00:01 +0200 Subject: [PATCH 23/90] remove statute (amendments) (#2506) * removed statutes * add migration and tests * cascading deletion with MigrationDeletionMixin * add MigrationDeletionSchema --- docs/Actions-Overview.md | 4 - docs/actions/meeting.create.md | 2 +- docs/actions/meeting.update.md | 3 - docs/actions/motion.create.md | 26 +- docs/actions/motion.create_forwarded.md | 2 +- .../motion_statute_paragraph.create.md | 15 - .../motion_statute_paragraph.delete.md | 10 - docs/actions/motion_statute_paragraph.sort.md | 13 - .../motion_statute_paragraph.update.md | 17 - docs/actions/motion_workflow.delete.md | 2 +- docs/actions/poll.create.md | 2 +- global/data/example-data.json | 6 +- global/data/initial-data.json | 2 +- global/meta | 2 +- openslides_backend/action/actions/__init__.py | 1 - .../action/actions/meeting/create.py | 1 - .../action/actions/meeting/update.py | 3 - .../action/actions/motion/create.py | 4 - .../action/actions/motion/create_base.py | 4 - .../action/actions/motion/create_forwarded.py | 7 +- .../motion/payload_validation_mixin.py | 14 +- .../motion_statute_paragraph/__init__.py | 2 - .../create_update_delete.py | 31 - .../actions/motion_statute_paragraph/sort.py | 32 - .../action/actions/motion_workflow/create.py | 2 - .../action/actions/motion_workflow/delete.py | 7 - .../migrations/0059_remove_statutes.py | 348 +++++ .../migrations/mixins/deletion_mixin.py | 270 ++++ openslides_backend/models/models.py | 39 - tests/system/action/committee/test_import.py | 2 - .../action/committee/test_json_upload.py | 4 - tests/system/action/meeting/test_clone.py | 5 - tests/system/action/meeting/test_create.py | 2 - tests/system/action/meeting/test_import.py | 40 +- tests/system/action/motion/test_create.py | 51 - .../motion/test_create_statute_amendment.py | 114 -- .../test_update.py | 3 - .../motion_statute_paragraph/__init__.py | 0 .../motion_statute_paragraph/test_create.py | 52 - .../motion_statute_paragraph/test_delete.py | 49 - .../motion_statute_paragraph/test_sort.py | 115 -- .../motion_statute_paragraph/test_update.py | 67 - .../action/motion_workflow/test_delete.py | 9 +- tests/system/action/test_action_worker.py | 10 +- .../system/action/user/test_merge_together.py | 4 - .../migrations/test_0059_remove_statutes.py | 1368 +++++++++++++++++ tests/system/presenter/test_check_database.py | 9 - .../presenter/test_check_database_all.py | 9 - tests/system/presenter/test_export_meeting.py | 1 - 49 files changed, 2044 insertions(+), 741 deletions(-) delete mode 100644 docs/actions/motion_statute_paragraph.create.md delete mode 100644 docs/actions/motion_statute_paragraph.delete.md delete mode 100644 docs/actions/motion_statute_paragraph.sort.md delete mode 100644 docs/actions/motion_statute_paragraph.update.md delete mode 100644 openslides_backend/action/actions/motion_statute_paragraph/__init__.py delete mode 100644 openslides_backend/action/actions/motion_statute_paragraph/create_update_delete.py delete mode 100644 openslides_backend/action/actions/motion_statute_paragraph/sort.py create mode 100644 openslides_backend/migrations/migrations/0059_remove_statutes.py create mode 100644 openslides_backend/migrations/mixins/deletion_mixin.py delete mode 100644 tests/system/action/motion/test_create_statute_amendment.py delete mode 100644 tests/system/action/motion_statute_paragraph/__init__.py delete mode 100644 tests/system/action/motion_statute_paragraph/test_create.py delete mode 100644 tests/system/action/motion_statute_paragraph/test_delete.py delete mode 100644 tests/system/action/motion_statute_paragraph/test_sort.py delete mode 100644 tests/system/action/motion_statute_paragraph/test_update.py create mode 100644 tests/system/migrations/test_0059_remove_statutes.py diff --git a/docs/Actions-Overview.md b/docs/Actions-Overview.md index bed33a0fbf..0ac0f5b718 100644 --- a/docs/Actions-Overview.md +++ b/docs/Actions-Overview.md @@ -113,10 +113,6 @@ A more general format description see in [Action-Service](https://github.com/Ope - [motion_comment_section.delete](actions/motion_comment_section.delete.md) - [motion_comment_section.sort](actions/motion_comment_section.sort.md) - [motion_comment_section.update](actions/motion_comment_section.update.md) -- [motion_statute_paragraph.create](actions/motion_statute_paragraph.create.md) -- [motion_statute_paragraph.delete](actions/motion_statute_paragraph.delete.md) -- [motion_statute_paragraph.sort](actions/motion_statute_paragraph.sort.md) -- [motion_statute_paragraph.update](actions/motion_statute_paragraph.update.md) - [motion_submitter.create](actions/motion_submitter.create.md) - [motion_submitter.delete](actions/motion_submitter.delete.md) - [motion_submitter.sort](actions/motion_submitter.sort.md) diff --git a/docs/actions/meeting.create.md b/docs/actions/meeting.create.md index 70b2aa1bea..6386c9a33d 100644 --- a/docs/actions/meeting.create.md +++ b/docs/actions/meeting.create.md @@ -27,7 +27,7 @@ When creating a meeting the following objects have to be created, too: - Groups: `Default`, `Admin`, `Delegates`, `Staff`, `Committees`. The first one is set as `meeting/default_group_id`, the second one as `meeting/admin_group_id`. The permissions can be found in the [initial-data.json](https://github.com/OpenSlides/openslides-backend/tree/main/global/data/initial-data.json)). - Projector: One projector named `"Default projector"` must be created and set as `meeting/reference_projector_id`. - All default projectors (`meeting/default_projector_*_ids`, see `models.yml`) must be set to the one projector -- Motion workflow and states: Create one workflow `"simple workflow"` which is set as `meeting/motions_default_workflow_id`, `meeting/motions_default_amendment_workflow_id` and `meeting/motions_default_statute_amendment_workflow_id`. Create four states (analog as in the [initial-data.json](https://github.com/OpenSlides/openslides-backend/tree/main/global/data/initial-data.json)). +- Motion workflow and states: Create one workflow `"simple workflow"` which is set as `meeting/motions_default_workflow_id` and `meeting/motions_default_amendment_workflow_id`. Create four states (analog as in the [initial-data.json](https://github.com/OpenSlides/openslides-backend/tree/main/global/data/initial-data.json)). - Two countdowns are created and set as `meeting/list_of_speakers_countdown` (name: "List of speakers countdown") and `meeting/voting_countdown` (name: "Voting countdown"). If `user_ids` are given, it must be checked that it is a subset of `committee/user_ids`. Each user is added to the meeting by being added to the default group. diff --git a/docs/actions/meeting.update.md b/docs/actions/meeting.update.md index 753183ff8b..6bffd3dc45 100644 --- a/docs/actions/meeting.update.md +++ b/docs/actions/meeting.update.md @@ -70,7 +70,6 @@ motions_default_workflow_id: Id; motions_default_amendment_workflow_id: Id; - motions_default_statute_amendment_workflow_id: Id; motions_preamble: string; motions_default_line_numbering: string; motions_line_length: number; @@ -83,13 +82,11 @@ motions_show_sequential_number: boolean; motions_recommendations_by: string; motions_block_slide_columns: number; - motions_statute_recommendations_by: string; motions_recommendation_text_mode: string; motions_default_sorting: string; motions_number_type: string; motions_number_min_digits: number; motions_number_with_blank: boolean; - motions_statutes_enabled: boolean; motions_amendments_enabled: boolean; motions_amendments_in_main_list: boolean; motions_amendments_of_amendments: boolean; diff --git a/docs/actions/motion.create.md b/docs/actions/motion.create.md index f4300f819c..703c2feaae 100644 --- a/docs/actions/motion.create.md +++ b/docs/actions/motion.create.md @@ -22,7 +22,6 @@ [paragraph_number: number]: HTML; }; // JSON Field lead_motion_id: Id; - statute_paragraph_id: Id; reason: HTML; // is required, if special settings are set // Optional special fields, see notes below @@ -42,41 +41,35 @@ ## Action Creates a new motion. -First, the type of the motion is identified by the values of `lead_motion_id`, `statute_paragraph_id`: +The motion is an amendment to another motion if `lead_motion_id` is given. Otherwise it is a normal motion. -- A normal motion: None of the fields are given. -- An amendment: `lead_motion_id` is given. -- A statute amendment: `statute_paragraph_id` is given. - -If `lead_motion_id` and `statute_paragraph_id` is given, it must result in an error. This is the logic for other fields depending on the motion type: +This is the logic for other fields depending on the motion type: - normal motion: - `text` required - error, if `amendment_paragraph` is given - amendment: - `text` XOR `amendment_paragraph` required -- statute amendment: - - `text` required - - error, if `amendment_paragraph` is given + - error otherwise -`reason` is independent must be given, if `meeting/motions_reason_required` is true. +`reason` must independently of the above be given, if `meeting/motions_reason_required` is true. There are some fields that need special attention: -- `workflow_id`: If it is given, the motion's state is set to the workflow's first state. The workflow must be from the same meeting. If the field is not given, one of the three default (`meeting/motions_default_workflow_id`, `meeting/motions_default_amendment_workflow_id` or `meeting/motions_default_statute_amendment_workflow_id`) workflows is used depending on the type of the motion to create. +- `workflow_id`: If it is given, the motion's state is set to the workflow's first state. The workflow must be from the same meeting. If the field is not given, one of the three default (`meeting/motions_default_workflow_id` or `meeting/motions_default_amendment_workflow_id`) workflows is used depending on the type of the motion to create. - `submitter_ids`: These are **user ids** and not ids of the `submitter` model. If nothing is given (`[]`), the request user's id is used. For each id in the list a `motion_submitter` model is created. The weight must be set to the order of the given list. - `agenda_*`: See [Agenda](https://github.com/OpenSlides/OpenSlides/wiki/Agenda#additional-fields-during-creation-of-agenda-content-objects). -Another things to do when creating a motions: +Other things to do when creating motions: - Set the field `sequential_number`: It is the `max+1` of `sequential_number` of all motions in the same meeting. If there are no other motions in this meeting (e.g. this is the first one), it gets 1. - Set timestamps: - always set `last_modified` and `created` to the current timestamp - if the state pointed to by `first_state_id` of the given workflow has the flag `set_workflow_timestamp` set, also set `workflow_timestamp`to the current timestamp. - Field `number`: Attention, it is a string, even if the field is named `number`. Note that the `number` must be unique within the meeting if it is set (so all numbers with length > 0 are unique). See the next paragraph how to get a value for `number`. -### Determinate a value for `number` +### Determine a value for `number` This is the procedure to determine what to set for the field `number`: * If `number` in the payload is a string with a length > 0, set it as the number and stop, but raise an error, if it exists. - * if `meeting/motions_number_type` == `"manually"` or not `motion.state.set_number`: Stop. We should not set the number automatically + * If `meeting/motions_number_type` == `"manually"` or not `motion.state.set_number`: stop. We should not set the number automatically * A _prefix_ is created: * If the motion is an amendment (it has a lead motion), the prefix is: ``` @@ -121,7 +114,7 @@ This is the procedure to determine what to set for the field `number`: 2) Create a motion without a category. It gets the number `001`. Set `meeting/motions_number_min_digits=1`. Create a plain motion. It must get the number `2`. `meeting/motions_number_type="per_category"`, `meeting/motions_number_min_digits=3`, `meeting/motions_number_with_blank=true`, `meeting/motions_amendments_prefix="X-"`. Create a category: `{name: "A", prefix: "A"}`. Make sure the state the motions get has `set_number=true`. -1) Create a motion in category A. It must get `A 001`. Create two amendments (motions wiuth `lead_motion_id` set to the id of `A 001`). The numbers are `A 001 X-001` and `A 001 X-002`. +1) Create a motion in category A. It must get `A 001`. Create two amendments (motions with `lead_motion_id` set to the id of `A 001`). The numbers are `A 001 X-001` and `A 001 X-002`. 2) Do 1) again, but with `meeting/motions_number_with_blank=false` and `meeting/motions_number_min_digits=1`. The numbers are `A1`, `A1X-1`, `A1X-2`. 3) Do 1) again, but set `meeting/motions_number_with_blank=false` and `meeting/motions_number_min_digits=1` after creating the first lead motion. The numbers are `A 001`, `A 001X-1`, `A 001X-2`. 4) Do 1) again. Create a new motion without an identifier and no `lead_motion_id`. It gets the number `002`. @@ -142,7 +135,6 @@ If the request user does not have `motion.can_manage`, the fields in the payload - `lead_motion_id` - `amendment_paragraph` - `category_id` -- `statute_paragraph_id` - `workflow_id` If `lead_motion_id` is given and `category_id` is empty, the value of `category_id` is set to the value of the lead motion. \ No newline at end of file diff --git a/docs/actions/motion.create_forwarded.md b/docs/actions/motion.create_forwarded.md index 7a5e5680b7..73198cd389 100644 --- a/docs/actions/motion.create_forwarded.md +++ b/docs/actions/motion.create_forwarded.md @@ -43,5 +43,5 @@ The request user needs `motion.can_forward` in the source meeting. `motion.can_m ### Exceptions -Although they would have meaningful title, text and reason, motions with a "parent", namely amendments (`lead_motion_id`) and statute amendments (`statute_paragraph_id`), should not be forwarded. The backend should throws an error when an amendment was forwarded. +Although they would have meaningful title, text and reason, amendment motions (motions with a field `lead_motion_id`), should not be forwarded directly. The backend should throw an error if an amendment was requested to be forwarded. The client does not offer to forward an amendment. diff --git a/docs/actions/motion_statute_paragraph.create.md b/docs/actions/motion_statute_paragraph.create.md deleted file mode 100644 index ef10dc11ee..0000000000 --- a/docs/actions/motion_statute_paragraph.create.md +++ /dev/null @@ -1,15 +0,0 @@ -## Payload -``` -{ -// Required - title: string; - text: HTML; - meeting_id: Id; -} -``` - -## Action -Creates a new statute paragraph. The `weight` must be set to `max+1` of all statute paragraphs of the meeting. - -## Permissions -The request user needs `motion.can_manage`. diff --git a/docs/actions/motion_statute_paragraph.delete.md b/docs/actions/motion_statute_paragraph.delete.md deleted file mode 100644 index a006f877c0..0000000000 --- a/docs/actions/motion_statute_paragraph.delete.md +++ /dev/null @@ -1,10 +0,0 @@ -## Payload -``` -{ id: Id; } -``` - -## Action -Deletes the statute paragraph. - -## Permissions -The request user needs `motion.can_manage`. diff --git a/docs/actions/motion_statute_paragraph.sort.md b/docs/actions/motion_statute_paragraph.sort.md deleted file mode 100644 index c9e3c601db..0000000000 --- a/docs/actions/motion_statute_paragraph.sort.md +++ /dev/null @@ -1,13 +0,0 @@ -## Payload -``` -{ - meeting_id: Id; - statute_paragraph_ids: Id[]; -} -``` - -## Action -All `statute_paragraph_ids` of the meeting must be given in the new order. Sets the `weight` field accordingly. - -## Permissions -The request user needs `motion.can_manage`. diff --git a/docs/actions/motion_statute_paragraph.update.md b/docs/actions/motion_statute_paragraph.update.md deleted file mode 100644 index 207c586468..0000000000 --- a/docs/actions/motion_statute_paragraph.update.md +++ /dev/null @@ -1,17 +0,0 @@ -## Payload -``` -{ -// Required - id: Id; - -// Optional - title: string; - text: HTML; -} -``` - -## Action -Updates the statute paragraph. - -## Permissions -The request user needs `motion.can_manage`. diff --git a/docs/actions/motion_workflow.delete.md b/docs/actions/motion_workflow.delete.md index d5ec60366b..1dbb085972 100644 --- a/docs/actions/motion_workflow.delete.md +++ b/docs/actions/motion_workflow.delete.md @@ -4,7 +4,7 @@ ``` ## Action -Deletes the motion workflow and all linked states. If the workflow is set as an default workflow for the meeting (`meeting/motions_default_workflow_id`, `meeting/motions_default_amendment_workflow_id` or `meeting/motions_default_statute_amendment_workflow_id`), an error must be returned. This implies, that always one workflow has to exists. +Deletes the motion workflow and all linked states. If the workflow is set as a default workflow for the meeting (`meeting/motions_default_workflow_id` or `meeting/motions_default_amendment_workflow_id`), an error must be returned. This implies, that always one workflow has to exists. If there exists a motion which is in the workflow, the deletion has to fail. diff --git a/docs/actions/poll.create.md b/docs/actions/poll.create.md index 04e57fd358..543194f794 100644 --- a/docs/actions/poll.create.md +++ b/docs/actions/poll.create.md @@ -56,7 +56,7 @@ Payload: ## Action If an analog poll with votes is given, the state is set to `finished` if at least one vote value is given. if `publish_immediately` is true and some vote value is given, the state is set to `published`. All options given are created as instances of the `option` model. If some options have values (for analog polls), `vote` objects have to be created, one for each option and vote value (`Y`, `N`, `A`). -The options must be unique in the way that each non-empty `text` and non-empty `content_object_id` can only exists once. The `option/weight` has to be set in the order the options are given in the payload. A global option has to be created. +The options must be unique in the way that each non-empty `text` and non-empty `content_object_id` can only exist once. The `option/weight` has to be set in the order the options are given in the payload. A global option has to be created. If the `type` is `pseudoanonymous`, `is_pseudoanonymized` has to be set to `true`. diff --git a/global/data/example-data.json b/global/data/example-data.json index 4dbe8bed54..74e5a81fb0 100644 --- a/global/data/example-data.json +++ b/global/data/example-data.json @@ -1,5 +1,5 @@ { - "_migration_index": 59, + "_migration_index": 60, "gender":{ "1":{ "id": 1, @@ -314,7 +314,6 @@ "list_of_speakers_present_users_only": false, "motions_default_workflow_id": 1, "motions_default_amendment_workflow_id": 1, - "motions_default_statute_amendment_workflow_id": 1, "motions_preamble": "The assembly may decide:", "motions_default_line_numbering": "outside", "motions_line_length": 85, @@ -327,13 +326,11 @@ "motions_show_sequential_number": true, "motions_reason_required": false, "motions_recommendations_by": "ABK", - "motions_statute_recommendations_by": "", "motions_recommendation_text_mode": "diff", "motions_default_sorting": "number", "motions_number_type": "per_category", "motions_number_min_digits": 2, "motions_number_with_blank": false, - "motions_statutes_enabled": true, "motions_amendments_enabled": true, "motions_amendments_in_main_list": true, "motions_amendments_of_amendments": true, @@ -1892,7 +1889,6 @@ "first_state_id": 1, "default_workflow_meeting_id": 1, "default_amendment_workflow_meeting_id": 1, - "default_statute_amendment_workflow_meeting_id": 1, "meeting_id": 1 }, "2": { diff --git a/global/data/initial-data.json b/global/data/initial-data.json index f684b378e0..4760ca9bec 100644 --- a/global/data/initial-data.json +++ b/global/data/initial-data.json @@ -1,5 +1,5 @@ { - "_migration_index": 59, + "_migration_index": 60, "gender":{ "1":{ "id": 1, diff --git a/global/meta b/global/meta index 2a34d9d4c0..85d328539b 160000 --- a/global/meta +++ b/global/meta @@ -1 +1 @@ -Subproject commit 2a34d9d4c0698de013854ec6b64fed3dac640568 +Subproject commit 85d328539bc82678973d83e40e5793d1b2a852c5 diff --git a/openslides_backend/action/actions/__init__.py b/openslides_backend/action/actions/__init__.py index 1c2636071f..98bf124578 100644 --- a/openslides_backend/action/actions/__init__.py +++ b/openslides_backend/action/actions/__init__.py @@ -27,7 +27,6 @@ def prepare_actions_map() -> None: motion_comment_section, motion_editor, motion_state, - motion_statute_paragraph, motion_submitter, motion_workflow, motion_working_group_speaker, diff --git a/openslides_backend/action/actions/meeting/create.py b/openslides_backend/action/actions/meeting/create.py index 027fd6b5a5..b882c92eba 100644 --- a/openslides_backend/action/actions/meeting/create.py +++ b/openslides_backend/action/actions/meeting/create.py @@ -231,7 +231,6 @@ def get_dependent_action_data( "name": _("Simple Workflow"), "default_workflow_meeting_id": instance["id"], "default_amendment_workflow_meeting_id": instance["id"], - "default_statute_amendment_workflow_meeting_id": instance["id"], "meeting_id": instance["id"], } ] diff --git a/openslides_backend/action/actions/meeting/update.py b/openslides_backend/action/actions/meeting/update.py index 0d4d95438d..27528d0020 100644 --- a/openslides_backend/action/actions/meeting/update.py +++ b/openslides_backend/action/actions/meeting/update.py @@ -94,7 +94,6 @@ "list_of_speakers_intervention_time", "motions_default_workflow_id", "motions_default_amendment_workflow_id", - "motions_default_statute_amendment_workflow_id", "motions_preamble", "motions_default_line_numbering", "motions_line_length", @@ -108,13 +107,11 @@ "motions_show_sequential_number", "motions_recommendations_by", "motions_block_slide_columns", - "motions_statute_recommendations_by", "motions_recommendation_text_mode", "motions_default_sorting", "motions_number_type", "motions_number_min_digits", "motions_number_with_blank", - "motions_statutes_enabled", "motions_amendments_enabled", "motions_amendments_in_main_list", "motions_amendments_of_amendments", diff --git a/openslides_backend/action/actions/motion/create.py b/openslides_backend/action/actions/motion/create.py index 9390a3937f..2a7128b83c 100644 --- a/openslides_backend/action/actions/motion/create.py +++ b/openslides_backend/action/actions/motion/create.py @@ -47,7 +47,6 @@ class MotionCreate( "tag_ids", "text", "lead_motion_id", - "statute_paragraph_id", "reason", "amendment_paragraphs", ], @@ -80,7 +79,6 @@ def prefetch(self, action_data: ActionData) -> None: "id", "motions_default_workflow_id", "motions_default_amendment_workflow_id", - "motions_default_statute_amendment_workflow_id", "motions_reason_required", "motion_submitter_ids", "motions_number_type", @@ -124,7 +122,6 @@ def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: [ "motions_default_workflow_id", "motions_default_amendment_workflow_id", - "motions_default_statute_amendment_workflow_id", ], ) @@ -172,7 +169,6 @@ def check_permissions(self, instance: dict[str, Any]) -> None: "lead_motion_id", "amendment_paragraphs", "category_id", - "statute_paragraph_id", "workflow_id", "id", "meeting_id", diff --git a/openslides_backend/action/actions/motion/create_base.py b/openslides_backend/action/actions/motion/create_base.py index 31bdbf2f83..b2f012e3f1 100644 --- a/openslides_backend/action/actions/motion/create_base.py +++ b/openslides_backend/action/actions/motion/create_base.py @@ -36,10 +36,6 @@ def set_state_from_workflow( if workflow_id is None: if instance.get("lead_motion_id"): workflow_id = meeting.get("motions_default_amendment_workflow_id") - elif instance.get("statute_paragraph_id"): - workflow_id = meeting.get( - "motions_default_statute_amendment_workflow_id" - ) else: workflow_id = meeting.get("motions_default_workflow_id") if workflow_id: diff --git a/openslides_backend/action/actions/motion/create_forwarded.py b/openslides_backend/action/actions/motion/create_forwarded.py index 247bb1b91b..7e85376f68 100644 --- a/openslides_backend/action/actions/motion/create_forwarded.py +++ b/openslides_backend/action/actions/motion/create_forwarded.py @@ -71,7 +71,6 @@ def prefetch(self, action_data: ActionData) -> None: [ "meeting_id", "lead_motion_id", - "statute_paragraph_id", "state_id", "all_origin_ids", "derived_motion_ids", @@ -205,13 +204,13 @@ def check_permissions(self, instance: dict[str, Any]) -> None: msg += f" Missing permission: {perm_origin}" raise PermissionDenied(msg) - # check if origin motion is amendment or statute_amendment + # check if origin motion is amendment origin = self.datastore.get( fqid_from_collection_and_id(self.model.collection, instance["origin_id"]), - ["lead_motion_id", "statute_paragraph_id"], + ["lead_motion_id"], lock_result=False, ) - if origin.get("lead_motion_id") or origin.get("statute_paragraph_id"): + if origin.get("lead_motion_id"): msg = "Amendments cannot be forwarded." raise PermissionDenied(msg) diff --git a/openslides_backend/action/actions/motion/payload_validation_mixin.py b/openslides_backend/action/actions/motion/payload_validation_mixin.py index 7299981343..3a7f621aa4 100644 --- a/openslides_backend/action/actions/motion/payload_validation_mixin.py +++ b/openslides_backend/action/actions/motion/payload_validation_mixin.py @@ -110,14 +110,7 @@ def _create_conduct_before_checks( {"type": MotionErrorType.TITLE, "message": "Title is required"} ) if instance.get("lead_motion_id"): - if instance.get("statute_paragraph_id"): - errors.append( - { - "type": MotionErrorType.MOTION_TYPE, - "message": "You can't give both of lead_motion_id and statute_paragraph_id.", - } - ) - elif not instance.get("text") and not instance.get("amendment_paragraphs"): + if not instance.get("text") and not instance.get("amendment_paragraphs"): errors.append( { "type": MotionErrorType.TEXT, @@ -157,17 +150,12 @@ def _create_conduct_after_checks( [ "motions_default_workflow_id", "motions_default_amendment_workflow_id", - "motions_default_statute_amendment_workflow_id", ], ) workflow_id = instance.get("workflow_id", None) if workflow_id is None: if instance.get("lead_motion_id"): workflow_id = meeting.get("motions_default_amendment_workflow_id") - elif instance.get("statute_paragraph_id"): - workflow_id = meeting.get( - "motions_default_statute_amendment_workflow_id" - ) else: workflow_id = meeting.get("motions_default_workflow_id") if not workflow_id: diff --git a/openslides_backend/action/actions/motion_statute_paragraph/__init__.py b/openslides_backend/action/actions/motion_statute_paragraph/__init__.py deleted file mode 100644 index f0d93aa414..0000000000 --- a/openslides_backend/action/actions/motion_statute_paragraph/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from . import create_update_delete # noqa -from . import sort # noqa diff --git a/openslides_backend/action/actions/motion_statute_paragraph/create_update_delete.py b/openslides_backend/action/actions/motion_statute_paragraph/create_update_delete.py deleted file mode 100644 index d8877b108e..0000000000 --- a/openslides_backend/action/actions/motion_statute_paragraph/create_update_delete.py +++ /dev/null @@ -1,31 +0,0 @@ -from ....models.models import MotionStatuteParagraph -from ....permissions.permissions import Permissions -from ...action_set import ActionSet -from ...generics.create import CreateAction -from ...mixins.sequential_numbers_mixin import SequentialNumbersMixin -from ...util.default_schema import DefaultSchema -from ...util.register import register_action_set - - -class MotionStatuteParagraphCreate(SequentialNumbersMixin, CreateAction): - pass - - -@register_action_set("motion_statute_paragraph") -class MotionStatuteParagraphActionSet(ActionSet): - """ - Actions to create, update and delete motion statute paragraph. - """ - - model = MotionStatuteParagraph() - create_schema = DefaultSchema(MotionStatuteParagraph()).get_create_schema( - required_properties=["meeting_id", "title", "text"], - optional_properties=[], - ) - update_schema = DefaultSchema(MotionStatuteParagraph()).get_update_schema( - optional_properties=["title", "text"] - ) - delete_schema = DefaultSchema(MotionStatuteParagraph()).get_delete_schema() - permission = Permissions.Motion.CAN_MANAGE - - CreateActionClass = MotionStatuteParagraphCreate diff --git a/openslides_backend/action/actions/motion_statute_paragraph/sort.py b/openslides_backend/action/actions/motion_statute_paragraph/sort.py deleted file mode 100644 index f3586ebd48..0000000000 --- a/openslides_backend/action/actions/motion_statute_paragraph/sort.py +++ /dev/null @@ -1,32 +0,0 @@ -from ....models.models import MotionStatuteParagraph -from ....permissions.permissions import Permissions -from ....shared.filters import FilterOperator -from ...generics.update import UpdateAction -from ...mixins.linear_sort_mixin import LinearSortMixin -from ...mixins.singular_action_mixin import SingularActionMixin -from ...util.default_schema import DefaultSchema -from ...util.register import register_action -from ...util.typing import ActionData - - -@register_action("motion_statute_paragraph.sort") -class MotionStatueParagraphSort(LinearSortMixin, SingularActionMixin, UpdateAction): - """ - Action to sort motion statue paragraph. - """ - - model = MotionStatuteParagraph() - schema = DefaultSchema(MotionStatuteParagraph()).get_linear_sort_schema( - "statute_paragraph_ids", - "meeting_id", - ) - permission = Permissions.Motion.CAN_MANAGE - - def get_updated_instances(self, action_data: ActionData) -> ActionData: - action_data = super().get_updated_instances(action_data) - # Action data is an iterable with exactly one item - instance = next(iter(action_data)) - yield from self.sort_linear( - instance["statute_paragraph_ids"], - FilterOperator("meeting_id", "=", instance["meeting_id"]), - ) diff --git a/openslides_backend/action/actions/motion_workflow/create.py b/openslides_backend/action/actions/motion_workflow/create.py index e3e4aa125e..35cdf86970 100644 --- a/openslides_backend/action/actions/motion_workflow/create.py +++ b/openslides_backend/action/actions/motion_workflow/create.py @@ -55,7 +55,6 @@ class MotionWorkflowCreateSimpleWorkflowAction(SequentialNumbersMixin, CreateAct [ "default_workflow_meeting_id", "default_amendment_workflow_meeting_id", - "default_statute_amendment_workflow_meeting_id", ], ) @@ -126,7 +125,6 @@ class MotionWorkflowCreateComplexWorkflowAction(SequentialNumbersMixin, CreateAc [ "default_workflow_meeting_id", "default_amendment_workflow_meeting_id", - "default_statute_amendment_workflow_meeting_id", ], ) diff --git a/openslides_backend/action/actions/motion_workflow/delete.py b/openslides_backend/action/actions/motion_workflow/delete.py index ae2fc90c86..f80d47aab4 100644 --- a/openslides_backend/action/actions/motion_workflow/delete.py +++ b/openslides_backend/action/actions/motion_workflow/delete.py @@ -33,7 +33,6 @@ def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: [ "motions_default_workflow_id", "motions_default_amendment_workflow_id", - "motions_default_statute_amendment_workflow_id", "motion_workflow_ids", ], ) @@ -45,12 +44,6 @@ def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: raise ActionException( "You cannot delete the workflow as long as it is selected as default workflow for new amendments in the settings. Please set another workflow as default in the settings and try to delete the workflow again." ) - if instance["id"] == meeting.get( - "motions_default_statute_amendment_workflow_id" - ): - raise ActionException( - "You cannot delete the workflow as long as it is selected as default workflow for new statute amendments in the settings. Please set another workflow as default in the settings and try to delete the workflow again." - ) workflow_ids = cast(list[int], meeting.get("motion_workflow_ids")) if len(workflow_ids) == 1: diff --git a/openslides_backend/migrations/migrations/0059_remove_statutes.py b/openslides_backend/migrations/migrations/0059_remove_statutes.py new file mode 100644 index 0000000000..3c3466de0d --- /dev/null +++ b/openslides_backend/migrations/migrations/0059_remove_statutes.py @@ -0,0 +1,348 @@ +from datastore.migrations import BaseModelMigration, MigrationException +from datastore.shared.util import fqid_from_collection_and_id +from datastore.writer.core import ( + BaseRequestEvent, + RequestDeleteEvent, + RequestUpdateEvent, +) + +from ...shared.filters import And, FilterOperator +from ..mixins.deletion_mixin import DeletionMixin, MigrationDeletionSchema + + +class Migration(BaseModelMigration, DeletionMixin): + """ + This migration removes all relations to and the statute motions/paragraphs themselves. + This requires to also delete all dangling models. However, lead motions, poll candidate lists + and their candidates of deleted polls will not be deleted since they are not expected + in this context. + """ + + target_migration_index = 60 + + deletion_schema: MigrationDeletionSchema = { + "motion": { + "deletes_models_from": { + "motion_change_recommendation": ["change_recommendation_ids"], + "motion_comment": ["comment_ids"], + "personal_note": ["personal_note_ids"], + "motion_working_group_speaker": ["working_group_speaker_ids"], + "motion_editor": ["editor_ids"], + "motion_submitter": ["submitter_ids"], + "list_of_speakers": ["list_of_speakers_id"], + "agenda_item": ["agenda_item_id"], + "projection": ["projection_ids"], + "poll": ["poll_ids"], + "motion": [ + "amendment_ids", + "origin_id", + "derived_motion_ids", + "all_origin_ids", + "all_derived_motion_ids", + ], + }, + "updates_models_from": { + "meeting": { + "meeting_id": "motion_ids", + }, + "tag": { + "tag_ids": "generic-tagged_ids", + }, + "motion_state": { + "state_id": "motion_ids", + "recommendation_id": "motion_recommendation_ids", + }, + "motion": { + "origin_meeting_id": "forwarded_motion_ids", + "sort_parent_id": "sort_child_ids", + "sort_child_ids": "sort_parent_id", + "state_extension_reference_ids": "referenced_in_motion_state_extension_ids", + "referenced_in_motion_state_extension_ids": "state_extension_reference_ids", + "recommendation_extension_reference_ids": "referenced_in_motion_recommendation_extension_ids", + "referenced_in_motion_recommendation_extension_ids": "recommendation_extension_reference_ids", + }, + "motion_category": { + "category_id": "motion_ids", + }, + "motion_block": { + "block_id": "motion_ids", + }, + "meeting_user": { + "supporter_meeting_user_ids": "supported_motion_ids", + }, + "meeting_mediafile": { + "attachment_meeting_mediafile_ids": "generic-attachment_ids", + }, + }, + }, + "motion_submitter": { + "deletes_models_from": {}, + "updates_models_from": { + "meeting": { + "meeting_id": "motion_submitter_ids", + }, + "meeting_user": { + "meeting_user_id": "motion_submitter_ids", + }, + }, + }, + "motion_editor": { + "deletes_models_from": {}, + "updates_models_from": { + "meeting": { + "meeting_id": "motion_editor_ids", + }, + "meeting_user": { + "meeting_user_id": "motion_editor_ids", + }, + }, + }, + "motion_working_group_speaker": { + "deletes_models_from": {}, + "updates_models_from": { + "meeting": { + "meeting_id": "motion_working_group_speaker_ids", + }, + "meeting_user": { + "meeting_user_id": "motion_working_group_speaker_ids", + }, + }, + }, + "personal_note": { + "deletes_models_from": {}, + "updates_models_from": { + "meeting": { + "meeting_id": "personal_note_ids", + }, + "meeting_user": { + "meeting_user_id": "personal_note_ids", + }, + }, + }, + "motion_comment": { + "deletes_models_from": {}, + "updates_models_from": { + "meeting": { + "meeting_id": "motion_comment_ids", + }, + "motion_comment_section": { + "section_id": "comment_ids", + }, + }, + }, + "motion_comment_section": { + "deletes_models_from": {}, + "updates_models_from": { + "meeting": { + "meeting_id": "motion_comment_section_ids", + }, + }, + }, + "motion_change_recommendation": { + "deletes_models_from": {}, + "updates_models_from": { + "meeting": { + "meeting_id": "motion_change_recommendation_ids", + }, + }, + }, + "poll": { + "deletes_models_from": { + "option": ["option_ids"], + "projection": ["projection_ids"], + }, + "updates_models_from": { + "meeting": { + "meeting_id": "poll_ids", + }, + "group": { + "entitled_group_ids": "poll_ids", + }, + "user": { + "voted_ids": "poll_voted_ids", + }, + }, + }, + "option": { + "deletes_models_from": { + "vote": ["vote_ids"], + }, + "updates_models_from": { + "meeting": { + "meeting_id": "option_ids", + }, + "motion": { + "content_object_id": "option_ids", + }, + "poll_candidate_list": { + "content_object_id": "option_id", + }, + "user": { + "content_object_id": "option_ids", + }, + }, + }, + "vote": { + "deletes_models_from": {}, + "updates_models_from": { + "meeting": { + "meeting_id": "vote_ids", + }, + "user": { + "user_id": "vote_ids", + "delegated_user_id": "delegated_vote_ids", + }, + }, + }, + "agenda_item": { + "deletes_models_from": { + "projection": ["projection_ids"], + }, + "updates_models_from": { + "meeting": { + "meeting_id": "agenda_item_ids", + }, + "agenda_item": { + "parent_id": "child_ids", + "child_ids": "parent_id", + }, + "tag": { + "tag_ids": "generic-tagged_ids", + }, + }, + }, + "list_of_speakers": { + "deletes_models_from": { + "projection": ["projection_ids"], + "speaker": ["speaker_ids"], + "structure_level_list_of_speakers": [ + "structure_level_list_of_speakers_ids" + ], + }, + "updates_models_from": { + "meeting": { + "meeting_id": "list_of_speakers_ids", + }, + }, + }, + "speaker": { + "deletes_models_from": {}, + "updates_models_from": { + "meeting": { + "meeting_id": "speaker_ids", + }, + "meeting_user": { + "meeting_user_id": "speaker_ids", + }, + "point_of_order_category": { + "point_of_order_category_id": "speaker_ids", + }, + }, + }, + "structure_level_list_of_speakers": { + "deletes_models_from": {}, + "updates_models_from": { + "meeting": { + "meeting_id": "structure_level_list_of_speakers_ids", + }, + "meeting_user": { + "meeting_user_ids": "structure_level_list_of_speakers_ids", + }, + "structure_level": { + "structure_level_id": "structure_level_list_of_speakers_ids", + }, + }, + }, + "structure_level": { + "deletes_models_from": {}, + "updates_models_from": { + "meeting": { + "meeting_id": "structure_level_ids", # TODO check in test # maybe test already corrupt? + }, + "meeting_user": { + "meeting_user_ids": "structure_level_ids", + }, + }, + }, + "projection": { + "deletes_models_from": {}, + "updates_models_from": { + "meeting": { + "meeting_id": "all_projection_ids", + }, + "projector": { + "current_projector_id": "current_projection_ids", + "preview_projector_id": "preview_projection_ids", + "history_projector_id": "history_projection_ids", + }, + }, + }, + } + + def migrate_models(self) -> list[BaseRequestEvent] | None: + """Migrates all models by deleting everything related to statutes and updating the relations.""" + events: list[BaseRequestEvent] = [] + + # delete statute related motions cascadingly and update related + statute_motions = self.reader.filter( + "motion", + And( + FilterOperator("statute_paragraph_id", "!=", None), + FilterOperator("meta_deleted", "!=", True), + ), + ) + for motion in statute_motions.values(): + if motion.get("lead_motion_id"): + raise MigrationException("A statute motion cannot have a lead motion.") + self.delete_update_by_schema( + {"motion": {motion_id for motion_id, motion in statute_motions.items()}}, + self.deletion_schema, + events, + ) + + # delete all statute paragraphs + statute_paragraphs = self.reader.get_all( + "motion_statute_paragraph", ["meeting_id"] + ) + for statute_paragraph_id, statute_paragraph in statute_paragraphs.items(): + events.append( + RequestDeleteEvent( + fqid_from_collection_and_id( + "motion_statute_paragraph", statute_paragraph_id + ) + ) + ) + + # find and update statute related motion workflows. That will make sure to get all. + motion_workflows = self.reader.get_all( + "motion_workflow", ["default_statute_amendment_workflow_meeting_id"] + ) + for workflow_id, workflow in motion_workflows.items(): + if workflow.get("default_statute_amendment_workflow_meeting_id", ""): + events.append( + RequestUpdateEvent( + fqid_from_collection_and_id("motion_workflow", workflow_id), + {"default_statute_amendment_workflow_meeting_id": None}, + ) + ) + + # update meetings + meetings = self.reader.get_all( + "meeting", + [], + ) + for meeting_id, meeting in meetings.items(): + meeting_fqid = fqid_from_collection_and_id("meeting", meeting_id) + events.append( + RequestUpdateEvent( + meeting_fqid, + { + "motions_statutes_enabled": None, + "motions_statute_recommendations_by": None, + "motion_statute_paragraph_ids": None, + "motions_default_statute_amendment_workflow_id": None, + }, + ) + ) + + return events diff --git a/openslides_backend/migrations/mixins/deletion_mixin.py b/openslides_backend/migrations/mixins/deletion_mixin.py new file mode 100644 index 0000000000..47ff352fa1 --- /dev/null +++ b/openslides_backend/migrations/mixins/deletion_mixin.py @@ -0,0 +1,270 @@ +from collections import defaultdict +from typing import Any, TypedDict + +from datastore.migrations.core.migration_reader import MigrationReader +from datastore.reader.core.requests import GetManyRequestPart +from datastore.shared.typing import Collection, Field, Fqid, Id, Model +from datastore.shared.util import ( + collection_and_id_from_fqid, + fqid_from_collection_and_id, + id_from_fqid, +) +from datastore.writer.core import ( + BaseRequestEvent, + RequestDeleteEvent, + RequestUpdateEvent, +) + + +class CollectionDeletionSchema(TypedDict): + deletes_models_from: dict[Collection, list[Field]] # [ids_field] + updates_models_from: dict[ + Collection, dict[Field, Field] + ] # {ids_field : foreign_ids_field} + + +MigrationDeletionSchema = dict[Collection, CollectionDeletionSchema] + + +class DeletionMixin: + """ + This class is used to delete cascading models genericly. + It should only be used in migration context, where a MigrationReader is available. + """ + + reader: MigrationReader + + def delete_update_by_schema( + self, + initial_deletions: dict[Collection, set[Id]], + deletion_schema: MigrationDeletionSchema, + events: list[BaseRequestEvent], + ) -> None: + """ + This recursively deletes all models specified by initial_deletions and the MigrationDeletionSchema. + Also updates all relations in other models related to deleted models by the specifics of the MigrationDeletionSchema. + This function auto magically handles 1:1, 1:n, n:m relations. It can also handle generic relations. + If the update relations foreign field is of generic type the field name needs to be prefixed with "generic-". + Can also delete models referenced within the same collection recursively. + Returns the list of delete and update requests. + """ + + update_schema: defaultdict[Collection, list[str]] = defaultdict(list) + for schema_part in deletion_schema.values(): + for collection, relation_fields in schema_part.get( + "updates_models_from", {} + ).items(): + for foreign_ids_field in relation_fields.values(): + if "generic-" in foreign_ids_field: + foreign_ids_field = foreign_ids_field.lstrip("generic-") + update_schema[collection].append(foreign_ids_field) + to_be_updated: dict[Collection, dict[Id, dict[Field, list[Id | Fqid]]]] = { + collection: defaultdict(lambda: defaultdict(list[Id | Fqid])) + for collection in update_schema.keys() + } + to_be_deleted: dict[Collection, set[Id]] = { + collection: set() for collection in deletion_schema.keys() + } + + self._recursively_stage_for_deletion_and_rel_for_update( + initial_deletions, + deletion_schema, + to_be_deleted, + to_be_updated, + ) + self.delete_all(to_be_deleted, events) + for collection, update_schema_part in update_schema.items(): + self.update_collection( + events, + collection, + update_schema_part, + to_be_updated[collection], + to_be_deleted, + ) + + def _recursively_stage_for_deletion_and_rel_for_update( + self, + initial_deletes: dict[Collection, set[Id]], + delete_schema: MigrationDeletionSchema, + to_be_deleted: dict[Collection, set[Id]], + to_be_updated: dict[Collection, dict[Id, dict[Field, list[Id | Fqid]]]], + ) -> None: + """ + Marks all models for deletion noted by the fields in collection_delete_schema. + Marks all models for update noted by the fields in collection_delete_schema. + """ + to_be_staged_recursively: defaultdict[Collection, set[Id]] = defaultdict(set) + for collection, model_ids in initial_deletes.items(): + to_be_deleted[collection].update(model_ids) + for collection, collection_delete_schema in delete_schema.items(): + # get models to be deleted later + if (to_be_deleted_ids := initial_deletes.get(collection)) and ( + fields := [ + field_name + for field_names in collection_delete_schema.get( + "deletes_models_from", {} + ).values() + for field_name in field_names + ] + + [ + field_name + for relation_fields in collection_delete_schema.get( + "updates_models_from", {} + ).values() + for field_name in relation_fields.keys() + ] + ): + models = self.reader.get_many( + [GetManyRequestPart(collection, list(to_be_deleted_ids), fields)] + ).get(collection, {}) + for model_id, model in models.items(): + self._stage_for_update( + collection_delete_schema.get("updates_models_from", {}), + model, + model_id, + to_be_updated, + collection, + ) + # stage related collection instances for later deletion + for foreign_collection, own_fields in collection_delete_schema.get( + "deletes_models_from", {} + ).items(): + for own_field in own_fields: + if foreign_id_or_ids := model.get(own_field): + if isinstance(foreign_id_or_ids, list) and isinstance( + foreign_id_or_ids[0], str + ): + foreign_id_or_ids = [ + id_from_fqid(foreign_id) + for foreign_id in foreign_id_or_ids + ] + elif isinstance(foreign_id_or_ids, str): + foreign_id_or_ids = [ + id_from_fqid(foreign_id_or_ids) + ] + elif isinstance(foreign_id_or_ids, int): + foreign_id_or_ids = [foreign_id_or_ids] + for foreign_id in foreign_id_or_ids: + if ( + foreign_id + not in to_be_deleted[foreign_collection] + ): + to_be_staged_recursively[ + foreign_collection + ].add(foreign_id) + if any(to_be_staged_recursively): + self._recursively_stage_for_deletion_and_rel_for_update( + to_be_staged_recursively, delete_schema, to_be_deleted, to_be_updated + ) + + def _stage_for_update( + self, + collection_schema_updates_models_from: dict[Collection, dict[Field, Field]], + model: Model, + model_id: Id, + to_be_updated: dict[Collection, dict[Id, dict[Field, list[Id | Fqid]]]], + collection: Collection, + ) -> None: + """stage instance ids for update in related collection instances""" + for ( + foreign_collection, + relation_fields, + ) in collection_schema_updates_models_from.items(): + for own_field, foreign_field in relation_fields.items(): + if "generic-" in foreign_field: + foreign_field = foreign_field.lstrip("generic-") + target_field_generic = True + else: + target_field_generic = False + if foreign_ids := model.get(own_field): + if not isinstance(foreign_ids, list): + foreign_ids = [foreign_ids] + for foreign_id in foreign_ids: + if isinstance(foreign_id, str): + tmp_foreign_collection, foreign_id = ( + collection_and_id_from_fqid(foreign_id) + ) + # generic fields can have different collections in fqid thus differing from target collection. + # will be treated by the next combination of this field and collection + if tmp_foreign_collection != foreign_collection: + continue + # need to store own collection context for generic foreign field + if target_field_generic: + model_id_or_fqid: Id | Fqid = fqid_from_collection_and_id( + collection, model_id + ) + else: + model_id_or_fqid = model_id + to_be_updated[foreign_collection][foreign_id][ + foreign_field + ].append(model_id_or_fqid) + + def delete_all( + self, to_be_deleted: dict[Collection, set[Id]], events: list[BaseRequestEvent] + ) -> None: + """Creates RequestDeleteEvents for models given in to_be_deleted""" + for collection, to_be_deleted_ids in to_be_deleted.items(): + if to_be_deleted_ids: + for to_be_deleted_id in to_be_deleted_ids: + events.append( + RequestDeleteEvent( + fqid_from_collection_and_id(collection, to_be_deleted_id) + ) + ) + + def update_collection( + self, + events: list, + collection: Collection, + collection_update_schema: list[Field], + to_be_updated_in_collection: dict[Id, dict[Field, Any]], + deleted_instances: dict[Collection, set[Id]], + ) -> None: + """ + Updates all models of the collection with the info provided by the collection_update_schema + but not those that were already deleted. + """ + to_remove = [] + # if there were no instances deleted we don't need to remove them from our update list. + if collections_deleted_ids := deleted_instances.get(collection): + for instance_id in to_be_updated_in_collection.keys(): + if instance_id in collections_deleted_ids: + to_remove.append(instance_id) + for instance_id in to_remove: + del to_be_updated_in_collection[instance_id] + + instances = self.reader.get_many( + [ + GetManyRequestPart( + collection, + [instance_id for instance_id in to_be_updated_in_collection.keys()], + collection_update_schema, + ) + ] + ).get(collection, {}) + for instance_id, fields_and_ids in to_be_updated_in_collection.items(): + instance = instances.get(instance_id, {}) + # save the instances data without the deleted ids + for field, without_ids in fields_and_ids.items(): + db_ids = instance.get(field, []) + if not isinstance(db_ids, list): + db_ids = [instance.get(field, [])] + fields_and_ids[field] = self.subtract_ids(db_ids, without_ids) + events.append( + RequestUpdateEvent( + fqid_from_collection_and_id(collection, instance_id), fields_and_ids + ) + ) + + def subtract_ids( + self, front_ids: list | None, without_ids: list | None + ) -> list | None: + """ + This subtracts items of a list from another list in an efficient manner. + Returns a list. + """ + if not front_ids: + return None + if not without_ids: + return front_ids + return list(set(front_ids) - set(without_ids)) or None diff --git a/openslides_backend/models/models.py b/openslides_backend/models/models.py index a06c12ed6d..c44e6e00bd 100644 --- a/openslides_backend/models/models.py +++ b/openslides_backend/models/models.py @@ -473,10 +473,6 @@ class Meeting(Model, MeetingModelMixin): motions_default_amendment_workflow_id = fields.RelationField( to={"motion_workflow": "default_amendment_workflow_meeting_id"}, required=True ) - motions_default_statute_amendment_workflow_id = fields.RelationField( - to={"motion_workflow": "default_statute_amendment_workflow_meeting_id"}, - required=True, - ) motions_preamble = fields.TextField(default="The assembly may decide:") motions_default_line_numbering = fields.CharField( default="outside", constraints={"enum": ["outside", "inline", "none"]} @@ -492,7 +488,6 @@ class Meeting(Model, MeetingModelMixin): motions_show_sequential_number = fields.BooleanField(default=True) motions_recommendations_by = fields.CharField() motions_block_slide_columns = fields.IntegerField(constraints={"minimum": 1}) - motions_statute_recommendations_by = fields.CharField() motions_recommendation_text_mode = fields.CharField( default="diff", constraints={"enum": ["original", "changed", "diff", "agreed"]} ) @@ -505,7 +500,6 @@ class Meeting(Model, MeetingModelMixin): ) motions_number_min_digits = fields.IntegerField(default=2) motions_number_with_blank = fields.BooleanField(default=False) - motions_statutes_enabled = fields.BooleanField(default=False) motions_amendments_enabled = fields.BooleanField(default=True) motions_amendments_in_main_list = fields.BooleanField(default=True) motions_amendments_of_amendments = fields.BooleanField(default=False) @@ -731,9 +725,6 @@ class Meeting(Model, MeetingModelMixin): motion_workflow_ids = fields.RelationListField( to={"motion_workflow": "meeting_id"}, on_delete=fields.OnDelete.CASCADE ) - motion_statute_paragraph_ids = fields.RelationListField( - to={"motion_statute_paragraph": "meeting_id"}, on_delete=fields.OnDelete.CASCADE - ) motion_comment_ids = fields.RelationListField( to={"motion_comment": "meeting_id"}, on_delete=fields.OnDelete.CASCADE ) @@ -1465,9 +1456,6 @@ class Motion(Model): on_delete=fields.OnDelete.CASCADE, equal_fields="meeting_id", ) - statute_paragraph_id = fields.RelationField( - to={"motion_statute_paragraph": "motion_ids"}, equal_fields="meeting_id" - ) comment_ids = fields.RelationListField( to={"motion_comment": "motion_id"}, on_delete=fields.OnDelete.CASCADE, @@ -1824,38 +1812,11 @@ class MotionWorkflow(Model): default_amendment_workflow_meeting_id = fields.RelationField( to={"meeting": "motions_default_amendment_workflow_id"} ) - default_statute_amendment_workflow_meeting_id = fields.RelationField( - to={"meeting": "motions_default_statute_amendment_workflow_id"} - ) meeting_id = fields.RelationField( to={"meeting": "motion_workflow_ids"}, required=True, constant=True ) -class MotionStatuteParagraph(Model): - collection = "motion_statute_paragraph" - verbose_name = "motion statute paragraph" - - id = fields.IntegerField(required=True, constant=True) - title = fields.CharField(required=True) - text = fields.HTMLStrictField() - weight = fields.IntegerField(default=10000) - sequential_number = fields.IntegerField( - required=True, - read_only=True, - constant=True, - constraints={ - "description": "The (positive) serial number of this model in its meeting. This number is auto-generated and read-only." - }, - ) - motion_ids = fields.RelationListField( - to={"motion": "statute_paragraph_id"}, equal_fields="meeting_id" - ) - meeting_id = fields.RelationField( - to={"meeting": "motion_statute_paragraph_ids"}, required=True, constant=True - ) - - class Poll(Model, PollModelMixin): collection = "poll" verbose_name = "poll" diff --git a/tests/system/action/committee/test_import.py b/tests/system/action/committee/test_import.py index ccb9744da3..4564c2fabb 100644 --- a/tests/system/action/committee/test_import.py +++ b/tests/system/action/committee/test_import.py @@ -437,7 +437,6 @@ def test_import_meeting_template(self) -> None: "language": "en", "default_group_id": 1, "motions_default_amendment_workflow_id": 1, - "motions_default_statute_amendment_workflow_id": 1, "motions_default_workflow_id": 1, "reference_projector_id": 1, "projector_ids": [1], @@ -465,7 +464,6 @@ def test_import_meeting_template(self) -> None: "name": "blup", "first_state_id": 1, "default_amendment_workflow_meeting_id": 1, - "default_statute_amendment_workflow_meeting_id": 1, "default_workflow_meeting_id": 1, "state_ids": [1], "sequential_number": 1, diff --git a/tests/system/action/committee/test_json_upload.py b/tests/system/action/committee/test_json_upload.py index e32893a7b9..de5e82e2b3 100644 --- a/tests/system/action/committee/test_json_upload.py +++ b/tests/system/action/committee/test_json_upload.py @@ -802,7 +802,6 @@ def json_upload_admin_defined_meeting_template_found(self) -> None: "motion_workflow_ids": [2], "motion_state_ids": [2], "motions_default_amendment_workflow_id": 2, - "motions_default_statute_amendment_workflow_id": 2, **{field: [1] for field in Meeting.all_default_projectors()}, }, "projector/1": { @@ -815,7 +814,6 @@ def json_upload_admin_defined_meeting_template_found(self) -> None: "motion_workflow/2": { "name": "yay", "default_amendment_workflow_meeting_id": 2, - "default_statute_amendment_workflow_meeting_id": 2, "sequential_number": 1, }, "motion_state/2": {"weight": 1, "name": "dismissed"}, @@ -870,7 +868,6 @@ def json_upload_meeting_template_with_admins_found(self) -> None: "motion_workflow_ids": [2], "motion_state_ids": [2], "motions_default_amendment_workflow_id": 2, - "motions_default_statute_amendment_workflow_id": 2, **{field: [1] for field in Meeting.all_default_projectors()}, }, "projector/1": { @@ -883,7 +880,6 @@ def json_upload_meeting_template_with_admins_found(self) -> None: "motion_workflow/2": { "name": "yay", "default_amendment_workflow_meeting_id": 2, - "default_statute_amendment_workflow_meeting_id": 2, "sequential_number": 1, }, "motion_state/2": {"weight": 1, "name": "dismissed"}, diff --git a/tests/system/action/meeting/test_clone.py b/tests/system/action/meeting/test_clone.py index 518e9fdf06..a1dc315adc 100644 --- a/tests/system/action/meeting/test_clone.py +++ b/tests/system/action/meeting/test_clone.py @@ -37,7 +37,6 @@ def setUp(self) -> None: "default_group_id": 1, "admin_group_id": 2, "motions_default_amendment_workflow_id": 1, - "motions_default_statute_amendment_workflow_id": 1, "motions_default_workflow_id": 1, "reference_projector_id": 1, "projector_countdown_default_time": 60, @@ -66,7 +65,6 @@ def setUp(self) -> None: "name": "blup", "first_state_id": 1, "default_amendment_workflow_meeting_id": 1, - "default_statute_amendment_workflow_meeting_id": 1, "default_workflow_meeting_id": 1, "state_ids": [1], "sequential_number": 1, @@ -122,7 +120,6 @@ def test_clone_without_users(self) -> None: "default_group_id": 3, "admin_group_id": 4, "motions_default_amendment_workflow_id": 2, - "motions_default_statute_amendment_workflow_id": 2, "motions_default_workflow_id": 2, "reference_projector_id": 2, "projector_countdown_default_time": 60, @@ -1160,13 +1157,11 @@ def test_clone_with_settings(self) -> None: "motions_show_referring_motions": True, "motions_show_sequential_number": True, "motions_recommendations_by": "rec", - "motions_statute_recommendations_by": "rec", "motions_recommendation_text_mode": "original", "motions_default_sorting": "weight", "motions_number_type": "manually", "motions_number_min_digits": 42, "motions_number_with_blank": True, - "motions_statutes_enabled": True, "motions_amendments_enabled": True, "motions_amendments_in_main_list": True, "motions_amendments_of_amendments": True, diff --git a/tests/system/action/meeting/test_create.py b/tests/system/action/meeting/test_create.py index 974749a5ed..b78eb6d190 100644 --- a/tests/system/action/meeting/test_create.py +++ b/tests/system/action/meeting/test_create.py @@ -67,7 +67,6 @@ def test_create_simple_and_complex_workflow(self) -> None: "motion_workflow_ids": [1, 2], "motions_default_workflow_id": 1, "motions_default_amendment_workflow_id": 1, - "motions_default_statute_amendment_workflow_id": 1, "motion_state_ids": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "list_of_speakers_countdown_id": 1, "poll_countdown_id": 2, @@ -92,7 +91,6 @@ def test_create_simple_and_complex_workflow(self) -> None: "meeting_id": 1, "default_workflow_meeting_id": 1, "default_amendment_workflow_meeting_id": 1, - "default_statute_amendment_workflow_meeting_id": 1, "state_ids": [1, 2, 3, 4], "first_state_id": 1, }, diff --git a/tests/system/action/meeting/test_import.py b/tests/system/action/meeting/test_import.py index 36afbcd07e..a209d75b98 100644 --- a/tests/system/action/meeting/test_import.py +++ b/tests/system/action/meeting/test_import.py @@ -62,7 +62,6 @@ def create_request_data( "admin_group_id": 1, "default_group_id": 2, "motions_default_amendment_workflow_id": 1, - "motions_default_statute_amendment_workflow_id": 1, "motions_default_workflow_id": 1, "projector_countdown_default_time": 60, "projector_countdown_warning_time": 60, @@ -133,13 +132,11 @@ def create_request_data( "motions_show_referring_motions": True, "motions_show_sequential_number": True, "motions_recommendations_by": "ABK", - "motions_statute_recommendations_by": "Statute ABK", "motions_recommendation_text_mode": "original", "motions_default_sorting": "number", "motions_number_type": "per_category", "motions_number_min_digits": 3, "motions_number_with_blank": False, - "motions_statutes_enabled": True, "motions_amendments_enabled": True, "motions_amendments_in_main_list": True, "motions_amendments_of_amendments": True, @@ -211,7 +208,6 @@ def create_request_data( "motion_category_ids": [], "motion_block_ids": [], "motion_workflow_ids": [1], - "motion_statute_paragraph_ids": [], "motion_change_recommendation_ids": [], "poll_ids": [], "option_ids": [], @@ -271,7 +267,6 @@ def create_request_data( "name": "blup", "first_state_id": 1, "default_amendment_workflow_meeting_id": 1, - "default_statute_amendment_workflow_meeting_id": 1, "default_workflow_meeting_id": 1, "state_ids": [1], "sequential_number": 1, @@ -2610,6 +2605,41 @@ def test_without_users(self) -> None: ) self.assert_model_not_exists("user/2") + def test_delete_statutes(self) -> None: + """test for deleted statute motions in event.data after migration. Uses migrations 0055 and onwards.""" + data = self.create_request_data() + data["meeting"]["meeting"]["1"][ + "motions_default_statute_amendment_workflow_id" + ] = 1 + data["meeting"]["meeting"]["1"][ + "motions_statute_recommendations_by" + ] = "Statute ABK" + data["meeting"]["meeting"]["1"]["motions_statutes_enabled"] = True + data["meeting"]["meeting"]["1"]["motion_statute_paragraph_ids"] = [] + + data["meeting"]["motion_workflow"]["1"][ + "default_statute_amendment_workflow_meeting_id" + ] = 1 + data["meeting"]["_migration_index"] = 55 + response = self.request("meeting.import", data) + self.assert_status_code(response, 200) + self.assert_model_not_exists("motion_workflow/2") + self.assert_model_exists( + "meeting/1", + { + "motions_default_statute_amendment_workflow_id": None, + "motions_statute_recommendations_by": None, + "motions_statutes_enabled": None, + "motion_statute_paragraph_ids": None, + }, + ) + self.assert_model_exists( + "motion_workflow/1", + { + "default_statute_amendment_workflow_meeting_id": None, + }, + ) + @pytest.mark.skip() def test_import_os3_data(self) -> None: data_raw = get_initial_data_file("global/data/export-OS3-demo.json") diff --git a/tests/system/action/motion/test_create.py b/tests/system/action/motion/test_create.py index e87f742bd9..4740c9e8a6 100644 --- a/tests/system/action/motion/test_create.py +++ b/tests/system/action/motion/test_create.py @@ -237,27 +237,6 @@ def test_create_reason_missing(self) -> None: self.assert_status_code(response, 400) assert "Reason is required" in response.json["message"] - def test_create_lead_motion_and_statute_paragraph_id_given(self) -> None: - self.set_models( - { - "motion_statute_paragraph/1": {"meeting_id": 1}, - } - ) - response = self.request( - "motion.create", - { - "title": "test_Xcdfgee", - "meeting_id": 1, - "text": "text", - "lead_motion_id": 1, - "statute_paragraph_id": 1, - }, - ) - self.assert_status_code(response, 400) - assert "both of lead_motion_id and statute_paragraph_id." in response.json.get( - "message", "" - ) - def test_create_with_submitters(self) -> None: self.set_models( { @@ -560,36 +539,6 @@ def test_create_check_not_unique_number(self) -> None: self.assert_status_code(response, 400) assert "Number is not unique." in response.json["message"] - def test_create_broken_motion_type(self) -> None: - self.set_models( - { - "meeting/1": { - "name": "name_uZXBoHMp", - "is_active_in_organization_id": 1, - "motion_ids": [1], - "motion_statute_paragraph_ids": [1], - }, - "motion/1": {"meeting_id": 1, "number": "T001"}, - "motion_statute_paragraph/1": {"meeting_id": 1, "title": "Paragraph"}, - } - ) - response = self.request( - "motion.create", - { - "title": "Title", - "text": "

of motion

", - "number": "A001", - "lead_motion_id": 1, - "statute_paragraph_id": 1, - "meeting_id": 1, - }, - ) - self.assert_status_code(response, 400) - assert ( - "You can't give both of lead_motion_id and statute_paragraph_id." - in response.json["message"] - ) - def test_create_amendment_paragraphs_where_not_allowed(self) -> None: self.set_models( { diff --git a/tests/system/action/motion/test_create_statute_amendment.py b/tests/system/action/motion/test_create_statute_amendment.py deleted file mode 100644 index bcfc115bdf..0000000000 --- a/tests/system/action/motion/test_create_statute_amendment.py +++ /dev/null @@ -1,114 +0,0 @@ -from tests.system.action.base import BaseActionTestCase - - -class MotionCreateAmendmentActionTest(BaseActionTestCase): - def setUp(self) -> None: - super().setUp() - # create parent paragraph and workflow - self.set_models( - { - "motion_workflow/12": { - "name": "name_workflow1", - "first_state_id": 34, - "state_ids": [34], - }, - "motion_state/34": {"name": "name_state34", "meeting_id": 222}, - "motion_statute_paragraph/1": { - "title": "title_eJveLQIh", - "meeting_id": 222, - }, - } - ) - - def test_create_statute_amendment(self) -> None: - self.set_models( - { - "meeting/222": {"is_active_in_organization_id": 1}, - "user/1": {"meeting_ids": [222]}, - } - ) - response = self.request( - "motion.create", - { - "title": "test_Xcdfgee", - "meeting_id": 222, - "workflow_id": 12, - "statute_paragraph_id": 1, - "text": "text", - }, - ) - self.assert_status_code(response, 200) - model = self.get_model("motion/1") - assert model.get("title") == "test_Xcdfgee" - assert model.get("meeting_id") == 222 - assert model.get("statute_paragraph_id") == 1 - assert model.get("text") == "text" - - def test_create_statute_amendment_default_workflow(self) -> None: - self.set_models( - { - "meeting/222": { - "motions_default_statute_amendment_workflow_id": 12, - "is_active_in_organization_id": 1, - }, - "user/1": {"meeting_ids": [222]}, - } - ) - response = self.request( - "motion.create", - { - "title": "test_Xcdfgee", - "meeting_id": 222, - "statute_paragraph_id": 1, - "text": "text", - }, - ) - self.assert_status_code(response, 200) - model = self.get_model("motion/1") - assert model.get("title") == "test_Xcdfgee" - assert model.get("meeting_id") == 222 - assert model.get("statute_paragraph_id") == 1 - assert model.get("text") == "text" - assert model.get("state_id") == 34 - - def test_create_with_amendment_paragraphs(self) -> None: - self.set_models( - { - "meeting/222": {"is_active_in_organization_id": 1}, - "user/1": {"meeting_ids": [222]}, - } - ) - response = self.request( - "motion.create", - { - "title": "test_Xcdfgee", - "meeting_id": 222, - "statute_paragraph_id": 1, - "text": "text", - "amendment_paragraphs": {4: "text"}, - }, - ) - self.assert_status_code(response, 400) - assert "give amendment_paragraphs in this context" in response.json["message"] - - def test_create_reason_missing(self) -> None: - self.set_models( - { - "meeting/222": { - "motions_reason_required": True, - "is_active_in_organization_id": 1, - }, - "user/1": {"meeting_ids": [222]}, - } - ) - response = self.request( - "motion.create", - { - "title": "test_Xcdfgee", - "meeting_id": 222, - "statute_paragraph_id": 1, - "text": "text", - }, - ) - self.assert_status_code(response, 400) - assert "Reason is required" in response.json["message"] diff --git a/tests/system/action/motion_change_recommendation/test_update.py b/tests/system/action/motion_change_recommendation/test_update.py index 8c45c08079..64b4bf05dd 100644 --- a/tests/system/action/motion_change_recommendation/test_update.py +++ b/tests/system/action/motion_change_recommendation/test_update.py @@ -10,7 +10,6 @@ def setUp(self) -> None: self.permission_test_models: dict[str, dict[str, Any]] = { "motion/25": { "title": "title_pheK0Ja3ai", - "statute_paragraph_id": None, "meeting_id": 1, }, "motion_change_recommendation/111": { @@ -28,7 +27,6 @@ def test_update_correct(self) -> None: "meeting/1": {"is_active_in_organization_id": 1}, "motion/25": { "title": "title_pheK0Ja3ai", - "statute_paragraph_id": None, "meeting_id": 1, }, "motion_change_recommendation/111": { @@ -68,7 +66,6 @@ def test_update_wrong_id(self) -> None: "meeting/1": {"is_active_in_organization_id": 1}, "motion/25": { "title": "title_pheK0Ja3ai", - "statute_paragraph_id": None, "meeting_id": 1, }, "motion_change_recommendation/111": { diff --git a/tests/system/action/motion_statute_paragraph/__init__.py b/tests/system/action/motion_statute_paragraph/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/system/action/motion_statute_paragraph/test_create.py b/tests/system/action/motion_statute_paragraph/test_create.py deleted file mode 100644 index 1e62388f08..0000000000 --- a/tests/system/action/motion_statute_paragraph/test_create.py +++ /dev/null @@ -1,52 +0,0 @@ -from openslides_backend.permissions.permissions import Permissions -from tests.system.action.base import BaseActionTestCase - - -class MotionStatuteParagraphActionTest(BaseActionTestCase): - def test_create(self) -> None: - self.create_model("meeting/42", {"is_active_in_organization_id": 1}) - response = self.request( - "motion_statute_paragraph.create", - {"meeting_id": 42, "title": "test_Xcdfgee", "text": "blablabla"}, - ) - self.assert_status_code(response, 200) - self.assert_model_exists("motion_statute_paragraph/1") - model = self.get_model("motion_statute_paragraph/1") - assert model.get("title") == "test_Xcdfgee" - assert model.get("weight") == 10000 - assert model.get("text") == "blablabla" - assert model.get("sequential_number") == 1 - - def test_create_empty_data(self) -> None: - response = self.request("motion_statute_paragraph.create", {}) - self.assert_status_code(response, 400) - self.assertIn( - "data must contain ['meeting_id', 'text', 'title'] properties", - response.json["message"], - ) - - def test_create_wrong_field(self) -> None: - self.create_model("meeting/42", {"is_active_in_organization_id": 1}) - response = self.request( - "motion_statute_paragraph.create", {"wrong_field": "text_AefohteiF8"} - ) - self.assert_status_code(response, 400) - self.assertIn( - "data must contain ['meeting_id', 'text', 'title'] properties", - response.json["message"], - ) - - def test_create_no_permissions(self) -> None: - self.base_permission_test( - {}, - "motion_statute_paragraph.create", - {"meeting_id": 1, "title": "test_Xcdfgee", "text": "blablabla"}, - ) - - def test_create_permissions(self) -> None: - self.base_permission_test( - {}, - "motion_statute_paragraph.create", - {"meeting_id": 1, "title": "test_Xcdfgee", "text": "blablabla"}, - Permissions.Motion.CAN_MANAGE, - ) diff --git a/tests/system/action/motion_statute_paragraph/test_delete.py b/tests/system/action/motion_statute_paragraph/test_delete.py deleted file mode 100644 index 24082770d6..0000000000 --- a/tests/system/action/motion_statute_paragraph/test_delete.py +++ /dev/null @@ -1,49 +0,0 @@ -from typing import Any - -from openslides_backend.permissions.permissions import Permissions -from tests.system.action.base import BaseActionTestCase - - -class MotionStatuteParagraphActionTest(BaseActionTestCase): - def setUp(self) -> None: - super().setUp() - self.permission_test_models: dict[str, dict[str, Any]] = { - "motion_statute_paragraph/111": {"meeting_id": 1} - } - - def test_delete_correct(self) -> None: - self.set_models( - { - "motion_statute_paragraph/111": {"meeting_id": 1}, - "meeting/1": {"is_active_in_organization_id": 1}, - } - ) - response = self.request("motion_statute_paragraph.delete", {"id": 111}) - self.assert_status_code(response, 200) - self.assert_model_deleted("motion_statute_paragraph/111") - - def test_delete_wrong_id(self) -> None: - self.set_models( - { - "motion_statute_paragraph/112": {"meeting_id": 1}, - "meeting/1": {"is_active_in_organization_id": 1}, - } - ) - response = self.request("motion_statute_paragraph.delete", {"id": 111}) - self.assert_status_code(response, 400) - self.assert_model_exists("motion_statute_paragraph/112") - - def test_delete_no_permissions(self) -> None: - self.base_permission_test( - self.permission_test_models, - "motion_statute_paragraph.delete", - {"id": 111}, - ) - - def test_delete_permissions(self) -> None: - self.base_permission_test( - self.permission_test_models, - "motion_statute_paragraph.delete", - {"id": 111}, - Permissions.Motion.CAN_MANAGE, - ) diff --git a/tests/system/action/motion_statute_paragraph/test_sort.py b/tests/system/action/motion_statute_paragraph/test_sort.py deleted file mode 100644 index 8faf854ea9..0000000000 --- a/tests/system/action/motion_statute_paragraph/test_sort.py +++ /dev/null @@ -1,115 +0,0 @@ -from typing import Any - -from openslides_backend.permissions.permissions import Permissions -from tests.system.action.base import BaseActionTestCase - - -class MotionStatuteParagraphSortActionTest(BaseActionTestCase): - def setUp(self) -> None: - super().setUp() - self.permission_test_models: dict[str, dict[str, Any]] = { - "motion_statute_paragraph/31": { - "meeting_id": 1, - "title": "title_loisueb", - }, - "motion_statute_paragraph/32": { - "meeting_id": 1, - "title": "title_blanumop", - }, - } - - def test_sort_correct_1(self) -> None: - self.set_models( - { - "meeting/222": { - "name": "name_SNLGsvIV", - "is_active_in_organization_id": 1, - }, - "motion_statute_paragraph/31": { - "meeting_id": 222, - "title": "title_loisueb", - }, - "motion_statute_paragraph/32": { - "meeting_id": 222, - "title": "title_blanumop", - }, - } - ) - response = self.request( - "motion_statute_paragraph.sort", - {"meeting_id": 222, "statute_paragraph_ids": [32, 31]}, - ) - self.assert_status_code(response, 200) - model_31 = self.get_model("motion_statute_paragraph/31") - assert model_31.get("weight") == 2 - model_32 = self.get_model("motion_statute_paragraph/32") - assert model_32.get("weight") == 1 - - def test_sort_missing_model(self) -> None: - self.set_models( - { - "meeting/222": { - "name": "name_SNLGsvIV", - "is_active_in_organization_id": 1, - }, - "motion_statute_paragraph/31": { - "meeting_id": 222, - "title": "title_loisueb", - }, - } - ) - response = self.request( - "motion_statute_paragraph.sort", - {"meeting_id": 222, "statute_paragraph_ids": [32, 31]}, - ) - self.assert_status_code(response, 400) - assert ( - "motion_statute_paragraph sorting failed, because element motion_statute_paragraph/32 doesn't exist." - in response.json["message"] - ) - - def test_sort_another_section_db(self) -> None: - self.set_models( - { - "meeting/222": { - "name": "name_SNLGsvIV", - "is_active_in_organization_id": 1, - }, - "motion_statute_paragraph/31": { - "meeting_id": 222, - "title": "title_loisueb", - }, - "motion_statute_paragraph/32": { - "meeting_id": 222, - "title": "title_blanumop", - }, - "motion_statute_paragraph/33": { - "meeting_id": 222, - "title": "title_polusiem", - }, - } - ) - response = self.request( - "motion_statute_paragraph.sort", - {"meeting_id": 222, "statute_paragraph_ids": [32, 31]}, - ) - self.assert_status_code(response, 400) - assert ( - "motion_statute_paragraph sorting failed, because some elements were not included in the call." - in response.json["message"] - ) - - def test_sort_no_permissions(self) -> None: - self.base_permission_test( - self.permission_test_models, - "motion_statute_paragraph.sort", - {"meeting_id": 1, "statute_paragraph_ids": [32, 31]}, - ) - - def test_sort_permissions(self) -> None: - self.base_permission_test( - self.permission_test_models, - "motion_statute_paragraph.sort", - {"meeting_id": 1, "statute_paragraph_ids": [32, 31]}, - Permissions.Motion.CAN_MANAGE, - ) diff --git a/tests/system/action/motion_statute_paragraph/test_update.py b/tests/system/action/motion_statute_paragraph/test_update.py deleted file mode 100644 index f1262550c6..0000000000 --- a/tests/system/action/motion_statute_paragraph/test_update.py +++ /dev/null @@ -1,67 +0,0 @@ -from typing import Any - -from openslides_backend.permissions.permissions import Permissions -from tests.system.action.base import BaseActionTestCase - - -class MotionStatuteParagraphActionTest(BaseActionTestCase): - def setUp(self) -> None: - super().setUp() - self.permission_test_models: dict[str, dict[str, Any]] = { - "motion_statute_paragraph/111": { - "title": "title_srtgb123", - "meeting_id": 1, - } - } - - def test_update_correct(self) -> None: - self.set_models( - { - "motion_statute_paragraph/111": { - "title": "title_srtgb123", - "meeting_id": 1, - }, - "meeting/1": {"is_active_in_organization_id": 1}, - } - ) - response = self.request( - "motion_statute_paragraph.update", - {"id": 111, "title": "title_Xcdfgee", "text": "text_blablabla"}, - ) - self.assert_status_code(response, 200) - self.assert_model_exists("motion_statute_paragraph/111") - model = self.get_model("motion_statute_paragraph/111") - assert model.get("title") == "title_Xcdfgee" - assert model.get("text") == "text_blablabla" - - def test_update_wrong_id(self) -> None: - self.set_models( - { - "motion_statute_paragraph/111": { - "title": "title_srtgb123", - "meeting_id": 1, - }, - "meeting/1": {"is_active_in_organization_id": 1}, - } - ) - response = self.request( - "motion_statute_paragraph.update", {"id": 112, "title": "title_Xcdfgee"} - ) - self.assert_status_code(response, 400) - model = self.get_model("motion_statute_paragraph/111") - assert model.get("title") == "title_srtgb123" - - def test_update_no_permissions(self) -> None: - self.base_permission_test( - self.permission_test_models, - "motion_statute_paragraph.update", - {"id": 111, "title": "title_Xcdfgee", "text": "text_blablabla"}, - ) - - def test_update_permissions(self) -> None: - self.base_permission_test( - self.permission_test_models, - "motion_statute_paragraph.update", - {"id": 111, "title": "title_Xcdfgee", "text": "text_blablabla"}, - Permissions.Motion.CAN_MANAGE, - ) diff --git a/tests/system/action/motion_workflow/test_delete.py b/tests/system/action/motion_workflow/test_delete.py index cd832ceae4..9a588415cc 100644 --- a/tests/system/action/motion_workflow/test_delete.py +++ b/tests/system/action/motion_workflow/test_delete.py @@ -9,7 +9,6 @@ def test_delete_correct(self) -> None: "meeting/90": { "name": "name_testtest", "motions_default_workflow_id": 12, - "motions_default_statute_amendment_workflow_id": 13, "motion_workflow_ids": [111, 2], "is_active_in_organization_id": 1, }, @@ -75,7 +74,6 @@ def test_delete_fail_case_default_1(self) -> None: "meeting/90": { "name": "name_testtest", "motions_default_workflow_id": 111, - "motions_default_statute_amendment_workflow_id": 13, "motion_workflow_ids": [111], "is_active_in_organization_id": 1, }, @@ -96,7 +94,6 @@ def test_delete_fail_case_default_2(self) -> None: "meeting/90": { "name": "name_testtest", "motions_default_workflow_id": 12, - "motions_default_statute_amendment_workflow_id": 111, "motion_workflow_ids": [111], "is_active_in_organization_id": 1, }, @@ -107,7 +104,7 @@ def test_delete_fail_case_default_2(self) -> None: self.assert_status_code(response, 400) self.assert_model_exists("motion_workflow/111") self.assertIn( - "You cannot delete the workflow as long as it is selected as default workflow for new statute amendments in the settings.", + "You cannot delete the last workflow of a meeting.", response.json["message"], ) @@ -117,7 +114,6 @@ def test_delete_fail_case_default_3(self) -> None: "meeting/90": { "name": "name_testtest", "motions_default_workflow_id": 12, - "motions_default_statute_amendment_workflow_id": 13, "motions_default_amendment_workflow_id": 111, "motion_workflow_ids": [111], "is_active_in_organization_id": 1, @@ -153,7 +149,6 @@ def test_delete_no_permissions(self) -> None: "meeting/1": { "name": "name_testtest", "motions_default_workflow_id": 12, - "motions_default_statute_amendment_workflow_id": 13, "motion_workflow_ids": [111, 2], "is_active_in_organization_id": 1, }, @@ -170,7 +165,6 @@ def test_delete_permissions(self) -> None: "meeting/1": { "name": "name_testtest", "motions_default_workflow_id": 12, - "motions_default_statute_amendment_workflow_id": 13, "motion_workflow_ids": [111, 2], "is_active_in_organization_id": 1, }, @@ -188,7 +182,6 @@ def test_delete_permissions_locked_meeting(self) -> None: "meeting/1": { "name": "name_testtest", "motions_default_workflow_id": 12, - "motions_default_statute_amendment_workflow_id": 13, "motion_workflow_ids": [111, 2], "is_active_in_organization_id": 1, }, diff --git a/tests/system/action/test_action_worker.py b/tests/system/action/test_action_worker.py index e9e2f0bde6..1a95df8c3b 100644 --- a/tests/system/action/test_action_worker.py +++ b/tests/system/action/test_action_worker.py @@ -184,7 +184,6 @@ def test_action_worker_permanent_stress(self) -> None: "motion_block": {}, "topic": {}, "assignment": {}, - "motion_statute_paragraph": {"text": "text"}, } def thread_method(self: ActionWorkerTest, collection: str) -> None: @@ -231,15 +230,10 @@ def test_action_worker_create_action_worker_during_running_db_action(self) -> No def thread_method(self: ActionWorkerTest) -> None: with self.lock: data = [ - { - "title": f"title{i}", - "meeting_id": 222, - "text": "text", - } - for i in range(1, self.number) + {"title": "boo", "username": "foo"} for i in range(1, self.number) ] self.start1 = datetime.now() - self.request_multi("motion_statute_paragraph.create", data) + self.request_multi("user.create", data) self.end1 = datetime.now() thread = Thread(target=thread_method, args=(self,)) diff --git a/tests/system/action/user/test_merge_together.py b/tests/system/action/user/test_merge_together.py index 1c4685b0fc..0e0180b5ea 100644 --- a/tests/system/action/user/test_merge_together.py +++ b/tests/system/action/user/test_merge_together.py @@ -103,7 +103,6 @@ def setUp(self) -> None: "language": "en", "motions_default_workflow_id": 1, "motions_default_amendment_workflow_id": 1, - "motions_default_statute_amendment_workflow_id": 1, "users_enable_vote_delegations": True, "committee_id": 1, "group_ids": [1, 2, 3], @@ -156,7 +155,6 @@ def setUp(self) -> None: "language": "en", "motions_default_workflow_id": 2, "motions_default_amendment_workflow_id": 2, - "motions_default_statute_amendment_workflow_id": 2, "users_enable_vote_delegations": True, "committee_id": 1, "group_ids": [4, 5, 6], @@ -215,7 +213,6 @@ def setUp(self) -> None: "language": "en", "motions_default_workflow_id": 3, "motions_default_amendment_workflow_id": 3, - "motions_default_statute_amendment_workflow_id": 3, "users_enable_vote_delegations": True, "committee_id": 2, "group_ids": [7, 8, 9], @@ -268,7 +265,6 @@ def setUp(self) -> None: "language": "en", "motions_default_workflow_id": 4, "motions_default_amendment_workflow_id": 4, - "motions_default_statute_amendment_workflow_id": 4, "users_enable_vote_delegations": True, "committee_id": 3, "group_ids": [10, 11, 12], diff --git a/tests/system/migrations/test_0059_remove_statutes.py b/tests/system/migrations/test_0059_remove_statutes.py new file mode 100644 index 0000000000..952d98b027 --- /dev/null +++ b/tests/system/migrations/test_0059_remove_statutes.py @@ -0,0 +1,1368 @@ +def write_comprehensive_data(write) -> dict: + data = { + "meeting/11": { + "id": 11, + "name": "string", + "language": "string", + "motions_statutes_enabled": True, + "motions_statute_recommendations_by": 12, + "motion_statute_paragraph_ids": [1, 2], + "motions_default_statute_amendment_workflow_id": 1, + "motion_ids": [1, 2], + "motion_workflow_ids": [1], + "motion_state_ids": [1], + "motion_category_ids": [1], + "motion_block_ids": [1], + "meeting_user_ids": [1, 2], + "tag_ids": [1, 2], + "mediafile_ids": [1], + "meeting_mediafile_ids": [1], + "motion_working_group_speaker_ids": [1], + "motion_change_recommendation_ids": [1], + "motion_submitter_ids": [1], + "motion_editor_ids": [1], + "motion_comment_ids": [1], + "motion_comment_section_ids": [1], + "all_projection_ids": [1, 2, 3, 4, 5, 6, 7], + "projector_ids": [1, 2], + "personal_note_ids": [1], + "group_ids": [1], + "poll_candidate_list_ids": [1], + "vote_ids": [1], + "option_ids": [1, 2, 3], + "poll_ids": [1], + "agenda_item_ids": [1, 2, 3], + "topic_ids": [1], + "structure_level_list_of_speakers_ids": [1], + "list_of_speakers_ids": [1], + "structure_level_ids": [1], + "speaker_ids": [1], + "point_of_order_category_ids": [1], + }, + "motion/1": { + "id": 1, + "statute_paragraph_id": 1, + "title": "text", + "recommendation_id": 1, + "amendment_ids": [666], + "state_extension_reference_ids": ["motion/2"], + "referenced_in_motion_state_extension_ids": [2], + "recommendation_extension_reference_ids": ["motion/2"], + "referenced_in_motion_recommendation_extension_ids": [2], + "category_id": 1, + "block_id": 1, + "supporter_meeting_user_ids": [1], + "tag_ids": [1, 2], + "attachment_meeting_mediafile_ids": [1], + "working_group_speaker_ids": [1], + "submitter_ids": [1], + "editor_ids": [1], + "comment_ids": [1], + "projection_ids": [1], + "personal_note_ids": [1], + "option_ids": [1], + "poll_ids": [1], + "agenda_item_id": 2, + "list_of_speakers_id": 1, + "meeting_id": 11, + }, + "motion/2": { + "id": 2, + "statute_paragraph_id": 2, + "title": "text", + "meeting_id": 11, + "recommendation_id": 1, + "state_extension_reference_ids": ["motion/1"], + "referenced_in_motion_state_extension_ids": [1], + "recommendation_extension_reference_ids": ["motion/1"], + "referenced_in_motion_recommendation_extension_ids": [1], + "change_recommendation_ids": [1], + "agenda_item_id": 1, + }, + "motion/666": {"id": 666, "lead_motion_id": 1}, + "motion_category/1": { + "id": 1, + "name": "AA", + "meeting_id": 11, + "motion_ids": [1], + }, + "motion_block/1": { + "id": 1, + "name": "AA", + "meeting_id": 11, + "motion_ids": [1], + }, + "meeting_user/1": { + "id": 1, + "user_id": 1, + "motion_editor_ids": [1], + "motion_submitter_ids": [1], + "motion_working_group_speaker_ids": [1], + "supported_motion_ids": [1], + "speaker_ids": [1], + "personal_note_ids": [1], + "meeting_id": 11, + }, + "meeting_user/2": { + "id": 2, + "user_id": 2, + "meeting_id": 11, + }, + "user/1": { + "id": 1, + "meeting_ids": [11], + "meeting_user_ids": [1], + "poll_voted_ids": [1], + "vote_ids": [1], + "option_ids": [2], + }, + "user/2": { + "id": 2, + "meeting_ids": [11], + "meeting_user_ids": [2], + "delegated_vote_ids": [1], + }, + "tag/1": { + "id": 1, + "name": "A Tag", + "meeting_id": 11, + "tagged_ids": ["motion/1", "agenda_item/3"], + }, + "tag/2": { + "id": 2, + "name": "A 2nd Tag", + "meeting_id": 11, + "tagged_ids": ["motion/1", "agenda_item/1", "agenda_item/2"], + }, + "meeting_mediafile/1": { + "id": 1, + "attachment_ids": ["motion/1"], + "mediafile_id": 1, + "is_public": True, + "meeting_id": 11, + }, + "mediafile/1": { + "id": 1, + "title": "A Media Attachment", + "owner_id": "meeting/11", + "is_public": True, + "meeting_mediafile_ids": [1], + }, + "motion_submitter/1": { + "id": 1, + "meeting_user_id": 1, + "motion_id": 1, + "meeting_id": 11, + }, + "motion_editor/1": { + "id": 1, + "meeting_user_id": 1, + "motion_id": 1, + "meeting_id": 11, + }, + "motion_working_group_speaker/1": { + "id": 1, + "meeting_user_id": 1, + "motion_id": 1, + "meeting_id": 11, + }, + "motion_change_recommendation/1": { + "id": 1, + "line_from": 1, + "line_to": 5, + "text": "HTML", + "motion_id": 2, + "meeting_id": 11, + }, + "motion_comment/1": { + "id": 1, + "comment": "HTML", + "motion_id": 1, + "section_id": 1, + "meeting_id": 11, + }, + "motion_comment_section/1": { + "id": 1, + "name": "A Comment Section", + "comment_ids": [1], + "meeting_id": 11, + }, + "projection/1": { + "id": 1, + "content_object_id": "motion/1", + "current_projector_id": 1, + "preview_projector_id": 1, + "history_projector_id": 2, + "meeting_id": 11, + }, + "projection/2": { + "id": 2, + "content_object_id": "poll/1", + "current_projector_id": 1, + "history_projector_id": 2, + "meeting_id": 11, + }, + "projection/3": { + "id": 3, + "content_object_id": "agenda_item/1", + "current_projector_id": 1, + "preview_projector_id": 2, + "meeting_id": 11, + }, + "projection/4": { + "id": 4, + "content_object_id": "agenda_item/2", + "current_projector_id": 2, + "meeting_id": 11, + }, + "projection/5": { + "id": 5, + "content_object_id": "agenda_item/3", + "history_projector_id": 1, + "meeting_id": 11, + }, + "projection/6": { + "id": 6, + "content_object_id": "topic/1", + "history_projector_id": 1, + "meeting_id": 11, + }, + "projection/7": { + "id": 7, + "content_object_id": "list_of_speakers/1", + "preview_projector_id": 2, + "meeting_id": 11, + }, + "projector/1": { + "id": 1, + "current_projection_ids": [1, 2, 3], + "preview_projection_ids": [1], + "history_projection_ids": [5, 6], + "meeting_id": 11, + }, + "projector/2": { + "id": 2, + "current_projection_ids": [4], + "preview_projection_ids": [3, 7], + "history_projection_ids": [1, 2], + "meeting_id": 11, + }, + "personal_note/1": { + "id": 1, + "meeting_user_id": 1, + "content_object_id": "motion/1", + "meeting_id": 11, + }, + "poll/1": { + "id": 1, + "content_object_id": "motion/1", + "option_ids": [1, 2, 3], + "global_option_id": 1, + "voted_ids": [1], + "entitled_group_ids": [1], + "projection_ids": [2], + "meeting_id": 11, + }, + "option/1": { + "id": 1, + "content_object_id": "motion/1", + "used_as_global_option_in_poll_id": 1, + "vote_ids": [1], + "poll_id": 1, + "meeting_id": 11, + }, + "option/2": { + "id": 2, + "content_object_id": "user/1", + "vote_ids": [], + "poll_id": 1, + "meeting_id": 11, + }, + "option/3": { + "id": 3, + "content_object_id": "poll_candidate_list/1", + "vote_ids": [], + "poll_id": 1, + "meeting_id": 11, + }, + "vote/1": { + "id": 1, + "delegated_user_id": 2, + "user_id": 1, + "meeting_id": 11, + "option_id": 1, + }, + "poll_candidate_list/1": { + "id": 1, + "option_id": 3, + "entries": {"user_id": 1, "weight": 20}, + "meeting_id": 11, + }, + "group/1": { + "id": 1, + "poll_ids": [1], + "name": "A Group", + "meeting_id": 11, + }, + "agenda_item/1": { + "id": 1, + "content_object_id": "motion/2", + "parent_id": None, + "child_ids": [2], + "tag_ids": [2], + "projection_ids": [3], + "meeting_id": 11, + }, + "agenda_item/2": { + "id": 2, + "content_object_id": "motion/1", + "parent_id": 1, + "child_ids": [3], + "tag_ids": [2], + "projection_ids": [4], + "meeting_id": 11, + }, + "agenda_item/3": { + "id": 3, + "content_object_id": "topic/1", + "parent_id": 2, + "child_ids": [], + "tag_ids": [1], + "projection_ids": [5], + "meeting_id": 11, + }, + "topic/1": { + "id": 1, + "title": "A tropical topic", + "agenda_item_id": 3, + "projection_ids": [6], + "meeting_id": 11, + }, + "list_of_speakers/1": { + "id": 1, + "content_object_id": "motion/1", + "speaker_ids": [1], + "structure_level_list_of_speakers_ids": [1], + "projection_ids": [7], + "meeting_id": 11, + }, + "structure_level_list_of_speakers/1": { + "id": 1, + "structure_level_id": 1, + "list_of_speakers_id": 1, + "initial_time": 30, + "speaker_ids": [1], + "meeting_id": 11, + }, + "structure_level/1": { + "id": 1, + "name": "ErstePartei", + "color": "#FF0000", + "default_time": 30, + "structure_level_list_of_speakers_ids": [1], + "meeting_id": 11, + }, + "speaker/1": { + "id": 1, + "meeting_user_id": 1, + "point_of_order_category_id": 1, + "list_of_speakers_id": 1, + "structure_level_list_of_speakers_id": 1, + "meeting_id": 11, + }, + "point_of_order_category/1": { + "id": 1, + "text": "A point of order category", + "speaker_ids": [1], + "rank": 1, + "meeting_id": 11, + }, + "motion_workflow/1": { + "id": 1, + "default_statute_amendment_workflow_meeting_id": 11, + "state_ids": [1], + "meeting_id": 11, + }, + "motion_statute_paragraph/1": { + "id": 1, + "title": "string", + "text": "HTML", + "meeting_id": 11, + "motion_ids": [1], + }, + "motion_statute_paragraph/2": { + "id": 2, + "title": "string", + "text": "HTML", + "meeting_id": 11, + "motion_ids": [2], + }, + "motion_state/1": { + "id": 1, + "name": "string", + "workflow_id": 1, + "motion_recommendation_ids": [1, 2], + "meeting_id": 11, + }, + } + write( + *[ + { + "type": "create", + "fqid": fqid, + "fields": fields, + } + for fqid, fields in data.items() + ] + ) + return data + + +def test_delete_motion_without_sideffects(write, finalize, assert_model): + write( + { + "type": "create", + "fqid": "meeting/11", + "fields": { + "id": 11, + "name": "string", + "language": "string", + "motions_statutes_enabled": True, + "motions_statute_recommendations_by": 12, + "motion_statute_paragraph_ids": [1], + "motion_ids": [1, 2], + }, + }, + { + "type": "create", + "fqid": "motion/1", + "fields": { + "id": 1, + "statute_paragraph_id": 1, + "title": "text", + "meeting_id": 11, + }, + }, + { + "type": "create", + "fqid": "motion/2", + "fields": { + "id": 2, + "title": "text", + "meeting_id": 11, + }, + }, + { + "type": "create", + "fqid": "motion_statute_paragraph/1", + "fields": { + "id": 1, + "title": "string", + "text": "HTML", + "meeting_id": 11, + "motion_ids": [1], + }, + }, + ) + finalize("0059_remove_statutes") + assert_model( + "motion/2", + { + "id": 2, + "title": "text", + "meeting_id": 11, + }, + ) + assert_model( + "motion/1", + { + "id": 1, + "statute_paragraph_id": 1, + "title": "text", + "meeting_id": 11, + "meta_deleted": True, + }, + ) + + +def test_no_sideffects_submodels(write, finalize, assert_model): + write_comprehensive_data(write) + data = { + "motion/3": { + "id": 3, + "title": "text", + "recommendation_id": 2, + "state_extension_reference_ids": ["motion/4"], + "referenced_in_motion_state_extension_ids": [4], + "recommendation_extension_reference_ids": ["motion/4"], + "referenced_in_motion_recommendation_extension_ids": [4], + "category_id": 1, + "block_id": 2, + "supporter_meeting_user_ids": [1], + "tag_ids": [1], + "attachment_meeting_mediafile_ids": [2], + "working_group_speaker_ids": [2], + "submitter_ids": [2], + "editor_ids": [2], + "comment_ids": [2], + "projection_ids": [11], + "personal_note_ids": [2], + "option_ids": [11], + "poll_ids": [2], + "agenda_item_id": 12, + "list_of_speakers_id": 2, + "meeting_id": 11, + }, + "motion/4": { + "id": 4, + "title": "text", + "recommendation_id": 2, + "state_extension_reference_ids": ["motion/3"], + "referenced_in_motion_state_extension_ids": [3], + "recommendation_extension_reference_ids": ["motion/3"], + "referenced_in_motion_recommendation_extension_ids": [3], + "change_recommendation_ids": [2], + "agenda_item_id": 11, + "meeting_id": 11, + }, + "motion_block/2": {"id": 2, "name": "AA", "meeting_id": 11, "motion_ids": [3]}, + "meeting_mediafile/2": { + "id": 2, + "mediafile_id": 2, + "attachment_ids": ["motion/3"], + "is_public": True, + "meeting_id": 11, + }, + "mediafile/2": { + "id": 2, + "title": "A Media Attachment", + "owner_id": "meeting/11", + "is_public": True, + "meeting_mediafile_ids": [2], + }, + "motion_submitter/2": { + "id": 2, + "meeting_user_id": 1, + "motion_id": 3, + "meeting_id": 11, + }, + "motion_editor/2": { + "id": 2, + "meeting_user_id": 1, + "motion_id": 3, + "meeting_id": 11, + }, + "motion_working_group_speaker/2": { + "id": 2, + "meeting_user_id": 1, + "motion_id": 3, + "meeting_id": 11, + }, + "motion_change_recommendation/2": { + "id": 2, + "line_from": 1, + "line_to": 5, + "text": "HTML", + "motion_id": 4, + "meeting_id": 11, + }, + "motion_comment/2": { + "id": 2, + "comment": "HTML", + "motion_id": 3, + "section_id": 2, + "meeting_id": 11, + }, + "motion_comment_section/2": { + "id": 2, + "name": "A Comment Section", + "comment_ids": [2], + "meeting_id": 11, + }, + "projection/11": { + "id": 11, + "content_object_id": "motion/3", + "current_projector_id": 3, + "preview_projector_id": 3, + "history_projector_id": 4, + "meeting_id": 11, + }, + "projection/12": { + "id": 12, + "content_object_id": "poll/2", + "current_projector_id": 3, + "history_projector_id": 4, + "meeting_id": 11, + }, + "projection/13": { + "id": 13, + "content_object_id": "agenda_item/11", + "current_projector_id": 3, + "preview_projector_id": 4, + "meeting_id": 11, + }, + "projection/14": { + "id": 14, + "content_object_id": "agenda_item/12", + "current_projector_id": 4, + "meeting_id": 11, + }, + "projection/15": { + "id": 15, + "content_object_id": "agenda_item/13", + "history_projector_id": 3, + "meeting_id": 11, + }, + "projection/16": { + "id": 16, + "content_object_id": "topic/2", + "history_projector_id": 3, + "meeting_id": 11, + }, + "projection/17": { + "id": 17, + "content_object_id": "list_of_speakers/2", + "preview_projector_id": 4, + "meeting_id": 11, + }, + "projector/3": { + "id": 3, + "current_projection_ids": [11, 12, 13], + "preview_projection_ids": [11], + "history_projection_ids": [15, 16], + "meeting_id": 11, + }, + "projector/4": { + "id": 4, + "current_projection_ids": [14], + "preview_projection_ids": [13, 17], + "history_projection_ids": [11, 12], + "meeting_id": 11, + }, + "personal_note/2": { + "id": 2, + "meeting_user_id": 1, + "content_object_id": "motion/3", + "meeting_id": 11, + }, + "poll/2": { + "id": 2, + "content_object_id": "motion/3", + "option_ids": [11, 12, 13], + "global_option_id": 11, + "voted_ids": [1, 2], + "entitled_group_ids": [1], + "projection_ids": [12], + "meeting_id": 11, + }, + "option/11": { + "id": 11, + "content_object_id": "motion/3", + "used_as_global_option_in_poll_id": 2, + "vote_ids": [2], + "poll_id": 2, + "meeting_id": 11, + }, + "option/12": { + "id": 12, + "content_object_id": "user/1", + "vote_ids": [], + "poll_id": 2, + "meeting_id": 11, + }, + "option/13": { + "id": 13, + "content_object_id": "poll_candidate_list/2", + "vote_ids": [], + "poll_id": 2, + "meeting_id": 11, + }, + "vote/2": { + "id": 2, + "delegated_user_id": 2, + "user_id": 1, + "meeting_id": 11, + "option_id": 11, + }, + "poll_candidate_list/2": { + "id": 2, + "option_id": 13, + "entries": {"user_id": 1, "weight": 20}, + "meeting_id": 11, + }, + "agenda_item/11": { + "id": 11, + "content_object_id": "motion/4", + "parent_id": None, + "child_ids": [12], + "tag_ids": [2], + "projection_ids": [13], + "meeting_id": 11, + }, + "agenda_item/12": { + "id": 12, + "content_object_id": "motion/3", + "parent_id": 11, + "child_ids": [13], + "tag_ids": [2], + "projection_ids": [14], + "meeting_id": 11, + }, + "agenda_item/13": { + "id": 13, + "content_object_id": "topic/2", + "parent_id": 12, + "child_ids": [], + "tag_ids": [1], + "projection_ids": [15], + "meeting_id": 11, + }, + "topic/2": { + "id": 2, + "title": "A topical tropic", + "agenda_item_id": 13, + "projection_ids": [16], + "meeting_id": 11, + }, + "list_of_speakers/2": { + "id": 2, + "content_object_id": "motion/3", + "speaker_ids": [2], + "structure_level_list_of_speakers_ids": [2], + "projection_ids": [17], + "meeting_id": 11, + }, + "structure_level_list_of_speakers/2": { + "id": 2, + "structure_level_id": 1, + "list_of_speakers_id": 2, + "initial_time": 30, + "speaker_ids": [2], + "meeting_id": 11, + }, + "speaker/2": { + "id": 2, + "meeting_user_id": 1, + "point_of_order_category_id": 2, + "list_of_speakers_id": 2, + "structure_level_list_of_speakers_id": 2, + "meeting_id": 11, + }, + "point_of_order_category/2": { + "id": 2, + "text": "A point of order category", + "speaker_ids": [2], + "rank": 1, + "meeting_id": 11, + }, + "motion_workflow/2": { + "id": 2, + "state_ids": [2], + "meeting_id": 11, + }, + "motion_state/2": { + "id": 2, + "name": "string", + "workflow_id": 2, + "motion_recommendation_ids": [3, 4], + "meeting_id": 11, + }, + } + write( + { + "type": "update", + "fqid": "meeting/11", + "fields": { + "motion_ids": [1, 2, 3, 4], + "motion_workflow_ids": [1, 2], + "motion_state_ids": [1, 2], + "motion_block_ids": [1, 2], + "mediafile_ids": [1, 2], + "meeting_mediafile_ids": [1, 2], + "motion_working_group_speaker_ids": [1, 2], + "motion_change_recommendation_ids": [1, 2], + "motion_submitter_ids": [1, 2], + "motion_editor_ids": [1, 2], + "motion_comment_ids": [1, 2], + "motion_comment_section_ids": [1, 2], + "all_projection_ids": [1, 2, 3, 4, 5, 6, 7, 11, 12, 13, 14, 15, 16, 17], + "projector_ids": [1, 2, 3, 4], + "personal_note_ids": [1, 2], + "poll_candidate_list_ids": [1, 2], + "vote_ids": [1, 2], + "option_ids": [1, 2, 3, 11, 12, 13], + "poll_ids": [1, 2], + "agenda_item_ids": [1, 2, 3, 11, 12, 13], + "topic_ids": [1, 2], + "list_of_speakers_ids": [1, 2], + "structure_level_list_of_speakers_ids": [1, 2], + "speaker_ids": [1, 2], + "point_of_order_category_ids": [1, 2], + }, + }, + { + "type": "update", + "fqid": "structure_level/1", + "fields": { + "structure_level_list_of_speakers_ids": [1, 2], + }, + }, + { + "type": "update", + "fqid": "motion_category/1", + "fields": {"motion_ids": [1, 3]}, + }, + { + "type": "update", + "fqid": "meeting_user/1", + "fields": { + "motion_editor_ids": [1, 2], + "motion_submitter_ids": [1, 2], + "motion_working_group_speaker_ids": [1, 2], + "supported_motion_ids": [1, 3], + "speaker_ids": [1, 2], + "personal_note_ids": [1, 2], + "meeting_id": 11, + }, + }, + { + "type": "update", + "fqid": "user/1", + "fields": { + "poll_voted_ids": [1, 2], + "vote_ids": [1, 2], + "option_ids": [2, 12], + }, + }, + { + "type": "update", + "fqid": "user/2", + "fields": { + "delegated_vote_ids": [1, 2], + "poll_voted_ids": [1, 2], + }, + }, + { + "type": "update", + "fqid": "group/1", + "fields": { + "poll_ids": [1, 2], + }, + }, + { + "type": "update", + "fqid": "poll/1", + "fields": { + "voted_ids": [1, 2], + }, + }, + { + "type": "update", + "fqid": "tag/1", + "fields": { + "tagged_ids": [ + "motion/1", + "agenda_item/3", + "motion/3", + "agenda_item/13", + ], + }, + }, + { + "type": "update", + "fqid": "tag/2", + "fields": { + "tagged_ids": [ + "agenda_item/1", + "agenda_item/2", + "agenda_item/11", + "agenda_item/12", + ], + }, + }, + *[ + { + "type": "create", + "fqid": fqid, + "fields": fields, + } + for fqid, fields in data.items() + ] + ) + finalize("0059_remove_statutes") + assert_model( + "motion/1", + { + "id": 1, + "statute_paragraph_id": 1, + "title": "text", + "amendment_ids": [666], + "agenda_item_id": 2, + "attachment_meeting_mediafile_ids": [1], + "block_id": 1, + "category_id": 1, + "comment_ids": [1], + "working_group_speaker_ids": [1], + "editor_ids": [1], + "list_of_speakers_id": 1, + "option_ids": [1], + "personal_note_ids": [1], + "poll_ids": [1], + "recommendation_extension_reference_ids": ["motion/2"], + "recommendation_id": 1, + "referenced_in_motion_recommendation_extension_ids": [2], + "referenced_in_motion_state_extension_ids": [2], + "state_extension_reference_ids": ["motion/2"], + "submitter_ids": [1], + "supporter_meeting_user_ids": [1], + "tag_ids": [1, 2], + "meeting_id": 11, + "projection_ids": [1], + "meta_deleted": True, + }, + ) + assert_model( + "motion/2", + { + "id": 2, + "title": "text", + "agenda_item_id": 1, + "change_recommendation_ids": [1], + "recommendation_extension_reference_ids": ["motion/1"], + "recommendation_id": 1, + "referenced_in_motion_recommendation_extension_ids": [1], + "referenced_in_motion_state_extension_ids": [1], + "state_extension_reference_ids": ["motion/1"], + "statute_paragraph_id": 2, + "meeting_id": 11, + "meta_deleted": True, + }, + ) + assert_model( + "tag/2", + { + "id": 2, + "name": "A 2nd Tag", + "meeting_id": 11, + "tagged_ids": ["agenda_item/11", "agenda_item/12"], + }, + ) + for fqid, fields in data.items(): + assert_model(fqid, fields) + + +def test_two_meetings(write, finalize, assert_model): + write_comprehensive_data(write) + write( + { + "type": "create", + "fqid": "meeting/12", + "fields": { + "id": 12, + "name": "string", + "language": "string", + "motions_statutes_enabled": True, + "motions_statute_recommendations_by": 12, + "motion_statute_paragraph_ids": [3], + "motion_ids": [3], + "motion_state_ids": [2], + "motion_workflow_ids": [2], + "motions_default_statute_amendment_workflow_id": 2, + }, + }, + { + "type": "create", + "fqid": "motion_statute_paragraph/3", + "fields": { + "id": 3, + "title": "string", + "text": "HTML", + "meeting_id": 12, + "motion_ids": [3], + }, + }, + { + "type": "create", + "fqid": "motion/3", + "fields": { + "id": 3, + "statute_paragraph_id": 3, + "title": "text", + "meeting_id": 12, + "recommendation_id": 2, + }, + }, + { + "type": "create", + "fqid": "motion_workflow/2", + "fields": { + "id": 2, + "default_statute_amendment_workflow_meeting_id": 12, + "state_ids": [2], + "meeting_id": 12, + }, + }, + { + "type": "create", + "fqid": "motion_state/2", + "fields": { + "id": 2, + "name": "string", + "workflow_id": 2, + "motion_recommendation_ids": [3], + "meeting_id": 12, + }, + }, + ) + finalize("0059_remove_statutes") + assert_model( + "motion/1", + { + "id": 1, + "statute_paragraph_id": 1, + "title": "text", + "meeting_id": 11, + "recommendation_id": 1, + "amendment_ids": [666], + "state_extension_reference_ids": ["motion/2"], + "referenced_in_motion_state_extension_ids": [2], + "recommendation_extension_reference_ids": ["motion/2"], + "referenced_in_motion_recommendation_extension_ids": [2], + "attachment_meeting_mediafile_ids": [1], + "block_id": 1, + "category_id": 1, + "supporter_meeting_user_ids": [1], + "tag_ids": [1, 2], + "comment_ids": [1], + "working_group_speaker_ids": [1], + "editor_ids": [1], + "personal_note_ids": [1], + "submitter_ids": [1], + "poll_ids": [1], + "option_ids": [1], + "agenda_item_id": 2, + "list_of_speakers_id": 1, + "projection_ids": [1], + "meta_deleted": True, + }, + ) + assert_model( + "motion/3", + { + "id": 3, + "statute_paragraph_id": 3, + "title": "text", + "meeting_id": 12, + "recommendation_id": 2, + "meta_deleted": True, + }, + ) + assert_model( + "meeting/11", + { + "id": 11, + "name": "string", + "language": "string", + "motion_workflow_ids": [1], + "motion_state_ids": [1], + "mediafile_ids": [1], + "meeting_mediafile_ids": [1], + "meeting_user_ids": [1, 2], + "motion_block_ids": [1], + "motion_category_ids": [1], + "motion_comment_section_ids": [1], + "tag_ids": [1, 2], + "group_ids": [1], + "poll_candidate_list_ids": [1], + "agenda_item_ids": [3], + "all_projection_ids": [5, 6], + "projector_ids": [1, 2], + "topic_ids": [1], + "point_of_order_category_ids": [1], + "structure_level_ids": [1], + }, + ) + assert_model( + "meeting/12", + { + "id": 12, + "name": "string", + "language": "string", + "motion_state_ids": [2], + "motion_workflow_ids": [2], + }, + ) + assert_model( + "motion_statute_paragraph/1", + { + "id": 1, + "title": "string", + "text": "HTML", + "meeting_id": 11, + "motion_ids": [1], + "meta_deleted": True, + }, + ) + assert_model( + "motion_statute_paragraph/2", + { + "id": 2, + "title": "string", + "text": "HTML", + "meeting_id": 11, + "motion_ids": [2], + "meta_deleted": True, + }, + ) + assert_model( + "motion_statute_paragraph/3", + { + "id": 3, + "title": "string", + "text": "HTML", + "meeting_id": 12, + "motion_ids": [3], + "meta_deleted": True, + }, + ) + assert_model( + "motion_workflow/2", + { + "id": 2, + "state_ids": [2], + "meeting_id": 12, + }, + ) + + +def test_migration_full(write, finalize, assert_model): + data = write_comprehensive_data(write) + finalize("0059_remove_statutes") + data_update = { + "meeting/11": { + "motion_statute_paragraph_ids": None, + "motions_statutes_enabled": None, + "motions_statute_recommendations_by": None, + "motions_default_statute_amendment_workflow_id": None, + "agenda_item_ids": [3], + "all_projection_ids": [5, 6], + "motion_change_recommendation_ids": None, + "motion_comment_ids": None, + "motion_editor_ids": None, + "motion_ids": None, + "motion_submitter_ids": None, + "list_of_speakers_ids": None, + "structure_level_list_of_speakers_ids": None, + "vote_ids": None, + "speaker_ids": None, + "poll_ids": None, + "personal_note_ids": None, + "option_ids": None, + "motion_working_group_speaker_ids": None, + }, + "motion/1": { + "meta_deleted": True, + }, + "motion/2": { + "meta_deleted": True, + }, + "motion/666": { + "meta_deleted": True, + }, + "motion_category/1": {"motion_ids": None}, + "motion_block/1": {"motion_ids": None}, + "meeting_user/1": { + "motion_editor_ids": None, + "motion_submitter_ids": None, + "motion_working_group_speaker_ids": None, + "personal_note_ids": None, + "speaker_ids": None, + "supported_motion_ids": None, + }, + "meeting_user/2": { + "motion_editor_ids": None, + "motion_submitter_ids": None, + "motion_working_group_speaker_ids": None, + "personal_note_ids": None, + "speaker_ids": None, + "supported_motion_ids": None, + }, + "user/1": { + "motion_ids": None, + "option_ids": None, + "poll_voted_ids": None, + "vote_ids": None, + }, + "user/2": { + "motion_ids": None, + "delegated_vote_ids": None, + }, + "tag/1": {"tagged_ids": ["agenda_item/3"]}, + "tag/2": {"tagged_ids": None}, + "meeting_mediafile/1": {"attachment_ids": None}, + "motion_submitter/1": { + "meta_deleted": True, + }, + "motion_editor/1": { + "meta_deleted": True, + }, + "motion_working_group_speaker/1": { + "meta_deleted": True, + }, + "motion_change_recommendation/1": { + "meta_deleted": True, + }, + "motion_statute_paragraph/1": { + "meta_deleted": True, + }, + "motion_statute_paragraph/2": { + "meta_deleted": True, + }, + "motion_comment/1": { + "meta_deleted": True, + }, + "motion_comment_section/1": {"comment_ids": None}, + "projection/1": { + "meta_deleted": True, + }, + "projection/2": { + "meta_deleted": True, + }, + "projection/3": { + "meta_deleted": True, + }, + "projection/4": { + "meta_deleted": True, + }, + "projection/7": { + "meta_deleted": True, + }, + "projector/1": { + "preview_projection_ids": None, + "current_projection_ids": None, + }, + "projector/2": { + "preview_projection_ids": None, + "current_projection_ids": None, + "history_projection_ids": None, + }, + "personal_note/1": { + "meta_deleted": True, + }, + "poll/1": { + "meta_deleted": True, + }, + "option/1": { + "meta_deleted": True, + }, + "option/2": { + "meta_deleted": True, + }, + "option/3": { + "meta_deleted": True, + }, + "vote/1": { + "meta_deleted": True, + }, + "poll_candidate_list/1": {"option_id": None}, + "group/1": {"poll_ids": None}, + "agenda_item/1": { + "meta_deleted": True, + }, + "agenda_item/2": { + "meta_deleted": True, + }, + "agenda_item/3": { + "parent_id": None, + }, + "list_of_speakers/1": { + "meta_deleted": True, + }, + "structure_level_list_of_speakers/1": { + "meta_deleted": True, + }, + "structure_level/1": { + "structure_level_list_of_speakers_ids": None, + }, + "speaker/1": { + "meta_deleted": True, + }, + "point_of_order_category/1": { + "speaker_ids": None, + }, + "motion_workflow/1": {"default_statute_amendment_workflow_meeting_id": None}, + "motion_state/1": { + "motion_recommendation_ids": None, + }, + } + for fqid, fields in data_update.items(): + data_fields = data.get(fqid) + data_fields.update(fields) + for key, value in fields.items(): + if value is None: + del data_fields[key] + for fqid, fields in data.items(): + assert_model(fqid, fields) + + +def test_non_deleted_motion_extension(write, finalize, assert_model): + """ + Tests if the fields state_extension_reference_ids and recommendation_extension_reference_ids + are correctly processed and the corresponding motions updated. + """ + write_comprehensive_data(write) + write( + { + "type": "update", + "fqid": "motion/2", + "fields": { + "state_extension_reference_ids": ["motion/1", "motion/3"], + "recommendation_extension_reference_ids": ["motion/1", "motion/3"], + }, + }, + {"type": "update", "fqid": "meeting/11", "fields": {"motion_ids": [1, 2, 3]}}, + { + "type": "update", + "fqid": "motion_state/1", + "fields": {"motion_recommendation_ids": [1, 2, 3]}, + }, + { + "type": "create", + "fqid": "motion/3", + "fields": { + "id": 3, + "title": "text", + "meeting_id": 11, + "recommendation_id": 1, + "referenced_in_motion_state_extension_ids": [2], + "referenced_in_motion_recommendation_extension_ids": [2], + }, + }, + ) + finalize("0059_remove_statutes") + assert_model( + "motion/3", + { + "id": 3, + "title": "text", + "meeting_id": 11, + "recommendation_id": 1, + }, + ) + assert_model( + "motion/2", + { + "id": 2, + "statute_paragraph_id": 2, + "title": "text", + "recommendation_id": 1, + "state_extension_reference_ids": ["motion/1", "motion/3"], + "referenced_in_motion_state_extension_ids": [1], + "recommendation_extension_reference_ids": ["motion/1", "motion/3"], + "referenced_in_motion_recommendation_extension_ids": [1], + "change_recommendation_ids": [1], + "agenda_item_id": 1, + "meeting_id": 11, + "meta_deleted": True, + }, + ) diff --git a/tests/system/presenter/test_check_database.py b/tests/system/presenter/test_check_database.py index 5996ea7d76..c4d1d3257c 100644 --- a/tests/system/presenter/test_check_database.py +++ b/tests/system/presenter/test_check_database.py @@ -98,7 +98,6 @@ def get_meeting_defaults(self) -> dict[str, Any]: "motions_number_type": "per_category", "motions_number_min_digits": 2, "motions_number_with_blank": False, - "motions_statutes_enabled": False, "motions_amendments_enabled": True, "motions_amendments_in_main_list": True, "motions_amendments_of_amendments": False, @@ -160,7 +159,6 @@ def test_correct(self) -> None: "default_group_id": 1, "admin_group_id": 2, "motions_default_amendment_workflow_id": 1, - "motions_default_statute_amendment_workflow_id": 1, "motions_default_workflow_id": 1, "reference_projector_id": 1, "projector_countdown_default_time": 60, @@ -190,7 +188,6 @@ def test_correct(self) -> None: "name": "blup", "first_state_id": 1, "default_amendment_workflow_meeting_id": 1, - "default_statute_amendment_workflow_meeting_id": 1, "default_workflow_meeting_id": 1, "state_ids": [1], "sequential_number": 1, @@ -294,7 +291,6 @@ def test_correct_relations(self) -> None: "default_group_id": 1, "admin_group_id": 2, "motions_default_amendment_workflow_id": 1, - "motions_default_statute_amendment_workflow_id": 1, "motions_default_workflow_id": 1, "reference_projector_id": 1, "projector_countdown_default_time": 60, @@ -415,7 +411,6 @@ def test_correct_relations(self) -> None: "name": "blup", "first_state_id": 1, "default_amendment_workflow_meeting_id": 1, - "default_statute_amendment_workflow_meeting_id": 1, "default_workflow_meeting_id": 1, "state_ids": [1], "sequential_number": 1, @@ -560,7 +555,6 @@ def test_relation_2(self) -> None: "default_group_id": 1, "admin_group_id": 2, "motions_default_amendment_workflow_id": 1, - "motions_default_statute_amendment_workflow_id": 1, "motions_default_workflow_id": 1, "reference_projector_id": 1, "projector_countdown_default_time": 60, @@ -592,7 +586,6 @@ def test_relation_2(self) -> None: "name": "blup", "first_state_id": 1, "default_amendment_workflow_meeting_id": 1, - "default_statute_amendment_workflow_meeting_id": 1, "default_workflow_meeting_id": 1, "state_ids": [1], "sequential_number": 1, @@ -646,7 +639,6 @@ def test_relation_2(self) -> None: "default_group_id": 3, "admin_group_id": 4, "motions_default_amendment_workflow_id": 2, - "motions_default_statute_amendment_workflow_id": 2, "motions_default_workflow_id": 2, "reference_projector_id": 2, "projector_countdown_default_time": 60, @@ -678,7 +670,6 @@ def test_relation_2(self) -> None: "name": "blup", "first_state_id": 2, "default_amendment_workflow_meeting_id": 2, - "default_statute_amendment_workflow_meeting_id": 2, "default_workflow_meeting_id": 2, "state_ids": [2], "sequential_number": 2, diff --git a/tests/system/presenter/test_check_database_all.py b/tests/system/presenter/test_check_database_all.py index 7f8967df88..3f99e6a2c0 100644 --- a/tests/system/presenter/test_check_database_all.py +++ b/tests/system/presenter/test_check_database_all.py @@ -91,7 +91,6 @@ def get_meeting_defaults(self) -> dict[str, Any]: "motions_number_type": "per_category", "motions_number_min_digits": 2, "motions_number_with_blank": False, - "motions_statutes_enabled": False, "motions_amendments_enabled": True, "motions_amendments_in_main_list": True, "motions_amendments_of_amendments": False, @@ -191,7 +190,6 @@ def test_correct(self) -> None: "default_group_id": 1, "admin_group_id": 2, "motions_default_amendment_workflow_id": 1, - "motions_default_statute_amendment_workflow_id": 1, "motions_default_workflow_id": 1, "reference_projector_id": 1, "projector_countdown_default_time": 60, @@ -221,7 +219,6 @@ def test_correct(self) -> None: "name": "blup", "first_state_id": 1, "default_amendment_workflow_meeting_id": 1, - "default_statute_amendment_workflow_meeting_id": 1, "default_workflow_meeting_id": 1, "state_ids": [1], "sequential_number": 1, @@ -377,7 +374,6 @@ def test_correct_relations(self) -> None: "default_group_id": 1, "admin_group_id": 2, "motions_default_amendment_workflow_id": 1, - "motions_default_statute_amendment_workflow_id": 1, "motions_default_workflow_id": 1, "reference_projector_id": 1, "projector_countdown_default_time": 60, @@ -498,7 +494,6 @@ def test_correct_relations(self) -> None: "name": "blup", "first_state_id": 1, "default_amendment_workflow_meeting_id": 1, - "default_statute_amendment_workflow_meeting_id": 1, "default_workflow_meeting_id": 1, "state_ids": [1], "sequential_number": 1, @@ -685,7 +680,6 @@ def test_relation_2(self) -> None: "default_group_id": 1, "admin_group_id": 2, "motions_default_amendment_workflow_id": 1, - "motions_default_statute_amendment_workflow_id": 1, "motions_default_workflow_id": 1, "reference_projector_id": 1, "projector_countdown_default_time": 60, @@ -717,7 +711,6 @@ def test_relation_2(self) -> None: "name": "blup", "first_state_id": 1, "default_amendment_workflow_meeting_id": 1, - "default_statute_amendment_workflow_meeting_id": 1, "default_workflow_meeting_id": 1, "state_ids": [1], "sequential_number": 1, @@ -776,7 +769,6 @@ def test_relation_2(self) -> None: "default_group_id": 3, "admin_group_id": 4, "motions_default_amendment_workflow_id": 2, - "motions_default_statute_amendment_workflow_id": 2, "motions_default_workflow_id": 2, "reference_projector_id": 2, "projector_countdown_default_time": 60, @@ -808,7 +800,6 @@ def test_relation_2(self) -> None: "name": "blup", "first_state_id": 2, "default_amendment_workflow_meeting_id": 2, - "default_statute_amendment_workflow_meeting_id": 2, "default_workflow_meeting_id": 2, "state_ids": [2], "sequential_number": 2, diff --git a/tests/system/presenter/test_export_meeting.py b/tests/system/presenter/test_export_meeting.py index ac64eb1425..440c57b4ba 100644 --- a/tests/system/presenter/test_export_meeting.py +++ b/tests/system/presenter/test_export_meeting.py @@ -29,7 +29,6 @@ def test_correct(self) -> None: "motion_change_recommendation", "motion_state", "motion_workflow", - "motion_statute_paragraph", "poll", "option", "vote", From fbab1b5dc8d129c2962ed8a29e515b56cf44cd82 Mon Sep 17 00:00:00 2001 From: "openslides-automation[bot]" <125256978+openslides-automation[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2024 14:15:39 +0000 Subject: [PATCH 24/90] Update meta repository (#2680) Co-authored-by: hjanott <12833127+hjanott@users.noreply.github.com> Co-authored-by: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> From 055cb83efa7a705629abc9804084475238ea1a06 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2024 16:32:12 +0200 Subject: [PATCH 25/90] Bump mypy from 1.11.2 to 1.12.0 in /requirements/partial (#2676) Bumps [mypy](https://github.com/python/mypy) from 1.11.2 to 1.12.0. - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v1.11.2...v1.12.0) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> --- requirements/partial/requirements_development.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/partial/requirements_development.txt b/requirements/partial/requirements_development.txt index 251fc97ccd..335d12ef34 100644 --- a/requirements/partial/requirements_development.txt +++ b/requirements/partial/requirements_development.txt @@ -4,7 +4,7 @@ black==24.10.0 debugpy==1.8.6 flake8==7.1.1 isort==5.13.2 -mypy==1.11.2 +mypy==1.12.0 pip-check==2.9 pytest==8.3.3 pytest-cov==5.0.0 From fdb682451915fb639ca5c9ef1c3818dd8e02f30a Mon Sep 17 00:00:00 2001 From: "openslides-automation[bot]" <125256978+openslides-automation[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2024 16:50:09 +0200 Subject: [PATCH 26/90] Update meta repository (#2681) Co-authored-by: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> --- global/meta | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global/meta b/global/meta index 85d328539b..5ecbba4daa 160000 --- a/global/meta +++ b/global/meta @@ -1 +1 @@ -Subproject commit 85d328539bc82678973d83e40e5793d1b2a852c5 +Subproject commit 5ecbba4daaca2579dd9fa90d27fd258510e39af5 From 5c7e05f7c4f0f6aeacb8b4128fc45cb0417ad9d9 Mon Sep 17 00:00:00 2001 From: Hannes Janott Date: Tue, 15 Oct 2024 18:15:05 +0200 Subject: [PATCH 27/90] make meeting clone with gender in user possible again (#2678) * add internal_target bool to prevent gender id conversion during meeting.clone --- .../action/actions/meeting/clone.py | 2 +- openslides_backend/shared/export_helper.py | 16 ++++++++++------ tests/system/action/meeting/test_clone.py | 5 ++++- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/openslides_backend/action/actions/meeting/clone.py b/openslides_backend/action/actions/meeting/clone.py index 27b4fae8a1..74a66224fa 100644 --- a/openslides_backend/action/actions/meeting/clone.py +++ b/openslides_backend/action/actions/meeting/clone.py @@ -92,7 +92,7 @@ def check_permissions(self, instance: dict[str, Any]) -> None: MeetingPermissionMixin.check_permissions(self, instance) def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: - meeting_json = export_meeting(self.datastore, instance["meeting_id"]) + meeting_json = export_meeting(self.datastore, instance["meeting_id"], True) instance["meeting"] = meeting_json additional_user_ids = instance.pop("user_ids", None) or [] additional_admin_ids = instance.pop("admin_ids", None) or [] diff --git a/openslides_backend/shared/export_helper.py b/openslides_backend/shared/export_helper.py index 5c05bec769..17d987ceb5 100644 --- a/openslides_backend/shared/export_helper.py +++ b/openslides_backend/shared/export_helper.py @@ -21,7 +21,9 @@ FORBIDDEN_FIELDS = ["forwarded_motion_ids"] -def export_meeting(datastore: DatastoreService, meeting_id: int) -> dict[str, Any]: +def export_meeting( + datastore: DatastoreService, meeting_id: int, internal_target: bool = False +) -> dict[str, Any]: export: dict[str, Any] = {} # fetch meeting @@ -129,7 +131,7 @@ def export_meeting(datastore: DatastoreService, meeting_id: int) -> dict[str, An elif collection_from_fqid(entry[field_name]) == "meeting_user": id_ = id_from_fqid(entry[field_name]) user_ids.add(results["meeting_user"][id_]["user_id"]) - add_users(list(user_ids), export, meeting_id, datastore) + add_users(list(user_ids), export, meeting_id, datastore, internal_target) return export @@ -138,6 +140,7 @@ def add_users( export_data: dict[str, Any], meeting_id: int, datastore: DatastoreService, + internal_target: bool, ) -> None: if not user_ids: return @@ -157,15 +160,16 @@ def add_users( ) for user in users.values(): - gender_dict = datastore.get_all("gender", ["name"], lock_result=False) user["meeting_ids"] = [meeting_id] if meeting_id in (user.get("is_present_in_meeting_ids") or []): user["is_present_in_meeting_ids"] = [meeting_id] else: user["is_present_in_meeting_ids"] = None - if user.get("gender_id"): - user["gender"] = gender_dict.get(user["gender_id"], {}).get("name") - del user["gender_id"] + if not internal_target: + gender_dict = datastore.get_all("gender", ["name"], lock_result=False) + if user.get("gender_id"): + user["gender"] = gender_dict.get(user["gender_id"], {}).get("name") + del user["gender_id"] # limit user fields to exported objects collection_field_tupels = [ ("meeting_user", "meeting_user_ids"), diff --git a/tests/system/action/meeting/test_clone.py b/tests/system/action/meeting/test_clone.py index a1dc315adc..2ce6016894 100644 --- a/tests/system/action/meeting/test_clone.py +++ b/tests/system/action/meeting/test_clone.py @@ -782,10 +782,12 @@ def test_clone_new_committee_and_user_with_group(self) -> None: "meeting_user_ids": [2], "meeting_ids": [1], "organization_id": 1, + "gender_id": 1, }, + "gender/1": {"name": "male", "organization_id": 1, "user_ids": [3]}, "group/1": {"meeting_user_ids": [2]}, "committee/2": {"organization_id": 1}, - "organization/1": {"committee_ids": [1, 2]}, + "organization/1": {"committee_ids": [1, 2], "gender_ids": [1]}, "meeting/1": {"user_ids": [1, 13], "meeting_user_ids": [1, 2]}, "meeting_user/2": { "meeting_id": 1, @@ -815,6 +817,7 @@ def test_clone_new_committee_and_user_with_group(self) -> None: "committee_ids": [1, 2], "meeting_ids": [1, 2], "meeting_user_ids": [2, 4], + "gender_id": 1, }, ) self.assert_model_exists( From 003bfe3448a59e720b3903e05290bf48d89c9bc0 Mon Sep 17 00:00:00 2001 From: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:09:55 +0200 Subject: [PATCH 28/90] Fix action worker test (#2683) --- tests/system/action/test_action_worker.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/system/action/test_action_worker.py b/tests/system/action/test_action_worker.py index 1a95df8c3b..0064a71971 100644 --- a/tests/system/action/test_action_worker.py +++ b/tests/system/action/test_action_worker.py @@ -230,10 +230,11 @@ def test_action_worker_create_action_worker_during_running_db_action(self) -> No def thread_method(self: ActionWorkerTest) -> None: with self.lock: data = [ - {"title": "boo", "username": "foo"} for i in range(1, self.number) + {"prefix": f"boo{i}", "name": f"foo{i}", "meeting_id": 222} + for i in range(1, self.number) ] self.start1 = datetime.now() - self.request_multi("user.create", data) + self.request_multi("motion_category.create", data) self.end1 = datetime.now() thread = Thread(target=thread_method, args=(self,)) From 50854d9cf791146ebc84d76fcad2b47b5aab836c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:50:19 +0200 Subject: [PATCH 29/90] Bump debugpy from 1.8.6 to 1.8.7 in /requirements/partial (#2671) Bumps [debugpy](https://github.com/microsoft/debugpy) from 1.8.6 to 1.8.7. - [Release notes](https://github.com/microsoft/debugpy/releases) - [Commits](https://github.com/microsoft/debugpy/compare/v1.8.6...v1.8.7) --- updated-dependencies: - dependency-name: debugpy dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> --- requirements/partial/requirements_development.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/partial/requirements_development.txt b/requirements/partial/requirements_development.txt index 335d12ef34..b35a0820e8 100644 --- a/requirements/partial/requirements_development.txt +++ b/requirements/partial/requirements_development.txt @@ -1,7 +1,7 @@ aiosmtpd==1.4.6 autoflake==2.3.1 black==24.10.0 -debugpy==1.8.6 +debugpy==1.8.7 flake8==7.1.1 isort==5.13.2 mypy==1.12.0 From 6442a6bf22a6bcf0ee651f36f44b71548ec857db Mon Sep 17 00:00:00 2001 From: Hannes Janott Date: Wed, 16 Oct 2024 12:06:04 +0200 Subject: [PATCH 30/90] fix participant import with meeting admin (#2684) delete and re add gender during participant json upload field validation --- .../actions/user/participant_json_upload.py | 7 +++ .../user/test_participant_json_upload.py | 58 +++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/openslides_backend/action/actions/user/participant_json_upload.py b/openslides_backend/action/actions/user/participant_json_upload.py index ed88a8312a..4dc108b6b1 100644 --- a/openslides_backend/action/actions/user/participant_json_upload.py +++ b/openslides_backend/action/actions/user/participant_json_upload.py @@ -103,7 +103,12 @@ def validate_entry(self, entry: dict[str, Any]) -> dict[str, Any]: ) payload_index = entry.pop("payload_index", None) + # only needed for get_failing_fields not to fail + if gender := entry.pop("gender", None): + entry["gender_id"] = {} failing_fields = self.permission_check.get_failing_fields(entry) + if gender: + entry.pop("gender_id") entry.pop("group_ids") entry.pop("structure_level_ids") entry.pop("meeting_id") @@ -152,6 +157,8 @@ def validate_entry(self, entry: dict[str, Any]) -> dict[str, Any]: if payload_index: entry["payload_index"] = payload_index + if gender: + entry["gender"] = gender return results diff --git a/tests/system/action/user/test_participant_json_upload.py b/tests/system/action/user/test_participant_json_upload.py index 08c17f9c9a..4f2a873e34 100644 --- a/tests/system/action/user/test_participant_json_upload.py +++ b/tests/system/action/user/test_participant_json_upload.py @@ -13,6 +13,10 @@ def setUp(self) -> None: self.set_models( { "organization/1": {"gender_ids": [1, 2, 3, 4]}, + "gender/1": {"name": "male"}, + "gender/2": {"name": "female"}, + "gender/3": {"name": "diverse"}, + "gender/4": {"name": "non-binary"}, "meeting/1": { "name": "test", "group_ids": [1, 7], @@ -361,6 +365,60 @@ def test_json_upload_permission_2(self) -> None: True, ) + def test_json_upload_permission_meeting_admin(self) -> None: + self.create_meeting() + user_id = self.create_user_for_meeting(1) + self.set_user_groups(user_id, [2]) + self.login(user_id) + response = self.request( + "participant.json_upload", + { + "meeting_id": 1, + "data": [ + {"username": "test", "gender": "male", "default_password": "secret"} + ], + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "import_preview/1", + { + "name": "participant", + "state": ImportState.DONE, + "result": { + "meeting_id": 1, + "rows": [ + { + "state": ImportState.NEW, + "messages": [], + "data": { + "username": { + "value": "test", + "info": ImportState.DONE, + }, + "groups": [ + { + "id": 1, + "info": ImportState.GENERATED, + "value": "group1", + } + ], + "gender": { + "id": 1, + "info": ImportState.DONE, + "value": "male", + }, + "default_password": { + "value": "secret", + "info": ImportState.DONE, + }, + }, + } + ], + }, + }, + ) + def test_json_upload_locked_meeting(self) -> None: self.base_locked_out_superadmin_permission_test( {}, From cce7b01a2c821b55b10a69275e124ca6c08b8420 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Oct 2024 12:56:26 +0200 Subject: [PATCH 31/90] Bump pypdf[crypto] from 5.0.0 to 5.0.1 in /requirements/partial (#2654) Bumps [pypdf[crypto]](https://github.com/py-pdf/pypdf) from 5.0.0 to 5.0.1. - [Release notes](https://github.com/py-pdf/pypdf/releases) - [Changelog](https://github.com/py-pdf/pypdf/blob/main/CHANGELOG.md) - [Commits](https://github.com/py-pdf/pypdf/compare/5.0.0...5.0.1) --- updated-dependencies: - dependency-name: pypdf[crypto] dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> --- requirements/partial/requirements_production.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/partial/requirements_production.txt b/requirements/partial/requirements_production.txt index be47d15dfb..fb1fc89c85 100644 --- a/requirements/partial/requirements_production.txt +++ b/requirements/partial/requirements_production.txt @@ -4,7 +4,7 @@ bleach[css]==6.1.0 dependency_injector==4.42.0 fastjsonschema==2.20.0 gunicorn==23.0.0 -pypdf[crypto]==5.0.0 +pypdf[crypto]==5.0.1 requests==2.32.3 roman==4.2 simplejson==3.19.3 From c185f00fe07057b39e0140de89d610fbc09034cf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Oct 2024 13:16:35 +0200 Subject: [PATCH 32/90] Bump pyupgrade from 3.17.0 to 3.18.0 in /requirements/partial (#2675) Bumps [pyupgrade](https://github.com/asottile/pyupgrade) from 3.17.0 to 3.18.0. - [Commits](https://github.com/asottile/pyupgrade/compare/v3.17.0...v3.18.0) --- updated-dependencies: - dependency-name: pyupgrade dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> --- requirements/partial/requirements_development.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/partial/requirements_development.txt b/requirements/partial/requirements_development.txt index b35a0820e8..8ddb6ecf86 100644 --- a/requirements/partial/requirements_development.txt +++ b/requirements/partial/requirements_development.txt @@ -9,7 +9,7 @@ pip-check==2.9 pytest==8.3.3 pytest-cov==5.0.0 pytest-profiling==1.7.0 -pyupgrade==3.17.0 +pyupgrade==3.18.0 pyyaml==6.0.2 # typing From fdc2fd4c3081c2c9c127719a51a0a8a0c3ede146 Mon Sep 17 00:00:00 2001 From: Hannes Janott Date: Wed, 16 Oct 2024 19:41:36 +0200 Subject: [PATCH 33/90] participant import error with gender and meeting admin (#2689) --- .../action/actions/user/participant_import.py | 6 +- .../actions/user/participant_json_upload.py | 12 ++-- .../action/user/test_participant_import.py | 67 +++++++++++-------- .../user/test_participant_json_upload.py | 35 +++++++--- 4 files changed, 72 insertions(+), 48 deletions(-) diff --git a/openslides_backend/action/actions/user/participant_import.py b/openslides_backend/action/actions/user/participant_import.py index fb4e1c1fc3..fec1cfd77a 100644 --- a/openslides_backend/action/actions/user/participant_import.py +++ b/openslides_backend/action/actions/user/participant_import.py @@ -120,9 +120,7 @@ def validate_entry(self, row: ImportRow) -> None: entry["meeting_id"] = self.meeting_id if isinstance(entry.get("gender"), dict): - if entry["gender"].get("info") != ImportState.WARNING: - entry["gender_id"] = entry["gender"]["id"] - entry.pop("gender") + entry["gender_id"] = entry.pop("gender") if "groups" not in entry: raise ActionException( @@ -199,6 +197,8 @@ def validate_entry(self, row: ImportRow) -> None: entry, row["messages"], entry.get("groups", []), row ) + if entry.get("gender_id"): + entry["gender"] = entry.pop("gender_id") entry.pop("meeting_id") if row["state"] == ImportState.ERROR and self.import_state == ImportState.DONE: self.import_state = ImportState.ERROR diff --git a/openslides_backend/action/actions/user/participant_json_upload.py b/openslides_backend/action/actions/user/participant_json_upload.py index 4dc108b6b1..b6dd27ed14 100644 --- a/openslides_backend/action/actions/user/participant_json_upload.py +++ b/openslides_backend/action/actions/user/participant_json_upload.py @@ -103,12 +103,10 @@ def validate_entry(self, entry: dict[str, Any]) -> dict[str, Any]: ) payload_index = entry.pop("payload_index", None) - # only needed for get_failing_fields not to fail - if gender := entry.pop("gender", None): - entry["gender_id"] = {} + # swapping needed for get_failing_fields and setting import states not to fail + if entry.get("gender"): + entry["gender_id"] = entry.pop("gender") failing_fields = self.permission_check.get_failing_fields(entry) - if gender: - entry.pop("gender_id") entry.pop("group_ids") entry.pop("structure_level_ids") entry.pop("meeting_id") @@ -157,8 +155,8 @@ def validate_entry(self, entry: dict[str, Any]) -> dict[str, Any]: if payload_index: entry["payload_index"] = payload_index - if gender: - entry["gender"] = gender + if entry.get("gender_id"): + entry["gender"] = entry.pop("gender_id") return results diff --git a/tests/system/action/user/test_participant_import.py b/tests/system/action/user/test_participant_import.py index c676b6f04d..4cf97c97a7 100644 --- a/tests/system/action/user/test_participant_import.py +++ b/tests/system/action/user/test_participant_import.py @@ -42,6 +42,13 @@ def setUp(self) -> None: "value": "male", "info": ImportState.DONE, }, + "groups": [ + { + "id": 1, + "value": "group1", + "info": ImportState.DONE, + } + ], }, }, ], @@ -75,6 +82,8 @@ def setUp(self) -> None: ) def test_import_without_any_group_in_import_data(self) -> None: + del self.import_preview1_data["result"]["rows"][0]["data"]["groups"] + self.update_model("import_preview/1", self.import_preview1_data) response = self.request("participant.import", {"id": 1, "import": True}) self.assert_status_code(response, 400) assert ( @@ -100,10 +109,6 @@ def test_import_wrong_invalid_name_in_preview(self) -> None: self.assert_model_exists("import_preview/1", {"name": "account"}) def test_import_names_and_email_and_create(self) -> None: - self.import_preview1_data["result"]["rows"][0]["data"]["groups"] = [ - {"info": ImportState.DONE, "value": "group1", "id": 1} - ] - self.update_model("import_preview/1", self.import_preview1_data) response = self.request("participant.import", {"id": 1, "import": True}) self.assert_status_code(response, 200) self.assert_model_exists( @@ -135,10 +140,6 @@ def test_import_names_and_email_and_create(self) -> None: ) def test_import_with_group_created_in_between(self) -> None: - self.import_preview1_data["result"]["rows"][0]["data"]["groups"] = [ - {"info": ImportState.DONE, "value": "group1", "id": 1} - ] - self.update_model("import_preview/1", self.import_preview1_data) self.set_models( { "group/123": {"meeting_id": 1, "name": "2Bcreated"}, @@ -195,9 +196,6 @@ def test_import_saml_id_error_new_and_saml_id_exists(self) -> None: "value": "testsaml", "info": ImportState.NEW, } - self.import_preview1_data["result"]["rows"][0]["data"]["groups"] = [ - {"info": ImportState.DONE, "value": "group1", "id": 1} - ] self.set_models( { "user/1": {"saml_id": "testsaml"}, @@ -219,9 +217,6 @@ def test_import_gender_warning(self) -> None: "value": "notAGender", "info": ImportState.WARNING, } - self.import_preview1_data["result"]["rows"][0]["data"]["groups"] = [ - {"info": ImportState.DONE, "value": "group1", "id": 1} - ] self.import_preview1_data["result"]["rows"][0]["messages"] = [ "Gender 'notAGender' is not in the allowed gender list." ] @@ -255,9 +250,6 @@ def test_import_error_state_done_missing_user_in_db(self) -> None: "id": 111, } self.import_preview1_data["result"]["rows"][0]["data"]["id"] = 111 - self.import_preview1_data["result"]["rows"][0]["data"]["groups"] = [ - {"info": ImportState.DONE, "value": "group1", "id": 1} - ] self.update_model("import_preview/1", self.import_preview1_data) response = self.request("participant.import", {"id": 1, "import": True}) self.assert_status_code(response, 200) @@ -278,10 +270,6 @@ def test_import_no_permission(self) -> None: self.base_permission_test({}, "participant.import", {"id": 1, "import": True}) def test_import_permission(self) -> None: - self.import_preview1_data["result"]["rows"][0]["data"]["groups"] = [ - {"info": ImportState.DONE, "value": "group1", "id": 1} - ] - self.update_model("import_preview/1", self.import_preview1_data) self.base_permission_test( {}, "participant.import", @@ -290,10 +278,6 @@ def test_import_permission(self) -> None: ) def test_import_permission_2(self) -> None: - self.import_preview1_data["result"]["rows"][0]["data"]["groups"] = [ - {"info": ImportState.DONE, "value": "group1", "id": 1} - ] - self.update_model("import_preview/1", self.import_preview1_data) self.base_permission_test( {}, "participant.import", @@ -302,6 +286,33 @@ def test_import_permission_2(self) -> None: True, ) + def test_import_permission_meeting_admin(self) -> None: + self.import_preview1_data["result"]["rows"][0]["data"]["id"] = 1 + self.import_preview1_data["result"]["rows"][0]["state"] = ImportState.DONE + self.import_preview1_data["result"]["rows"][0]["data"]["gender"][ + "info" + ] = ImportState.REMOVE + self.update_model("import_preview/1", self.import_preview1_data) + + self.create_meeting() + self.create_meeting(4) + user_id = self.create_user_for_meeting(1) + other_user_id = 3 + self.set_models( + { + f"user/{other_user_id}": self._get_user_data("jonny", {1: [], 4: []}), + } + ) + self.set_user_groups(user_id, [2]) + self.set_user_groups(other_user_id, [1, 4]) + self.login(user_id) + + response = self.request("participant.import", {"id": 1, "import": True}) + + self.assert_status_code(response, 200) + self.assert_model_exists("meeting_user/3", {"user_id": 3, "meeting_id": 4}) + self.assert_model_not_exists("meeting_user/4") + def test_import_locked_meeting(self) -> None: self.base_locked_out_superadmin_permission_test( {}, "participant.import", {"id": 1, "import": True} @@ -734,7 +745,7 @@ def test_json_upload_one_structure_level_newly_created(self) -> None: {"id": created_groups["group4"], "info": "new", "value": "group4"}, ], "structure_level": [{"info": "new", "value": "level up", "id": 2}], - "gender_id": 3, + "gender": {"id": 3, "info": "done", "value": "diverse"}, } row = result["rows"][1] @@ -780,6 +791,7 @@ def test_json_upload_one_structure_level_newly_created(self) -> None: {"info": ImportState.NEW, "value": "level up", "id": 2}, {"info": ImportState.DONE, "value": "no. 5", "id": 1}, ], + "gender": {"info": "warning", "value": "unknown"}, } self.assert_model_exists("structure_level/2", {"name": "level up"}) @@ -847,7 +859,7 @@ def test_json_upload_update_multiple_users_all_error(self) -> None: {"info": "new", "value": "group4"}, ], "structure_level": [{"info": "new", "value": "level up"}], - "gender_id": 3, + "gender": {"id": 3, "info": "done", "value": "diverse"}, } row = result["rows"][1] @@ -898,6 +910,7 @@ def test_json_upload_update_multiple_users_all_error(self) -> None: {"info": ImportState.NEW, "value": "level up"}, {"info": ImportState.NEW, "value": "no. 5"}, ], + "gender": {"info": "warning", "value": "unknown"}, } row = result["rows"][4] diff --git a/tests/system/action/user/test_participant_json_upload.py b/tests/system/action/user/test_participant_json_upload.py index 4f2a873e34..6005ffc1ad 100644 --- a/tests/system/action/user/test_participant_json_upload.py +++ b/tests/system/action/user/test_participant_json_upload.py @@ -9,6 +9,7 @@ class ParticipantJsonUpload(BaseActionTestCase): def setUp(self) -> None: + self.maxDiff = None super().setUp() self.set_models( { @@ -365,10 +366,18 @@ def test_json_upload_permission_2(self) -> None: True, ) - def test_json_upload_permission_meeting_admin(self) -> None: + def test_json_upload_no_permission_meeting_admin(self) -> None: self.create_meeting() + self.create_meeting(4) user_id = self.create_user_for_meeting(1) + other_user_id = 3 + self.set_models( + { + f"user/{other_user_id}": self._get_user_data("test", {1: [], 4: []}), + } + ) self.set_user_groups(user_id, [2]) + self.set_user_groups(other_user_id, [1, 4]) self.login(user_id) response = self.request( "participant.json_upload", @@ -389,28 +398,32 @@ def test_json_upload_permission_meeting_admin(self) -> None: "meeting_id": 1, "rows": [ { - "state": ImportState.NEW, - "messages": [], + "state": ImportState.DONE, + "messages": [ + "Following fields were removed from payload, because the user has no permissions to change them: username, gender_id, default_password" + ], "data": { "username": { "value": "test", - "info": ImportState.DONE, + "info": ImportState.REMOVE, + "id": 3, + }, + "default_password": { + "value": "secret", + "info": ImportState.REMOVE, }, + "id": 3, "groups": [ { - "id": 1, "info": ImportState.GENERATED, "value": "group1", + "id": 1, } ], "gender": { - "id": 1, - "info": ImportState.DONE, + "info": ImportState.REMOVE, "value": "male", - }, - "default_password": { - "value": "secret", - "info": ImportState.DONE, + "id": 1, }, }, } From f31358c321bcc969d68cfb23b01b00fe5814a553 Mon Sep 17 00:00:00 2001 From: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> Date: Thu, 17 Oct 2024 12:28:09 +0200 Subject: [PATCH 34/90] Move moderator_note to LoS (#2593) * Remove can_see from can_see_moderator_note * Write migration --- docs/actions/agenda_item.create.md | 4 +- docs/actions/agenda_item.update.md | 5 +- docs/actions/group.update.md | 2 +- docs/actions/list_of_speakers.update.md | 4 +- global/data/example-data.json | 2 +- global/data/initial-data.json | 2 +- global/meta | 2 +- .../action/actions/agenda_item/create.py | 4 +- .../actions/agenda_item/permission_mixin.py | 25 -- .../action/actions/agenda_item/update.py | 4 +- .../action/actions/list_of_speakers/update.py | 20 +- .../migrations/0060_move_moderator_notes.py | 114 ++++++++ openslides_backend/models/models.py | 6 +- .../permissions/permission_helper.py | 2 +- openslides_backend/permissions/permissions.py | 15 +- .../system/action/agenda_item/test_create.py | 20 -- .../system/action/agenda_item/test_update.py | 17 -- .../action/list_of_speakers/test_update.py | 17 ++ .../test_0060_move_moderator_notes.py | 276 ++++++++++++++++++ 19 files changed, 449 insertions(+), 92 deletions(-) delete mode 100644 openslides_backend/action/actions/agenda_item/permission_mixin.py create mode 100644 openslides_backend/migrations/migrations/0060_move_moderator_notes.py create mode 100644 tests/system/migrations/test_0060_move_moderator_notes.py diff --git a/docs/actions/agenda_item.create.md b/docs/actions/agenda_item.create.md index 610abbcc20..5270818c9a 100644 --- a/docs/actions/agenda_item.create.md +++ b/docs/actions/agenda_item.create.md @@ -13,7 +13,6 @@ duration: number; // in seconds weight: number; tag_ids: Id[]; - moderator_notes: HTML; } ``` @@ -23,5 +22,4 @@ item or the content object cannot have an agenda item (see available collections `models.yml`). `tag_ids` must be from the same meeting. ## Permissions -The request user needs `agenda_item.can_manage_moderator_notes` to set `moderator_notes` and -`agenda_item.can_manage` for all other fields. +The request user needs `agenda_item.can_manage`. diff --git a/docs/actions/agenda_item.update.md b/docs/actions/agenda_item.update.md index 8cab4855b8..0d20b57df3 100644 --- a/docs/actions/agenda_item.update.md +++ b/docs/actions/agenda_item.update.md @@ -12,7 +12,6 @@ duration: number; // in minutes weight: number; tag_ids: Id[]; - moderator_notes: HTML; } ``` @@ -21,5 +20,5 @@ Updates the agenda item. `tag_ids` must be from the same meeting. The `type` attribute of one `agenda_item` must be one of [`common`, `internal`, `hidden`]. ## Permissions -The request user needs `agenda_item.can_manage_moderator_notes` to set `moderator_notes` and -`agenda_item.can_manage` for all other fields. +The request user needs `agenda_item.can_manage`. + diff --git a/docs/actions/group.update.md b/docs/actions/group.update.md index 75b52e7e6e..ac4b815345 100644 --- a/docs/actions/group.update.md +++ b/docs/actions/group.update.md @@ -17,9 +17,9 @@ Updates the group. Permissions are restricted to the following enum: https://git If the group is the meetings anonymous group, the name may not be changed and the permissions have to be in the following whitelist: - agenda_item.can_see, - agenda_item.can_see_internal, -- agenda_item.can_see_moderator_notes, - assignment.can_see, - list_of_speakers.can_see, +- list_of_speakers.can_see_moderator_notes, - mediafile.can_see, - meeting.can_see_autopilot, - meeting.can_see_frontpage, diff --git a/docs/actions/list_of_speakers.update.md b/docs/actions/list_of_speakers.update.md index bfe9b44d00..adc0163d05 100644 --- a/docs/actions/list_of_speakers.update.md +++ b/docs/actions/list_of_speakers.update.md @@ -6,6 +6,7 @@ // Optional closed: boolean; + moderator_notes: HTML; } ``` @@ -13,4 +14,5 @@ Updates a list of speakers. ## Permissions -The request user needs `list_of_speakers.can_manage`. +The request user needs `list_of_speakers.can_manage_moderator_notes` to set `moderator_notes` and +`list_of_speakers.can_manage` for all other fields. diff --git a/global/data/example-data.json b/global/data/example-data.json index 74e5a81fb0..13ce494082 100644 --- a/global/data/example-data.json +++ b/global/data/example-data.json @@ -1,5 +1,5 @@ { - "_migration_index": 60, + "_migration_index": 61, "gender":{ "1":{ "id": 1, diff --git a/global/data/initial-data.json b/global/data/initial-data.json index 4760ca9bec..88eecdaf3c 100644 --- a/global/data/initial-data.json +++ b/global/data/initial-data.json @@ -1,5 +1,5 @@ { - "_migration_index": 60, + "_migration_index": 61, "gender":{ "1":{ "id": 1, diff --git a/global/meta b/global/meta index 5ecbba4daa..858abfc958 160000 --- a/global/meta +++ b/global/meta @@ -1 +1 @@ -Subproject commit 5ecbba4daaca2579dd9fa90d27fd258510e39af5 +Subproject commit 858abfc958df72703711b30f45f2a42c49d1fa4b diff --git a/openslides_backend/action/actions/agenda_item/create.py b/openslides_backend/action/actions/agenda_item/create.py index 5d9fd2d1ef..3996e8f18d 100644 --- a/openslides_backend/action/actions/agenda_item/create.py +++ b/openslides_backend/action/actions/agenda_item/create.py @@ -9,11 +9,10 @@ ) from ...util.default_schema import DefaultSchema from ...util.register import register_action -from .permission_mixin import AgendaItemPermissionMixin @register_action("agenda_item.create") -class AgendaItemCreate(AgendaItemPermissionMixin, CreateActionWithInferredMeeting): +class AgendaItemCreate(CreateActionWithInferredMeeting): """ Action to create agenda items. """ @@ -29,7 +28,6 @@ class AgendaItemCreate(AgendaItemPermissionMixin, CreateActionWithInferredMeetin "duration", "weight", "tag_ids", - "moderator_notes", ], ) permission = Permissions.AgendaItem.CAN_MANAGE diff --git a/openslides_backend/action/actions/agenda_item/permission_mixin.py b/openslides_backend/action/actions/agenda_item/permission_mixin.py deleted file mode 100644 index 17146e43cd..0000000000 --- a/openslides_backend/action/actions/agenda_item/permission_mixin.py +++ /dev/null @@ -1,25 +0,0 @@ -from typing import Any - -from openslides_backend.action.action import Action -from openslides_backend.permissions.permission_helper import has_perm -from openslides_backend.permissions.permissions import Permissions -from openslides_backend.shared.exceptions import MissingPermission - - -class AgendaItemPermissionMixin(Action): - def check_permissions(self, instance: dict[str, Any]) -> None: - fields = set(instance.keys()) - if "id" in fields: - fields.remove("id") - if "moderator_notes" in fields: - fields.remove("moderator_notes") - perm = Permissions.AgendaItem.CAN_MANAGE_MODERATOR_NOTES - if not has_perm( - self.datastore, - self.user_id, - perm, - self.get_meeting_id(instance), - ): - raise MissingPermission(perm) - if fields: - super().check_permissions(instance) diff --git a/openslides_backend/action/actions/agenda_item/update.py b/openslides_backend/action/actions/agenda_item/update.py index c051327db9..3fbcc6f49a 100644 --- a/openslides_backend/action/actions/agenda_item/update.py +++ b/openslides_backend/action/actions/agenda_item/update.py @@ -6,11 +6,10 @@ from ...util.default_schema import DefaultSchema from ...util.register import register_action from ...util.typing import ActionData -from .permission_mixin import AgendaItemPermissionMixin @register_action("agenda_item.update") -class AgendaItemUpdate(AgendaItemPermissionMixin, UpdateAction): +class AgendaItemUpdate(UpdateAction): """ Action to update agenda items. """ @@ -25,7 +24,6 @@ class AgendaItemUpdate(AgendaItemPermissionMixin, UpdateAction): "weight", "tag_ids", "duration", - "moderator_notes", ] ) permission = Permissions.AgendaItem.CAN_MANAGE diff --git a/openslides_backend/action/actions/list_of_speakers/update.py b/openslides_backend/action/actions/list_of_speakers/update.py index bb9a6f272f..bb65c0037e 100644 --- a/openslides_backend/action/actions/list_of_speakers/update.py +++ b/openslides_backend/action/actions/list_of_speakers/update.py @@ -1,5 +1,9 @@ +from typing import Any + from ....models.models import ListOfSpeakers +from ....permissions.permission_helper import has_perm from ....permissions.permissions import Permissions +from ....shared.exceptions import MissingPermission from ...generics.update import UpdateAction from ...util.default_schema import DefaultSchema from ...util.register import register_action @@ -13,6 +17,20 @@ class ListOfSpeakersUpdateAction(UpdateAction): model = ListOfSpeakers() schema = DefaultSchema(ListOfSpeakers()).get_update_schema( - optional_properties=["closed"] + optional_properties=["closed", "moderator_notes"] ) permission = Permissions.ListOfSpeakers.CAN_MANAGE + + def check_permissions(self, instance: dict[str, Any]) -> None: + if "moderator_notes" in instance: + perm = Permissions.ListOfSpeakers.CAN_MANAGE_MODERATOR_NOTES + if not has_perm( + self.datastore, + self.user_id, + perm, + self.get_meeting_id(instance), + ): + raise MissingPermission(perm) + if len(instance) == 2: + return + super().check_permissions(instance) diff --git a/openslides_backend/migrations/migrations/0060_move_moderator_notes.py b/openslides_backend/migrations/migrations/0060_move_moderator_notes.py new file mode 100644 index 0000000000..204a4329d9 --- /dev/null +++ b/openslides_backend/migrations/migrations/0060_move_moderator_notes.py @@ -0,0 +1,114 @@ +from collections import defaultdict + +from datastore.migrations import BaseModelMigration +from datastore.reader.core import GetManyRequestPart +from datastore.writer.core import BaseRequestEvent, RequestUpdateEvent + +from openslides_backend.shared.patterns import ( + collection_and_id_from_fqid, + fqid_from_collection_and_id, +) + + +class Migration(BaseModelMigration): + """ + This migration moves all set moderator notes to the related models list_of_speakers. + It also gives any group which had the permissions agenda_item.can_see_moderator_notes or agenda_item.can_manage_moderator_notes + the permission list_of_speakers.can_see_moderator_notes or list_of_speakers.can_manage_moderator_notes instead. + """ + + target_migration_index = 61 + + def migrate_models(self) -> list[BaseRequestEvent] | None: + agenda_items = self.reader.get_all( + "agenda_item", ["id", "content_object_id", "moderator_notes"] + ) + events: list[BaseRequestEvent] = [ + RequestUpdateEvent( + fqid_from_collection_and_id("agenda_item", agenda_item["id"]), + {"moderator_notes": None}, + ) + for agenda_item in agenda_items.values() + if (mod_note := agenda_item.get("moderator_notes")) + ] + if len(events): + content_object_id_to_mod_note: dict[tuple[str, int], str] = { + collection_and_id_from_fqid(agenda_item["content_object_id"]): mod_note + for agenda_item in agenda_items.values() + if (mod_note := agenda_item.get("moderator_notes")) + } + content_object_collection_to_ids: dict[str, list[int]] = defaultdict(list) + for co_collection, co_id in content_object_id_to_mod_note.keys(): + content_object_collection_to_ids[co_collection].append(co_id) + content_objects = self.reader.get_many( + [ + GetManyRequestPart(co_collection, co_ids, ["list_of_speakers_id"]) + for co_collection, co_ids in content_object_collection_to_ids.items() + ] + ) + events.extend( + [ + RequestUpdateEvent( + fqid_from_collection_and_id( + "list_of_speakers", + content_objects[co_collection][co_id][ + "list_of_speakers_id" + ], + ), + {"moderator_notes": mod_note}, + ) + for ( + co_collection, + co_id, + ), mod_note in content_object_id_to_mod_note.items() + ] + ) + groups = self.reader.get_all("group", ["id", "permissions"]) + events.extend( + [ + RequestUpdateEvent( + fqid_from_collection_and_id("group", group["id"]), + {}, + { + "add": { + "permissions": [ + *( + ["agenda_item.can_see"] + if not any( + "agenda_item." + suffix + in group.get("permissions", []) + or [] + for suffix in ["can_see_internal", "can_manage"] + ) + else [] + ), + *[ + "list_of_speakers" + substr + for substr in perm_substrings + ], + ] + }, + "remove": { + "permissions": [ + "agenda_item" + substr for substr in perm_substrings + ] + }, + }, + ) + for group in groups.values() + if ( + perm_substrings := [ + substr + for substr in [ + ".can_see_moderator_notes", + ".can_manage_moderator_notes", + ] + if any( + permission == "agenda_item" + substr + for permission in group.get("permissions", []) or [] + ) + ] + ) + ] + ) + return events diff --git a/openslides_backend/models/models.py b/openslides_backend/models/models.py index c44e6e00bd..20c6d405af 100644 --- a/openslides_backend/models/models.py +++ b/openslides_backend/models/models.py @@ -954,8 +954,6 @@ class Group(Model): "agenda_item.can_manage", "agenda_item.can_see", "agenda_item.can_see_internal", - "agenda_item.can_manage_moderator_notes", - "agenda_item.can_see_moderator_notes", "assignment.can_manage", "assignment.can_nominate_other", "assignment.can_nominate_self", @@ -964,6 +962,8 @@ class Group(Model): "list_of_speakers.can_be_speaker", "list_of_speakers.can_manage", "list_of_speakers.can_see", + "list_of_speakers.can_manage_moderator_notes", + "list_of_speakers.can_see_moderator_notes", "mediafile.can_manage", "mediafile.can_see", "meeting.can_manage_logos_and_fonts", @@ -1098,7 +1098,6 @@ class AgendaItem(Model, AgendaItemModelMixin): duration = fields.IntegerField( constraints={"description": "Given in seconds", "minimum": 0} ) - moderator_notes = fields.HTMLStrictField() is_internal = fields.BooleanField( read_only=True, constraints={"description": "Calculated by the server"} ) @@ -1153,6 +1152,7 @@ class ListOfSpeakers(Model): "description": "The (positive) serial number of this model in its meeting. This number is auto-generated and read-only." }, ) + moderator_notes = fields.HTMLStrictField() content_object_id = fields.GenericRelationField( to={ "motion": "list_of_speakers_id", diff --git a/openslides_backend/permissions/permission_helper.py b/openslides_backend/permissions/permission_helper.py index c8b17a439d..4d79cd83da 100644 --- a/openslides_backend/permissions/permission_helper.py +++ b/openslides_backend/permissions/permission_helper.py @@ -193,9 +193,9 @@ def is_admin(datastore: DatastoreService, user_id: int, meeting_id: int) -> bool anonymous_perms_whitelist: set[Permission] = { Permissions.AgendaItem.CAN_SEE, Permissions.AgendaItem.CAN_SEE_INTERNAL, - Permissions.AgendaItem.CAN_SEE_MODERATOR_NOTES, Permissions.Assignment.CAN_SEE, Permissions.ListOfSpeakers.CAN_SEE, + Permissions.ListOfSpeakers.CAN_SEE_MODERATOR_NOTES, Permissions.Mediafile.CAN_SEE, Permissions.Meeting.CAN_SEE_AUTOPILOT, Permissions.Meeting.CAN_SEE_FRONTPAGE, diff --git a/openslides_backend/permissions/permissions.py b/openslides_backend/permissions/permissions.py index eec5e46b4e..3caa90864e 100644 --- a/openslides_backend/permissions/permissions.py +++ b/openslides_backend/permissions/permissions.py @@ -7,10 +7,8 @@ class _AgendaItem(str, Permission, Enum): CAN_MANAGE = "agenda_item.can_manage" - CAN_MANAGE_MODERATOR_NOTES = "agenda_item.can_manage_moderator_notes" CAN_SEE = "agenda_item.can_see" CAN_SEE_INTERNAL = "agenda_item.can_see_internal" - CAN_SEE_MODERATOR_NOTES = "agenda_item.can_see_moderator_notes" class _Assignment(str, Permission, Enum): @@ -27,7 +25,9 @@ class _Chat(str, Permission, Enum): class _ListOfSpeakers(str, Permission, Enum): CAN_BE_SPEAKER = "list_of_speakers.can_be_speaker" CAN_MANAGE = "list_of_speakers.can_manage" + CAN_MANAGE_MODERATOR_NOTES = "list_of_speakers.can_manage_moderator_notes" CAN_SEE = "list_of_speakers.can_see" + CAN_SEE_MODERATOR_NOTES = "list_of_speakers.can_see_moderator_notes" class _Mediafile(str, Permission, Enum): @@ -95,14 +95,9 @@ class Permissions: # Holds the corresponding parent for each permission. permission_parents: dict[Permission, list[Permission]] = { - _AgendaItem.CAN_SEE: [ - _AgendaItem.CAN_SEE_INTERNAL, - _AgendaItem.CAN_SEE_MODERATOR_NOTES, - ], + _AgendaItem.CAN_SEE: [_AgendaItem.CAN_SEE_INTERNAL], _AgendaItem.CAN_SEE_INTERNAL: [_AgendaItem.CAN_MANAGE], _AgendaItem.CAN_MANAGE: [], - _AgendaItem.CAN_SEE_MODERATOR_NOTES: [_AgendaItem.CAN_MANAGE_MODERATOR_NOTES], - _AgendaItem.CAN_MANAGE_MODERATOR_NOTES: [], _Assignment.CAN_SEE: [ _Assignment.CAN_NOMINATE_OTHER, _Assignment.CAN_NOMINATE_SELF, @@ -114,6 +109,10 @@ class Permissions: _ListOfSpeakers.CAN_SEE: [_ListOfSpeakers.CAN_MANAGE], _ListOfSpeakers.CAN_MANAGE: [], _ListOfSpeakers.CAN_BE_SPEAKER: [], + _ListOfSpeakers.CAN_SEE_MODERATOR_NOTES: [ + _ListOfSpeakers.CAN_MANAGE_MODERATOR_NOTES + ], + _ListOfSpeakers.CAN_MANAGE_MODERATOR_NOTES: [], _Mediafile.CAN_SEE: [_Mediafile.CAN_MANAGE], _Mediafile.CAN_MANAGE: [], _Meeting.CAN_MANAGE_SETTINGS: [], diff --git a/tests/system/action/agenda_item/test_create.py b/tests/system/action/agenda_item/test_create.py index 700fc7a980..5e6dcfcb0e 100644 --- a/tests/system/action/agenda_item/test_create.py +++ b/tests/system/action/agenda_item/test_create.py @@ -333,26 +333,6 @@ def test_create_permissions_with_locked_meeting(self) -> None: {"content_object_id": "topic/1"}, ) - def test_create_moderator_notes_no_permissions(self) -> None: - self.base_permission_test( - {"topic/1": {"meeting_id": 1}}, - "agenda_item.create", - {"content_object_id": "topic/1", "moderator_notes": "test"}, - Permissions.AgendaItem.CAN_MANAGE, - fail=True, - ) - - def test_create_moderator_notes_permissions(self) -> None: - self.base_permission_test( - {"topic/1": {"meeting_id": 1}}, - "agenda_item.create", - {"content_object_id": "topic/1", "moderator_notes": "test"}, - [ - Permissions.AgendaItem.CAN_MANAGE, - Permissions.AgendaItem.CAN_MANAGE_MODERATOR_NOTES, - ], - ) - def test_create_replace_reverse_of_multi_content_object_id_required_error( self, ) -> None: diff --git a/tests/system/action/agenda_item/test_update.py b/tests/system/action/agenda_item/test_update.py index 67233bb539..0e9073d3dc 100644 --- a/tests/system/action/agenda_item/test_update.py +++ b/tests/system/action/agenda_item/test_update.py @@ -226,20 +226,3 @@ def test_update_permissions_locked_meeting(self) -> None: "agenda_item.update", {"id": 111, "duration": 1200}, ) - - def test_update_moderator_notes_no_permissions(self) -> None: - self.base_permission_test( - {}, - "agenda_item.update", - {"id": 111, "moderator_notes": "test"}, - Permissions.AgendaItem.CAN_MANAGE, - fail=True, - ) - - def test_update_moderator_notes_permissions(self) -> None: - self.base_permission_test( - {}, - "agenda_item.update", - {"id": 111, "moderator_notes": "test"}, - Permissions.AgendaItem.CAN_MANAGE_MODERATOR_NOTES, - ) diff --git a/tests/system/action/list_of_speakers/test_update.py b/tests/system/action/list_of_speakers/test_update.py index b334eb7e03..2c4ee196ab 100644 --- a/tests/system/action/list_of_speakers/test_update.py +++ b/tests/system/action/list_of_speakers/test_update.py @@ -64,3 +64,20 @@ def test_update_permissions_locked_meeting(self) -> None: "list_of_speakers.update", {"id": 111, "closed": True}, ) + + def test_update_moderator_notes_no_permissions(self) -> None: + self.base_permission_test( + self.permission_test_models, + "list_of_speakers.update", + {"id": 111, "moderator_notes": "test"}, + Permissions.ListOfSpeakers.CAN_MANAGE, + fail=True, + ) + + def test_update_moderator_notes_permissions(self) -> None: + self.base_permission_test( + self.permission_test_models, + "list_of_speakers.update", + {"id": 111, "moderator_notes": "test"}, + Permissions.ListOfSpeakers.CAN_MANAGE_MODERATOR_NOTES, + ) diff --git a/tests/system/migrations/test_0060_move_moderator_notes.py b/tests/system/migrations/test_0060_move_moderator_notes.py new file mode 100644 index 0000000000..2f254342a3 --- /dev/null +++ b/tests/system/migrations/test_0060_move_moderator_notes.py @@ -0,0 +1,276 @@ +from typing import Any + +from openslides_backend.shared.patterns import fqid_from_collection_and_id + + +def generate_agenda_item_data( + collection: str, base: int, note: str | None +) -> list[dict[str, Any]]: + co_id = base * 11 + co_fqid = fqid_from_collection_and_id(collection, co_id) + los_id = base * 111 + return [ + { + "type": "create", + "fqid": fqid_from_collection_and_id("agenda_item", base), + "fields": { + "id": base, + "content_object_id": co_fqid, + **({"moderator_notes": note} if note else {}), + }, + }, + { + "type": "create", + "fqid": co_fqid, + "fields": { + "id": co_id, + "agenda_item_id": base, + "list_of_speakers_id": los_id, + }, + }, + { + "type": "create", + "fqid": fqid_from_collection_and_id("list_of_speakers", los_id), + "fields": { + "id": los_id, + "content_object_id": co_fqid, + }, + }, + ] + + +def test_migration_everything(write, finalize, assert_model): + collection_base_note = [ + ("motion", 1, "ABCDEFGHIJKLMNOPQRSTUVWXYZ"), + ("motion", 2, "Hello world!"), + ("assignment", 3, "Let the election begin"), + ("topic", 4, "To pick or not to pick a topic"), + ("motion_block", 5, "A block?"), + ("topic", 6, None), + ] + write( + { + "type": "create", + "fqid": "group/1", + "fields": { + "id": 1, + "permissions": [ + "agenda_item.can_manage", + "agenda_item.can_manage_moderator_notes", + "mediafile.can_see", + ], + }, + }, + { + "type": "create", + "fqid": "group/2", + "fields": { + "id": 2, + "permissions": [ + "agenda_item.can_see_internal", + "agenda_item.can_see_moderator_notes", + "motion.can_create", + ], + }, + }, + { + "type": "create", + "fqid": "group/3", + "fields": { + "id": 3, + "permissions": [ + "agenda_item.can_manage_moderator_notes", + "agenda_item.can_see", + "agenda_item.can_see_moderator_notes", + "list_of_speakers.can_be_speaker", + ], + }, + }, + { + "type": "create", + "fqid": "group/4", + "fields": { + "id": 4, + "permissions": [ + "agenda_item.can_manage_moderator_notes", + "list_of_speakers.can_see", + "user.can_manage", + ], + }, + }, + { + "type": "create", + "fqid": "group/5", + "fields": { + "id": 5, + "permissions": [ + "agenda_item.can_see_internal", + "list_of_speakers.can_see", + ], + }, + }, + { + "type": "create", + "fqid": "group/10", + "fields": { + "id": 10, + "permissions": None, + }, + }, + *[ + event + for collection, base, note in collection_base_note + for event in generate_agenda_item_data(collection, base, note) + ], + { + "type": "create", + "fqid": "motion/77", + "fields": {"id": 77, "list_of_speakers_id": 777}, + }, + { + "type": "create", + "fqid": "list_of_speakers/777", + "fields": { + "id": 777, + "content_object_id": "motion/77", + }, + }, + ) + + finalize("0060_move_moderator_notes") + + assert_model( + "group/1", + { + "id": 1, + "permissions": [ + "agenda_item.can_manage", + "list_of_speakers.can_manage_moderator_notes", + "mediafile.can_see", + ], + }, + ) + assert_model( + "group/2", + { + "id": 2, + "permissions": [ + "agenda_item.can_see_internal", + "list_of_speakers.can_see_moderator_notes", + "motion.can_create", + ], + }, + ) + assert_model( + "group/3", + { + "id": 3, + "permissions": [ + "agenda_item.can_see", + "list_of_speakers.can_manage_moderator_notes", + "list_of_speakers.can_see_moderator_notes", + "list_of_speakers.can_be_speaker", + ], + }, + ) + assert_model( + "group/4", + { + "id": 4, + "permissions": [ + "agenda_item.can_see", + "list_of_speakers.can_manage_moderator_notes", + "list_of_speakers.can_see", + "user.can_manage", + ], + }, + ) + assert_model( + "group/5", + { + "id": 5, + "permissions": [ + "agenda_item.can_see_internal", + "list_of_speakers.can_see", + ], + }, + ) + for collection, base, note in collection_base_note: + co_fqid = fqid_from_collection_and_id(collection, base * 11) + assert_model( + fqid_from_collection_and_id("agenda_item", base), + {"id": base, "content_object_id": co_fqid}, + ) + expect_los = { + "id": base * 111, + "content_object_id": co_fqid, + } + if note: + expect_los["moderator_notes"] = note + assert_model( + fqid_from_collection_and_id("list_of_speakers", base * 111), + expect_los, + ) + assert_model( + "list_of_speakers/777", + { + "id": 777, + "content_object_id": "motion/77", + }, + ) + + +def test_migration_some_collections(write, finalize, assert_model): + """ + Just to see if leaving some collections out will cause errors + """ + collection_base_note = [ + ("motion", 1, "ABCDEFGHIJKLMNOPQRSTUVWXYZ"), + ("motion", 2, None), + ] + write( + *[ + event + for collection, base, note in collection_base_note + for event in generate_agenda_item_data(collection, base, note) + ], + { + "type": "create", + "fqid": "motion/33", + "fields": {"id": 33, "list_of_speakers_id": 333}, + }, + { + "type": "create", + "fqid": "list_of_speakers/333", + "fields": { + "id": 333, + "content_object_id": "motion/33", + }, + }, + ) + + finalize("0060_move_moderator_notes") + + for collection, base, note in collection_base_note: + co_fqid = fqid_from_collection_and_id(collection, base * 11) + assert_model( + fqid_from_collection_and_id("agenda_item", base), + {"id": base, "content_object_id": co_fqid}, + ) + expect_los = { + "id": base * 111, + "content_object_id": co_fqid, + } + if note: + expect_los["moderator_notes"] = note + assert_model( + fqid_from_collection_and_id("list_of_speakers", base * 111), + expect_los, + ) + assert_model( + "list_of_speakers/333", + { + "id": 333, + "content_object_id": "motion/33", + }, + ) From 71812784ed57748fd8e57c191233d4f3edd2d1d7 Mon Sep 17 00:00:00 2001 From: "openslides-automation[bot]" <125256978+openslides-automation[bot]@users.noreply.github.com> Date: Thu, 17 Oct 2024 10:44:02 +0000 Subject: [PATCH 35/90] Update meta repository (#2691) Co-authored-by: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> --- global/meta | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global/meta b/global/meta index 858abfc958..f010fd2c34 160000 --- a/global/meta +++ b/global/meta @@ -1 +1 @@ -Subproject commit 858abfc958df72703711b30f45f2a42c49d1fa4b +Subproject commit f010fd2c346353413a191abb3fbcd89888d6ffc6 From 0ff540a82f19ca8f4611c595bd34a8b1f04eb285 Mon Sep 17 00:00:00 2001 From: "openslides-automation[bot]" <125256978+openslides-automation[bot]@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:48:55 +0100 Subject: [PATCH 36/90] Update meta repository (#2702) --- global/meta | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global/meta b/global/meta index f010fd2c34..21bc4fd12a 160000 --- a/global/meta +++ b/global/meta @@ -1 +1 @@ -Subproject commit f010fd2c346353413a191abb3fbcd89888d6ffc6 +Subproject commit 21bc4fd12a604933cd8732a130e973e34835f541 From d4eac2ae48094457abdda8802810b53ba3c7958a Mon Sep 17 00:00:00 2001 From: Ludwig Reiter Date: Wed, 30 Oct 2024 13:41:17 +0100 Subject: [PATCH 37/90] Fix mismatch of perm check of get-forwarding-meetings permissions and error message(#2707) --- openslides_backend/presenter/get_forwarding_meetings.py | 2 +- tests/system/presenter/test_get_forwarding_meetings.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openslides_backend/presenter/get_forwarding_meetings.py b/openslides_backend/presenter/get_forwarding_meetings.py index 89a5702fa0..f3b52b8c5e 100644 --- a/openslides_backend/presenter/get_forwarding_meetings.py +++ b/openslides_backend/presenter/get_forwarding_meetings.py @@ -41,7 +41,7 @@ def get_result(self) -> Any: self.data["meeting_id"], ): msg = "You are not allowed to perform presenter get_forwarding_meetings" - msg += f" Missing permission: {Permissions.Motion.CAN_MANAGE}" + msg += f" Missing permission: {Permissions.Motion.CAN_FORWARD}" raise PermissionDenied(msg) meeting = self.datastore.get( diff --git a/tests/system/presenter/test_get_forwarding_meetings.py b/tests/system/presenter/test_get_forwarding_meetings.py index 78984caf2d..bd02973aad 100644 --- a/tests/system/presenter/test_get_forwarding_meetings.py +++ b/tests/system/presenter/test_get_forwarding_meetings.py @@ -311,4 +311,4 @@ def test_with_locked_meeting(self) -> None: ) status_code, data = self.request("get_forwarding_meetings", {"meeting_id": 3}) assert status_code == 403 - assert "Missing permission: motion.can_manage" in data["message"] + assert "Missing permission: motion.can_forward" in data["message"] From bd550955d563a692fbfbe4a66f59ceb3ebb5d57f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20B=C3=B6hlke?= Date: Mon, 4 Nov 2024 15:06:10 +0100 Subject: [PATCH 38/90] KRY-149 migration script --- .../0057_user_upload_to_keycloak.py | 79 +++++++++++++++++++ .../migrations/0058_user_remove_saml_id.py | 11 +++ 2 files changed, 90 insertions(+) create mode 100644 openslides_backend/migrations/migrations/0057_user_upload_to_keycloak.py create mode 100644 openslides_backend/migrations/migrations/0058_user_remove_saml_id.py diff --git a/openslides_backend/migrations/migrations/0057_user_upload_to_keycloak.py b/openslides_backend/migrations/migrations/0057_user_upload_to_keycloak.py new file mode 100644 index 0000000000..54b0a2d56b --- /dev/null +++ b/openslides_backend/migrations/migrations/0057_user_upload_to_keycloak.py @@ -0,0 +1,79 @@ +from datastore.shared.typing import JSON +from datastore.migrations import BaseEventMigration, BaseEvent +from datastore.shared.util import collection_and_id_from_fqid + +class Migration(BaseEventMigration): + """ + This migration adds `group/weight` with the id as the default weight. + """ + + target_migration_index = 58 + + collection = "user" + field = "kc_id" + + def migrate_event( + self, + event: BaseEvent, + ) -> list[BaseEvent] | None: + collection, id_ = collection_and_id_from_fqid(event.fqid) + + if collection != "user": + return None + + if isinstance(event, CreateEvent): + if event.data.get("is_active_in_organization_id") != ONE_ORGANIZATION_ID: + event.data["is_archived_in_organization_id"] = ONE_ORGANIZATION_ID + self.meeting_ids_to_add.add(id_) + return [event] + elif isinstance(event, DeleteEvent): + data, _ = self.new_accessor.get_model_ignore_deleted(event.fqid) + if data.get("is_active_in_organization_id") != ONE_ORGANIZATION_ID: + if id_ in self.meeting_ids_to_add: + self.meeting_ids_to_add.remove(id_) + else: + self.meeting_ids_to_remove.add(id_) + elif isinstance(event, RestoreEvent): + data, _ = self.new_accessor.get_model_ignore_deleted(event.fqid) + if data.get("is_active_in_organization_id") != ONE_ORGANIZATION_ID: + if id_ in self.meeting_ids_to_remove: + self.meeting_ids_to_remove.remove(id_) + else: + self.meeting_ids_to_add.add(id_) + elif isinstance(event, DeleteFieldsEvent): + if "is_active_in_organization_id" in event.data: + data, _ = self.new_accessor.get_model_ignore_deleted(event.fqid) + if data.get("is_active_in_organization_id") == ONE_ORGANIZATION_ID: + update_event = UpdateEvent( + event.fqid, + {"is_archived_in_organization_id": ONE_ORGANIZATION_ID}, + ) + if id_ in self.meeting_ids_to_remove: + self.meeting_ids_to_remove.remove(id_) + else: + self.meeting_ids_to_add.add(id_) + return [event, update_event] + elif isinstance(event, UpdateEvent): + if ( + "is_active_in_organization_id" in event.data + and event.data["is_active_in_organization_id"] == ONE_ORGANIZATION_ID + ): + delete_field_event = DeleteFieldsEvent( + event.fqid, ["is_archived_in_organization_id"] + ) + if id_ in self.meeting_ids_to_add: + self.meeting_ids_to_add.remove(id_) + else: + self.meeting_ids_to_remove.add(id_) + return [event, delete_field_event] + elif ( + "is_active_in_organization_id" in event.data + and event.data["is_active_in_organization_id"] != ONE_ORGANIZATION_ID + ): + event.data["is_archived_in_organization_id"] = ONE_ORGANIZATION_ID + if id_ in self.meeting_ids_to_remove: + self.meeting_ids_to_remove.remove(id_) + else: + self.meeting_ids_to_add.add(id_) + return [event] + return None diff --git a/openslides_backend/migrations/migrations/0058_user_remove_saml_id.py b/openslides_backend/migrations/migrations/0058_user_remove_saml_id.py new file mode 100644 index 0000000000..ad5abacaac --- /dev/null +++ b/openslides_backend/migrations/migrations/0058_user_remove_saml_id.py @@ -0,0 +1,11 @@ +from datastore.migrations import RemoveFieldsMigration + + +class Migration(RemoveFieldsMigration): + """ + This migration removes field `user/saml_id` from database-events + """ + + target_migration_index = 59 + + collection_fields_map = {"user": ["saml_id"]} From ce0d1243ec49c6d2186798b5c4a94a9e4292ae58 Mon Sep 17 00:00:00 2001 From: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> Date: Wed, 6 Nov 2024 09:23:51 +0100 Subject: [PATCH 39/90] Fix empty string translator (#2701) * Fix empty string translator * Write test, fix language header content sorting --- openslides_backend/i18n/translator.py | 8 +-- tests/unit/test_translator.py | 70 +++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_translator.py diff --git a/openslides_backend/i18n/translator.py b/openslides_backend/i18n/translator.py index 9d22d80ee3..fcabbab1d5 100644 --- a/openslides_backend/i18n/translator.py +++ b/openslides_backend/i18n/translator.py @@ -10,7 +10,7 @@ class _Translator: translations: dict[str, Catalog] = {} current_language: str - def __init__(self) -> None: + def __init__(self, extra_translations: dict[str, Catalog] = {}) -> None: # read all po files at startup path = Path(__file__).parent / "messages" for file in path.glob("*.po"): @@ -18,11 +18,13 @@ def __init__(self) -> None: self.translations[file.stem] = read_po(f) # empty catalog for en since it is not used anyway self.translations[DEFAULT_LANGUAGE] = Catalog() + if extra_translations: + self.translations.update(extra_translations) self.current_language = DEFAULT_LANGUAGE def translate(self, msg: str) -> str: translation = self.translations[self.current_language].get(msg) - if translation: + if translation and translation.string: return translation.string else: return msg @@ -56,7 +58,7 @@ def parse_language_header(self, lang_header: str) -> list[str]: code = code.split("-")[0] result.append((q, code)) # sort by quality value and return only the codes - return [t[1] for t in sorted(result)] + return [t[1] for t in sorted(result, key=lambda tup: tup[0])] Translator = _Translator() diff --git a/tests/unit/test_translator.py b/tests/unit/test_translator.py new file mode 100644 index 0000000000..094c71fa5b --- /dev/null +++ b/tests/unit/test_translator.py @@ -0,0 +1,70 @@ +from unittest import TestCase + +from babel import Locale +from babel.messages.catalog import Catalog +from babel.messages.pofile import PoFileParser + +from openslides_backend.i18n.translator import _Translator + + +class TranslatorTest(TestCase): + def setUp(self) -> None: + catalog = Catalog( + locale=Locale("de"), last_translator="Bob", charset="utf-8", fuzzy=False + ) + parser = PoFileParser(catalog) + parser.parse( + [ + 'msgid "a"', + 'msgstr "b"', + "", + 'msgid "c"', + 'msgstr "d"', + "", + 'msgid "e"', + 'msgstr ""', + ] + ) + self.translator = _Translator({"de": catalog}) + + def test_translate_wrong_language(self) -> None: + assert self.translator.translate("c") == "c" + + def test_translate_normal(self) -> None: + self.translator.set_translation_language("de") + assert self.translator.translate("a") == "b" + + def test_translate_unknown_phrase(self) -> None: + self.translator.set_translation_language("de") + assert ( + self.translator.translate("abcdefghijklmnopqrstuvwxyz") + == "abcdefghijklmnopqrstuvwxyz" + ) + + def test_translate_untranslated(self) -> None: + self.translator.set_translation_language("de") + assert self.translator.translate("e") == "e" + + def test_translate_order_sorted_header_non_alphabetical(self) -> None: + self.translator.set_translation_language("en,de") + assert self.translator.translate("c") == "c" + + def test_translate_order_sorted_header_alphabetical(self) -> None: + self.translator.set_translation_language("de,en") + assert self.translator.translate("c") == "d" + + def test_translate_weight_sorted_header_asc(self) -> None: + self.translator.set_translation_language("en;q=0.5,de;q=0.8") + assert self.translator.translate("c") == "c" + + def test_translate_weight_sorted_header_desc(self) -> None: + self.translator.set_translation_language("en;q=0.8,de;q=0.5") + assert self.translator.translate("c") == "d" + + def test_translate_half_weight_sorted_header_asc(self) -> None: + self.translator.set_translation_language("en;q=0.5,de") + assert self.translator.translate("c") == "c" + + def test_translate_half_weight_sorted_header_desc(self) -> None: + self.translator.set_translation_language("en,de;q=0.5") + assert self.translator.translate("c") == "d" From e2c091a223f8783b57c44faa48902d2448eac128 Mon Sep 17 00:00:00 2001 From: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> Date: Wed, 6 Nov 2024 09:39:27 +0100 Subject: [PATCH 40/90] More email placeholders (#2679) --- docs/actions/user.send_invitation_email.md | 10 +++ .../actions/user/send_invitation_email.py | 29 +++++++ .../action/user/test_send_invitation_email.py | 76 +++++++++++++++++++ 3 files changed, 115 insertions(+) diff --git a/docs/actions/user.send_invitation_email.md b/docs/actions/user.send_invitation_email.md index a48b4e6a02..2a385bb1bb 100644 --- a/docs/actions/user.send_invitation_email.md +++ b/docs/actions/user.send_invitation_email.md @@ -35,6 +35,11 @@ Use `meeting/users_email_subject` (if a meeting is given by `meeting_id`); other - `event_name` as `meeting/name` (if a meeting is given, otherwise `organization/name`) and - `username` as `user/username` - `name` as the users short name +- `title` as `user/title` +- `given_name` as `user/first_name` +- `surname` as `user/last_name` +- `groups` as a listing of the names of the users groups in the meeting +- `structure_levels` as a listing of the names of the users structure_levels in the meeting to be replaced in the template (see [here](https://github.com/OpenSlides/OpenSlides/blob/7315626e18c0515b6ff61551c705156cbd5056cb/server/openslides/users/models.py#L266)) ### Body @@ -44,6 +49,11 @@ It does an equal string formatting as for the subject. Use `meeting/users_email - `url` as `meeting/users_pdf_url` (this can only be used, if a meeting is given; otherwise it's the `organization/url`) - `username` as `user/username` - `password` as `user/default_password` +- `title` as `user/title` +- `first_name` as `user/first_name` +- `last_name` as `user/last_name` +- `groups` as a listing of the names of the users groups in the meeting. +- `structure_levels` as a listing of the names of the users structure_levels. ### Unknown keywords in Subject or Body Sending email is no longer refused, the wrong keyword will be injected instead. diff --git a/openslides_backend/action/actions/user/send_invitation_email.py b/openslides_backend/action/actions/user/send_invitation_email.py index a2c995fd13..f724c62fdc 100644 --- a/openslides_backend/action/actions/user/send_invitation_email.py +++ b/openslides_backend/action/actions/user/send_invitation_email.py @@ -16,6 +16,7 @@ from openslides_backend.shared.util import ONE_ORGANIZATION_FQID +from ....action.mixins.meeting_user_helper import get_meeting_user from ....models.models import User from ....permissions.management_levels import OrganizationManagementLevel from ....permissions.permission_helper import ( @@ -23,6 +24,7 @@ has_perm, ) from ....permissions.permissions import Permissions +from ....services.datastore.commands import GetManyRequest from ....shared.exceptions import DatastoreException, MissingPermission from ....shared.interfaces.write_request import WriteRequest from ....shared.patterns import fqid_from_collection_and_id @@ -214,8 +216,35 @@ def __missing__(self, key: str) -> str: "event_name": mail_data.get("name", ""), "name": self.get_verbose_username(user), "username": user.get("username", ""), + "title": user.get("title", ""), + "first_name": user.get("first_name", ""), + "last_name": user.get("last_name", ""), }, ) + if meeting_id: + m_user = get_meeting_user( + self.datastore, + meeting_id, + user_id, + ["structure_level_ids", "group_ids"], + ) + gmr = [ + GetManyRequest(coll, coll_ids, ["name"]) + for coll in ["group", "structure_level"] + if m_user and (coll_ids := m_user.get(coll + "_ids")) + ] + gm_result: dict[str, dict[int, dict[str, Any]]] = {} + if len(gmr): + gm_result = self.datastore.get_many(gmr, lock_result=False) + subject_format.update( + { + coll + + "s": ", ".join( + [model["name"] for model in gm_result.get(coll, {}).values()] + ) + for coll in ["group", "structure_level"] + } + ) body_dict = { "password": user.get("default_password", ""), "url": mail_data.get("url", ""), diff --git a/tests/system/action/user/test_send_invitation_email.py b/tests/system/action/user/test_send_invitation_email.py index edfa5a76bd..59d191c669 100644 --- a/tests/system/action/user/test_send_invitation_email.py +++ b/tests/system/action/user/test_send_invitation_email.py @@ -766,3 +766,79 @@ def test_with_locked_meeting(self) -> None: "Missing Permission: user.can_update Mail 1 from 1", response.json["results"][0][0]["message"], ) + + def test_correct_modified_body_new_placeholders(self) -> None: + self.set_models( + { + "user/2": { + "title": "Mr.", + }, + "meeting_user/2": { + "structure_level_ids": [1, 2, 3], + }, + "meeting/1": { + "users_email_subject": "Instructions for {title} {last_name} of group(s) {groups}", + "users_email_body": """Hello {first_name}! +Your shopping list: {structure_levels}. +Please ensure all of it is bought and brought over at least a week before new year's eve.""", + "structure_level_ids": [1, 2, 3], + }, + "structure_level/1": {"name": "Rock sugar", "meeting_user_ids": [2]}, + "structure_level/2": {"name": "Anise", "meeting_user_ids": [2]}, + "structure_level/3": {"name": "Cardamom", "meeting_user_ids": [2]}, + } + ) + handler = AIOHandler() + with AiosmtpdServerManager(handler): + response = self.request( + "user.send_invitation_email", + { + "id": 2, + "meeting_id": 1, + }, + ) + self.assert_status_code(response, 200) + self.assertEqual(response.json["results"][0][0]["sent"], True) + self.assertIn( + "Subject: Instructions for Mr. Beam of group(s) group1", + handler.emails[0]["data"], + ) + self.assertIn( + "Hello Jim!\r\nYour shopping list: Rock sugar, Anise, Cardamom.\r\nPlease ensure all of it is bought and brought over at least a week before new=\r\n year's eve.", + handler.emails[0]["data"], + ) + + def test_correct_organization_new_placeholders(self) -> None: + self.set_models( + { + "user/2": { + "title": "Mr.", + }, + ONE_ORGANIZATION_FQID: { + "name": "test orga name", + "users_email_subject": "Instructions for {title} {last_name} of group(s) {groups}", + "users_email_body": """Hello {first_name}! +Your shopping list: {structure_levels}. +Please ensure all of it is bought and brought over at least a week before new year's eve.""", + }, + } + ) + handler = AIOHandler() + with AiosmtpdServerManager(handler): + response = self.request( + "user.send_invitation_email", + { + "id": 2, + }, + ) + self.assert_status_code(response, 200) + print(response.json["results"]) + self.assertEqual(response.json["results"][0][0]["sent"], True) + self.assertIn( + "Subject: Instructions for Mr. Beam of group(s) 'groups'", + handler.emails[0]["data"], + ) + self.assertIn( + "Hello Jim!\r\nYour shopping list: 'structure_levels'.\r\nPlease ensure all of it is bought and brought over at least a week before new=\r\n year's eve.", + handler.emails[0]["data"], + ) From 9bbed167d6a824a5f9eff2ab89ad4ac6b9db94c9 Mon Sep 17 00:00:00 2001 From: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> Date: Thu, 7 Nov 2024 12:31:33 +0100 Subject: [PATCH 41/90] Add user_id to action_worker (#2717) * Add user_id to action_worker * Add description * Add migration --- global/data/example-data.json | 2 +- global/data/initial-data.json | 2 +- global/meta | 2 +- openslides_backend/action/action_worker.py | 1 + .../0061_add_user_ids_to_action_workers.py | 23 ++++++ openslides_backend/models/models.py | 7 ++ tests/system/action/test_action_worker.py | 71 ++++++++++++++++- ...est_0061_add_user_ids_to_action_workers.py | 78 +++++++++++++++++++ 8 files changed, 179 insertions(+), 7 deletions(-) create mode 100644 openslides_backend/migrations/migrations/0061_add_user_ids_to_action_workers.py create mode 100644 tests/system/migrations/test_0061_add_user_ids_to_action_workers.py diff --git a/global/data/example-data.json b/global/data/example-data.json index 13ce494082..3f2550c9f0 100644 --- a/global/data/example-data.json +++ b/global/data/example-data.json @@ -1,5 +1,5 @@ { - "_migration_index": 61, + "_migration_index": 62, "gender":{ "1":{ "id": 1, diff --git a/global/data/initial-data.json b/global/data/initial-data.json index 88eecdaf3c..57df08e21b 100644 --- a/global/data/initial-data.json +++ b/global/data/initial-data.json @@ -1,5 +1,5 @@ { - "_migration_index": 61, + "_migration_index": 62, "gender":{ "1":{ "id": 1, diff --git a/global/meta b/global/meta index 21bc4fd12a..6a8b38544c 160000 --- a/global/meta +++ b/global/meta @@ -1 +1 @@ -Subproject commit 21bc4fd12a604933cd8732a130e973e34835f541 +Subproject commit 6a8b38544c9aa3172c2c0bf2fd5962993431673c diff --git a/openslides_backend/action/action_worker.py b/openslides_backend/action/action_worker.py index 9364710373..6cd9fb5fac 100644 --- a/openslides_backend/action/action_worker.py +++ b/openslides_backend/action/action_worker.py @@ -130,6 +130,7 @@ def initial_action_worker_write(self) -> str: "state": ActionWorkerState.RUNNING, "created": self.start_time, "timestamp": current_time, + "user_id": self.user_id, }, ) ], diff --git a/openslides_backend/migrations/migrations/0061_add_user_ids_to_action_workers.py b/openslides_backend/migrations/migrations/0061_add_user_ids_to_action_workers.py new file mode 100644 index 0000000000..f68f4c8bb5 --- /dev/null +++ b/openslides_backend/migrations/migrations/0061_add_user_ids_to_action_workers.py @@ -0,0 +1,23 @@ +from datastore.migrations import BaseModelMigration +from datastore.writer.core import BaseRequestEvent, RequestUpdateEvent + +from openslides_backend.shared.patterns import fqid_from_collection_and_id + + +class Migration(BaseModelMigration): + """ + This migration adds the user_id "-1" to all existing action_workers. + This is the number usually used for calls using the internal route. + """ + + target_migration_index = 62 + + def migrate_models(self) -> list[BaseRequestEvent] | None: + action_workers = self.reader.get_all("action_worker", ["id"]) + return [ + RequestUpdateEvent( + fqid_from_collection_and_id("action_worker", worker["id"]), + {"user_id": -1}, + ) + for worker in action_workers.values() + ] diff --git a/openslides_backend/models/models.py b/openslides_backend/models/models.py index 20c6d405af..e3670a62da 100644 --- a/openslides_backend/models/models.py +++ b/openslides_backend/models/models.py @@ -2462,6 +2462,13 @@ class ActionWorker(Model): created = fields.TimestampField(required=True) timestamp = fields.TimestampField(required=True) result = fields.JSONField() + user_id = fields.IntegerField( + required=True, + constant=True, + constraints={ + "description": "Id of the calling user. If the action is called via internal route, the value will be -1." + }, + ) class ImportPreview(Model): diff --git a/tests/system/action/test_action_worker.py b/tests/system/action/test_action_worker.py index 0064a71971..2597ae1066 100644 --- a/tests/system/action/test_action_worker.py +++ b/tests/system/action/test_action_worker.py @@ -107,7 +107,67 @@ def test_action_worker_not_ready_before_timeout_okay(self) -> None: self.assert_model_exists( f"motion/{count_motions}", {"title": f"test_title {count_motions}"} ) - self.assert_model_exists("action_worker/1", {"state": ActionWorkerState.END}) + self.assert_model_exists( + "action_worker/1", {"state": ActionWorkerState.END, "user_id": 1} + ) + + def test_internal_action_worker_not_ready_before_timeout_okay(self) -> None: + """action thread used, main process ends before action_worker is ready, + but the final result will be okay. + """ + self.create_meeting(222) + self.set_user_groups(1, [222]) + chat_message_ids = [i + 1 for i in range(50)] + self.set_models( + { + "meeting/222": { + "chat_group_ids": [22], + "chat_message_ids": chat_message_ids, + }, + "meeting_user/1": {"chat_message_ids": chat_message_ids}, + "chat_group/22": { + "name": "blob", + "chat_message_ids": chat_message_ids, + "read_group_ids": [222, 223, 224], + "write_group_ids": [222], + "meeting_id": 222, + }, + **{ + fqid_from_collection_and_id("chat_message", id_): { + "content": f"Message {id_}", + "created": 1600000000 + id_, + "meeting_user_id": 1, + "chat_group_id": 22, + "meeting_id": 222, + } + for id_ in chat_message_ids + }, + } + ) + self.set_thread_watch_timeout(0) + response = self.request("chat_group.clear", {"id": 22}, internal=True) + + self.assert_status_code(response, 202) + self.assertIn( + "Action (chat_group.clear) lasts too long. action_worker/1 written to database. Get the result from database, when the job is done.", + response.json["message"], + ) + self.assertFalse( + response.json["success"], + "Action worker still not finished, success must be False.", + ) + self.assertEqual( + response.json["results"][0][0], + {"fqid": "action_worker/1", "name": "chat_group.clear", "written": True}, + ) + if action_worker := self.get_thread_by_name("action_worker"): + action_worker.join() + self.assert_model_exists("chat_group/22", {"chat_message_ids": []}) + for id_ in chat_message_ids: + self.assert_model_deleted(fqid_from_collection_and_id("chat_message", id_)) + self.assert_model_exists( + "action_worker/1", {"state": ActionWorkerState.END, "user_id": -1} + ) def test_action_worker_not_ready_before_timeout_exception(self) -> None: """action thread used, ended after timeout""" @@ -150,7 +210,7 @@ def test_action_worker_not_ready_before_timeout_exception(self) -> None: action_worker.join() self.assert_model_not_exists("motion/1") action_worker1 = self.assert_model_exists( - "action_worker/1", {"state": ActionWorkerState.END} + "action_worker/1", {"state": ActionWorkerState.END, "user_id": 1} ) self.assertFalse(action_worker1["result"]["success"]) self.assertIn("Text is required", action_worker1["result"]["message"]) @@ -255,6 +315,7 @@ def thread_method(self: ActionWorkerTest) -> None: "id": self.new_id, "name": "test", "state": ActionWorkerState.RUNNING, + "user_id": 1, }, ) ], @@ -265,7 +326,8 @@ def thread_method(self: ActionWorkerTest) -> None: end2 = datetime.now() thread.join() self.assert_model_exists( - "action_worker/1", {"name": "test", "state": ActionWorkerState.RUNNING} + "action_worker/1", + {"name": "test", "state": ActionWorkerState.RUNNING, "user_id": 1}, ) assert ( self.start1 < start2 and self.end1 > end2 @@ -285,6 +347,7 @@ def test_action_worker_delete_by_ids(self) -> None: "id": new_id, "name": "test", "state": ActionWorkerState.RUNNING, + "user_id": 1, }, ) ], @@ -294,7 +357,7 @@ def test_action_worker_delete_by_ids(self) -> None: ) self.assert_model_exists( f"action_worker/{new_id}", - {"name": "test", "state": ActionWorkerState.RUNNING}, + {"name": "test", "state": ActionWorkerState.RUNNING, "user_id": 1}, ) self.datastore.write_without_events( diff --git a/tests/system/migrations/test_0061_add_user_ids_to_action_workers.py b/tests/system/migrations/test_0061_add_user_ids_to_action_workers.py new file mode 100644 index 0000000000..cdc45d27ec --- /dev/null +++ b/tests/system/migrations/test_0061_add_user_ids_to_action_workers.py @@ -0,0 +1,78 @@ +from openslides_backend.action.action_worker import ActionWorkerState + + +def test_migration(write, finalize, assert_model): + write( + { + "type": "create", + "fqid": "action_worker/1", + "fields": { + "id": 1, + "name": "meeting.update", + "state": ActionWorkerState.RUNNING, + "created": 123456789, + "timestamp": 123456790, + }, + }, + { + "type": "create", + "fqid": "action_worker/2", + "fields": { + "id": 2, + "name": "meeting.update", + "state": ActionWorkerState.ABORTED, + "created": 123456789, + "timestamp": 123456789, + }, + }, + { + "type": "create", + "fqid": "action_worker/3", + "fields": { + "id": 3, + "name": "meeting.update", + "state": ActionWorkerState.END, + "created": 123456790, + "timestamp": 123456790, + }, + }, + ) + write( + {"type": "delete", "fqid": "action_worker/2", "fields": {}}, + ) + + finalize("0061_add_user_ids_to_action_workers") + + assert_model( + "action_worker/1", + { + "id": 1, + "name": "meeting.update", + "state": ActionWorkerState.RUNNING, + "created": 123456789, + "timestamp": 123456790, + "user_id": -1, + }, + ) + assert_model( + "action_worker/2", + { + "id": 2, + "name": "meeting.update", + "state": ActionWorkerState.ABORTED, + "created": 123456789, + "timestamp": 123456789, + "meta_deleted": True, + }, + ) + assert_model( + "action_worker/3", + { + "id": 3, + "name": "meeting.update", + "state": ActionWorkerState.END, + "created": 123456790, + "timestamp": 123456790, + "user_id": -1, + }, + ) From f2942ffeebced931c16a0e4eb374ee2018d99d59 Mon Sep 17 00:00:00 2001 From: "openslides-automation[bot]" <125256978+openslides-automation[bot]@users.noreply.github.com> Date: Thu, 7 Nov 2024 11:46:53 +0000 Subject: [PATCH 42/90] Update meta repository (#2719) Co-authored-by: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> --- global/meta | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global/meta b/global/meta index 6a8b38544c..97d0647de3 160000 --- a/global/meta +++ b/global/meta @@ -1 +1 @@ -Subproject commit 6a8b38544c9aa3172c2c0bf2fd5962993431673c +Subproject commit 97d0647de38fa73d64ee9d5e1363ef507dd27dd3 From 24d913dc2e488b193735b6200260e78dc2834fa0 Mon Sep 17 00:00:00 2001 From: peb-adr Date: Fri, 8 Nov 2024 10:28:31 +0100 Subject: [PATCH 43/90] Don't wait for DS if ANONYMOUS_ONLY=1 (#2713) --- entrypoint.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 30ef30c2bd..cc4d80e879 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,6 +1,9 @@ #!/bin/bash source scripts/export_datastore_variables.sh -scripts/wait.sh $DATASTORE_WRITER_HOST $DATASTORE_WRITER_PORT + +if [ ! $ANONYMOUS_ONLY ]; then + scripts/wait.sh $DATASTORE_WRITER_HOST $DATASTORE_WRITER_PORT +fi exec "$@" From ed34503a71eec1d3c3e0f8683a41dbc3e996118c Mon Sep 17 00:00:00 2001 From: Hannes Janott Date: Fri, 8 Nov 2024 12:23:36 +0100 Subject: [PATCH 44/90] Update export_service_commits.sh to new datastore commit (#2716) --- requirements/export_service_commits.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/export_service_commits.sh b/requirements/export_service_commits.sh index 8e62af5325..f647b4bc96 100755 --- a/requirements/export_service_commits.sh +++ b/requirements/export_service_commits.sh @@ -1,3 +1,3 @@ #!/bin/bash -export DATASTORE_COMMIT_HASH=46ee759b96507ed62c7dfe95972cbae4a826b2e2 -export AUTH_COMMIT_HASH=c12b89bd2bcefc6868fcc5cd3d57c23d04bd47b0 +export DATASTORE_COMMIT_HASH=ff32410426631356f67013ebdd607c099cd676a1 +export AUTH_COMMIT_HASH=31bf8d3e965f59ab4203223fff1d9a640f3a3444 From 9996193202e2b3275cc3a8ed7ee207117ea1ec37 Mon Sep 17 00:00:00 2001 From: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> Date: Mon, 11 Nov 2024 11:14:55 +0100 Subject: [PATCH 45/90] Case insensitive email matching for forget password (#2698) * Case insensitive email matching for forget password * Update docs/actions/user.forget_password.md Co-authored-by: Hannes Janott --------- Co-authored-by: rrenkert Co-authored-by: Hannes Janott --- docs/actions/user.forget_password.md | 4 +- .../action/actions/user/forget_password.py | 8 +-- .../action/user/test_forget_password.py | 49 +++++++++++++++++++ 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/docs/actions/user.forget_password.md b/docs/actions/user.forget_password.md index 5076ab18aa..72bb68b3f9 100644 --- a/docs/actions/user.forget_password.md +++ b/docs/actions/user.forget_password.md @@ -9,11 +9,11 @@ ## Action -A user can change their password by this action without being authenticated. To perform a changing of their password the user enters their email-address. The client then sends a hard-coded reset email to the given email-address (as `email`). TODO: translations +A user can change their password by this action without being authenticated. To perform a changing of their password the user enters their email-address (case insensitive). The client then sends a hard-coded reset email to the given email-address (as `email`). TODO: translations Regardless if the email-address is used by a user, the client shows the user a successful message (for example "An email was successfully sent to the given email-address"). This is necessary to avoid filtering which email-address is used by an OpenSlides-user. -In the case that an email-address is used by a user, an email is sent to that email-address with the given text including a link to set a new password. If multiple users use the same email address, one email is sent per user. +In the case that an email-address is used by a user, an email is sent to that email-address of that user including a link to set a new password. If multiple users use the same email address (case insensitive), one email is sent per user (case-sensitive). The link redirects a user to `/login/forget-password-confirm?user_id=&token=`. As you can see, the user_id of the user and a token are given as query-parameters. The token is a [jsonwebtoken](https://jwt.io/) (specified by [RFC7519](https://datatracker.ietf.org/doc/html/rfc7519)), which is self-contained and up to ten minutes valid. The user_id and the email-address (as `email`) are given as payload to the token. Furthermore, the token is signed. The secret to sign the token is the secret which is used to sign `access_token`s. The algorithm `HS256` will be used. The token is given as a base64-encoded string. diff --git a/openslides_backend/action/actions/user/forget_password.py b/openslides_backend/action/actions/user/forget_password.py index 40de46e003..60db56ac4f 100644 --- a/openslides_backend/action/actions/user/forget_password.py +++ b/openslides_backend/action/actions/user/forget_password.py @@ -56,9 +56,9 @@ def get_updated_instances(self, action_data: ActionData) -> ActionData: raise ActionException(f"'{email}' is not a valid email adress.") # search for users with email - filter_ = FilterOperator("email", "=", email) + filter_ = FilterOperator("email", "~=", email) results = self.datastore.filter( - self.model.collection, filter_, ["id", "username", "saml_id"] + self.model.collection, filter_, ["id", "username", "saml_id", "email"] ) organization = self.datastore.get( @@ -79,11 +79,11 @@ def get_updated_instances(self, action_data: ActionData) -> ActionData: mail_client, self.logger, EmailSettings.default_from_email, - email, + user["email"], self.PW_FORGET_EMAIL_SUBJECT + f": {username}", self.get_email_body( user["id"], - self.get_token(user["id"], email), + self.get_token(user["id"], user["email"]), user["username"], url, ), diff --git a/tests/system/action/user/test_forget_password.py b/tests/system/action/user/test_forget_password.py index c138b007d8..92259b0e7b 100644 --- a/tests/system/action/user/test_forget_password.py +++ b/tests/system/action/user/test_forget_password.py @@ -142,6 +142,55 @@ def test_forget_password_two_users_with_email(self) -> None: ) assert "https://openslides.example.com" in handler.emails[1]["data"] + def test_forget_password_two_users_case_insensitive(self) -> None: + id_to_email = {1: "Test@ntvtn.dE", 2: "tEst@nTVTN.de"} + self.set_models( + { + ONE_ORGANIZATION_FQID: {"url": "https://openslides.example.com"}, + "user/1": {"email": id_to_email[1]}, + "user/2": {"email": id_to_email[2], "username": "test2"}, + } + ) + start_time = int(time()) + handler = AIOHandler() + with AiosmtpdServerManager(handler): + response = self.request( + "user.forget_password", {"email": "TEST@ntvtn.de"}, anonymous=True + ) + self.assert_status_code(response, 200) + user = self.get_model("user/1") + assert user.get("last_email_sent", 0) >= start_time + user2 = self.get_model("user/2") + assert user2.get("last_email_sent", 0) >= start_time + assert handler.emails[0]["from"] == EmailSettings.default_from_email + + index_of_user_1 = 0 if handler.emails[0]["to"][0] == id_to_email[1] else 1 + assert handler.emails[index_of_user_1]["to"][0] == id_to_email[1] + assert ( + "Reset your OpenSlides password: admin" + in handler.emails[index_of_user_1]["data"] + ) + assert ( + "https://openslides.example.com" in handler.emails[index_of_user_1]["data"] + ) + + assert ( + handler.emails[1 - index_of_user_1]["from"] + == EmailSettings.default_from_email + ) + assert handler.emails[1 - index_of_user_1]["to"][0] == id_to_email[2] + assert ( + "Reset your OpenSlides password: test2" + in handler.emails[1 - index_of_user_1]["data"] + ) + assert ("test2" in handler.emails[0]["data"]) != ( + "test2" in handler.emails[1]["data"] + ) + assert ( + "https://openslides.example.com" + in handler.emails[1 - index_of_user_1]["data"] + ) + def test_forget_password_no_user_found(self) -> None: self.set_models({ONE_ORGANIZATION_FQID: {"url": None}}) handler = AIOHandler() From 7abf5f68967f28abd5097978e13a4da29808978c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Nov 2024 13:25:06 +0000 Subject: [PATCH 46/90] Bump mypy from 1.12.0 to 1.13.0 in /requirements/partial (#2700) Bumps [mypy](https://github.com/python/mypy) from 1.12.0 to 1.13.0. - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v1.12.0...v1.13.0) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/partial/requirements_development.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/partial/requirements_development.txt b/requirements/partial/requirements_development.txt index 8ddb6ecf86..5328345c56 100644 --- a/requirements/partial/requirements_development.txt +++ b/requirements/partial/requirements_development.txt @@ -4,7 +4,7 @@ black==24.10.0 debugpy==1.8.7 flake8==7.1.1 isort==5.13.2 -mypy==1.12.0 +mypy==1.13.0 pip-check==2.9 pytest==8.3.3 pytest-cov==5.0.0 From ae316b6929d2cbd7e9fbb30d920d14394ef9f1c7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Nov 2024 14:48:35 +0100 Subject: [PATCH 47/90] Bump pyupgrade from 3.18.0 to 3.19.0 in /requirements/partial (#2699) Bumps [pyupgrade](https://github.com/asottile/pyupgrade) from 3.18.0 to 3.19.0. - [Commits](https://github.com/asottile/pyupgrade/compare/v3.18.0...v3.19.0) --- updated-dependencies: - dependency-name: pyupgrade dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/partial/requirements_development.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/partial/requirements_development.txt b/requirements/partial/requirements_development.txt index 5328345c56..5c472d079f 100644 --- a/requirements/partial/requirements_development.txt +++ b/requirements/partial/requirements_development.txt @@ -9,7 +9,7 @@ pip-check==2.9 pytest==8.3.3 pytest-cov==5.0.0 pytest-profiling==1.7.0 -pyupgrade==3.18.0 +pyupgrade==3.19.0 pyyaml==6.0.2 # typing From 0387c08c96de9eac42612d0c7863e427f2a760bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Nov 2024 14:04:18 +0000 Subject: [PATCH 48/90] Bump types-requests in /requirements/partial (#2688) Bumps [types-requests](https://github.com/python/typeshed) from 2.32.0.20240914 to 2.32.0.20241016. - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-requests dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/partial/requirements_development.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/partial/requirements_development.txt b/requirements/partial/requirements_development.txt index 5c472d079f..95b47575cf 100644 --- a/requirements/partial/requirements_development.txt +++ b/requirements/partial/requirements_development.txt @@ -17,6 +17,6 @@ types-babel==2.11.0.15 types-beautifulsoup4==4.12.0.20240907 types-bleach==6.1.0.20240331 types-PyYAML==6.0.12.20240917 -types-requests==2.32.0.20240914 +types-requests==2.32.0.20241016 types-simplejson==3.19.0.20240801 types-Pygments==2.18.0.20240506 From 11201219a87d80661964634af7fe992fe0d3660b Mon Sep 17 00:00:00 2001 From: "openslides-automation[bot]" <125256978+openslides-automation[bot]@users.noreply.github.com> Date: Mon, 11 Nov 2024 14:23:04 +0000 Subject: [PATCH 49/90] Update meta repository (#2725) Co-authored-by: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> --- global/meta | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global/meta b/global/meta index 97d0647de3..0c7ea66ede 160000 --- a/global/meta +++ b/global/meta @@ -1 +1 @@ -Subproject commit 97d0647de38fa73d64ee9d5e1363ef507dd27dd3 +Subproject commit 0c7ea66edebd9902e1223446e70a22aab0222c3d From 688fdb7dddc8678702a16d4f91433f9e96073385 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Nov 2024 15:00:41 +0000 Subject: [PATCH 50/90] Bump debugpy from 1.8.7 to 1.8.8 in /requirements/partial (#2720) Bumps [debugpy](https://github.com/microsoft/debugpy) from 1.8.7 to 1.8.8. - [Release notes](https://github.com/microsoft/debugpy/releases) - [Commits](https://github.com/microsoft/debugpy/compare/v1.8.7...v1.8.8) --- updated-dependencies: - dependency-name: debugpy dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> --- requirements/partial/requirements_development.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/partial/requirements_development.txt b/requirements/partial/requirements_development.txt index 95b47575cf..c2c17266c4 100644 --- a/requirements/partial/requirements_development.txt +++ b/requirements/partial/requirements_development.txt @@ -1,7 +1,7 @@ aiosmtpd==1.4.6 autoflake==2.3.1 black==24.10.0 -debugpy==1.8.7 +debugpy==1.8.8 flake8==7.1.1 isort==5.13.2 mypy==1.13.0 From 4d786d09b1adf77682d512a1d8e2408347b4e77d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Nov 2024 15:17:54 +0000 Subject: [PATCH 51/90] Bump pytest-cov from 5.0.0 to 6.0.0 in /requirements/partial (#2708) Bumps [pytest-cov](https://github.com/pytest-dev/pytest-cov) from 5.0.0 to 6.0.0. - [Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-cov/compare/v5.0.0...v6.0.0) --- updated-dependencies: - dependency-name: pytest-cov dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/partial/requirements_development.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/partial/requirements_development.txt b/requirements/partial/requirements_development.txt index c2c17266c4..479338e0b7 100644 --- a/requirements/partial/requirements_development.txt +++ b/requirements/partial/requirements_development.txt @@ -7,7 +7,7 @@ isort==5.13.2 mypy==1.13.0 pip-check==2.9 pytest==8.3.3 -pytest-cov==5.0.0 +pytest-cov==6.0.0 pytest-profiling==1.7.0 pyupgrade==3.19.0 pyyaml==6.0.2 From e16f70d48cffcef29773d26e00b60707c6840717 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Nov 2024 15:34:45 +0000 Subject: [PATCH 52/90] Bump dependency-injector from 4.42.0 to 4.43.0 in /requirements/partial (#2712) Bumps [dependency-injector](https://github.com/ets-labs/python-dependency-injector) from 4.42.0 to 4.43.0. - [Commits](https://github.com/ets-labs/python-dependency-injector/compare/4.42.0...4.43.0) --- updated-dependencies: - dependency-name: dependency-injector dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/partial/requirements_production.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/partial/requirements_production.txt b/requirements/partial/requirements_production.txt index fb1fc89c85..c0b1241800 100644 --- a/requirements/partial/requirements_production.txt +++ b/requirements/partial/requirements_production.txt @@ -1,7 +1,7 @@ Babel==2.16.0 beautifulsoup4==4.12.3 bleach[css]==6.1.0 -dependency_injector==4.42.0 +dependency_injector==4.43.0 fastjsonschema==2.20.0 gunicorn==23.0.0 pypdf[crypto]==5.0.1 From 936a4cc84fc96bec92fd2ed54e9a3b66da02c4a4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Nov 2024 15:49:56 +0000 Subject: [PATCH 53/90] Bump types-beautifulsoup4 in /requirements/partial (#2693) Bumps [types-beautifulsoup4](https://github.com/python/typeshed) from 4.12.0.20240907 to 4.12.0.20241020. - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-beautifulsoup4 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/partial/requirements_development.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/partial/requirements_development.txt b/requirements/partial/requirements_development.txt index 479338e0b7..47e94684e5 100644 --- a/requirements/partial/requirements_development.txt +++ b/requirements/partial/requirements_development.txt @@ -14,7 +14,7 @@ pyyaml==6.0.2 # typing types-babel==2.11.0.15 -types-beautifulsoup4==4.12.0.20240907 +types-beautifulsoup4==4.12.0.20241020 types-bleach==6.1.0.20240331 types-PyYAML==6.0.12.20240917 types-requests==2.32.0.20241016 From b1fe0838a2c2fa41ddd68a5659a43d00695204db Mon Sep 17 00:00:00 2001 From: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> Date: Mon, 11 Nov 2024 17:30:39 +0100 Subject: [PATCH 54/90] Bump opentelemetry dependencies update datastore and auth hashes (#2726) * Bump the opentelemetry-dependencies group across 1 directory with 5 updates Bumps the opentelemetry-dependencies group with 5 updates in the /requirements/partial directory: | Package | From | To | | --- | --- | --- | | [opentelemetry-api](https://github.com/open-telemetry/opentelemetry-python) | `1.27.0` | `1.28.1` | | [opentelemetry-sdk](https://github.com/open-telemetry/opentelemetry-python) | `1.27.0` | `1.28.1` | | [opentelemetry-exporter-otlp](https://github.com/open-telemetry/opentelemetry-python) | `1.27.0` | `1.28.1` | | [opentelemetry-instrumentation-flask](https://github.com/open-telemetry/opentelemetry-python-contrib) | `0.48b0` | `0.49b1` | | [opentelemetry-instrumentation-requests](https://github.com/open-telemetry/opentelemetry-python-contrib) | `0.48b0` | `0.49b1` | Updates `opentelemetry-api` from 1.27.0 to 1.28.1 - [Release notes](https://github.com/open-telemetry/opentelemetry-python/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-python/blob/v1.28.1/CHANGELOG.md) - [Commits](https://github.com/open-telemetry/opentelemetry-python/compare/v1.27.0...v1.28.1) Updates `opentelemetry-sdk` from 1.27.0 to 1.28.1 - [Release notes](https://github.com/open-telemetry/opentelemetry-python/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-python/blob/v1.28.1/CHANGELOG.md) - [Commits](https://github.com/open-telemetry/opentelemetry-python/compare/v1.27.0...v1.28.1) Updates `opentelemetry-exporter-otlp` from 1.27.0 to 1.28.1 - [Release notes](https://github.com/open-telemetry/opentelemetry-python/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-python/blob/v1.28.1/CHANGELOG.md) - [Commits](https://github.com/open-telemetry/opentelemetry-python/compare/v1.27.0...v1.28.1) Updates `opentelemetry-instrumentation-flask` from 0.48b0 to 0.49b1 - [Release notes](https://github.com/open-telemetry/opentelemetry-python-contrib/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-python-contrib/blob/main/CHANGELOG.md) - [Commits](https://github.com/open-telemetry/opentelemetry-python-contrib/commits) Updates `opentelemetry-instrumentation-requests` from 0.48b0 to 0.49b1 - [Release notes](https://github.com/open-telemetry/opentelemetry-python-contrib/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-python-contrib/blob/main/CHANGELOG.md) - [Commits](https://github.com/open-telemetry/opentelemetry-python-contrib/commits) --- updated-dependencies: - dependency-name: opentelemetry-api dependency-type: direct:production update-type: version-update:semver-minor dependency-group: opentelemetry-dependencies - dependency-name: opentelemetry-sdk dependency-type: direct:production update-type: version-update:semver-minor dependency-group: opentelemetry-dependencies - dependency-name: opentelemetry-exporter-otlp dependency-type: direct:production update-type: version-update:semver-minor dependency-group: opentelemetry-dependencies - dependency-name: opentelemetry-instrumentation-flask dependency-type: direct:production dependency-group: opentelemetry-dependencies - dependency-name: opentelemetry-instrumentation-requests dependency-type: direct:production dependency-group: opentelemetry-dependencies ... Signed-off-by: dependabot[bot] * Bump opentelemetry dependencies update datastore and auth hashes --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/export_service_commits.sh | 4 ++-- requirements/partial/requirements_production.txt | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/requirements/export_service_commits.sh b/requirements/export_service_commits.sh index f647b4bc96..82a9e372bf 100755 --- a/requirements/export_service_commits.sh +++ b/requirements/export_service_commits.sh @@ -1,3 +1,3 @@ #!/bin/bash -export DATASTORE_COMMIT_HASH=ff32410426631356f67013ebdd607c099cd676a1 -export AUTH_COMMIT_HASH=31bf8d3e965f59ab4203223fff1d9a640f3a3444 +export DATASTORE_COMMIT_HASH=827476ebb2f796a2a8ac665d5dbb577e870fc4de +export AUTH_COMMIT_HASH=85f5d0e10f7ab455b45f8da73ea7c69ce953e569 diff --git a/requirements/partial/requirements_production.txt b/requirements/partial/requirements_production.txt index c0b1241800..09aac616bd 100644 --- a/requirements/partial/requirements_production.txt +++ b/requirements/partial/requirements_production.txt @@ -13,8 +13,8 @@ python-magic==0.4.27 pygments==2.18.0 # opentelemetry -opentelemetry-api==1.27.0 -opentelemetry-sdk==1.27.0 -opentelemetry-exporter-otlp==1.27.0 -opentelemetry-instrumentation-flask==0.48b0 -opentelemetry-instrumentation-requests==0.48b0 +opentelemetry-api==1.28.1 +opentelemetry-sdk==1.28.1 +opentelemetry-exporter-otlp==1.28.1 +opentelemetry-instrumentation-flask==0.49b1 +opentelemetry-instrumentation-requests==0.49b1 From 45bdd24b638223c3bef87986f6955fe3d3016434 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Nov 2024 09:32:27 +0100 Subject: [PATCH 55/90] Bump pypdf[crypto] from 5.0.1 to 5.1.0 in /requirements/partial (#2705) Bumps [pypdf[crypto]](https://github.com/py-pdf/pypdf) from 5.0.1 to 5.1.0. - [Release notes](https://github.com/py-pdf/pypdf/releases) - [Changelog](https://github.com/py-pdf/pypdf/blob/main/CHANGELOG.md) - [Commits](https://github.com/py-pdf/pypdf/compare/5.0.1...5.1.0) --- updated-dependencies: - dependency-name: pypdf[crypto] dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/partial/requirements_production.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/partial/requirements_production.txt b/requirements/partial/requirements_production.txt index 09aac616bd..ee1a3c7080 100644 --- a/requirements/partial/requirements_production.txt +++ b/requirements/partial/requirements_production.txt @@ -4,7 +4,7 @@ bleach[css]==6.1.0 dependency_injector==4.43.0 fastjsonschema==2.20.0 gunicorn==23.0.0 -pypdf[crypto]==5.0.1 +pypdf[crypto]==5.1.0 requests==2.32.3 roman==4.2 simplejson==3.19.3 From cb07383d8cc109a9b892cf354e07b5d62a38400c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Nov 2024 09:53:12 +0100 Subject: [PATCH 56/90] Bump bleach[css] from 6.1.0 to 6.2.0 in /requirements/partial (#2709) Bumps [bleach[css]](https://github.com/mozilla/bleach) from 6.1.0 to 6.2.0. - [Changelog](https://github.com/mozilla/bleach/blob/main/CHANGES) - [Commits](https://github.com/mozilla/bleach/compare/v6.1.0...v6.2.0) --- updated-dependencies: - dependency-name: bleach[css] dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/partial/requirements_production.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/partial/requirements_production.txt b/requirements/partial/requirements_production.txt index ee1a3c7080..83ac300a93 100644 --- a/requirements/partial/requirements_production.txt +++ b/requirements/partial/requirements_production.txt @@ -1,6 +1,6 @@ Babel==2.16.0 beautifulsoup4==4.12.3 -bleach[css]==6.1.0 +bleach[css]==6.2.0 dependency_injector==4.43.0 fastjsonschema==2.20.0 gunicorn==23.0.0 From b5921c5f6d29a2b3f7b2c070befb9e7eda569d06 Mon Sep 17 00:00:00 2001 From: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> Date: Wed, 13 Nov 2024 12:14:41 +0100 Subject: [PATCH 57/90] Check for poll permission correctly (#2622) * Check for poll permission correctly * Update documentation --- docs/actions/option.update.md | 2 +- docs/actions/poll.anonymize.md | 2 +- docs/actions/poll.create.md | 2 +- docs/actions/poll.delete.md | 2 +- docs/actions/poll.publish.md | 2 +- docs/actions/poll.reset.md | 2 +- docs/actions/poll.start.md | 2 +- docs/actions/poll.stop.md | 2 +- docs/actions/poll.update.md | 2 +- .../action/actions/poll/functions.py | 10 +++++++--- openslides_backend/action/actions/poll/mixins.py | 4 +++- tests/system/action/poll/test_create.py | 6 ++---- tests/system/action/poll/test_delete.py | 14 +++++++++++--- tests/system/action/poll/test_publish.py | 9 +++++++-- 14 files changed, 39 insertions(+), 22 deletions(-) diff --git a/docs/actions/option.update.md b/docs/actions/option.update.md index 8a197c2010..81d99e4bdf 100644 --- a/docs/actions/option.update.md +++ b/docs/actions/option.update.md @@ -20,4 +20,4 @@ If the poll's state is *created* and at least one vote value is given (`Y`, `N` The request user needs: - `motion.can_manage_polls` if the poll's content object is a motion - `assignment.can_manage` if the poll's content object is an assignment -- `poll.can_manage` else +- `poll.can_manage` if the poll's content object is a topic diff --git a/docs/actions/poll.anonymize.md b/docs/actions/poll.anonymize.md index ac0fbeb14f..442ffb07e5 100644 --- a/docs/actions/poll.anonymize.md +++ b/docs/actions/poll.anonymize.md @@ -12,4 +12,4 @@ Only for non-analog polls in the state *finished* or *published*. Sets all `vote The request user needs: - `motion.can_manage_polls` if the poll's content object is a motion - `assignment.can_manage` if the poll's content object is an assignment -- `poll.can_manage` else +- `poll.can_manage` if the poll's content object is a topic diff --git a/docs/actions/poll.create.md b/docs/actions/poll.create.md index 543194f794..7398cad8d8 100644 --- a/docs/actions/poll.create.md +++ b/docs/actions/poll.create.md @@ -68,4 +68,4 @@ The `entitled_group_ids` may not contain the meetings `anonymous_group_id`. The request user needs: - `motion.can_manage_polls` if the poll's content object is a motion - `assignment.can_manage` if the poll's content object is an assignment -- `poll.can_manage` else +- `poll.can_manage` if the poll's content object is a topic diff --git a/docs/actions/poll.delete.md b/docs/actions/poll.delete.md index f6ef35e38d..cfea2d9a28 100644 --- a/docs/actions/poll.delete.md +++ b/docs/actions/poll.delete.md @@ -10,4 +10,4 @@ Deletes the given poll and all linked options with all votes, too. The request user needs: - `motion.can_manage_polls` if the poll's content object is a motion - `assignment.can_manage` if the poll's content object is an assignment -- `poll.can_manage` else +- `poll.can_manage` if the poll's content object is a topic diff --git a/docs/actions/poll.publish.md b/docs/actions/poll.publish.md index 38df11f899..a5a0c8af5e 100644 --- a/docs/actions/poll.publish.md +++ b/docs/actions/poll.publish.md @@ -10,4 +10,4 @@ Sets the state to *published*. Only allowed for polls in the *finished* state. The request user needs: - `motion.can_manage_polls` if the poll's content object is a motion - `assignment.can_manage` if the poll's content object is an assignment -- `poll.can_manage` else +- `poll.can_manage` if the poll's content object is a topic diff --git a/docs/actions/poll.reset.md b/docs/actions/poll.reset.md index 33081bc0eb..0d8368b0bc 100644 --- a/docs/actions/poll.reset.md +++ b/docs/actions/poll.reset.md @@ -12,4 +12,4 @@ If `type != "pseudoanonymous"`, `is_pseudoanonymized` may be reset to `false` (i The request user needs: - `motion.can_manage_polls` if the poll's content object is a motion - `assignment.can_manage` if the poll's content object is an assignment -- `poll.can_manage` else +- `poll.can_manage` if the poll's content object is a topic diff --git a/docs/actions/poll.start.md b/docs/actions/poll.start.md index d50b49991c..1278779135 100644 --- a/docs/actions/poll.start.md +++ b/docs/actions/poll.start.md @@ -12,4 +12,4 @@ If `meeting/poll_couple_countdown` is true and the poll is an electronic poll, t The request user needs: - `motion.can_manage_polls` if the poll's content object is a motion - `assignment.can_manage` if the poll's content object is an assignment -- `poll.can_manage` else +- `poll.can_manage` if the poll's content object is a topic diff --git a/docs/actions/poll.stop.md b/docs/actions/poll.stop.md index a44d7abd42..d271a3e2bb 100644 --- a/docs/actions/poll.stop.md +++ b/docs/actions/poll.stop.md @@ -16,4 +16,4 @@ Some fields have to be calculated upon stopping a poll: The request user needs: - `motion.can_manage_polls` if the poll's content object is a motion - `assignment.can_manage` if the poll's content object is an assignment -- `poll.can_manage` else +- `poll.can_manage` if the poll's content object is a topic diff --git a/docs/actions/poll.update.md b/docs/actions/poll.update.md index c880c442da..4544c4d64c 100644 --- a/docs/actions/poll.update.md +++ b/docs/actions/poll.update.md @@ -43,4 +43,4 @@ The `entitled_group_ids` may not contain the meetings `anonymous_group_id`. The request user needs: - `motion.can_manage_polls` if the poll's content object is a motion - `assignment.can_manage` if the poll's content object is an assignment -- `poll.can_manage` else +- `poll.can_manage` if the poll's content object is a topic diff --git a/openslides_backend/action/actions/poll/functions.py b/openslides_backend/action/actions/poll/functions.py index ebd9666f9f..d4b9bbcc24 100644 --- a/openslides_backend/action/actions/poll/functions.py +++ b/openslides_backend/action/actions/poll/functions.py @@ -1,8 +1,8 @@ from ....permissions.permission_helper import has_perm from ....permissions.permissions import Permission, Permissions from ....services.datastore.interface import DatastoreService -from ....shared.exceptions import MissingPermission -from ....shared.patterns import KEYSEPARATOR +from ....shared.exceptions import ActionException, MissingPermission +from ....shared.patterns import KEYSEPARATOR, collection_from_fqid def check_poll_or_option_perms( @@ -15,7 +15,11 @@ def check_poll_or_option_perms( perm: Permission = Permissions.Motion.CAN_MANAGE_POLLS elif content_object_id.startswith("assignment" + KEYSEPARATOR): perm = Permissions.Assignment.CAN_MANAGE - else: + elif content_object_id.startswith("topic" + KEYSEPARATOR): perm = Permissions.Poll.CAN_MANAGE + else: + raise ActionException( + f"'{collection_from_fqid(content_object_id)}' is not a valid poll collection." + ) if not has_perm(datastore, user_id, perm, meeting_id): raise MissingPermission(perm) diff --git a/openslides_backend/action/actions/poll/mixins.py b/openslides_backend/action/actions/poll/mixins.py index 557b72c203..6616952e55 100644 --- a/openslides_backend/action/actions/poll/mixins.py +++ b/openslides_backend/action/actions/poll/mixins.py @@ -5,7 +5,7 @@ from openslides_backend.shared.typing import HistoryInformation from ....services.datastore.commands import GetManyRequest -from ....shared.exceptions import VoteServiceException +from ....shared.exceptions import ActionException, VoteServiceException from ....shared.interfaces.write_request import WriteRequest from ....shared.patterns import ( collection_from_fqid, @@ -33,6 +33,8 @@ def check_permissions(self, instance: dict[str, Any]) -> None: ) content_object_id = poll.get("content_object_id", "") meeting_id = poll["meeting_id"] + if not content_object_id: + raise ActionException("No 'content_object_id' was given") check_poll_or_option_perms( content_object_id, self.datastore, self.user_id, meeting_id ) diff --git a/tests/system/action/poll/test_create.py b/tests/system/action/poll/test_create.py index eb3e7bea6f..75f6d765f2 100644 --- a/tests/system/action/poll/test_create.py +++ b/tests/system/action/poll/test_create.py @@ -276,6 +276,7 @@ def test_create_wrong_publish_immediately(self) -> None: response = self.request( "poll.create", { + "content_object_id": "assignment/1", "title": "test_title_ahThai4pae1pi4xoogoo", "pollmethod": "YN", "type": "pseudoanonymous", @@ -799,10 +800,7 @@ def test_create_without_content_object(self) -> None: }, ) self.assert_status_code(response, 400) - assert ( - response.json["message"] - == "Creation of poll/1: You try to set following required fields to an empty value: ['content_object_id']" - ) + assert response.json["message"] == "No 'content_object_id' was given" def test_create_no_permissions_assignment(self) -> None: self.base_permission_test( diff --git a/tests/system/action/poll/test_delete.py b/tests/system/action/poll/test_delete.py index 639d04c9f8..714034d1f7 100644 --- a/tests/system/action/poll/test_delete.py +++ b/tests/system/action/poll/test_delete.py @@ -33,15 +33,18 @@ def test_delete_wrong_id(self) -> None: def test_delete_correct_cascading(self) -> None: self.set_models( { + "topic/1": {"poll_ids": [111], "meeting_id": 1}, "poll/111": { "option_ids": [42], "meeting_id": 1, "projection_ids": [1], + "content_object_id": "topic/1", }, "option/42": {"poll_id": 111, "meeting_id": 1}, "meeting/1": { "is_active_in_organization_id": 1, "all_projection_ids": [1], + "topic_ids": [1], }, "projection/1": { "content_object_id": "poll/111", @@ -64,9 +67,11 @@ def test_delete_correct_cascading(self) -> None: def test_delete_cascading_poll_candidate_list(self) -> None: self.set_models( { + "topic/1": {"poll_ids": [111], "meeting_id": 1}, "poll/111": { "option_ids": [42], "meeting_id": 1, + "content_object_id": "topic/1", }, "option/42": { "poll_id": 111, @@ -77,6 +82,7 @@ def test_delete_cascading_poll_candidate_list(self) -> None: "is_active_in_organization_id": 1, "poll_candidate_list_ids": [12], "poll_candidate_ids": [13], + "topic_ids": [1], }, "poll_candidate_list/12": { "meeting_id": 1, @@ -100,12 +106,14 @@ def test_delete_cascading_poll_candidate_list(self) -> None: def test_delete_no_permissions(self) -> None: self.base_permission_test( - {"poll/111": {"meeting_id": 1}}, "poll.delete", {"id": 111} + {"poll/111": {"meeting_id": 1, "content_object_id": "topic/1"}}, + "poll.delete", + {"id": 111}, ) def test_delete_permissions(self) -> None: self.base_permission_test( - {"poll/111": {"meeting_id": 1}}, + {"poll/111": {"meeting_id": 1, "content_object_id": "topic/1"}}, "poll.delete", {"id": 111}, Permissions.Poll.CAN_MANAGE, @@ -113,7 +121,7 @@ def test_delete_permissions(self) -> None: def test_delete_permissions_locked_meeting(self) -> None: self.base_locked_out_superadmin_permission_test( - {"poll/111": {"meeting_id": 1}}, + {"poll/111": {"meeting_id": 1, "content_object_id": "topic/1"}}, "poll.delete", {"id": 111}, ) diff --git a/tests/system/action/poll/test_publish.py b/tests/system/action/poll/test_publish.py index b7d5fad5cd..b2c9126c5e 100644 --- a/tests/system/action/poll/test_publish.py +++ b/tests/system/action/poll/test_publish.py @@ -42,8 +42,13 @@ def test_publish_assignment(self) -> None: def test_publish_wrong_state(self) -> None: self.set_models( { - "poll/1": {"state": "created", "meeting_id": 1}, - "meeting/1": {"is_active_in_organization_id": 1}, + "topic/1": {"poll_ids": [111], "meeting_id": 1}, + "poll/1": { + "state": "created", + "meeting_id": 1, + "content_object_id": "topic/1", + }, + "meeting/1": {"is_active_in_organization_id": 1, "topic_ids": [1]}, } ) response = self.request("poll.publish", {"id": 1}) From efdc41958e162372892b82e7914177e555ef0470 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20B=C3=B6hlke?= Date: Mon, 7 Oct 2024 10:03:27 +0200 Subject: [PATCH 58/90] Work on deeper authlib integration --- openslides_backend/action/action_handler.py | 6 +- openslides_backend/action/action_worker.py | 11 ++-- openslides_backend/http/views/action_view.py | 11 ++-- openslides_backend/http/views/auth.py | 66 +++++++++++++++++++ openslides_backend/services/media/adapter.py | 12 ++-- .../services/media/interface.py | 8 --- .../partial/requirements_production.txt | 1 + .../action/test_action_command_format.py | 2 +- tests/system/util.py | 1 - 9 files changed, 89 insertions(+), 29 deletions(-) create mode 100644 openslides_backend/http/views/auth.py diff --git a/openslides_backend/action/action_handler.py b/openslides_backend/action/action_handler.py index 2a56056b70..27bec3abae 100644 --- a/openslides_backend/action/action_handler.py +++ b/openslides_backend/action/action_handler.py @@ -4,6 +4,7 @@ from typing import Any, TypeVar, cast import fastjsonschema +from authlib.jose import JWTClaims from ..shared.exceptions import ( ActionException, @@ -29,6 +30,7 @@ Payload, PayloadElement, ) +from ..http.views.auth import AuthContext T = TypeVar("T") @@ -94,7 +96,7 @@ def get_health_info(cls) -> Iterable[tuple[str, dict[str, Any]]]: def handle_request( self, payload: Payload, - user_id: int, + user_id: AuthContext, atomic: bool = True, internal: bool = False, ) -> ActionsResponse: @@ -154,7 +156,7 @@ def execute_internal_action(self, action: str, data: dict[str, Any]) -> None: "data": [data], } ], - -1, + AuthContext(-1, "", JWTClaims({})), internal=True, ) diff --git a/openslides_backend/action/action_worker.py b/openslides_backend/action/action_worker.py index 9364710373..fd4d8c05fd 100644 --- a/openslides_backend/action/action_worker.py +++ b/openslides_backend/action/action_worker.py @@ -18,6 +18,7 @@ from ..shared.interfaces.write_request import WriteRequest from .action_handler import ActionHandler from .util.typing import ActionsResponse, Payload +from ..http.views.auth import AuthContext, token_storage class ActionWorkerState(str, Enum): @@ -44,7 +45,7 @@ def handle_action_in_worker_thread( ) action_worker_thread = ActionWorker( payload, - user_id, + AuthContext(user_id, token_storage.access_token, token_storage.claims), is_atomic, handler, lock, @@ -220,7 +221,7 @@ class ActionWorker(threading.Thread): def __init__( self, payload: Payload, - user_id: int, + auth_context: AuthContext, is_atomic: bool, handler: ActionHandler, lock: threading.Lock, @@ -229,7 +230,7 @@ def __init__( super().__init__(name="action_worker") self.handler = handler self.payload = payload - self.user_id = user_id + self.auth_context = auth_context self.is_atomic = is_atomic self.lock = lock self.internal = internal @@ -240,8 +241,8 @@ def run(self): # type: ignore self.started = True try: self.response = self.handler.handle_request( - self.payload, self.user_id, self.is_atomic, self.internal - ) + self.payload, self.auth_context, self.is_atomic, self.internal + ) except Exception as exception: self.exception = exception diff --git a/openslides_backend/http/views/action_view.py b/openslides_backend/http/views/action_view.py index bb2a568388..44bdb5579b 100644 --- a/openslides_backend/http/views/action_view.py +++ b/openslides_backend/http/views/action_view.py @@ -3,6 +3,8 @@ from base64 import b64decode from pathlib import Path +from authlib.jose import JWTClaims + from os_authlib.message_bus import MessageBus from ...action.action_handler import ActionHandler from ...action.action_worker import handle_action_in_worker_thread @@ -16,10 +18,10 @@ from ..http_exceptions import Unauthorized from ..request import Request from .base_view import BaseView, route +from .auth import token_required INTERNAL_AUTHORIZATION_HEADER = "Authorization" - VERSION_PATH = Path(__file__).parent / ".." / ".." / "version.txt" @@ -34,15 +36,14 @@ def __init__(self, *args, **kwargs): self.message_bus = MessageBus() @route(["handle_request", "handle_separately"]) - def action_route(self, request: Request) -> RouteResponse: + @token_required + def action_route(self, request: Request, claims: JWTClaims) -> RouteResponse: self.logger.debug("Start dispatching action request.") assert_migration_index() # Get user id. - user_id, access_token = self.get_user_id_from_headers( - request.headers, request.cookies - ) + user_id, access_token = int(claims.get("userId")), claims.get("access_token") # Set Headers and Cookies in services. self.services.vote().set_authentication( request.headers.get(AUTHENTICATION_HEADER, ""), diff --git a/openslides_backend/http/views/auth.py b/openslides_backend/http/views/auth.py new file mode 100644 index 0000000000..30fa97b7d7 --- /dev/null +++ b/openslides_backend/http/views/auth.py @@ -0,0 +1,66 @@ +import threading + +import requests +from authlib.jose import JsonWebKey, jwt, JWTClaims +from authlib.oauth2.rfc9068 import JWTBearerTokenValidator +from authlib.oidc.discovery import OpenIDProviderMetadata, get_well_known_url +from werkzeug.exceptions import Unauthorized, Forbidden +from werkzeug.local import Local + +token_storage = Local() + +KEYCLOAK_DOMAIN = 'http://keycloak:8080' +KEYCLOAK_REALM = 'os' +ISSUER = f"{KEYCLOAK_DOMAIN}/realms/{KEYCLOAK_REALM}" + +class MyBearerTokenValidator(JWTBearerTokenValidator): + # Cache the JWKS keys to avoid fetching them repeatedly + jwk_set = None + + def get_jwks(self): + if self.jwk_set is None: + oidc_configuration = OpenIDProviderMetadata(requests.get(get_well_known_url(ISSUER, True)).json()) + response = requests.get(oidc_configuration.get('jwks_uri')) + response.raise_for_status() + jwks_keys = response.json() + self.jwk_set = JsonWebKey.import_key_set(jwks_keys) + return self.jwk_set + + # def verify_token(self, token): + # try: + # claims = jwt.decode(token, key=self.get_jwks_key_set()) + # claims.validate() + # return claims + # except Exception as e: + # return None + +def token_required(f): + def decorated_function(view, request, *args, **kwargs): + auth_header = request.headers.get('Authentication') + if not auth_header: + raise Unauthorized('missing token') + + token = auth_header.split(" ")[1] + validator = MyBearerTokenValidator(ISSUER, 'https://localhost:8000/system') + claims = validator.authenticate_token(token) + + if not claims: + raise Forbidden('missing or invalid token') + + view.logger.debug(f"Saving Token claims to thread: {threading.get_ident()}") + + token_storage.claims = claims + token_storage.access_token = token + + return f(view, request, claims, *args, **kwargs) + return decorated_function + +class AuthContext: + user_id: int + access_token: str + claims: JWTClaims + + def __init__(self, user_id: int, access_token: str, claims: JWTClaims): + self.user_id = user_id + self.access_token = access_token + self.claims = claims \ No newline at end of file diff --git a/openslides_backend/services/media/adapter.py b/openslides_backend/services/media/adapter.py index c46216038c..ccaab5262c 100644 --- a/openslides_backend/services/media/adapter.py +++ b/openslides_backend/services/media/adapter.py @@ -1,3 +1,4 @@ +import threading from typing import Any import requests @@ -5,7 +6,7 @@ from ...shared.exceptions import MediaServiceException from ...shared.interfaces.logging import LoggingModule from .interface import MediaService - +from ...http.views.auth import token_storage class MediaServiceAdapter(MediaService): """ @@ -27,10 +28,6 @@ def upload_mediafile(self, file: str, id: int, mimetype: str) -> None: subpath = "upload_mediafile" self._upload(file, id, mimetype, subpath) - def upload_resource(self, file: str, id: int, mimetype: str) -> None: - subpath = "upload_resource" - self._upload(file, id, mimetype, subpath) - def duplicate_mediafile(self, source_id: int, target_id: int) -> None: url = self.media_url + "duplicate_mediafile/" payload = {"source_id": source_id, "target_id": target_id} @@ -41,7 +38,8 @@ def _handle_upload( self, url: str, payload: dict[str, Any], description: str ) -> None: try: - response = requests.post(url, json=payload) + self.logger.debug(f"Getting access token from : {threading.get_ident()}") + response = requests.post(url, json=payload, headers={"Authentication": token_storage.access_token}) except requests.exceptions.ConnectionError as e: msg = f"Connect to mediaservice failed. {e}" self.logger.debug(description + msg) @@ -50,4 +48,4 @@ def _handle_upload( if response.status_code != 200: msg = f"Mediaservice Error: {str(response.content)}" self.logger.debug(description + msg) - raise MediaServiceException(msg) + raise MediaServiceException(msg) \ No newline at end of file diff --git a/openslides_backend/services/media/interface.py b/openslides_backend/services/media/interface.py index 99a8226c3b..9a0b421ea0 100644 --- a/openslides_backend/services/media/interface.py +++ b/openslides_backend/services/media/interface.py @@ -15,14 +15,6 @@ def upload_mediafile(self, file: str, id: int, mimetype: str) -> None: """ ... - @abstractmethod - def upload_resource(self, file: str, id: int, mimetype: str) -> None: - """ - Throws a MediaServiceException, if there is a ConnectionError or - any Error reported from MediaService-Request - """ - ... - @abstractmethod def duplicate_mediafile(self, source_id: int, target_id: int) -> None: """ diff --git a/requirements/partial/requirements_production.txt b/requirements/partial/requirements_production.txt index be47d15dfb..202df5e910 100644 --- a/requirements/partial/requirements_production.txt +++ b/requirements/partial/requirements_production.txt @@ -11,6 +11,7 @@ simplejson==3.19.3 Werkzeug==3.0.4 python-magic==0.4.27 pygments==2.18.0 +authlib==1.3.1 # opentelemetry opentelemetry-api==1.27.0 diff --git a/tests/system/action/test_action_command_format.py b/tests/system/action/test_action_command_format.py index 6beb68d955..56b68c70d0 100644 --- a/tests/system/action/test_action_command_format.py +++ b/tests/system/action/test_action_command_format.py @@ -16,7 +16,7 @@ def get_action_handler(self) -> ActionHandler: logger = Mock() config = Mock() handler = ActionHandler(config, self.services, logger) - handler.user_id = 1 + handler.auth_context = 1 handler.internal = False return handler diff --git a/tests/system/util.py b/tests/system/util.py index 7d77ca8bbc..5717904f1c 100644 --- a/tests/system/util.py +++ b/tests/system/util.py @@ -87,7 +87,6 @@ def create_test_application(view: type[View]) -> OpenSlidesBackendWSGIApplicatio mock_media_service.upload_mediafile = Mock( side_effect=side_effect_for_upload_method ) - mock_media_service.upload_resource = Mock(side_effect=side_effect_for_upload_method) services.media = MagicMock(return_value=mock_media_service) return create_base_test_application(view, services, env) From 68549f942e42a91bc473b4355a3b822e3a9c611a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20B=C3=B6hlke?= Date: Thu, 10 Oct 2024 12:34:53 +0200 Subject: [PATCH 59/90] Replace os_authlib with authlib --- openslides_backend/action/action_handler.py | 4 ++-- openslides_backend/action/action_worker.py | 4 ++-- openslides_backend/http/auth_context.py | 11 +++++++++++ openslides_backend/http/token_storage.py | 3 +++ openslides_backend/http/views/auth.py | 16 ++-------------- 5 files changed, 20 insertions(+), 18 deletions(-) create mode 100644 openslides_backend/http/auth_context.py create mode 100644 openslides_backend/http/token_storage.py diff --git a/openslides_backend/action/action_handler.py b/openslides_backend/action/action_handler.py index 27bec3abae..983cf8e6ce 100644 --- a/openslides_backend/action/action_handler.py +++ b/openslides_backend/action/action_handler.py @@ -30,7 +30,7 @@ Payload, PayloadElement, ) -from ..http.views.auth import AuthContext +from ..http.auth_context import AuthContext T = TypeVar("T") @@ -156,7 +156,7 @@ def execute_internal_action(self, action: str, data: dict[str, Any]) -> None: "data": [data], } ], - AuthContext(-1, "", JWTClaims({})), + AuthContext(-1, "", JWTClaims({}, {})), internal=True, ) diff --git a/openslides_backend/action/action_worker.py b/openslides_backend/action/action_worker.py index fd4d8c05fd..43bef1d305 100644 --- a/openslides_backend/action/action_worker.py +++ b/openslides_backend/action/action_worker.py @@ -18,8 +18,8 @@ from ..shared.interfaces.write_request import WriteRequest from .action_handler import ActionHandler from .util.typing import ActionsResponse, Payload -from ..http.views.auth import AuthContext, token_storage - +from ..http.token_storage import token_storage +from ..http.auth_context import AuthContext class ActionWorkerState(str, Enum): RUNNING = "running" diff --git a/openslides_backend/http/auth_context.py b/openslides_backend/http/auth_context.py new file mode 100644 index 0000000000..6e7099feca --- /dev/null +++ b/openslides_backend/http/auth_context.py @@ -0,0 +1,11 @@ +from authlib.jose import JWTClaims + +class AuthContext: + user_id: int + access_token: str + claims: JWTClaims + + def __init__(self, user_id: int, access_token: str, claims: JWTClaims): + self.user_id = user_id + self.access_token = access_token + self.claims = claims \ No newline at end of file diff --git a/openslides_backend/http/token_storage.py b/openslides_backend/http/token_storage.py new file mode 100644 index 0000000000..a7459879a4 --- /dev/null +++ b/openslides_backend/http/token_storage.py @@ -0,0 +1,3 @@ +from werkzeug.local import Local + +token_storage = Local() \ No newline at end of file diff --git a/openslides_backend/http/views/auth.py b/openslides_backend/http/views/auth.py index 30fa97b7d7..b5a048e4b1 100644 --- a/openslides_backend/http/views/auth.py +++ b/openslides_backend/http/views/auth.py @@ -1,13 +1,11 @@ import threading import requests -from authlib.jose import JsonWebKey, jwt, JWTClaims +from authlib.jose import JsonWebKey from authlib.oauth2.rfc9068 import JWTBearerTokenValidator from authlib.oidc.discovery import OpenIDProviderMetadata, get_well_known_url from werkzeug.exceptions import Unauthorized, Forbidden -from werkzeug.local import Local - -token_storage = Local() +from ..token_storage import token_storage KEYCLOAK_DOMAIN = 'http://keycloak:8080' KEYCLOAK_REALM = 'os' @@ -54,13 +52,3 @@ def decorated_function(view, request, *args, **kwargs): return f(view, request, claims, *args, **kwargs) return decorated_function - -class AuthContext: - user_id: int - access_token: str - claims: JWTClaims - - def __init__(self, user_id: int, access_token: str, claims: JWTClaims): - self.user_id = user_id - self.access_token = access_token - self.claims = claims \ No newline at end of file From e0687e057c00b8002cc6d689167df3606d98e24e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20B=C3=B6hlke?= Date: Mon, 4 Nov 2024 12:06:13 +0100 Subject: [PATCH 60/90] Work on deeper integration of keycloak and dev reviews --- openslides_backend/action/action_handler.py | 2 +- openslides_backend/action/action_worker.py | 5 ++++- openslides_backend/models/models.py | 1 - openslides_backend/services/media/adapter.py | 9 +++++++-- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/openslides_backend/action/action_handler.py b/openslides_backend/action/action_handler.py index 983cf8e6ce..dd4c5ec5f2 100644 --- a/openslides_backend/action/action_handler.py +++ b/openslides_backend/action/action_handler.py @@ -105,7 +105,7 @@ def handle_request( parsing all actions. In the end it sends everything to the event store. """ with make_span(self.env, "handle request"): - self.user_id = user_id + self.user_id = user_id.user_id self.internal = internal try: diff --git a/openslides_backend/action/action_worker.py b/openslides_backend/action/action_worker.py index 43bef1d305..add464c5f0 100644 --- a/openslides_backend/action/action_worker.py +++ b/openslides_backend/action/action_worker.py @@ -9,7 +9,7 @@ from gunicorn.http.wsgi import Response from gunicorn.workers.gthread import ThreadWorker -from openslides_backend.shared.patterns import fqid_from_collection_and_id +from ..shared.patterns import fqid_from_collection_and_id from ..services.datastore.interface import DatastoreService from ..shared.exceptions import ActionException, DatastoreException @@ -239,6 +239,9 @@ def __init__( def run(self): # type: ignore with self.lock: self.started = True + # set global werkzeug context + token_storage.access_token = self.auth_context.access_token + token_storage.claims = self.auth_context.claims try: self.response = self.handler.handle_request( self.payload, self.auth_context, self.is_atomic, self.internal diff --git a/openslides_backend/models/models.py b/openslides_backend/models/models.py index c9a5db379b..c6e4d567ba 100644 --- a/openslides_backend/models/models.py +++ b/openslides_backend/models/models.py @@ -935,7 +935,6 @@ class StructureLevel(Model): to={"meeting": "structure_level_ids"}, required=True ) - class Group(Model): collection = "group" verbose_name = "group" diff --git a/openslides_backend/services/media/adapter.py b/openslides_backend/services/media/adapter.py index ccaab5262c..85cb87287f 100644 --- a/openslides_backend/services/media/adapter.py +++ b/openslides_backend/services/media/adapter.py @@ -3,6 +3,7 @@ import requests +from os_authlib import AUTHORIZATION_HEADER, AUTHENTICATION_HEADER from ...shared.exceptions import MediaServiceException from ...shared.interfaces.logging import LoggingModule from .interface import MediaService @@ -28,6 +29,10 @@ def upload_mediafile(self, file: str, id: int, mimetype: str) -> None: subpath = "upload_mediafile" self._upload(file, id, mimetype, subpath) + def upload_resource(self, file: str, id: int, mimetype: str) -> None: + subpath = "upload_resource" + self._upload(file, id, mimetype, subpath) + def duplicate_mediafile(self, source_id: int, target_id: int) -> None: url = self.media_url + "duplicate_mediafile/" payload = {"source_id": source_id, "target_id": target_id} @@ -38,8 +43,8 @@ def _handle_upload( self, url: str, payload: dict[str, Any], description: str ) -> None: try: - self.logger.debug(f"Getting access token from : {threading.get_ident()}") - response = requests.post(url, json=payload, headers={"Authentication": token_storage.access_token}) + self.logger.debug(f"Getting access token from : {threading.get_ident()} -> {token_storage.access_token}") + response = requests.post(url, json=payload, headers={AUTHORIZATION_HEADER: f'Bearer {token_storage.access_token}'}) except requests.exceptions.ConnectionError as e: msg = f"Connect to mediaservice failed. {e}" self.logger.debug(description + msg) From 92bd338f0ca473e87ac729ba7bd306cecf844492 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20B=C3=B6hlke?= Date: Wed, 6 Nov 2024 12:00:02 +0100 Subject: [PATCH 61/90] WIP: Work on keycloak service for migration and other features needing Keycloak Admin API connection --- .../migrations/0057_user_keycloak_upload.py | 26 +++++++ .../services/keycloak/__init__.py | 0 .../services/keycloak/adapter.py | 74 +++++++++++++++++++ .../services/keycloak/interface.py | 23 ++++++ openslides_backend/services/vote/adapter.py | 4 +- .../shared/interfaces/services.py | 4 +- openslides_backend/wsgi.py | 1 + 7 files changed, 128 insertions(+), 4 deletions(-) create mode 100644 openslides_backend/migrations/migrations/0057_user_keycloak_upload.py create mode 100644 openslides_backend/services/keycloak/__init__.py create mode 100644 openslides_backend/services/keycloak/adapter.py create mode 100644 openslides_backend/services/keycloak/interface.py diff --git a/openslides_backend/migrations/migrations/0057_user_keycloak_upload.py b/openslides_backend/migrations/migrations/0057_user_keycloak_upload.py new file mode 100644 index 0000000000..403db7ac47 --- /dev/null +++ b/openslides_backend/migrations/migrations/0057_user_keycloak_upload.py @@ -0,0 +1,26 @@ +from datastore.migrations import BaseModelMigration +from datastore.shared.util import fqid_from_collection_and_id +from datastore.writer.core import BaseRequestEvent, RequestUpdateEvent + + +class Migration(BaseModelMigration): + """ + This migration removes all default_number fields from user models + """ + + target_migration_index = 52 + + def migrate_models(self) -> list[BaseRequestEvent] | None: + events: list[BaseRequestEvent] = [] + db_models = self.reader.get_all("user") + for id_, model in db_models.items(): + if "default_number" in model: + events.append( + RequestUpdateEvent( + fqid_from_collection_and_id("user", id_), + { + "default_number": None, + }, + ) + ) + return events diff --git a/openslides_backend/services/keycloak/__init__.py b/openslides_backend/services/keycloak/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openslides_backend/services/keycloak/adapter.py b/openslides_backend/services/keycloak/adapter.py new file mode 100644 index 0000000000..76e3972271 --- /dev/null +++ b/openslides_backend/services/keycloak/adapter.py @@ -0,0 +1,74 @@ +from typing import Any + +import requests +import simplejson as json + +from ...shared.exceptions import VoteServiceException +from ...shared.interfaces.logging import LoggingModule +from ..shared.authenticated_service import AuthenticatedService +from .interface import KeycloakAdminService + + +class KeycloakAdminAdapter(KeycloakAdminService, AuthenticatedService): + """ + Adapter to connect to the vote service. + """ + + def __init__(self, vote_url: str, logging: LoggingModule) -> None: + self.url = vote_url + self.logger = logging.getLogger(__name__) + + def retrieve(self, endpoint: str, payload: dict[str, Any] | None = None) -> Any: + response = self.make_request(endpoint, payload) + message = f"Vote service sends HTTP {response.status_code} with the following content: {str(response.content)}." + if response.status_code < 400: + self.logger.debug(message) + elif response.status_code == 500: + self.logger.error(message) + raise VoteServiceException( + "Vote service sends HTTP 500 Internal Server Error." + ) + else: + self.logger.error(message) + raise VoteServiceException(message) + if response.content: + return response.json() + + def make_request(self, endpoint: str, payload: dict[str, Any] | None = None) -> Any: + if not self.access_token or not self.refresh_id: + raise VoteServiceException("You must be logged in to vote") + payload_json = json.dumps(payload, separators=(",", ":")) if payload else None + try: + return requests.post( + url=endpoint, + data=payload_json, + headers={ + "Content-Type": "application/json", + **self.get_auth_header(), + }, + cookies=self.get_auth_cookie(), + ) + except requests.exceptions.ConnectionError as e: + self.logger.error( + f"Cannot reach the vote service on {endpoint}. Error: {e}" + ) + raise VoteServiceException(f"Cannot reach the vote service on {endpoint}.") + + def start(self, id: int) -> None: + endpoint = self.get_endpoint("start", id) + self.retrieve(endpoint) + + def stop(self, id: int) -> dict[str, Any]: + endpoint = self.get_endpoint("stop", id) + return self.retrieve(endpoint) + + def clear(self, id: int) -> None: + endpoint = self.get_endpoint("clear", id) + self.retrieve(endpoint) + + def clear_all(self) -> None: + endpoint = self.get_endpoint("clear_all") + self.retrieve(endpoint) + + def get_endpoint(self, route: str, id: int | None = None) -> str: + return f"{self.url}/{route}" + (f"?id={id}" if id else "") diff --git a/openslides_backend/services/keycloak/interface.py b/openslides_backend/services/keycloak/interface.py new file mode 100644 index 0000000000..9f487b2639 --- /dev/null +++ b/openslides_backend/services/keycloak/interface.py @@ -0,0 +1,23 @@ +from abc import abstractmethod +from typing import Any, Protocol + +from ..shared.authenticated_service import AuthenticatedServiceInterface + + +class KeycloakAdminService(AuthenticatedServiceInterface, Protocol): + """ + Interface of the vote service. + """ + + @abstractmethod + def start(self, id: int) -> None: ... + + @abstractmethod + def stop(self, id: int) -> dict[str, Any]: ... + + @abstractmethod + def clear(self, id: int) -> None: ... + + @abstractmethod + def clear_all(self) -> None: + """Only for testing purposes.""" diff --git a/openslides_backend/services/vote/adapter.py b/openslides_backend/services/vote/adapter.py index 67a000bbc8..76e3972271 100644 --- a/openslides_backend/services/vote/adapter.py +++ b/openslides_backend/services/vote/adapter.py @@ -6,10 +6,10 @@ from ...shared.exceptions import VoteServiceException from ...shared.interfaces.logging import LoggingModule from ..shared.authenticated_service import AuthenticatedService -from .interface import VoteService +from .interface import KeycloakAdminService -class VoteAdapter(VoteService, AuthenticatedService): +class KeycloakAdminAdapter(KeycloakAdminService, AuthenticatedService): """ Adapter to connect to the vote service. """ diff --git a/openslides_backend/shared/interfaces/services.py b/openslides_backend/shared/interfaces/services.py index 75580e4e87..588356c5a7 100644 --- a/openslides_backend/shared/interfaces/services.py +++ b/openslides_backend/shared/interfaces/services.py @@ -3,7 +3,7 @@ from ...services.auth.interface import AuthenticationService from ...services.datastore.interface import DatastoreService from ...services.media.interface import MediaService -from ...services.vote.interface import VoteService +from ...services.vote.interface import KeycloakAdminService class Services(Protocol): @@ -17,4 +17,4 @@ def datastore(self) -> DatastoreService: ... def media(self) -> MediaService: ... - def vote(self) -> VoteService: ... + def vote(self) -> KeycloakAdminService: ... diff --git a/openslides_backend/wsgi.py b/openslides_backend/wsgi.py index 5cdde890b2..ff5be33e9d 100644 --- a/openslides_backend/wsgi.py +++ b/openslides_backend/wsgi.py @@ -28,6 +28,7 @@ class OpenSlidesBackendServices(containers.DeclarativeContainer): ) datastore = providers.Factory(ExtendedDatastoreAdapter, engine, logging, env) vote = providers.Singleton(VoteAdapter, config.vote_url, logging) + keycloak = providers.Singleton(KeycloakAdminAdapter, config.vote_url, logging) class OpenSlidesBackendWSGI(containers.DeclarativeContainer): From 7d0685c29a8e4132e6064fc88cceeb4fe2b77e0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20B=C3=B6hlke?= Date: Sun, 10 Nov 2024 09:45:42 +0100 Subject: [PATCH 62/90] Revert --- openslides_backend/services/vote/adapter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openslides_backend/services/vote/adapter.py b/openslides_backend/services/vote/adapter.py index 76e3972271..67a000bbc8 100644 --- a/openslides_backend/services/vote/adapter.py +++ b/openslides_backend/services/vote/adapter.py @@ -6,10 +6,10 @@ from ...shared.exceptions import VoteServiceException from ...shared.interfaces.logging import LoggingModule from ..shared.authenticated_service import AuthenticatedService -from .interface import KeycloakAdminService +from .interface import VoteService -class KeycloakAdminAdapter(KeycloakAdminService, AuthenticatedService): +class VoteAdapter(VoteService, AuthenticatedService): """ Adapter to connect to the vote service. """ From d21d2e29acc6a1cbcacde22db2c00411cb3b9fc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20B=C3=B6hlke?= Date: Sun, 10 Nov 2024 13:44:34 +0100 Subject: [PATCH 63/90] Scatch implementation of idp migration --- .../migrations/migration_wrapper.py | 4 + .../migrations/0057_user_keycloak_upload.py | 9 ++- .../services/keycloak/adapter.py | 75 +++---------------- .../services/keycloak/interface.py | 18 ++--- .../shared/interfaces/services.py | 7 +- openslides_backend/wsgi.py | 3 +- tests/system/base.py | 3 + .../test_0057_user_keycloak_upload.py | 41 ++++++++++ tests/system/util.py | 9 +++ 9 files changed, 86 insertions(+), 83 deletions(-) create mode 100644 tests/system/migrations/test_0057_user_keycloak_upload.py diff --git a/openslides_backend/migrations/migration_wrapper.py b/openslides_backend/migrations/migration_wrapper.py index 5ddc524623..883f190258 100644 --- a/openslides_backend/migrations/migration_wrapper.py +++ b/openslides_backend/migrations/migration_wrapper.py @@ -11,6 +11,9 @@ setup, ) from datastore.shared.typing import Fqid, Model +from datastore.shared.di import injector +from openslides_backend.services.keycloak.adapter import KeycloakAdminAdapter +from openslides_backend.services.keycloak.interface import IdpAdminService class BadMigrationModule(MigrationException): @@ -32,6 +35,7 @@ def __init__( memory_only: bool = False, ) -> None: migrations = MigrationWrapper.load_migrations() + injector.register_as_singleton(IdpAdminService, KeycloakAdminAdapter) self.handler = setup(verbose, print_fn, memory_only) self.handler.register_migrations(*migrations) diff --git a/openslides_backend/migrations/migrations/0057_user_keycloak_upload.py b/openslides_backend/migrations/migrations/0057_user_keycloak_upload.py index 403db7ac47..4171847b8f 100644 --- a/openslides_backend/migrations/migrations/0057_user_keycloak_upload.py +++ b/openslides_backend/migrations/migrations/0057_user_keycloak_upload.py @@ -1,12 +1,16 @@ from datastore.migrations import BaseModelMigration +from datastore.shared.di import service_as_singleton from datastore.shared.util import fqid_from_collection_and_id from datastore.writer.core import BaseRequestEvent, RequestUpdateEvent +from openslides_backend.services.keycloak.interface import IdpAdminService +@service_as_singleton class Migration(BaseModelMigration): """ This migration removes all default_number fields from user models """ + idpAdmin: IdpAdminService target_migration_index = 52 @@ -14,12 +18,13 @@ def migrate_models(self) -> list[BaseRequestEvent] | None: events: list[BaseRequestEvent] = [] db_models = self.reader.get_all("user") for id_, model in db_models.items(): - if "default_number" in model: + if not "kc_id" in model: + idp_id = self.idpAdmin.create_user(model.get("username"), model.get("saml_id")) events.append( RequestUpdateEvent( fqid_from_collection_and_id("user", id_), { - "default_number": None, + "idp_id": idp_id, }, ) ) diff --git a/openslides_backend/services/keycloak/adapter.py b/openslides_backend/services/keycloak/adapter.py index 76e3972271..bbfed2e516 100644 --- a/openslides_backend/services/keycloak/adapter.py +++ b/openslides_backend/services/keycloak/adapter.py @@ -1,74 +1,19 @@ -from typing import Any +from os import environ -import requests -import simplejson as json - -from ...shared.exceptions import VoteServiceException -from ...shared.interfaces.logging import LoggingModule +from .interface import IdpAdminService from ..shared.authenticated_service import AuthenticatedService -from .interface import KeycloakAdminService +from ...models.models import User +from ...shared.interfaces.logging import LoggingModule -class KeycloakAdminAdapter(KeycloakAdminService, AuthenticatedService): +class KeycloakAdminAdapter(IdpAdminService, AuthenticatedService): """ Adapter to connect to the vote service. """ - def __init__(self, vote_url: str, logging: LoggingModule) -> None: - self.url = vote_url - self.logger = logging.getLogger(__name__) - - def retrieve(self, endpoint: str, payload: dict[str, Any] | None = None) -> Any: - response = self.make_request(endpoint, payload) - message = f"Vote service sends HTTP {response.status_code} with the following content: {str(response.content)}." - if response.status_code < 400: - self.logger.debug(message) - elif response.status_code == 500: - self.logger.error(message) - raise VoteServiceException( - "Vote service sends HTTP 500 Internal Server Error." - ) - else: - self.logger.error(message) - raise VoteServiceException(message) - if response.content: - return response.json() - - def make_request(self, endpoint: str, payload: dict[str, Any] | None = None) -> Any: - if not self.access_token or not self.refresh_id: - raise VoteServiceException("You must be logged in to vote") - payload_json = json.dumps(payload, separators=(",", ":")) if payload else None - try: - return requests.post( - url=endpoint, - data=payload_json, - headers={ - "Content-Type": "application/json", - **self.get_auth_header(), - }, - cookies=self.get_auth_cookie(), - ) - except requests.exceptions.ConnectionError as e: - self.logger.error( - f"Cannot reach the vote service on {endpoint}. Error: {e}" - ) - raise VoteServiceException(f"Cannot reach the vote service on {endpoint}.") - - def start(self, id: int) -> None: - endpoint = self.get_endpoint("start", id) - self.retrieve(endpoint) - - def stop(self, id: int) -> dict[str, Any]: - endpoint = self.get_endpoint("stop", id) - return self.retrieve(endpoint) - - def clear(self, id: int) -> None: - endpoint = self.get_endpoint("clear", id) - self.retrieve(endpoint) - - def clear_all(self) -> None: - endpoint = self.get_endpoint("clear_all") - self.retrieve(endpoint) + def __init__(self, keycloak_url: str | None = None, logging: LoggingModule | None = None) -> None: + self.url = keycloak_url if keycloak_url else environ.get("OPENSLIDES_KEYCLOAK_URL") + self.logger = logging.getLogger(__name__) if logging else None - def get_endpoint(self, route: str, id: int | None = None) -> str: - return f"{self.url}/{route}" + (f"?id={id}" if id else "") + def create_user(self, user: User) -> str: + pass diff --git a/openslides_backend/services/keycloak/interface.py b/openslides_backend/services/keycloak/interface.py index 9f487b2639..67e46660e1 100644 --- a/openslides_backend/services/keycloak/interface.py +++ b/openslides_backend/services/keycloak/interface.py @@ -2,22 +2,14 @@ from typing import Any, Protocol from ..shared.authenticated_service import AuthenticatedServiceInterface +from ...models.models import User -class KeycloakAdminService(AuthenticatedServiceInterface, Protocol): +class IdpAdminService(AuthenticatedServiceInterface, Protocol): """ - Interface of the vote service. + Interface of a idp admin service. """ @abstractmethod - def start(self, id: int) -> None: ... - - @abstractmethod - def stop(self, id: int) -> dict[str, Any]: ... - - @abstractmethod - def clear(self, id: int) -> None: ... - - @abstractmethod - def clear_all(self) -> None: - """Only for testing purposes.""" + def create_user(self, username: str, saml_id: str | None) -> str: + """Create user and return new IDP user ID""" diff --git a/openslides_backend/shared/interfaces/services.py b/openslides_backend/shared/interfaces/services.py index 588356c5a7..71d6178c06 100644 --- a/openslides_backend/shared/interfaces/services.py +++ b/openslides_backend/shared/interfaces/services.py @@ -2,8 +2,9 @@ from ...services.auth.interface import AuthenticationService from ...services.datastore.interface import DatastoreService +from ...services.keycloak.interface import IdpAdminService from ...services.media.interface import MediaService -from ...services.vote.interface import KeycloakAdminService +from ...services.vote.interface import VoteService class Services(Protocol): @@ -17,4 +18,6 @@ def datastore(self) -> DatastoreService: ... def media(self) -> MediaService: ... - def vote(self) -> KeycloakAdminService: ... + def vote(self) -> VoteService: ... + + def idp_admin(self) -> IdpAdminService: ... diff --git a/openslides_backend/wsgi.py b/openslides_backend/wsgi.py index ff5be33e9d..298fc8c543 100644 --- a/openslides_backend/wsgi.py +++ b/openslides_backend/wsgi.py @@ -7,6 +7,7 @@ from .services.auth.adapter import AuthenticationHTTPAdapter from .services.datastore.extended_adapter import ExtendedDatastoreAdapter from .services.datastore.http_engine import HTTPEngine +from .services.keycloak.adapter import KeycloakAdminAdapter from .services.media.adapter import MediaServiceAdapter from .services.vote.adapter import VoteAdapter from .shared.interfaces.logging import LoggingModule @@ -28,7 +29,7 @@ class OpenSlidesBackendServices(containers.DeclarativeContainer): ) datastore = providers.Factory(ExtendedDatastoreAdapter, engine, logging, env) vote = providers.Singleton(VoteAdapter, config.vote_url, logging) - keycloak = providers.Singleton(KeycloakAdminAdapter, config.vote_url, logging) + idp_admin = providers.Singleton(KeycloakAdminAdapter, config.keycloak_url, logging) class OpenSlidesBackendWSGI(containers.DeclarativeContainer): diff --git a/tests/system/base.py b/tests/system/base.py index 72e37d31cd..d9b68727e8 100644 --- a/tests/system/base.py +++ b/tests/system/base.py @@ -16,6 +16,7 @@ from openslides_backend.services.datastore.with_database_context import ( with_database_context, ) +from openslides_backend.services.keycloak.interface import IdpAdminService from openslides_backend.shared.env import Environment from openslides_backend.shared.exceptions import ActionException, DatastoreException from openslides_backend.shared.filters import FilterOperator @@ -46,6 +47,7 @@ class BaseSystemTestCase(TestCase): datastore: DatastoreService vote_service: TestVoteService media: Any # Any is needed because it is mocked and has magic methods + idp_admin: IdpAdminService client: Client anon_client: Client @@ -65,6 +67,7 @@ def setUp(self) -> None: self.vote_service = cast(TestVoteService, self.services.vote()) self.datastore = self.services.datastore() self.datastore.truncate_db() + self.idp_admin = self.services.idp_admin() self.set_thread_watch_timeout(-1) self.created_fqids = set() diff --git a/tests/system/migrations/test_0057_user_keycloak_upload.py b/tests/system/migrations/test_0057_user_keycloak_upload.py new file mode 100644 index 0000000000..6b3680823e --- /dev/null +++ b/tests/system/migrations/test_0057_user_keycloak_upload.py @@ -0,0 +1,41 @@ +def test_migration(write, finalize, assert_model): + write( + { + "type": "create", + "fqid": "user/1", + "fields": { + "id": 1, + "username": "adam_sandler", + }, + }, + ) + write( + { + "type": "create", + "fqid": "user/2", + "fields": { + "id": 2, + "username": "stifflers", + "saml_id": "mom" + }, + }, + ) + + finalize("0057_user_keycloak_upload") + + assert_model( + "user/1", + { + "id": 1, + "username": "adam_sandler", + "idp_id": "adam_sandler" + }, + ) + assert_model( + "user/2", + { + "id": 2, + "username": "stifflers", + "idp_id": "stifflers_mom" + }, + ) \ No newline at end of file diff --git a/tests/system/util.py b/tests/system/util.py index 5717904f1c..494ce1adb3 100644 --- a/tests/system/util.py +++ b/tests/system/util.py @@ -14,6 +14,7 @@ from openslides_backend.http.views import ActionView, PresenterView from openslides_backend.http.views.base_view import ROUTE_OPTIONS_ATTR, RouteFunction from openslides_backend.services.datastore.adapter import DatastoreAdapter +from openslides_backend.services.keycloak.interface import IdpAdminService from openslides_backend.services.media.interface import MediaService from openslides_backend.services.vote.adapter import VoteAdapter from openslides_backend.services.vote.interface import VoteService @@ -52,6 +53,13 @@ def vote(self, data: dict[str, Any]) -> Response: ) return convert_to_test_response(response) +class TestIdpAdminAdapter(IdpAdminService): + def set_authentication(self, access_token: str, refresh_id: str) -> None: + pass + + def create_user(self, username: str, saml_id: str | None) -> str: + return username + f"_{saml_id}" if saml_id is not None else "" + def create_action_test_application() -> OpenSlidesBackendWSGIApplication: return create_test_application(ActionView) @@ -88,6 +96,7 @@ def create_test_application(view: type[View]) -> OpenSlidesBackendWSGIApplicatio side_effect=side_effect_for_upload_method ) services.media = MagicMock(return_value=mock_media_service) + services.idp_admin = providers.Singleton(TestIdpAdminAdapter) return create_base_test_application(view, services, env) From dca8b9f64be3ab7ba9b106ab74197edb09523834 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20B=C3=B6hlke?= Date: Tue, 12 Nov 2024 11:24:38 +0100 Subject: [PATCH 64/90] Work on actions --- openslides_backend/action/actions/user/create.py | 2 ++ .../migrations/0057_user_keycloak_upload.py | 4 ++-- openslides_backend/services/keycloak/adapter.py | 14 ++++++++++++-- openslides_backend/services/keycloak/interface.py | 2 +- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/openslides_backend/action/actions/user/create.py b/openslides_backend/action/actions/user/create.py index 49c5edbc2c..963c958e92 100644 --- a/openslides_backend/action/actions/user/create.py +++ b/openslides_backend/action/actions/user/create.py @@ -70,6 +70,8 @@ def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: if instance.get("is_active"): self.check_limit_of_user(1) + + idp_id = instance.get("idp_id") saml_id = instance.get("saml_id") if not instance.get("username"): if saml_id: diff --git a/openslides_backend/migrations/migrations/0057_user_keycloak_upload.py b/openslides_backend/migrations/migrations/0057_user_keycloak_upload.py index 4171847b8f..522510c7fc 100644 --- a/openslides_backend/migrations/migrations/0057_user_keycloak_upload.py +++ b/openslides_backend/migrations/migrations/0057_user_keycloak_upload.py @@ -19,12 +19,12 @@ def migrate_models(self) -> list[BaseRequestEvent] | None: db_models = self.reader.get_all("user") for id_, model in db_models.items(): if not "kc_id" in model: - idp_id = self.idpAdmin.create_user(model.get("username"), model.get("saml_id")) + idp_id = self.idpAdmin.create_user(model.get("username"), model.get("password"), model.get("saml_id")) events.append( RequestUpdateEvent( fqid_from_collection_and_id("user", id_), { - "idp_id": idp_id, + "idp_id": idp_id }, ) ) diff --git a/openslides_backend/services/keycloak/adapter.py b/openslides_backend/services/keycloak/adapter.py index bbfed2e516..f68924f52a 100644 --- a/openslides_backend/services/keycloak/adapter.py +++ b/openslides_backend/services/keycloak/adapter.py @@ -2,7 +2,6 @@ from .interface import IdpAdminService from ..shared.authenticated_service import AuthenticatedService -from ...models.models import User from ...shared.interfaces.logging import LoggingModule @@ -15,5 +14,16 @@ def __init__(self, keycloak_url: str | None = None, logging: LoggingModule | Non self.url = keycloak_url if keycloak_url else environ.get("OPENSLIDES_KEYCLOAK_URL") self.logger = logging.getLogger(__name__) if logging else None - def create_user(self, user: User) -> str: + def create_user(self, username: str, password_hash: str, saml_id: str | None) -> str: + ARGON2_HASH_START = "$argon2" + ''' + def is_sha512_hash(self, hash: str) -> bool: + return ( + not hash.startswith(ARGON2_HASH_START) and len(hash) == SHA512_HASHED_LENGTH + ) + + def is_argon2_hash(self, hash: str) -> bool: + return hash.startswith(ARGON2_HASH_START) + ''' pass + diff --git a/openslides_backend/services/keycloak/interface.py b/openslides_backend/services/keycloak/interface.py index 67e46660e1..801ad1050c 100644 --- a/openslides_backend/services/keycloak/interface.py +++ b/openslides_backend/services/keycloak/interface.py @@ -11,5 +11,5 @@ class IdpAdminService(AuthenticatedServiceInterface, Protocol): """ @abstractmethod - def create_user(self, username: str, saml_id: str | None) -> str: + def create_user(self, username: str, password_hash: str, saml_id: str | None) -> str: """Create user and return new IDP user ID""" From cf020c471b66f3d7f0d2e681a296eb7e8d3db9b5 Mon Sep 17 00:00:00 2001 From: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> Date: Wed, 20 Nov 2024 10:01:09 +0100 Subject: [PATCH 65/90] Agenda item permission checks for motion.create (#2728) --- .../action/actions/motion/create.py | 5 +++ tests/system/action/motion/test_create.py | 43 +++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/openslides_backend/action/actions/motion/create.py b/openslides_backend/action/actions/motion/create.py index 2a7128b83c..01c241e999 100644 --- a/openslides_backend/action/actions/motion/create.py +++ b/openslides_backend/action/actions/motion/create.py @@ -149,6 +149,11 @@ def check_permissions(self, instance: dict[str, Any]) -> None: # whitelist the fields depending on the user's permissions whitelist = [] forbidden_fields = set() + perm = Permissions.AgendaItem.CAN_MANAGE + if has_perm(self.datastore, self.user_id, perm, instance["meeting_id"]): + whitelist = [*agenda_creation_properties.keys()] + elif contained := set(agenda_creation_properties.keys()).intersection(instance): + forbidden_fields.update(contained) perm = Permissions.Mediafile.CAN_SEE if has_perm(self.datastore, self.user_id, perm, instance["meeting_id"]): whitelist.append("attachment_mediafile_ids") diff --git a/tests/system/action/motion/test_create.py b/tests/system/action/motion/test_create.py index 4740c9e8a6..5e4b305fea 100644 --- a/tests/system/action/motion/test_create.py +++ b/tests/system/action/motion/test_create.py @@ -4,6 +4,7 @@ from openslides_backend.action.mixins.delegation_based_restriction_mixin import ( DelegationBasedRestriction, ) +from openslides_backend.models.models import AgendaItem from openslides_backend.permissions.base_classes import Permission from openslides_backend.permissions.permissions import Permissions from tests.system.action.base import BaseActionTestCase @@ -422,6 +423,48 @@ def setup_permission_test( if additional_data: self.set_models(additional_data) + def test_create_permission_agenda_allowed(self) -> None: + self.setup_permission_test( + [ + Permissions.AgendaItem.CAN_MANAGE, + Permissions.Motion.CAN_CREATE, + Permissions.Motion.CAN_MANAGE_METADATA, + ] + ) + response = self.request( + "motion.create", + { + "title": "test_Xcdfgee", + "meeting_id": 1, + "text": "test", + "agenda_create": True, + "agenda_type": AgendaItem.INTERNAL_ITEM, + }, + ) + self.assert_status_code(response, 200) + + def test_create_permission_agenda_forbidden(self) -> None: + self.setup_permission_test( + [ + Permissions.Motion.CAN_CREATE, + Permissions.Motion.CAN_MANAGE_METADATA, + ] + ) + response = self.request( + "motion.create", + { + "title": "test_Xcdfgee", + "meeting_id": 1, + "text": "test", + "agenda_create": True, + "agenda_type": AgendaItem.INTERNAL_ITEM, + }, + ) + self.assert_status_code(response, 403) + assert "Forbidden fields: " in response.json["message"] + assert "agenda_create" in response.json["message"] + assert "agenda_type" in response.json["message"] + def test_create_permission_missing_can_manage(self) -> None: self.setup_permission_test([Permissions.Motion.CAN_CREATE]) response = self.request( From dec9a053cc99851f9d2223cb03d5129a5fea9f85 Mon Sep 17 00:00:00 2001 From: Hannes Janott Date: Mon, 25 Nov 2024 14:56:05 +0100 Subject: [PATCH 66/90] beautify presenter docs (#2732) Improving js syntax Adding js highlighting Deleting deleted presenters Specifying required and optional parameters --- docs/migration_route.md | 4 +-- docs/presenters/check_database.md | 12 +++---- docs/presenters/check_database_all.md | 10 +++--- docs/presenters/export_meeting.md | 4 +-- docs/presenters/get_active_users_amount.md | 2 +- docs/presenters/get_forwarding_committees.md | 6 ++-- docs/presenters/get_forwarding_meetings.md | 12 +++---- docs/presenters/get_mediafile_context.md | 18 +++++----- docs/presenters/get_meetings.md | 28 --------------- docs/presenters/get_user_related_models.md | 20 +++++------ docs/presenters/get_user_scope.md | 4 +-- docs/presenters/get_users.md | 11 +++--- docs/presenters/search_deleted_models.md | 36 ------------------- .../search_for_id_by_external_id.md | 12 +++---- docs/presenters/search_users.md | 5 +-- 15 files changed, 61 insertions(+), 123 deletions(-) delete mode 100644 docs/presenters/get_meetings.md delete mode 100644 docs/presenters/search_deleted_models.md diff --git a/docs/migration_route.md b/docs/migration_route.md index 26bd427f6d..8a80883d2a 100644 --- a/docs/migration_route.md +++ b/docs/migration_route.md @@ -23,7 +23,7 @@ enum MigrationState { "success": true, "status"?: MigrationState, "output"?: str, - "exception"?: str, + "exception"?: str } ``` `output` always contains the full output of the migration command up to this point. `exception` contains the thrown exception, if any, which can only be the case if the command is finished (meaning `status != "migration_running"`). After issuing a migration command, it is waited a short period of time for the thread to finish, so the status can be all of these things for any command (e.g. after calling `migrate`, the returned status can be either `MIGRATION_RUNNING` if the migrations did not finish directly or `FINALIZATION_REQUIRED` if the migration is already done). @@ -41,7 +41,7 @@ The `stats` return value is the following: "positions": int, "events": int, "partially_migrated_positions": int, - "fully_migrated_positions": int, + "fully_migrated_positions": int } } ``` diff --git a/docs/presenters/check_database.md b/docs/presenters/check_database.md index 57518ec938..cf1574bf81 100644 --- a/docs/presenters/check_database.md +++ b/docs/presenters/check_database.md @@ -1,23 +1,23 @@ # Payload -``` +```js { // optional - meeting_id: integer; + meeting_id: integer } ``` # Returns If okay: -``` +```js { - ok: boolean, + ok: boolean } ``` else: -``` +```js { ok: boolean, - errors: string, + errors: string } ``` diff --git a/docs/presenters/check_database_all.md b/docs/presenters/check_database_all.md index 393c25ae7e..e2881c5bac 100644 --- a/docs/presenters/check_database_all.md +++ b/docs/presenters/check_database_all.md @@ -1,21 +1,21 @@ # Payload -``` +```js { } ``` # Returns If okay: -``` +```js { - ok: boolean, + ok: boolean } ``` else: -``` +```js { ok: boolean, - errors: string, + errors: string } ``` diff --git a/docs/presenters/export_meeting.md b/docs/presenters/export_meeting.md index 02d73abc1b..b94aaab652 100644 --- a/docs/presenters/export_meeting.md +++ b/docs/presenters/export_meeting.md @@ -1,8 +1,8 @@ # Payload -``` +```js { - meeting_id: Id + meeting_id: Id // required } ``` diff --git a/docs/presenters/get_active_users_amount.md b/docs/presenters/get_active_users_amount.md index 018973a645..5e6ce7a141 100644 --- a/docs/presenters/get_active_users_amount.md +++ b/docs/presenters/get_active_users_amount.md @@ -6,7 +6,7 @@ Nothing ```js { - active_users_amount: Number; + active_users_amount: Number } ``` diff --git a/docs/presenters/get_forwarding_committees.md b/docs/presenters/get_forwarding_committees.md index df01a69435..afc410ec47 100644 --- a/docs/presenters/get_forwarding_committees.md +++ b/docs/presenters/get_forwarding_committees.md @@ -1,14 +1,14 @@ # Payload -``` +```js { - meeting_id: Id + meeting_id: Id // required } ``` # Returns -``` +```js string[] ``` diff --git a/docs/presenters/get_forwarding_meetings.md b/docs/presenters/get_forwarding_meetings.md index b76a5c454a..d971356012 100644 --- a/docs/presenters/get_forwarding_meetings.md +++ b/docs/presenters/get_forwarding_meetings.md @@ -1,19 +1,19 @@ # Payload -``` +```js { - meeting_id: Id + meeting_id: Id // required } ``` # Returns -``` +```js [ { - id: Id - name: string - default_meeting_id: Id + id: Id, + name: string, + default_meeting_id: Id, meeting: [{id: Id, name: string, start_time:timestamp|null, end_time:timestamp|null}, ...] }, ... diff --git a/docs/presenters/get_mediafile_context.md b/docs/presenters/get_mediafile_context.md index e67fb81923..cd91e4352d 100644 --- a/docs/presenters/get_mediafile_context.md +++ b/docs/presenters/get_mediafile_context.md @@ -2,7 +2,7 @@ ```js { - mediafile_ids: Id[]; + mediafile_ids: Id[] // required } ``` @@ -15,16 +15,16 @@ published: boolean, meetings_of_interest: { [meeting_id: Id]: { - name: string; - holds_attachments: boolean; - holds_logos: boolean; - holds_fonts: boolean; - holds_current_projections: boolean; - holds_history_projections: boolean; - holds_preview_projections: boolean; + name: string, + holds_attachments: boolean, + holds_logos: boolean, + holds_fonts: boolean, + holds_current_projections: boolean, + holds_history_projections: boolean, + holds_preview_projections: boolean } }, - children_amount: int, + children_amount: int } } ``` diff --git a/docs/presenters/get_meetings.md b/docs/presenters/get_meetings.md deleted file mode 100644 index 00567e7c34..0000000000 --- a/docs/presenters/get_meetings.md +++ /dev/null @@ -1,28 +0,0 @@ -# Payload - -``` -{ - with_deleted: boolean, # default: False - with_archived: boolean, # default: False -} -``` - -# Returns - -``` -[ - { - id: Id, - name: string, - deleted: boolean, - is_active_in_organization: int, - }, - ... -] -``` - -# Logic - -The request user needs OML `can_manage_users` or higher or CML `can_manage`. - -This presenter creates a filtered list of meetings for various situations. With CML permission the list shows only meetings of committees, where the user has the needed permission. diff --git a/docs/presenters/get_user_related_models.md b/docs/presenters/get_user_related_models.md index db71df47f8..6f7a9808de 100644 --- a/docs/presenters/get_user_related_models.md +++ b/docs/presenters/get_user_related_models.md @@ -2,7 +2,7 @@ ```js { - user_ids: Id[]; + user_ids: Id[] // required } ``` @@ -12,16 +12,16 @@ { [user_id: Id]: { organization_management_level: OML-String, - committees: [{ id: Id; name: String; cml: CML-String; }], + committees: [{ id: Id, name: String, cml: CML-String }], meetings: [{ - id: Id; - name: String; - is_active_in_organization_id: Id; - is_locked: boolean; - motion_submitter_ids: Id[]; - assignment_candidate_ids: Id[]; - speaker_ids: Id[]; - locked_out: boolean; + id: Id, + name: String, + is_active_in_organization_id: Id, + is_locked: boolean, + motion_submitter_ids: Id[], + assignment_candidate_ids: Id[], + speaker_ids: Id[], + locked_out: boolean }] } } diff --git a/docs/presenters/get_user_scope.md b/docs/presenters/get_user_scope.md index 928d8fc250..b4b4eca3a6 100644 --- a/docs/presenters/get_user_scope.md +++ b/docs/presenters/get_user_scope.md @@ -2,7 +2,7 @@ ```js { - user_ids: Id[]; + user_ids: Id[] // required } ``` @@ -13,7 +13,7 @@ user_id: Id: { collection: String, # one of "meeting", "committee" or "organization" id: Id, - user_oml: String, # one of "superadmin", "can_manage_organization", "can_manage_users", "" + user_oml: String, # one of "superadmin", "can_manage_organization", "can_manage_users", "" committee_ids: int[] // Ids of all committees the user is part of } } diff --git a/docs/presenters/get_users.md b/docs/presenters/get_users.md index b20707f5f4..65b147a663 100644 --- a/docs/presenters/get_users.md +++ b/docs/presenters/get_users.md @@ -1,24 +1,25 @@ # Payload -``` +```js { + // optional start_index: number, entries: number, sort_criteria: string[], // can contain ["username", "first_name", "last_name"], reverse: boolean, - filter?: string, + filter?: string } ``` # Returns -``` +```js [ { id: Id, username: string, first_name: string, - last_name: string, + last_name: string }, ... ] @@ -28,4 +29,4 @@ The request user needs OML `can_manage_users` or higher. Otherwise an error is returned. -Returns all users, that have `filer` in `username`, `first_name`, `last_name`. If filter is `null`, all users are returned. The users are sorted by `sort_criteria`. If it is not given, the default is `["username", "first_name", "last_name"]`. If `reverse` is true, the order is reversed. Lastly, the users are paginated beginning at `start_index` with at max `entries` number of users. +Returns all users, that have `filter` in `username`, `first_name`, `last_name`. If filter is `null`, all users are returned. The users are sorted by `sort_criteria`. If it is not given, the default is `["username", "first_name", "last_name"]`. If `reverse` is true, the order is reversed. Lastly, the users can be paginated beginning at `start_index` with at max `entries` number of users. diff --git a/docs/presenters/search_deleted_models.md b/docs/presenters/search_deleted_models.md deleted file mode 100644 index 3abb15b104..0000000000 --- a/docs/presenters/search_deleted_models.md +++ /dev/null @@ -1,36 +0,0 @@ -## Payload - -```js -{ - collection: string, - filter_string: string, - meeting_id: Id, -} -``` - -## Presenter - -Searches all deleted models of the given collection in the given meeting for the given filter string. The fields which -are searched differ from collection to collection: -``` -{ - "assignment": ["title"], - "motion": ["number", "title"], - "user": [ - "username", - "first_name", - "last_name", - "title", - "pronoun", - "structure_level", - "number", - "email", - ], -} -``` -These 3 are also the only allowed collections. The result list is returned as a mapping from the -model's id to the searched fields of the model (see above). - -## Permissions - -TODO diff --git a/docs/presenters/search_for_id_by_external_id.md b/docs/presenters/search_for_id_by_external_id.md index ebbd42a818..b90145c899 100644 --- a/docs/presenters/search_for_id_by_external_id.md +++ b/docs/presenters/search_for_id_by_external_id.md @@ -2,22 +2,22 @@ ```js { // required - collection: string; - external_id: string; - context_id: Id; + collection: string, + external_id: string, + context_id: Id } ``` ## Returns ```js { - id: Id; + id: Id } ``` in the case one id is found. ```js { - id: null; - error: string; + id: null, + error: string } ``` else. diff --git a/docs/presenters/search_users.md b/docs/presenters/search_users.md index 416cafead2..a220075874 100644 --- a/docs/presenters/search_users.md +++ b/docs/presenters/search_users.md @@ -2,6 +2,7 @@ ```js { + // required permission_type: "meeting" | "committee" | "organization" permission_id: number, // Id of permission scope object search: { @@ -10,7 +11,7 @@ "first_name": string, "last_name": string, "email": string, - "member_number": string, + "member_number": string }[] } ``` @@ -24,7 +25,7 @@ "first_name": string, "last_name": string, "email": string, - "member_number": string, + "member_number": string }[][] ``` A double array: The outer array has the same length as the request's `search` array and contains From 1bdd315cf7ffe5eaa35c1a2a7bee029ae543acfe Mon Sep 17 00:00:00 2001 From: Hannes Janott Date: Mon, 25 Nov 2024 15:48:00 +0100 Subject: [PATCH 67/90] remove participant presence status when removed from meeting (#2730) --- global/data/example-data.json | 2 +- global/data/initial-data.json | 2 +- .../action/actions/meeting_user/delete.py | 4 +- .../user/conditional_speaker_cascade_mixin.py | 14 +++ .../action/actions/user/user_mixins.py | 6 +- .../0062_unset_presence_of_removed_users.py | 87 +++++++++++++++++ .../system/action/meeting_user/test_delete.py | 18 +++- tests/system/action/user/test_delete.py | 6 +- tests/system/action/user/test_update.py | 16 ++++ ...st_0062_unset_presence_of_removed_users.py | 93 +++++++++++++++++++ 10 files changed, 238 insertions(+), 10 deletions(-) create mode 100644 openslides_backend/migrations/migrations/0062_unset_presence_of_removed_users.py create mode 100644 tests/system/migrations/test_0062_unset_presence_of_removed_users.py diff --git a/global/data/example-data.json b/global/data/example-data.json index 3f2550c9f0..d842581ee9 100644 --- a/global/data/example-data.json +++ b/global/data/example-data.json @@ -1,5 +1,5 @@ { - "_migration_index": 62, + "_migration_index": 63, "gender":{ "1":{ "id": 1, diff --git a/global/data/initial-data.json b/global/data/initial-data.json index 57df08e21b..ba6880235f 100644 --- a/global/data/initial-data.json +++ b/global/data/initial-data.json @@ -1,5 +1,5 @@ { - "_migration_index": 62, + "_migration_index": 63, "gender":{ "1":{ "id": 1, diff --git a/openslides_backend/action/actions/meeting_user/delete.py b/openslides_backend/action/actions/meeting_user/delete.py index c4ac6fb585..da82a4a221 100644 --- a/openslides_backend/action/actions/meeting_user/delete.py +++ b/openslides_backend/action/actions/meeting_user/delete.py @@ -34,9 +34,11 @@ def get_history_information(self) -> HistoryInformation | None: def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: meeting_user = self.datastore.get( - fqid_from_collection_and_id("meeting_user", instance["id"]), ["speaker_ids"] + fqid_from_collection_and_id("meeting_user", instance["id"]), + ["speaker_ids", "user_id", "meeting_id"], ) speaker_ids = meeting_user.get("speaker_ids", []) self.conditionally_delete_speakers(speaker_ids) + self.remove_presence(meeting_user["user_id"], meeting_user["meeting_id"]) return super().update_instance(instance) diff --git a/openslides_backend/action/actions/user/conditional_speaker_cascade_mixin.py b/openslides_backend/action/actions/user/conditional_speaker_cascade_mixin.py index 35ef5f1bb3..77fede8615 100644 --- a/openslides_backend/action/actions/user/conditional_speaker_cascade_mixin.py +++ b/openslides_backend/action/actions/user/conditional_speaker_cascade_mixin.py @@ -6,6 +6,7 @@ from ....shared.patterns import fqid_from_collection_and_id from ...action import Action from ..speaker.delete import SpeakerDeleteAction +from .set_present import UserSetPresentAction class ConditionalSpeakerCascadeMixinHelper(Action): @@ -41,6 +42,18 @@ def conditionally_delete_speakers(self, speaker_ids: list[int]) -> None: [{"id": speaker["id"]} for speaker in speakers_to_delete], ) + def remove_presence(self, user_id: int, meeting_id: int) -> None: + self.execute_other_action( + UserSetPresentAction, + [ + { + "id": user_id, + "meeting_id": meeting_id, + "present": False, + } + ], + ) + class ConditionalSpeakerCascadeMixin(ConditionalSpeakerCascadeMixinHelper): """ @@ -65,6 +78,7 @@ def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: for speaker_id in val.get("speaker_ids", []) ] self.conditionally_delete_speakers(speaker_ids) + self.remove_presence(instance["id"], removed_meeting_id) return super().update_instance(instance) diff --git a/openslides_backend/action/actions/user/user_mixins.py b/openslides_backend/action/actions/user/user_mixins.py index f2b3c836d5..9f5c99f899 100644 --- a/openslides_backend/action/actions/user/user_mixins.py +++ b/openslides_backend/action/actions/user/user_mixins.py @@ -142,10 +142,8 @@ def strip_field(self, field: str, instance: dict[str, Any]) -> None: def check_meeting_and_users( self, instance: dict[str, Any], user_fqid: FullQualifiedId ) -> None: - if instance.get("meeting_id") is not None: - self.datastore.apply_changed_model( - user_fqid, {"meeting_id": instance.get("meeting_id")} - ) + if (meeting_id := instance.get("meeting_id")) is not None: + self.datastore.apply_changed_model(user_fqid, {"meeting_id": meeting_id}) def meeting_user_set_data(self, instance: dict[str, Any]) -> None: meeting_user_data = {} diff --git a/openslides_backend/migrations/migrations/0062_unset_presence_of_removed_users.py b/openslides_backend/migrations/migrations/0062_unset_presence_of_removed_users.py new file mode 100644 index 0000000000..4ee49676de --- /dev/null +++ b/openslides_backend/migrations/migrations/0062_unset_presence_of_removed_users.py @@ -0,0 +1,87 @@ +from collections import defaultdict + +from datastore.migrations import BaseModelMigration +from datastore.reader.core import GetManyRequestPart +from datastore.writer.core import BaseRequestEvent, RequestUpdateEvent + +from openslides_backend.shared.patterns import fqid_from_collection_and_id + +from ...shared.filters import And, FilterOperator + + +class Migration(BaseModelMigration): + """ + This migration removes the presence status if the user is not part of the meeting anymore. + """ + + target_migration_index = 63 + + def migrate_models(self) -> list[BaseRequestEvent] | None: + present_users_per_meeting: dict[int, list[int]] = defaultdict(list) + meetings_per_present_user: dict[int, list[int]] = defaultdict(list) + + meeting_users = self.reader.filter( + "meeting_user", + And( + FilterOperator("group_ids", "=", "[]"), + FilterOperator("meta_deleted", "!=", True), + ), + ["user_id", "meeting_id"], + ) + for meeting_user in meeting_users.values(): + user_id = meeting_user["user_id"] + meeting_id = meeting_user["meeting_id"] + meetings_per_present_user[user_id].append(meeting_id) + present_users_per_meeting[meeting_id].append(user_id) + + meetings = self.reader.get_many( + [ + GetManyRequestPart( + "meeting", + [meeting_id for meeting_id in present_users_per_meeting], + ["present_user_ids"], + ) + ] + ).get("meeting", dict()) + users = self.reader.get_many( + [ + GetManyRequestPart( + "user", + [present_user_id for present_user_id in meetings_per_present_user], + ["is_present_in_meeting_ids"], + ) + ] + ).get("user", dict()) + return [ + *[ + RequestUpdateEvent( + fqid_from_collection_and_id("meeting", meeting_id), + { + "present_user_ids": [ + id_ + for id_ in meetings.get(meeting_id, dict()).get( + "present_user_ids", [] + ) + if id_ not in user_ids + ] + }, + ) + for meeting_id, user_ids in present_users_per_meeting.items() + if meetings + ], + *[ + RequestUpdateEvent( + fqid_from_collection_and_id("user", user_id), + { + "is_present_in_meeting_ids": [ + id_ + for id_ in users.get(user_id, dict()).get( + "is_present_in_meeting_ids", [] + ) + if id_ not in meeting_ids + ] + }, + ) + for user_id, meeting_ids in meetings_per_present_user.items() + ], + ] diff --git a/tests/system/action/meeting_user/test_delete.py b/tests/system/action/meeting_user/test_delete.py index d2e50b2fa9..54d9841acd 100644 --- a/tests/system/action/meeting_user/test_delete.py +++ b/tests/system/action/meeting_user/test_delete.py @@ -16,7 +16,18 @@ def test_delete(self) -> None: def test_delete_with_speaker(self) -> None: self.set_models( { - "meeting/10": {"is_active_in_organization_id": 1}, + "meeting/10": { + "is_active_in_organization_id": 1, + "present_user_ids": [1], + }, + "meeting/101": { + "is_active_in_organization_id": 1, + "present_user_ids": [1], + }, + "user/1": { + "is_present_in_meeting_ids": [10, 101], + "meeting_user_ids": [5], + }, "meeting_user/5": { "user_id": 1, "meeting_id": 10, @@ -38,6 +49,11 @@ def test_delete_with_speaker(self) -> None: self.assert_model_deleted("meeting_user/5") self.assert_model_deleted("speaker/1") self.assert_model_exists("speaker/2", {"meeting_id": 10, "begin_time": 123456}) + self.assert_model_exists( + "user/1", {"is_present_in_meeting_ids": [101], "meeting_user_ids": []} + ) + self.assert_model_exists("meeting/10", {"present_user_ids": []}) + self.assert_model_exists("meeting/101", {"present_user_ids": [1]}) def test_delete_with_chat_message(self) -> None: self.set_models( diff --git a/tests/system/action/user/test_delete.py b/tests/system/action/user/test_delete.py index a21ceac962..890ff3a31e 100644 --- a/tests/system/action/user/test_delete.py +++ b/tests/system/action/user/test_delete.py @@ -74,13 +74,14 @@ def test_delete_with_speaker(self) -> None: "user/111": { "username": "username_srtgb123", "meeting_user_ids": [1111], + "is_present_in_meeting_ids": [1], }, "meeting_user/1111": { "meeting_id": 1, "user_id": 111, "speaker_ids": [15, 16], }, - "meeting/1": {}, + "meeting/1": {"present_user_ids": [111]}, "speaker/15": { "meeting_user_id": 1111, "meeting_id": 1, @@ -95,13 +96,14 @@ def test_delete_with_speaker(self) -> None: response = self.request("user.delete", {"id": 111}) self.assert_status_code(response, 200) - self.assert_model_deleted("user/111") + self.assert_model_deleted("user/111", {"is_present_in_meeting_ids": []}) self.assert_model_deleted("meeting_user/1111") self.assert_model_exists( "speaker/15", {"meeting_user_id": None, "meeting_id": 1, "begin_time": 12345678}, ) self.assert_model_deleted("speaker/16") + self.assert_model_exists("meeting/1", {"present_user_ids": []}) def test_delete_with_candidate(self) -> None: self.set_models( diff --git a/tests/system/action/user/test_update.py b/tests/system/action/user/test_update.py index 75405ef691..2e3ace2e9c 100644 --- a/tests/system/action/user/test_update.py +++ b/tests/system/action/user/test_update.py @@ -2436,6 +2436,7 @@ def test_group_removal_with_speaker(self) -> None: "user/1234": { "username": "username_abcdefgh123", "meeting_user_ids": [4444, 5555], + "is_present_in_meeting_ids": [4, 5], }, "meeting_user/4444": { "meeting_id": 4, @@ -2453,11 +2454,13 @@ def test_group_removal_with_speaker(self) -> None: "is_active_in_organization_id": 1, "meeting_user_ids": [4444], "committee_id": 1, + "present_user_ids": [1234], }, "meeting/5": { "is_active_in_organization_id": 1, "meeting_user_ids": [5555], "committee_id": 1, + "present_user_ids": [1234], }, "committee/1": {"meeting_ids": [4, 5]}, "speaker/14": {"meeting_user_id": 4444, "meeting_id": 4}, @@ -2481,6 +2484,19 @@ def test_group_removal_with_speaker(self) -> None: { "username": "username_abcdefgh123", "meeting_user_ids": [4444, 5555], + "is_present_in_meeting_ids": [5], + }, + ) + self.assert_model_exists( + "meeting/4", + { + "present_user_ids": [], + }, + ) + self.assert_model_exists( + "meeting/5", + { + "present_user_ids": [1234], }, ) self.assert_model_exists( diff --git a/tests/system/migrations/test_0062_unset_presence_of_removed_users.py b/tests/system/migrations/test_0062_unset_presence_of_removed_users.py new file mode 100644 index 0000000000..19481a8e0c --- /dev/null +++ b/tests/system/migrations/test_0062_unset_presence_of_removed_users.py @@ -0,0 +1,93 @@ +from typing import Any + + +def create_data() -> dict[str, dict[str, Any]]: + return { + "meeting/1": { + "id": 1, + "name": "meeting name", + "present_user_ids": [1, 3, 2], + "meeting_user_ids": [3, 4, 5], + "group_ids": [10], + }, + "meeting/2": { + "id": 1, + "name": "meeting name", + "present_user_ids": [2], + "meeting_user_ids": [6], + }, + "user/1": { + "id": 1, + "username": "correct_user", + "is_present_in_meeting_ids": [1], + "meeting_user_ids": [3], + }, + "user/2": { + "id": 2, + "username": "wrong_user", + "is_present_in_meeting_ids": [2, 1], + # meeting users do exist after user left meeting but have no group ids + "meeting_user_ids": [ + 5, + 6, + ], + }, + "user/3": { + "id": 3, + "username": "correct_user", + "is_present_in_meeting_ids": [1], + "meeting_user_ids": [4], + }, + "meeting_user/3": {"id": 3, "user_id": 1, "meeting_id": 1, "group_ids": [10]}, + "meeting_user/4": {"id": 4, "user_id": 3, "meeting_id": 1, "group_ids": [10]}, + "meeting_user/5": {"id": 5, "user_id": 2, "meeting_id": 1, "group_ids": []}, + "meeting_user/6": {"id": 6, "user_id": 2, "meeting_id": 2, "group_ids": []}, + "group/10": {"id": 10, "meeting_user_ids": [3, 4], "meeting_id": 1}, + } + + +def test_migration_both_ways(write, finalize, assert_model): + data = create_data() + for fqid, fields in data.items(): + write({"type": "create", "fqid": fqid, "fields": fields}) + + finalize("0062_unset_presence_of_removed_users") + + data["meeting/1"]["present_user_ids"] = [1, 3] + data["meeting/2"]["present_user_ids"] = [] + data["user/2"]["is_present_in_meeting_ids"] = [] + + for fqid, fields in data.items(): + assert_model(fqid, fields) + + +def test_migration_one_way(write, finalize, assert_model): + data = create_data() + data["meeting/1"]["present_user_ids"] = [1, 3] + data["meeting/2"]["present_user_ids"] = [] + + for fqid, fields in data.items(): + write({"type": "create", "fqid": fqid, "fields": fields}) + + finalize("0062_unset_presence_of_removed_users") + + data["user/2"]["is_present_in_meeting_ids"] = [] + + for fqid, fields in data.items(): + assert_model(fqid, fields) + + +def test_migration_other_way(write, finalize, assert_model): + data = create_data() + data["user/2"]["is_present_in_meeting_ids"] = [] + + for fqid, fields in data.items(): + write({"type": "create", "fqid": fqid, "fields": fields}) + + finalize("0062_unset_presence_of_removed_users") + + data["meeting/1"]["present_user_ids"] = [1, 3] + data["meeting/2"]["present_user_ids"] = [] + + for fqid, fields in data.items(): + assert_model(fqid, fields) From deac9e116c0d5ff3e3fbde035b9218bfd11a716c Mon Sep 17 00:00:00 2001 From: Hannes Janott Date: Mon, 25 Nov 2024 16:04:14 +0100 Subject: [PATCH 68/90] new saml meeting mapping (#2722) remove old direct meeting mapping --------- Co-authored-by: rrenkert --- docs/actions/user.save_saml_account.md | 58 +- global/data/example-data.json | 3 +- global/data/initial-data.json | 3 +- .../action/actions/meeting_user/create.py | 58 +- .../action/actions/organization/update.py | 23 +- .../action/actions/speaker/create.py | 2 +- .../action/actions/user/save_saml_account.py | 447 +++++++++++--- .../system/action/organization/test_update.py | 73 ++- tests/system/action/speaker/test_create.py | 2 +- tests/system/action/user/test_create.py | 2 +- .../action/user/test_save_saml_account.py | 584 ++++++++++++++++-- tests/system/action/user/test_update.py | 8 +- 12 files changed, 1091 insertions(+), 172 deletions(-) diff --git a/docs/actions/user.save_saml_account.md b/docs/actions/user.save_saml_account.md index 523507efce..15e2891bd6 100644 --- a/docs/actions/user.save_saml_account.md +++ b/docs/actions/user.save_saml_account.md @@ -11,6 +11,8 @@ pronoun: string, is_active: boolean, is_physical_person: boolean, + member_number: string, + // Additional meeting related data can be given. See below explanation on meeting mappers. } ``` @@ -31,7 +33,61 @@ Extras to do on creation: As you can see there is no password for local login and the user can't change it. -- Add user to the meeting by adding him to the group given in the organization-wide field-mapping as `"meeting": { "external_id": "xyz", "external_group_id": "delegates"}` if a `meeting`-entry is given. If it fails for any reason, a log entry is written, but no exception thrown. Add the user always to the group, if it fails try to add him to the default group. +### Meeting Mappers +- The saml attribute mapping can have a list of 'meeting_mappers' that can be used to assign users meeting related data. (See example below.) + - A mapper can be given a 'name' for debugging purposes. + - The 'external_id' maps to the meeting and is required (logged as warning if meeting does not exist). Multiple mappers can map to the same meeting. + - If 'allow_update' is set to false, the mapper is only used if the user does not already exist. If it is not given it defaults to true. + - Mappers are only used if every condition in the list of 'conditions' resolves to true. For this the 'attribute' in the payload data needs to match the string or regex given in 'condition'. If no condition is given this defaults to true. + - The actual mappings are objects or lists of objects of attribute-default pairs (exception: number, which only has the option of an attribute). + - The attribute refers to the payloads data. + - A default value can be given in case the payloads attribute does not exist or contains no data. (Logged as debug) + - Groups and structure levels are given as a list of attribute-default pairs. +- On conflict of multiple mappers mappings on a same meetings field the last given mappers data for that field is used. Exception to this are groups and structure levels. Their data is combined. +- Values for groups and structure levels can additionally be given in comma separated lists composed as a single string. +- Values for groups are interpreted as their external ID and structure levels as their name within that meeting. +- If no group exists for a meeting and no default is given, the meetings default group is used. (Logged as warning) +- If a structure level does not exist, it is created. +- Vote weights need to be given as 6 digit decimal strings. + +``` +"meeting_mappers": [{ + "name": "Mapper-Name", + "external_id": "M2025", + "allow_update": "false", + "conditions": [{ + "attribute": "membernumber", + "condition": "1426\d{4,6}$" + }, { + "attribute": "function", + "condition": "board" + }], + "mappings": { + "groups": [{ + "attribute": "membership", + "default": "admin, standard" + }], + "structure_levels": [{ + "attribute": "ovname", + "default": "struct1, struct2" + }], + "number": {"attribute": "p_number"}, + "comment": { + "attribute": "idp_comment", + "default": "Group set via SSO" + }, + "vote_weight": { + "attribute": "vote", + "default":"1.000000" + }, + "present": { + "attribute": "present_key", + "default":"True" + } + } +}] +``` +If you are using Keycloak as your SAML-server, make sure to fill the attributes of all users. Then you also need to configure for each attribute in 'Clients' a mapping for your Openslides services 'Client Scopes'. Choose 'User Attribute' and assign the 'User Attribute' as in the step before and the 'SAML Attribut Name' as defined in Openslides 'meeting_mappers'. ## Return Value diff --git a/global/data/example-data.json b/global/data/example-data.json index d842581ee9..f0a1400f69 100644 --- a/global/data/example-data.json +++ b/global/data/example-data.json @@ -72,7 +72,8 @@ "gender": "gender", "pronoun": "pronoun", "is_active": "is_active", - "is_physical_person": "is_person" + "is_physical_person": "is_person", + "member_number": "member_number" } } }, diff --git a/global/data/initial-data.json b/global/data/initial-data.json index ba6880235f..a219af8dd1 100644 --- a/global/data/initial-data.json +++ b/global/data/initial-data.json @@ -58,7 +58,8 @@ "gender": "gender", "pronoun": "pronoun", "is_active": "is_active", - "is_physical_person": "is_person" + "is_physical_person": "is_person", + "member_number": "member_number" } } }, diff --git a/openslides_backend/action/actions/meeting_user/create.py b/openslides_backend/action/actions/meeting_user/create.py index 929cfc4083..5b08281db6 100644 --- a/openslides_backend/action/actions/meeting_user/create.py +++ b/openslides_backend/action/actions/meeting_user/create.py @@ -54,24 +54,22 @@ def get_history_information(self) -> HistoryInformation | None: information = {} for instance in self.instances: instance_information = [] - if "group_ids" in instance: - if len(instance["group_ids"]) == 1: - instance_information.extend( - [ - "Participant added to group {} in meeting {}", - fqid_from_collection_and_id( - "group", instance["group_ids"][0] - ), - ] + fqids_per_collection = { + collection_name: [ + fqid_from_collection_and_id( + collection_name, + _id, ) - else: - instance_information.append( - "Participant added to multiple groups in meeting {}", - ) - else: - instance_information.append( - "Participant added to meeting {}", - ) + for _id in ids + ] + for collection_name in ["group", "structure_level"] + if (ids := instance.get(f"{collection_name}_ids")) + } + instance_information.append( + self.compose_history_string(list(fqids_per_collection.items())) + ) + for collection_name, fqids in fqids_per_collection.items(): + instance_information.extend(fqids) instance_information.append( fqid_from_collection_and_id("meeting", instance["meeting_id"]), ) @@ -79,3 +77,29 @@ def get_history_information(self) -> HistoryInformation | None: instance_information ) return information + + def compose_history_string( + self, fqids_per_collection: list[tuple[str, list[str]]] + ) -> str: + """ + Composes a string of the shape: + Participant added to groups {}, {} and structure levels {} in meeting {}. + """ + middle_sentence_parts = [ + " ".join( + [ # prefix and to collection name if it's not the first in list + ("and " if collection_name != fqids_per_collection[0][0] else "") + + collection_name.replace("_", " ") # replace for human readablity + + ("s" if len(fqids) != 1 else ""), # plural s + ", ".join(["{}" for _ in range(len(fqids))]), + ] + ) + for collection_name, fqids in fqids_per_collection + ] + return " ".join( + [ + "Participant added to", + *middle_sentence_parts, + ("in " if fqids_per_collection else "") + "meeting {}.", + ] + ) diff --git a/openslides_backend/action/actions/organization/update.py b/openslides_backend/action/actions/organization/update.py index c82a1c90fc..95ff496010 100644 --- a/openslides_backend/action/actions/organization/update.py +++ b/openslides_backend/action/actions/organization/update.py @@ -60,13 +60,24 @@ class OrganizationUpdate( field: {**optional_str_schema, "max_length": 256} for field in allowed_user_fields } - saml_props["meeting"] = { - "type": ["object", "null"], - "properties": { - field: {**optional_str_schema, "max_length": 256} - for field in ("external_id", "external_group_id") + saml_props["meeting_mappers"] = { + "type": ["array", "null"], + "items": { + "type": "object", + "properties": { + **{ + field: {**optional_str_schema, "max_length": 256} + for field in ("external_id", "name", "allow_update") + }, + "conditions": { + "type": ["array", "null"], + "max_length": 256, + }, # , "items": {"object"} + "mappings": {"type": ["object", "array"], "max_length": 256}, + }, + "required": ["external_id"], + "additionalProperties": False, }, - "additionalProperties": False, } schema = DefaultSchema(Organization()).get_update_schema( optional_properties=group_A_fields + group_B_fields, diff --git a/openslides_backend/action/actions/speaker/create.py b/openslides_backend/action/actions/speaker/create.py index cf9aaed91f..e0eff2684a 100644 --- a/openslides_backend/action/actions/speaker/create.py +++ b/openslides_backend/action/actions/speaker/create.py @@ -294,7 +294,7 @@ def validate_fields(self, instance: dict[str, Any]) -> dict[str, Any]: user = self.datastore.get(user_fqid, ["is_present_in_meeting_ids"]) if meeting_id not in user.get("is_present_in_meeting_ids", ()): raise ActionException( - "Only present users can be on the lists of speakers." + "Only present users can be on the list of speakers." ) if not meeting.get("list_of_speakers_allow_multiple_speakers"): diff --git a/openslides_backend/action/actions/user/save_saml_account.py b/openslides_backend/action/actions/user/save_saml_account.py index 1989cde148..35648b3fb4 100644 --- a/openslides_backend/action/actions/user/save_saml_account.py +++ b/openslides_backend/action/actions/user/save_saml_account.py @@ -1,4 +1,6 @@ -from collections.abc import Iterable +import re +from collections import defaultdict +from collections.abc import Generator, Iterable from typing import Any, cast import fastjsonschema @@ -7,7 +9,7 @@ from ....models.models import User from ....shared.exceptions import ActionException -from ....shared.filters import And, FilterOperator +from ....shared.filters import And, FilterOperator, Or from ....shared.interfaces.event import Event from ....shared.schema import schema_version from ....shared.typing import Schema @@ -18,6 +20,7 @@ from ...util.register import register_action from ...util.typing import ActionData, ActionResultElement from ..gender.create import GenderCreate +from ..structure_level.create import StructureLevelCreateAction from .create import UserCreate from .update import UserUpdate from .user_mixins import UsernameMixin @@ -32,6 +35,16 @@ "pronoun", "is_active", "is_physical_person", + "member_number", +] + +allowed_meeting_user_fields = [ + "groups", + "structure_levels", + "number", + "comment", + "vote_weight", + "present", ] @@ -63,7 +76,9 @@ def validate_instance(self, instance: dict[str, Any]) -> None: raise ActionException( "SingleSignOn is not enabled in OpenSlides configuration" ) - self.saml_attr_mapping = organization.get("saml_attr_mapping", {}) + self.saml_attr_mapping: dict[str, Any] = organization.get( + "saml_attr_mapping", dict() + ) if not self.saml_attr_mapping or not isinstance(self.saml_attr_mapping, dict): raise ActionException( "SingleSignOn field attributes are not configured in OpenSlides" @@ -111,7 +126,10 @@ def validate_instance(self, instance: dict[str, Any]) -> None: def validate_fields(self, instance_old: dict[str, Any]) -> dict[str, Any]: """ - Transforms the payload fields into model fields, removes the possible array-wrapped format + Transforms the payload fields into model fields, removes the possible array-wrapped format. + Mapper data is comprised on a per meeting basis. On conflicts the last statement is used. + Groups and structure levels are combined, however. + Meeting related data will be transformed via the idp attributes to the actual model data. """ instance: dict[str, Any] = dict() for model_field, payload_field in self.saml_attr_mapping.items(): @@ -120,14 +138,14 @@ def validate_fields(self, instance_old: dict[str, Any]) -> dict[str, Any]: and payload_field in instance_old and model_field in allowed_user_fields ): - value = ( + idp_attribute = ( tx[0] if isinstance((tx := instance_old[payload_field]), list) and len(tx) else tx ) - if value not in (None, []): - instance[model_field] = value - + if idp_attribute not in (None, []): + instance[model_field] = idp_attribute + self.apply_meeting_mapping(instance, instance_old) return super().validate_fields(instance) def prepare_action_data(self, action_data: ActionData) -> ActionData: @@ -138,49 +156,49 @@ def check_permissions(self, instance: dict[str, Any]) -> None: pass def base_update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: - meeting_id, group_id = self.check_for_group_add() users = self.datastore.filter( "user", FilterOperator("saml_id", "=", instance["saml_id"]), - ["id", "gender_id", *allowed_user_fields], + [ + "id", + "meeting_user_ids", + "is_present_in_meeting_ids", + "gender_id", + *allowed_user_fields, + ], ) - - if gender := instance.get("gender"): - if gender == "": - instance["gender_id"] = None + if gender := instance.pop("gender", None): + gender_dict = self.datastore.filter( + "gender", + FilterOperator("name", "=", gender), + ["id"], + ) + gender_id = None + if gender_dict: + gender_id = next(iter(gender_dict.keys())) else: - gender_dict = self.datastore.filter( - "gender", - FilterOperator("name", "=", gender), - ["id"], + action_result = self.execute_other_action( + GenderCreate, [{"name": gender}] ) - if gender_dict: - gender_id = next(iter(gender_dict.keys())) - else: - action_result = self.execute_other_action( - GenderCreate, [{"name": gender}] - ) - gender_id = action_result[0].get("id", 0) # type: ignore + if action_result and action_result[0]: + gender_id = action_result[0].get("id", 0) + if gender_id: instance["gender_id"] = gender_id - del instance["gender"] + else: + self.logger.warning( + f"save_saml_account could neither find nor create {gender}. Not handling gender." + ) + # Empty string: remove gender_id elif gender == "": instance["gender_id"] = None - del instance["gender"] + meeting_users: dict[int, dict[str, Any]] | None = dict() + user_id = None if len(users) == 1: self.user = next(iter(users.values())) instance["id"] = (user_id := cast(int, self.user["id"])) - if meeting_id and group_id: - meeting_user = get_meeting_user( - self.datastore, meeting_id, user_id, ["id", "group_ids"] - ) - if meeting_user: - old_group_ids = meeting_user["group_ids"] - if group_id not in old_group_ids: - instance["meeting_id"] = meeting_id - instance["group_ids"] = old_group_ids + [group_id] - else: - instance["meeting_id"] = meeting_id - instance["group_ids"] = [group_id] + meeting_users = self.apply_meeting_user_data(instance, user_id, True) + if meeting_users: + self.update_meeting_users_from_db(meeting_users, user_id) instance = { k: v for k, v in instance.items() if k == "id" or v != self.user.get(k) } @@ -188,19 +206,24 @@ def base_update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: self.execute_other_action(UserUpdate, [instance]) elif len(users) == 0: instance = self.set_defaults(instance) - if group_id: - instance["meeting_id"] = meeting_id - instance["group_ids"] = [group_id] - self.execute_other_action(UserCreate, [instance]) + meeting_users = instance.pop("meeting_user_data", None) + response = self.execute_other_action(UserCreate, [instance]) + if response and response[0]: + user_id = response[0].get("id") + instance["meeting_user_data"] = meeting_users + if user_id: + meeting_users = self.apply_meeting_user_data(instance, user_id, False) else: ActionException( f"More than one existing user found in database with saml_id {instance['saml_id']}" ) + if meeting_users: + self.execute_other_action(UserUpdate, [mu for mu in meeting_users.values()]) return instance def create_events(self, instance: dict[str, Any]) -> Iterable[Event]: """ - delegated to execute_other_actions + delegated to execute_other_action """ return [] @@ -218,46 +241,312 @@ def set_defaults(self, instance: dict[str, Any]) -> dict[str, Any]: instance["username"] = self.generate_usernames([instance.get("saml_id", "")])[0] return instance - def check_for_group_add(self) -> tuple[int, int] | tuple[None, None]: - NoneResult = (None, None) - if not ( - meeting_info := cast(dict, self.saml_attr_mapping.get("meeting")) - ) or not (external_id := meeting_info.get("external_id")): - return NoneResult - - meetings = self.datastore.filter( - collection="meeting", - filter=FilterOperator("external_id", "=", external_id), - mapped_fields=["id", "default_group_id"], + def validate_meeting_mapper( + self, instance: dict[str, Any], meeting_mapper: dict[str, Any] + ) -> bool: + """ + Validates the meeting mapper to be complete. Returns False if not. + Returns True if the mapper matches its criteria on instances values or no conditions were given. + Instances values can not be None or empty string. + """ + if not meeting_mapper.get("external_id"): + return False + if not (mapper_conditions := meeting_mapper.get("conditions")): + return True + return all( + ( + (instance_value := instance.get(mapper_condition.get("attribute"))) + and regex_condition.search(instance_value) + ) + for mapper_condition in mapper_conditions + if (regex_condition := re.compile(mapper_condition.get("condition"))) ) - if len(meetings) == 1: - meeting = next(iter(meetings.values())) - group_id = meeting["default_group_id"] - else: + + def apply_meeting_mapping( + self, instance: dict[str, Any], instance_old: dict[str, Any] + ) -> None: + if meeting_mappers := cast( + list[dict[str, Any]], + self.saml_attr_mapping.get("meeting_mappers", []), + ): + meeting_user_data: dict[str, Any] = defaultdict(dict) + for meeting_mapper in meeting_mappers: + if self.validate_meeting_mapper(instance_old, meeting_mapper): + meeting_external_id = cast(str, meeting_mapper["external_id"]) + mapping_results = meeting_user_data[meeting_external_id] + allow_update: str | bool + if isinstance( + allow_update := cast( + str, meeting_mapper.get("allow_update", "True") + ), + str, + ): + allow_update = allow_update.casefold() != "False".casefold() + result = { + **{ + key: value + for key, value in self.get_field_data( + instance_old, + mapping_results.get("for_create", dict()), + meeting_mapper, + ) + }, + } + if allow_update: + mapping_results["for_create"] = result + mapping_results["for_update"] = { + **{ + key: value + for key, value in self.get_field_data( + instance_old, + mapping_results.get("for_update", dict()), + meeting_mapper, + ) + }, + } + else: + mapping_results["for_create"] = result + if meeting_user_data: + instance["meeting_user_data"] = meeting_user_data + else: + self.logger.warning( + "save_saml_account found no matching meeting mappers." + ) + + def apply_meeting_user_data( + self, instance: dict[str, Any], user_id: int, is_update: bool + ) -> dict[int, dict[str, Any]] | None: + if not (meeting_user_data := instance.pop("meeting_user_data", None)) or not ( + external_meeting_ids := sorted( + [ext_id for ext_id in meeting_user_data.keys()] + ) + ): + return None + meetings = { + meeting_id: meeting + for meeting_id, meeting in sorted( + self.datastore.filter( + "meeting", + Or( + FilterOperator("external_id", "=", external_meeting_id) + for external_meeting_id in external_meeting_ids + ), + ["id", "default_group_id", "external_id"], + ).items() + ) + } + missing_meetings = [ + external_meeting_id + for external_meeting_id in external_meeting_ids + if external_meeting_id + not in {meeting.get("external_id") for meeting in meetings.values()} + ] + if missing_meetings: self.logger.warning( - f"save_saml_account found {len(meetings)} meetings with external_id '{external_id}'" + f"save_saml_account found no meetings for {len(missing_meetings)} meetings with external_ids {missing_meetings}" + ) + # declare and half way through initialize mu data + result: dict[int, dict[str, Any]] = dict() + for ( + meeting_id, + meeting, + ) in meetings.items(): + if not ( + instance_meeting_user_data := meeting_user_data.get( + meeting["external_id"] + ) + ): + continue + if is_update: + instance_meeting_user = instance_meeting_user_data.get("for_update") + else: + instance_meeting_user = instance_meeting_user_data.get("for_create") + if instance_meeting_user is not None: + instance_meeting_user["id"] = user_id + instance_meeting_user["meeting_id"] = meeting_id + for saml_meeting_user_field in ["groups", "structure_levels"]: + names = sorted( + instance_meeting_user.pop(saml_meeting_user_field, []) + ) + if saml_meeting_user_field == "groups": + ids = self.get_group_ids(names, meeting) + elif saml_meeting_user_field == "structure_levels": + ids = self.get_structure_level_ids(names, meeting) + if ids: + instance_meeting_user[ + f"{saml_meeting_user_field.rstrip('s')}_ids" + ] = ids + if instance_meeting_user.pop("present", ""): + present_in_meeting_ids = instance.get( + "is_present_in_meeting_ids", [] + ) + if meeting_id not in present_in_meeting_ids: + present_in_meeting_ids.append(meeting_id) + instance["is_present_in_meeting_ids"] = present_in_meeting_ids + result[meeting_id] = instance_meeting_user + return result + + def update_meeting_users_from_db( + self, meeting_users: dict[int, dict[str, Any]], user_id: int + ) -> None: + """updates meeting users with groups and structure level relations from database""" + for meeting_id, meeting_user in meeting_users.items(): + if meeting_user_db := get_meeting_user( + self.datastore, + meeting_id, + user_id, + ["id", "group_ids", "structure_level_ids"], + ): + for field_name in ["group_ids", "structure_level_ids"]: + if old_ids := meeting_user_db.get(field_name): + ids = meeting_user.get(field_name, []) + for _id in ids: + if _id not in old_ids: + meeting_user[field_name] = old_ids + [_id] + + def get_field_data( + self, + instance: dict[str, Any], + meeting_user: dict[str, Any], + meeting_mapper: dict[str, dict[str, Any]], + ) -> Generator[tuple[str, Any]]: + """ + returns the field data for the given idp mapping field. Groups the groups and structure levels for each meeting. + Uses mappers for generating default values. + """ + missing_attributes = [] + for saml_meeting_user_field in allowed_meeting_user_fields: + result: set[str] | str | bool = "" + meeting_mapping = meeting_mapper.get("mappings", dict()) + result = meeting_user.get(saml_meeting_user_field, "") + if saml_meeting_user_field in ["groups", "structure_levels"]: + attr_default_list = meeting_mapping.get(saml_meeting_user_field, []) + else: + attr_default_list = [ + meeting_mapping.get(saml_meeting_user_field, dict()) + ] + for attr_default in attr_default_list: + idp_attribute = attr_default.get("attribute", "") + if saml_meeting_user_field == "number": + # Number cannot have a default. + if value := instance.get(idp_attribute): + result = cast(str, value) + else: + missing_attributes.append(idp_attribute) + elif not (value := instance.get(idp_attribute)): + missing_attributes.append(idp_attribute) + value = attr_default.get("default") + if value: + if saml_meeting_user_field in ["groups", "structure_levels"]: + # Need to append to group and structure_level for same meeting. + if not result: + result = set() + cast(set, result).update(value.split(", ")) + elif saml_meeting_user_field == "comment": + # Want comments from all matching mappers. + if result: + result = cast(str, result) + " " + value + else: + result = value + elif saml_meeting_user_field == "present": + # Result is int or bool. int will later be interpreted as bool. + result = ( + value + if not isinstance(value, str) + else ( + False + if value.casefold() == "false".casefold() + else True + ) + ) + else: + result = value + if result: + yield saml_meeting_user_field, result + if fields := ",".join(missing_attributes): + mapper_name = meeting_mapper.get("name", "unnamed") + self.logger.debug( + f"Meeting mapper: {mapper_name} could not find value in idp data for fields: {fields}. Using default if available." ) - return NoneResult - if external_group_id := meeting_info.get("external_group_id"): + + def get_group_ids(self, group_names: list[str], meeting: dict) -> list[int]: + """ + Gets the group ids from given group names in that meeting. + If none of the groups exists in the meeting, the meetings default group is returned. + """ + if group_names: groups = self.datastore.filter( - collection="group", - filter=And( - [ - FilterOperator("external_id", "=", external_group_id), - FilterOperator("meeting_id", "=", meeting.get("id")), - ] + "group", + And( + FilterOperator("meeting_id", "=", meeting["id"]), + Or( + FilterOperator("external_id", "=", group_name) + for group_name in group_names + ), ), - mapped_fields=["id"], + ["meeting_user_ids"], ) - if len(groups) == 1: - group_id = next(iter(groups.keys())) - else: - self.logger.warning( - f"save_saml_account found no group in meeting '{external_id}' for '{external_group_id}', but use default_group of meeting" - ) - if not group_id: + if len(groups) > 0: + return sorted(groups) + if default_group_id := meeting["default_group_id"]: + external_meeting_id = meeting["external_id"] self.logger.warning( - f"save_saml_account found no group in meeting '{external_id}' for '{external_group_id}'" + f"save_saml_account found no group in meeting '{external_meeting_id}' for {group_names}, but used default_group of meeting" ) - return NoneResult - return meeting.get("id"), group_id + return [default_group_id] + else: + assert False + + def get_structure_level_ids( + self, structure_level_names: list[str], meeting: dict[str, Any] + ) -> list[int]: + """ + Gets the structure level ids from given structure level names in that meeting. + For this also creates new structure levels not already existing in the meeting. + """ + if structure_level_names: + meeting_id = meeting["id"] + found_structure_levels = self.datastore.filter( + "structure_level", + And( + FilterOperator("meeting_id", "=", meeting_id), + Or( + FilterOperator("name", "=", structure_level_name) + for structure_level_name in structure_level_names + if structure_level_name + ), + ), + ["meeting_user_ids"], + ) + found_structure_level_ids = list(found_structure_levels.keys()) + if len(found_structure_levels) == len(structure_level_names): + return found_structure_level_ids + else: + found_structure_level_names = [ + structure_level.get("name") + for structure_level in found_structure_levels.values() + ] + to_be_created_structure_levels = [ + sl_name + for sl_name in structure_level_names + if sl_name and sl_name not in found_structure_level_names + ] + # meeting_user_ids are only known during UserUpdate. Hence we cannot do batch create for all meeting users + if structure_levels_result := ( + self.execute_other_action( + StructureLevelCreateAction, + [ + {"name": structure_level_name, "meeting_id": meeting_id} + for structure_level_name in to_be_created_structure_levels + ], + ) + ): + return sorted( + [ + structure_level["id"] + for structure_level in structure_levels_result + if structure_level + ] + + found_structure_level_ids + ) + return [] diff --git a/tests/system/action/organization/test_update.py b/tests/system/action/organization/test_update.py index 182db03a65..d13555d9a6 100644 --- a/tests/system/action/organization/test_update.py +++ b/tests/system/action/organization/test_update.py @@ -6,7 +6,11 @@ class OrganizationUpdateActionTest(BaseActionTestCase): - saml_attr_mapping: dict[str, str | dict[str, str]] = { + ListOfDicts = list[dict[str, str]] + MeetingMappers = list[ + dict[str, str | ListOfDicts | dict[str, str | dict[str, str] | ListOfDicts]] + ] + saml_attr_mapping: dict[str, str | MeetingMappers] = { "saml_id": "username", "title": "title", "first_name": "firstName", @@ -16,6 +20,7 @@ class OrganizationUpdateActionTest(BaseActionTestCase): "pronoun": "pronoun", "is_active": "is_active", "is_physical_person": "is_person", + "member_number": "member_number", } def setUp(self) -> None: @@ -64,7 +69,46 @@ def test_update(self) -> None: def test_update_with_meeting(self) -> None: self.saml_attr_mapping.update( - {"meeting": {"external_id": "Landtag", "external_group_id": "Delegated"}} + { + "meeting_mappers": [ + { + "name": "Mapper-Name", + "external_id": "Landtag", + "allow_update": "false", + "conditions": [ + {"attribute": "membernumber", "condition": r"1426\d{4,6}$"}, + {"attribute": "function", "condition": "board"}, + ], + "mappings": { + "groups": [ + { + "attribute": "membership", + "default": "admin, standard", + } + ], + "structure_levels": [ + { + "attribute": "ovname", + "default": "struct1, struct2", + } + ], + "number": {"attribute": "p_number"}, + "comment": { + "attribute": "idp_comment", + "default": "Group set via SSO", + }, + "vote_weight": { + "attribute": "vote", + "default": "1.000000", + }, + "present": { + "attribute": "present_key", + "default": "True", + }, + }, + } + ] + } ), response = self.request( "organization.update", @@ -85,9 +129,28 @@ def test_update_with_meeting(self) -> None: }, ) - def test_update_with_meeting_error(self) -> None: + def test_update_with_meeting_missing_ext_id(self) -> None: + self.saml_attr_mapping.update( + {"meeting_mappers": [{"external_idx": "Landtag"}]} + ), + response = self.request( + "organization.update", + { + "id": 1, + "name": "testtest", + "description": "blablabla", + "saml_attr_mapping": self.saml_attr_mapping, + }, + ) + self.assert_status_code(response, 400) + assert ( + "data.saml_attr_mapping.meeting_mappers[0] must contain ['external_id'] properties" + in response.json["message"] + ) + + def test_update_with_meeting_wrong_attr(self) -> None: self.saml_attr_mapping.update( - {"meeting": {"external_idx": "Landtag", "external_group_id": "Delegated"}} + {"meeting_mappers": [{"external_id": "Landtag", "unkown_field": " "}]} ), response = self.request( "organization.update", @@ -100,7 +163,7 @@ def test_update_with_meeting_error(self) -> None: ) self.assert_status_code(response, 400) assert ( - "data.saml_attr_mapping.meeting must not contain {'external_idx'} properties" + "data.saml_attr_mapping.meeting_mappers[0] must not contain {'unkown_field'} properties" in response.json["message"] ) diff --git a/tests/system/action/speaker/test_create.py b/tests/system/action/speaker/test_create.py index 93000e44d4..c5a04b445f 100644 --- a/tests/system/action/speaker/test_create.py +++ b/tests/system/action/speaker/test_create.py @@ -353,7 +353,7 @@ def test_create_user_not_present(self) -> None: self.assert_status_code(response, 400) self.assert_model_not_exists("speaker/1") self.assertIn( - "Only present users can be on the lists of speakers.", + "Only present users can be on the list of speakers.", response.json["message"], ) diff --git a/tests/system/action/user/test_create.py b/tests/system/action/user/test_create.py index 96e76bf4d3..dfcc537b8f 100644 --- a/tests/system/action/user/test_create.py +++ b/tests/system/action/user/test_create.py @@ -140,7 +140,7 @@ def test_create_some_more_fields(self) -> None: ) self.assert_history_information( "user/2", - ["Account created", "Participant added to meeting {}", "meeting/111"], + ["Account created", "Participant added to meeting {}.", "meeting/111"], ) def test_create_comment(self) -> None: diff --git a/tests/system/action/user/test_save_saml_account.py b/tests/system/action/user/test_save_saml_account.py index 795f948812..4374924d00 100644 --- a/tests/system/action/user/test_save_saml_account.py +++ b/tests/system/action/user/test_save_saml_account.py @@ -500,6 +500,7 @@ def setUp(self) -> None: self.organization = { "saml_enabled": True, "saml_attr_mapping": { + "member_number": "member_number", "saml_id": "username", "title": "title", "first_name": "firstName", @@ -509,130 +510,575 @@ def setUp(self) -> None: "pronoun": "pronoun", "is_active": "is_active", "is_physical_person": "is_person", - "meeting": { - "external_id": "Landtag", - "external_group_id": "Delegates", - }, }, } + self.meeting_mappers = [ + { + "name": "works", + "external_id": "Landtag", + "conditions": [ + {"attribute": "member_number", "condition": "LV_.*"}, + { + "attribute": "email", + "condition": "[\\w\\.]+@([\\w-]+\\.)+[\\w]{2,4}", + }, + ], + "mappings": { + "comment": { + "attribute": "idp_commentary", + "default": "Vote weight, groups and structure levels set via SSO.", + }, + "number": {"attribute": "participant_number"}, + "structure_levels": [ + { + "attribute": "structure", + "default": "structure1", + } + ], + "groups": [ + { + "attribute": "idp_group_attribute", + "default": "not_a_group", + } + ], + "vote_weight": {"attribute": "vw", "default": "1.000000"}, + "present": {"attribute": "presence", "default": "True"}, + }, + }, + { + "name": "works_too", + "external_id": "Kreistag", + "conditions": [{"attribute": "kv_member_number", "condition": "KV_.*"}], + "mappings": { + "comment": { + "attribute": "idp_commentary", + "default": "Vote weight, groups and structure levels set via SSO.", + }, + "number": {"attribute": "participant_kv_number"}, + "structure_levels": [ + { + "attribute": "kv_structure", + "default": "structure1", + } + ], + "groups": [ + { + "attribute": "kv_group_attribute", + "default": "not_a_group", + } + ], + "vote_weight": { + "attribute": "kv_vw", + "default": "1.000000", + }, + "present": {"attribute": "kv_presence", "default": "True"}, + }, + }, + ] + self.organization["saml_attr_mapping"]["meeting_mappers"] = self.meeting_mappers # type: ignore self.create_meeting() + self.create_meeting(4) self.set_models( { "organization/1": self.organization, "group/1": {"external_id": "Default"}, "group/2": {"external_id": "Delegates"}, "group/3": {"external_id": "Admin"}, + "group/4": {"external_id": "Default"}, + "group/5": {"external_id": "Delegates"}, + "group/6": {"external_id": "Admin"}, "meeting/1": {"external_id": "Landtag", "default_group_id": 1}, + "meeting/4": {"external_id": "Kreistag", "default_group_id": 4}, "user/1": {"saml_id": "admin_saml"}, } ) - def test_create_user_with_membership(self) -> None: - response = self.request("user.save_saml_account", {"username": ["111"]}) + def test_create_user_with_multi_membership(self) -> None: + """ + Shows: + * generally works for multiple matching mappers on different meetings + * if default for 'groups' doesn't resolve, default group of that meeting is used + """ + response = self.request( + "user.save_saml_account", + { + "username": ["111"], + "member_number": "LV_Königholz", + "email": "holzi@holz.de", + "participant_number": "MG_1254", + "idp_group_attribute": "Delegates", + "kv_member_number": "KV_Könighols", + "kv_email": "hols@holz.de", + "participant_kv_number": "MG_1254", + "idp_kv_group_attribute": "Delegates", + "kv_structure": "structure2", + "idp_commentary": "normal data used", + }, + ) self.assert_status_code(response, 200) + self.app.logger.warning.assert_called_with( # type: ignore + "save_saml_account found no group in meeting 'Kreistag' for ['not_a_group'], but used default_group of meeting" + ) self.assert_model_exists( "user/2", { "saml_id": "111", "username": "111", - "meeting_user_ids": [1], - "meeting_ids": [1], + "email": "holzi@holz.de", + "meeting_user_ids": [1, 2], + "meeting_ids": [1, 4], + "is_present_in_meeting_ids": [1, 4], }, ) self.assert_model_exists( - "meeting_user/1", {"user_id": 2, "group_ids": [2], "meeting_id": 1} + "meeting_user/1", + { + "user_id": 2, + "meeting_id": 1, + "group_ids": [2], + "structure_level_ids": [1], + "vote_weight": "1.000000", + "number": "MG_1254", + "comment": "normal data used", + }, + ) + self.assert_model_exists( + "meeting_user/2", + { + "user_id": 2, + "group_ids": [4], + "meeting_id": 4, + "comment": "normal data used", + "structure_level_ids": [2], + "vote_weight": "1.000000", + "number": "MG_1254", + }, ) self.assert_model_exists( "group/2", {"meeting_user_ids": [1], "external_id": "Delegates"} ) + self.assert_model_exists( + "structure_level/1", + {"meeting_user_ids": [1], "name": "structure1"}, + ) + self.assert_model_exists( + "structure_level/2", + {"meeting_user_ids": [2], "name": "structure2"}, + ) - def test_update_user_with_membership(self) -> None: - response = self.request("user.save_saml_account", {"username": ["admin_saml"]}) + def test_create_user_with_multi_membership_multi(self) -> None: + """ + Shows: + * group and structure levels can be multiple values in concatenated, comma separated list string for: + * default values + * saml datas values + * multiple entries in structure level and group lists are respected. + * multiple values can be repeated + * mappers can share idp data fields + """ + self.meeting_mappers[1]["mappings"]["groups"] = [ # type: ignore + { + "attribute": "use_default", + "default": "Default, Delegates, Delegates", + }, + {"attribute": "idp_group_attribute", "default": ""}, + ] + self.set_models({"organization/1": self.organization}) + response = self.request( + "user.save_saml_account", + { + "username": ["111"], + "member_number": "LV_Königholz", + "email": "holzi@holz.de", + "participant_number": "MG_1254", + "idp_group_attribute": "Delegates, Admin, Delegates", + "kv_member_number": "KV_Könighols", + "participant_kv_number": "MG_1254", + "idp_kv_group_attribute": "Delegates", + "kv_structure": "structure2", + }, + ) self.assert_status_code(response, 200) self.assert_model_exists( - "user/1", + "user/2", { - "saml_id": "admin_saml", - "username": "admin", + "saml_id": "111", + "username": "111", + "meeting_user_ids": [1, 2], + "meeting_ids": [1, 4], + }, + ) + self.assert_model_exists( + "meeting_user/1", {"user_id": 2, "group_ids": [2, 3], "meeting_id": 1} + ) + self.assert_model_exists( + "meeting_user/2", {"user_id": 2, "group_ids": [4, 5, 6], "meeting_id": 4} + ) + self.assert_model_exists( + "group/2", {"meeting_user_ids": [1], "external_id": "Delegates"} + ) + self.assert_model_exists( + "structure_level/1", + {"meeting_user_ids": [1], "name": "structure1"}, + ) + self.assert_model_exists( + "structure_level/2", + {"meeting_user_ids": [2], "name": "structure2"}, + ) + + def test_create_user_with_multi_membership_not_matching(self) -> None: + """ + shows: + * matching only one mapper -> creating only one meeting user + """ + response = self.request( + "user.save_saml_account", + { + "username": ["111"], + "member_number": "LV_Königholz", + "email": "holzi@holz.de", + "participant_number": "MG_1254", + "idp_group_attribute": "Delegates", + "kv_member_number": "LV_Königholz", + "kv_email": "hols@holz.de", + "participant_kv_number": "MG_1254", + "idp_kv_group_attribute": "Delegates", + "kv_structure": "structure2", + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/2", + { + "saml_id": "111", + "username": "111", "meeting_user_ids": [1], "meeting_ids": [1], }, ) self.assert_model_exists( - "meeting_user/1", {"user_id": 1, "group_ids": [2], "meeting_id": 1} + "meeting_user/1", {"user_id": 2, "group_ids": [2], "meeting_id": 1} ) self.assert_model_exists( "group/2", {"meeting_user_ids": [1], "external_id": "Delegates"} ) + self.assert_model_exists( + "structure_level/1", + {"meeting_user_ids": [1], "name": "structure1"}, + ) + self.assert_model_not_exists("structure_level/2") - def test_create_user_invalid_meeting(self) -> None: - """silent fail, user created and logged in""" - self.organization["saml_attr_mapping"]["meeting"]["external_id"] = "Kreistag" # type: ignore + def test_create_user_mapping_no_mapper(self) -> None: + del self.organization["saml_attr_mapping"]["meeting_mappers"] # type: ignore self.set_models({"organization/1": self.organization}) - response = self.request("user.save_saml_account", {"username": ["111"]}) - self.assert_status_code(response, 200) - self.app.logger.warning.assert_called_with( # type: ignore - "save_saml_account found 0 meetings with external_id 'Kreistag'" + response = self.request( + "user.save_saml_account", + { + "username": ["111"], + "member_number": "LV_Königholz", + "email": "holzi@holz.de", + "participant_number": "MG_1254", + "idp_group_attribute": "Delegates", + "kv_member_number": "KV_Könighols", + "kv_email": "hols@holz.de", + "participant_kv_number": "MG_1254", + "idp_kv_group_attribute": "Delegates", + "kv_structure": "structure2", + "kv_presence": "True", + }, ) + self.assert_status_code(response, 200) self.assert_model_exists( - "user/2", {"saml_id": "111", "username": "111", "meeting_user_ids": None} + "user/2", + { + "saml_id": "111", + "username": "111", + }, ) self.assert_model_not_exists("meeting_user/1") + self.assert_model_not_exists("structure_level/1") + + def test_create_user_mapping_no_mapping(self) -> None: + del self.meeting_mappers[0]["mappings"] + del self.meeting_mappers[1] + self.set_models({"organization/1": self.organization}) + response = self.request( + "user.save_saml_account", + { + "username": ["111"], + "member_number": "LV_Königholz", + "email": "holzi@holz.de", + "participant_number": "MG_1254", + "idp_group_attribute": "Delegates", + "kv_member_number": "KV_Könighols", + "kv_email": "hols@holz.de", + "participant_kv_number": "MG_1254", + "idp_kv_group_attribute": "Delegates", + "kv_structure": "structure2", + "kv_presence": "True", + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/2", + { + "saml_id": "111", + "username": "111", + }, + ) self.assert_model_exists( - "group/2", {"meeting_user_ids": None, "external_id": "Delegates"} + "meeting_user/1", {"user_id": 2, "group_ids": [1], "meeting_id": 1} ) + self.assert_model_exists( + "group/1", {"meeting_user_ids": [1], "external_id": "Default"} + ) + self.assert_model_not_exists("structure_level/1") - def test_create_user_invalid_group_but_default(self) -> None: - """silent fail, but added to default group and logged in""" - self.organization["saml_attr_mapping"]["meeting"][ # type: ignore - "external_group_id" - ] = "Developers" + def test_create_user_mapping_empty_mappings(self) -> None: + self.meeting_mappers[0]["mappings"] = dict() + del self.meeting_mappers[1] self.set_models({"organization/1": self.organization}) - response = self.request("user.save_saml_account", {"username": ["111"]}) - self.assert_status_code(response, 200) - self.app.logger.warning.assert_called_with( # type: ignore - "save_saml_account found no group in meeting 'Landtag' for 'Developers', but use default_group of meeting" + response = self.request( + "user.save_saml_account", + { + "username": ["111"], + "member_number": "LV_Königholz", + "email": "holzi@holz.de", + "participant_number": "MG_1254", + "idp_group_attribute": "Delegates", + "kv_member_number": "KV_Könighols", + "kv_email": "hols@holz.de", + "participant_kv_number": "MG_1254", + "idp_kv_group_attribute": "Delegates", + "kv_structure": "structure2", + "kv_presence": "True", + }, ) + self.assert_status_code(response, 200) self.assert_model_exists( - "user/2", {"saml_id": "111", "meeting_user_ids": [1], "meeting_ids": [1]} + "user/2", + { + "saml_id": "111", + "username": "111", + }, ) self.assert_model_exists( "meeting_user/1", {"user_id": 2, "group_ids": [1], "meeting_id": 1} ) self.assert_model_exists( - "group/1", + "group/1", {"meeting_user_ids": [1], "external_id": "Default"} + ) + self.assert_model_not_exists("structure_level/1") + + def test_create_user_meeting_not_exists(self) -> None: + """ + Shows: if meeting does not exist error is logged. + """ + self.meeting_mappers[0]["external_id"] = "Bundestag" + del self.meeting_mappers[1] + self.set_models({"organization/1": self.organization}) + response = self.request( + "user.save_saml_account", { - "meeting_user_ids": [1], - "external_id": "Default", - "default_group_for_meeting_id": 1, + "username": ["111"], + "member_number": "LV_Königholz", + "email": "holzi@holz.de", + "participant_number": "MG_1254", + "idp_group_attribute": "Delegates", + "kv_member_number": "KV_Könighols", + "kv_email": "hols@holz.de", + "participant_kv_number": "MG_1254", + "kv_group_attribute": "Delegates", + "kv_structure": "structure2", + }, + ) + self.assert_status_code(response, 200) + self.app.logger.warning.assert_called_with( # type: ignore + "save_saml_account found no meetings for 1 meetings with external_ids ['Bundestag']" + ) + self.assert_model_exists( + "user/2", + { + "saml_id": "111", + "username": "111", + "meeting_user_ids": None, + "meeting_ids": None, }, ) + self.assert_model_not_exists("meeting_user/1") + self.assert_model_not_exists("structure_level/1") - def test_create_user_only_meeting_given(self) -> None: - """silent fail, but added to default group and logged in""" - del self.organization["saml_attr_mapping"]["meeting"]["external_group_id"] # type: ignore - self.set_models({"organization/1": self.organization}) - response = self.request("user.save_saml_account", {"username": ["111"]}) + def test_create_user_only_one_sl_exists(self) -> None: + """Shows: no errors if one structure level exists and the other doesn't. Latter being created.""" + self.create_model("structure_level/1", {"name": "structure1", "meeting_id": 1}) + response = self.request( + "user.save_saml_account", + { + "username": ["111"], + "member_number": "LV_Königholz", + "email": "holzi@holz.de", + "participant_number": "MG_1254", + "idp_group_attribute": "Delegates", + "kv_member_number": "KV_Könighols", + "kv_email": "hols@holz.de", + "participant_kv_number": "MG_1254", + "idp_kv_group_attribute": "Delegates", + "kv_structure": "structure2", + }, + ) self.assert_status_code(response, 200) self.assert_model_exists( - "user/2", {"saml_id": "111", "meeting_user_ids": [1], "meeting_ids": [1]} + "user/2", + { + "saml_id": "111", + "username": "111", + "meeting_user_ids": [1, 2], + "meeting_ids": [1, 4], + }, + ) + self.assert_model_exists( + "meeting_user/1", {"user_id": 2, "group_ids": [2], "meeting_id": 1} ) self.assert_model_exists( - "meeting_user/1", {"user_id": 2, "group_ids": [1], "meeting_id": 1} + "group/2", {"meeting_user_ids": [1], "external_id": "Delegates"} + ) + self.assert_model_exists( + "structure_level/1", + {"meeting_user_ids": [1], "name": "structure1"}, + ) + self.assert_model_exists( + "structure_level/2", + {"meeting_user_ids": [2], "name": "structure2"}, + ) + + def test_create_user_mapping_one_meeting_twice(self) -> None: + self.meeting_mappers[1]["external_id"] = "Landtag" + self.meeting_mappers[1]["mappings"]["groups"] = [ # type: ignore + { + "attribute": "use_default", + "default": "Default", + } + ] + self.set_models({"organization/1": self.organization}) + response = self.request( + "user.save_saml_account", + { + "username": ["111"], + "member_number": "LV_Königholz", + "email": "holzi@holz.de", + "participant_number": "MG_1254", + "idp_group_attribute": "Delegates", + "kv_member_number": "KV_Könighols", + "kv_email": "hols@holz.de", + "participant_kv_number": "MG_1254", + "idp_kv_group_attribute": "Delegates", + "kv_structure": "structure2", + "kv_presence": "True", + }, ) + self.assert_status_code(response, 200) self.assert_model_exists( - "group/1", + "user/2", { + "saml_id": "111", + "username": "111", "meeting_user_ids": [1], - "external_id": "Default", - "default_group_for_meeting_id": 1, + "meeting_ids": [1], }, ) + self.assert_model_exists( + "meeting_user/1", {"user_id": 2, "group_ids": [1, 2], "meeting_id": 1} + ) + self.assert_model_exists( + "group/1", {"meeting_user_ids": [1], "external_id": "Default"} + ) + self.assert_model_exists( + "group/2", {"meeting_user_ids": [1], "external_id": "Delegates"} + ) + self.assert_model_exists( + "structure_level/1", + {"meeting_user_ids": [1], "name": "structure1"}, + ) + self.assert_model_exists( + "structure_level/2", + {"meeting_user_ids": [1], "name": "structure2"}, + ) - def test_update_user_existing_member_in_group(self) -> None: - """user created and logged in""" + def test_update_user_with_default_membership(self) -> None: + """ + Shows: + * deleting all conditions of a single mapper defaults this mapper to true + * updating without any group data in saml payload and not existing group in default inserts user in meetings default group + * updating without group data in saml payload but existing group in default works + """ + del self.meeting_mappers[0]["conditions"] + self.meeting_mappers[1]["conditions"] = [ + {"attribute": "yes", "condition": ".*"} + ] + self.meeting_mappers[1]["mappings"]["groups"][0]["default"] = "Delegates" # type: ignore + self.set_models({"organization/1": self.organization}) + response = self.request( + "user.save_saml_account", {"username": ["admin_saml"], "yes": "to_all"} + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/1", + { + "saml_id": "admin_saml", + "username": "admin", + "meeting_user_ids": [1, 2], + "meeting_ids": [1, 4], + }, + ) + self.assert_model_exists( + "meeting_user/1", {"user_id": 1, "group_ids": [1], "meeting_id": 1} + ) + self.assert_model_exists( + "meeting_user/2", {"user_id": 1, "group_ids": [5], "meeting_id": 4} + ) + self.assert_model_exists( + "group/1", {"meeting_user_ids": [1], "external_id": "Default"} + ) + self.assert_model_exists( + "group/5", {"meeting_user_ids": [2], "external_id": "Delegates"} + ) + + def test_update_user_participant_already_in_group(self) -> None: + """ + Shows: + * user stays in group 2 + * structure_level/1 left untouched structure_level/2 added + * second mapper is ignored due to being an update and allow_update being false + """ + self.meeting_mappers[1]["allow_update"] = "False" + self.set_models({"organization/1": self.organization}) self.set_user_groups(1, [2]) - response = self.request("user.save_saml_account", {"username": ["admin_saml"]}) + self.set_models( + { + "structure_level/1": { + "name": "structure1", + "meeting_user_ids": [1], + "meeting_id": 1, + } + } + ) + self.update_model("meeting_user/1", {"structure_level_ids": [1]}) + response = self.request( + "user.save_saml_account", + { + "username": ["admin_saml"], + "member_number": "LV_Königholz", + "email": "holzi@holz.de", + "participant_number": "MG_1254", + "idp_group_attribute": "Delegates", + "kv_member_number": "KV_Könighols", + "kv_email": "hols@holz.de", + "participant_kv_number": "MG_1254", + "kv_group_attribute": "Delegates", + "kv_structure": "structure2", + "structure": "structure2", + }, + ) self.assert_status_code(response, 200) self.assert_model_exists( "user/1", @@ -644,29 +1090,57 @@ def test_update_user_existing_member_in_group(self) -> None: }, ) self.assert_model_exists( - "meeting_user/1", {"user_id": 1, "group_ids": [2], "meeting_id": 1} + "meeting_user/1", + { + "user_id": 1, + "group_ids": [2], + "meeting_id": 1, + "structure_level_ids": [1, 2], + }, ) + self.assert_model_not_exists("meeting_user/2") self.assert_model_exists( "group/2", {"meeting_user_ids": [1], "external_id": "Delegates"} ) + self.assert_model_exists( + "structure_level/1", {"name": "structure1", "meeting_user_ids": [1]} + ) + self.assert_model_exists( + "structure_level/2", {"name": "structure2", "meeting_user_ids": [1]} + ) + self.assert_model_not_exists("structure_level/3") def test_update_user_add_group_to_existing_groups(self) -> None: """group added, user created and logged in""" self.set_user_groups(1, [1, 3]) - response = self.request("user.save_saml_account", {"username": ["admin_saml"]}) + response = self.request( + "user.save_saml_account", + { + "username": ["admin_saml"], + "member_number": "LV_Königholz", + "email": "holzi@holz.de", + "idp_group_attribute": "Delegates", + "kv_member_number": "KV_Könighols", + "kv_group_attribute": "Delegates", + "kv_structure": "structure2", + }, + ) self.assert_status_code(response, 200) self.assert_model_exists( "user/1", { "saml_id": "admin_saml", "username": "admin", - "meeting_user_ids": [1], - "meeting_ids": [1], + "meeting_user_ids": [1, 2], + "meeting_ids": [1, 4], }, ) self.assert_model_exists( "meeting_user/1", {"user_id": 1, "group_ids": [1, 3, 2], "meeting_id": 1} ) + self.assert_model_exists( + "meeting_user/2", {"user_id": 1, "group_ids": [5], "meeting_id": 4} + ) self.assert_model_exists( "group/2", {"meeting_user_ids": [1], "external_id": "Delegates"} ) diff --git a/tests/system/action/user/test_update.py b/tests/system/action/user/test_update.py index 2e3ace2e9c..0b1b33e3a6 100644 --- a/tests/system/action/user/test_update.py +++ b/tests/system/action/user/test_update.py @@ -153,7 +153,7 @@ def test_update_with_meeting_user_fields(self) -> None: self.assert_history_information( "user/22", [ - "Participant added to meeting {}", + "Participant added to meeting {}.", "meeting/1", "Committee management changed", ], @@ -2354,7 +2354,7 @@ def test_update_participant_data_with_existing_meetings(self) -> None: self.assert_history_information( "user/222", [ - "Participant added to meeting {}", + "Participant added to meeting {}.", "meeting/2", ], ) @@ -2395,9 +2395,9 @@ def test_update_participant_data_in_multiple_meetings_with_existing_meetings( self.assert_history_information( "user/222", [ - "Participant added to meeting {}", + "Participant added to meeting {}.", "meeting/2", - "Participant added to meeting {}", + "Participant added to meeting {}.", "meeting/3", ], ) From c13b26f64d1e94ad2f1f3688b4130a08e76ca9b7 Mon Sep 17 00:00:00 2001 From: Hannes Janott Date: Mon, 25 Nov 2024 16:38:09 +0100 Subject: [PATCH 69/90] Allow meeting admin user to update a non admin user that shares all his meetings with requesting user. (#2576) * Allow meeting admin user to update a non admin user that shares all his meetings with requesting admin user. * Use user.can_update and user.can_manage. * Implement get_user_editable presenter with payload field names to support all payload field groups. --------- Co-authored-by: Elblinator <69210919+Elblinator@users.noreply.github.com> Co-authored-by: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> --- docs/Presenters-Overview.md | 1 + docs/actions/user.set_password.md | 1 - docs/presenters/get_user_editable.md | 31 ++ docs/presenters/get_user_related_models.md | 4 +- docs/presenters/get_user_scope.md | 11 +- .../action/actions/user/create.py | 5 +- .../action/actions/user/participant_common.py | 9 +- .../action/actions/user/update.py | 5 +- .../action/actions/user/user_mixins.py | 4 + .../action/mixins/import_mixins.py | 2 +- openslides_backend/presenter/__init__.py | 1 + openslides_backend/presenter/base.py | 1 + .../presenter/get_user_editable.py | 85 +++ .../presenter/get_user_scope.py | 5 +- openslides_backend/shared/exceptions.py | 25 +- .../user_create_update_permissions_mixin.py} | 146 ++--- .../shared/mixins/user_scope_mixin.py | 201 ++++++- .../action/user/scope_permissions_mixin.py | 28 +- tests/system/action/user/test_delete.py | 4 +- .../action/user/test_generate_new_password.py | 4 +- .../user/test_participant_json_upload.py | 1 - .../user/test_reset_password_to_default.py | 4 +- tests/system/action/user/test_set_password.py | 65 ++- tests/system/action/user/test_update.py | 498 ++++++++++++++++- tests/system/presenter/base.py | 71 +++ .../presenter/test_get_user_editable.py | 515 ++++++++++++++++++ .../presenter/test_get_user_related_models.py | 56 ++ 27 files changed, 1671 insertions(+), 112 deletions(-) create mode 100644 docs/presenters/get_user_editable.md create mode 100644 openslides_backend/presenter/get_user_editable.py rename openslides_backend/{action/actions/user/create_update_permissions_mixin.py => shared/mixins/user_create_update_permissions_mixin.py} (83%) create mode 100644 tests/system/presenter/test_get_user_editable.py diff --git a/docs/Presenters-Overview.md b/docs/Presenters-Overview.md index 5ffc1db58a..58ef16d284 100644 --- a/docs/Presenters-Overview.md +++ b/docs/Presenters-Overview.md @@ -6,6 +6,7 @@ Available presenters: - [get_forwarding_meetings](presenters/get_forwarding_meetings.md) - [get_meetings](presenters/get_meetings.md) - [get_users](presenters/get_users.md) +- [get_user_editable](presenters/get_user_editable.md) - [get_user_related_models](presenters/get_user_related_models.md) - [get_user_scope](presenters/get_user_scope.md) - [search_deleted_models](presenters/search_deleted_models.md) diff --git a/docs/actions/user.set_password.md b/docs/actions/user.set_password.md index af0a4e856d..cc64952402 100644 --- a/docs/actions/user.set_password.md +++ b/docs/actions/user.set_password.md @@ -9,7 +9,6 @@ set_as_default: boolean; // default false, if not given } ``` - ## Action Sets the password of the user given by `id` to `password`. If `set_as_default` is true, the `default_password` is also updated. diff --git a/docs/presenters/get_user_editable.md b/docs/presenters/get_user_editable.md new file mode 100644 index 0000000000..480e5bafb2 --- /dev/null +++ b/docs/presenters/get_user_editable.md @@ -0,0 +1,31 @@ +## Payload + +```js +{ + user_ids: Id[], // required + fields: string[] // required +} +``` + +## Returns + +```js +{ + user_id: Id: { + field: str: ( + editable: boolean, // true if user can be updated or deleted, + message?: string // error message if an exception was caught + ), + ... + }, + ... +} +``` + +## Logic + +It iterates over the given `user_ids` and calculates whether a user can be updated depending on the given payload fields, permissions in shared committees and meetings, OML and the user-scope. The user scope is defined [here](https://github.com/OpenSlides/OpenSlides/wiki/Users#user-scopes). The payload field permissions are described [here](https://github.com/OpenSlides/openslides-backend/blob/main/docs/actions/user.update.md) and [here](https://github.com/OpenSlides/openslides-backend/blob/main/docs/actions/user.create.md). + +## Permissions + +There are no special permissions necessary. \ No newline at end of file diff --git a/docs/presenters/get_user_related_models.md b/docs/presenters/get_user_related_models.md index 6f7a9808de..b5e057be54 100644 --- a/docs/presenters/get_user_related_models.md +++ b/docs/presenters/get_user_related_models.md @@ -1,6 +1,6 @@ ## Payload -```js +``` { user_ids: Id[] // required } @@ -8,7 +8,7 @@ ## Returns -```js +``` { [user_id: Id]: { organization_management_level: OML-String, diff --git a/docs/presenters/get_user_scope.md b/docs/presenters/get_user_scope.md index b4b4eca3a6..0119f79b2e 100644 --- a/docs/presenters/get_user_scope.md +++ b/docs/presenters/get_user_scope.md @@ -1,6 +1,6 @@ ## Payload -```js +``` { user_ids: Id[] // required } @@ -8,14 +8,15 @@ ## Returns -```js +``` { user_id: Id: { collection: String, # one of "meeting", "committee" or "organization" id: Id, - user_oml: String, # one of "superadmin", "can_manage_organization", "can_manage_users", "" - committee_ids: int[] // Ids of all committees the user is part of - } + user_oml: String, # one of "superadmin", "can_manage_organization", "can_manage_users", "" + committee_ids: Id[] // Ids of all committees the user is part of + }, + ... } ``` diff --git a/openslides_backend/action/actions/user/create.py b/openslides_backend/action/actions/user/create.py index 49494ec5f1..e4eef4b38b 100644 --- a/openslides_backend/action/actions/user/create.py +++ b/openslides_backend/action/actions/user/create.py @@ -2,6 +2,9 @@ from typing import Any from openslides_backend.permissions.permissions import Permissions +from openslides_backend.shared.mixins.user_create_update_permissions_mixin import ( + CreateUpdatePermissionsMixin, +) from ....models.models import User from ....shared.exceptions import ActionException @@ -15,13 +18,13 @@ from ...util.register import register_action from ...util.typing import ActionResultElement from ..meeting_user.mixin import CheckLockOutPermissionMixin -from .create_update_permissions_mixin import CreateUpdatePermissionsMixin from .password_mixins import SetPasswordMixin from .user_mixins import LimitOfUserMixin, UserMixin, UsernameMixin, check_gender_exists @register_action("user.create") class UserCreate( + UserMixin, EmailCheckMixin, CreateAction, CreateUpdatePermissionsMixin, diff --git a/openslides_backend/action/actions/user/participant_common.py b/openslides_backend/action/actions/user/participant_common.py index 2069cc94c0..54f11e63e4 100644 --- a/openslides_backend/action/actions/user/participant_common.py +++ b/openslides_backend/action/actions/user/participant_common.py @@ -11,14 +11,14 @@ ) from openslides_backend.permissions.permissions import Permissions from openslides_backend.shared.exceptions import MissingPermission +from openslides_backend.shared.mixins.user_create_update_permissions_mixin import ( + CreateUpdatePermissionsFailingFields, + PermissionVarStore, +) from openslides_backend.shared.patterns import fqid_from_collection_and_id from ....shared.filters import And, FilterOperator, Or from ..meeting_user.mixin import CheckLockOutPermissionMixin -from ..user.create_update_permissions_mixin import ( - CreateUpdatePermissionsFailingFields, - PermissionVarStore, -) class ParticipantCommon(BaseImportJsonUploadAction, CheckLockOutPermissionMixin): @@ -51,6 +51,7 @@ def check_permissions(self, instance: dict[str, Any]) -> None: ) self.permission_check = CreateUpdatePermissionsFailingFields( + self.user_id, permstore, self.services, self.datastore, diff --git a/openslides_backend/action/actions/user/update.py b/openslides_backend/action/actions/user/update.py index a5df918baa..d3fe226832 100644 --- a/openslides_backend/action/actions/user/update.py +++ b/openslides_backend/action/actions/user/update.py @@ -2,6 +2,9 @@ from typing import Any from openslides_backend.permissions.permissions import Permissions +from openslides_backend.shared.mixins.user_create_update_permissions_mixin import ( + CreateUpdatePermissionsMixin, +) from ....action.action import original_instances from ....action.util.typing import ActionData @@ -17,7 +20,6 @@ from ...util.register import register_action from ..meeting_user.mixin import CheckLockOutPermissionMixin from .conditional_speaker_cascade_mixin import ConditionalSpeakerCascadeMixin -from .create_update_permissions_mixin import CreateUpdatePermissionsMixin from .user_mixins import ( AdminIntegrityCheckMixin, LimitOfUserMixin, @@ -29,6 +31,7 @@ @register_action("user.update") class UserUpdate( + UserMixin, EmailCheckMixin, CreateUpdatePermissionsMixin, UpdateAction, diff --git a/openslides_backend/action/actions/user/user_mixins.py b/openslides_backend/action/actions/user/user_mixins.py index 9f5c99f899..807e00fb91 100644 --- a/openslides_backend/action/actions/user/user_mixins.py +++ b/openslides_backend/action/actions/user/user_mixins.py @@ -91,6 +91,10 @@ class UserMixin(CheckForArchivedMeetingMixin): "locked_out": {"type": "boolean"}, } + def check_permissions(self, instance: dict[str, Any]) -> None: + self.assert_not_anonymous() + super().check_permissions(instance) + def validate_instance(self, instance: dict[str, Any]) -> None: super().validate_instance(instance) if "meeting_id" not in instance and any( diff --git a/openslides_backend/action/mixins/import_mixins.py b/openslides_backend/action/mixins/import_mixins.py index 87df99e143..dfe4e68b7b 100644 --- a/openslides_backend/action/mixins/import_mixins.py +++ b/openslides_backend/action/mixins/import_mixins.py @@ -317,7 +317,7 @@ def flatten_copied_object_fields( ) -> list[ImportRow]: """The self.rows will be deepcopied, flattened and returned, without changes on the self.rows. - This is necessary for using the data in the executution of actions. + This is necessary for using the data in the execution of actions. The requests response should be given with the unchanged self.rows. Parameter: hook_method: diff --git a/openslides_backend/presenter/__init__.py b/openslides_backend/presenter/__init__.py index 432c547183..f06f93cb07 100644 --- a/openslides_backend/presenter/__init__.py +++ b/openslides_backend/presenter/__init__.py @@ -7,6 +7,7 @@ get_forwarding_meetings, get_history_information, get_mediafile_context, + get_user_editable, get_user_related_models, get_user_scope, get_users, diff --git a/openslides_backend/presenter/base.py b/openslides_backend/presenter/base.py index 72e44e971e..81a0ff7994 100644 --- a/openslides_backend/presenter/base.py +++ b/openslides_backend/presenter/base.py @@ -17,6 +17,7 @@ class BasePresenter(BaseServiceProvider): Base class for presenters. """ + internal: bool = False data: Any schema: Callable[[Any], None] | None = None diff --git a/openslides_backend/presenter/get_user_editable.py b/openslides_backend/presenter/get_user_editable.py new file mode 100644 index 0000000000..7104da3190 --- /dev/null +++ b/openslides_backend/presenter/get_user_editable.py @@ -0,0 +1,85 @@ +from collections import defaultdict +from typing import Any + +import fastjsonschema + +from openslides_backend.permissions.permissions import Permissions +from openslides_backend.shared.exceptions import ( + ActionException, + MissingPermission, + PermissionDenied, + PresenterException, +) +from openslides_backend.shared.mixins.user_create_update_permissions_mixin import ( + CreateUpdatePermissionsMixin, +) +from openslides_backend.shared.schema import id_list_schema, str_list_schema + +from ..shared.schema import schema_version +from .base import BasePresenter +from .presenter import register_presenter + +get_user_editable_schema = fastjsonschema.compile( + { + "$schema": schema_version, + "type": "object", + "title": "get_user_editable", + "description": "get user editable", + "properties": { + "user_ids": id_list_schema, + "fields": str_list_schema, + }, + "required": ["user_ids", "fields"], + "additionalProperties": False, + } +) + + +@register_presenter("get_user_editable") +class GetUserEditable(CreateUpdatePermissionsMixin, BasePresenter): + """ + Checks for each given user whether the given fields are editable by calling user on a per payload group basis. + """ + + schema = get_user_editable_schema + name = "get_user_editable" + permission = Permissions.User.CAN_MANAGE + + def get_result(self) -> Any: + if not self.data["fields"]: + raise PresenterException( + "Need at least one field name to check editability." + ) + reversed_field_rights = { + field: group + for group, fields in self.field_rights.items() + for field in fields + } + one_field_per_group = { + group_fields[0] + for field_name in self.data["fields"] + for group_fields in self.field_rights.values() + if field_name in group_fields + } + result: defaultdict[str, dict[str, tuple[bool, str]]] = defaultdict(dict) + for user_id in self.data["user_ids"]: + result[str(user_id)] = {} + groups_editable = {} + for field_name in one_field_per_group: + try: + self.check_permissions({"id": user_id, field_name: None}) + groups_editable[reversed_field_rights[field_name]] = (True, "") + except (PermissionDenied, MissingPermission, ActionException) as e: + groups_editable[reversed_field_rights[field_name]] = ( + False, + e.message, + ) + result[str(user_id)].update( + { + data_field_name: groups_editable[ + reversed_field_rights[data_field_name] + ] + for data_field_name in self.data["fields"] + } + ) + return result diff --git a/openslides_backend/presenter/get_user_scope.py b/openslides_backend/presenter/get_user_scope.py index 8c07f485a9..fa609bad92 100644 --- a/openslides_backend/presenter/get_user_scope.py +++ b/openslides_backend/presenter/get_user_scope.py @@ -36,7 +36,10 @@ def get_result(self) -> Any: result: dict[str, Any] = {} user_ids = self.data["user_ids"] for user_id in user_ids: - scope, scope_id, user_oml, committee_ids = self.get_user_scope(user_id) + scope, scope_id, user_oml, committee_meeting_ids = self.get_user_scope( + user_id + ) + committee_ids = [ci for ci in committee_meeting_ids.keys()] result[str(user_id)] = { "collection": scope, "id": scope_id, diff --git a/openslides_backend/shared/exceptions.py b/openslides_backend/shared/exceptions.py index 59404c29a6..94ea1811d6 100644 --- a/openslides_backend/shared/exceptions.py +++ b/openslides_backend/shared/exceptions.py @@ -131,16 +131,29 @@ def __init__(self, action_name: str) -> None: class MissingPermission(PermissionDenied): def __init__( self, - permissions: AnyPermission | dict[AnyPermission, int], + permissions: AnyPermission | dict[AnyPermission, int | set[int]], ) -> None: if isinstance(permissions, dict): - self.message = ( - "Missing permission" + ("s" if len(permissions) > 1 else "") + ": " - ) + to_remove = [] + for permission, id_or_ids in permissions.items(): + if isinstance(id_or_ids, set) and not id_or_ids: + to_remove.append(permission) + for permission in to_remove: + del permissions[permission] + self.message = "Missing permission" + self._plural_s(permissions) + ": " self.message += " or ".join( - f"{permission.get_verbose_type()} {permission} in {permission.get_base_model()} {id}" - for permission, id in permissions.items() + f"{permission.get_verbose_type()} {permission} in {permission.get_base_model()}{self._plural_s(id_or_ids)} {id_or_ids}" + for permission, id_or_ids in permissions.items() ) else: self.message = f"Missing {permissions.get_verbose_type()}: {permissions}" super().__init__(self.message) + + def _plural_s(self, permission_or_id_or_ids: dict | int | set[int]) -> str: + if ( + isinstance(permission_or_id_or_ids, set) + or (isinstance(permission_or_id_or_ids, dict)) + ) and len(permission_or_id_or_ids) > 1: + return "s" + else: + return "" diff --git a/openslides_backend/action/actions/user/create_update_permissions_mixin.py b/openslides_backend/shared/mixins/user_create_update_permissions_mixin.py similarity index 83% rename from openslides_backend/action/actions/user/create_update_permissions_mixin.py rename to openslides_backend/shared/mixins/user_create_update_permissions_mixin.py index 4a64e86a96..a7d0874c3e 100644 --- a/openslides_backend/action/actions/user/create_update_permissions_mixin.py +++ b/openslides_backend/shared/mixins/user_create_update_permissions_mixin.py @@ -3,7 +3,6 @@ from functools import reduce from typing import Any, cast -from openslides_backend.action.action import Action from openslides_backend.action.relations.relation_manager import RelationManager from openslides_backend.permissions.base_classes import Permission from openslides_backend.permissions.management_levels import ( @@ -13,8 +12,10 @@ from openslides_backend.permissions.permissions import Permissions, permission_parents from openslides_backend.services.datastore.commands import GetManyRequest from openslides_backend.services.datastore.interface import DatastoreService +from openslides_backend.shared.base_service_provider import BaseServiceProvider from openslides_backend.shared.exceptions import ( ActionException, + AnyPermission, MissingPermission, PermissionDenied, ) @@ -24,8 +25,6 @@ from openslides_backend.shared.mixins.user_scope_mixin import UserScope, UserScopeMixin from openslides_backend.shared.patterns import fqid_from_collection_and_id -from .user_mixins import UserMixin - class PermissionVarStore: permission: Permission @@ -171,9 +170,10 @@ def _get_user_meetings_with_permission( return user_meetings -class CreateUpdatePermissionsMixin(UserMixin, UserScopeMixin, Action): +class CreateUpdatePermissionsMixin(UserScopeMixin, BaseServiceProvider): permstore: PermissionVarStore permission: Permission + internal: bool field_rights: dict[str, list] = { "A": [ @@ -203,7 +203,7 @@ class CreateUpdatePermissionsMixin(UserMixin, UserScopeMixin, Action): "is_present", # participant import ], "C": ["meeting_id", "group_ids"], - "D": ["committee_ids", "committee_management_ids"], + "D": ["committee_management_ids"], "E": ["organization_management_level"], "F": ["default_password"], "G": ["is_demo_user"], @@ -213,10 +213,12 @@ class CreateUpdatePermissionsMixin(UserMixin, UserScopeMixin, Action): def check_permissions(self, instance: dict[str, Any]) -> None: """ Checks the permissions on a per field and user.scope base, details see - https://github.com/OpenSlides/OpenSlides/wiki/user.update or user.create + https://github.com/OpenSlides/OpenSlides/wiki/Users + https://github.com/OpenSlides/OpenSlides/wiki/Permission-System + https://github.com/OpenSlides/OpenSlides/wiki/Restrictions-Overview The fields groups and their necessary permissions are also documented there. + Returns true if permissions are given. """ - self.assert_not_anonymous() if "forwarding_committee_ids" in instance: raise PermissionDenied("forwarding_committee_ids is not allowed.") @@ -227,12 +229,12 @@ def check_permissions(self, instance: dict[str, Any]) -> None: ) actual_group_fields = self._get_actual_grouping_from_instance(instance) - # store scope, id and OML-permission for requested user + # store scope, scope id, OML-permission and committee ids including the the respective meetings for requested user ( self.instance_user_scope, self.instance_user_scope_id, self.instance_user_oml_permission, - self.instance_committee_ids, + self.instance_committee_meeting_ids, ) = self.get_user_scope(instance.get("id") or instance) if self.permstore.user_oml != OrganizationManagementLevel.SUPERADMIN: @@ -247,40 +249,38 @@ def check_permissions(self, instance: dict[str, Any]) -> None: lock_result=False, ).get("locked_from_inside", False) - # Ordered by supposed velocity advantages. Changing order can only effect the sequence of detected errors for tests + # Ordered by supposed speed advantages. Changing order can only effect the sequence of detected errors for tests self.check_group_H(actual_group_fields["H"]) self.check_group_E(actual_group_fields["E"], instance) self.check_group_D(actual_group_fields["D"], instance) self.check_group_C(actual_group_fields["C"], instance, locked_from_inside) self.check_group_B(actual_group_fields["B"], instance, locked_from_inside) - self.check_group_A(actual_group_fields["A"]) - self.check_group_F(actual_group_fields["F"]) + self.check_group_A(actual_group_fields["A"], instance) + self.check_group_F(actual_group_fields["F"], instance) self.check_group_G(actual_group_fields["G"]) - def check_group_A( - self, - fields: list[str], - ) -> None: + def check_group_A(self, fields: list[str], instance: dict[str, Any]) -> None: """Check Group A: Depending on scope of user to act on""" if ( - self.permstore.user_oml == OrganizationManagementLevel.SUPERADMIN - or not fields + not fields or self.permstore.user_oml >= OrganizationManagementLevel.CAN_MANAGE_USERS ): return + missing_permissions: dict[AnyPermission, int | set[int]] = dict() if self.instance_user_scope == UserScope.Organization: - if self.permstore.user_committees.intersection(self.instance_committee_ids): - return - raise MissingPermission({OrganizationManagementLevel.CAN_MANAGE_USERS: 1}) - if self.instance_user_scope == UserScope.Committee: - if self.instance_user_scope_id not in self.permstore.user_committees: - raise MissingPermission( - { - OrganizationManagementLevel.CAN_MANAGE_USERS: 1, - CommitteeManagementLevel.CAN_MANAGE: self.instance_user_scope_id, - } + if not ( + self.permstore.user_committees.intersection( + self.instance_committee_meeting_ids ) + ): + missing_permissions = {OrganizationManagementLevel.CAN_MANAGE_USERS: 1} + elif self.instance_user_scope == UserScope.Committee: + if self.instance_user_scope_id not in self.permstore.user_committees: + missing_permissions = { + OrganizationManagementLevel.CAN_MANAGE_USERS: 1, + CommitteeManagementLevel.CAN_MANAGE: self.instance_user_scope_id, + } elif ( self.instance_user_scope_id not in self.permstore.user_committees_meetings and self.instance_user_scope_id not in self.permstore.user_meetings @@ -290,13 +290,26 @@ def check_group_A( ["committee_id"], lock_result=False, ) - raise MissingPermission( + missing_permissions = { + OrganizationManagementLevel.CAN_MANAGE_USERS: 1, + CommitteeManagementLevel.CAN_MANAGE: meeting["committee_id"], + self.permission: self.instance_user_scope_id, + } + if missing_permissions and not self.check_for_admin_in_all_meetings( + instance.get("id", 0) + ): + missing_permissions.update( { - OrganizationManagementLevel.CAN_MANAGE_USERS: 1, - CommitteeManagementLevel.CAN_MANAGE: meeting["committee_id"], - self.permission: self.instance_user_scope_id, + Permissions.User.CAN_UPDATE: { + meeting_id + for meeting_ids in self.instance_committee_meeting_ids.values() + if meeting_ids is not None + for meeting_id in meeting_ids + if meeting_id is not None + }, } ) + raise MissingPermission(missing_permissions) def check_group_B( self, fields: list[str], instance: dict[str, Any], locked_from_inside: bool @@ -360,10 +373,7 @@ def check_group_E(self, fields: list[str], instance: dict[str, Any]) -> None: f"Your organization management level is not high enough to set a Level of {instance.get('organization_management_level', OrganizationManagementLevel.CAN_MANAGE_USERS.get_verbose_type())}." ) - def check_group_F( - self, - fields: list[str], - ) -> None: + def check_group_F(self, fields: list[str], instance: dict[str, Any]) -> None: """Check F common fields: scoped permissions necessary, but if instance user has an oml-permission, that of the request user must be higher""" if ( @@ -372,6 +382,7 @@ def check_group_F( ): return + missing_permissions: dict[AnyPermission, int | set[int]] = dict() if ( self.instance_user_oml_permission or self.instance_user_scope == UserScope.Organization @@ -382,25 +393,22 @@ def check_group_F( ) else: if self.permstore.user_committees.intersection( - self.instance_committee_ids + self.instance_committee_meeting_ids ): return expected_oml_permission = OrganizationManagementLevel.CAN_MANAGE_USERS if expected_oml_permission > self.permstore.user_oml: - raise MissingPermission({expected_oml_permission: 1}) + missing_permissions = {expected_oml_permission: 1} else: return - else: - if self.permstore.user_oml >= OrganizationManagementLevel.CAN_MANAGE_USERS: - return - if self.instance_user_scope == UserScope.Committee: + elif self.permstore.user_oml >= OrganizationManagementLevel.CAN_MANAGE_USERS: + return + elif self.instance_user_scope == UserScope.Committee: if self.instance_user_scope_id not in self.permstore.user_committees: - raise MissingPermission( - { - OrganizationManagementLevel.CAN_MANAGE_USERS: 1, - CommitteeManagementLevel.CAN_MANAGE: self.instance_user_scope_id, - } - ) + missing_permissions = { + OrganizationManagementLevel.CAN_MANAGE_USERS: 1, + CommitteeManagementLevel.CAN_MANAGE: self.instance_user_scope_id, + } elif ( self.instance_user_scope_id not in self.permstore.user_committees_meetings and self.instance_user_scope_id not in self.permstore.user_meetings @@ -410,13 +418,26 @@ def check_group_F( ["committee_id"], lock_result=False, ) - raise MissingPermission( + missing_permissions = { + OrganizationManagementLevel.CAN_MANAGE_USERS: 1, + CommitteeManagementLevel.CAN_MANAGE: meeting["committee_id"], + self.permission: self.instance_user_scope_id, + } + if missing_permissions and not self.check_for_admin_in_all_meetings( + instance.get("id", 0) + ): + missing_permissions.update( { - OrganizationManagementLevel.CAN_MANAGE_USERS: 1, - CommitteeManagementLevel.CAN_MANAGE: meeting["committee_id"], - self.permission: self.instance_user_scope_id, + Permissions.User.CAN_UPDATE: { + meeting_id + for meeting_ids in self.instance_committee_meeting_ids.values() + if meeting_ids is not None + for meeting_id in meeting_ids + if meeting_id is not None + }, } ) + raise MissingPermission(missing_permissions) def check_group_G(self, fields: list[str]) -> None: """Group G: OML SUPERADMIN necessary""" @@ -475,8 +496,9 @@ def _get_actual_grouping_from_instance( """ Returns a dictionary with an entry for each field group A-E with a list of fields from payload instance. - The field groups A-F refer to https://github.com/OpenSlides/OpenSlides/wiki/user.create - or user.update + The field groups A-F refer to https://github.com/OpenSlides/openslides-meta/blob/main/models.yml + or https://github.com/OpenSlides/openslides-backend/blob/main/docs/actions/user.create.md + or https://github.com/OpenSlides/openslides-backend/blob/main/docs/actions/user.update.md """ act_grouping: dict[str, list[str]] = defaultdict(list) for key, _ in instance.items(): @@ -519,7 +541,9 @@ def _meetings_from_group_B_fields_from_instance( any other group B field. """ meetings: set[int] = set(instance.get("is_present_in_meeting_ids", [])) - meetings.add(cast(int, instance.get("meeting_id"))) + meeting_id = cast(int, instance.get("meeting_id")) + if meeting_id: + meetings.add(meeting_id) return meetings @@ -528,6 +552,7 @@ class CreateUpdatePermissionsFailingFields(CreateUpdatePermissionsMixin): def __init__( self, + user_id: int, permstore: PermissionVarStore, services: Services, datastore: DatastoreService, @@ -538,14 +563,11 @@ def __init__( use_meeting_ids_for_archived_meeting_check: bool | None = None, ) -> None: self.permstore = permstore + self.user_id = user_id super().__init__( services, datastore, - relation_manager, logging, - env, - skip_archived_meeting_check, - use_meeting_ids_for_archived_meeting_check, ) def get_failing_fields(self, instance: dict[str, Any]) -> list[str]: @@ -570,7 +592,7 @@ def get_failing_fields(self, instance: dict[str, Any]) -> list[str]: self.instance_user_scope, self.instance_user_scope_id, self.instance_user_oml_permission, - self.instance_committee_ids, + self.instance_committee_meeting_ids, ) = self.get_user_scope(instance.get("id") or instance) instance_meeting_id = instance.get("meeting_id") @@ -598,8 +620,8 @@ def get_failing_fields(self, instance: dict[str, Any]) -> list[str]: instance, locked_from_inside, ), - (self.check_group_A, actual_group_fields["A"], None, None), - (self.check_group_F, actual_group_fields["F"], None, None), + (self.check_group_A, actual_group_fields["A"], instance, None), + (self.check_group_F, actual_group_fields["F"], instance, None), (self.check_group_G, actual_group_fields["G"], None, None), ]: try: diff --git a/openslides_backend/shared/mixins/user_scope_mixin.py b/openslides_backend/shared/mixins/user_scope_mixin.py index b84b6050fa..47c22b5c3e 100644 --- a/openslides_backend/shared/mixins/user_scope_mixin.py +++ b/openslides_backend/shared/mixins/user_scope_mixin.py @@ -1,3 +1,4 @@ +from collections import defaultdict from enum import Enum from typing import Any @@ -29,21 +30,26 @@ def __repr__(self) -> str: class UserScopeMixin(BaseServiceProvider): + instance_committee_meeting_ids: dict + name: str + def get_user_scope( self, id_or_instance: int | dict[str, Any] - ) -> tuple[UserScope, int, str, list[int]]: + ) -> tuple[UserScope, int, str, dict[int, Any]]: """ Parameter id_or_instance: id for existing user or instance for user to create Returns the scope of the given user id together with the relevant scope id (either meeting, committee or organization), the OML level of the user as string (empty string if the user - has none) and the ids of all committees that the user is either a manager in or a member of. + has none) and the ids of all committees that the user is either a manager in or a member of + together with their respective meetings the user being part of. A committee can have no meetings if the + user just has committee management rights and is not part of any of its meetings. """ - meetings: set[int] = set() + meeting_ids: set[int] = set() committees_manager: set[int] = set() if isinstance(id_or_instance, dict): if "group_ids" in id_or_instance: if "meeting_id" in id_or_instance: - meetings.add(id_or_instance["meeting_id"]) + meeting_ids.add(id_or_instance["meeting_id"]) committees_manager.update( set(id_or_instance.get("committee_management_ids", [])) ) @@ -56,15 +62,16 @@ def get_user_scope( "organization_management_level", "committee_management_ids", ], + lock_result=False, ) - meetings.update(user.get("meeting_ids", [])) + meeting_ids.update(user.get("meeting_ids", [])) committees_manager.update(set(user.get("committee_management_ids") or [])) oml_right = user.get("organization_management_level", "") result = self.datastore.get_many( [ GetManyRequest( "meeting", - list(meetings), + list(meeting_ids), ["committee_id", "is_active_in_organization_id"], ) ] @@ -76,25 +83,32 @@ def get_user_scope( if meeting_data.get("is_active_in_organization_id") } committees = committees_manager | set(meetings_committee.values()) + committee_meetings: dict[int, Any] = defaultdict(list) + for meeting, committee in meetings_committee.items(): + committee_meetings[committee].append(meeting) + for committee in committees: + if committee not in committee_meetings.keys(): + committee_meetings[committee] = None + if len(meetings_committee) == 1 and len(committees) == 1: return ( UserScope.Meeting, next(iter(meetings_committee)), oml_right, - list(committees), + committee_meetings, ) elif len(committees) == 1: return ( UserScope.Committee, next(iter(committees)), oml_right, - list(committees), + committee_meetings, ) - return UserScope.Organization, 1, oml_right, list(committees) + return UserScope.Organization, 1, oml_right, committee_meetings def check_permissions_for_scope( self, - id: int, + instance_id: int, always_check_user_oml: bool = True, meeting_permission: Permission = Permissions.User.CAN_MANAGE, ) -> None: @@ -105,7 +119,9 @@ def check_permissions_for_scope( Reason: A user with OML-level-permission has scope "meeting" or "committee" if he belongs to only 1 meeting or 1 committee. """ - scope, scope_id, user_oml, committees = self.get_user_scope(id) + scope, scope_id, user_oml, committees_to_meetings = self.get_user_scope( + instance_id + ) if ( always_check_user_oml and user_oml @@ -160,7 +176,166 @@ def check_permissions_for_scope( self.datastore, self.user_id, CommitteeManagementLevel.CAN_MANAGE, - committees, + [ci for ci in committees_to_meetings.keys()], ): return - raise MissingPermission({OrganizationManagementLevel.CAN_MANAGE_USERS: 1}) + meeting_ids = { + meeting_id + for mids in committees_to_meetings.values() + for meeting_id in mids + } + if not meeting_ids or not self.check_for_admin_in_all_meetings( + instance_id, meeting_ids + ): + raise MissingPermission( + { + OrganizationManagementLevel.CAN_MANAGE_USERS: 1, + **{ + Permissions.User.CAN_UPDATE: meeting_id + for meeting_id in meeting_ids + }, + } + ) + + def check_for_admin_in_all_meetings( + self, + instance_id: int, + b_meeting_ids: set[int] | None = None, + ) -> bool: + """ + This function checks the special permission condition for scope request, user.update/create with + payload fields A and F and other user altering actions like user.delete or set_default_password. + This function returns true if: + * requested user is no committee manager and + * requested user doesn't have any admin/user.can_update/user.can_manage rights in his meetings and + * requesting user has those permissions in all of those meetings + """ + if not self._check_not_committee_manager(instance_id): + return False + + if not (meetings := self._get_meetings_if_subset(b_meeting_ids)): + return False + admin_meeting_users = self._collect_admin_meeting_users(meetings) + return self._analyze_meeting_admins(admin_meeting_users, meetings) + + def _check_not_committee_manager(self, instance_id: int) -> bool: + """ + Helper function used in method check_for_admin_in_all_meetings. + Checks that requested user is not a committee manager. + """ + if not (hasattr(self, "name") and self.name == "user.create"): + if self.datastore.get( + fqid_from_collection_and_id("user", instance_id), + ["committee_management_ids"], + lock_result=False, + use_changed_models=False, + ).get("committee_management_ids", []): + return False + return True + + def _get_meetings_if_subset(self, b_meeting_ids: set[int] | None) -> dict[int, Any]: + """ + Helper function used in method check_for_admin_in_all_meetings. + Gets the requested users meetings if these are subset of requesting user. Returns False if this is not possible. + """ + if not b_meeting_ids and not ( + b_meeting_ids := { + m_id + for m_ids in self.instance_committee_meeting_ids.values() + for m_id in m_ids + } + ): + return {} + if not ( + a_meeting_ids := set( + self.datastore.get( + fqid_from_collection_and_id("user", self.user_id), + ["meeting_ids"], + lock_result=False, + ).get("meeting_ids", []) + ) + ): + return {} + if not b_meeting_ids.issubset(a_meeting_ids): + return {} + return self.datastore.get_many( + [ + GetManyRequest( + "meeting", + list(b_meeting_ids), + ["admin_group_id", "group_ids"], + ) + ], + lock_result=False, + ).get("meeting", {}) + + def _collect_admin_meeting_users(self, meetings: dict[int, Any]) -> set[int]: + """ + Gets the admin groups and those groups with permission User.CAN_UPDATE and USER.CAN_MANAGE of the meetings. + Returns a set of the groups meeting_user_ids. + """ + group_ids = [ + group_id + for meeting_id, meeting_dict in meetings.items() + for group_id in meeting_dict.get("group_ids", []) + ] + return { + mu_id + for group_id, group in self.datastore.get_many( + [ + GetManyRequest( + "group", + group_ids, + [ + "meeting_user_ids", + "permissions", + "admin_group_for_meeting_id", + ], + ) + ], + lock_result=False, + ) + .get("group", {}) + .items() + if ( + group.get("admin_group_for_meeting_id") + or "user.can_update" in group.get("permissions", []) + or "user.can_manage" in group.get("permissions", []) + ) + for mu_id in group.get("meeting_user_ids", []) + } + + def _analyze_meeting_admins( + self, + admin_meeting_user_ids: set[int], + all_meetings: dict[int, Any], + ) -> bool: + """ + Helper function used in method check_for_admin_in_all_meetings. + Compares the users of admin meeting users of all meetings with the ids of requested user and requesting user. + Requesting user must be admin in all meetings. Requested user cannot be admin in any. + """ + meeting_id_to_admin_user_ids: dict[int, set[int]] = { + meeting_id: set() for meeting_id in all_meetings + } + for meeting_user in ( + self.datastore.get_many( + [ + GetManyRequest( + "meeting_user", + list(admin_meeting_user_ids), + ["user_id", "meeting_id"], + ) + ], + lock_result=False, + ) + .get("meeting_user", {}) + .values() + ): + meeting_id_to_admin_user_ids[meeting_user["meeting_id"]].add( + meeting_user["user_id"] + ) + return all( + self.user_id in admin_users + for admin_users in meeting_id_to_admin_user_ids.values() + ) diff --git a/tests/system/action/user/scope_permissions_mixin.py b/tests/system/action/user/scope_permissions_mixin.py index a2e78f506d..cbe016abcd 100644 --- a/tests/system/action/user/scope_permissions_mixin.py +++ b/tests/system/action/user/scope_permissions_mixin.py @@ -32,6 +32,26 @@ def setup_admin_scope_permissions( self.set_organization_management_level(None) self.set_user_groups(1, [3]) self.set_group_permissions(3, [meeting_permission]) + self.set_models( + { + "user/777": { + "username": "admin_group_filler", + "meeting_user_ids": [666, 667], + }, + "meeting_user/666": { + "group_ids": [12, 23], + "meeting_id": 1, + "user_id": 777, + }, + "meeting_user/667": { + "group_ids": [12, 23], + "meeting_id": 2, + "user_id": 777, + }, + "group/12": {"meeting_user_ids": [666]}, + "group/23": {"meeting_user_ids": [667]}, + } + ) def setup_scoped_user(self, scope: UserScope) -> None: """ @@ -45,13 +65,15 @@ def setup_scoped_user(self, scope: UserScope) -> None: "meeting/1": { "user_ids": [111], "committee_id": 1, - "group_ids": [11], + "group_ids": [11, 12], + "admin_group_id": 12, "is_active_in_organization_id": 1, }, "meeting/2": { "user_ids": [111], "committee_id": 2, - "group_ids": [22], + "group_ids": [22, 23], + "admin_group_id": 23, "is_active_in_organization_id": 1, }, "user/111": { @@ -70,7 +92,9 @@ def setup_scoped_user(self, scope: UserScope) -> None: "group_ids": [22], }, "group/11": {"meeting_id": 1, "meeting_user_ids": [11]}, + "group/12": {"meeting_id": 1, "meeting_user_ids": [666]}, "group/22": {"meeting_id": 2, "meeting_user_ids": [22]}, + "group/23": {"meeting_id": 2, "meeting_user_ids": [667]}, } ) elif scope == UserScope.Committee: diff --git a/tests/system/action/user/test_delete.py b/tests/system/action/user/test_delete.py index 890ff3a31e..250f212465 100644 --- a/tests/system/action/user/test_delete.py +++ b/tests/system/action/user/test_delete.py @@ -429,7 +429,7 @@ def test_delete_scope_organization_no_permission(self) -> None: response = self.request("user.delete", {"id": 111}) self.assert_status_code(response, 403) self.assertIn( - "You are not allowed to perform action user.delete. Missing permission: OrganizationManagementLevel can_manage_users in organization 1", + "You are not allowed to perform action user.delete. Missing permissions: OrganizationManagementLevel can_manage_users in organization 1 or Permission user.can_update in meeting 2", response.json["message"], ) @@ -479,7 +479,7 @@ def test_delete_scope_organization_permission_in_meeting(self) -> None: response = self.request("user.delete", {"id": 111}) self.assert_status_code(response, 403) self.assertIn( - "You are not allowed to perform action user.delete. Missing permission: OrganizationManagementLevel can_manage_users in organization 1", + "You are not allowed to perform action user.delete. Missing permissions: OrganizationManagementLevel can_manage_users in organization 1 or Permission user.can_update in meeting 2", response.json["message"], ) diff --git a/tests/system/action/user/test_generate_new_password.py b/tests/system/action/user/test_generate_new_password.py index 922b99fc6f..bae8c8fcc8 100644 --- a/tests/system/action/user/test_generate_new_password.py +++ b/tests/system/action/user/test_generate_new_password.py @@ -111,7 +111,7 @@ def test_scope_organization_no_permission(self) -> None: response = self.request("user.generate_new_password", {"id": 111}) self.assert_status_code(response, 403) self.assertIn( - "You are not allowed to perform action user.generate_new_password. Missing permission: OrganizationManagementLevel can_manage_users in organization 1", + "You are not allowed to perform action user.generate_new_password. Missing permissions: OrganizationManagementLevel can_manage_users in organization 1 or Permission user.can_update in meeting 2", response.json["message"], ) @@ -165,7 +165,7 @@ def test_scope_organization_permission_in_meeting(self) -> None: response = self.request("user.generate_new_password", {"id": 111}) self.assert_status_code(response, 403) self.assertIn( - "You are not allowed to perform action user.generate_new_password. Missing permission: OrganizationManagementLevel can_manage_users in organization 1", + "You are not allowed to perform action user.generate_new_password. Missing permissions: OrganizationManagementLevel can_manage_users in organization 1 or Permission user.can_update in meeting 2", response.json["message"], ) diff --git a/tests/system/action/user/test_participant_json_upload.py b/tests/system/action/user/test_participant_json_upload.py index 6005ffc1ad..e8c180b88d 100644 --- a/tests/system/action/user/test_participant_json_upload.py +++ b/tests/system/action/user/test_participant_json_upload.py @@ -9,7 +9,6 @@ class ParticipantJsonUpload(BaseActionTestCase): def setUp(self) -> None: - self.maxDiff = None super().setUp() self.set_models( { diff --git a/tests/system/action/user/test_reset_password_to_default.py b/tests/system/action/user/test_reset_password_to_default.py index dbd33d406d..491f017420 100644 --- a/tests/system/action/user/test_reset_password_to_default.py +++ b/tests/system/action/user/test_reset_password_to_default.py @@ -118,7 +118,7 @@ def test_scope_organization_no_permission(self) -> None: response = self.request("user.reset_password_to_default", {"id": 111}) self.assert_status_code(response, 403) self.assertIn( - "You are not allowed to perform action user.reset_password_to_default. Missing permission: OrganizationManagementLevel can_manage_users in organization 1", + "You are not allowed to perform action user.reset_password_to_default. Missing permissions: OrganizationManagementLevel can_manage_users in organization 1 or Permission user.can_update in meeting 2", response.json["message"], ) @@ -172,7 +172,7 @@ def test_scope_organization_permission_in_meeting(self) -> None: response = self.request("user.reset_password_to_default", {"id": 111}) self.assert_status_code(response, 403) self.assertIn( - "You are not allowed to perform action user.reset_password_to_default. Missing permission: OrganizationManagementLevel can_manage_users in organization 1", + "You are not allowed to perform action user.reset_password_to_default. Missing permissions: OrganizationManagementLevel can_manage_users in organization 1 or Permission user.can_update in meeting 2", response.json["message"], ) diff --git a/tests/system/action/user/test_set_password.py b/tests/system/action/user/test_set_password.py index 0ab5e700ad..e607d801e8 100644 --- a/tests/system/action/user/test_set_password.py +++ b/tests/system/action/user/test_set_password.py @@ -23,6 +23,67 @@ def test_update_correct(self) -> None: self.assert_history_information("user/2", ["Password changed"]) self.assert_logged_in() + def test_two_meetings(self) -> None: + self.create_meeting() + self.create_meeting(4) # meeting 4 + user_id = self.create_user("test", group_ids=[1]) + self.login(user_id) + self.set_models( + { + "user/111": {"password": "old_pw"}, + "meeting_user/666": { + "group_ids": [12, 23], + "meeting_id": 12, + "user_id": 1, + }, + } + ) + self.update_model( + "user/1", + {"meeting_user_ids": [666]}, + ) + # only to make sure every meeting has an admin at all times + self.set_user_groups(1, [2, 5]) + # Admin groups of meeting/1 for test user meeting/2 as normal user + self.set_user_groups(user_id, [2, 4]) + # 111 into both meetings + self.set_user_groups(111, [1, 4]) + response = self.request( + "user.set_password", {"id": 111, "password": self.PASSWORD} + ) + self.assert_status_code(response, 403) + self.assertIn( + "You are not allowed to perform action user.set_password. Missing permissions: OrganizationManagementLevel can_manage_users in organization 1 or Permission user.can_update in meeting 4", + response.json["message"], + ) + model = self.get_model("user/111") + assert "old_pw" == model.get("password", "") + # Admin groups of meeting/1 for test user + self.set_user_groups(user_id, [2]) + # 111 into both meetings + self.set_user_groups(111, [1, 4]) + response = self.request( + "user.set_password", {"id": 111, "password": self.PASSWORD} + ) + self.assert_status_code(response, 403) + self.assertIn( + "You are not allowed to perform action user.set_password. Missing permissions: OrganizationManagementLevel can_manage_users in organization 1 or Permission user.can_update in meeting 4", + response.json["message"], + ) + model = self.get_model("user/111") + assert "old_pw" == model.get("password", "") + # Admin groups of meeting/1 and meeting/4 for test user + self.set_user_groups(user_id, [2, 5]) + # 111 into both meetings + self.set_user_groups(111, [1, 4]) + response = self.request( + "user.set_password", {"id": 111, "password": self.PASSWORD} + ) + self.assert_status_code(response, 200) + model = self.get_model("user/111") + assert self.auth.is_equal(self.PASSWORD, model.get("password", "")) + self.assert_history_information("user/111", ["Password changed"]) + def test_update_correct_default_case(self) -> None: self.update_model("user/1", {"password": "old_pw"}) response = self.request( @@ -143,7 +204,7 @@ def test_scope_organization_no_permission(self) -> None: ) self.assert_status_code(response, 403) self.assertIn( - "You are not allowed to perform action user.set_password. Missing permission: OrganizationManagementLevel can_manage_users in organization 1", + "You are not allowed to perform action user.set_password. Missing permissions: OrganizationManagementLevel can_manage_users in organization 1 or Permission user.can_update in meeting 2", response.json["message"], ) @@ -205,7 +266,7 @@ def test_scope_organization_permission_in_meeting(self) -> None: ) self.assert_status_code(response, 403) self.assertIn( - "You are not allowed to perform action user.set_password. Missing permission: OrganizationManagementLevel can_manage_users in organization 1", + "You are not allowed to perform action user.set_password. Missing permissions: OrganizationManagementLevel can_manage_users in organization 1 or Permission user.can_update in meeting 2", response.json["message"], ) diff --git a/tests/system/action/user/test_update.py b/tests/system/action/user/test_update.py index 0b1b33e3a6..f91b7dd938 100644 --- a/tests/system/action/user/test_update.py +++ b/tests/system/action/user/test_update.py @@ -18,6 +18,136 @@ def permission_setup(self) -> None: } ) + def two_meetings_test_fail_ADEFGH( + self, committee_id: None | int = None, group_B_success: bool = False + ) -> None: + # test group A + response = self.request( + "user.update", + { + "id": 111, + "pronoun": "I'm not gonna get updated.", + }, + ) + self.assert_status_code(response, 403) + self.assertIn( + "You are not allowed to perform action user.update. Missing permissions: OrganizationManagementLevel can_manage_users in organization 1 or Permission user.can_update in meetings {1, 4}", + response.json["message"], + ) + self.assert_model_exists( + "user/111", + { + "pronoun": None, + }, + ) + # test group D + response = self.request( + "user.update", + { + "id": 111, + "committee_management_ids": [1, 2], + }, + ) + self.assert_status_code(response, 403) + if committee_id: + self.assertIn( + f"You are not allowed to perform action user.update. Missing permission: CommitteeManagementLevel can_manage in committee {committee_id}", + response.json["message"], + ) + self.assert_model_exists( + "user/111", + { + "committee_management_ids": [committee_id], + }, + ) + else: + self.assertIn( + "You are not allowed to perform action user.update. Missing permission: CommitteeManagementLevel can_manage in committee ", + response.json["message"], + ) + self.assert_model_exists( + "user/111", + { + "committee_management_ids": None, + }, + ) + # test group E + response = self.request( + "user.update", + { + "id": 111, + "organization_management_level": OrganizationManagementLevel.CAN_MANAGE_USERS, + }, + ) + self.assert_status_code(response, 403) + self.assertIn( + "Your organization management level is not high enough to set a Level of can_manage_users.", + response.json["message"], + ) + self.assert_model_exists( + "user/111", + { + "organization_management_level": None, + }, + ) + # test group F + response = self.request( + "user.update", + { + "id": 111, + "default_password": "I'm not gonna get updated.", + }, + ) + self.assert_status_code(response, 403) + self.assertIn( + "You are not allowed to perform action user.update. Missing permissions: OrganizationManagementLevel can_manage_users in organization 1 or Permission user.can_update in meetings {1, 4}", + response.json["message"], + ) + self.assert_model_exists( + "user/111", + { + "default_password": None, + }, + ) + # test group G + response = self.request( + "user.update", + { + "id": 111, + "is_demo_user": True, + }, + ) + self.assert_status_code(response, 403) + self.assertIn( + "You are not allowed to perform action user.update. Missing OrganizationManagementLevel: superadmin", + response.json["message"], + ) + self.assert_model_exists( + "user/111", + { + "is_demo_user": None, + }, + ) + # test group H + response = self.request( + "user.update", + { + "id": 111, + "saml_id": "I'm not gonna get updated.", + }, + ) + self.assert_status_code(response, 400) + self.assertIn( + "The field 'saml_id' can only be used in internal action calls", + response.json["message"], + ) + self.assert_model_exists( + "user/111", + { + "saml_id": None, + }, + ) + def test_update_correct(self) -> None: self.create_model( "user/111", @@ -843,11 +973,61 @@ def test_perm_group_A_cml_manage_user_archived_meeting_in_other_committee( self.assertCountEqual(user111["meeting_ids"], [1, 4]) def test_perm_group_A_meeting_manage_user(self) -> None: - """May update group A fields on meeting scope. User belongs to 1 meeting without being part of a committee""" + """ + May update group A fields on meeting scope. User belongs to 1 meeting without being part of a committee. + Testing various scenarios: + * both default group + * default group has user.can_update permission + * requesting user is in admin group + """ self.permission_setup() - self.set_user_groups(self.user_id, [2]) + self.set_user_groups(self.user_id, [1]) self.set_user_groups(111, [1]) + response = self.request( + "user.update", + { + "id": 111, + "username": "new_username", + "pronoun": "pronoun", + }, + ) + self.assert_status_code(response, 403) + self.assertIn( + "You are not allowed to perform action user.update. Missing permissions: OrganizationManagementLevel can_manage_users in organization 1 or CommitteeManagementLevel can_manage in committee 60 or Permission user.can_update in meeting {1}", + response.json["message"], + ) + self.assert_model_exists( + "user/111", + { + "username": "User111", + "pronoun": None, + "meeting_ids": [1], + "committee_ids": None, + }, + ) + + self.update_model("group/1", {"permissions": ["user.can_update"]}) + response = self.request( + "user.update", + { + "id": 111, + "username": "new_user", + "pronoun": "pro", + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/111", + { + "username": "new_user", + "pronoun": "pro", + "meeting_ids": [1], + "committee_ids": None, + }, + ) + + self.set_user_groups(self.user_id, [2]) response = self.request( "user.update", { @@ -867,6 +1047,172 @@ def test_perm_group_A_meeting_manage_user(self) -> None: }, ) + def test_perm_group_A_belongs_to_same_meetings(self) -> None: + """May update group A fields on any scope as long as admin user Ann belongs to all meetings user Ben belongs to. See issue 2522.""" + self.permission_setup() # meeting 1 + logged in test user + user 111 + self.create_meeting(4) # meeting 4 + # Admin groups of meeting/1 and meeting/4 for requesting user + self.set_user_groups(self.user_id, [1, 4]) + # 111 into both meetings + self.set_user_groups(111, [1, 4]) + self.two_meetings_test_fail_ADEFGH() + # Admin group of meeting/1 and default group for meeting/4 for request user + self.set_user_groups(self.user_id, [2, 4]) + # 111 into both meetings (admin group for meeting/4) + self.set_user_groups(111, [1, 5]) + self.two_meetings_test_fail_ADEFGH() + # test group B and C + response = self.request( + "user.update", + {"id": 111, "number": "I'm not gonna get updated.", "meeting_id": 4}, + ) + self.assertIn( + "The user needs OrganizationManagementLevel.can_manage_users or CommitteeManagementLevel.can_manage for committee of following meeting or Permission user.can_update for meeting 4", + response.json["message"], + ) + self.assert_status_code(response, 403) + self.assert_model_exists( + "user/111", + { + "number": None, + }, + ) + # Admin groups of meeting/1 and meeting/4 for request user + self.set_user_groups(self.user_id, [2, 5]) + # 111 into both meetings + self.set_user_groups(111, [1, 4]) + response = self.request( + "user.update", + { + "id": 111, + "pronoun": "I'm gonna get updated.", + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/111", + { + "pronoun": "I'm gonna get updated.", + }, + ) + + def test_perm_group_A_belongs_to_same_meetings_can_update(self) -> None: + """ + May update group A fields on any scope as long as requesting user has + user.can_update rights in requested users meetings. + Also makes sure being in multiple groups of a single meeting is no problem. + """ + self.permission_setup() # meeting 1 + logged in test user + user 111 + self.create_meeting(4) # meeting 4 + self.update_model( + "group/6", + {"permissions": ["user.can_update"]}, + ) + # Admin group of meeting/1 and default group of meeting/4 for requesting user + self.set_user_groups(self.user_id, [2, 4]) + # 111 into both meetings + self.set_user_groups(111, [1, 4]) + self.two_meetings_test_fail_ADEFGH() + # Admin groups of meeting/1 and meeting/4 (via group permission) for requesting user + self.set_user_groups(self.user_id, [2, 4, 6]) + response = self.request( + "user.update", + { + "id": 111, + "pronoun": "I'm gonna get updated.", + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/111", + { + "pronoun": "I'm gonna get updated.", + }, + ) + + def test_perm_group_A_belongs_to_same_meetings_can_manage(self) -> None: + """ + May update group A fields on any scope as long as requesting user has + user.can_update rights in requested users meetings. + Also makes sure being in multiple groups of a single meeting is no problem. + """ + self.permission_setup() # meeting 1 + logged in requesting user + user 111 + self.create_meeting(4) # meeting 4 + self.update_model( + "group/6", + {"permissions": ["user.can_manage"]}, + ) + # Admin group of meeting/1 and default group of meeting/4 for requesting user + self.set_user_groups(self.user_id, [2, 4]) + # 111 into both meetings + self.set_user_groups(111, [1, 4]) + self.two_meetings_test_fail_ADEFGH() + # Admin groups of meeting/1 and meeting/4 (via group permission) for requesting user + self.set_user_groups(self.user_id, [1, 2, 4, 6]) + response = self.request( + "user.update", + { + "id": 111, + "pronoun": "I'm gonna get updated.", + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/111", + { + "pronoun": "I'm gonna get updated.", + }, + ) + + def test_perm_group_A_belongs_to_same_meetings_committee_admin(self) -> None: + """May not update group A fields on any scope as long as admin user Ann belongs + to all meetings user Ben belongs to but Ben is committee admin. See issue 2522. + """ + self.permission_setup() # meeting 1 + logged in requesting user + user 111 + self.create_meeting(4) # meeting 4 + # Admin groups of meeting/1 and meeting/4 for requesting user + self.set_user_groups(self.user_id, [2, 5]) + # 111 into both meetings + self.set_user_groups(111, [1, 4]) + # 111 is committee admin + committee_id = 60 + self.set_committee_management_level([committee_id], 111) + self.two_meetings_test_fail_ADEFGH(committee_id) + # test group B and C + response = self.request( + "user.update", + {"id": 111, "number": "I'm not gonna get updated.", "meeting_id": 4}, + ) + self.assert_status_code(response, 200) + self.assertIn( + "Actions handled successfully", + response.json["message"], + ) + self.assert_model_exists( + "user/111", + { + "number": None, + }, + ) + response = self.request( + "user.update", + { + "id": 111, + "pronoun": "I'm not gonna get updated.", + }, + ) + self.assert_status_code(response, 403) + self.assertIn( + "You are not allowed to perform action user.update. Missing permissions: OrganizationManagementLevel can_manage_users in organization 1 or Permission user.can_update in meetings {1, 4}", + response.json["message"], + ) + self.assert_model_exists( + "user/111", + { + "pronoun": None, + }, + ) + def test_perm_group_A_meeting_manage_user_archived_meeting(self) -> None: self.perm_group_A_meeting_manage_user_archived_meeting( Permissions.User.CAN_UPDATE @@ -929,7 +1275,7 @@ def test_perm_group_A_no_permission(self) -> None: ) self.assert_status_code(response, 403) self.assertIn( - "You are not allowed to perform action user.update. Missing permission: OrganizationManagementLevel can_manage_users in organization 1", + "You are not allowed to perform action user.update. Missing permissions: OrganizationManagementLevel can_manage_users in organization 1 or Permission user.can_update in meetings {1, 4}", response.json["message"], ) @@ -992,7 +1338,7 @@ def test_perm_group_F_default_password_for_superadmin_no_permission(self) -> Non ) def test_perm_group_F_cml_manage_user_with_two_committees(self) -> None: - """May update group A fields on committee scope. User belongs to 1 meeting in 1 committee""" + """May update group F fields on committee scope. User belongs to two meetings.""" self.permission_setup() self.create_meeting(4) self.set_committee_management_level([60], self.user_id) @@ -1014,6 +1360,150 @@ def test_perm_group_F_cml_manage_user_with_two_committees(self) -> None: }, ) + def test_perm_group_F_with_meeting_scope(self) -> None: + """ + Test user update with various scenarios (admin in different meeting and committee no interference) + * not in same meeting fails + * same meeting but requesting user not in admin or permission group fails + * same meeting requesting user with permission user.can_update works + * same meeting both admin works + * same meeting requesting user is committee admin works + """ + self.permission_setup() + self.create_meeting(4) + self.set_user_groups(111, [2]) + self.set_user_groups(self.user_id, [5]) + self.set_committee_management_level([63], self.user_id) + + response = self.request( + "user.update", + { + "id": 111, + "default_password": "new_one", + }, + ) + self.assert_status_code(response, 403) + self.assertIn( + "You are not allowed to perform action user.update. Missing permissions: OrganizationManagementLevel can_manage_users in organization 1 or CommitteeManagementLevel can_manage in committee 60 or Permission user.can_update in meeting {1}", + response.json["message"], + ) + self.assert_model_exists( + "user/111", + { + "default_password": None, + }, + ) + + self.set_user_groups(self.user_id, [1, 5]) + response = self.request( + "user.update", + { + "id": 111, + "default_password": "new_one", + }, + ) + self.assert_status_code(response, 403) + self.assert_model_exists( + "user/111", + { + "default_password": None, + }, + ) + + self.update_model("group/1", {"permissions": ["user.can_update"]}) + response = self.request( + "user.update", + { + "id": 111, + "default_password": "new_one", + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/111", + { + "default_password": "new_one", + }, + ) + + self.set_user_groups(self.user_id, [2, 5]) + response = self.request( + "user.update", + { + "id": 111, + "default_password": "newer_one", + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/111", + { + "default_password": "newer_one", + }, + ) + + self.set_committee_management_level([60], self.user_id) + response = self.request( + "user.update", + { + "id": 111, + "default_password": "newest_one", + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/111", + { + "default_password": "newest_one", + }, + ) + + def test_perm_group_F_with_two_meeting_across_committees(self) -> None: + """ + May not update group F fields unless requesting user has admin rights in + all of requested users meetings. Also requested user can be admin himself. + """ + self.permission_setup() + self.create_meeting(4) + self.set_user_groups(111, [1, 4]) + self.set_user_groups(self.user_id, [2, 4]) + + response = self.request( + "user.update", + { + "id": 111, + "default_password": "new_one", + }, + ) + self.assert_status_code(response, 403) + self.assertIn( + "You are not allowed to perform action user.update. Missing permissions: OrganizationManagementLevel can_manage_users in organization 1 or Permission user.can_update in meetings {1, 4}", + response.json["message"], + ) + self.assert_model_exists( + "user/111", + { + "default_password": None, + }, + ) + # assert meeting admin can change normal user + self.set_user_groups(111, [1, 5]) + self.set_user_groups(self.user_id, [2, 5]) + response = self.request( + "user.update", + { + "id": 111, + "default_password": "new_one", + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/111", + { + "default_password": "new_one", + }, + ) + def test_perm_group_B_user_can_update(self) -> None: """update group B fields for 2 meetings with simple user.can_update permissions""" self.permission_setup() diff --git a/tests/system/presenter/base.py b/tests/system/presenter/base.py index 286e68c396..4230f12090 100644 --- a/tests/system/presenter/base.py +++ b/tests/system/presenter/base.py @@ -1,3 +1,4 @@ +from collections import defaultdict from typing import Any from openslides_backend.http.application import OpenSlidesBackendWSGIApplication @@ -26,3 +27,73 @@ def request( if isinstance(response.json, list) and len(response.json) == 1: return (response.status_code, response.json[0]) return (response.status_code, response.json) + + def create_meeting_for_two_users( + self, user1: int, user2: int, base: int = 1 + ) -> None: + """ + Creates meeting with id 1, committee 60 and groups with ids 1, 2, 3 by default. + With base you can setup other meetings, but be cautious because of group-ids + The groups have no permissions and no users by default. + Uses usernumber to create meeting users with the concatenation of base and usernumber. + """ + committee_id = base + 59 + self.set_models( + { + f"meeting/{base}": { + "group_ids": [base, base + 1, base + 2], + "default_group_id": base, + "admin_group_id": base + 1, + "committee_id": committee_id, + "is_active_in_organization_id": 1, + }, + f"group/{base}": { + "meeting_id": base, + "default_group_for_meeting_id": base, + "name": f"group{base}", + }, + f"group/{base+1}": { + "meeting_id": base, + "admin_group_for_meeting_id": base, + "name": f"group{base+1}", + }, + f"group/{base+2}": { + "meeting_id": base, + "name": f"group{base+2}", + }, + f"committee/{committee_id}": { + "organization_id": 1, + "name": f"Commitee{committee_id}", + "meeting_ids": [base], + }, + "organization/1": { + "limit_of_meetings": 0, + "active_meeting_ids": [base], + "enable_electronic_voting": True, + }, + f"meeting_user/{base}{user1}": {"user_id": user1, "meeting_id": base}, + f"meeting_user/{base}{user2}": {"user_id": user2, "meeting_id": base}, + } + ) + + def move_user_to_group(self, meeting_user_to_groups: dict[int, Any]) -> None: + """ + Sets the users groups, returns the meeting_user_ids + Be careful as it does not reset previously set groups if the related meeting + users are not in meeting_user_to_groups. + """ + groups_to_meeting_user = defaultdict(list) + for meeting_user_id, group_id in meeting_user_to_groups.items(): + if group_id: + self.update_model( + f"meeting_user/{meeting_user_id}", {"group_ids": [group_id]} + ) + groups_to_meeting_user[group_id].append(meeting_user_id) + else: + self.update_model( + f"meeting_user/{meeting_user_id}", {"group_ids": None} + ) + for group_id, meeting_user_ids in groups_to_meeting_user.items(): + self.update_model( + f"group/{group_id}", {"meeting_user_ids": meeting_user_ids} + ) diff --git a/tests/system/presenter/test_get_user_editable.py b/tests/system/presenter/test_get_user_editable.py new file mode 100644 index 0000000000..a000997432 --- /dev/null +++ b/tests/system/presenter/test_get_user_editable.py @@ -0,0 +1,515 @@ +from openslides_backend.permissions.management_levels import OrganizationManagementLevel + +from .base import BasePresenterTestCase + + +class TestGetUSerEditable(BasePresenterTestCase): + def set_up(self) -> None: + self.create_model( + "user/111", + { + "username": "Helmhut", + "last_name": "Schmidt", + "is_active": True, + "password": self.auth.hash("Kohl"), + "default_password": "Kohl", + }, + ) + self.login(111) + self.set_models( + { + "meeting/1": { + "committee_id": 2, + "is_active_in_organization_id": 1, + }, + # archived meeting + "meeting/2": { + "committee_id": 2, + "is_active_in_organization_id": None, + "is_archived_in_organization_id": 1, + }, + "committee/1": {}, + "committee/2": {"meeting_ids": [1, 2]}, + "user/2": { + "username": "only_oml_level", + "organization_management_level": OrganizationManagementLevel.CAN_MANAGE_USERS, + }, + "user/3": { + "username": "only_cml_level", + "committee_management_ids": [1], + "meeting_ids": [], + }, + "user/4": { + "username": "cml_and_meeting", + "meeting_ids": [1], + "committee_management_ids": [2], + }, + "user/5": { + "username": "no_organization", + "meeting_ids": [], + }, + "user/6": { + "username": "oml_and_meeting", + "organization_management_level": OrganizationManagementLevel.SUPERADMIN, + "meeting_ids": [1], + }, + "user/7": { + "username": "meeting_and_archived_meeting", + "meeting_ids": [1, 2], + }, + } + ) + + def test_with_oml(self) -> None: + self.set_up() + self.update_model( + "user/111", + { + "organization_management_level": OrganizationManagementLevel.CAN_MANAGE_USERS, + }, + ) + status_code, data = self.request( + "get_user_editable", + { + "user_ids": [2, 3, 4, 5, 6, 7], + "fields": ["first_name", "default_password"], + }, + ) + self.assertEqual(status_code, 200) + self.assertEqual( + data, + { + "2": { + "default_password": [True, ""], + "first_name": [True, ""], + }, + "3": { + "default_password": [True, ""], + "first_name": [True, ""], + }, + "4": { + "default_password": [True, ""], + "first_name": [True, ""], + }, + "5": { + "default_password": [True, ""], + "first_name": [True, ""], + }, + "6": { + "default_password": [ + False, + "Your organization management level is not high enough to change a user with a Level of superadmin!", + ], + "first_name": [ + False, + "Your organization management level is not high enough to change a user with a Level of superadmin!", + ], + }, + "7": { + "default_password": [True, ""], + "first_name": [True, ""], + }, + }, + ) + + def test_with_cml(self) -> None: + self.set_up() + self.update_model( + "user/111", + { + "committee_management_ids": [2], + }, + ) + status_code, data = self.request( + "get_user_editable", + { + "user_ids": [2, 3, 4, 5, 6, 7], + "fields": ["first_name", "default_password"], + }, + ) + self.assertEqual(status_code, 200) + self.assertEqual( + data, + { + "2": { + "default_password": [ + False, + "Your organization management level is not high enough to change a user with a Level of can_manage_users!", + ], + "first_name": [ + False, + "Your organization management level is not high enough to change a user with a Level of can_manage_users!", + ], + }, + "3": { + "default_password": [ + False, + "Missing permissions: OrganizationManagementLevel can_manage_users in organization 1 or CommitteeManagementLevel can_manage in committee 1", + ], + "first_name": [ + False, + "Missing permissions: OrganizationManagementLevel can_manage_users in organization 1 or CommitteeManagementLevel can_manage in committee 1", + ], + }, + "4": { + "default_password": [True, ""], + "first_name": [True, ""], + }, + "5": { + "default_password": [ + False, + "Missing permission: OrganizationManagementLevel can_manage_users in organization 1", + ], + "first_name": [ + False, + "Missing permission: OrganizationManagementLevel can_manage_users in organization 1", + ], + }, + "6": { + "default_password": [ + False, + "Your organization management level is not high enough to change a user with a Level of superadmin!", + ], + "first_name": [ + False, + "Your organization management level is not high enough to change a user with a Level of superadmin!", + ], + }, + "7": { + "default_password": [True, ""], + "first_name": [True, ""], + }, + }, + ) + + def test_with_same_meeting(self) -> None: + """ + User 5 can be edited because he is only in meetings which User 111 is admin of. + User 7 can not be edited because he is in two of the same meetings but User 111 is not admin in all of them. + """ + self.set_up() + self.create_meeting_for_two_users(5, 111, 1) + self.create_meeting_for_two_users(5, 111, 4) + self.create_meeting_for_two_users(7, 111, 7) + self.create_meeting_for_two_users(7, 111, 10) + self.update_model("meeting/1", {"committee_id": 1}) + self.update_model("meeting/4", {"committee_id": 2}) + self.update_model("meeting/7", {"committee_id": 1}) + self.update_model("meeting/10", {"committee_id": 2}) + # User 111 is meeting admin in meeting 1, 4 and 7 but normal user in 10 + # User 5 is normal user in meeting 1 and 4 + # User 7 is normal user in meeting 7 and 10 + meeting_user_to_group = { + 1111: 2, + 4111: 5, + 15: 1, + 45: 4, + 7111: 8, + 10111: 10, + 77: 7, + 107: 10, + } + self.move_user_to_group(meeting_user_to_group) + self.update_model( + "user/5", + { + "meeting_user_ids": [ + 15, + 45, + ], + "meeting_ids": [1, 4], + }, + ) + self.update_model( + "user/7", + { + "meeting_user_ids": [77, 107], + "meeting_ids": [7, 10], + }, + ) + self.update_model( + "user/111", + { + "meeting_user_ids": [1111, 4111, 7111, 10111], + "meeting_ids": [1, 4, 7, 10], + }, + ) + status_code, data = self.request( + "get_user_editable", + { + "user_ids": [5, 7], + "fields": ["first_name", "default_password"], + }, + ) + self.assertEqual(status_code, 200) + self.assertEqual( + data, + { + "5": {"default_password": [True, ""], "first_name": [True, ""]}, + "7": { + "default_password": [ + False, + "Missing permissions: OrganizationManagementLevel can_manage_users in organization 1 or Permission user.can_update in meetings {10, 7}", + ], + "first_name": [ + False, + "Missing permissions: OrganizationManagementLevel can_manage_users in organization 1 or Permission user.can_update in meetings {10, 7}", + ], + }, + }, + ) + + def test_with_same_meeting_can_update(self) -> None: + """ + User 5 can be edited because he is only in meetings which User 111 has can_manage of. + User 7 can be edited because he is only in meetings which User 111 has can_update of. + """ + self.set_up() + self.create_meeting_for_two_users(5, 111, 1) + self.create_meeting_for_two_users(5, 111, 4) + self.create_meeting_for_two_users(7, 111, 7) + self.create_meeting_for_two_users(7, 111, 10) + self.update_model("meeting/1", {"committee_id": 1}) + self.update_model("meeting/4", {"committee_id": 2}) + self.update_model("meeting/7", {"committee_id": 1}) + self.update_model("meeting/10", {"committee_id": 2}) + self.update_model("group/3", {"permissions": ["user.can_update"]}) + self.update_model("group/6", {"permissions": ["user.can_update"]}) + self.update_model("group/9", {"permissions": ["user.can_manage"]}) + self.update_model("group/12", {"permissions": ["user.can_manage"]}) + # User 111 has sufficient group rights in meeting 1, 4, 7 and 10 + # User 5 is normal user in meeting 1 and 4 + # User 7 is normal user in meeting 7 and 10 + meeting_user_to_group = { + 1111: 3, + 4111: 6, + 15: 1, + 45: 4, + 7111: 9, + 10111: 12, + 77: 7, + 107: 10, + } + self.move_user_to_group(meeting_user_to_group) + self.update_model( + "user/5", + { + "meeting_user_ids": [ + 15, + 45, + ], + "meeting_ids": [1, 4], + }, + ) + self.update_model( + "user/7", + { + "meeting_user_ids": [77, 107], + "meeting_ids": [7, 10], + }, + ) + self.update_model( + "user/111", + { + "meeting_user_ids": [1111, 4111, 7111, 10111], + "meeting_ids": [1, 4, 7, 10], + }, + ) + status_code, data = self.request( + "get_user_editable", + { + "user_ids": [5, 7], + "fields": ["first_name", "default_password"], + }, + ) + self.assertEqual(status_code, 200) + self.assertEqual( + data, + { + "5": {"default_password": [True, ""], "first_name": [True, ""]}, + "7": {"default_password": [True, ""], "first_name": [True, ""]}, + }, + ) + + def test_with_all_payload_groups(self) -> None: + """ + Tests all user.create/update payload field groups. Especially the field 'saml_id'. + """ + self.set_up() + self.update_model( + "user/111", + { + "organization_management_level": OrganizationManagementLevel.CAN_MANAGE_USERS, + }, + ) + status_code, data = self.request( + "get_user_editable", + { + "user_ids": [2, 3, 4, 5, 6, 7], + "fields": [ + "first_name", + "default_password", + "vote_weight", + "group_ids", + "committee_management_ids", + "organization_management_level", + "is_demo_user", + "saml_id", + ], + }, + ) + self.assertEqual(status_code, 200) + self.assertEqual( + data, + { + "2": { + "committee_management_ids": [True, ""], + "default_password": [True, ""], + "first_name": [True, ""], + "group_ids": [True, ""], + "is_demo_user": [ + False, + "Missing OrganizationManagementLevel: superadmin", + ], + "organization_management_level": [True, ""], + "saml_id": [ + False, + "The field 'saml_id' can only be used in internal action calls", + ], + "vote_weight": [True, ""], + }, + "3": { + "committee_management_ids": [True, ""], + "default_password": [True, ""], + "first_name": [True, ""], + "group_ids": [True, ""], + "is_demo_user": [ + False, + "Missing OrganizationManagementLevel: superadmin", + ], + "organization_management_level": [True, ""], + "saml_id": [ + False, + "The field 'saml_id' can only be used in internal action calls", + ], + "vote_weight": [True, ""], + }, + "4": { + "committee_management_ids": [True, ""], + "default_password": [True, ""], + "first_name": [True, ""], + "group_ids": [True, ""], + "is_demo_user": [ + False, + "Missing OrganizationManagementLevel: superadmin", + ], + "organization_management_level": [True, ""], + "saml_id": [ + False, + "The field 'saml_id' can only be used in internal action calls", + ], + "vote_weight": [True, ""], + }, + "5": { + "committee_management_ids": [True, ""], + "default_password": [True, ""], + "first_name": [True, ""], + "group_ids": [True, ""], + "is_demo_user": [ + False, + "Missing OrganizationManagementLevel: superadmin", + ], + "organization_management_level": [True, ""], + "saml_id": [ + False, + "The field 'saml_id' can only be used in internal action calls", + ], + "vote_weight": [True, ""], + }, + "6": { + "committee_management_ids": [ + False, + "Your organization management level is not high enough to change a user with a Level of superadmin!", + ], + "default_password": [ + False, + "Your organization management level is not high enough to change a user with a Level of superadmin!", + ], + "first_name": [ + False, + "Your organization management level is not high enough to change a user with a Level of superadmin!", + ], + "group_ids": [True, ""], + "is_demo_user": [ + False, + "Your organization management level is not high enough to change a user with a Level of superadmin!", + ], + "organization_management_level": [ + False, + "Your organization management level is not high enough to change a user with a Level of superadmin!", + ], + "saml_id": [ + False, + "Your organization management level is not high enough to change a user with a Level of superadmin!", + ], + "vote_weight": [True, ""], + }, + "7": { + "committee_management_ids": [True, ""], + "default_password": [True, ""], + "first_name": [True, ""], + "group_ids": [True, ""], + "is_demo_user": [ + False, + "Missing OrganizationManagementLevel: superadmin", + ], + "organization_management_level": [True, ""], + "saml_id": [ + False, + "The field 'saml_id' can only be used in internal action calls", + ], + "vote_weight": [True, ""], + }, + }, + ) + + def test_payload_list_of_None(self) -> None: + status_code, data = self.request( + "get_user_editable", + { + "user_ids": [None], + "fields": [ + "first_name", + "default_password", + ], + }, + ) + self.assertEqual(status_code, 400) + self.assertIn("data.user_ids[0] must be integer", data["message"]) + status_code, data = self.request( + "get_user_editable", {"user_ids": [1, 2], "fields": [None]} + ) + self.assertEqual(status_code, 400) + self.assertIn("data.fields[0] must be string", data["message"]) + + def test_payload_empty_list(self) -> None: + status_code, data = self.request( + "get_user_editable", + { + "user_ids": [], + "fields": [ + "first_name", + "default_password", + ], + }, + ) + self.assertEqual(status_code, 200) + assert data == {} + status_code, data = self.request( + "get_user_editable", {"user_ids": [1, 2], "fields": []} + ) + self.assertEqual(status_code, 400) + assert data == { + "message": "Need at least one field name to check editability.", + "success": False, + } diff --git a/tests/system/presenter/test_get_user_related_models.py b/tests/system/presenter/test_get_user_related_models.py index b4c8846473..4fe09c80b5 100644 --- a/tests/system/presenter/test_get_user_related_models.py +++ b/tests/system/presenter/test_get_user_related_models.py @@ -137,6 +137,62 @@ def test_get_user_related_models_meeting(self) -> None: } } + def test_two_meetings(self) -> None: + user_id = 2 + self.set_models( + { + f"user/{user_id}": { + "username": "executor", + "default_password": "DEFAULT_PASSWORD", + "password": self.auth.hash("DEFAULT_PASSWORD"), + "is_active": True, + "meeting_ids": [1, 4], + }, + f"user/{111}": {"username": "untouchable", "meeting_ids": [1, 4]}, + } + ) + self.create_meeting_for_two_users(user_id, 111) + self.create_meeting_for_two_users(user_id, 111, 4) # meeting 4 + self.set_models( + { + "user/777": {"meeting_user_ids": [666]}, + "meeting_user/666": { + "meeting_id": 1, + "user_id": 777, + }, + } + ) + self.update_model("group/5", {"meeting_user_ids": [666]}) + self.login(user_id) + # Admin groups of meeting/1 for requesting user meeting/2 as normal user + # 111 into both meetings + # 777 additional admin for meeting/2 doesn't affect outcome + meeting_user_to_group = {12: 2, 42: 4, 1111: 1, 4111: 4, 666: 5} + self.move_user_to_group(meeting_user_to_group) + status_code, data = self.request( + "get_user_related_models", {"user_ids": [111, 777]} + ) + self.assertEqual(status_code, 403) + self.assertEqual( + "Missing permissions: OrganizationManagementLevel can_manage_users in organization 1 or Permission user.can_update in meeting 4", + data["message"], + ) + # Admin groups of meeting/1 for requesting user + # 111 into both meetings + self.move_user_to_group({12: 2, 42: None, 1111: 1, 4111: 4}) + status_code, data = self.request("get_user_related_models", {"user_ids": [111]}) + self.assertEqual(status_code, 403) + self.assertEqual( + "Missing permissions: OrganizationManagementLevel can_manage_users in organization 1 or Permission user.can_update in meeting 4", + data["message"], + ) + # Admin groups of meeting/1 and meeting/4 for requesting user + # 111 into both meetings + meeting_user_to_group = {12: 2, 42: 5, 1111: 1, 4111: 4} + self.move_user_to_group(meeting_user_to_group) + status_code, data = self.request("get_user_related_models", {"user_ids": [111]}) + self.assertEqual(status_code, 200) + def test_get_user_related_models_meetings_more_users(self) -> None: self.set_models( { From d0773328e88d6ac46198331e331521d52ddec36c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Nov 2024 15:57:21 +0000 Subject: [PATCH 70/90] Bump werkzeug from 3.0.4 to 3.1.3 in /requirements/partial (#2721) Bumps [werkzeug](https://github.com/pallets/werkzeug) from 3.0.4 to 3.1.3. - [Release notes](https://github.com/pallets/werkzeug/releases) - [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/werkzeug/compare/3.0.4...3.1.3) --- updated-dependencies: - dependency-name: werkzeug dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/partial/requirements_production.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/partial/requirements_production.txt b/requirements/partial/requirements_production.txt index 83ac300a93..9136f005f9 100644 --- a/requirements/partial/requirements_production.txt +++ b/requirements/partial/requirements_production.txt @@ -8,7 +8,7 @@ pypdf[crypto]==5.1.0 requests==2.32.3 roman==4.2 simplejson==3.19.3 -Werkzeug==3.0.4 +Werkzeug==3.1.3 python-magic==0.4.27 pygments==2.18.0 From 5e8f15358525c9e552425f9b412aee1df70d6e91 Mon Sep 17 00:00:00 2001 From: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> Date: Mon, 25 Nov 2024 17:29:29 +0100 Subject: [PATCH 71/90] Remove motion import (#2736) --- docs/actions/motion.import.md | 21 - docs/actions/motion.json_upload.md | 54 - .../action/actions/motion/__init__.py | 2 - .../action/actions/motion/import_.py | 635 --------- .../action/actions/motion/json_upload.py | 599 --------- tests/system/action/motion/test_import.py | 1161 ----------------- .../system/action/motion/test_json_upload.py | 974 -------------- 7 files changed, 3446 deletions(-) delete mode 100644 docs/actions/motion.import.md delete mode 100644 docs/actions/motion.json_upload.md delete mode 100644 openslides_backend/action/actions/motion/import_.py delete mode 100644 openslides_backend/action/actions/motion/json_upload.py delete mode 100644 tests/system/action/motion/test_import.py delete mode 100644 tests/system/action/motion/test_json_upload.py diff --git a/docs/actions/motion.import.md b/docs/actions/motion.import.md deleted file mode 100644 index d93d50c8ee..0000000000 --- a/docs/actions/motion.import.md +++ /dev/null @@ -1,21 +0,0 @@ -## Payload -```js -{ -// required - id: Id; // action worker id - import: boolean; -} -``` - - -## Action -If `import` is `True`, use the rows from the given action worker and check that the import type -matches and whether it should still be created (row state `new`) or update (row state `done`). -On row state `new`, the username must not exist yet. On row state `done`, -the record with the matching `id` should still have the same username. On error, don't import anything, -but create data as in json_upload. Do the actual import with bulk actions. - -If `import` is `False` or the import was successful, remove the action worker. - -## Permission -The request user needs permission `motion.can_manage`, but only allow importing data if there are no errors in preview. \ No newline at end of file diff --git a/docs/actions/motion.json_upload.md b/docs/actions/motion.json_upload.md deleted file mode 100644 index eae90a8ac2..0000000000 --- a/docs/actions/motion.json_upload.md +++ /dev/null @@ -1,54 +0,0 @@ -## Payload - -Because the data fields are all converted from CSV import file, **they are all of type `string`**. -The types noted below are the internal types after conversion in the backend. See [here](preface_special_imports.md#internal-types) for the representation of the types. -```js -{ - // required for new motions - data: { - // required in create - title: string, // info: done, error - text: string, // info: done, error - // all optional, but see rules below - number: string, // unique when set, info: done, generated or error - reason: string, // required for create if the meeting has "motions_reason_required", info: done or error - submitters_verbose: string[], - submitters_username: string[], // info: done, generated, warning, error if len(submitters_verbose) > len(submitters_username) - supporters_verbose: string[], - supporters_username: string[], // info: done, warning, error if len(supporters_verbose) > len(supporters_username) - category_name: string, // info: done or warning, partial reference to: motion_category - category_prefix: string, - tags: string[], // info: done or warning, reference to: tag - block: string, // info: done or warning, reference to: motion_block - motion_amendment: boolean, // info: done or warning, if True, warning, that motion amendments cannot be imported - }[]; - meeting_id: Id, // id of the current meeting. -} -``` -## Return value and object fields - -Besides the usual headers as seen in payload (name and type), there are these differences: - -- `submitters`, `supporter_users`, `category_name/prefix`, `tags` and `block`: Objects that show if the model has been found (`done`) or not (`warning`). -- `text`: will be surrounded in html `

` tags if the string isn't encased in html tags already. -- `number`: will be object with error, if the field is set and another row in the payload has the same number. If the `number` field is left empty and the motion is going to be created in a `motion_state` where `set_number` is true, a new `number` will be generated and the object is going to have the info `generated`. - -The row state can be one of "new", "done" or "error". In case of an error, no import should be possible. - -See [common description](preface_special_imports.md#general-format-of-the-result-send-to-the-client-for-preview). - -Other than the validity check for the username-fields, `submitters_verbose` and `supporters_verbose` are NOT otherwise used or taken note of in the import. They are merely accepted in order to check that someone didn't accidentally edit the wrong column in a file that has both verbose and non-verbose columns. -They are not included in the return value. - - -## Action -The data will create or update motions. - -### Motion matching - -Motions can be updated via their `number`. -If a motion has a `number`, it will be matched with and updated with the data of any import date that has the same `number`. -Therefore motions that don't have a number can not be overwritten. - -## Permission -Permission `motion.can_manage` \ No newline at end of file diff --git a/openslides_backend/action/actions/motion/__init__.py b/openslides_backend/action/actions/motion/__init__.py index f4b731bfe1..9f504ec2a3 100644 --- a/openslides_backend/action/actions/motion/__init__.py +++ b/openslides_backend/action/actions/motion/__init__.py @@ -3,8 +3,6 @@ create_forwarded, delete, follow_recommendation, - import_, - json_upload, reset_recommendation, reset_state, set_recommendation, diff --git a/openslides_backend/action/actions/motion/import_.py b/openslides_backend/action/actions/motion/import_.py deleted file mode 100644 index 3e17e5d6c6..0000000000 --- a/openslides_backend/action/actions/motion/import_.py +++ /dev/null @@ -1,635 +0,0 @@ -from typing import Any, cast - -from openslides_backend.action.mixins.import_mixins import ( - BaseImportAction, - ImportRow, - ImportState, - Lookup, - ResultType, -) -from openslides_backend.action.util.register import register_action -from openslides_backend.permissions.permissions import Permissions -from openslides_backend.shared.exceptions import ActionException -from openslides_backend.shared.filters import And, Filter, FilterOperator, Or - -from ....models.models import ImportPreview -from ....shared.patterns import fqid_from_collection_and_id -from ....shared.schema import required_id_schema -from ...util.default_schema import DefaultSchema -from ..meeting_user.create import MeetingUserCreate -from ..motion_submitter.create import MotionSubmitterCreateAction -from ..motion_submitter.delete import MotionSubmitterDeleteAction -from ..motion_submitter.sort import MotionSubmitterSort -from .create import MotionCreate -from .payload_validation_mixin import ( - MotionActionErrorData, - MotionCreatePayloadValidationMixin, - MotionErrorType, - MotionUpdatePayloadValidationMixin, -) -from .update import MotionUpdate - - -@register_action("motion.import") -class MotionImport( - BaseImportAction, - MotionCreatePayloadValidationMixin, - MotionUpdatePayloadValidationMixin, -): - """ - Action to import a result from the import_preview. - """ - - model = ImportPreview() - schema = DefaultSchema(ImportPreview()).get_default_schema( - additional_required_fields={ - "id": required_id_schema, - "import": {"type": "boolean"}, - } - ) - permission = Permissions.Motion.CAN_MANAGE - skip_archived_meeting_check = True - import_name = "motion" - number_lookup: Lookup - username_lookup: dict[str, list[dict[str, Any]]] - category_lookup: dict[str, list[dict[str, Any]]] - tags_lookup: dict[str, list[dict[str, Any]]] - block_lookup: dict[str, list[dict[str, Any]]] - _user_ids_to_meeting_user: dict[int, Any] - _submitter_ids_to_user_id: dict[int, int] - - def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: - if not instance["import"]: - return {} - - instance = super().update_instance(instance) - meeting_id = self.get_meeting_id(instance) - self.setup_lookups(meeting_id) - - self.rows = [self.validate_entry(row) for row in self.result["rows"]] - - if self.import_state != ImportState.ERROR: - create_action_payload: list[dict[str, Any]] = [] - update_action_payload: list[dict[str, Any]] = [] - submitter_create_action_payload: list[dict[str, Any]] = [] - submitter_delete_action_payload: list[dict[str, Any]] = [] - - motion_to_submitter_user_ids: dict[int, list[int]] = {} - old_submitters: dict[int, dict[int, int]] = ( - {} - ) # {motion_id: {user_id:submitter_id}} - for row in self.rows: - payload: dict[str, Any] = row["data"].copy() - used_list = ["text", "reason", "title", "number"] - for field in used_list: - if field in payload: - if type(dvalue := payload[field]) is dict: - payload[field] = dvalue["value"] - self.remove_fields_from_data( - payload, - ["submitters_verbose", "supporters_verbose", "motion_amendment"], - ) - if (category := payload.pop("category_name", None)) and category[ - "info" - ] == ImportState.DONE: - payload["category_id"] = ( - category["id"] if category.get("id") else None - ) - if (block := payload.pop("block", None)) and block[ - "info" - ] == ImportState.DONE: - payload["block_id"] = block["id"] if block.get("id") else None - payload["tag_ids"] = self.get_ids_from_object_list( - payload.pop("tags", []) - ) - meeting_users_to_create = [ - {"user_id": submitter["id"], "meeting_id": meeting_id} - for submitter in payload["submitters_username"] - if submitter["info"] == ImportState.GENERATED - and submitter["id"] not in self._user_ids_to_meeting_user.keys() - ] - if len(meeting_users_to_create): - meeting_users = cast( - list[dict[str, int]], - self.execute_other_action( - MeetingUserCreate, meeting_users_to_create - ), - ) - for i in range(len(meeting_users)): - self._user_ids_to_meeting_user[ - meeting_users_to_create[i]["user_id"] - ] = meeting_users[i] - submitters = self.get_ids_from_object_list( - payload.pop("submitters_username") - ) - supporters = [ - self._user_ids_to_meeting_user[supporter_id]["id"] - for supporter_id in self.get_ids_from_object_list( - payload.pop("supporters_username", []) - ) - ] - payload["supporter_meeting_user_ids"] = supporters - payload.pop("category_prefix", None) - errors: list[MotionActionErrorData] = [] - if row["state"] == ImportState.NEW: - payload.update({"submitter_ids": submitters}) - create_action_payload.append(payload) - errors = self.get_create_payload_integrity_error_message( - payload, meeting_id - ) - else: - id_ = payload["id"] - motion_to_submitter_user_ids[id_] = submitters - motion = { - k: v - for k, v in ( - [ - motion - for motion in self.number_lookup.name_to_ids.get( - payload["number"], [] - ) - if motion.get("id") - ][0] - ).items() - } - for field in ["category_id", "block_id"]: - if payload.get(field) is None: - if not motion.get(field): - payload.pop(field, None) - if len(submitters): - motion_submitter_ids: list[int] = ( - motion.get("submitter_ids", []) or [] - ) - matched_submitters = { - self._submitter_ids_to_user_id[submitter_id]: submitter_id - for submitter_id in motion_submitter_ids - if self._submitter_ids_to_user_id.get(submitter_id) - in submitters - } - submitter_create_action_payload.extend( - [ - { - "meeting_user_id": self._user_ids_to_meeting_user[ - user_id - ]["id"], - "motion_id": id_, - } - for user_id in submitters - if user_id not in matched_submitters.keys() - ] - ) - submitter_delete_action_payload.extend( - [ - {"id": submitter_id} - for submitter_id in motion_submitter_ids - if submitter_id not in matched_submitters.values() - ] - ) - old_submitters[id_] = matched_submitters - - payload.pop("meeting_id", None) - update_action_payload.append(payload) - errors = self.get_update_payload_integrity_error_message( - payload, meeting_id - ) - for err in errors: - if err["type"] != MotionErrorType.REASON: - raise ActionException("Error: " + err["message"]) - if not ( - row["data"].get("reason") - and isinstance(row["data"]["reason"], dict) - ): - row["data"]["reason"] = { - "value": row["data"].get("reason", ""), - "info": ImportState.ERROR, - } - else: - row["data"]["reason"]["info"] = ImportState.ERROR - row["data"]["reason"].pop("id", 0) - row["messages"].append("Error: " + err["message"]) - row["state"] = ImportState.ERROR - self.import_state = ImportState.ERROR - if self.import_state != ImportState.ERROR: - created_submitters: list[dict[str, int]] = [] - if create_action_payload: - self.execute_other_action(MotionCreate, create_action_payload) - if update_action_payload: - self.execute_other_action(MotionUpdate, update_action_payload) - if len(submitter_create_action_payload): - created_submitters = cast( - list[dict[str, int]], - self.execute_other_action( - MotionSubmitterCreateAction, submitter_create_action_payload - ), - ) - if len(submitter_delete_action_payload): - self.execute_other_action( - MotionSubmitterDeleteAction, submitter_delete_action_payload - ) - new_submitters: dict[int, dict[int, int]] = ( - {} - ) # {motion_id: {meeting_user_id:submitter_id}} - for i in range(len(created_submitters)): - motion_id = submitter_create_action_payload[i]["motion_id"] - new_submitters[motion_id] = { - **new_submitters.get(motion_id, {}), - submitter_create_action_payload[i][ - "meeting_user_id" - ]: created_submitters[i]["id"], - } - sort_payload: list[dict[str, Any]] = [] - for motion_id in motion_to_submitter_user_ids: - sorted_motion_submitter_ids: list[int] = [] - for submitter_user_id in motion_to_submitter_user_ids[motion_id]: - meeting_user_id = cast( - int, self._user_ids_to_meeting_user[submitter_user_id]["id"] - ) - if ( - submitter_user_id - in old_submitters.get(motion_id, {}).keys() - ): - sorted_motion_submitter_ids.append( - old_submitters[motion_id][submitter_user_id] - ) - elif ( - meeting_user_id in new_submitters.get(motion_id, {}).keys() - ): - sorted_motion_submitter_ids.append( - new_submitters[motion_id][meeting_user_id] - ) - else: - raise Exception( - f"Submitter sorting failed due to submitter for user/{submitter_user_id} not being found" - ) - if len(sorted_motion_submitter_ids): - sort_payload.append( - { - "motion_id": motion_id, - "motion_submitter_ids": sorted_motion_submitter_ids, - } - ) - for payload in sort_payload: - self.execute_other_action(MotionSubmitterSort, [payload]) - - return {} - - def get_ids_from_object_list(self, object_list: list[dict[str, Any]]) -> list[int]: - return [ - obj["id"] - for obj in object_list - if obj.get("info") != ImportState.WARNING - and obj.get("info") != ImportState.ERROR - ] - - def remove_fields_from_data( - self, data: dict[str, Any], fieldnames: list[str] - ) -> None: - for fieldname in fieldnames: - data.pop(fieldname, None) - - def validate_entry(self, row: ImportRow) -> ImportRow: - entry = row["data"] - - if ("id" in entry) != ("id" in entry.get("number", {})): - raise ActionException( - f"Invalid JsonUpload data: A data row with state '{ImportState.DONE}' must have an 'id'" - ) - - number = self.get_value_from_union_str_object(entry.get("number")) - if number: - check_result = self.number_lookup.check_duplicate(number) - id_ = cast(int, self.number_lookup.get_field_by_name(number, "id")) - - if check_result == ResultType.FOUND_ID and id_ != 0: - if row["state"] != ImportState.DONE: - row["messages"].append( - f"Error: Row state expected to be '{ImportState.DONE}', but it is '{row['state']}'." - ) - row["state"] = ImportState.ERROR - entry["number"]["info"] = ImportState.ERROR - elif entry["id"] != id_: - row["state"] = ImportState.ERROR - entry["number"]["info"] = ImportState.ERROR - row["messages"].append( - f"Error: Number '{number}' found in different id ({id_} instead of {entry['id']})" - ) - elif check_result == ResultType.FOUND_MORE_IDS: - row["state"] = ImportState.ERROR - entry["number"]["info"] = ImportState.ERROR - row["messages"].append( - f"Error: Number '{number}' is duplicated in import." - ) - elif check_result == ResultType.NOT_FOUND_ANYMORE: - row["messages"].append( - f"Error: Motion {entry['number']['id']} not found anymore for updating motion '{number}'." - ) - row["state"] = ImportState.ERROR - - category_name = self.get_value_from_union_str_object(entry.get("category_name")) - if category_name and entry["category_name"].get("info") == ImportState.DONE: - category_prefix = entry.get("category_prefix") or None - if "id" not in entry["category_name"]: - raise ActionException( - f"Invalid JsonUpload data: A category_name entry with state '{ImportState.DONE}' must have an 'id'" - ) - categories = self.category_lookup.get(category_name, []) - categories = [ - category - for category in categories - if category.get("prefix") == category_prefix - ] - if len(categories) > 0: - if not any( - [ - category.get("id") == entry["category_name"].get("id") - for category in categories - ] - ): - row["messages"].append( - "Error: Category search didn't deliver the same result as in the preview" - ) - entry["category_name"] = { - "value": category_name, - "info": ImportState.ERROR, - } - row["state"] = ImportState.ERROR - else: - entry["category_name"] = { - "value": category_name, - "info": ImportState.ERROR, - } - row["state"] = ImportState.ERROR - row["messages"].append("Error: Category could not be found anymore") - - block = self.get_value_from_union_str_object(entry.get("block")) - if block and entry["block"].get("info") == ImportState.DONE: - if "id" not in entry["block"]: - raise ActionException( - f"Invalid JsonUpload data: A block entry with state '{ImportState.DONE}' must have an 'id'" - ) - found_blocks = self.block_lookup.get(block, []) - if len(found_blocks) > 0: - if not any( - [block.get("id") == entry["block"]["id"] for block in found_blocks] - ): - entry["block"] = {"value": block, "info": ImportState.ERROR} - row["messages"].append( - "Error: Motion block search didn't deliver the same result as in the preview" - ) - row["state"] = ImportState.ERROR - else: - entry["block"] = { - "value": block, - "info": ImportState.ERROR, - } - row["messages"].append("Error: Couldn't find motion block anymore") - row["state"] = ImportState.ERROR - - if isinstance(entry.get("tags"), list): - different: list[str] = [] - not_found: list[str] = [] - for tag_entry in entry.get("tags", []): - tag = self.get_value_from_union_str_object(tag_entry) - if tag and tag_entry.get("info") == ImportState.DONE: - if "id" not in tag_entry: - raise ActionException( - f"Invalid JsonUpload data: A tag entry with state '{ImportState.DONE}' must have an 'id'" - ) - found_tags = self.tags_lookup.get(tag, []) - if len(found_tags) > 0: - if not any( - [tag.get("id") == tag_entry["id"] for tag in found_tags] - ): - tag_entry["info"] = ImportState.ERROR - tag_entry.pop("id") - different.append(tag) - else: - tag_entry["info"] = ImportState.ERROR - tag_entry.pop("id") - not_found.append(tag) - if len(different): - row["messages"].append( - "Error: Tag search didn't deliver the same result as in the preview: " - + ", ".join(different) - ) - row["state"] = ImportState.ERROR - if len(not_found): - row["messages"].append( - "Error: Couldn't find tag anymore: " + ", ".join(not_found) - ) - row["state"] = ImportState.ERROR - - for fieldname in ["submitter", "supporter"]: - if isinstance(entry.get(f"{fieldname}s_username"), list): - different = [] - not_found = [] - for user_entry in entry.get(f"{fieldname}s_username", []): - user = self.get_value_from_union_str_object(user_entry) - if user and ( - user_entry.get("info") == ImportState.DONE - or user_entry.get("info") == ImportState.GENERATED - ): - if "id" not in user_entry: - raise ActionException( - f"Invalid JsonUpload data: A {fieldname} entry with state '{ImportState.DONE}' or '{ImportState.GENERATED}' must have an 'id'" - ) - found_users = self.username_lookup.get(user, []) - if len(found_users) == 1: - if found_users[0].get("id") != user_entry["id"]: - user_entry["info"] = ImportState.ERROR - user_entry.pop("id") - different.append(user) - elif len(found_users) > 1: - raise ActionException( - f"Database corrupt: Found multiple users with the username {user}." - ) - else: - user_entry["info"] = ImportState.ERROR - user_entry.pop("id") - not_found.append(user) - if len(different): - row["messages"].append( - f"Error: {fieldname[0].capitalize() + fieldname[1:]} search didn't deliver the same result as in the preview: " - + ", ".join(different) - ) - row["state"] = ImportState.ERROR - if len(not_found): - row["messages"].append( - f"Error: Couldn't find {fieldname} anymore: " - + ", ".join(not_found) - ) - row["state"] = ImportState.ERROR - - row["messages"] = list(set(row["messages"])) - - if row["state"] == ImportState.ERROR and self.import_state == ImportState.DONE: - self.import_state = ImportState.ERROR - return { - "state": row["state"], - "data": row["data"], - "messages": row.get("messages", []), - } - - def setup_lookups(self, meeting_id: int) -> None: - rows = self.result["rows"] - self.number_lookup = Lookup( - self.datastore, - "motion", - [ - (entry["number"]["value"], entry) - for row in rows - if "number" in (entry := row["data"]) - and entry["number"].get("info") != ImportState.WARNING - ], - field="number", - mapped_fields=["submitter_ids", "category_id", "block_id"], - global_and_filter=FilterOperator("meeting_id", "=", meeting_id), - ) - self.block_lookup = self.get_lookup_dict( - "motion_block", - [ - entry["block"]["value"] - for row in rows - if "block" in (entry := row["data"]) - and entry["block"].get("info") != ImportState.WARNING - ], - "title", - and_filters=[FilterOperator("meeting_id", "=", meeting_id)], - ) - self.category_lookup = self.get_lookup_dict( - "motion_category", - [ - entry["category_name"]["value"] - for row in rows - if "category_name" in (entry := row["data"]) - and entry["category_name"].get("info") != ImportState.WARNING - ], - "name", - ["prefix"], - and_filters=[FilterOperator("meeting_id", "=", meeting_id)], - ) - - self.username_lookup = self.get_lookup_dict( - "user", - list( - { - user["value"] - for row in rows - if ( - users := [ - *row["data"].get("submitters_username", []), - *row["data"].get("supporters_username", []), - ] - ) - for user in users - if user and user.get("info") != ImportState.WARNING - } - ), - "username", - ["meeting_ids", "meeting_user_ids"], - ) - self.username_lookup = { - username: [ - date - for date in self.username_lookup[username] - if ( - date.get("meeting_ids") - and (meeting_id in date["meeting_ids"]) - or date.get("id") == self.user_id - ) - ] - for username in self.username_lookup - } - all_user_ids = list( - [ - submitter["id"] - for submitters in self.username_lookup.values() - for submitter in submitters - ] - ) - all_meeting_users: dict[int, dict[str, Any]] = {} - if len(all_user_ids): - all_meeting_users = self.datastore.filter( - "meeting_user", - And( - FilterOperator("meeting_id", "=", meeting_id), - Or( - *[ - FilterOperator("user_id", "=", user_id) - for user_id in all_user_ids - ] - ), - ), - [ - "id", - "user_id", - "motion_submitter_ids", - "supported_motion_ids", - ], - lock_result=False, - ) - self._user_ids_to_meeting_user = { - all_meeting_users[meeting_user_id]["user_id"]: all_meeting_users[ - meeting_user_id - ] - for meeting_user_id in all_meeting_users - if all_meeting_users[meeting_user_id].get("user_id") - } - self._submitter_ids_to_user_id = { - submitter_id: all_meeting_users[meeting_user_id]["user_id"] - for meeting_user_id in all_meeting_users - for submitter_id in ( - all_meeting_users[meeting_user_id].get("motion_submitter_ids", []) or [] - ) - if all_meeting_users[meeting_user_id].get("user_id") - } - self.tags_lookup = self.get_lookup_dict( - "tag", - [ - tag["value"] - for row in rows - if "tags" in (entry := row["data"]) - for tag in entry["tags"] - if tag.get("info") != ImportState.WARNING - ], - "name", - and_filters=[FilterOperator("meeting_id", "=", meeting_id)], - ) - - def get_lookup_dict( - self, - collection: str, - entries: list[str], - fieldname: str = "name", - mapped_fields: list[str] = [], - and_filters: list[Filter] = [], - ) -> dict[str, list[dict[str, Any]]]: - lookup: dict[str, list[dict[str, Any]]] = {} - if len(entries): - data = self.datastore.filter( - collection, - And( - *and_filters, - Or([FilterOperator(fieldname, "=", name) for name in set(entries)]), - ), - [*mapped_fields, "id", fieldname], - lock_result=False, - ) - for date_id in data: - date = data[date_id] - lookup[date[fieldname]] = [ - *lookup.get(date[fieldname], []), - date, - ] - return lookup - - def get_meeting_id(self, instance: dict[str, Any]) -> int: - store_id = instance["id"] - worker = self.datastore.get( - fqid_from_collection_and_id("import_preview", store_id), - ["name", "result"], - lock_result=False, - ) - if worker.get("name") == self.import_name: - return next(iter(worker.get("result", {})["rows"]))["data"]["meeting_id"] - raise ActionException("Import data cannot be found.") diff --git a/openslides_backend/action/actions/motion/json_upload.py b/openslides_backend/action/actions/motion/json_upload.py deleted file mode 100644 index fcb5741a41..0000000000 --- a/openslides_backend/action/actions/motion/json_upload.py +++ /dev/null @@ -1,599 +0,0 @@ -from collections import defaultdict -from collections.abc import Iterable -from re import search, sub -from typing import Any, cast - -from openslides_backend.shared.filters import And, Filter, FilterOperator, Or - -from ....models.models import Motion -from ....permissions.permissions import Permissions -from ....shared.exceptions import ActionException -from ....shared.schema import required_id_schema -from ...mixins.import_mixins import ( - BaseJsonUploadAction, - ImportState, - Lookup, - ResultType, -) -from ...util.default_schema import DefaultSchema -from ...util.register import register_action -from .payload_validation_mixin import ( - MotionActionErrorData, - MotionCreatePayloadValidationMixin, - MotionErrorType, - MotionUpdatePayloadValidationMixin, -) - -LIST_TYPE = { - "anyOf": [ - { - "type": "array", - "items": {"type": "string"}, - }, - {"type": "string"}, - ] -} - - -@register_action("motion.json_upload") -class MotionJsonUpload( - BaseJsonUploadAction, - MotionCreatePayloadValidationMixin, - MotionUpdatePayloadValidationMixin, -): - """ - Action to allow to upload a json. It is used as first step of an import. - """ - - model = Motion() - schema = DefaultSchema(Motion()).get_default_schema( - additional_required_fields={ - "data": { - "type": "array", - "items": { - "type": "object", - "properties": { - **model.get_properties( - "title", - "text", - "number", - "reason", - ), - **{ - "submitters_verbose": LIST_TYPE, - "submitters_username": LIST_TYPE, - "supporters_verbose": LIST_TYPE, - "supporters_username": LIST_TYPE, - "category_name": {"type": "string"}, - "category_prefix": {"type": "string"}, - "tags": LIST_TYPE, - "block": {"type": "string"}, - "motion_amendment": {"type": "boolean"}, - }, - }, - "required": [], - "additionalProperties": False, - }, - "minItems": 1, - "uniqueItems": False, - }, - "meeting_id": required_id_schema, - } - ) - - headers = [ - {"property": "title", "type": "string", "is_object": True}, - {"property": "text", "type": "string", "is_object": True}, - {"property": "number", "type": "string", "is_object": True}, - {"property": "reason", "type": "string", "is_object": True}, - { - "property": "submitters_verbose", - "type": "string", - "is_list": True, - "is_hidden": True, - }, - { - "property": "submitters_username", - "type": "string", - "is_object": True, - "is_list": True, - }, - { - "property": "supporters_verbose", - "type": "string", - "is_list": True, - "is_hidden": True, - }, - { - "property": "supporters_username", - "type": "string", - "is_object": True, - "is_list": True, - }, - {"property": "category_name", "type": "string", "is_object": True}, - {"property": "category_prefix", "type": "string"}, - {"property": "tags", "type": "string", "is_object": True, "is_list": True}, - {"property": "block", "type": "string", "is_object": True}, - { - "property": "motion_amendment", - "type": "boolean", - "is_object": True, - "is_hidden": True, - }, - ] - permission = Permissions.Motion.CAN_MANAGE - row_state: ImportState - number_lookup: Lookup - username_lookup: dict[str, list[dict[str, Any]]] = {} - category_lookup: dict[str, list[dict[str, Any]]] = {} - tags_lookup: dict[str, list[dict[str, Any]]] = {} - block_lookup: dict[str, list[dict[str, Any]]] = {} - _first_state_id: int | None = None - _operator_username: str | None = None - _previous_numbers: list[str] - import_name = "motion" - - def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: - # transform instance into a correct create/update payload - # try to find a pre-existing motion with the same number - # if there is one, validate for a motion.update, otherwise for a motion.create - # using get_update_payload_integrity_error_message and get_create_payload_integrity_error_message - - data = instance.pop("data") - data = self.add_payload_index_to_action_data(data) - self.setup_lookups(data, instance["meeting_id"]) - - # enrich data with meeting_id - for entry in data: - entry["meeting_id"] = instance["meeting_id"] - - self._previous_numbers = [] - self.rows = [self.validate_entry(entry) for entry in data] - - # generate statistics - self.generate_statistics() - return {} - - def validate_entry(self, entry: dict[str, Any]) -> dict[str, Any]: - messages: list[str] = [] - id_: int | None = None - meeting_id: int = entry["meeting_id"] - set_entry_id = False - - if (is_amendment := entry.get("motion_amendment")) is not None: - entry["motion_amendment"] = { - "value": is_amendment, - "info": ImportState.DONE, - } - if is_amendment: - entry["motion_amendment"]["info"] = ImportState.WARNING - messages.append("Amendments cannot be correctly imported") - - if category_name := entry.get("category_name"): - category_prefix = entry.get("category_prefix") - categories = self.category_lookup.get(category_name, []) - categories = [ - category - for category in categories - if category.get("prefix") == category_prefix - ] - if len(categories) == 1 and categories[0].get("id") != 0: - entry["category_name"] = { - "value": category_name, - "info": ImportState.DONE, - "id": categories[0].get("id"), - } - else: - entry["category_name"] = { - "value": category_name, - "info": ImportState.WARNING, - } - messages.append("Category could not be found") - elif category_prefix := entry.get("category_prefix"): - entry["category_name"] = {"value": "", "info": ImportState.WARNING} - messages.append("Category could not be found") - - if number := entry.get("number"): - check_result = self.number_lookup.check_duplicate(number) - id_ = cast(int, self.number_lookup.get_field_by_name(number, "id")) - if check_result == ResultType.FOUND_ID and id_ != 0: - self.row_state = ImportState.DONE - set_entry_id = True - entry["number"] = { - "value": number, - "info": ImportState.DONE, - "id": id_, - } - elif check_result == ResultType.NOT_FOUND or id_ == 0: - self.row_state = ImportState.NEW - entry["number"] = { - "value": number, - "info": ImportState.DONE, - } - elif check_result == ResultType.FOUND_MORE_IDS: - self.row_state = ImportState.ERROR - entry["number"] = { - "value": number, - "info": ImportState.ERROR, - } - messages.append("Error: Found multiple motions with the same number") - else: - category_id: int | None = None - if entry.get("category_name"): - category_id = entry["category_name"].get("id") - self.row_state = ImportState.NEW - value: dict[str, Any] = {} - self.set_number( - value, - meeting_id, - self._get_first_workflow_state_id(meeting_id), - None, - category_id, - other_forbidden_numbers=self._previous_numbers, - ) - if number := value.get("number"): - entry["number"] = {"value": number, "info": ImportState.GENERATED} - self._previous_numbers.append(number) - - has_submitter_error: bool = False - for fieldname in ["submitter", "supporter"]: - if users := entry.get(f"{fieldname}s_username"): - verbose = entry.get(f"{fieldname}s_verbose", []) - verbose_user_mismatch = len(verbose) > len(users) - username_set: set[str] = set() - entry_list: list[dict[str, Any]] = [] - duplicates: set[str] = set() - not_found: set[str] = set() - for user in users: - if verbose_user_mismatch: - entry_list.append({"value": user, "info": ImportState.ERROR}) - elif user in username_set: - entry_list.append({"value": user, "info": ImportState.WARNING}) - duplicates.add(user) - else: - username_set.add(user) - found_users = self.username_lookup.get(user, []) - if len(found_users) == 1 and found_users[0].get("id") != 0: - user_id = cast(int, found_users[0].get("id")) - entry_list.append( - { - "value": user, - "info": ImportState.DONE, - "id": user_id, - } - ) - elif len(found_users) <= 1: - entry_list.append( - { - "value": user, - "info": ImportState.WARNING, - } - ) - not_found.add(user) - else: - raise ActionException( - f"Database corrupt: Found multiple users with the username {user}" - ) - entry[f"{fieldname}s_username"] = entry_list - if verbose_user_mismatch: - self.row_state = ImportState.ERROR - messages.append( - f"Error: Verbose field is set and has more entries than the username field for {fieldname}s" - ) - if fieldname == "submitter": - has_submitter_error = True - if len(duplicates): - messages.append( - f"At least one {fieldname} has been referenced multiple times: " - + ", ".join(duplicates) - ) - if len(not_found): - messages.append( - f"Could not find at least one {fieldname}: " - + ", ".join(not_found) - ) - - if not has_submitter_error: - if ( - len(cast(list[dict[str, Any]], entry.get("submitters_username", []))) - == 0 - ): - entry["submitters_username"] = [self._get_self_username_object()] - elif ( - len( - [ - entry - for entry in ( - cast( - list[dict[str, Any]], - entry.get("submitters_username", []), - ) - ) - if entry.get("info") and (entry["info"] != ImportState.WARNING) - ] - ) - == 0 - ): - entry["submitters_username"].append(self._get_self_username_object()) - - if tags := entry.get("tags"): - entry_list = [] - duplicates = set() - not_found = set() - multiple: set[str] = set() - tags_set: set[str] = set() - for tag in tags: - if tag in tags_set: - entry_list.append({"value": tag, "info": ImportState.WARNING}) - duplicates.add(tag) - else: - tags_set.add(tag) - found_tags = self.tags_lookup.get(tag, []) - if len(found_tags) == 1 and found_tags[0].get("id") != 0: - tag_id = cast(int, found_tags[0].get("id")) - entry_list.append( - { - "value": tag, - "info": ImportState.DONE, - "id": tag_id, - } - ) - elif len(found_tags) <= 1: - entry_list.append( - { - "value": tag, - "info": ImportState.WARNING, - } - ) - not_found.add(tag) - else: - entry_list.append( - { - "value": tag, - "info": ImportState.WARNING, - } - ) - multiple.add(tag) - entry["tags"] = entry_list - if len(duplicates): - messages.append( - "At least one tag has been referenced multiple times: " - + ", ".join(duplicates) - ) - if len(not_found): - messages.append( - "Could not find at least one tag: " + ", ".join(not_found) - ) - if len(multiple): - messages.append( - "Found multiple tags with the same name: " + ", ".join(multiple) - ) - - if (block := entry.get("block")) and isinstance(block, str): - found_blocks = self.block_lookup.get(block, []) - if len(found_blocks) == 1 and found_blocks[0].get("id") != 0: - block_id = cast(int, found_blocks[0].get("id")) - entry["block"] = { - "value": block, - "info": ImportState.DONE, - "id": block_id, - } - elif len(found_blocks) <= 1: - entry["block"] = { - "value": block, - "info": ImportState.WARNING, - } - messages.append("Could not find motion block") - else: - entry["block"] = { - "value": block, - "info": ImportState.WARNING, - } - messages.append("Found multiple motion blocks with the same name") - - if id_ and set_entry_id: - entry["id"] = id_ - - if (text := entry.get("text")) and not search( - r"^<\w+[^>]*>[\w\W]*?<\/\w>$", text - ): - entry["text"] = ( - "

" - + sub(r"\n", "
", sub(r"\n([ \t]*\n)+", "

", text)) - + "

" - ) - - for field in ["title", "text", "reason"]: - if (date := entry.get(field)) and isinstance(date, str): - if date == "": - del entry[field] - else: - entry[field] = {"value": date, "info": ImportState.DONE} - - # check via mixin - payload = { - **{ - k: v.get("value") - for k, v in entry.items() - if k in ["title", "text", "number", "reason"] - }, - **{ - k: self._get_field_ids(entry, v) - for k, v in { - "submitter_ids": "submitters_username", - "supporter_meeting_user_ids": "supporters_username", - "tag_ids": "tags", - }.items() - }, - **{ - k: self._get_field_id(entry, v) - for k, v in { - "category_id": "category_name", - "block_id": "block", - }.items() - if entry.get(v) - }, - } - - errors: list[MotionActionErrorData] = [] - if id_: - payload = {"id": id_, **payload} - errors = self.get_update_payload_integrity_error_message( - payload, meeting_id - ) - else: - payload = {"meeting_id": meeting_id, **payload} - errors = self.get_create_payload_integrity_error_message( - payload, meeting_id - ) - - for err in errors: - entry = self._add_error_to_entry(entry, err) - messages.append("Error: " + err["message"]) - - return {"state": self.row_state, "messages": messages, "data": entry} - - def setup_lookups(self, data: Iterable[dict[str, Any]], meeting_id: int) -> None: - self.number_lookup = Lookup( - self.datastore, - "motion", - [(number, entry) for entry in data if (number := entry.get("number"))], - field="number", - mapped_fields=[], - global_and_filter=FilterOperator("meeting_id", "=", meeting_id), - ) - self.block_lookup = self.get_lookup_dict( - "motion_block", - [title for entry in data if (title := entry.get("block"))], - "title", - and_filters=[FilterOperator("meeting_id", "=", meeting_id)], - ) - self.category_lookup = self.get_lookup_dict( - "motion_category", - [name for entry in data if (name := entry.get("category_name"))], - "name", - ["prefix"], - and_filters=[FilterOperator("meeting_id", "=", meeting_id)], - ) - self.username_lookup = self.get_lookup_dict( - "user", - list( - { - username - for entry in data - if ( - usernames := [ - *entry.get("submitters_username", []), - *entry.get("supporters_username", []), - ] - ) - for username in usernames - if username - } - ), - "username", - ["meeting_ids"], - ) - self.username_lookup = { - username: [ - date - for date in self.username_lookup[username] - if date.get("meeting_ids") and (meeting_id in date["meeting_ids"]) - ] - for username in self.username_lookup - } - self.tags_lookup = self.get_lookup_dict( - "tag", - [ - name - for entry in data - if (names := entry.get("tags")) - for name in names - if name - ], - "name", - and_filters=[FilterOperator("meeting_id", "=", meeting_id)], - ) - - def get_lookup_dict( - self, - collection: str, - entries: list[str], - fieldname: str = "name", - mapped_fields: list[str] = [], - and_filters: list[Filter] = [], - ) -> dict[str, list[dict[str, Any]]]: - lookup: dict[str, list[dict[str, Any]]] = defaultdict(list) - if len(entries): - data = self.datastore.filter( - collection, - And( - *and_filters, - Or([FilterOperator(fieldname, "=", name) for name in set(entries)]), - ), - [*mapped_fields, "id", fieldname], - lock_result=False, - ) - for date in data.values(): - lookup[date[fieldname]].append(date) - return lookup - - def _get_self_username_object(self) -> dict[str, Any]: - if not self._operator_username: - user = self.datastore.get("user/" + str(self.user_id), ["username"]) - if not (user and user.get("username")): - raise ActionException("Couldn't find operator's username") - self._operator_username = cast(str, user["username"]) - return { - "value": self._operator_username, - "info": ImportState.GENERATED, - "id": self.user_id, - } - - def _get_first_workflow_state_id(self, meeting_id: int) -> int: - if not self._first_state_id: - default_workflows = self.datastore.filter( - "motion_workflow", - FilterOperator("default_workflow_meeting_id", "=", meeting_id), - mapped_fields=["first_state_id"], - ).values() - if len(default_workflows) != 1: - raise ActionException("Couldn't determine default workflow") - self._first_state_id = cast( - int, list(default_workflows)[0].get("first_state_id") - ) - return self._first_state_id - - def _get_field_ids(self, entry: dict[str, Any], fieldname: str) -> list[int]: - value = entry.get(fieldname, []) - if not isinstance(value, list): - value = [entry[fieldname]] - return [val["id"] for val in value if val.get("id")] - - def _get_field_id(self, entry: dict[str, Any], fieldname: str) -> int: - return entry[fieldname].get("id") - - def _add_error_to_entry( - self, entry: dict[str, Any], err: MotionActionErrorData - ) -> dict[str, Any]: - fieldname = "" - match err["type"]: - case MotionErrorType.UNIQUE_NUMBER: - fieldname = "number" - case MotionErrorType.TEXT: - fieldname = "text" - case MotionErrorType.REASON: - fieldname = "reason" - case MotionErrorType.TITLE: - fieldname = "title" - case _: - raise ActionException("Error: " + err["message"]) - if not (entry.get(fieldname) and isinstance(entry[fieldname], dict)): - entry[fieldname] = { - "value": entry.get(fieldname, ""), - "info": ImportState.ERROR, - } - else: - entry[fieldname]["info"] = ImportState.ERROR - self.row_state = ImportState.ERROR - return entry diff --git a/tests/system/action/motion/test_import.py b/tests/system/action/motion/test_import.py deleted file mode 100644 index da996334e5..0000000000 --- a/tests/system/action/motion/test_import.py +++ /dev/null @@ -1,1161 +0,0 @@ -from openslides_backend.action.mixins.import_mixins import ImportState - -from .test_json_upload import MotionJsonUploadForUseInImport - - -class MotionImport(MotionJsonUploadForUseInImport): - def test_import_database_corrupt(self) -> None: - self.create_meeting(42) - self.set_models( - { - "import_preview/2": { - "state": ImportState.DONE, - "name": "motion", - "result": { - "rows": [ - { - "state": (ImportState.DONE), - "messages": [], - "data": { - "title": { - "value": "New", - "info": ImportState.DONE, - }, - "text": { - "value": "Motion", - "info": ImportState.DONE, - }, - "reason": {"value": "", "info": ImportState.DONE}, - "supporters_username": [], - "category_name": { - "value": "", - "info": ImportState.DONE, - }, - "tags": [], - "block": {"value": "", "info": ImportState.DONE}, - "number": { - "info": ImportState.DONE, - "id": 101, - "value": "NUM01", - }, - "submitters_username": [ - { - "info": ImportState.DONE, - "id": 12345678, - "value": "bob", - } - ], - "id": 101, - "meeting_id": 42, - }, - } - ], - }, - }, - "user/12345678": { - "username": "bob", - "meeting_ids": [42], - "meeting_user_ids": [12345678], - }, - "meeting_user/12345678": {}, - "user/123456789": { - "username": "bob", - "meeting_ids": [42], - "meeting_user_ids": [123456789], - }, - "meeting_user/123456789": {}, - "meeting/42": {"meeting_user_ids": [12345678, 123456789]}, - } - ) - response = self.request("motion.import", {"id": 2, "import": True}) - self.assert_status_code(response, 400) - assert ( - "Database corrupt: Found multiple users with the username bob." - in response.json["message"] - ) - - def test_import_abort(self) -> None: - self.create_meeting(42) - self.set_models( - { - "import_preview/2": { - "state": ImportState.DONE, - "name": "motion", - "result": { - "rows": [ - { - "state": ImportState.NEW, - "messages": [], - "data": { - "title": { - "value": "New", - "info": ImportState.DONE, - }, - "text": { - "value": "Motion", - "info": ImportState.DONE, - }, - "number": {"value": "", "info": ImportState.DONE}, - "reason": {"value": "", "info": ImportState.DONE}, - "submitters_username": [ - { - "value": "admin", - "info": ImportState.GENERATED, - "id": 1, - } - ], - "supporters_username": [], - "category_name": { - "value": "", - "info": ImportState.DONE, - }, - "tags": [], - "block": {"value": "", "info": ImportState.DONE}, - "meeting_id": 42, - }, - } - ], - }, - }, - } - ) - response = self.request("motion.import", {"id": 2, "import": False}) - self.assert_status_code(response, 200) - self.assert_model_not_exists("import_preview/2") - self.assert_model_not_exists("motion/1") - - def test_import_wrong_import_preview(self) -> None: - self.create_meeting(42) - self.set_models( - { - "import_preview/3": {"result": None}, - "import_preview/4": { - "state": ImportState.DONE, - "name": "topic", - "result": { - "rows": [ - { - "state": ImportState.NEW, - "messages": [], - "data": { - "title": {"value": "test", "info": ImportState.NEW}, - "meeting_id": 42, - }, - }, - ], - }, - }, - } - ) - response = self.request("motion.import", {"id": 3, "import": True}) - self.assert_status_code(response, 400) - assert ( - "Wrong id doesn't point on motion import data." in response.json["message"] - ) - - def test_import_non_existant_ids(self) -> None: - self.create_meeting(42) - preview_rows = [ - { - "state": ImportState.DONE, - "messages": [], - "data": { - "id": 1, - "title": { - "value": "Update", - "info": ImportState.DONE, - }, - "text": { - "value": "

of non-existant motion", - "info": ImportState.DONE, - }, - "number": { - "id": 1, - "value": "NOMNOMNOM1", - "info": ImportState.DONE, - }, - "submitters_username": [ - { - "value": "nonExistantUser", - "info": ImportState.DONE, - "id": 2, - } - ], - "supporters_username": [ - { - "value": "NoOne", - "info": ImportState.DONE, - "id": 3, - } - ], - "category_name": { - "id": 8, - "value": "NonCategory", - "info": ImportState.DONE, - }, - "tags": [ - { - "id": 9, - "value": "NonTag", - "info": ImportState.DONE, - } - ], - "block": {"id": 9, "value": "NonBlock", "info": ImportState.DONE}, - "meeting_id": 42, - }, - }, - { - "state": ImportState.NEW, - "messages": [], - "data": { - "title": { - "value": "Create", - "info": ImportState.DONE, - }, - "text": { - "value": "

a new motion

", - "info": ImportState.DONE, - }, - "number": {"value": "NEW", "info": ImportState.DONE}, - "submitters_username": [ - { - "value": "nonUser", - "info": ImportState.DONE, - "id": 4, - } - ], - "supporters_username": [ - { - "value": "NoOne", - "info": ImportState.DONE, - "id": 5, - } - ], - "category_name": { - "id": 10, - "value": "NonCategoryTwoElectricBoogaloo", - "info": ImportState.DONE, - }, - "tags": [ - { - "id": 11, - "value": "TagNot", - "info": ImportState.DONE, - } - ], - "block": { - "id": 12, - "value": "JustBlockIt", - "info": ImportState.DONE, - }, - "meeting_id": 42, - }, - }, - ] - self.set_models( - { - "import_preview/2": { - "state": ImportState.DONE, - "name": "motion", - "result": { - "rows": preview_rows, - }, - }, - } - ) - response = self.request("motion.import", {"id": 2, "import": True}) - self.assert_status_code(response, 200) - self.assert_model_not_exists("motion/1") - self.assert_model_not_exists("motion/2") - meeting = self.assert_model_exists("meeting/42") - assert "motion_ids" not in meeting - response_rows = response.json["results"][0][0]["rows"] - assert response_rows[0]["data"] == { - "id": 1, - "tags": [{"info": "error", "value": "NonTag"}], - "text": {"info": "done", "value": "

of non-existant motion"}, - "block": {"value": "NonBlock", "info": "error"}, - "title": {"info": "done", "value": "Update"}, - "number": {"id": 1, "info": "error", "value": "NOMNOMNOM1"}, - "meeting_id": 42, - "category_name": {"value": "NonCategory", "info": "error"}, - "submitters_username": [{"info": "error", "value": "nonExistantUser"}], - "supporters_username": [{"info": "error", "value": "NoOne"}], - } - assert sorted(response_rows[0]["messages"]) == sorted( - [ - "Error: Couldn't find motion block anymore", - "Error: Couldn't find supporter anymore: NoOne", - "Error: Couldn't find tag anymore: NonTag", - "Error: Category could not be found anymore", - "Error: Couldn't find submitter anymore: nonExistantUser", - "Error: Motion 1 not found anymore for updating motion 'NOMNOMNOM1'.", - ] - ) - assert response_rows[1]["data"] == { - "tags": [{"info": "error", "value": "TagNot"}], - "text": {"info": "done", "value": "

a new motion

"}, - "block": {"value": "JustBlockIt", "info": "error"}, - "title": {"info": "done", "value": "Create"}, - "number": {"info": "done", "value": "NEW"}, - "meeting_id": 42, - "category_name": { - "value": "NonCategoryTwoElectricBoogaloo", - "info": "error", - }, - "submitters_username": [{"info": "error", "value": "nonUser"}], - "supporters_username": [{"info": "error", "value": "NoOne"}], - } - assert sorted(response_rows[1]["messages"]) == sorted( - [ - "Error: Couldn't find motion block anymore", - "Error: Couldn't find supporter anymore: NoOne", - "Error: Couldn't find tag anymore: TagNot", - "Error: Category could not be found anymore", - "Error: Couldn't find submitter anymore: nonUser", - ] - ) - - def test_import_with_deleted_references(self) -> None: - self.json_upload_multi_row() - self.request("motion.delete", {"id": 100}) - self.request("user.delete", {"id": 2}) - self.request("meeting_user.delete", {"id": 3}) - self.request("motion_category.delete", {"id": 100}) - self.request("motion_category.delete", {"id": 1000}) - self.request("motion_block.delete", {"id": 1}) - self.request("tag.delete", {"id": 1}) - response = self.request("motion.import", {"id": 1, "import": True}) - self.assert_status_code(response, 200) - self.assert_model_exists( - "motion/1", - { - "title": "A", - "text": "

nice little

", - "reason": "motion", - }, - ) - self.assert_model_exists("meeting/1", {"motion_ids": [1]}) - self.assert_model_deleted("motion/100") - self.assert_model_not_exists("motion/101") - self.assert_model_not_exists("motion/102") - self.assert_model_not_exists("motion/103") - rows = response.json["results"][0][0]["rows"] - assert len(rows) == 5 - row = rows[0] - assert row["state"] == ImportState.ERROR - assert row["messages"] == ["Error: Couldn't find supporter anymore: user1"] - assert row["data"]["supporters_username"] == [ - {"info": ImportState.ERROR, "value": "user1"} - ] - row = rows[1] - assert row["state"] == ImportState.ERROR - assert sorted(row["messages"]) == sorted( - [ - "Error: Motion 100 not found anymore for updating motion 'NUM02'.", - "Error: Couldn't find submitter anymore: user1", - "Error: Category could not be found anymore", - ] - ) - assert row["data"]["number"] == { - "id": 100, - "value": "NUM02", - "info": ImportState.ERROR, - } - assert row["data"]["category_name"] == { - "info": ImportState.ERROR, - "value": "Other motion", - } - assert row["data"]["submitters_username"] == [ - {"info": ImportState.ERROR, "value": "user1"} - ] - row = rows[2] - assert row["state"] == ImportState.ERROR - assert row["messages"] == ["Error: Couldn't find tag anymore: Tag1"] - assert row["data"]["tags"] == [{"info": ImportState.ERROR, "value": "Tag1"}] - row = rows[3] - assert row["state"] == ImportState.ERROR - assert sorted(row["messages"]) == sorted( - [ - "Error: Couldn't find supporter anymore: user1, anotherUser", - "Error: Couldn't find motion block anymore", - ] - ) - assert row["data"]["supporters_username"] == [ - {"info": ImportState.ERROR, "value": "user1"}, - {"id": 1, "info": ImportState.DONE, "value": "admin"}, - {"info": ImportState.ERROR, "value": "anotherUser"}, - ] - assert row["data"]["block"] == {"info": ImportState.ERROR, "value": "Block1"} - row = rows[4] - assert row["state"] == ImportState.ERROR - assert row["messages"] == ["Error: Category could not be found anymore"] - assert row["data"]["category_name"] == { - "info": ImportState.ERROR, - "value": "Other motion", - } - - def test_import_with_changed_references(self) -> None: - self.json_upload_multi_row() - self.set_models( - { - "user/2": {"username": "changedName"}, - "user/3": {"username": "changedNameToo"}, - "motion_category/100": {"name": "changedName"}, - "motion_category/1000": {"prefix": "changedPREFIX"}, - "motion_block/1": {"title": "changedTitle"}, - "tag/1": {"name": "changedName"}, - } - ) - self.create_user("anotherUser", [1]) - response = self.request("motion.import", {"id": 1, "import": True}) - self.assert_status_code(response, 200) - rows = response.json["results"][0][0]["rows"] - assert len(rows) == 5 - row = rows[0] - assert row["state"] == ImportState.ERROR - assert row["messages"] == ["Error: Couldn't find supporter anymore: user1"] - assert row["data"]["supporters_username"] == [ - {"info": ImportState.ERROR, "value": "user1"} - ] - row = rows[1] - assert row["state"] == ImportState.ERROR - assert sorted(row["messages"]) == sorted( - [ - "Error: Couldn't find submitter anymore: user1", - "Error: Category could not be found anymore", - ] - ) - assert row["data"]["category_name"] == { - "info": ImportState.ERROR, - "value": "Other motion", - } - assert row["data"]["submitters_username"] == [ - {"info": ImportState.ERROR, "value": "user1"} - ] - row = rows[2] - assert row["state"] == ImportState.ERROR - assert row["messages"] == ["Error: Couldn't find tag anymore: Tag1"] - assert row["data"]["tags"] == [{"info": ImportState.ERROR, "value": "Tag1"}] - row = rows[3] - assert row["state"] == ImportState.ERROR - assert sorted(row["messages"]) == sorted( - [ - "Error: Supporter search didn't deliver the same result as in the preview: anotherUser", - "Error: Couldn't find motion block anymore", - "Error: Couldn't find supporter anymore: user1", - ] - ) - assert row["data"]["supporters_username"] == [ - {"info": ImportState.ERROR, "value": "user1"}, - {"id": 1, "info": ImportState.DONE, "value": "admin"}, - {"info": ImportState.ERROR, "value": "anotherUser"}, - ] - assert row["data"]["block"] == {"info": ImportState.ERROR, "value": "Block1"} - row = rows[4] - assert row["state"] == ImportState.ERROR - assert row["messages"] == ["Error: Category could not be found anymore"] - assert row["data"]["category_name"] == { - "info": ImportState.ERROR, - "value": "Other motion", - } - - def test_import_wrong_meeting_model_import_preview(self) -> None: - self.create_meeting(42) - self.set_models( - { - "import_preview/4": { - "state": ImportState.DONE, - "name": "topic", - "result": { - "rows": [ - { - "state": ImportState.NEW, - "messages": [], - "data": { - "title": {"value": "test", "info": ImportState.NEW}, - "meeting_id": 42, - }, - }, - ], - }, - }, - } - ) - response = self.request("motion.import", {"id": 4, "import": True}) - self.assert_status_code(response, 400) - assert ( - "Wrong id doesn't point on motion import data." in response.json["message"] - ) - - def test_json_upload_amendment(self) -> None: - self.json_upload_amendment() - response = self.request("motion.import", {"id": 1, "import": True}) - self.assert_status_code(response, 200) - assert response.json["results"][0][0]["state"] == ImportState.DONE - - def test_json_upload_with_errors(self) -> None: - self.json_upload_create_missing_title() - response = self.request("motion.import", {"id": 1, "import": True}) - self.assert_status_code(response, 400) - assert "Error in import. Data will not be imported." in response.json["message"] - - def test_json_upload_update_missing_title(self) -> None: - self.json_upload_update_missing_title() - response = self.request("motion.import", {"id": 1, "import": True}) - self.assert_status_code(response, 200) - assert response.json["results"][0][0]["state"] == ImportState.DONE - self.assert_model_exists("motion/42", {"title": "A"}) - - def test_json_upload_update_missing_reason_although_required(self) -> None: - self.json_upload_update_missing_reason_although_required() - response = self.request("motion.import", {"id": 1, "import": True}) - self.assert_status_code(response, 200) - assert response.json["results"][0][0]["state"] == ImportState.DONE - self.assert_model_exists("motion/42", {"reason": "motion"}) - - def test_json_upload_multi_row(self) -> None: - self.json_upload_multi_row() - response = self.request("motion.import", {"id": 1, "import": True}) - self.assert_status_code(response, 200) - self.assert_model_exists( - "motion/1", - { - "title": "first", - "text": "

test my stuff

", - "reason": "motion", # retained from before - "supporter_meeting_user_ids": [1], - "submitter_ids": [4], - }, - ) - self.assert_model_exists("motion_submitter/4", {"meeting_user_id": 2}) - self.assert_model_exists("meeting_user/1", {"user_id": 2}) - self.assert_model_exists("meeting_user/2", {"user_id": 1}) - self.assert_model_exists( - "motion/100", - { - "title": "then", - "text": "

nice little

", # retained from before - "reason": "test my other stuff", - "submitter_ids": [5], - "category_id": 1000, - }, - ) - self.assert_model_exists("motion_submitter/5", {"meeting_user_id": 1}) - self.assert_model_exists( - "motion/101", - { - "number": "NUM03", - "title": "also", - "text": "

test the other peoples stuff

", - "submitter_ids": [1], - "tag_ids": [1], - }, - ) - self.assert_model_exists("motion_submitter/1", {"meeting_user_id": 2}) - self.assert_model_exists( - "motion/102", - { - "number": "03", - "title": "after that", - "text": "

test even more stuff

", - "submitter_ids": [2], - "supporter_meeting_user_ids": [1, 2, 3], - "block_id": 1, - }, - ) - self.assert_model_exists("motion_submitter/2", {"meeting_user_id": 2}) - self.assert_model_exists( - "motion/103", - { - "number": "OTHER01", - "title": "finally", - "text": "

finish testing

", - "submitter_ids": [3], - "category_id": 100, - }, - ) - self.assert_model_exists("motion_submitter/3", {"meeting_user_id": 2}) - - def test_simple_create(self) -> None: - self.json_upload_simple_create() - response = self.request("motion.import", {"id": 1, "import": True}) - self.assert_status_code(response, 200) - motion = self.assert_model_exists( - "motion/4201", - { - "title": "test", - "text": "

my

", - "reason": "stuff", - "submitter_ids": [1], - }, - ) - assert "number" not in motion - self.assert_model_exists("motion_submitter/1", {"meeting_user_id": 2}) - self.assert_model_exists("meeting_user/2", {"user_id": 1}) - - def test_simple_create_with_reason_required(self) -> None: - self.json_upload_simple_create(True) - response = self.request("motion.import", {"id": 1, "import": True}) - self.assert_status_code(response, 200) - motion = self.assert_model_exists( - "motion/4201", - { - "title": "test", - "text": "

my

", - "reason": "stuff", - "submitter_ids": [1], - }, - ) - assert "number" not in motion - self.assert_model_exists("motion_submitter/1", {"meeting_user_id": 2}) - self.assert_model_exists("meeting_user/2", {"user_id": 1}) - - def test_simple_create_with_set_number(self) -> None: - self.json_upload_simple_create(is_set_number=True) - response = self.request("motion.import", {"id": 1, "import": True}) - self.assert_status_code(response, 200) - self.assert_model_exists( - "motion/4201", - { - "number": "03", - "title": "test", - "text": "

my

", - "reason": "stuff", - "submitter_ids": [1], - }, - ) - self.assert_model_exists("motion_submitter/1", {"meeting_user_id": 2}) - self.assert_model_exists("meeting_user/2", {"user_id": 1}) - - def test_simple_update(self) -> None: - self.json_upload_simple_update() - self.assert_simple_update_successful() - - def test_simple_update_with_reason_required(self) -> None: - self.json_upload_simple_update(True) - self.assert_simple_update_successful() - - def test_simple_update_with_set_number(self) -> None: - self.json_upload_simple_update(is_set_number=True) - self.assert_simple_update_successful() - - def assert_simple_update_successful(self) -> None: - response = self.request("motion.import", {"id": 1, "import": True}) - self.assert_status_code(response, 200) - self.assert_model_exists( - "motion/42", - { - "number": "NUM01", - "title": "test", - "text": "

my

", - "reason": "stuff", - "submitter_ids": [1], - }, - ) - self.assert_model_exists("motion_submitter/1", {"meeting_user_id": 2}) - self.assert_model_exists("meeting_user/2", {"user_id": 1}) - - def test_json_upload_update_with_foreign_meeting(self) -> None: - self.json_upload_update_with_foreign_meeting() - response = self.request("motion.import", {"id": 1, "import": True}) - self.assert_status_code(response, 200) - self.assert_model_exists( - "motion/42", - { - "number": "NUM01", - "title": "test", - "text": "

my

", - "reason": "stuff", - "category_id": 42, - "submitter_ids": [1], - "supporter_meeting_user_ids": [1], - }, - ) - self.assert_model_exists("motion_submitter/1", {"meeting_user_id": 3}) - self.assert_model_exists("meeting_user/3", {"user_id": 1}) - - def test_json_upload_custom_number_create(self) -> None: - self.json_upload_custom_number_create() - response = self.request("motion.import", {"id": 1, "import": True}) - self.assert_status_code(response, 200) - self.assert_model_exists( - "motion/4201", - { - "number": "Z01", - "title": "test", - "text": "

my

", - "reason": "stuff", - "category_id": 420, - }, - ) - - def test_json_upload_custom_number_create_with_set_number(self) -> None: - self.json_upload_custom_number_create_with_set_number() - response = self.request("motion.import", {"id": 1, "import": True}) - self.assert_status_code(response, 200) - self.assert_model_exists( - "motion/4201", - { - "number": "Z01", - "title": "test", - "text": "

my

", - "reason": "stuff", - "category_id": 420, - }, - ) - - def test_with_warnings(self) -> None: - self.json_upload_with_warnings() - response = self.request("motion.import", {"id": 1, "import": True}) - self.assert_status_code(response, 200) - motion = self.assert_model_exists("motion/10") - assert "category_id" not in motion - assert motion["submitter_ids"] == [3] - self.assert_model_exists("motion_submitter/3", {"meeting_user_id": 1}) - self.assert_model_exists("meeting_user/1", {"user_id": 2}) - - motion = self.assert_model_exists("motion/1001") - assert "category_id" not in motion - assert motion["submitter_ids"] == [1] - self.assert_model_exists("motion_submitter/1", {"meeting_user_id": 2}) - self.assert_model_exists("meeting_user/2", {"user_id": 1}) - assert motion["supporter_meeting_user_ids"] == [1] - - motion = self.assert_model_exists("motion/1002") - assert "category_id" not in motion - assert motion["submitter_ids"] == [2] - self.assert_model_exists("motion_submitter/2", {"meeting_user_id": 2}) - assert len(motion["supporter_meeting_user_ids"]) == 0 - - def test_with_non_matching_verbose_users_okay(self) -> None: - self.json_upload_with_non_matching_verbose_users_okay() - response = self.request("motion.import", {"id": 1, "import": True}) - self.assert_status_code(response, 200) - self.assert_model_exists( - "motion/123", - { - "number": "NUM01", - "title": "Up", - "text": "

date

", - "submitter_ids": [2, 3], - "supporter_meeting_user_ids": [1, 2], - }, - ) - newMotion = self.assert_model_exists( - "motion/12301", - { - "title": "Newer", - "text": "

motion

", - "submitter_ids": [1], - }, - ) - assert len(newMotion.get("supporter_meeting_user_ids", [])) == 0 - self.assert_model_exists( - "meeting_user/1", {"user_id": 2, "motion_submitter_ids": [2]} - ) - self.assert_model_exists( - "meeting_user/2", {"user_id": 3, "motion_submitter_ids": [3]} - ) - self.assert_model_exists( - "meeting_user/3", {"user_id": 1, "motion_submitter_ids": [1]} - ) - - def test_with_tags_and_blocks(self) -> None: - self.json_upload_with_tags_and_blocks() - response = self.request("motion.import", {"id": 1, "import": True}) - self.assert_status_code(response, 200) - self.assert_model_exists( - "motion/42", - { - "number": "NUM01", - "tag_ids": [1, 2], - "block_id": 1, - }, - ) - self.assert_model_exists("motion/5501", {"tag_ids": [1, 2], "block_id": 2}) - mot3 = self.assert_model_exists("motion/5502", {"tag_ids": []}) - assert "block_id" not in mot3 - mot4 = self.assert_model_exists("motion/5503", {"tag_ids": []}) - assert "block_id" not in mot4 - - def test_with_new_duplicate_tags_and_blocks(self) -> None: - self.json_upload_with_tags_and_blocks() - self.set_models( - { - "tag/7": {"name": "Tag-liatelle", "meeting_id": 42}, - "motion_block/7": {"title": "Blockolade", "meeting_id": 42}, - "meeting/42": { - "tag_ids": [1, 2, 3, 4, 7], - "motion_block_ids": [1, 2, 3, 4, 7], - }, - } - ) - response = self.request("motion.import", {"id": 1, "import": True}) - self.assert_status_code(response, 200) - self.assert_model_exists( - "motion/42", - { - "number": "NUM01", - "tag_ids": [1, 2], - "block_id": 1, - }, - ) - self.assert_model_exists("motion/5501", {"tag_ids": [1, 2], "block_id": 2}) - mot3 = self.assert_model_exists("motion/5502", {"tag_ids": []}) - assert "block_id" not in mot3 - mot4 = self.assert_model_exists("motion/5503", {"tag_ids": []}) - assert "block_id" not in mot4 - - def test_update_with_changed_number(self) -> None: - self.json_upload_simple_update() - self.set_models({"motion/42": {"number": "CHANGED01"}}) - response = self.request("motion.import", {"id": 1, "import": True}) - self.assert_status_code(response, 200) - assert response.json["results"][0][0]["rows"][0]["state"] == ImportState.ERROR - assert response.json["results"][0][0]["rows"][0]["data"]["number"] == { - "id": 42, - "info": ImportState.ERROR, - "value": "NUM01", - } - assert response.json["results"][0][0]["rows"][0]["messages"] == [ - "Error: Motion 42 not found anymore for updating motion 'NUM01'." - ] - - def test_import_with_newly_duplicate_number(self) -> None: - self.setup_meeting_with_settings(5, True, True) - self.set_models( - { - "import_preview/55": { - "state": ImportState.DONE, - "name": "motion", - "result": { - "rows": [ - { - "state": ImportState.NEW, - "messages": [], - "data": { - "title": { - "value": "New", - "info": ImportState.DONE, - }, - "text": { - "value": "Motion", - "info": ImportState.DONE, - }, - "number": { - "info": ImportState.DONE, - "value": "NUM01", - }, - "submitters_username": [ - { - "value": "admin", - "info": ImportState.GENERATED, - "id": 1, - } - ], - "meeting_id": 5, - }, - } - ], - }, - } - } - ) - response = self.request("motion.import", {"id": 55, "import": True}) - self.assert_status_code(response, 200) - assert response.json["results"][0][0]["rows"][0]["state"] == ImportState.ERROR - assert response.json["results"][0][0]["rows"][0]["messages"] == [ - "Error: Row state expected to be 'done', but it is 'new'." - ] - assert ( - response.json["results"][0][0]["rows"][0]["data"]["number"]["info"] - == ImportState.ERROR - ) - - def test_import_without_reason_when_required(self) -> None: - self.setup_meeting_with_settings(5, True, True) - self.set_models( - { - "import_preview/55": { - "state": ImportState.DONE, - "name": "motion", - "result": { - "rows": [ - { - "state": ImportState.NEW, - "messages": [], - "data": { - "title": { - "value": "New", - "info": ImportState.DONE, - }, - "text": { - "value": "Motion", - "info": ImportState.DONE, - }, - "submitters_username": [ - { - "value": "admin", - "info": ImportState.GENERATED, - "id": 1, - } - ], - "meeting_id": 5, - }, - } - ], - }, - } - } - ) - response = self.request("motion.import", {"id": 55, "import": True}) - self.assert_status_code(response, 200) - assert response.json["results"][0][0]["rows"][0]["state"] == ImportState.ERROR - assert response.json["results"][0][0]["rows"][0]["messages"] == [ - "Error: Reason is required" - ] - assert response.json["results"][0][0]["rows"][0]["data"]["reason"] == { - "value": "", - "info": ImportState.ERROR, - } - - def test_update_with_replaced_number(self) -> None: - self.json_upload_simple_update() - self.set_models( - { - "motion/42": {"number": "CHANGED01"}, - "motion/56": { - "meeting_id": 42, - "number": "NUM01", - "title": "Impostor", - "text": "

motion

", - }, - "meeting/42": {"motion_ids": [42, 56, 4200]}, - } - ) - response = self.request("motion.import", {"id": 1, "import": True}) - self.assert_status_code(response, 200) - assert response.json["results"][0][0]["rows"][0]["state"] == ImportState.ERROR - assert response.json["results"][0][0]["rows"][0]["data"]["number"] == { - "id": 42, - "info": ImportState.ERROR, - "value": "NUM01", - } - assert response.json["results"][0][0]["rows"][0]["messages"] == [ - "Error: Number 'NUM01' found in different id (56 instead of 42)" - ] - - def test_update_with_duplicated_number(self) -> None: - self.json_upload_simple_update() - self.set_models( - { - "motion/56": { - "meeting_id": 42, - "number": "NUM01", - "title": "Impostor", - "text": "

motion

", - }, - "meeting/42": {"motion_ids": [42, 56, 4200]}, - } - ) - response = self.request("motion.import", {"id": 1, "import": True}) - self.assert_status_code(response, 200) - assert response.json["results"][0][0]["rows"][0]["state"] == ImportState.ERROR - assert response.json["results"][0][0]["rows"][0]["data"]["number"] == { - "id": 42, - "info": ImportState.ERROR, - "value": "NUM01", - } - assert response.json["results"][0][0]["rows"][0]["messages"] == [ - "Error: Number 'NUM01' is duplicated in import." - ] - - def test_update_with_duplicated_number_2(self) -> None: - self.setup_meeting_with_settings(5, True, True) - row = { - "state": ImportState.DONE, - "messages": [], - "data": { - "id": 5, - "title": { - "value": "New", - "info": ImportState.DONE, - }, - "text": { - "value": "Motion", - "info": ImportState.DONE, - }, - "number": {"value": "NUM01", "info": ImportState.DONE, "id": 5}, - "submitters_username": [ - { - "value": "admin", - "info": ImportState.GENERATED, - "id": 1, - } - ], - "meeting_id": 5, - }, - } - self.set_models( - { - "import_preview/123": { - "state": ImportState.DONE, - "name": "motion", - "result": { - "rows": [row, row.copy()], - }, - } - } - ) - response = self.request("motion.import", {"id": 123, "import": True}) - self.assert_status_code(response, 200) - for i in range(2): - assert ( - response.json["results"][0][0]["rows"][i]["state"] == ImportState.ERROR - ) - assert response.json["results"][0][0]["rows"][i]["data"]["number"] == { - "id": 5, - "info": ImportState.ERROR, - "value": "NUM01", - } - assert response.json["results"][0][0]["rows"][i]["messages"] == [ - "Error: Number 'NUM01' is duplicated in import." - ] - - def test_with_replaced_tags_and_blocks(self) -> None: - self.json_upload_with_tags_and_blocks() - self.set_models( - { - "tag/1": {"name": "Changed"}, - "tag/7": {"name": "Tag-liatelle", "meeting_id": 42}, - "motion_block/1": {"title": "Changed"}, - "motion_block/7": {"title": "Blockolade", "meeting_id": 42}, - "meeting/42": { - "tag_ids": [1, 2, 3, 4, 7], - "motion_block_ids": [1, 2, 3, 4, 7], - }, - } - ) - response = self.request("motion.import", {"id": 1, "import": True}) - self.assert_status_code(response, 200) - assert response.json["results"][0][0]["rows"][0]["state"] == ImportState.ERROR - assert response.json["results"][0][0]["rows"][0]["data"]["tags"] == [ - { - "info": ImportState.ERROR, - "value": "Tag-liatelle", - }, - {"id": 2, "info": ImportState.DONE, "value": "Tag-you're-it"}, - {"value": "Tag-ether", "info": "warning"}, - {"value": "Price tag", "info": "warning"}, - {"value": "Not a tag", "info": "warning"}, - ] - assert response.json["results"][0][0]["rows"][0]["data"]["block"] == { - "info": ImportState.ERROR, - "value": "Blockolade", - } - assert ( - "Error: Tag search didn't deliver the same result as in the preview: Tag-liatelle" - in response.json["results"][0][0]["rows"][0]["messages"] - ) - assert ( - "Error: Motion block search didn't deliver the same result as in the preview" - in response.json["results"][0][0]["rows"][0]["messages"] - ) - - def test_update_with_broken_id_entries(self) -> None: - self.setup_meeting_with_settings(5, True, True) - self.set_models( - { - "import_preview/123": { - "state": ImportState.DONE, - "name": "motion", - "result": { - "rows": [ - { - "id": 5, - "state": ImportState.DONE, - "messages": [], - "data": { - "title": { - "value": "New", - "info": ImportState.DONE, - }, - "text": { - "value": "Motion", - "info": ImportState.DONE, - }, - "number": { - "value": "NUM01", - "info": ImportState.DONE, - "id": 5, - }, - "submitters_username": [ - { - "value": "admin", - "info": ImportState.GENERATED, - "id": 1, - } - ], - "meeting_id": 5, - }, - }, - ], - }, - } - } - ) - response = self.request("motion.import", {"id": 123, "import": True}) - self.assert_status_code(response, 400) - assert ( - "Invalid JsonUpload data: A data row with state 'done' must have an 'id'" - in response.json["message"] - ) - - def test_update_with_broken_id_entries_2(self) -> None: - self.setup_meeting_with_settings(5, True, True) - self.set_models( - { - "import_preview/123": { - "state": ImportState.DONE, - "name": "motion", - "result": { - "rows": [ - { - "state": ImportState.DONE, - "messages": [], - "data": { - "id": 5, - "title": { - "value": "New", - "info": ImportState.DONE, - }, - "text": { - "value": "Motion", - "info": ImportState.DONE, - }, - "number": { - "value": "NUM01", - "info": ImportState.DONE, - }, - "submitters_username": [ - { - "value": "admin", - "info": ImportState.GENERATED, - "id": 1, - } - ], - "meeting_id": 5, - }, - }, - ], - }, - } - } - ) - response = self.request("motion.import", {"id": 123, "import": True}) - self.assert_status_code(response, 400) - assert ( - "Invalid JsonUpload data: A data row with state 'done' must have an 'id'" - in response.json["message"] - ) diff --git a/tests/system/action/motion/test_json_upload.py b/tests/system/action/motion/test_json_upload.py deleted file mode 100644 index 903708ae47..0000000000 --- a/tests/system/action/motion/test_json_upload.py +++ /dev/null @@ -1,974 +0,0 @@ -from openslides_backend.action.mixins.import_mixins import ImportState -from tests.system.action.base import BaseActionTestCase - - -class BaseMotionJsonUpload(BaseActionTestCase): - def setup_meeting_with_settings( - self, - id_: int = 42, - is_reason_required: bool = False, - is_set_number: bool = False, - ) -> None: - self.create_meeting(id_) - self.create_user(f"user{id_}", [id_], None) - self.set_models( - { - f"meeting/{id_}": { - "motions_reason_required": is_reason_required, - "motions_number_type": "per_category", - "motions_number_min_digits": 2, - "motion_ids": [id_, id_ * 100], - "motion_category_ids": [id_, id_ * 10, id_ * 100, id_ * 1000], - }, - f"motion_state/{id_}": {"set_number": is_set_number}, - f"motion/{id_}": { - "title": "A", - "text": "

nice little

", - "reason": "motion", - "meeting_id": id_, - "number": "NUM01", - "number_value": 1, - }, - f"motion/{id_ * 100}": { - "title": "Another", - "text": "

nice little

", - "reason": "motion", - "meeting_id": id_, - "number": "NUM02", - "number_value": 2, - }, - f"motion_category/{id_}": { - "name": "Normal motion", - "prefix": "NORM", - "meeting_id": id_, - }, - f"motion_category/{id_ * 10}": { - "name": "Other motion", - "prefix": "NORM", - "meeting_id": id_, - }, - f"motion_category/{id_ * 100}": { - "name": "Other motion", - "prefix": "OTHER", - "meeting_id": id_, - }, - f"motion_category/{id_ * 1000}": { - "name": "Other motion", - "meeting_id": id_, - }, - } - ) - - -class MotionJsonUpload(BaseMotionJsonUpload): - def test_json_upload_empty_data(self) -> None: - response = self.request( - "motion.json_upload", - {"data": [], "meeting_id": 42}, - ) - self.assert_status_code(response, 400) - assert "data.data must contain at least 1 items" in response.json["message"] - - def test_json_upload_unknown_meeting_id(self) -> None: - self.setup_meeting_with_settings(42) - response = self.request( - "motion.json_upload", - { - "data": [{"title": "test", "reason": "stuff"}], - "meeting_id": 41, - }, - ) - self.assert_status_code(response, 400) - assert "Import tries to use non-existent meeting 41" in response.json["message"] - - def test_json_upload_create_missing_text(self) -> None: - self.setup_meeting_with_settings(42) - response = self.request( - "motion.json_upload", - { - "data": [{"title": "test", "reason": "stuff"}], - "meeting_id": 42, - }, - ) - self.assert_status_code(response, 200) - assert response.json["results"][0][0]["state"] == ImportState.ERROR - assert ( - "Error: Text is required" - in response.json["results"][0][0]["rows"][0]["messages"] - ) - assert response.json["results"][0][0]["rows"][0]["data"]["text"] == { - "value": "", - "info": ImportState.ERROR, - } - - def test_json_upload_create_missing_reason_although_required(self) -> None: - self.setup_meeting_with_settings(42, is_reason_required=True) - response = self.request( - "motion.json_upload", - { - "data": [{"title": "test", "text": "my"}], - "meeting_id": 42, - }, - ) - self.assert_status_code(response, 200) - assert response.json["results"][0][0]["state"] == ImportState.ERROR - assert ( - "Error: Reason is required" - in response.json["results"][0][0]["rows"][0]["messages"] - ) - assert response.json["results"][0][0]["rows"][0]["data"]["reason"] == { - "value": "", - "info": ImportState.ERROR, - } - - def assert_duplicate_numbers(self, number: str) -> None: - self.setup_meeting_with_settings(22) - response = self.request( - "motion.json_upload", - { - "data": [ - { - "number": "NUM01", - "title": "test", - "text": "my", - "reason": "stuff", - }, - { - "number": "NUM01", - "title": "test also", - "text": "

my other

", - "reason": "stuff", - }, - { - "number": "NUM04", - "title": "test", - "text": "my", - "reason": "stuff", - }, - { - "number": "NUM04", - "title": "test also", - "text": "

my other

", - "reason": "stuff", - }, - ], - "meeting_id": 22, - }, - ) - self.assert_status_code(response, 200) - assert len(response.json["results"][0][0]["rows"]) == 4 - result = response.json["results"][0][0] - assert result["state"] == ImportState.ERROR - for i in range(4): - assert result["rows"][i]["state"] == ImportState.ERROR - assert ( - "Error: Found multiple motions with the same number" - in result["rows"][i]["messages"] - ) - assert result["rows"][i]["data"]["number"] == { - "value": number, - "info": ImportState.ERROR, - } - - def test_duplicate_numbers_in_datastore(self) -> None: - self.setup_meeting_with_settings(22) - self.set_models( - { - "motion/23": { - "meeting_id": 22, - "number": "NUM01", - "title": "Title", - "text": "

Text

", - }, - "meeting/22": {"motion_ids": [22, 23, 2200]}, - } - ) - response = self.request( - "motion.json_upload", - { - "data": [ - { - "number": "NUM01", - "title": "test", - "text": "my", - "reason": "stuff", - }, - ], - "meeting_id": 22, - }, - ) - self.assert_status_code(response, 200) - assert len(response.json["results"][0][0]["rows"]) == 1 - result = response.json["results"][0][0] - assert result["state"] == ImportState.ERROR - assert result["rows"][0]["state"] == ImportState.ERROR - assert sorted(result["rows"][0]["messages"]) == sorted( - [ - "Error: Found multiple motions with the same number", - "Error: Number is not unique.", - ] - ) - assert result["rows"][0]["data"]["number"] == { - "value": "NUM01", - "info": ImportState.ERROR, - } - - def test_with_non_matching_verbose_users_with_errors(self) -> None: - self.setup_meeting_with_settings(123) - self.create_user("anotherOne", [123]) - knights = [ - "Sir Lancelot the Brave", - "Sir Galahad the Pure", - "Sir Bedivere the Wise", - "Sir Robin the-not-quite-so-brave-as-Sir-Lancelot", - "Arthur, King of the Britons", - ] - response = self.request( - "motion.json_upload", - { - "data": [ - { - "title": "New", - "text": "motion", - "submitters_username": ["user123", "anotherOne"], - "submitters_verbose": knights, - "supporters_username": ["user123", "anotherOne"], - "supporters_verbose": knights, - }, - ], - "meeting_id": 123, - }, - ) - self.assert_status_code(response, 200) - assert len(response.json["results"][0][0]["rows"]) == 1 - result = response.json["results"][0][0] - assert result["state"] == ImportState.ERROR - row = result["rows"][0] - assert row["state"] == ImportState.ERROR - assert row["messages"] == [ - "Error: Verbose field is set and has more entries than the username field for submitters", - "Error: Verbose field is set and has more entries than the username field for supporters", - ] - assert row["data"]["submitters_username"] == [ - {"info": ImportState.ERROR, "value": "user123"}, - {"info": ImportState.ERROR, "value": "anotherOne"}, - ] - assert row["data"]["supporters_username"] == [ - {"info": ImportState.ERROR, "value": "user123"}, - {"info": ImportState.ERROR, "value": "anotherOne"}, - ] - - -class MotionJsonUploadForUseInImport(BaseMotionJsonUpload): - def json_upload_amendment(self) -> None: - self.create_meeting(42) - response = self.request( - "motion.json_upload", - { - "data": [{"title": "test", "text": "my", "motion_amendment": "1"}], - "meeting_id": 42, - }, - ) - self.assert_status_code(response, 200) - assert len(response.json["results"][0][0]["rows"]) == 1 - assert response.json["results"][0][0]["state"] == ImportState.WARNING - data = { - "meeting_id": 42, - "title": {"value": "test", "info": ImportState.DONE}, - "text": {"value": "

my

", "info": ImportState.DONE}, - "motion_amendment": {"value": True, "info": ImportState.WARNING}, - "submitters_username": [{"id": 1, "info": "generated", "value": "admin"}], - } - expected = { - "state": ImportState.NEW, - "messages": ["Amendments cannot be correctly imported"], - "data": data, - } - assert response.json["results"][0][0]["rows"][0] == expected - - def json_upload_create_missing_title(self) -> None: - self.setup_meeting_with_settings(42) - response = self.request( - "motion.json_upload", - { - "data": [{"text": "my", "reason": "stuff"}], - "meeting_id": 42, - }, - ) - self.assert_status_code(response, 200) - assert response.json["results"][0][0]["state"] == ImportState.ERROR - assert ( - "Error: Title is required" - in response.json["results"][0][0]["rows"][0]["messages"] - ) - - assert response.json["results"][0][0]["rows"][0]["data"]["title"] == { - "value": "", - "info": ImportState.ERROR, - } - - def json_upload_update_missing_title(self) -> None: - self.setup_meeting_with_settings(42) - response = self.request( - "motion.json_upload", - { - "data": [{"text": "my", "reason": "stuff", "number": "NUM01"}], - "meeting_id": 42, - }, - ) - self.assert_status_code(response, 200) - assert response.json["results"][0][0]["state"] == ImportState.DONE - assert response.json["results"][0][0]["rows"][0]["messages"] == [] - assert "title" not in response.json["results"][0][0]["rows"][0]["data"] - - def json_upload_update_missing_reason_although_required(self) -> None: - self.setup_meeting_with_settings(42, is_reason_required=True) - response = self.request( - "motion.json_upload", - { - "data": [{"title": "test", "text": "my", "number": "NUM01"}], - "meeting_id": 42, - }, - ) - self.assert_status_code(response, 200) - assert response.json["results"][0][0]["state"] == ImportState.DONE - assert response.json["results"][0][0]["rows"][0]["messages"] == [] - assert "reason" not in response.json["results"][0][0]["rows"][0]["data"] - - def json_upload_multi_row(self) -> None: - self.setup_meeting_with_settings(1, is_set_number=True) - self.set_user_groups(1, [1]) - self.create_user("anotherUser", [1]) - self.set_models( - { - "meeting/1": {"tag_ids": [1], "motion_block_ids": [1]}, - "tag/1": {"meeting_id": 1, "name": "Tag1"}, - "motion_block/1": {"meeting_id": 1, "title": "Block1"}, - } - ) - response = self.request( - "motion.json_upload", - { - "data": [ - { - "number": "NUM01", - "title": "first", - "text": "test my stuff", - "supporters_username": ["user1"], - }, - { - "number": "NUM02", - "title": "then", - "reason": "test my other stuff", - "category_name": "Other motion", - "submitters_username": ["user1"], - "submitters_verbose": ["Lancelot the brave"], - }, - { - "number": "NUM03", - "title": "also", - "text": "test the other peoples stuff", - "tags": ["Tag1"], - }, - { - "title": "after that", - "text": "test even more stuff", - "supporters_username": ["user1", "admin", "anotherUser"], - "block": "Block1", - }, - { - "title": "finally", - "text": "finish testing", - "category_name": "Other motion", - "category_prefix": "OTHER", - }, - ], - "meeting_id": 1, - }, - ) - self.assert_status_code(response, 200) - rows = response.json["results"][0][0]["rows"] - simple_payload_addition = { - "meeting_id": 1, - "submitters_username": [ - {"id": 1, "info": ImportState.GENERATED, "value": "admin"} - ], - } - assert len(rows) == 5 - row = rows[0] - assert row["state"] == ImportState.DONE - assert row["messages"] == [] - assert row["data"] == { - **simple_payload_addition, - "id": 1, - "number": {"value": "NUM01", "info": ImportState.DONE, "id": 1}, - "title": {"info": ImportState.DONE, "value": "first"}, - "text": {"info": ImportState.DONE, "value": "

test my stuff

"}, - "supporters_username": [ - {"id": 2, "info": ImportState.DONE, "value": "user1"} - ], - } - row = rows[1] - assert row["state"] == ImportState.DONE - assert row["messages"] == [] - assert row["data"] == { - **simple_payload_addition, - "id": 100, - "number": {"value": "NUM02", "info": ImportState.DONE, "id": 100}, - "title": {"info": ImportState.DONE, "value": "then"}, - "reason": {"info": ImportState.DONE, "value": "test my other stuff"}, - "category_name": { - "info": ImportState.DONE, - "value": "Other motion", - "id": 1000, - }, - "submitters_username": [ - {"id": 2, "info": ImportState.DONE, "value": "user1"} - ], - "submitters_verbose": ["Lancelot the brave"], - } - row = rows[2] - assert row["state"] == ImportState.NEW - assert row["messages"] == [] - assert row["data"] == { - **simple_payload_addition, - "number": {"value": "NUM03", "info": ImportState.DONE}, - "title": {"info": ImportState.DONE, "value": "also"}, - "text": { - "info": ImportState.DONE, - "value": "

test the other peoples stuff

", - }, - "tags": [{"id": 1, "info": ImportState.DONE, "value": "Tag1"}], - } - row = rows[3] - assert row["state"] == ImportState.NEW - assert row["messages"] == [] - assert row["data"] == { - **simple_payload_addition, - "number": {"value": "03", "info": ImportState.GENERATED}, - "title": {"info": ImportState.DONE, "value": "after that"}, - "text": {"info": ImportState.DONE, "value": "

test even more stuff

"}, - "supporters_username": [ - {"id": 2, "info": ImportState.DONE, "value": "user1"}, - {"id": 1, "info": ImportState.DONE, "value": "admin"}, - {"id": 3, "info": ImportState.DONE, "value": "anotherUser"}, - ], - "block": {"id": 1, "info": ImportState.DONE, "value": "Block1"}, - } - row = rows[4] - assert row["state"] == ImportState.NEW - assert row["messages"] == [] - assert row["data"] == { - **simple_payload_addition, - "number": {"value": "OTHER01", "info": ImportState.GENERATED}, - "title": {"info": ImportState.DONE, "value": "finally"}, - "text": {"info": ImportState.DONE, "value": "

finish testing

"}, - "category_name": { - "info": ImportState.DONE, - "value": "Other motion", - "id": 100, - }, - "category_prefix": "OTHER", - } - - def json_upload_simple_create( - self, - is_reason_required: bool = False, - is_set_number: bool = False, - ) -> None: - self.setup_meeting_with_settings(42, is_reason_required, is_set_number) - response = self.request( - "motion.json_upload", - { - "data": [{"title": "test", "text": "my", "reason": "stuff"}], - "meeting_id": 42, - }, - ) - self.assert_status_code(response, 200) - assert len(response.json["results"][0][0]["rows"]) == 1 - data = { - "meeting_id": 42, - "title": {"value": "test", "info": ImportState.DONE}, - "text": {"value": "

my

", "info": ImportState.DONE}, - "reason": {"value": "stuff", "info": ImportState.DONE}, - "submitters_username": [{"id": 1, "info": "generated", "value": "admin"}], - } - if is_set_number: - data.update({"number": {"info": ImportState.GENERATED, "value": "03"}}) - expected = { - "state": ImportState.NEW, - "messages": [], - "data": data, - } - assert response.json["results"][0][0]["rows"][0] == expected - - def json_upload_simple_update( - self, - is_reason_required: bool = False, - is_set_number: bool = False, - ) -> None: - self.setup_meeting_with_settings(42, is_reason_required, is_set_number) - response = self.request( - "motion.json_upload", - { - "data": [ - { - "number": "NUM01", - "title": "test", - "text": "my", - "reason": "stuff", - } - ], - "meeting_id": 42, - }, - ) - self.assert_status_code(response, 200) - assert len(response.json["results"][0][0]["rows"]) == 1 - expected = { - "state": ImportState.DONE, - "messages": [], - "data": { - "id": 42, - "meeting_id": 42, - "number": {"id": 42, "value": "NUM01", "info": ImportState.DONE}, - "title": {"value": "test", "info": ImportState.DONE}, - "text": {"value": "

my

", "info": ImportState.DONE}, - "reason": {"value": "stuff", "info": ImportState.DONE}, - "submitters_username": [ - {"id": 1, "info": "generated", "value": "admin"} - ], - }, - } - assert response.json["results"][0][0]["rows"][0] == expected - - def json_upload_update_with_foreign_meeting(self) -> None: - self.setup_meeting_with_settings(42, is_set_number=True) - self.setup_meeting_with_settings(55) - self.create_user("orgaUser") - response = self.request( - "motion.json_upload", - { - "data": [ - { - "number": "NUM01", - "title": "test", - "text": "my", - "reason": "stuff", - "category_name": "Normal motion", - "category_prefix": "NORM", - "submitters_username": ["user55"], - "supporters_username": ["user42", "nonExistant", "orgaUser"], - } - ], - "meeting_id": 42, - }, - ) - self.assert_status_code(response, 200) - assert response.json["results"][0][0]["state"] == ImportState.WARNING - assert response.json["results"][0][0]["headers"] == [ - {"property": "title", "type": "string", "is_object": True}, - {"property": "text", "type": "string", "is_object": True}, - {"property": "number", "type": "string", "is_object": True}, - {"property": "reason", "type": "string", "is_object": True}, - { - "property": "submitters_verbose", - "type": "string", - "is_list": True, - "is_hidden": True, - }, - { - "property": "submitters_username", - "type": "string", - "is_object": True, - "is_list": True, - }, - { - "property": "supporters_verbose", - "type": "string", - "is_list": True, - "is_hidden": True, - }, - { - "property": "supporters_username", - "type": "string", - "is_object": True, - "is_list": True, - }, - {"property": "category_name", "type": "string", "is_object": True}, - {"property": "category_prefix", "type": "string"}, - {"property": "tags", "type": "string", "is_object": True, "is_list": True}, - {"property": "block", "type": "string", "is_object": True}, - { - "property": "motion_amendment", - "type": "boolean", - "is_object": True, - "is_hidden": True, - }, - ] - assert response.json["results"][0][0]["statistics"] == [ - {"name": "total", "value": 1}, - {"name": "created", "value": 0}, - {"name": "updated", "value": 1}, - {"name": "error", "value": 0}, - {"name": "warning", "value": 1}, - ] - assert len(response.json["results"][0][0]["rows"]) == 1 - assert response.json["results"][0][0]["rows"][0]["state"] == ImportState.DONE - messages = response.json["results"][0][0]["rows"][0]["messages"] - assert len(messages) == 2 - assert messages[0] == "Could not find at least one submitter: user55" - assert messages[1].startswith("Could not find at least one supporter:") - assert " nonExistant" in messages[1] - assert " orgaUser" in messages[1] - assert response.json["results"][0][0]["rows"][0]["data"] == { - "number": {"value": "NUM01", "info": ImportState.DONE, "id": 42}, - "title": {"value": "test", "info": ImportState.DONE}, - "text": {"value": "

my

", "info": ImportState.DONE}, - "reason": {"value": "stuff", "info": ImportState.DONE}, - "category_name": { - "value": "Normal motion", - "info": ImportState.DONE, - "id": 42, - }, - "category_prefix": "NORM", - "meeting_id": 42, - "submitters_username": [ - {"value": "user55", "info": ImportState.WARNING}, - {"id": 1, "info": ImportState.GENERATED, "value": "admin"}, - ], - "id": 42, - "supporters_username": [ - {"value": "user42", "info": ImportState.DONE, "id": 2}, - {"value": "nonExistant", "info": ImportState.WARNING}, - {"value": "orgaUser", "info": ImportState.WARNING}, - ], - } - - def json_upload_custom_number_create(self) -> None: - self.assert_custom_number_create() - - def json_upload_custom_number_create_with_set_number(self) -> None: - self.assert_custom_number_create(True) - - def assert_custom_number_create(self, is_set_number: bool = False) -> None: - self.setup_meeting_with_settings(42, is_set_number) - response = self.request( - "motion.json_upload", - { - "data": [ - { - "number": "Z01", - "title": "test", - "text": "my", - "reason": "stuff", - "category_name": "Other motion", - "category_prefix": "NORM", - } - ], - "meeting_id": 42, - }, - ) - self.assert_status_code(response, 200) - rows = response.json["results"][0][0]["rows"] - assert len(rows) == 1 - assert rows[0] == { - "state": ImportState.NEW, - "messages": [], - "data": { - "meeting_id": 42, - "number": {"value": "Z01", "info": ImportState.DONE}, - "title": {"value": "test", "info": ImportState.DONE}, - "text": {"value": "

my

", "info": ImportState.DONE}, - "reason": {"value": "stuff", "info": ImportState.DONE}, - "submitters_username": [ - {"id": 1, "info": "generated", "value": "admin"} - ], - "category_prefix": "NORM", - "category_name": { - "info": ImportState.DONE, - "value": "Other motion", - "id": 420, - }, - }, - } - - def json_upload_with_warnings(self) -> None: - self.setup_meeting_with_settings(10) - self.set_models( - { - "motion_category/100000": { - "name": "Normal motion", - "prefix": "NORM", - "meeting_id": 10, - }, - "meeting/10": {"motion_category_ids": [10, 100, 1000, 10000, 100000]}, - } - ) - response = self.request( - "motion.json_upload", - { - "data": [ - { - "number": "NUM01", - "title": "Up", - "text": "date", - "category_name": "Unknown", - "category_prefix": "CAT", - "submitters_username": ["user10", "user10"], - }, - { - "title": "New", - "text": "motion", - "category_prefix": "Shouldn't be found", - "supporters_username": ["user10", "user10", "user10", "user10"], - }, - { - "title": "Newer", - "text": "motion", - "category_name": "Normal motion", - "category_prefix": "NORM", - "submitters_username": ["nonExistant"], - "supporters_username": ["nonExistant", "nonExistant"], - }, - ], - "meeting_id": 10, - }, - ) - self.assert_status_code(response, 200) - assert len(response.json["results"][0][0]["rows"]) == 3 - result = response.json["results"][0][0] - assert result["state"] == ImportState.WARNING - row = result["rows"][0] - assert row["state"] == ImportState.DONE - assert sorted(row["messages"]) == sorted( - [ - "Category could not be found", - "At least one submitter has been referenced multiple times: user10", - ] - ) - assert row["data"] == { - "number": {"value": "NUM01", "info": ImportState.DONE, "id": 10}, - "title": {"value": "Up", "info": ImportState.DONE}, - "text": {"value": "

date

", "info": ImportState.DONE}, - "category_name": {"value": "Unknown", "info": ImportState.WARNING}, - "category_prefix": "CAT", - "meeting_id": 10, - "submitters_username": [ - {"value": "user10", "info": ImportState.DONE, "id": 2}, - {"value": "user10", "info": ImportState.WARNING}, - ], - "id": 10, - } - row = result["rows"][1] - assert row["state"] == ImportState.NEW - assert sorted(row["messages"]) == sorted( - [ - "Category could not be found", - "At least one supporter has been referenced multiple times: user10", - ] - ) - assert row["data"] == { - "title": {"value": "New", "info": ImportState.DONE}, - "text": {"value": "

motion

", "info": ImportState.DONE}, - "category_name": {"value": "", "info": ImportState.WARNING}, - "category_prefix": "Shouldn't be found", - "meeting_id": 10, - "submitters_username": [ - {"value": "admin", "info": ImportState.GENERATED, "id": 1} - ], - "supporters_username": [ - {"value": "user10", "info": ImportState.DONE, "id": 2}, - {"value": "user10", "info": ImportState.WARNING}, - {"value": "user10", "info": ImportState.WARNING}, - {"value": "user10", "info": ImportState.WARNING}, - ], - } - row = result["rows"][2] - assert row["state"] == ImportState.NEW - assert sorted(row["messages"]) == sorted( - [ - "Category could not be found", - "Could not find at least one submitter: nonExistant", - "At least one supporter has been referenced multiple times: nonExistant", - "Could not find at least one supporter: nonExistant", - ] - ) - assert row["data"] == { - "title": {"value": "Newer", "info": ImportState.DONE}, - "text": {"value": "

motion

", "info": ImportState.DONE}, - "category_name": {"value": "Normal motion", "info": ImportState.WARNING}, - "category_prefix": "NORM", - "meeting_id": 10, - "submitters_username": [ - {"value": "nonExistant", "info": ImportState.WARNING}, - {"value": "admin", "info": ImportState.GENERATED, "id": 1}, - ], - "supporters_username": [ - {"value": "nonExistant", "info": ImportState.WARNING}, - {"value": "nonExistant", "info": ImportState.WARNING}, - ], - } - - def json_upload_with_non_matching_verbose_users_okay(self) -> None: - self.setup_meeting_with_settings(123) - self.create_user("anotherOne", [123]) - knights = [ - "Sir Lancelot the Brave", - "Sir Galahad the Pure", - "Sir Bedivere the Wise", - "Sir Robin the-not-quite-so-brave-as-Sir-Lancelot", - "Arthur, King of the Britons", - ] - response = self.request( - "motion.json_upload", - { - "data": [ - { - "number": "NUM01", - "title": "Up", - "text": "date", - "submitters_username": ["user123", "anotherOne"], - "submitters_verbose": [knights[0]], - "supporters_username": ["user123", "anotherOne"], - "supporters_verbose": [knights[0]], - }, - { - "title": "Newer", - "text": "motion", - "submitters_verbose": knights, - "supporters_verbose": knights, - }, - ], - "meeting_id": 123, - }, - ) - self.assert_status_code(response, 200) - assert len(response.json["results"][0][0]["rows"]) == 2 - result = response.json["results"][0][0] - assert result["state"] == ImportState.DONE - row = result["rows"][0] - assert row["state"] == ImportState.DONE - assert row["messages"] == [] - assert row["data"]["submitters_username"] == [ - {"id": 2, "info": ImportState.DONE, "value": "user123"}, - {"id": 3, "info": ImportState.DONE, "value": "anotherOne"}, - ] - assert row["data"]["supporters_username"] == [ - {"id": 2, "info": ImportState.DONE, "value": "user123"}, - {"id": 3, "info": ImportState.DONE, "value": "anotherOne"}, - ] - row = result["rows"][1] - assert row["state"] == ImportState.NEW - assert row["messages"] == [] - assert row["data"]["submitters_username"] == [ - {"value": "admin", "info": ImportState.GENERATED, "id": 1} - ] - assert row["data"]["submitters_verbose"] == knights - assert "supporters_username" not in row["data"] - assert row["data"]["supporters_verbose"] == knights - - def json_upload_with_tags_and_blocks(self) -> None: - self.setup_meeting_with_settings(42) - self.setup_meeting_with_settings(55) - self.set_models( - { - "meeting/42": { - "tag_ids": [1, 2, 3, 4], - "motion_block_ids": [1, 2, 3, 4], - }, - "meeting/55": {"tag_ids": [5, 6], "motion_block_ids": [5, 6]}, - "tag/1": {"name": "Tag-liatelle", "meeting_id": 42}, - "tag/2": {"name": "Tag-you're-it", "meeting_id": 42}, - "tag/3": {"name": "Tag-ether", "meeting_id": 42}, - "tag/4": {"name": "Tag-ether", "meeting_id": 42}, - "tag/5": {"name": "Tag-you're-it", "meeting_id": 55}, - "tag/6": {"name": "Price tag", "meeting_id": 55}, - "motion_block/1": {"title": "Blockolade", "meeting_id": 42}, - "motion_block/2": {"title": "Blockodile", "meeting_id": 42}, - "motion_block/3": {"title": "Block chain", "meeting_id": 42}, - "motion_block/4": {"title": "Block chain", "meeting_id": 42}, - "motion_block/5": {"title": "Blockodile", "meeting_id": 55}, - "motion_block/6": {"title": "Blockoli", "meeting_id": 55}, - } - ) - response = self.request( - "motion.json_upload", - { - "data": [ - { - "number": "NUM01", - "title": "Up", - "text": "date", - "tags": [ - "Tag-liatelle", - "Tag-you're-it", - "Tag-ether", - "Price tag", - "Not a tag", - ], - "block": "Blockolade", - }, - { - "title": "New", - "text": "motion", - "tags": [ - "Tag-liatelle", - "Tag-liatelle", - "Tag-you're-it", - "Tag-you're-it", - ], - "block": "Blockodile", - }, - {"title": "Newer", "text": "motion", "block": "Block chain"}, - {"title": "Newest", "text": "motion", "block": "Blockoli"}, - ], - "meeting_id": 42, - }, - ) - self.assert_status_code(response, 200) - result = response.json["results"][0][0] - assert result["state"] == ImportState.WARNING - assert len(result["rows"]) == 4 - row = result["rows"][0] - assert row["state"] == ImportState.DONE - assert row["messages"][0].startswith("Could not find at least one tag:") - assert " Not a tag" in row["messages"][0] - assert " Price tag" in row["messages"][0] - assert row["messages"][1] == "Found multiple tags with the same name: Tag-ether" - assert row["data"]["tags"] == [ - {"value": "Tag-liatelle", "info": "done", "id": 1}, - {"value": "Tag-you're-it", "info": "done", "id": 2}, - {"value": "Tag-ether", "info": "warning"}, - {"value": "Price tag", "info": "warning"}, - {"value": "Not a tag", "info": "warning"}, - ] - assert row["data"]["block"] == {"value": "Blockolade", "info": "done", "id": 1} - row = result["rows"][1] - assert row["state"] == ImportState.NEW - assert len(row["messages"]) == 1 - assert row["messages"][0].startswith( - "At least one tag has been referenced multiple times:" - ) - assert "Tag-liatelle" in row["messages"][0] - assert "Tag-you're-it" in row["messages"][0] - assert row["data"]["tags"] == [ - {"value": "Tag-liatelle", "info": "done", "id": 1}, - {"value": "Tag-liatelle", "info": ImportState.WARNING}, - {"value": "Tag-you're-it", "info": "done", "id": 2}, - {"value": "Tag-you're-it", "info": ImportState.WARNING}, - ] - assert row["data"]["block"] == {"value": "Blockodile", "info": "done", "id": 2} - row = result["rows"][2] - assert row["state"] == ImportState.NEW - assert row["messages"] == ["Found multiple motion blocks with the same name"] - assert row["data"]["block"] == { - "value": "Block chain", - "info": ImportState.WARNING, - } - row = result["rows"][3] - assert row["state"] == ImportState.NEW - assert row["messages"] == ["Could not find motion block"] - assert row["data"]["block"] == { - "value": "Blockoli", - "info": ImportState.WARNING, - } From 3ef90521b9e04e51883574226a00b2a278847a87 Mon Sep 17 00:00:00 2001 From: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> Date: Tue, 26 Nov 2024 09:33:21 +0100 Subject: [PATCH 72/90] Update debugpy and datastore commit hash (#2738) --- requirements/export_service_commits.sh | 2 +- requirements/partial/requirements_development.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/export_service_commits.sh b/requirements/export_service_commits.sh index 82a9e372bf..7a2f20256f 100755 --- a/requirements/export_service_commits.sh +++ b/requirements/export_service_commits.sh @@ -1,3 +1,3 @@ #!/bin/bash -export DATASTORE_COMMIT_HASH=827476ebb2f796a2a8ac665d5dbb577e870fc4de +export DATASTORE_COMMIT_HASH=f3ef84416fb3dcf4a949234e6b9b459fdc949d85 export AUTH_COMMIT_HASH=85f5d0e10f7ab455b45f8da73ea7c69ce953e569 diff --git a/requirements/partial/requirements_development.txt b/requirements/partial/requirements_development.txt index 47e94684e5..a602883211 100644 --- a/requirements/partial/requirements_development.txt +++ b/requirements/partial/requirements_development.txt @@ -1,7 +1,7 @@ aiosmtpd==1.4.6 autoflake==2.3.1 black==24.10.0 -debugpy==1.8.8 +debugpy==1.8.9 flake8==7.1.1 isort==5.13.2 mypy==1.13.0 From a2d779e2b0346a565ef376dfd07d8d72a18ada05 Mon Sep 17 00:00:00 2001 From: Hannes Janott Date: Fri, 29 Nov 2024 10:04:03 +0100 Subject: [PATCH 73/90] make database check work again (#2747) --- openslides_backend/presenter/check_database.py | 2 +- openslides_backend/shared/export_helper.py | 6 ++---- tests/system/presenter/test_check_database.py | 2 ++ 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/openslides_backend/presenter/check_database.py b/openslides_backend/presenter/check_database.py index 624e9b2821..b051ee690c 100644 --- a/openslides_backend/presenter/check_database.py +++ b/openslides_backend/presenter/check_database.py @@ -41,7 +41,7 @@ def check_meetings( errors: dict[int, str] = {} for meeting_id in meeting_ids: - export = export_meeting(datastore, meeting_id) + export = export_meeting(datastore, meeting_id, True) try: Checker( data=export, diff --git a/openslides_backend/shared/export_helper.py b/openslides_backend/shared/export_helper.py index 17d987ceb5..f2ccb62e39 100644 --- a/openslides_backend/shared/export_helper.py +++ b/openslides_backend/shared/export_helper.py @@ -165,11 +165,9 @@ def add_users( user["is_present_in_meeting_ids"] = [meeting_id] else: user["is_present_in_meeting_ids"] = None - if not internal_target: + if not internal_target and (gender_id := user.pop("gender_id", None)): gender_dict = datastore.get_all("gender", ["name"], lock_result=False) - if user.get("gender_id"): - user["gender"] = gender_dict.get(user["gender_id"], {}).get("name") - del user["gender_id"] + user["gender"] = gender_dict.get(gender_id, {}).get("name") # limit user fields to exported objects collection_field_tupels = [ ("meeting_user", "meeting_user_ids"), diff --git a/tests/system/presenter/test_check_database.py b/tests/system/presenter/test_check_database.py index c4d1d3257c..d7cebc7780 100644 --- a/tests/system/presenter/test_check_database.py +++ b/tests/system/presenter/test_check_database.py @@ -250,6 +250,7 @@ def get_new_user(self, username: str, datapart: dict[str, Any]) -> dict[str, Any } def test_correct_relations(self) -> None: + """Also asserts that the internal flag of meeting export is used for the gender""" self.set_models( { "organization/1": { @@ -340,6 +341,7 @@ def test_correct_relations(self) -> None: "is_physical_person": True, "default_vote_weight": "1.000000", "organization_id": 1, + "gender_id": 2, }, "user/2": self.get_new_user( "present_user", From eb967ae718f7492a8edd4fe0c65504d95f7b191f Mon Sep 17 00:00:00 2001 From: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com> Date: Tue, 3 Dec 2024 10:11:07 +0100 Subject: [PATCH 74/90] Translate public group name (#2748) --- .../action/actions/meeting/update.py | 12 ++++++++-- tests/system/action/base.py | 1 + tests/system/action/meeting/test_settings.py | 10 ++++++--- tests/system/action/meeting/test_update.py | 22 +++++++++++++++++++ .../action/test_action_command_format.py | 4 ++++ 5 files changed, 44 insertions(+), 5 deletions(-) diff --git a/openslides_backend/action/actions/meeting/update.py b/openslides_backend/action/actions/meeting/update.py index 27528d0020..edb0848e24 100644 --- a/openslides_backend/action/actions/meeting/update.py +++ b/openslides_backend/action/actions/meeting/update.py @@ -4,6 +4,8 @@ CheckUniqueInContextMixin, ) +from ....i18n.translator import Translator +from ....i18n.translator import translate as _ from ....models.models import Meeting from ....permissions.management_levels import ( CommitteeManagementLevel, @@ -214,9 +216,15 @@ def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: set_as_template = instance.pop("set_as_template", None) db_meeting = self.datastore.get( fqid_from_collection_and_id("meeting", instance["id"]), - ["template_for_organization_id", "locked_from_inside", "admin_group_id"], + [ + "template_for_organization_id", + "locked_from_inside", + "admin_group_id", + "language", + ], lock_result=False, ) + Translator.set_translation_language(db_meeting["language"]) lock_meeting = ( instance.get("locked_from_inside") if instance.get("locked_from_inside") is not None @@ -311,7 +319,7 @@ def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: if instance.get("enable_anonymous") and not anonymous_group_id: group_result = self.execute_other_action( GroupCreate, - [{"name": "Public", "weight": 0, "meeting_id": instance["id"]}], + [{"name": _("Public"), "weight": 0, "meeting_id": instance["id"]}], ) instance["anonymous_group_id"] = anonymous_group_id = cast( list[dict[str, Any]], group_result diff --git a/tests/system/action/base.py b/tests/system/action/base.py index 90a638d7df..cd66fb5136 100644 --- a/tests/system/action/base.py +++ b/tests/system/action/base.py @@ -177,6 +177,7 @@ def create_meeting(self, base: int = 1) -> None: "motions_default_workflow_id": base, "committee_id": committee_id, "is_active_in_organization_id": 1, + "language": "en", }, f"group/{base}": { "meeting_id": base, diff --git a/tests/system/action/meeting/test_settings.py b/tests/system/action/meeting/test_settings.py index 4b5c22b156..1174712428 100644 --- a/tests/system/action/meeting/test_settings.py +++ b/tests/system/action/meeting/test_settings.py @@ -8,6 +8,7 @@ def test_group_ids(self) -> None: "meeting/1": { "motion_poll_default_group_ids": [1], "is_active_in_organization_id": 1, + "language": "en", }, "group/1": {"used_as_motion_poll_default_id": 1}, "group/2": {"name": "2", "used_as_motion_poll_default_id": None}, @@ -29,7 +30,8 @@ def test_group_ids(self) -> None: def test_html_field_iframe(self) -> None: self.create_model( - "meeting/1", {"welcome_text": "Hi", "is_active_in_organization_id": 1} + "meeting/1", + {"welcome_text": "Hi", "is_active_in_organization_id": 1, "language": "en"}, ) response = self.request( "meeting.update", {"id": 1, "welcome_text": '", + "number": "number1", + "structure_level_ids": [31], + "about_me": "

about

", + "vote_weight": "1.000000", + "committee_management_ids": [2], + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/223", + { + "committee_management_ids": [2], + "committee_ids": [1, 2], + "meeting_user_ids": [2], + "meeting_ids": [1], + }, + ) + result = response.json["results"][0][0] + assert result == {"id": 223, "meeting_user_id": 2} + self.assert_model_exists( + "meeting_user/2", + { + "group_ids": [11], + "vote_delegations_from_ids": [1], + "comment": "comment<iframe></iframe>", + "number": "number1", + "structure_level_ids": [31], + "about_me": "

about

<iframe></iframe>", + "vote_weight": "1.000000", + }, + ) + self.assert_model_exists("user/222", {"meeting_user_ids": [1]}) + self.assert_model_exists("meeting_user/1", {"vote_delegated_to_id": 2}) + self.assert_model_exists("group/11", {"meeting_user_ids": [2]}) + self.assert_model_exists("meeting/1", {"user_ids": [223]}) + + def test_invalid_committee_management_ids(self) -> None: + self.set_models( + { + "committee/1": {"name": "C1", "meeting_ids": [1]}, + "meeting/1": {"committee_id": 1}, + "user/222": {"meeting_ids": [1]}, + } + ) + response = self.request( + "user.create", + { + "username": "test_Xcdfgee", + "committee_management_ids": [2], + }, + ) + self.assert_status_code(response, 400) + self.assertIn("'committee/2' does not exist.", response.json["message"]) + + def test_invalid_invalid_meeting_for_meeting_user(self) -> None: + self.create_model("meeting/1") + response = self.request( + "user.create", + { + "username": "test_Xcdfgee", + "meeting_id": 2, + "comment": "comment", + "organization_management_level": OrganizationManagementLevel.CAN_MANAGE_USERS, + }, + ) + self.assert_status_code(response, 400) + self.assertIn( + "'meeting/2' does not exist", + response.json["message"], + ) + + def test_create_invalid_group_id(self) -> None: + self.set_models( + { + "committee/1": {"meeting_ids": [1, 2]}, + "meeting/1": {"committee_id": 1}, + "meeting/2": { + "is_active_in_organization_id": ONE_ORGANIZATION_ID, + "committee_id": 1, + }, + "group/11": {"meeting_id": 1}, + } + ) + response = self.request( + "user.create", + { + "username": "test_Xcdfgee", + "meeting_id": 2, + "group_ids": [11], + }, + ) + self.assert_status_code(response, 400) + self.assertIn( + "The following models do not belong to meeting 2: ['group/11']", + response.json["message"], + ) + + def test_create_broken_email(self) -> None: + response = self.request( + "user.create", + { + "username": "test_Xcdfgee", + "email": "broken@@", + }, + ) + self.assert_status_code(response, 400) + assert "email must be valid email." in response.json["message"] + + def test_create_empty_data(self) -> None: + response = self.request("user.create", {}) + self.assert_status_code(response, 400) + assert "Need username or first_name or last_name" in response.json["message"] + + def test_create_wrong_field(self) -> None: + response = self.request( + "user.create", {"wrong_field": "text_AefohteiF8", "username": "test1"} + ) + self.assert_status_code(response, 400) + self.assertIn( + "data must not contain {'wrong_field'} properties", + response.json["message"], + ) + + def test_username_already_exists(self) -> None: + response = self.request( + "user.create", + { + "username": "admin", + "organization_management_level": OrganizationManagementLevel.CAN_MANAGE_USERS, + }, + ) + self.assert_status_code(response, 400) + assert ( + response.json["message"] == "A user with the username admin already exists." + ) + + def test_member_number_already_exists(self) -> None: + response = self.request( + "user.create", + {"username": "user1", "member_number": "14m4m3m832"}, + ) + self.assert_status_code(response, 200) + response = self.request( + "user.create", + {"username": "user2", "member_number": "14m4m3m832"}, + ) + self.assert_status_code(response, 400) + assert ( + response.json["message"] + == "A user with the member_number 14m4m3m832 already exists." + ) + + def test_member_number_none(self) -> None: + response = self.request( + "user.create", + {"username": "user2", "member_number": None}, + ) + self.assert_status_code(response, 200) + self.assert_model_exists("user/2", {"member_number": None}) + + def test_user_create_with_empty_vote_delegation_from_ids(self) -> None: + self.set_models( + { + "meeting/1": {"is_active_in_organization_id": 1}, + } + ) + response = self.request( + "user.create", + { + "username": "testname", + "meeting_id": 1, + "vote_delegations_from_ids": [], + "organization_management_level": OrganizationManagementLevel.CAN_MANAGE_USERS, + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/2", {"username": "testname", "meeting_user_ids": [1]} + ) + self.assert_model_exists( + "meeting_user/1", + {"meeting_id": 1, "user_id": 2, "vote_delegations_from_ids": []}, + ) + + def test_create_committee_manager_without_committee_ids(self) -> None: + """create has to add a missing committee to the user, because cml permission is demanded""" + self.set_models( + { + "committee/60": {"name": "c60"}, + "committee/63": {"name": "c63"}, + } + ) + + response = self.request( + "user.create", + { + "username": "usersname", + "committee_management_ids": [60, 63], + }, + ) + self.assert_status_code(response, 200) + user = self.get_model("user/2") + self.assertCountEqual((60, 63), user["committee_ids"]) + self.assertCountEqual((60, 63), user["committee_management_ids"]) + self.assert_model_exists("committee/60", {"manager_ids": [2], "user_ids": [2]}) + self.assert_model_exists("committee/63", {"manager_ids": [2], "user_ids": [2]}) + + def test_create_empty_username(self) -> None: + response = self.request("user.create", {"username": ""}) + self.assert_status_code(response, 400) + self.assertIn( + "data.username must be longer than or equal to 1 characters", + response.json["message"], + ) + + def test_create_user_without_explicit_scope(self) -> None: + response = self.request("user.create", {"username": "user/2"}) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/2", + { + "meeting_ids": None, + "organization_management_level": None, + "committee_management_ids": None, + }, + ) + + def test_create_strip_spaces(self) -> None: + response = self.request( + "user.create", + { + "first_name": " first name test ", + "last_name": " last name test ", + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/2", + { + "username": "firstnametestlastnametest", + "first_name": "first name test", + "last_name": "last name test", + }, + ) + + def test_create_permission_nothing(self) -> None: + self.permission_setup() + response = self.request( + "user.create", + { + "username": "username", + "meeting_id": 1, + "vote_weight": "1.000000", + "group_ids": [1], + }, + ) + self.assert_status_code(response, 403) + self.assertIn( + "The user needs OrganizationManagementLevel.can_manage_users or CommitteeManagementLevel.can_manage for committee of following meeting or Permission user.can_manage for meeting 1", + response.json["message"], + ) + + def test_create_permission_auth_error(self) -> None: + self.permission_setup() + response = self.request( + "user.create", + { + "username": "username_Neu", + "meeting_id": 1, + "vote_weight": "1.000000", + "group_ids": [1], + }, + anonymous=True, + ) + self.assert_status_code(response, 403) + self.assertIn( + "Anonymous is not allowed to execute user.create", + response.json["message"], + ) + + def test_create_permission_superadmin(self) -> None: + """ + SUPERADMIN may set fields of all groups and may set an other user as SUPERADMIN, too. + The SUPERADMIN don't need to belong to a meeting in any way to change data! + """ + self.permission_setup() + self.set_organization_management_level( + OrganizationManagementLevel.SUPERADMIN, self.user_id + ) + + response = self.request( + "user.create", + { + "username": "username_new", + "organization_management_level": OrganizationManagementLevel.SUPERADMIN, + "meeting_id": 1, + "vote_weight": "1.000000", + "group_ids": [1], + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/3", + { + "username": "username_new", + "organization_management_level": OrganizationManagementLevel.SUPERADMIN, + "meeting_user_ids": [2], + "meeting_ids": [1], + }, + ) + self.assert_model_exists( + "meeting_user/2", + { + "user_id": 3, + "meeting_id": 1, + "vote_weight": "1.000000", + "group_ids": [1], + }, + ) + self.assert_model_exists( + "meeting_user/2", + { + "user_id": 3, + "meeting_id": 1, + "vote_weight": "1.000000", + "group_ids": [1], + }, + ) + + def test_create_permission_group_A_oml_manage_user(self) -> None: + """May create group A fields on organsisation scope, because belongs to 2 meetings in 2 committees, requiring OML level permission""" + self.permission_setup() + self.create_meeting(base=4) + self.set_organization_management_level( + OrganizationManagementLevel.CAN_MANAGE_USERS, self.user_id + ) + self.set_models( + {"organization/1": {"genders": ["male", "female", "diverse", "non-binary"]}} + ) + + response = self.request_json( + [ + { + "action": "user.create", + "data": [ + { + "username": "new_username", + "title": "new title", + "first_name": "new first_name", + "last_name": "new last_name", + "is_active": True, + "is_physical_person": True, + "default_password": "new default_password", + "gender": "female", + "email": "info@openslides.com", + "default_vote_weight": "1.234000", + "can_change_own_password": False, + "meeting_id": 1, + "group_ids": [1], + } + ], + }, + { + "action": "user.update", + "data": [ + { + "id": 3, + "meeting_id": 4, + "group_ids": [4], + } + ], + }, + ], + atomic=False, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/3", + { + "username": "new_username", + "title": "new title", + "first_name": "new first_name", + "last_name": "new last_name", + "is_active": True, + "is_physical_person": True, + "default_password": "new default_password", + "gender": "female", + "email": "info@openslides.com", + "default_vote_weight": "1.234000", + "can_change_own_password": False, + "committee_ids": [60, 63], + "meeting_ids": [1, 4], + "meeting_user_ids": [2, 3], + }, + ) + self.assert_model_exists("meeting_user/2", {"meeting_id": 1, "group_ids": [1]}) + self.assert_model_exists("meeting_user/3", {"meeting_id": 4, "group_ids": [4]}) + + def test_create_permission_group_A_cml_manage_user(self) -> None: + """May create group A fields on cml scope""" + self.permission_setup() + self.create_meeting(base=4) + self.set_models( + { + f"user/{self.user_id}": { + "committee_management_ids": [60], + "committee_ids": [60], + }, + "meeting/4": {"committee_id": 60, "is_active_in_organization_id": 1}, + "committee/60": {"meeting_ids": [1, 4]}, + } + ) + + response = self.request_json( + [ + { + "action": "user.create", + "data": [ + { + "username": "usersname", + "meeting_id": 1, + "group_ids": [1], + } + ], + }, + { + "action": "user.update", + "data": [ + { + "id": 3, + "meeting_id": 4, + "group_ids": [4], + } + ], + }, + ], + atomic=False, + ) + + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/3", + { + "username": "usersname", + "meeting_ids": [1, 4], + "committee_ids": [60], + "meeting_user_ids": [2, 3], + }, + ) + self.assert_model_exists("meeting_user/2", {"meeting_id": 1, "group_ids": [1]}) + self.assert_model_exists("meeting_user/3", {"meeting_id": 4, "group_ids": [4]}) + + def test_create_permission_group_A_user_can_manage(self) -> None: + """May create group A fields on meeting scope""" + self.permission_setup() + self.set_user_groups(self.user_id, [2]) + response = self.request( + "user.create", + { + "username": "usersname", + "meeting_id": 1, + "group_ids": [1], + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/3", + { + "username": "usersname", + "meeting_user_ids": [2], + "meeting_ids": [1], + "committee_ids": [60], + }, + ) + self.assert_model_exists("meeting_user/2", {"meeting_id": 1, "group_ids": [1]}) + + def test_create_permission_group_A_both_committee_permissions(self) -> None: + """May not create group A fields on organsisation scope, although having both committee permissions""" + self.permission_setup() + self.create_meeting(base=4) + self.update_model( + f"user/{self.user_id}", + { + "committee_management_ids": [60, 63], + "committee_ids": [60, 63], + }, + ) + + response = self.request( + "user.create", + { + "username": "new_username", + "committee_management_ids": [60], + "meeting_id": 4, + "group_ids": [4], + }, + ) + self.assert_status_code(response, 200) + + def test_create_permission_group_B_user_can_manage(self) -> None: + """create group B fields with simple user.can_manage permissions""" + self.permission_setup() + self.set_organization_management_level(None, self.user_id) + self.set_user_groups(self.user_id, [2]) # Admin groups of meeting/1 + + self.set_models( + { + "user/5": {"username": "user5"}, + "user/6": {"username": "user6"}, + "meeting/1": { + "structure_level_ids": [31], + }, + "structure_level/31": {"meeting_id": 1}, + } + ) + self.set_user_groups(5, [1]) + self.set_user_groups(6, [1]) + + response = self.request( + "user.create", + { + "username": "username7", + "meeting_id": 1, + "number": "number1", + "structure_level_ids": [31], + "vote_weight": "12.002345", + "about_me": "about me 1", + "comment": "comment for meeting/1", + "vote_delegations_from_ids": [2, 3], + "group_ids": [1], + "is_present_in_meeting_ids": [1], + "locked_out": True, + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/7", + { + "username": "username7", + "meeting_ids": [1], + "meeting_user_ids": [4], + "is_present_in_meeting_ids": [1], + }, + ) + self.assert_model_exists( + "meeting_user/4", + { + "meeting_id": 1, + "user_id": 7, + "number": "number1", + "structure_level_ids": [31], + "vote_weight": "12.002345", + "about_me": "about me 1", + "comment": "comment for meeting/1", + "vote_delegations_from_ids": [2, 3], + "group_ids": [1], + "locked_out": True, + }, + ) + self.assert_model_exists( + "meeting_user/2", + { + "meeting_id": 1, + "user_id": 5, + "vote_delegated_to_id": 4, + }, + ) + self.assert_model_exists( + "meeting_user/3", + { + "meeting_id": 1, + "user_id": 6, + "vote_delegated_to_id": 4, + }, + ) + + def test_create_permission_group_B_user_can_manage_no_permission(self) -> None: + """Group B fields needs explicit user.can_manage permission for meeting""" + self.permission_setup() + self.set_organization_management_level( + OrganizationManagementLevel.CAN_MANAGE_USERS, self.user_id + ) + self.set_user_groups(self.user_id, [3]) # Empty group of meeting/1 + + response = self.request( + "user.create", + { + "username": "usersname", + "meeting_id": 1, + "group_ids": [1], + "is_present_in_meeting_ids": [1], + "number": "number1", + }, + ) + self.assert_status_code(response, 403) + self.assertIn( + "You are not allowed to perform action user.create. Missing permission: Permission user.can_manage in meeting 1", + response.json["message"], + ) + + def test_create_permission_group_B_locked_meeting(self) -> None: + """Group B fields needs explicit user.can_manage permission for meeting""" + self.permission_setup() + self.set_organization_management_level( + OrganizationManagementLevel.SUPERADMIN, self.user_id + ) + self.create_meeting(4) + self.set_models({"meeting/4": {"locked_from_inside": True}}) + + response = self.request( + "user.create", + { + "username": "usersname", + "meeting_id": 4, + "group_ids": [4], + "is_present_in_meeting_ids": [4], + "number": "number1", + }, + ) + self.assert_status_code(response, 403) + self.assertIn( + "The user needs Permission user.can_manage for meeting 4", + response.json["message"], + ) + + def test_create_permission_group_C_oml_manager(self) -> None: + """May create group C group_ids by OML permission""" + self.permission_setup() + self.set_organization_management_level( + OrganizationManagementLevel.CAN_MANAGE_USERS, self.user_id + ) + + response = self.request( + "user.create", + { + "username": "usersname", + "meeting_id": 1, + "group_ids": [1], + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists("user/3", {"meeting_user_ids": [2]}) + self.assert_model_exists("meeting_user/2", {"group_ids": [1]}) + + def test_create_permission_group_C_locked_meeting(self) -> None: + """May not create group C group_ids by OML permission with a locked meeting""" + self.permission_setup() + self.set_organization_management_level( + OrganizationManagementLevel.SUPERADMIN, self.user_id + ) + self.create_meeting(4) + self.set_models({"meeting/4": {"locked_from_inside": True}}) + + response = self.request( + "user.create", + { + "username": "usersname", + "meeting_id": 4, + "group_ids": [4], + }, + ) + self.assert_status_code(response, 403) + self.assertIn( + "The user needs Permission user.can_manage for meeting 4", + response.json["message"], + ) + + def test_create_permission_group_C_committee_manager(self) -> None: + """May create group C group_ids by committee permission""" + self.permission_setup() + self.set_committee_management_level([60], self.user_id) + + response = self.request( + "user.create", + { + "username": "usersname", + "meeting_id": 1, + "group_ids": [1], + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/3", + { + "username": "usersname", + "meeting_user_ids": [2], + "meeting_ids": [1], + }, + ) + self.assert_model_exists( + "meeting_user/2", + { + "group_ids": [1], + "meeting_id": 1, + }, + ) + + def test_create_permission_group_C_user_can_manage(self) -> None: + """May create group C group_ids by user.can_manage permission""" + self.permission_setup() + self.set_user_groups(self.user_id, [2]) # Admin-group + + response = self.request( + "user.create", + { + "username": "usersname", + "meeting_id": 1, + "group_ids": [2], + }, + ) + + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/3", + { + "username": "usersname", + "meeting_user_ids": [2], + "meeting_ids": [1], + }, + ) + self.assert_model_exists( + "meeting_user/2", + { + "group_ids": [2], + "meeting_id": 1, + }, + ) + + def test_create_permission_group_C_no_permission(self) -> None: + """May not create group C group_ids""" + self.permission_setup() + + response = self.request( + "user.create", + { + "username": "usersname", + "meeting_id": 1, + "group_ids": [1], + }, + ) + self.assert_status_code(response, 403) + self.assertIn( + "The user needs OrganizationManagementLevel.can_manage_users or CommitteeManagementLevel.can_manage for committee of following meeting or Permission user.can_manage for meeting 1", + response.json["message"], + ) + + def test_create_permission_group_C_cml_locked_meeting(self) -> None: + """May not create group C group_ids in locked meetings as a committee manager""" + self.permission_setup() + self.create_meeting(4) + self.set_committee_management_level([63], self.user_id) + self.set_models({"meeting/4": {"locked_from_inside": True}}) + + response = self.request( + "user.create", + { + "username": "usersname", + "meeting_id": 4, + "group_ids": [4], + }, + ) + self.assert_status_code(response, 403) + self.assertIn( + "The user needs Permission user.can_manage for meeting 4", + response.json["message"], + ) + + def test_create_permission_group_D_permission_with_OML(self) -> None: + """May create Group D committee fields with OML level permission for more than one committee""" + self.permission_setup() + self.create_meeting(base=4) + self.set_organization_management_level( + OrganizationManagementLevel.CAN_MANAGE_USERS, self.user_id + ) + + response = self.request( + "user.create", + { + "username": "usersname", + "committee_management_ids": [60, 63], + "organization_management_level": None, + }, + ) + self.assert_status_code(response, 200) + user3 = self.assert_model_exists( + "user/3", + { + "committee_ids": [60, 63], + "organization_management_level": None, + "committee_management_ids": [60, 63], + "username": "usersname", + }, + ) + self.assertCountEqual(user3.get("committee_management_ids", []), [60, 63]) + + def test_create_permission_group_D_permission_with_CML(self) -> None: + """ + May create Group D committee fields with CML permission for one committee only. + Note: For 2 committees he can't set the username, which is required, but within 2 committees + it would be organizational scope with organizational rights required. + To do this he could create a user with 1 committee and later he could update the + same user with second committee, if he has the permission for the committees. + """ + self.permission_setup() + self.set_committee_management_level([60], self.user_id) + + response = self.request( + "user.create", + { + "username": "usersname", + "committee_management_ids": [60], + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/3", + { + "committee_ids": [60], + "committee_management_ids": [60], + "username": "usersname", + }, + ) + + def test_create_permission_group_D_no_permission(self) -> None: + """May not create Group D committee fields, because of missing CML permission for one committee""" + self.permission_setup() + self.create_meeting(base=4) + self.set_committee_management_level([60], self.user_id) + + response = self.request( + "user.create", + { + "username": "usersname", + "committee_management_ids": [60, 63], + }, + ) + self.assert_status_code(response, 403) + self.assertIn( + "You are not allowed to perform action user.create. Missing permission: CommitteeManagementLevel can_manage in committee 63", + response.json["message"], + ) + + def test_create_permission_group_E_OML_high_enough(self) -> None: + """OML level to set is sufficient""" + self.permission_setup() + self.set_organization_management_level( + OrganizationManagementLevel.CAN_MANAGE_USERS, self.user_id + ) + + response = self.request( + "user.create", + { + "username": "usersname", + "organization_management_level": OrganizationManagementLevel.CAN_MANAGE_USERS, + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/3", + { + "organization_management_level": OrganizationManagementLevel.CAN_MANAGE_USERS, + "username": "usersname", + }, + ) + + def test_create_permission_group_E_OML_not_high_enough(self) -> None: + """OML level to set is higher than level of request user""" + self.permission_setup() + self.set_organization_management_level( + OrganizationManagementLevel.CAN_MANAGE_USERS, self.user_id + ) + + response = self.request( + "user.create", + { + "username": "usersname", + "organization_management_level": OrganizationManagementLevel.CAN_MANAGE_ORGANIZATION, + }, + ) + self.assert_status_code(response, 403) + self.assertIn( + "Your organization management level is not high enough to set a Level of can_manage_organization.", + response.json["message"], + ) + + def test_create_permission_group_H_internal_idp_id(self) -> None: + self.permission_setup() + self.set_user_groups(self.user_id, [2]) # Admin-group + + response = self.request( + "user.create", + { + "username": "username", + "idp_id": "11111", + "meeting_id": 1, + "group_ids": [2], + }, + ) + self.assert_status_code(response, 400) + self.assertIn( + "The field 'idp_id' can only be used in internal action calls", + response.json["message"], + ) + + def test_create_permission_group_H_oml_can_manage_user_idp_id(self) -> None: + self.set_organization_management_level( + OrganizationManagementLevel.CAN_MANAGE_USERS + ) + + response = self.request( + "user.create", + { + "idp_id": "11111", + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/2", + { + "username": "11111", + "idp_id": "11111", + "can_change_own_password": False, + "default_password": None, + }, + ) + + def test_create_permission_group_F_demo_user_permission(self) -> None: + """demo_user only editable by Superadmin""" + self.permission_setup() + self.set_organization_management_level( + OrganizationManagementLevel.SUPERADMIN, self.user_id + ) + + response = self.request( + "user.create", + { + "username": "username3", + "is_demo_user": True, + }, + ) + + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/3", + { + "username": "username3", + "is_demo_user": True, + }, + ) + + def test_create_permission_group_F_demo_user_no_permission(self) -> None: + """demo_user only editable by Superadmin""" + self.permission_setup() + self.set_organization_management_level( + OrganizationManagementLevel.CAN_MANAGE_ORGANIZATION, self.user_id + ) + response = self.request( + "user.create", + { + "username": "username3", + "is_demo_user": True, + }, + ) + + self.assert_status_code(response, 403) + self.assertIn( + "You are not allowed to perform action user.create. Missing OrganizationManagementLevel: superadmin", + response.json["message"], + ) + + def test_create_forbidden_username(self) -> None: + response = self.request( + "user.create", + { + "username": " ", + "organization_management_level": OrganizationManagementLevel.CAN_MANAGE_USERS, + }, + ) + self.assert_status_code(response, 400) + assert "Need username or first_name or last_name" in response.json["message"] + + def test_create_username_with_spaces(self) -> None: + response = self.request( + "user.create", + { + "username": "test name", + }, + ) + self.assert_status_code(response, 400) + assert "Username may not contain spaces" in response.json["message"] + + def test_create_gender(self) -> None: + self.set_models({"organization/1": {"genders": ["male", "female"]}}) + response = self.request( + "user.create", + { + "username": "test_Xcdfgee", + "gender": "test", + }, + ) + self.assert_status_code(response, 400) + assert ( + "Gender 'test' is not in the allowed gender list." + in response.json["message"] + ) + + def test_exceed_limit_of_users(self) -> None: + self.set_models( + { + ONE_ORGANIZATION_FQID: {"limit_of_users": 3}, + "user/2": {"is_active": True}, + "user/3": {"is_active": True}, + } + ) + response = self.request( + "user.create", + { + "username": "test_Xcdfgee", + "is_active": True, + }, + ) + self.assert_status_code(response, 400) + assert ( + "The number of active users cannot exceed the limit of users." + == response.json["message"] + ) + + def test_create_inactive_user(self) -> None: + self.set_models( + { + ONE_ORGANIZATION_FQID: {"limit_of_users": 1}, + } + ) + response = self.request( + "user.create", + { + "username": "test_Xcdfgee", + "is_active": False, + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/2", + { + "username": "test_Xcdfgee", + "is_active": False, + }, + ) + + def test_create_negative_default_vote_weight(self) -> None: + response = self.request( + "user.create", + { + "username": "test_Xcdfgee", + "default_vote_weight": "-1.500000", + }, + ) + self.assert_status_code(response, 400) + self.assertIn( + "default_vote_weight must be bigger than or equal to 0.", + response.json["message"], + ) + + def test_create_default_vote_weight_none(self) -> None: + response = self.request( + "user.create", + { + "username": "test_Xcdfgee", + "default_vote_weight": None, + }, + ) + self.assert_status_code(response, 200) + user = self.get_model("user/2") + assert "default_vote_weight" not in user + + def test_create_forwarding_committee_ids_not_allowed(self) -> None: + self.set_models({"meeting/1": {"is_active_in_organization_id": 1}}) + response = self.request( + "user.create", + { + "username": "test_Xcdfgee", + "forwarding_committee_ids": [], + }, + ) + self.assert_status_code(response, 403) + assert "forwarding_committee_ids is not allowed." in response.json["message"] + + def test_create_negative_vote_weight(self) -> None: + self.set_models( + { + "meeting/1": {"is_active_in_organization_id": 1}, + "meeting/2": {"is_active_in_organization_id": 1}, + } + ) + response = self.request( + "user.create", + { + "username": "test_Xcdfgee", + "meeting_id": 1, + "vote_weight": "-1.000000", + }, + ) + self.assert_status_code(response, 400) + self.assertIn( + "vote_weight must be bigger than or equal to 0.", + response.json["message"], + ) + + def test_create_variant(self) -> None: + """ + The replacement on both sides user and committe is the committee_management_level, + the ids are the user_ids and on user-side the committee_ids. + """ + self.set_models( + { + "committee/1": { + "name": "C1", + "meeting_ids": [1], + "user_ids": [222], + "manager_ids": [222], + }, + "committee/2": { + "name": "C2", + "meeting_ids": [2], + "user_ids": [222], + "manager_ids": [222], + }, + "meeting/1": {"committee_id": 1, "is_active_in_organization_id": 1}, + "meeting/2": {"committee_id": 2, "is_active_in_organization_id": 1}, + "user/222": { + "committee_management_ids": [1, 2], + }, + "group/22": {"meeting_id": 2}, + } + ) + response = self.request( + "user.create", + { + "username": "test_Xcdfgee", + "committee_management_ids": [1], + "meeting_id": 2, + "group_ids": [22], + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/223", + { + "committee_management_ids": [1], + "meeting_ids": [2], + "committee_ids": [1, 2], + "meeting_user_ids": [1], + }, + ) + self.assert_model_exists( + "meeting_user/1", + { + "meeting_id": 2, + "user_id": 223, + "group_ids": [22], + }, + ) + + self.assert_model_exists( + "committee/1", {"user_ids": [222, 223], "manager_ids": [222, 223]} + ) + self.assert_model_exists( + "committee/2", {"user_ids": [222, 223], "manager_ids": [222]} + ) + self.assert_model_exists("group/22", {"meeting_user_ids": [1]}) + self.assert_model_exists("meeting/1", {"user_ids": None}) + self.assert_model_exists( + "meeting/2", {"user_ids": [223], "meeting_user_ids": [1]} + ) + + def assert_lock_out_user( + self, + meeting_id: int, + other_payload_data: dict[str, Any], + errormsg: str | None = None, + ) -> None: + self.create_meeting() # committee:60; groups: default:1, admin:2, can_manage:3 + self.create_meeting(4) # committee:63; groups: default:4, admin:5, can_update:6 + self.add_group_permissions(3, [Permissions.User.CAN_MANAGE]) + self.add_group_permissions(6, [Permissions.User.CAN_UPDATE]) + response = self.request( + "user.create", + { + "username": "test", + "meeting_id": meeting_id, + "locked_out": True, + **other_payload_data, + }, + ) + if errormsg is not None: + self.assert_status_code(response, 400) + self.assertIn( + errormsg, + response.json["message"], + ) + else: + self.assert_status_code(response, 200) + + def test_create_locked_out_user_foreign_cml_allowed(self) -> None: + self.assert_lock_out_user(1, {"committee_management_ids": [63]}) + + def test_create_locked_out_user_superadmin_error(self) -> None: + self.assert_lock_out_user( + 1, + {"organization_management_level": "superadmin"}, + errormsg="Cannot lock user from meeting 1 as long as he has the OrganizationManagementLevel superadmin", + ) + + def test_create_locked_out_user_other_oml_error(self) -> None: + self.assert_lock_out_user( + 1, + {"organization_management_level": "can_manage_users"}, + errormsg="Cannot lock user from meeting 1 as long as he has the OrganizationManagementLevel can_manage_users", + ) + + def test_create_locked_out_user_cml_error(self) -> None: + self.assert_lock_out_user( + 1, + {"committee_management_ids": [60]}, + errormsg="Cannot lock user out of meeting 1 as he is manager of the meetings committee", + ) + + def test_create_locked_out_user_meeting_admin_error(self) -> None: + self.assert_lock_out_user( + 1, + {"group_ids": [2]}, + errormsg="Group(s) 2 have user.can_manage permissions and may therefore not be used by users who are locked out", + ) + + def test_create_locked_out_user_can_manage_error(self) -> None: + self.assert_lock_out_user( + 1, + {"group_ids": [3]}, + errormsg="Group(s) 3 have user.can_manage permissions and may therefore not be used by users who are locked out", + ) + + def test_create_locked_out_user_can_update_allowed(self) -> None: + self.assert_lock_out_user( + 4, + {"group_ids": [6]}, + ) + + +class UserCreateActionTestInternal(BaseInternalActionTest): + def test_create_empty_idp_id_and_empty_values(self) -> None: + response = self.internal_request( + "user.create", + {"idp_id": " ", "username": "x"}, + ) + self.assert_status_code(response, 400) + self.assertIn("This idp_id is forbidden.", response.json["message"]) + + def test_create_idp_id_and_default_pasword(self) -> None: + response = self.internal_request( + "user.create", + { + "username": "username_test", + "saml_id": "123saml", + "default_password": "test", + }, + ) + self.assert_status_code(response, 400) + assert ( + "user 123saml is a Single Sign On user and may not set the local default_passwort or the right to change it locally." + in response.json["message"] + ) + + def test_create_saml_id_and_empty_values(self) -> None: + response = self.internal_request( + "user.create", + { + "saml_id": "123saml", + "default_password": "", + "can_change_own_password": False, + }, + ) + self.assert_status_code(response, 200) + self.assert_model_exists( + "user/2", + { + "username": "123saml", + "saml_id": "123saml", + "default_password": "", + "can_change_own_password": False, + "password": None, + "is_physical_person": True, + "is_active": True, + }, + ) + + def test_create_saml_id_but_duplicate_error1(self) -> None: + self.set_models({"user/2": {"username": "x", "saml_id": "123saml"}}) + response = self.internal_request( + "user.create", + { + "saml_id": "123saml", + "default_password": "", + "can_change_own_password": False, + }, + ) + self.assert_status_code(response, 400) + self.assertIn( + "A user with the saml_id 123saml already exists.", response.json["message"] + ) + + def test_create_saml_id_but_duplicate_error2(self) -> None: + self.set_models({"user/2": {"username": "123saml"}}) + response = self.internal_request( + "user.create", + { + "saml_id": "123saml", + "default_password": "", + "can_change_own_password": False, + }, + ) + self.assert_status_code(response, 400) + self.assertIn( + "A user with the username 123saml already exists.", response.json["message"] + ) + + def test_create_anonymous_group_id(self) -> None: + self.create_meeting() + self.set_models( + { + "meeting/1": {"group_ids": [1, 2, 3, 4]}, + "group/4": {"anonymous_group_for_meeting_id": 1}, + } + ) + response = self.request( + "user.create", + { + "username": "test_Xcdfgee", + "meeting_id": 1, + "group_ids": [4], + }, + ) + self.assert_status_code(response, 400) + self.assertIn( + "Cannot add explicit users to a meetings anonymous group", + response.json["message"], + ) + + def test_create_permission_as_locked_out(self) -> None: + self.create_meeting() + self.user_id = self.create_user("user") + self.login(self.user_id) + meeting_user_id = self.set_user_groups(self.user_id, [3])[0] + self.set_models({f"meeting_user/{meeting_user_id}": {"locked_out": True}}) + self.set_group_permissions(3, [Permissions.User.CAN_MANAGE]) + response = self.request( + "user.create", + { + "username": "test_Xcdfgee", + "meeting_id": 1, + "group_ids": [1], + "number": "123456", + }, + ) + self.assert_status_code(response, 403) + self.assertIn( + "The user needs OrganizationManagementLevel.can_manage_users or CommitteeManagementLevel.can_manage for committee of following meeting or Permission user.can_manage for meeting 1", + response.json["message"], + ) From 6e1f38145db2e828cf9a03ddfb989acf1e74449b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20B=C3=B6hlke?= Date: Wed, 8 Jan 2025 17:42:28 +0100 Subject: [PATCH 87/90] Work on keycloak authenticator --- .../migrations/0063_user_keycloak_upload.py | 11 +++--- .../services/keycloak/adapter.py | 38 ++++++++++++------- .../shared/base_service_provider.py | 3 ++ openslides_backend/wsgi.py | 2 +- 4 files changed, 34 insertions(+), 20 deletions(-) diff --git a/openslides_backend/migrations/migrations/0063_user_keycloak_upload.py b/openslides_backend/migrations/migrations/0063_user_keycloak_upload.py index 82cbf55f98..fe21f22834 100644 --- a/openslides_backend/migrations/migrations/0063_user_keycloak_upload.py +++ b/openslides_backend/migrations/migrations/0063_user_keycloak_upload.py @@ -1,3 +1,5 @@ +from openslides_backend.services.keycloak.adapter import MigrationKeycloakAdminAdapter + from datastore.migrations import BaseModelMigration from datastore.shared.di import service_as_singleton from datastore.shared.util import fqid_from_collection_and_id @@ -5,23 +7,22 @@ from openslides_backend.services.keycloak.interface import IdpAdminService -@service_as_singleton class Migration(BaseModelMigration): """ This migration removes all default_number fields from user models """ - idpAdmin: IdpAdminService target_migration_index = 64 - def __init__(self, idpAdmin: IdpAdminService) -> None: - self.idpAdmin = idpAdmin + def __init__(self) -> None: + self.idpAdmin = MigrationKeycloakAdminAdapter() def migrate_models(self) -> list[BaseRequestEvent] | None: events: list[BaseRequestEvent] = [] db_models = self.reader.get_all("user") for id_, model in db_models.items(): - if not "kc_id" in model: + if not "kc_id" in model and model.get('username') != 'admin': + print(f"Creating user {model.get('username')} in keycloak...") idp_id = self.idpAdmin.create_user(model.get("username"), model.get("password"), model.get("saml_id")) events.append( RequestUpdateEvent( diff --git a/openslides_backend/services/keycloak/adapter.py b/openslides_backend/services/keycloak/adapter.py index 432d07e396..877fe946bf 100644 --- a/openslides_backend/services/keycloak/adapter.py +++ b/openslides_backend/services/keycloak/adapter.py @@ -1,6 +1,7 @@ import base64 import json from os import environ +import logging import requests from keycloak import KeycloakAdmin @@ -36,20 +37,29 @@ def set_user_password_hash(self, user_id, secret_data, credential_data): else: raise Exception(f"Fehler: {response.status_code}, {response.text}") +@service_as_singleton class KeycloakAdminAdapter(IdpAdminService, AuthenticatedService): """ Adapter to connect keycloak. """ + keycloak_admin_obj = None - def __init__(self, keycloak_url: str, logging: LoggingModule) -> None: + def __init__(self) -> None: + keycloak_url = environ.get("OPENSLIDES_KEYCLOAK_URL") self.url = keycloak_url - self.logger = logging.getLogger(__name__) if logging else None + self.logger = logging.getLogger(__name__) def create_keycloak_admin(self): access_token = token_storage.access_token keycloak_realm_name = environ.get("OPENSLIDES_AUTH_REALM") + print(f"Creating Keycloak admin with realm: {keycloak_realm_name} on {self.url}") return CustomKeycloakAdmin(server_url=self.url, token=access_token, realm_name=keycloak_realm_name) + def keycloak_admin(self): + if self.keycloak_admin_obj is None: + self.keycloak_admin_obj = self.create_keycloak_admin() + return self.keycloak_admin_obj + def create_user(self, username: str, password_hash: str, saml_id: str | None) -> str: ''' public static final String TYPE_KEY = "type"; @@ -70,11 +80,12 @@ def create_user(self, username: str, password_hash: str, saml_id: str | None) -> if not self.is_sha512_hash(password_hash) and not self.is_argon2_hash(password_hash): raise ValueError("The password hash is not a valid hash.") if self.is_sha512_hash(password_hash): - algorithm = "sha512" secret_data = { "value": password_hash } - credential_data = {} + credential_data = { + "algorithm": "sha512" + } else: # example value: $argon2id$v=19$m=65536,t=3,p=4$ag1cK0W8DxJ6VnUlOdgRKQ$wi/8MnuLaOWZVhO/7p4N+XWgnh6S2qTnrDylY+Z/tQc hash_data = self.parse_argon2_hash(password_hash) @@ -96,11 +107,13 @@ def create_user(self, username: str, password_hash: str, saml_id: str | None) -> } } - keycloak_admin = self.create_keycloak_admin() - existing_users = keycloak_admin.get_users({"username": username}) - user_id = existing_users[0]['id'] if existing_users else keycloak_admin.create_user({"username": username}) + print(f"Creating user {username} with password hash {password_hash}") + keycloak_admin = self.keycloak_admin() + existing_user_id = keycloak_admin.get_user_id(username) + user_id = existing_user_id if existing_user_id else keycloak_admin.create_user({"username": username}) keycloak_admin.set_user_password_hash(user_id, secret_data, credential_data) if saml_id: + print(f"Setting saml_id {saml_id} for user {user_id}") keycloak_admin.update_user({"id": user_id, "attributes": {"saml_id": saml_id}}) return user_id @@ -152,15 +165,12 @@ class MigrationKeycloakAdminAdapter(KeycloakAdminAdapter): """ Adapter to connect keycloak getting admin credentials from environment variables. """ - - def __init__(self) -> None: - keycloak_url = environ.get("OPENSLIDES_KEYCLOAK_URL") - super().__init__(keycloak_url, None) - def create_keycloak_admin(self): keycloak_realm_name = environ.get("OPENSLIDES_AUTH_REALM") keycloak_admin_username = environ.get("OPENSLIDES_KEYCLOAK_ADMIN_USERNAME") keycloak_admin_password = environ.get("OPENSLIDES_KEYCLOAK_ADMIN_PASSWORD") print(f"Creating Keycloak admin with realm: {keycloak_realm_name}, username: {keycloak_admin_username} on {self.url}, password: {keycloak_admin_password}") - return CustomKeycloakAdmin(server_url="http://keycloak:8080/idp/", username=keycloak_admin_username, password=keycloak_admin_password, realm_name="master") - # return CustomKeycloakAdmin(server_url=self.url, username=keycloak_admin_username, password=keycloak_admin_password, realm_name=keycloak_realm_name) + admin = CustomKeycloakAdmin(server_url="http://keycloak:8080/idp/", username="admin", + password="admin", realm_name="master", client_id="admin-cli", verify=False) + # admin.connection.realm_name = keycloak_realm_name + return admin diff --git a/openslides_backend/shared/base_service_provider.py b/openslides_backend/shared/base_service_provider.py index d3564192b2..bf4b398304 100644 --- a/openslides_backend/shared/base_service_provider.py +++ b/openslides_backend/shared/base_service_provider.py @@ -1,5 +1,6 @@ from openslides_backend.services.auth.interface import AuthenticationService from openslides_backend.services.datastore.interface import DatastoreService +from openslides_backend.services.keycloak.interface import IdpAdminService from openslides_backend.services.media.interface import MediaService from openslides_backend.services.vote.interface import VoteService from openslides_backend.shared.interfaces.logging import Logger, LoggingModule @@ -16,6 +17,7 @@ class BaseServiceProvider: auth: AuthenticationService media: MediaService vote: VoteService + idp_admin: IdpAdminService logging: LoggingModule logger: Logger @@ -32,5 +34,6 @@ def __init__( self.auth = services.authentication() self.media = services.media() self.vote_service = services.vote() + self.idp_admin = services.idp_admin() self.datastore = datastore self.logging = logging diff --git a/openslides_backend/wsgi.py b/openslides_backend/wsgi.py index ee1a1ddc7b..f3ac469777 100644 --- a/openslides_backend/wsgi.py +++ b/openslides_backend/wsgi.py @@ -28,7 +28,7 @@ class OpenSlidesBackendServices(containers.DeclarativeContainer): ) datastore = providers.Factory(ExtendedDatastoreAdapter, engine, logging, env) vote = providers.Singleton(VoteAdapter, config.vote_url, logging) - idp_admin = providers.Singleton(KeycloakAdminAdapter, config.keycloak_url, logging) + idp_admin = providers.Singleton(KeycloakAdminAdapter) class OpenSlidesBackendWSGI(containers.DeclarativeContainer): From 115560f989ae5775f00aa5c5e5c42db56fbd4656 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20B=C3=B6hlke?= Date: Wed, 25 Sep 2024 14:44:18 +0200 Subject: [PATCH 88/90] Work on keycloak integration --- openslides_backend/action/action.py | 11 +++- .../action/actions/user/backchannel_login.py | 57 ++++++++++--------- .../action/actions/user/update.py | 8 +-- .../action/actions/user/user_mixins.py | 4 +- openslides_backend/http/views/action_view.py | 7 ++- openslides_backend/http/views/auth.py | 14 +++-- .../permissions/management_levels.py | 5 ++ openslides_backend/permissions/permissions.py | 4 +- openslides_backend/services/media/adapter.py | 2 +- 9 files changed, 67 insertions(+), 45 deletions(-) diff --git a/openslides_backend/action/action.py b/openslides_backend/action/action.py index acf14ea543..97305e948c 100644 --- a/openslides_backend/action/action.py +++ b/openslides_backend/action/action.py @@ -11,7 +11,7 @@ from ..models.fields import BaseRelationField from ..permissions.management_levels import ( CommitteeManagementLevel, - OrganizationManagementLevel, + OrganizationManagementLevel, SystemManagementLevel, ) from ..permissions.permission_helper import has_organization_management_level, has_perm from ..permissions.permissions import Permission @@ -85,7 +85,7 @@ class Action(BaseServiceProvider, metaclass=SchemaProvider): is_singular: bool = False action_type: ActionType = ActionType.PUBLIC - permission: Permission | OrganizationManagementLevel | None = None + permission: Permission | OrganizationManagementLevel | SystemManagementLevel | None = None permission_model: Model | None = None permission_id: str | None = None skip_archived_meeting_check: bool = False @@ -197,8 +197,12 @@ def check_permissions(self, instance: dict[str, Any]) -> None: """ Checks permission by requesting permission service or using internal check. """ + print("check_permissions " + str(self.permission)) if self.permission: - if isinstance(self.permission, OrganizationManagementLevel): + if isinstance(self.permission, SystemManagementLevel): + if self.user_id == -1: + return + elif isinstance(self.permission, OrganizationManagementLevel): if has_organization_management_level( self.datastore, self.user_id, @@ -228,6 +232,7 @@ def check_permissions(self, instance: dict[str, Any]) -> None: def check_for_archived_meeting(self, instance: dict[str, Any]) -> None: """Do not allow changing any data in an archived meeting""" + print("check_for_archived_meeting " + str(self.skip_archived_meeting_check)) if self.skip_archived_meeting_check: return try: diff --git a/openslides_backend/action/actions/user/backchannel_login.py b/openslides_backend/action/actions/user/backchannel_login.py index 7cca0fb65f..6ca95f5d9c 100644 --- a/openslides_backend/action/actions/user/backchannel_login.py +++ b/openslides_backend/action/actions/user/backchannel_login.py @@ -1,22 +1,20 @@ import time -from typing import Any - -import time -from typing import Any - -import fastjsonschema +from typing import Any, Iterable +from datastore.shared.util import FilterOperator from openslides_backend.permissions.permissions import Permissions -from ...mixins.singular_action_mixin import SingularActionMixin +from ...action import Action from ...util.register import register_action +from ...util.typing import ActionResultElement from ....models.models import User +from ....permissions.management_levels import SystemManagementLevel from ....shared.exceptions import ActionException +from ....shared.interfaces.event import Event from ....shared.schema import schema_version + @register_action("user.backchannel_login") -class UserBackchannelLogin( - SingularActionMixin, -): +class UserBackchannelLogin(Action): """ Action to login a user via back-channel. """ @@ -24,27 +22,32 @@ class UserBackchannelLogin( model = User() # must contain an object with a string attribute "idp_id" schema = { - "$schema": schema_version, - "title": "User login hook schema", - "type": "object", - "properties": { - "idp_id": {"type": "string"} - }, - "required": ["idp_id"], - "additionalProperties": False - } - - permission = Permissions.System.CAN_LOGIN + "$schema": schema_version, + "title": "User login hook schema", + "type": "object", + "properties": { + "idp_id": {"type": "string"} + }, + "required": ["idp_id"], + "additionalProperties": False + } + + permission = SystemManagementLevel(Permissions.System.CAN_LOGIN) history_information = "User back-channel login" + skip_archived_meeting_check = True + + def create_action_result_element(self, instance: dict[str, Any]) -> ActionResultElement | None: + return instance def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: idp_id = instance.get("idp_id") - user = self.datastore.get( - self.model.collection, - self.model.default_values, - {"idp_id": idp_id}, - ) - if not user: + user = self.datastore.filter(self.model.collection, FilterOperator("idp_id", "=", idp_id), ["id"]) + if len(user) != 1: raise ActionException(f"User with idp_id {idp_id} not found.") instance["last_login"] = int(time.time()) + user = next(iter(user.values())) + instance["id"] = user["id"] return instance + + def create_events(self, instance: dict[str, Any]) -> Iterable[Event]: + return [] diff --git a/openslides_backend/action/actions/user/update.py b/openslides_backend/action/actions/user/update.py index 9ea0dee00c..218d2cad8b 100644 --- a/openslides_backend/action/actions/user/update.py +++ b/openslides_backend/action/actions/user/update.py @@ -105,17 +105,17 @@ def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: mapped_fields=[ "is_active", "organization_management_level", - "saml_id", + "idp_id", "password", ], ) - if user.get("saml_id") and ( + if user.get("idp_id") and ( instance.get("can_change_own_password") or instance.get("default_password") ): raise ActionException( - f"user {user['saml_id']} is a Single Sign On user and may not set the local default_passwort or the right to change it locally." + f"user {user['idp_id']} is a Single Sign On user and may not set the local default_passwort or the right to change it locally." ) - if instance.get("saml_id") and user.get("password"): + if instance.get("idp_id") and user.get("password"): instance["can_change_own_password"] = False instance["default_password"] = "" instance["password"] = "" diff --git a/openslides_backend/action/actions/user/user_mixins.py b/openslides_backend/action/actions/user/user_mixins.py index 807e00fb91..0d1757a05c 100644 --- a/openslides_backend/action/actions/user/user_mixins.py +++ b/openslides_backend/action/actions/user/user_mixins.py @@ -107,7 +107,7 @@ def validate_instance(self, instance: dict[str, Any]) -> None: @original_instances def get_updated_instances(self, action_data: ActionData) -> ActionData: for instance in action_data: - for field in ("username", "first_name", "last_name", "email", "saml_id"): + for field in ("username", "first_name", "last_name", "email", "idp_id"): self.strip_field(field, instance) return super().get_updated_instances(action_data) @@ -129,7 +129,7 @@ def check_existence(what: str) -> None: ) check_existence("username") - check_existence("saml_id") + check_existence("idp_id") if instance.get("member_number") is not None: check_existence("member_number") diff --git a/openslides_backend/http/views/action_view.py b/openslides_backend/http/views/action_view.py index d68536ee38..88436b41fd 100644 --- a/openslides_backend/http/views/action_view.py +++ b/openslides_backend/http/views/action_view.py @@ -51,7 +51,7 @@ def action_route(self, request: Request, claims: JWTClaims) -> RouteResponse: assert_migration_index() # Get user id. - user_id, access_token = int(claims.get("userId")), claims.get("access_token") + user_id, access_token = self.get_uid(claims), claims.get("access_token") # Set Headers and Cookies in services. self.services.vote().set_authentication( request.headers.get(AUTHENTICATION_HEADER, ""), @@ -67,6 +67,11 @@ def action_route(self, request: Request, claims: JWTClaims) -> RouteResponse: ) return response, access_token + def get_uid(self, claims): + if not claims.get("os_uid"): + return -1 + return int(claims.get("os_uid")) + @route("handle_request", internal=True) def internal_action_route(self, request: Request) -> RouteResponse: self.logger.debug("Start dispatching internal action request.") diff --git a/openslides_backend/http/views/auth.py b/openslides_backend/http/views/auth.py index b19cdce6aa..eb52e829cd 100644 --- a/openslides_backend/http/views/auth.py +++ b/openslides_backend/http/views/auth.py @@ -5,11 +5,14 @@ from authlib.oauth2.rfc9068 import JWTBearerTokenValidator from authlib.oidc.discovery import OpenIDProviderMetadata, get_well_known_url from werkzeug.exceptions import Unauthorized, Forbidden + +from os_authlib.token_validator import JWTBearerOpenSlidesTokenValidator, create_openslides_token_validator from ..token_storage import token_storage, TokenStorageUpdate -KEYCLOAK_DOMAIN = 'http://keycloak:8080' +KEYCLOAK_DOMAIN = 'http://keycloak:8080/idp' KEYCLOAK_REALM = 'os' ISSUER = f"{KEYCLOAK_DOMAIN}/realms/{KEYCLOAK_REALM}" +CERTS_URI = f"{KEYCLOAK_DOMAIN}/realms/{KEYCLOAK_REALM}/protocol/openid-connect/certs" class MyBearerTokenValidator(JWTBearerTokenValidator): # Cache the JWKS keys to avoid fetching them repeatedly @@ -17,8 +20,9 @@ class MyBearerTokenValidator(JWTBearerTokenValidator): def get_jwks(self): if self.jwk_set is None: - oidc_configuration = OpenIDProviderMetadata(requests.get(get_well_known_url(ISSUER, True)).json()) - response = requests.get(oidc_configuration.get('jwks_uri')) + # oidc_configuration = OpenIDProviderMetadata(requests.get(get_well_known_url(ISSUER, True)).json()) + # jwks_uri = oidc_configuration.get('jwks_uri') + response = requests.get(CERTS_URI) response.raise_for_status() jwks_keys = response.json() self.jwk_set = JsonWebKey.import_key_set(jwks_keys) @@ -34,12 +38,12 @@ def get_jwks(self): def token_required(f): def decorated_function(view, request, *args, **kwargs): - auth_header = request.headers.get('Authentication') + auth_header = request.headers.get('Authentication') or request.headers.get('Authorization') if not auth_header: raise Unauthorized('missing token') token = auth_header.split(" ")[1] - validator = MyBearerTokenValidator(ISSUER, 'https://localhost:8000/system') + validator = create_openslides_token_validator() claims = validator.authenticate_token(token) if not claims: diff --git a/openslides_backend/permissions/management_levels.py b/openslides_backend/permissions/management_levels.py index 69f299fb46..6a3972e13d 100644 --- a/openslides_backend/permissions/management_levels.py +++ b/openslides_backend/permissions/management_levels.py @@ -58,3 +58,8 @@ class CommitteeManagementLevel(CompareRightLevel): def get_base_model(self) -> str: return "committee" + +class SystemManagementLevel: + + def __init__(self, permission: str): + self.permission = permission \ No newline at end of file diff --git a/openslides_backend/permissions/permissions.py b/openslides_backend/permissions/permissions.py index f01f58a719..9729044458 100644 --- a/openslides_backend/permissions/permissions.py +++ b/openslides_backend/permissions/permissions.py @@ -79,8 +79,8 @@ class _User(str, Permission, Enum): CAN_UPDATE = "user.can_update" class _System(str, Permission, Enum): - CAN_LOGIN = "user.can_login" - CAN_LOGOUT = "user.can_logout" + CAN_LOGIN = "system.can_login" + CAN_LOGOUT = "system.can_logout" class Permissions: AgendaItem = _AgendaItem diff --git a/openslides_backend/services/media/adapter.py b/openslides_backend/services/media/adapter.py index 85cb87287f..70afc80115 100644 --- a/openslides_backend/services/media/adapter.py +++ b/openslides_backend/services/media/adapter.py @@ -44,7 +44,7 @@ def _handle_upload( ) -> None: try: self.logger.debug(f"Getting access token from : {threading.get_ident()} -> {token_storage.access_token}") - response = requests.post(url, json=payload, headers={AUTHORIZATION_HEADER: f'Bearer {token_storage.access_token}'}) + response = requests.post(url, json=payload, headers={'Authorization': f'Bearer {token_storage.access_token}'}) except requests.exceptions.ConnectionError as e: msg = f"Connect to mediaservice failed. {e}" self.logger.debug(description + msg) From e4e35ba6dd2c86566005f6901471bb5d963acf2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20B=C3=B6hlke?= Date: Fri, 10 Jan 2025 16:49:47 +0100 Subject: [PATCH 89/90] deactivate migration, merge main --- .../action/actions/organization/update.py | 73 ------------------- .../action/actions/user/create.py | 1 + .../migrations/0063_user_keycloak_upload.py | 20 ++--- openslides_backend/services/auth/adapter.py | 7 ++ 4 files changed, 18 insertions(+), 83 deletions(-) diff --git a/openslides_backend/action/actions/organization/update.py b/openslides_backend/action/actions/organization/update.py index b9a6362f7c..a8837de1ec 100644 --- a/openslides_backend/action/actions/organization/update.py +++ b/openslides_backend/action/actions/organization/update.py @@ -13,7 +13,6 @@ from ...mixins.send_email_mixin import EmailCheckMixin, EmailSenderCheckMixin from ...util.default_schema import DefaultSchema from ...util.register import register_action -from ..user.save_saml_account import allowed_user_fields from ..user.update import UserUpdate @@ -51,78 +50,6 @@ class OrganizationUpdate( ) model = Organization() - saml_props = { - field: {**optional_str_schema, "max_length": 256} - for field in allowed_user_fields - } - saml_props["meeting_mappers"] = { - "type": ["array", "null"], - "items": { - "type": "object", - "properties": { - **{ - field: {**optional_str_schema, "max_length": 256} - for field in ("external_id", "name", "allow_update") - }, - "conditions": { - "type": ["array", "null"], - "items": { - "type": ["object", "null"], - "properties": { - **{ - field: {**optional_str_schema, "max_length": 256} - for field in ("attribute", "condition") - }, - }, - }, - }, - "mappings": { - "type": ["object", "null"], - "properties": { - **{ - mapping_field: { - "type": ["object", "null"], - "properties": { - field: {**optional_str_schema, "max_length": 256} - for field in ("attribute", "default") - }, - "additionalProperties": False, - } - for mapping_field in [ - "number", - "comment", - "vote_weight", - "present", - ] - }, - **{ - mapping_field: { - "type": ["array", "null"], - "items": { - "type": ["object", "null"], - "properties": { - field: { - **optional_str_schema, - "max_length": 256, - } - for field in ("attribute", "default") - }, - "additionalProperties": False, - }, - } - for mapping_field in [ - "groups", - "structure_levels", - ] - }, - }, - "additionalProperties": False, - }, - }, - "required": ["external_id"], - "additionalProperties": False, - }, - } schema = DefaultSchema(Organization()).get_update_schema( optional_properties=group_A_fields + group_B_fields, additional_optional_fields={ diff --git a/openslides_backend/action/actions/user/create.py b/openslides_backend/action/actions/user/create.py index 6ce9e4ba13..e1b0232185 100644 --- a/openslides_backend/action/actions/user/create.py +++ b/openslides_backend/action/actions/user/create.py @@ -18,6 +18,7 @@ from ...util.register import register_action from ...util.typing import ActionResultElement from ..meeting_user.mixin import CheckLockOutPermissionMixin +from .create_update_permissions_mixin import CreateUpdatePermissionsMixin from .user_mixins import LimitOfUserMixin, UserMixin, UsernameMixin, check_gender_exists diff --git a/openslides_backend/migrations/migrations/0063_user_keycloak_upload.py b/openslides_backend/migrations/migrations/0063_user_keycloak_upload.py index fe21f22834..8e446353f2 100644 --- a/openslides_backend/migrations/migrations/0063_user_keycloak_upload.py +++ b/openslides_backend/migrations/migrations/0063_user_keycloak_upload.py @@ -23,14 +23,14 @@ def migrate_models(self) -> list[BaseRequestEvent] | None: for id_, model in db_models.items(): if not "kc_id" in model and model.get('username') != 'admin': print(f"Creating user {model.get('username')} in keycloak...") - idp_id = self.idpAdmin.create_user(model.get("username"), model.get("password"), model.get("saml_id")) - events.append( - RequestUpdateEvent( - fqid_from_collection_and_id("user", id_), - { - "idp_id": idp_id, - "saml_id": None - }, - ) - ) + # idp_id = self.idpAdmin.create_user(model.get("username"), model.get("password"), model.get("saml_id")) + # events.append( + # RequestUpdateEvent( + # fqid_from_collection_and_id("user", id_), + # { + # "idp_id": idp_id, + # "saml_id": None + # }, + # ) + # ) return events diff --git a/openslides_backend/services/auth/adapter.py b/openslides_backend/services/auth/adapter.py index ddd1ebc022..946990634c 100644 --- a/openslides_backend/services/auth/adapter.py +++ b/openslides_backend/services/auth/adapter.py @@ -47,6 +47,13 @@ def is_equal(self, toHash: str, toCompare: str) -> bool: def is_anonymous(self, user_id: int) -> bool: return user_id == ANONYMOUS_USER + def verify_authorization_token(self, user_id: int, token: str) -> bool: + try: + found_user_id, _ = self.auth_handler.verify_authorization_token(token) + except (AuthenticateException, AuthorizationException) as e: + raise AuthenticationException(e.message) + return user_id == found_user_id + def clear_all_sessions(self) -> None: self.auth_handler.clear_all_sessions( self.access_token, parse.unquote(self.refresh_id) From daf7c1f89f5a12b986ce98fe5160416081fa918b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20B=C3=B6hlke?= Date: Fri, 10 Jan 2025 17:13:50 +0100 Subject: [PATCH 90/90] Fix merge bug --- openslides_backend/action/actions/user/create.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openslides_backend/action/actions/user/create.py b/openslides_backend/action/actions/user/create.py index e1b0232185..6ce9e4ba13 100644 --- a/openslides_backend/action/actions/user/create.py +++ b/openslides_backend/action/actions/user/create.py @@ -18,7 +18,6 @@ from ...util.register import register_action from ...util.typing import ActionResultElement from ..meeting_user.mixin import CheckLockOutPermissionMixin -from .create_update_permissions_mixin import CreateUpdatePermissionsMixin from .user_mixins import LimitOfUserMixin, UserMixin, UsernameMixin, check_gender_exists