Merge staging/4.1.5 into stable/4.1.x. Update 20240320

* commit '639d9e725b4b4846c2df6559c4101881fc870d71': (50 commits) Fix KeyError in entitled users calculation (OpenSlides#2314) (OpenSlides#2321) Fix participant import re-upload structure_level error (OpenSlides#2313) (OpenSlides#2318) Update all translations (OpenSlides#2282) Implement certain preventions for changing active speakers state (OpenSlides#2272) Implement cascade delete for amendments (OpenSlides#2253) Add staging branches to CI (OpenSlides#2255) Fix \update-meta command (OpenSlides#2303) Fix \update-meta command (OpenSlides#2302) Fix \update-meta command (OpenSlides#2301) Fix \update-meta command (OpenSlides#2300) Fix \update-meta command (OpenSlides#2299) Update meta repository (OpenSlides#2298) Bump autoflake from 2.3.0 to 2.3.1 in /requirements/partial (OpenSlides#2297) Bump mypy from 1.8.0 to 1.9.0 in /requirements/partial (OpenSlides#2283) Bump types-simplejson in /requirements/partial (OpenSlides#2291) Bump types-bleach in /requirements/partial (OpenSlides#2289) Bump types-requests in /requirements/partial (OpenSlides#2286) Update meta repository (OpenSlides#2294) Bump types-pygments in /requirements/partial (OpenSlides#2292) Bump types-redis in /requirements/partial (OpenSlides#2290) ...
peb-adr · Mar 20, 2024 · 7a04231 · 7a04231
2 parents e843058 + 639d9e7
commit 7a04231
Show file tree

Hide file tree

Showing 55 changed files with 981 additions and 367 deletions.
diff --git a/.github/workflows/close-issues.yml b/.github/workflows/close-issues.yml
@@ -0,0 +1,52 @@
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - "feature/*"
+
+jobs:
+  close-issue:
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.merged
+    steps:
+      - name: Generate access token
+        uses: tibdex/github-app-token@v2
+        id: generate-token
+        with:
+          app_id: ${{ secrets.AUTOMATION_APP_ID }}
+          private_key: ${{ secrets.AUTOMATION_APP_PRIVATE_KEY }}
+
+      - uses: octokit/[email protected]
+        id: get-issues
+        env:
+          GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }}
+        with:
+          query: |
+            query getLinkedIssues($owner: String!, $name: String!, $number: Int!) {
+              repository(owner: $owner, name: $name) {
+                pullRequest(number: $number) {
+                  closingIssuesReferences(first: 100) {
+                    nodes {
+                      number
+                      repository {
+                        nameWithOwner
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          variables: |
+            owner: ${{ github.event.repository.owner.name }}
+            repo: ${{ github.event.repository.name }}
+            number: ${{ github.event.pull_request.number }}
+
+      - name: Close issues
+        env:
+          GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }}
+        run: |
+          issue_data="$(echo "${{ steps.get-issues.outputs.data }}" | jq -r '.data.repository.pullRequest.closingIssuesReferences.nodes[] | [.number,.repository.nameWithOwner] | @tsv')"
+          echo "$issue_data" | while read number nameWithOwner; do
+            gh issue close "$number" -r "$nameWithOwner"
+          done
diff --git a/.github/workflows/commands.yml b/.github/workflows/commands.yml
@@ -0,0 +1,87 @@
+name: Commands for the openslides-automation
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  update-meta-repo:
+    name: Recursively merge the main branch
+    runs-on: ubuntu-latest
+    if: |
+      github.event.issue.pull_request != '' && (
+        contains(github.event.comment.body, '\update-meta') ||
+        contains(github.event.comment.body, '\update_meta') ||
+        contains(github.event.comment.body, '\updatemeta')
+      )
+
+    steps:
+      - name: Generate access token
+        uses: tibdex/github-app-token@v2
+        id: generate-token
+        with:
+          app_id: ${{ secrets.AUTOMATION_APP_ID }}
+          private_key: ${{ secrets.AUTOMATION_APP_PRIVATE_KEY }}
+
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: recursive
+          token: ${{ steps.generate-token.outputs.token }}
+
+      - name: Checkout PR
+        env:
+          GH_TOKEN: ${{ steps.generate-token.outputs.token }}
+        run: |
+          git config --global submodule.recurse true
+          gh pr checkout ${{ github.event.issue.number }}
+
+      - name: Set git credentials
+        run: |
+          git config --global user.name openslides-automation
+          git config --global user.email [email protected]
+      
+      - name: Merge main branch
+        id: merge
+        run: git merge --no-edit main || echo "result=1" >> $GITHUB_OUTPUT
+
+      - name: Push conflict-free merge
+        if: steps.merge.outputs.result == ''
+        run: git push
+
+      - name: Get submodule path
+        id: path
+        if: steps.merge.outputs.result != ''
+        run: echo "path=$(git config --file .gitmodules --get submodule.openslides-meta.path)" >> $GITHUB_OUTPUT
+
+      - name: Determine whether the conflicts are too complex to continue
+        id: conflicts
+        if: steps.merge.outputs.result != ''
+        run: |
+          [[ "$(git diff --name-only --diff-filter=U)" = "${{ steps.path.outputs.path }}" ]] || echo "result=1" >> $GITHUB_OUTPUT
+      
+      - name: Write failure comment
+        if: steps.merge.outputs.result != '' && steps.conflicts.outputs.result != ''
+        uses: actions/github-script@v7
+        with:
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: 'Merging failed since the conflicts are too complex to continue. Please resolve the conflicts manually.'
+            })
+
+      - name: Merge meta repo
+        if: steps.merge.outputs.result != '' && steps.conflicts.outputs.result == ''
+        working-directory: ${{ steps.path.outputs.path }}
+        env:
+          GH_TOKEN: ${{ steps.generate-token.outputs.token }}
+        run: |
+          gh pr checkout $(gh pr list --search $(git rev-parse HEAD) --json number --jq ".[0].number")
+          git fetch origin main
+          git merge --no-edit origin/main
+          git push
+      
+      - name: Finish merge & push result
+        if: steps.merge.outputs.result != '' && steps.conflicts.outputs.result == ''
+        run: git commit -a --no-edit && git push
diff --git a/Makefile b/Makefile
@@ -54,7 +54,7 @@ generate-permissions:
 	black openslides_backend/permissions/permissions.py
 
 check-permissions:
-	python cli/generate_permissions.py check
+	python cli/generate_permissions.py --check
 
 check-initial-data-json:
 	python cli/check_json.py global/data/initial-data.json

diff --git a/cli/__init__.py b/cli/__init__.py
diff --git a/cli/generate_models.py b/cli/generate_models.py
@@ -1,15 +1,16 @@
-import argparse
 import os
 import string
-import subprocess
 from collections import ChainMap
-from io import StringIO, TextIOBase
 from textwrap import dedent
-from typing import Any, Optional, cast
-
-import requests
-import yaml
-
+from typing import Any, Optional
+
+from cli.util.util import (
+    ROOT,
+    assert_equal,
+    open_output,
+    open_yml_file,
+    parse_arguments,
+)
 from openslides_backend.models.base import Model as BaseModel
 from openslides_backend.models.fields import OnDelete
 from openslides_backend.models.mixins import (
@@ -21,11 +22,6 @@
 
 SOURCE = "./global/meta/models.yml"
 
-ROOT = os.path.join(
-    os.path.dirname(os.path.abspath(__file__)),
-    "..",
-)
-
 DESTINATION = os.path.abspath(
     os.path.join(
         ROOT,
@@ -97,30 +93,12 @@ def main() -> None:
             type: relation_list
             to: some_model/some_attribute_id
     """
-    parser = argparse.ArgumentParser()
-    parser.add_argument("filename", nargs="?", default=SOURCE)
-    parser.add_argument("--check", action="store_true")
-    args = parser.parse_args()
-
-    # Retrieve models.yml from call-parameter for testing purposes, local file or GitHub
-    file = args.filename
-    if os.path.isfile(file):
-        with open(file, "rb") as x:
-            models_yml = x.read()
-    else:
-        models_yml = requests.get(file).content
-
-    # open output stream
-    dest: TextIOBase
-    if args.check:
-        dest = StringIO()
-    else:
-        dest = open(DESTINATION, "w")
+    args = parse_arguments(SOURCE)
+    global MODELS
+    MODELS = open_yml_file(args.filename)
 
     # Load and parse models.yml
-    global MODELS
-    MODELS = yaml.safe_load(models_yml)
-    with dest:
+    with open_output(DESTINATION, args.check) as dest:
         dest.write(FILE_TEMPLATE)
         dest.write(
             "from .mixins import "
@@ -134,15 +112,7 @@ def main() -> None:
             dest.write(model.get_code())
 
         if args.check:
-            result = subprocess.run(
-                ["black", "-c", cast(StringIO, dest).getvalue()],
-                capture_output=True,
-                text=True,
-                cwd=ROOT,
-            )
-            result.check_returncode()
-            with open(DESTINATION) as f:
-                assert f.read() == result.stdout
+            assert_equal(dest, DESTINATION)
             print("Models file up-to-date.")
         else:
             print(f"Models file {DESTINATION} successfully created.")

diff --git a/cli/generate_permissions.py b/cli/generate_permissions.py
@@ -1,15 +1,13 @@
-import hashlib
 import os
-import sys
 from collections import defaultdict
 from collections.abc import Iterable
 from pathlib import Path
 from textwrap import dedent
 from typing import Any
 
-import requests
 import yaml
 
+from cli.util.util import assert_equal, open_output, open_yml_file, parse_arguments
 from openslides_backend.permissions.get_permission_parts import get_permission_parts
 
 SOURCE = "./global/meta/permission.yml"
@@ -29,11 +27,8 @@
     # Code generated. DO NOT EDIT.
 
     from enum import Enum
-    from typing import Dict, List
 
     from .base_classes import Permission
-
-    PERMISSION_YML_CHECKSUM = "{}"
     """
 )
 
@@ -48,23 +43,8 @@ def main() -> None:
     """
     Main entry point for this script to generate the permissions.py from permission.yml.
     """
-    # Retrieve models.yml from call-parameter for testing purposes, local file or GitHub
-    if len(sys.argv) > 1 and sys.argv[1] != "check":
-        file = sys.argv[1]
-    else:
-        file = SOURCE
-
-    if os.path.isfile(file):
-        with open(file, "rb") as x:
-            permissions_yml = x.read()
-    else:
-        permissions_yml = requests.get(file).content
-
-    # calc checksum to assert the permissions.py is up-to-date
-    checksum = hashlib.md5(permissions_yml).hexdigest()
-
-    # Load and parse permissions.yml
-    permissions = yaml.safe_load(permissions_yml)
+    args = parse_arguments(SOURCE)
+    permissions = open_yml_file(args.filename)
     all_parents: dict[str, list[str]] = {}
     all_permissions: dict[str, set[str]] = defaultdict(set)
     for collection, children in permissions.items():
@@ -77,49 +57,48 @@ def main() -> None:
             if pair[1] is not None:
                 all_parents[pair[0]] += [pair[1]]
 
-    if len(sys.argv) > 1 and sys.argv[1] == "check":
-        from openslides_backend.permissions.permissions import PERMISSION_YML_CHECKSUM
-
-        assert checksum == PERMISSION_YML_CHECKSUM
-        print("permissions.py is up to date (checksum-comparison)")
-
-        # check group.permissions enum in models.yml, if possible
-        models_file = Path(file).parent / "models.yml"
-        if os.path.isfile(models_file):
-            with open(models_file, "rb") as f:
-                models = yaml.safe_load(f.read())
-            enum = set(models["group"]["permissions"]["items"]["enum"])
-            permissions = {
-                str(permission)
-                for permissions in all_permissions.values()
-                for permission in permissions
-            }
-            assert enum == permissions, (
-                "Missing permissions: "
-                + str(permissions - enum)
-                + "\nAdditional permissions: "
-                + str(enum - permissions)
-            )
-            print("models.yml field group/permissions enum contains all permissions")
-    else:
-        # Write permissions.py
-        with open(DESTINATION, "w") as dest:
-            dest.write(FILE_TEMPLATE.format(checksum))
-            for collection, permissions in all_permissions.items():
-                dest.write(f"\nclass _{collection}(str, Permission, Enum):\n")
-                for permission in sorted(permissions):
-                    _, perm_str = get_permission_parts(permission)
-                    dest.write(f"    {perm_str} = '{permission}'\n")
-
-            dest.write("class Permissions:\n")
-            for collection in all_permissions.keys():
-                dest.write(f"    {collection} = _{collection}\n")
-
-            dest.write("\n# Holds the corresponding parent for each permission.\n")
-            dest.write("permission_parents: dict[Permission, list[Permission]] = ")
-            dest.write(repr(all_parents))
-
-        print(f"Permissions file {DESTINATION} successfully created.")
+    with open_output(DESTINATION, args.check) as dest:
+        dest.write(FILE_TEMPLATE)
+        for collection, permissions in all_permissions.items():
+            dest.write(f"\nclass _{collection}(str, Permission, Enum):\n")
+            for permission in sorted(permissions):
+                _, perm_str = get_permission_parts(permission)
+                dest.write(f"    {perm_str} = '{permission}'\n")
+
+        dest.write("class Permissions:\n")
+        for collection in all_permissions.keys():
+            dest.write(f"    {collection} = _{collection}\n")
+
+        dest.write("\n# Holds the corresponding parent for each permission.\n")
+        dest.write("permission_parents: dict[Permission, list[Permission]] = ")
+        dest.write(repr(all_parents))
+
+        if args.check:
+            assert_equal(dest, DESTINATION)
+            print("Permissions file up-to-date.")
+
+            # check group.permissions enum in models.yml, if possible
+            models_file = Path(args.filename).parent / "models.yml"
+            if os.path.isfile(models_file):
+                with open(models_file, "rb") as f:
+                    models = yaml.safe_load(f.read())
+                enum = set(models["group"]["permissions"]["items"]["enum"])
+                permissions = {
+                    str(permission)
+                    for permissions in all_permissions.values()
+                    for permission in permissions
+                }
+                assert enum == permissions, (
+                    "Missing permissions: "
+                    + str(permissions - enum)
+                    + "\nAdditional permissions: "
+                    + str(enum - permissions)
+                )
+                print(
+                    "models.yml field group/permissions enum contains all permissions"
+                )
+        else:
+            print(f"Permissions file {DESTINATION} successfully created.")
 
 
 def process_permission_level(

diff --git a/cli/util/__init__.py b/cli/util/__init__.py