Merge main into stable/4.0.x. Update 20230906

* commit '13629938d89a45f241cbe54c4c5cec84e8cff684': Remove test which isn't working Manage versioning (#231) Fix test not working after merge Add auth-*-key secrets to media (#230) Extend project automation (#227) Fix backendAction secrets (#226) rename env-vars (#225) add search service (#221)
OpenSlides · Sep 6, 2023 · 7042379 · 7042379
2 parents e4fc8ef + 1362993
commit 7042379
Show file tree

Hide file tree

Showing 18 changed files with 260 additions and 89 deletions.
diff --git a/.github/workflows/project-automation.yml b/.github/workflows/project-automation.yml
@@ -0,0 +1,32 @@
+name: Project automation
+on:
+  workflow_call:
+    inputs:
+      resource_node_id:
+        required: true
+        type: string
+      status_value:
+        required: true
+        type: string
+    secrets:
+      AUTOMATION_APP_ID:
+        required: true
+      AUTOMATION_APP_INSTALLATION_ID:
+        required: true
+      AUTOMATION_APP_PRIVATE_KEY:
+        required: true
+
+jobs:
+  workflow_call:
+    name: Set status
+    runs-on: ubuntu-latest
+    steps:
+      - uses: leonsteinhaeuser/[email protected]
+        with:
+          gh_app_ID: ${{ secrets.AUTOMATION_APP_ID }}
+          gh_app_installation_ID: ${{ secrets.AUTOMATION_APP_INSTALLATION_ID }}
+          gh_app_secret_key: ${{ secrets.AUTOMATION_APP_PRIVATE_KEY }}
+          organization: OpenSlides
+          project_id: 2
+          resource_node_id: ${{ inputs.resource_node_id }}
+          status_value: ${{ inputs.status_value }}
diff --git a/.github/workflows/project-issue-closed.yml b/.github/workflows/project-issue-closed.yml
@@ -0,0 +1,14 @@
+name: Project automation
+on:
+  issues:
+    types:
+      - closed
+
+jobs:
+  issue_closed:
+    name: Issue closed
+    uses: ./.github/workflows/project-automation.yml
+    secrets: inherit
+    with:
+      resource_node_id: ${{ github.event.issue.node_id }}
+      status_value: "Done"
diff --git a/.github/workflows/project-issue-opened.yml b/.github/workflows/project-issue-opened.yml
@@ -0,0 +1,15 @@
+name: Project automation
+on:
+  issues:
+    types:
+      - opened
+      - reopened
+
+jobs:
+  issue_opened:
+    name: Issue opened
+    uses: ./.github/workflows/project-automation.yml
+    secrets: inherit
+    with:
+      resource_node_id: ${{ github.event.issue.node_id }}
+      status_value: "Backlog"
diff --git a/.github/workflows/project-pull-request-closed.yml b/.github/workflows/project-pull-request-closed.yml
@@ -0,0 +1,14 @@
+name: Project automation
+on:
+  pull_request_target:
+    types:
+      - closed
+
+jobs:
+  pull_request_closed:
+    name: Pull request closed
+    uses: ./.github/workflows/project-automation.yml
+    secrets: inherit
+    with:
+      resource_node_id: ${{ github.event.pull_request.node_id }}
+      status_value: "Done"
diff --git a/.github/workflows/project-pull-request-opened.yml b/.github/workflows/project-pull-request-opened.yml
@@ -0,0 +1,15 @@
+name: Project automation
+on:
+  pull_request_target:
+    types:
+      - opened
+      - reopened
+
+jobs:
+  pull_request_opened:
+    name: Pull request opened
+    uses: ./.github/workflows/project-automation.yml
+    secrets: inherit
+    with:
+      resource_node_id: ${{ github.event.pull_request.node_id }}
+      status_value: "Work in progress"
diff --git a/.github/workflows/project-pull-request-review-requested.yml b/.github/workflows/project-pull-request-review-requested.yml
@@ -0,0 +1,14 @@
+name: Project automation
+on:
+  pull_request_target:
+    types:
+      - review_requested
+
+jobs:
+  pull_request_review_requested:
+    name: Pull request review requested
+    uses: ./.github/workflows/project-automation.yml
+    secrets: inherit
+    with:
+      resource_node_id: ${{ github.event.pull_request.node_id }}
+      status_value: "Review in progress"
diff --git a/.github/workflows/release.yml → .github/workflows/release-create.yml b/.github/workflows/release.yml → .github/workflows/release-create.yml
@@ -1,7 +1,20 @@
-name: GitHub Release
+name: GitHub Release (reusable)
 on:
-  push:
-    branches: [main]
+  workflow_call:
+    inputs:
+      branch:
+        type: string
+        default: main
+      tag:
+        type: string
+        default: dev
+      title:
+        type: string
+        default: Development Build
+      prerelease:
+        type: boolean
+        default: true
+
 jobs:
   release:
     name: Create GitHub Release and upload binary
@@ -14,6 +27,8 @@ jobs:
 
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
+      with:
+        ref: "${{ inputs.branch }}"
 
     - name: Build binaries
       run: go build ./cmd/openslides
@@ -22,8 +37,8 @@ jobs:
       uses: marvinpinto/action-automatic-releases@latest
       with:
         repo_token: "${{ secrets.GITHUB_TOKEN }}"
-        automatic_release_tag: "latest"
-        prerelease: true
-        title: "Development Build"
+        automatic_release_tag: "${{ inputs.tag }}"
+        prerelease: "${{ inputs.prerelease }}"
+        title: "${{ inputs.title }}"
         files: |
           openslides
diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml
@@ -0,0 +1,9 @@
+name: GitHub Release (dev)
+on:
+  push:
+    branches: [main]
+
+jobs:
+  release-dev:
+    name: "Create dev Release"
+    uses: ./.github/workflows/release-create.yml
diff --git a/.github/workflows/release-stable.yml b/.github/workflows/release-stable.yml
@@ -0,0 +1,24 @@
+name: GitHub Release (stable)
+on:
+  repository_dispatch:
+    types: [stable-update]
+
+jobs:
+  release-stable:
+    name: "Create GitHub Release (4.x.x) and upload binary"
+    if: startsWith(github.event.client_payload.branch, 'stable/4')
+    uses: ./.github/workflows/release-create.yml
+    with:
+      branch: "${{ github.event.client_payload.branch }}"
+      tag: "${{ github.event.client_payload.version }}"
+      title: Stable Build
+      prerelease: false
+  release-stable-latest:
+    name: "Create GitHub Release (latest) and upload binary"
+    if: startsWith(github.event.client_payload.branch, 'stable/4')
+    uses: ./.github/workflows/release-create.yml
+    with:
+      branch: "${{ github.event.client_payload.branch }}"
+      tag: latest
+      title: Stable Build
+      prerelease: false
diff --git a/.github/workflows/set-project.yml b/.github/workflows/set-project.yml
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -25,6 +25,3 @@ jobs:
 
     - name: go test
       run: go test -timeout 10s -race ./...
-
-    - name: go test for special system tests
-      run: OPENSLIDES_MANAGE_SERVICE_SYSTEM_TEST=true go test -timeout 300s -v pkg/client/system_test.go
diff --git a/pkg/config/config.go b/pkg/config/config.go
@@ -239,6 +239,7 @@ func NewYmlConfig(configFiles [][]byte) (*YmlConfig, error) {
 		"autoupdate",
 		"auth",
 		"vote",
+		"search",
 		"redis",
 		"media",
 		"icc",

diff --git a/pkg/config/default-config.yml b/pkg/config/default-config.yml
@@ -29,60 +29,58 @@ defaultEnvironment:
   DATASTORE_READER_PORT: 9010
   DATASTORE_WRITER_HOST: datastoreWriter
   DATASTORE_WRITER_PORT: 9011
-  DATASTORE_DATABASE_HOST: postgres
-  DATASTORE_DATABASE_PORT: 5432
-  DATASTORE_DATABASE_NAME: openslides
-  DATASTORE_DATABASE_USER: openslides
-  DATASTORE_DATABASE_PASSWORD_FILE: /run/secrets/postgres_password
 
   AUTOUPDATE_HOST: autoupdate
   AUTOUPDATE_PORT: 9012
 
   AUTH_HOST: auth
   AUTH_PORT: 9004
 
+  SEARCH_HOST: search
+  SEARCH_PORT: 9050
+  RESTRICTER_URL: http://autoupdate:9012/internal/autoupdate
+
   VOTE_HOST: vote
   VOTE_PORT: 9013
-  VOTE_DATABASE_HOST: postgres
-  VOTE_DATABASE_PORT: 5432
-  VOTE_DATABASE_NAME: openslides
-  VOTE_DATABASE_USER: openslides
-  VOTE_DATABASE_PASSWORD_FILE: /run/secrets/postgres_password
-  VOTE_REDIS_HOST: redis
-  VOTE_REDIS_PORT: 6379
 
   CACHE_HOST: redis
   CACHE_PORT: 6379
-
   MESSAGE_BUS_HOST: redis
   MESSAGE_BUS_PORT: 6379
 
   MEDIA_HOST: media
   MEDIA_PORT: 9006
-  MEDIA_DATABASE_HOST: postgres
-  MEDIA_DATABASE_PORT: 5432
-  MEDIA_DATABASE_NAME: openslides
-  MEDIA_DATABASE_USER: openslides
-  MEDIA_DATABASE_PASSWORD_FILE: /run/secrets/postgres_password
-  MEDIA_BLOCK_SIZE: 4096
-  MEDIA_PRESENTER_HOST: backendPresenter
-  MEDIA_PRESENTER_PORT: 9003
 
   ICC_HOST: icc
   ICC_PORT: 9007
-  ICC_REDIS_HOST: redis
-  ICC_REDIS_PORT: 6379
 
   MANAGE_HOST: manage
   MANAGE_PORT: 9008
-  MANAGE_AUTH_PASSWORD_FILE: /run/secrets/manage_auth_password
-  MANAGE_ACTION_HOST: backendManage
+
+  DATABASE_HOST: postgres
+  DATABASE_PORT: 5432
+  DATABASE_NAME: openslides
+  DATABASE_USER: openslides
+  DATABASE_PASSWORD_FILE: /run/secrets/postgres_password
+  MEDIA_DATABASE_HOST: postgres
+  MEDIA_DATABASE_PORT: 5432
+  MEDIA_DATABASE_NAME: openslides
+  MEDIA_DATABASE_USER: openslides
+  MEDIA_DATABASE_PASSWORD_FILE: /run/secrets/postgres_password
+  VOTE_DATABASE_HOST: postgres
+  VOTE_DATABASE_PORT: 5432
+  VOTE_DATABASE_NAME: openslides
+  VOTE_DATABASE_USER: openslides
+  VOTE_DATABASE_PASSWORD_FILE: /run/secrets/postgres_password
 
   INTERNAL_AUTH_PASSWORD_FILE: /run/secrets/internal_auth_password
+  MANAGE_AUTH_PASSWORD_FILE: /run/secrets/manage_auth_password
+  AUTH_TOKEN_KEY_FILE: /run/secrets/auth_token_key
+  AUTH_COOKIE_KEY_FILE: /run/secrets/auth_cookie_key
+  SUPERADMIN_PASSWORD_FILE: /run/secrets/superadmin
 
   OPENSLIDES_LOGLEVEL: info
   OPENSLIDES_DEVELOPMENT: "false"
-
   SYSTEM_URL: localhost:8000
 
 # You can extend or replace parts of the defaultEnvironment.
@@ -97,6 +95,9 @@ services:
   datastoreReader:
     environment:
       NUM_WORKERS: 8
+  manage:
+    environment:
+      ACTION_HOST: backendManage
 
 # All properties from the "defaults" section are available here.
 #

diff --git a/pkg/config/default-docker-compose.yml b/pkg/config/default-docker-compose.yml
@@ -15,6 +15,7 @@ services:
       - backendAction
       - backendPresenter
       - autoupdate
+      - search
       - auth
       - media
       - icc
@@ -54,6 +55,7 @@ services:
       - backendAction
       - backendPresenter
       - autoupdate
+      - search
       - auth
       - media
       - icc
@@ -91,6 +93,7 @@ services:
     secrets:
       - auth_token_key
       - auth_cookie_key
+      - internal_auth_password
       - postgres_password
     {{- with .AdditionalContent }}{{ marshalContent 4 . }}{{- end }}
   {{- end }}
@@ -227,6 +230,30 @@ services:
   {{- end }}
 
 
+  {{- with .Services.search }}
+
+  search:
+    image: {{ .ContainerRegistry }}/openslides-search:{{ .Tag }}
+    {{- if checkFlag $.DisableDependsOn }}{{ else }}
+    depends_on:
+      - datastoreReader
+      - postgres
+      - autoupdate
+    {{- end }}
+    environment:
+      << : *default-environment
+      {{- with .Environment }}{{ marshalContent 6 . }}{{- end }}
+    networks:
+      - frontend
+      - data
+    secrets:
+      - auth_token_key
+      - auth_cookie_key
+      - postgres_password
+    {{- with .AdditionalContent }}{{ marshalContent 4 . }}{{- end }}
+  {{- end }}
+
+
   {{- with .Services.auth }}
 
   auth:
@@ -245,6 +272,7 @@ services:
     secrets:
       - auth_token_key
       - auth_cookie_key
+      - internal_auth_password
     {{- with .AdditionalContent }}{{ marshalContent 4 . }}{{- end }}
   {{- end }}
 
@@ -301,6 +329,8 @@ services:
       - frontend
       - data
     secrets:
+      - auth_token_key
+      - auth_cookie_key
       - postgres_password
     {{- with .AdditionalContent }}{{ marshalContent 4 . }}{{- end }}
   {{- end }}