From 429d6d64eaa4b2a3a9e23a093b101f2dd9a50d02 Mon Sep 17 00:00:00 2001 From: Erik van Velzen Date: Mon, 15 Jul 2024 15:55:47 +0200 Subject: [PATCH] WIP docker swarm --- .editorconfig | 2 +- .github/workflows/build-push-deploy.yml | 43 ++++++++++++++++- .github/workflows/get-variables.js | 12 +++++ docker/compose-prod.yaml | 61 +++++++++++++++++++++++++ docker/config/python.example.env | 1 + src/holon/views/holon.py | 42 ++++++++++++----- src/requirements/local.txt | 2 +- 7 files changed, 148 insertions(+), 15 deletions(-) create mode 100644 docker/compose-prod.yaml diff --git a/.editorconfig b/.editorconfig index 11b226077..5c17cd8d7 100644 --- a/.editorconfig +++ b/.editorconfig @@ -14,7 +14,7 @@ indent_size = 4 [*.{yml,yaml}] indent_style = space -indent_size = 2 +indent_size = 4 [*.md] trim_trailing_whitespace = false diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index 05cf9d853..9abd6619d 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -6,6 +6,7 @@ on: - main - production - acceptance + - swarm workflow_dispatch: permissions: @@ -74,7 +75,7 @@ jobs: --push ## This job takes pretty long and can be split up to parallelize the deploy. - deploy-wagtail: + deploy-wagtail-azure: needs: - build-and-push-images - variables @@ -142,7 +143,7 @@ jobs: --hostname ${{ fromJson(needs.variables.outputs.result).WAGTAIL_HOSTNAME }} - deploy-next: + deploy-next-azure: needs: - build-and-push-images - variables @@ -195,3 +196,41 @@ jobs: --environment holon-env --name next-${{ github.ref_name }} --hostname www.holontool.nl + + deploy-swarm: + runs-on: ubuntu-latest + needs: + - build-and-push-images + - variables + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + sparse-checkout: docker + - name: Deploy to Docker Swarm + uses: sagebind/docker-swarm-deploy-action@v2 + env: + # Shared + TAG: ${{ needs.variables.outputs.tag }} + WAGTAIL_HOSTNAME: https://${{ fromJson(needs.variables.outputs.result).WAGTAIL_HOSTNAME }} + # Wagtail + AZURE_STORAGE_KEY: ${{ secrets.AZURE_STORAGE_KEY }} + MEDIA_LOCATION: ${{ fromJson(needs.variables.outputs.result).MEDIA_LOCATION }} + STATIC_LOCATION: ${{ fromJson(needs.variables.outputs.result).STATIC_LOCATION }} + DB_USER: ${{ fromJson(needs.variables.outputs.result).DB_USER }} + DB_NAME: ${{ fromJson(needs.variables.outputs.result).DB_NAME }} + DB_PASSWORD: ${{ secrets[fromJson(needs.variables.outputs.result).DB_PASSWORD_KEY] }} + RETURN_SCENARIO: ${{ fromJson(needs.variables.outputs.result).RETURN_SCENARIO }} + SECRET_KEY: "${{ secrets.SECRET_KEY }}" + SENTRY_ENVIRONMENT: ${{ fromJson(needs.variables.outputs.result).SENTRY_ENVIRONMENT }} + DOMAIN_HOST: ${{ fromJson(needs.variables.outputs.result).DOMAIN_HOST }} + N_WORKERS: ${{ fromJson(needs.variables.outputs.result).wagtail.N_WORKERS }} + EMAIL_HOST_PASSWORD: ${{ secrets.EMAIL_HOST_PASSWORD }} + # NextJS + NEXT_HOSTNAME: ${{ fromJson(needs.variables.outputs.result).NEXT_HOSTNAME }} + NEXT_PUBLIC_TINY_URL_API_KEY: ${{ secrets.TINY_URL_API_KEY }} + with: + remote_host: ssh://root@server.zenmo.com + ssh_private_key: ${{ secrets.SWARM_SSH_PRIVATE_KEY }} + ssh_public_key: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJ1E4LUG22qgzc8U7oNYGWCn0cyA31+iyX2pck9wcPMS + args: stack deploy --compose-file ./docker/compose-prod.yaml holon-${{ github.ref_name }} diff --git a/.github/workflows/get-variables.js b/.github/workflows/get-variables.js index 7e44858ee..415f456f1 100644 --- a/.github/workflows/get-variables.js +++ b/.github/workflows/get-variables.js @@ -52,6 +52,18 @@ const configPerBranch = { N_WORKERS: '4', } }, + swarm: { + DB_NAME: 'holon-test', + DB_USER: 'holon-test', + DB_PASSWORD_KEY: 'POSTGRES_PASSWORD', + RETURN_SCENARIO: 'True', + SENTRY_ENVIRONMENT: 'swarm-test', + NEXT_HOSTNAME: 'swarm.holontool.nl', + DOMAIN_HOST: 'https://swarm.holontool.nl', + WAGTAIL_HOSTNAME: 'cms-swarm.holontool.nl', + MEDIA_LOCATION: 'media-test', + STATIC_LOCATION: 'static-test', + }, } module.exports = (branchName) => { diff --git a/docker/compose-prod.yaml b/docker/compose-prod.yaml new file mode 100644 index 000000000..9449d611e --- /dev/null +++ b/docker/compose-prod.yaml @@ -0,0 +1,61 @@ +## Compose file for Docker Swarm environments +version: "3.8" + +services: + wagtail: + image: ghcr.io/zenmo/holon-wagtail:${TAG} + environment: + ALLOWED_HOSTS: "*" + AZURE_ACCOUNT_NAME: holonstorage + AZURE_STORAGE_KEY: ${AZURE_STORAGE_KEY} + MEDIA_LOCATION: ${MEDIA_LOCATION} + STATIC_LOCATION: ${STATIC_LOCATION} + DB_HOST: postgres + DB_USER: ${DB_USER} + DB_NAME: ${DB_NAME} + DB_PASSWORD: ${DB_PASSWORD} + RETURN_SCENARIO: ${RETURN_SCENARIO} + SECRET_KEY: ${SECRET_KEY} + SENTRY_DSN: "https://764e9f2b886741bcbcfd2acd74a7f7b0@o4505045746384896.ingest.sentry.io/4505045759361024" + SENTRY_ENVIRONMENT: ${SENTRY_ENVIRONMENT} + DOMAIN_HOST: ${DOMAIN_HOST} + N_WORKERS: ${N_WORKERS} + EMAIL_HOST_PASSWORD: ${EMAIL_HOST_PASSWORD} + WAGTAILADMIN_BASE_URL: https://${WAGTAIL_HOSTNAME} + labels: + caddy: ${WAGTAIL_HOSTNAME} + caddy.reverse_proxy: "{{upstreams 8000}}" + networks: + - caddy_default + - postgres_default + - default + deploy: + resources: + limits: + cpus: "8" + memory: 8G + + nextjs: + image: ghcr.io/zenmo/holon-nextjs:${TAG} + environment: + WAGTAIL_API_URL: http://wagtail:8000/wt/api/nextjs + NEXT_PUBLIC_WAGTAIL_API_URL: https://${WAGTAIL_HOSTNAME}/wt/api/nextjs + NEXT_PUBLIC_TINY_URL_API_KEY: ${NEXT_PUBLIC_TINY_URL_API_KEY} + labels: + caddy: ${NEXT_HOSTNAME}, www.${NEXT_HOSTNAME} + caddy.reverse_proxy: "{{upstreams 3000}}" + networks: + - caddy_default + - default + deploy: + resources: + limits: + cpus: "4" + memory: 4G + +networks: + caddy_default: + external: true + postgres_default: + external: true + default: diff --git a/docker/config/python.example.env b/docker/config/python.example.env index c95a7a8b6..c3b3d6dd6 100644 --- a/docker/config/python.example.env +++ b/docker/config/python.example.env @@ -1,3 +1,4 @@ +# For development AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_BUCKET_NAME= diff --git a/src/holon/views/holon.py b/src/holon/views/holon.py index d380787e3..eadb4874e 100644 --- a/src/holon/views/holon.py +++ b/src/holon/views/holon.py @@ -3,7 +3,6 @@ from django.apps import apps from django.shortcuts import render -from etm_service.etm_session.session import ETMConnectionError from rest_framework import generics, status from rest_framework.request import Request from rest_framework.response import Response @@ -15,9 +14,10 @@ from holon.models.config import QueryCovertModuleType from holon.models.scenario_rule import ModelType, DatamodelQueryRule from holon.models.util import all_subclasses, is_exclude_field -from holon.serializers import HolonRequestSerializer, ScenarioSerializer +from holon.serializers import HolonRequestSerializer from holon.services import CostTables, ETMConnect from holon.services.cloudclient import CloudClient +from holon.services.cloudclient.input import inputs_to_debug_values from holon.services.cloudclient.output import AnyLogicOutput from holon.services.data import Results from holon.utils.logging import HolonLogger @@ -58,6 +58,13 @@ def post(self, request: Request): use_caching = use_result_cache(request) scenario: Scenario | None = None anylogic_output: AnyLogicOutput | None = None + debug_values = { + "scenario": {}, + "anylogic": { + "input": {}, + "output": {}, + }, + } try: if not serializer.is_valid(): @@ -84,15 +91,29 @@ def post(self, request: Request): HolonV2Service.logger.log_print("Applying interactive elements to scenario") scenario = Scenario.queryset_with_relations().get(id=scenario_id) + debug_values["scenario"] = { + "id": scenario.id, + "name": scenario.name, + } scenario_aggregate = ScenarioAggregate(scenario) scenario_aggregate = rule_mapping.apply_rules(scenario_aggregate, interactive_elements) - cc_payload = scenario_aggregate.serialize_to_json() - # RUN ANYLOGIC HolonV2Service.logger.log_print("Running Anylogic model") - anylogic_output = CloudClient(payload=cc_payload, scenario=scenario).run() + anylogic_client = CloudClient(payload=scenario_aggregate, scenario=scenario) + debug_values["anylogic"]["input"] = { + # TODO: this is quite computationally expensive, see if we can do it only when needed + "debug": inputs_to_debug_values(anylogic_client.create_inputs()) + } + anylogic_output = anylogic_client.run() + # Disabled because it makes the response too big for common tooling + # debug_values["anylogic"]["output"] = { + # # TODO: this is quite computationally expensive, see if we can do it only when needed + # "debug": anylogic_output.get_debug_output(), + # # It would be useful to include the interpreted values but the response is getting too big + # # "decoded": anylogic_output.decoded, + # } # ETM Module # We want to be resilient in the face of errors. @@ -113,7 +134,7 @@ def post(self, request: Request): ) results = Results( - cc_payload=cc_payload, + debug_values=debug_values, request=request, anylogic_outcomes=anylogic_output, cost_benefit_overview=cost_benefit_tables.main_table(), @@ -141,11 +162,10 @@ def post(self, request: Request): traceback.print_exc() capture_exception(e) - response_body = {"error_msg": f"something went wrong: {e}"} - if scenario: - response_body["scenario"] = ScenarioSerializer(scenario).data - if anylogic_output: - response_body["anylogic_outcomes"] = anylogic_output.decoded + response_body = { + "error_msg": f"something went wrong: {e}", + "debug_values": debug_values, + } return Response( response_body, diff --git a/src/requirements/local.txt b/src/requirements/local.txt index a2f241264..31456abbc 100644 --- a/src/requirements/local.txt +++ b/src/requirements/local.txt @@ -2,7 +2,7 @@ -r test.txt # Add local extra requirements here (django-debug etc) -pydevd-pycharm~=241.17011.79 +pydevd-pycharm~=242.21829.44 black django-debug-toolbar isort