From a60dee52de2e6d974e699fad8903a4d0f3c52e8c Mon Sep 17 00:00:00 2001 From: pdmurray Date: Fri, 18 Oct 2024 12:45:35 -0700 Subject: [PATCH] [ENH] Add support for hot reloading of the development environment --- .gitignore | 3 + .../_internal/server/app.py | 25 ++++-- conda-store-server/pyproject.toml | 6 +- docker-compose-core.yml | 42 +++++++++ docker-compose.yaml | 52 +++-------- .../community/contribute/contribute-code.md | 9 +- tests/assets/conda_store_config.py | 1 + tests/assets/local_dev_config.py | 87 +++++++++++++++++++ 8 files changed, 173 insertions(+), 52 deletions(-) create mode 100644 docker-compose-core.yml create mode 100644 tests/assets/local_dev_config.py diff --git a/.gitignore b/.gitignore index 0ec82f551..b4224ac18 100644 --- a/.gitignore +++ b/.gitignore @@ -61,4 +61,7 @@ yarn-error.log* # conda-store conda-store.sqlite +# Development workspace +conda-store-workspace/ + *.lockb diff --git a/conda-store-server/conda_store_server/_internal/server/app.py b/conda-store-server/conda_store_server/_internal/server/app.py index ffe66fd11..a78d3daf4 100644 --- a/conda-store-server/conda_store_server/_internal/server/app.py +++ b/conda-store-server/conda_store_server/_internal/server/app.py @@ -207,9 +207,6 @@ def initialize(self, *args, **kwargs): f"Running conda-store with store directory: {self.conda_store.store_directory}" ) - if self.conda_store.upgrade_db: - dbutil.upgrade(self.conda_store.database_url) - self.authentication = self.authentication_class( parent=self, log=self.log, @@ -389,7 +386,9 @@ def _check_worker(self, delay=5): ) def start(self): - fastapi_app = self.init_fastapi_app() + """Start the CondaStoreServer application, and run a FastAPI-based webserver.""" + if self.conda_store.upgrade_db: + dbutil.upgrade(self.conda_store.database_url) with self.conda_store.session_factory() as db: self.conda_store.ensure_settings(db) @@ -440,7 +439,7 @@ def start(self): logger.info(f"Starting server on {self.address}:{self.port}") uvicorn.run( - fastapi_app, + "conda_store_server._internal.server.app:CondaStoreServer.create_webserver", host=self.address, port=self.port, workers=1, @@ -452,7 +451,9 @@ def start(self): if self.reload else [] ), + factory=True, ) + except: import traceback @@ -462,3 +463,17 @@ def start(self): if self.standalone: process.join() worker_checker.join() + + @classmethod + def create_webserver(cls: type) -> FastAPI: + """Create a CondaStoreServer instance to load the config, then return a FastAPI app. + + Returns + ------- + FastAPI + A FastAPI app configured using a fresh CondaStoreServer instance + + """ + app = cls() + app.initialize() + return app.init_fastapi_app() diff --git a/conda-store-server/pyproject.toml b/conda-store-server/pyproject.toml index 7a04593a0..ece19de06 100644 --- a/conda-store-server/pyproject.toml +++ b/conda-store-server/pyproject.toml @@ -96,9 +96,13 @@ dependencies = [ "pytest-playwright", "twine>=5.0.0", "pkginfo>=1.10", # Needed to support metadata 2.3 - ] +[tool.hatch.envs.dev.scripts] +server = "conda-store-server --config ../tests/assets/local_dev_config.py" +worker = "conda-store-worker --config ../tests/assets/local_dev_config.py" +services = "docker compose -f ../docker-compose-core.yml up --build" + [tool.hatch.envs.lint] dependencies = ["pre-commit"] diff --git a/docker-compose-core.yml b/docker-compose-core.yml new file mode 100644 index 000000000..abc42f0d5 --- /dev/null +++ b/docker-compose-core.yml @@ -0,0 +1,42 @@ +name: conda-store-services +services: + minio: + image: minio/minio:RELEASE.2020-11-10T21-02-24Z + ports: + - "9000:9000" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 10s + timeout: 5s + retries: 5 + entrypoint: sh + command: -c 'mkdir -p /data/conda-store && /usr/bin/minio server /data' + environment: + MINIO_ACCESS_KEY: admin + MINIO_SECRET_KEY: password + + postgres: + image: postgres:13 + user: postgres + ports: + - 5432:5432 + healthcheck: + test: ["CMD-SHELL", "pg_isready"] + interval: 10s + timeout: 5s + retries: 5 + environment: + POSTGRES_PASSWORD: password + POSTGRES_DB: conda-store + + redis: + image: bitnami/redis + ports: + - 6379:6379 + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + environment: + REDIS_PASSWORD: password diff --git a/docker-compose.yaml b/docker-compose.yaml index 33ec36d38..d5304c084 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,4 +1,17 @@ services: + minio: + extends: + file: docker-compose-core.yml + service: minio + postgres: + extends: + file: docker-compose-core.yml + service: postgres + redis: + extends: + file: docker-compose-core.yml + service: redis + conda-store-worker: build: context: conda-store-server @@ -47,42 +60,3 @@ services: ] ports: - "8080:8080" - - minio: - image: minio/minio:RELEASE.2020-11-10T21-02-24Z - ports: - - "9000:9000" - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 10s - timeout: 5s - retries: 5 - entrypoint: sh - command: -c 'mkdir -p /data/conda-store && /usr/bin/minio server /data' - environment: - MINIO_ACCESS_KEY: admin - MINIO_SECRET_KEY: password - - postgres: - image: postgres:13 - user: postgres - ports: - - 5432:5432 - healthcheck: - test: ["CMD-SHELL", "pg_isready"] - interval: 10s - timeout: 5s - retries: 5 - environment: - POSTGRES_PASSWORD: password - POSTGRES_DB: conda-store - - redis: - image: bitnami/redis - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 10s - timeout: 5s - retries: 5 - environment: - REDIS_PASSWORD: password diff --git a/docusaurus-docs/community/contribute/contribute-code.md b/docusaurus-docs/community/contribute/contribute-code.md index 28820abc3..d90003557 100644 --- a/docusaurus-docs/community/contribute/contribute-code.md +++ b/docusaurus-docs/community/contribute/contribute-code.md @@ -61,13 +61,8 @@ After running the `docker compose` command, the following resources will be avai On a fast machine, this deployment should only take 10 or so seconds assuming the Docker images have been partially built before. -If you are making any changes to `conda-store-server` and would like to see -those changes in the deployment, run: - -```shell -docker compose down -v # not always necessary -docker compose up --build -``` +Any changes made to the `conda-store-server` will be hot reloaded, so there's no +need to bring the services down and then up again between edits. To stop the deployment, run: diff --git a/tests/assets/conda_store_config.py b/tests/assets/conda_store_config.py index 523a3c8b1..e833516b1 100644 --- a/tests/assets/conda_store_config.py +++ b/tests/assets/conda_store_config.py @@ -46,6 +46,7 @@ c.CondaStoreServer.enable_ui = True c.CondaStoreServer.enable_api = True c.CondaStoreServer.enable_registry = True +c.CondaStoreServer.reload = True c.CondaStoreServer.enable_metrics = True c.CondaStoreServer.address = "0.0.0.0" c.CondaStoreServer.port = 8080 diff --git a/tests/assets/local_dev_config.py b/tests/assets/local_dev_config.py new file mode 100644 index 000000000..6c68e1051 --- /dev/null +++ b/tests/assets/local_dev_config.py @@ -0,0 +1,87 @@ +# Copyright (c) conda-store development team. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import logging + +from conda_store_server.server.auth import DummyAuthentication +from conda_store_server.storage import S3Storage + + +# ================================== +# conda-store settings +# ================================== +# The default storage_threshold limit was reached on CI, which caused test +# failures +c.CondaStore.storage_threshold = 1024**3 +c.CondaStore.storage_class = S3Storage +c.CondaStore.store_directory = "./conda-store-workspace/" +c.CondaStore.environment_directory = "./conda-store-workspace/envs/{namespace}-{name}" +# c.CondaStore.database_url = "mysql+pymysql://admin:password@mysql/conda-store" +c.CondaStore.database_url = ( + "postgresql+psycopg2://postgres:password@localhost/conda-store" +) +c.CondaStore.redis_url = "redis://:password@localhost:6379/0" +c.CondaStore.default_uid = 1000 +c.CondaStore.default_gid = 1000 +c.CondaStore.default_permissions = "775" +c.CondaStore.conda_included_packages = ["ipykernel"] + +c.CondaStore.pypi_included_packages = ["nothing"] + + +c.S3Storage.internal_endpoint = "minio:9000" +c.S3Storage.external_endpoint = "localhost:9000" +c.S3Storage.access_key = "admin" +c.S3Storage.secret_key = "password" +c.S3Storage.region = "us-east-1" # minio region default +c.S3Storage.bucket_name = "conda-store" +c.S3Storage.internal_secure = False +c.S3Storage.external_secure = False + +# ================================== +# server settings +# ================================== +c.CondaStoreServer.log_level = logging.INFO +c.CondaStoreServer.enable_ui = True +c.CondaStoreServer.enable_api = True +c.CondaStoreServer.enable_registry = True +c.CondaStoreServer.reload = True +c.CondaStoreServer.enable_metrics = True +c.CondaStoreServer.address = "0.0.0.0" +c.CondaStoreServer.port = 8080 +# This MUST start with `/` +c.CondaStoreServer.url_prefix = "/conda-store" + + +# ================================== +# auth settings +# ================================== +c.CondaStoreServer.authentication_class = DummyAuthentication +c.CondaStoreServer.template_vars = { + "banner": '', + "logo": "https://raw.githubusercontent.com/conda-incubator/conda-store/main/docusaurus-docs/community/assets/logos/conda-store-logo-horizontal-lockup.svg", +} + +# ================================== +# worker settings +# ================================== +c.CondaStoreWorker.log_level = logging.INFO +c.CondaStoreWorker.watch_paths = ["./conda-store-workspace/environments"] +c.CondaStoreWorker.concurrency = 4 + +# ================================== +# registry settings +# ================================== +# from python_docker.registry import Registry +# import os + +# def _configure_docker_registry(registry_url: str): +# return Registry( +# "https://registry-1.docker.io", +# username=os.environ.get('DOCKER_USERNAME'), +# password=os.environ.get('DOCKER_PASSWORD')) + +# c.ContainerRegistry.container_registries = { +# 'https://registry-1.docker.io': _configure_docker_registry +# }