[ENH] Add hot reloading for development

.gitignore

@@ -61,4 +61,7 @@ yarn-error.log*
 # conda-store
 conda-store.sqlite
 
+# Development workspace
+conda-store-workspace/
+
 *.lockb

conda-store-server/conda_store_server/_internal/server/app.py

@@ -207,9 +207,6 @@ def initialize(self, *args, **kwargs):
             f"Running conda-store with store directory: {self.conda_store.store_directory}"
         )
 
-        if self.conda_store.upgrade_db:
-            dbutil.upgrade(self.conda_store.database_url)
-
         self.authentication = self.authentication_class(
             parent=self,
             log=self.log,
@@ -389,7 +386,9 @@ def _check_worker(self, delay=5):
             )
 
     def start(self):
-        fastapi_app = self.init_fastapi_app()
+        """Start the CondaStoreServer application, and run a FastAPI-based webserver."""
+        if self.conda_store.upgrade_db:
+            dbutil.upgrade(self.conda_store.database_url)
 
         with self.conda_store.session_factory() as db:
             self.conda_store.ensure_settings(db)
@@ -440,7 +439,7 @@ def start(self):
             logger.info(f"Starting server on {self.address}:{self.port}")
 
             uvicorn.run(
-                fastapi_app,
+                "conda_store_server._internal.server.app:CondaStoreServer.create_webserver",
                 host=self.address,
                 port=self.port,
                 workers=1,
@@ -452,7 +451,9 @@ def start(self):
                     if self.reload
                     else []
                 ),
+                factory=True,
             )
+
         except:
             import traceback
 
@@ -462,3 +463,17 @@ def start(self):
             if self.standalone:
                 process.join()
             worker_checker.join()
+
+    @classmethod
+    def create_webserver(cls: type) -> FastAPI:
+        """Create a CondaStoreServer instance to load the config, then return a FastAPI app.
+
+        Returns
+        -------
+        FastAPI
+            A FastAPI app configured using a fresh CondaStoreServer instance
+
+        """
+        app = cls()
+        app.initialize()
+        return app.init_fastapi_app()

conda-store-server/pyproject.toml

@@ -96,9 +96,13 @@ dependencies = [
   "pytest-playwright",
   "twine>=5.0.0",
   "pkginfo>=1.10", # Needed to support metadata 2.3
-
 ]
 
+[tool.hatch.envs.dev.scripts]
+server = "conda-store-server --config ../tests/assets/local_dev_config.py"
+worker = "conda-store-worker --config ../tests/assets/local_dev_config.py"
+services = "docker compose -f ../docker-compose-core.yml up --build"
+
 [tool.hatch.envs.lint]
 dependencies = ["pre-commit"]
 

docker-compose-core.yml

@@ -0,0 +1,42 @@
+name: conda-store-services
+services:
+  minio:
+    image: minio/minio:RELEASE.2020-11-10T21-02-24Z
+    ports:
+      - "9000:9000"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    entrypoint: sh
+    command: -c 'mkdir -p /data/conda-store && /usr/bin/minio server /data'
+    environment:
+      MINIO_ACCESS_KEY: admin
+      MINIO_SECRET_KEY: password
+
+  postgres:
+    image: postgres:13
+    user: postgres
+    ports:
+      - 5432:5432
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    environment:
+      POSTGRES_PASSWORD: password
+      POSTGRES_DB: conda-store
+
+  redis:
+    image: bitnami/redis
+    ports:
+      - 6379:6379
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    environment:
+      REDIS_PASSWORD: password

docker-compose.yaml

@@ -1,4 +1,17 @@
 services:
+  minio:
+    extends:
+      file: docker-compose-core.yml
+      service: minio
+  postgres:
+    extends:
+      file: docker-compose-core.yml
+      service: postgres
+  redis:
+    extends:
+      file: docker-compose-core.yml
+      service: redis
+
   conda-store-worker:
     build:
       context: conda-store-server
@@ -47,42 +60,3 @@ services:
       ]
     ports:
       - "8080:8080"
-
-  minio:
-    image: minio/minio:RELEASE.2020-11-10T21-02-24Z
-    ports:
-      - "9000:9000"
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-    entrypoint: sh
-    command: -c 'mkdir -p /data/conda-store && /usr/bin/minio server /data'
-    environment:
-      MINIO_ACCESS_KEY: admin
-      MINIO_SECRET_KEY: password
-
-  postgres:
-    image: postgres:13
-    user: postgres
-    ports:
-      - 5432:5432
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-    environment:
-      POSTGRES_PASSWORD: password
-      POSTGRES_DB: conda-store
-
-  redis:
-    image: bitnami/redis
-    healthcheck:
-      test: ["CMD", "redis-cli", "ping"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-    environment:
-      REDIS_PASSWORD: password

docusaurus-docs/community/contribute/contribute-code.md

@@ -61,13 +61,8 @@ After running the `docker compose` command, the following resources will be avai
 On a fast machine, this deployment should only take 10 or so seconds
 assuming the Docker images have been partially built before.
 
-If you are making any changes to `conda-store-server` and would like to see
-those changes in the deployment, run:
-
-```shell
-docker compose down -v # not always necessary
-docker compose up --build
-```
+Any changes made to the `conda-store-server` will be hot reloaded, so there's no
+need to bring the services down and then up again between edits.
 
 To stop the deployment, run:
 

tests/assets/conda_store_config.py

@@ -46,6 +46,7 @@
 c.CondaStoreServer.enable_ui = True
 c.CondaStoreServer.enable_api = True
 c.CondaStoreServer.enable_registry = True
+c.CondaStoreServer.reload = True
 c.CondaStoreServer.enable_metrics = True
 c.CondaStoreServer.address = "0.0.0.0"
 c.CondaStoreServer.port = 8080

tests/assets/local_dev_config.py

@@ -0,0 +1,87 @@
+# Copyright (c) conda-store development team. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+import logging
+
+from conda_store_server.server.auth import DummyAuthentication
+from conda_store_server.storage import S3Storage
+
+
+# ==================================
+#      conda-store settings
+# ==================================
+# The default storage_threshold limit was reached on CI, which caused test
+# failures
+c.CondaStore.storage_threshold = 1024**3
+c.CondaStore.storage_class = S3Storage
+c.CondaStore.store_directory = "./conda-store-workspace/"
+c.CondaStore.environment_directory = "./conda-store-workspace/envs/{namespace}-{name}"
+# c.CondaStore.database_url = "mysql+pymysql://admin:password@mysql/conda-store"
+c.CondaStore.database_url = (
+    "postgresql+psycopg2://postgres:password@localhost/conda-store"
+)
+c.CondaStore.redis_url = "redis://:password@localhost:6379/0"
+c.CondaStore.default_uid = 1000
+c.CondaStore.default_gid = 1000
+c.CondaStore.default_permissions = "775"
+c.CondaStore.conda_included_packages = ["ipykernel"]
+
+c.CondaStore.pypi_included_packages = ["nothing"]
+
+
+c.S3Storage.internal_endpoint = "minio:9000"
+c.S3Storage.external_endpoint = "localhost:9000"
+c.S3Storage.access_key = "admin"
+c.S3Storage.secret_key = "password"
+c.S3Storage.region = "us-east-1"  # minio region default
+c.S3Storage.bucket_name = "conda-store"
+c.S3Storage.internal_secure = False
+c.S3Storage.external_secure = False
+
+# ==================================
+#        server settings
+# ==================================
+c.CondaStoreServer.log_level = logging.INFO
+c.CondaStoreServer.enable_ui = True
+c.CondaStoreServer.enable_api = True
+c.CondaStoreServer.enable_registry = True
+c.CondaStoreServer.reload = True
+c.CondaStoreServer.enable_metrics = True
+c.CondaStoreServer.address = "0.0.0.0"
+c.CondaStoreServer.port = 8080
+# This MUST start with `/`
+c.CondaStoreServer.url_prefix = "/conda-store"
+
+
+# ==================================
+#         auth settings
+# ==================================
+c.CondaStoreServer.authentication_class = DummyAuthentication
+c.CondaStoreServer.template_vars = {
+    "banner": '<div class="alert alert-danger" role="alert">This is a localhost server</div>',
+    "logo": "https://raw.githubusercontent.com/conda-incubator/conda-store/main/docusaurus-docs/community/assets/logos/conda-store-logo-horizontal-lockup.svg",
+}
+
+# ==================================
+#         worker settings
+# ==================================
+c.CondaStoreWorker.log_level = logging.INFO
+c.CondaStoreWorker.watch_paths = ["./conda-store-workspace/environments"]
+c.CondaStoreWorker.concurrency = 4
+
+# ==================================
+#         registry settings
+# ==================================
+# from python_docker.registry import Registry
+# import os
+
+# def _configure_docker_registry(registry_url: str):
+#     return Registry(
+#         "https://registry-1.docker.io",
+#         username=os.environ.get('DOCKER_USERNAME'),
+#         password=os.environ.get('DOCKER_PASSWORD'))
+
+# c.ContainerRegistry.container_registries = {
+#     'https://registry-1.docker.io': _configure_docker_registry
+# }