Split services to enable easier hot reloading in development

.gitignore

@@ -61,4 +61,7 @@ yarn-error.log*
 # conda-store
 conda-store.sqlite
 
+# Development workspace
+conda-store-workspace/
+
 *.lockb

conda-store-server/pyproject.toml

@@ -94,9 +94,13 @@ dependencies = [
   "pytest-playwright",
   "twine>=5.0.0",
   "pkginfo>=1.10", # Needed to support metadata 2.3
-
 ]
 
+[tool.hatch.envs.dev.scripts]
+server = "conda-store-server --config ../tests/assets/local_dev_config.py"
+worker = "conda-store-worker --config ../tests/assets/local_dev_config.py"
+services = "docker compose -f ../docker-compose-core.yml up --build"
+
 [tool.hatch.envs.lint]
 dependencies = ["pre-commit"]
 

docker-compose-core.yml

@@ -0,0 +1,42 @@
+name: conda-store-services
+services:
+  minio:
+    image: minio/minio:RELEASE.2020-11-10T21-02-24Z
+    ports:
+      - "9000:9000"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    entrypoint: sh
+    command: -c 'mkdir -p /data/conda-store && /usr/bin/minio server /data'
+    environment:
+      MINIO_ACCESS_KEY: admin
+      MINIO_SECRET_KEY: password
+
+  postgres:
+    image: postgres:13
+    user: postgres
+    ports:
+      - 5432:5432
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    environment:
+      POSTGRES_PASSWORD: password
+      POSTGRES_DB: conda-store
+
+  redis:
+    image: bitnami/redis
+    ports:
+      - 6379:6379
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    environment:
+      REDIS_PASSWORD: password

docker-compose.yaml

@@ -1,4 +1,17 @@
 services:
+  minio:
+    extends:
+      file: docker-compose-core.yml
+      service: minio
+  postgres:
+    extends:
+      file: docker-compose-core.yml
+      service: postgres
+  redis:
+    extends:
+      file: docker-compose-core.yml
+      service: redis
+
   conda-store-worker:
     build:
       context: conda-store-server
@@ -7,7 +20,6 @@ services:
     volumes:
       - ./tests/assets/environments:/opt/environments:ro
       - ./tests/assets/conda_store_config.py:/opt/conda_store/conda_store_config.py:ro
-      - ./conda-store-server/:/opt/conda-store-server/:delegated
     depends_on:
       conda-store-server:
         condition: service_healthy
@@ -33,7 +45,6 @@ services:
         condition: service_healthy
     volumes:
       - ./tests/assets/conda_store_config.py:/opt/conda_store/conda_store_config.py:ro
-      - ./conda-store-server/:/opt/conda-store-server/:delegated
     healthcheck:
       test:
         ["CMD", "curl", "--fail", "http://localhost:8080/conda-store/api/v1/"]
@@ -49,42 +60,3 @@ services:
       ]
     ports:
       - "8080:8080"
-
-  minio:
-    image: minio/minio:RELEASE.2020-11-10T21-02-24Z
-    ports:
-      - "9000:9000"
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-    entrypoint: sh
-    command: -c 'mkdir -p /data/conda-store && /usr/bin/minio server /data'
-    environment:
-      MINIO_ACCESS_KEY: admin
-      MINIO_SECRET_KEY: password
-
-  postgres:
-    image: postgres:13
-    user: postgres
-    ports:
-      - 5432:5432
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-    environment:
-      POSTGRES_PASSWORD: password
-      POSTGRES_DB: conda-store
-
-  redis:
-    image: bitnami/redis
-    healthcheck:
-      test: ["CMD", "redis-cli", "ping"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-    environment:
-      REDIS_PASSWORD: password

tests/assets/local_dev_config.py

@@ -0,0 +1,86 @@
+# Copyright (c) conda-store development team. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+import logging
+
+from conda_store_server.server.auth import DummyAuthentication
+from conda_store_server.storage import S3Storage
+
+# ==================================
+#      conda-store settings
+# ==================================
+# The default storage_threshold limit was reached on CI, which caused test
+# failures
+c.CondaStore.storage_threshold = 1024**3
+c.CondaStore.storage_class = S3Storage
+c.CondaStore.store_directory = "./conda-store-workspace/"
+c.CondaStore.environment_directory = "./conda-store-workspace/envs/{namespace}-{name}"
+# c.CondaStore.database_url = "mysql+pymysql://admin:password@mysql/conda-store"
+c.CondaStore.database_url = (
+    "postgresql+psycopg2://postgres:password@localhost/conda-store"
+)
+c.CondaStore.redis_url = "redis://:password@localhost:6379/0"
+c.CondaStore.default_uid = 1000
+c.CondaStore.default_gid = 1000
+c.CondaStore.default_permissions = "775"
+c.CondaStore.conda_included_packages = ["ipykernel"]
+
+c.CondaStore.pypi_included_packages = ["nothing"]
+
+
+c.S3Storage.internal_endpoint = "minio:9000"
+c.S3Storage.external_endpoint = "localhost:9000"
+c.S3Storage.access_key = "admin"
+c.S3Storage.secret_key = "password"
+c.S3Storage.region = "us-east-1"  # minio region default
+c.S3Storage.bucket_name = "conda-store"
+c.S3Storage.internal_secure = False
+c.S3Storage.external_secure = False
+
+# ==================================
+#        server settings
+# ==================================
+c.CondaStoreServer.log_level = logging.INFO
+c.CondaStoreServer.enable_ui = True
+c.CondaStoreServer.enable_api = True
+c.CondaStoreServer.enable_registry = True
+c.CondaStoreServer.reload = True
+c.CondaStoreServer.enable_metrics = True
+c.CondaStoreServer.address = "0.0.0.0"
+c.CondaStoreServer.port = 8080
+# This MUST start with `/`
+c.CondaStoreServer.url_prefix = "/conda-store"
+
+
+# ==================================
+#         auth settings
+# ==================================
+c.CondaStoreServer.authentication_class = DummyAuthentication
+c.CondaStoreServer.template_vars = {
+    "banner": '<div class="alert alert-danger" role="alert">This is a localhost server</div>',
+    "logo": "https://raw.githubusercontent.com/conda-incubator/conda-store/main/docusaurus-docs/community/assets/logos/conda-store-logo-horizontal-lockup.svg",
+}
+
+# ==================================
+#         worker settings
+# ==================================
+c.CondaStoreWorker.log_level = logging.INFO
+c.CondaStoreWorker.watch_paths = ["./conda-store-workspace/environments"]
+c.CondaStoreWorker.concurrency = 4
+
+# ==================================
+#         registry settings
+# ==================================
+# from python_docker.registry import Registry
+# import os
+
+# def _configure_docker_registry(registry_url: str):
+#     return Registry(
+#         "https://registry-1.docker.io",
+#         username=os.environ.get('DOCKER_USERNAME'),
+#         password=os.environ.get('DOCKER_PASSWORD'))
+
+# c.ContainerRegistry.container_registries = {
+#     'https://registry-1.docker.io': _configure_docker_registry
+# }