From 9a34d3d18f5d8a69b8f40234ea8bfb89052c2e82 Mon Sep 17 00:00:00 2001 From: pdmurray Date: Thu, 24 Oct 2024 13:25:01 -0700 Subject: [PATCH] Split services to enable easier hot reloading in development --- .gitignore | 3 ++ conda-store-server/pyproject.toml | 6 ++- docker-compose-core.yml | 42 +++++++++++++++ docker-compose.yaml | 54 +++++-------------- tests/assets/local_dev_config.py | 86 +++++++++++++++++++++++++++++++ 5 files changed, 149 insertions(+), 42 deletions(-) create mode 100644 docker-compose-core.yml create mode 100644 tests/assets/local_dev_config.py diff --git a/.gitignore b/.gitignore index 0ec82f551..b4224ac18 100644 --- a/.gitignore +++ b/.gitignore @@ -61,4 +61,7 @@ yarn-error.log* # conda-store conda-store.sqlite +# Development workspace +conda-store-workspace/ + *.lockb diff --git a/conda-store-server/pyproject.toml b/conda-store-server/pyproject.toml index d2e2a970d..1e40b10ae 100644 --- a/conda-store-server/pyproject.toml +++ b/conda-store-server/pyproject.toml @@ -94,9 +94,13 @@ dependencies = [ "pytest-playwright", "twine>=5.0.0", "pkginfo>=1.10", # Needed to support metadata 2.3 - ] +[tool.hatch.envs.dev.scripts] +server = "conda-store-server --config ../tests/assets/local_dev_config.py" +worker = "conda-store-worker --config ../tests/assets/local_dev_config.py" +services = "docker compose -f ../docker-compose-core.yml up --build" + [tool.hatch.envs.lint] dependencies = ["pre-commit"] diff --git a/docker-compose-core.yml b/docker-compose-core.yml new file mode 100644 index 000000000..abc42f0d5 --- /dev/null +++ b/docker-compose-core.yml @@ -0,0 +1,42 @@ +name: conda-store-services +services: + minio: + image: minio/minio:RELEASE.2020-11-10T21-02-24Z + ports: + - "9000:9000" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 10s + timeout: 5s + retries: 5 + entrypoint: sh + command: -c 'mkdir -p /data/conda-store && /usr/bin/minio server /data' + environment: + MINIO_ACCESS_KEY: admin + MINIO_SECRET_KEY: password + + postgres: + image: postgres:13 + user: postgres + ports: + - 5432:5432 + healthcheck: + test: ["CMD-SHELL", "pg_isready"] + interval: 10s + timeout: 5s + retries: 5 + environment: + POSTGRES_PASSWORD: password + POSTGRES_DB: conda-store + + redis: + image: bitnami/redis + ports: + - 6379:6379 + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + environment: + REDIS_PASSWORD: password diff --git a/docker-compose.yaml b/docker-compose.yaml index d276e5b69..d5304c084 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,4 +1,17 @@ services: + minio: + extends: + file: docker-compose-core.yml + service: minio + postgres: + extends: + file: docker-compose-core.yml + service: postgres + redis: + extends: + file: docker-compose-core.yml + service: redis + conda-store-worker: build: context: conda-store-server @@ -7,7 +20,6 @@ services: volumes: - ./tests/assets/environments:/opt/environments:ro - ./tests/assets/conda_store_config.py:/opt/conda_store/conda_store_config.py:ro - - ./conda-store-server/:/opt/conda-store-server/:delegated depends_on: conda-store-server: condition: service_healthy @@ -33,7 +45,6 @@ services: condition: service_healthy volumes: - ./tests/assets/conda_store_config.py:/opt/conda_store/conda_store_config.py:ro - - ./conda-store-server/:/opt/conda-store-server/:delegated healthcheck: test: ["CMD", "curl", "--fail", "http://localhost:8080/conda-store/api/v1/"] @@ -49,42 +60,3 @@ services: ] ports: - "8080:8080" - - minio: - image: minio/minio:RELEASE.2020-11-10T21-02-24Z - ports: - - "9000:9000" - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 10s - timeout: 5s - retries: 5 - entrypoint: sh - command: -c 'mkdir -p /data/conda-store && /usr/bin/minio server /data' - environment: - MINIO_ACCESS_KEY: admin - MINIO_SECRET_KEY: password - - postgres: - image: postgres:13 - user: postgres - ports: - - 5432:5432 - healthcheck: - test: ["CMD-SHELL", "pg_isready"] - interval: 10s - timeout: 5s - retries: 5 - environment: - POSTGRES_PASSWORD: password - POSTGRES_DB: conda-store - - redis: - image: bitnami/redis - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 10s - timeout: 5s - retries: 5 - environment: - REDIS_PASSWORD: password diff --git a/tests/assets/local_dev_config.py b/tests/assets/local_dev_config.py new file mode 100644 index 000000000..802c49cc4 --- /dev/null +++ b/tests/assets/local_dev_config.py @@ -0,0 +1,86 @@ +# Copyright (c) conda-store development team. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import logging + +from conda_store_server.server.auth import DummyAuthentication +from conda_store_server.storage import S3Storage + +# ================================== +# conda-store settings +# ================================== +# The default storage_threshold limit was reached on CI, which caused test +# failures +c.CondaStore.storage_threshold = 1024**3 +c.CondaStore.storage_class = S3Storage +c.CondaStore.store_directory = "./conda-store-workspace/" +c.CondaStore.environment_directory = "./conda-store-workspace/envs/{namespace}-{name}" +# c.CondaStore.database_url = "mysql+pymysql://admin:password@mysql/conda-store" +c.CondaStore.database_url = ( + "postgresql+psycopg2://postgres:password@localhost/conda-store" +) +c.CondaStore.redis_url = "redis://:password@localhost:6379/0" +c.CondaStore.default_uid = 1000 +c.CondaStore.default_gid = 1000 +c.CondaStore.default_permissions = "775" +c.CondaStore.conda_included_packages = ["ipykernel"] + +c.CondaStore.pypi_included_packages = ["nothing"] + + +c.S3Storage.internal_endpoint = "minio:9000" +c.S3Storage.external_endpoint = "localhost:9000" +c.S3Storage.access_key = "admin" +c.S3Storage.secret_key = "password" +c.S3Storage.region = "us-east-1" # minio region default +c.S3Storage.bucket_name = "conda-store" +c.S3Storage.internal_secure = False +c.S3Storage.external_secure = False + +# ================================== +# server settings +# ================================== +c.CondaStoreServer.log_level = logging.INFO +c.CondaStoreServer.enable_ui = True +c.CondaStoreServer.enable_api = True +c.CondaStoreServer.enable_registry = True +c.CondaStoreServer.reload = True +c.CondaStoreServer.enable_metrics = True +c.CondaStoreServer.address = "0.0.0.0" +c.CondaStoreServer.port = 8080 +# This MUST start with `/` +c.CondaStoreServer.url_prefix = "/conda-store" + + +# ================================== +# auth settings +# ================================== +c.CondaStoreServer.authentication_class = DummyAuthentication +c.CondaStoreServer.template_vars = { + "banner": '', + "logo": "https://raw.githubusercontent.com/conda-incubator/conda-store/main/docusaurus-docs/community/assets/logos/conda-store-logo-horizontal-lockup.svg", +} + +# ================================== +# worker settings +# ================================== +c.CondaStoreWorker.log_level = logging.INFO +c.CondaStoreWorker.watch_paths = ["./conda-store-workspace/environments"] +c.CondaStoreWorker.concurrency = 4 + +# ================================== +# registry settings +# ================================== +# from python_docker.registry import Registry +# import os + +# def _configure_docker_registry(registry_url: str): +# return Registry( +# "https://registry-1.docker.io", +# username=os.environ.get('DOCKER_USERNAME'), +# password=os.environ.get('DOCKER_PASSWORD')) + +# c.ContainerRegistry.container_registries = { +# 'https://registry-1.docker.io': _configure_docker_registry +# }