From 15ed04cba11097404945302fe0a6fcaadda9d3bf Mon Sep 17 00:00:00 2001 From: Justin Brooks Date: Wed, 24 May 2023 00:45:38 -0400 Subject: [PATCH 1/5] chore: Docker compose init --- docker-compose.yaml | 124 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 docker-compose.yaml diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..4812b2a --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,124 @@ +version: '3' + +services: + db: + container_name: wandb_db + image: mysql:8.0 + restart: always + networks: + - wandb + environment: + MYSQL_DATABASE: 'wandb' + MYSQL_USER: 'wandb' + MYSQL_PASSWORD: 'wandb' + MYSQL_ROOT_PASSWORD: 'wandb' + ports: + - 3306:3306 + volumes: + - db:/var/lib/mysql + healthcheck: + test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"] + timeout: 20s + interval: 5s + retries: 10 + + redis: + container_name: wandb_redis + image: redis:6 + restart: always + networks: + - wandb + ports: + - 6379:6379 + volumes: + - redis:/data + healthcheck: + interval: 5s + test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ] + + minio: + container_name: wandb_storage + image: minio/minio + restart: always + networks: + - wandb + ports: + - 9000:9000 + - 9001:9001 + volumes: + - files:/data + environment: + MINIO_ROOT_USER: wandb + MINIO_ROOT_PASSWORD: wandbadmin + MINIO_REGION_NAME: us-east-1 + command: server /data + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 5s + timeout: 20s + retries: 5 + + createbucket: + container_name: wandb_createbucket + image: minio/mc + depends_on: + minio: + condition: service_healthy + restart: on-failure + entrypoint: > + /bin/sh -c " + /usr/bin/mc alias set myminio http://minio:9000 wandb wandbadmin; + /usr/bin/mc mb myminio/data; + /usr/bin/mc policy set public myminio/data; + " + networks: + - wandb + + wandb: + container_name: wandb_server + image: wandb/local:latest + restart: always + networks: + - wandb + links: + - db + - redis + - minio + ports: + - 8080:8080 + environment: + MYSQL: mysql://wandb:wandb@db:3306/wandb + BUCKET_QUEUE: internal:// + AWS_REGION: us-east-1 + BUCKET: s3://wandb:wandbadmin@minio:9000/data + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + minio: + condition: service_healthy + + watchtower: + container_name: wandb_autoupgrade + image: containrrr/watchtower + restart: always + environment: + WATCHTOWER_SCHEDULE: 0 0 0 * * 0 # Every sunday + WATCHTOWER_CLEANUP: "true" + WATCHTOWER_DEBUG: "true" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: wandb_server + +volumes: + db: + driver: local + redis: + driver: local + files: + driver: local + +networks: + wandb: + name: wa \ No newline at end of file From c71a96a55d2a78eb69305098e0928f3f0cf00bf1 Mon Sep 17 00:00:00 2001 From: Justin Brooks Date: Wed, 24 May 2023 00:46:11 -0400 Subject: [PATCH 2/5] fix network --- docker-compose.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 4812b2a..efd46a3 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -120,5 +120,4 @@ volumes: driver: local networks: - wandb: - name: wa \ No newline at end of file + wandb: \ No newline at end of file From b185c4a24fac8296816c89d16f334040c030a408 Mon Sep 17 00:00:00 2001 From: Justin Brooks Date: Wed, 24 May 2023 00:46:25 -0400 Subject: [PATCH 3/5] nl --- docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index efd46a3..a5a1767 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -120,4 +120,4 @@ volumes: driver: local networks: - wandb: \ No newline at end of file + wandb: From 1410eeeb6b7b73a0491c35e50e27be9603fd7805 Mon Sep 17 00:00:00 2001 From: Justin Brooks Date: Wed, 24 May 2023 00:59:27 -0400 Subject: [PATCH 4/5] include stoped --- docker-compose.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index a5a1767..814eaee 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -107,6 +107,8 @@ services: WATCHTOWER_SCHEDULE: 0 0 0 * * 0 # Every sunday WATCHTOWER_CLEANUP: "true" WATCHTOWER_DEBUG: "true" + WATCHTOWER_INCLUDE_RESTARTING: "true" + WATCHTOWER_INCLUDE_STOPPED: "true" volumes: - /var/run/docker.sock:/var/run/docker.sock command: wandb_server From 8b6f9e629356a98a231c92b1afb8ebcc44b060ef Mon Sep 17 00:00:00 2001 From: Zachary Blasczyk <77289967+wandb-zacharyblasczyk@users.noreply.github.com> Date: Wed, 24 May 2023 09:21:49 -0500 Subject: [PATCH 5/5] Create docker-compose.md --- docker-compose.md | 73 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 docker-compose.md diff --git a/docker-compose.md b/docker-compose.md new file mode 100644 index 0000000..ba5042b --- /dev/null +++ b/docker-compose.md @@ -0,0 +1,73 @@ +# W&B Server Docker Compose README + +This repository contains a Docker Compose configuration for setting up a W&B (Weights & Biases) server. This configuration includes various services necessary for the operation of the W&B server, including a database, cache, storage, and a system for updating the W&B server container. + +## Services + +1. `db` - A MySQL database used for storing W&B data. +2. `redis` - A Redis cache used by the W&B server. +3. `minio` - A MinIO server that acts as an S3-compatible storage for the W&B server. +4. `createbucket` - A service that sets up the MinIO bucket for the W&B server. +5. `wandb` - The W&B server itself. +6. `watchtower` - A service that automatically updates the W&B server container. + +## Environment Variables + +Some of the services use environment variables for configuration. These should be changed as per your requirements. + +- `MYSQL_DATABASE`, `MYSQL_USER`, `MYSQL_PASSWORD`, and `MYSQL_ROOT_PASSWORD` are used for the MySQL database configuration. +- `MINIO_ROOT_USER` and `MINIO_ROOT_PASSWORD` are used for the MinIO server configuration. +- `WATCHTOWER_SCHEDULE`, `WATCHTOWER_CLEANUP`, `WATCHTOWER_DEBUG`, `WATCHTOWER_INCLUDE_RESTARTING`, and `WATCHTOWER_INCLUDE_STOPPED` are used for the Watchtower configuration. + +## How to Run + +Make sure Docker and Docker Compose are installed on your machine. + +1. Clone this repository. + + ``` + git clone + ``` + +2. Navigate to the repository folder. + + ``` + cd + ``` + +3. Run the Docker Compose configuration. + + ``` + docker-compose up + ``` + +## Ports + +The following ports are used by the services: + +- `db` - 3306 +- `redis` - 6379 +- `minio` - 9000, 9001 +- `wandb` - 8080 + +## Healthchecks + +Healthchecks are set up for the `db`, `redis`, and `minio` services. These are used to ensure that these services are running correctly. + +## Volumes + +Docker volumes are used for data persistence across container restarts. The following volumes are used: + +- `db` - Used for storing MySQL database data. +- `redis` - Used for storing Redis cache data. +- `files` - Used for storing MinIO files. + +## Networks + +A Docker network named `wandb` is used for communication between the services. + +## Autoupgrade + +The `watchtower` service is set up to automatically upgrade the `wandb_server` container every Sunday. This can be adjusted as per your requirements. + +Please note, this setup requires Docker and Docker Compose to run. If you need any further assistance, feel free to reach out or open an issue.