From 05f471246d3e780995b173339e6668997a2acec8 Mon Sep 17 00:00:00 2001 From: Jeongkyu Shin Date: Thu, 4 Jul 2024 21:51:14 +0900 Subject: [PATCH] update: halfstack packages to point latest stable versions --- changes/2367.deps.md | 1 + docker-compose.halfstack-2409.yml | 69 +++++++++++ docker-compose.halfstack-main.yml | 2 +- docs/dev/daily-workflows.rst | 20 ++++ docs/dev/version-management-and-upgrades.rst | 109 ++++++++++++++++++ .../install-from-package/prepare-database.rst | 2 +- 6 files changed, 201 insertions(+), 2 deletions(-) create mode 100644 changes/2367.deps.md create mode 100644 docker-compose.halfstack-2409.yml diff --git a/changes/2367.deps.md b/changes/2367.deps.md new file mode 100644 index 00000000000..1ee9043434d --- /dev/null +++ b/changes/2367.deps.md @@ -0,0 +1 @@ +Update the halfstack containers to point the latest stable versions diff --git a/docker-compose.halfstack-2409.yml b/docker-compose.halfstack-2409.yml new file mode 100644 index 00000000000..6397fe451aa --- /dev/null +++ b/docker-compose.halfstack-2409.yml @@ -0,0 +1,69 @@ +services: + + backendai-half-db: + image: postgres:16.3-alpine + restart: unless-stopped + command: postgres -c 'max_connections=256' + networks: + - half + ports: + - "8100:5432" + environment: + - POSTGRES_PASSWORD=develove + - POSTGRES_DB=backend + volumes: + - "./volumes/${DATADIR_PREFIX:-.}/postgres-data:/var/lib/postgresql/data:rw" + healthcheck: + test: ["CMD", "pg_isready", "-U", "postgres"] + interval: 5s + timeout: 3s + retries: 10 + + backendai-half-redis: + image: redis:7.2.4-alpine + restart: unless-stopped + networks: + - half + ports: + - "8110:6379" + volumes: + - "./volumes/${DATADIR_PREFIX:-.}/redis-data:/data:rw" + command: > + redis-server + --appendonly yes + healthcheck: + test: ["CMD-SHELL", "redis-cli ping | grep PONG"] + interval: 5s + timeout: 3s + retries: 10 + + backendai-half-etcd: + image: quay.io/coreos/etcd:v3.5.14 + restart: unless-stopped + volumes: + - "./volumes/${DATADIR_PREFIX:-.}/etcd-data:/etcd-data:rw" + networks: + - half + ports: + - "8120:2379" + command: > + /usr/local/bin/etcd + --name backendai-etcd + --data-dir /etcd-data + --listen-client-urls http://0.0.0.0:2379 + --advertise-client-urls http://0.0.0.0:2379 + --listen-peer-urls http://0.0.0.0:2380 + --initial-advertise-peer-urls http://0.0.0.0:2380 + --initial-cluster backendai-etcd=http://0.0.0.0:2380 + --initial-cluster-token backendai-etcd-token + --initial-cluster-state new + --enable-v2=true + --auto-compaction-retention 1 + healthcheck: + test: ["CMD", "etcdctl", "endpoint", "health"] + interval: 5s + timeout: 3s + retries: 10 + +networks: + half: diff --git a/docker-compose.halfstack-main.yml b/docker-compose.halfstack-main.yml index 747569b752b..d7c00a40505 120000 --- a/docker-compose.halfstack-main.yml +++ b/docker-compose.halfstack-main.yml @@ -1 +1 @@ -docker-compose.halfstack-2303.yml \ No newline at end of file +./docker-compose.halfstack-2409.yml \ No newline at end of file diff --git a/docs/dev/daily-workflows.rst b/docs/dev/daily-workflows.rst index 5de0ce25539..611332e6953 100644 --- a/docs/dev/daily-workflows.rst +++ b/docs/dev/daily-workflows.rst @@ -733,6 +733,26 @@ Making a new release To make workflow above effective, be aware that backporting DB revisions to older major releases will no longer be permitted after major release version is switched. +Making a new release branch +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This example shows the case when the current release is 24.03 and the next upcoming release is 24.09. +It makes the main branch to stand for the upcoming release 24.09, by branching out the current release 24.03. + +* Make a new git branch for the current release in the ``YY.MM`` format (like ``24.03``) from the main branch. + +* Update ``./VERSION`` file to indicate the next development version (like ``24.09.0dev0``). + +* Create a new halfstack compose configuration for the next release by copying and updating the halfstack config of the current release. + + .. code-block:: console + + $ cp docker-compose.halfstack-2403.yml docker-compose.halfstack-2409.yml + $ edit docker-compose.halfstack-2409.yml # update the container versions + $ rm docker-compose.halfstack-main.yml + $ ln -s docker-compose.halfstack-2409.yml docker-compose.halfstack-main.yml + $ git add docker-compose.*.yml + Backporting to legacy per-pkg repositories ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/dev/version-management-and-upgrades.rst b/docs/dev/version-management-and-upgrades.rst index f2cdc76fa60..a04c52e5228 100644 --- a/docs/dev/version-management-and-upgrades.rst +++ b/docs/dev/version-management-and-upgrades.rst @@ -50,3 +50,112 @@ It is recommended to re-run ``pip install -U -r requirements.txt`` as dependenci For the manager, ensure that your database schema is up-to-date by running ``alembic upgrade head``. If you setup your development environment with Pants and ``install-dev.sh`` script, keep your database schema up-to-date via ``./py -m alembic upgrade head`` instead of plain alembic command above. Also check if any manual etcd configuration scheme change is required, though we will try to keep it compatible and automatically upgrade when first executed. + + +Halfstack / Component Update Guide +================================== + +Backend.AI relies on various components for its operation, including a database, etcd registry, container registry, and kvstore. Backend.AI primarily uses PostgreSQL as the database, etcd as the registry, and Redis as the kvstore. + +These components also get updated periodically due to reasons such as stability improvements, security enhancements, and new features. Generally, these component updates are automated, but for PostgreSQL, the version upgrade must be performed manually. Below is the guide on how to do this. + +Even for the halfstack setup for development, these upgrades are necessary. If you are installing from scratch, this is not required. However, if you need to upgrade the stack while maintaining the database contents of your development environment, follow the steps below. + +General Update (Components Installed via Container) +--------------------------------------------------- + +1. Turn off Backend.AI manager/agent/storage-proxy. +2. Backup the current database:: + + docker compose -p [DOCKER_COMPOSE_PREFIX] -f [DOCKER_COMPOSE_YAML_FILE] exec -T [SERVICE_NAME] pg_dump -U [DB_USER] backend > [BACKUP_FILE_NAME] + +3. Stop all containers:: + + docker compose -f [DOCKER_COMPOSE_YAML_FILE] -p [DOCKER_COMPOSE_PREFIX] down + +4. Delete the database volume. + + .. note:: + Be careful, this will delete all Backend.AI database! Ensure that your backup can be restored before you start. + + :: + + # Data backup + cp -Rp [POSTGRES_DATA_DIR] [POSTGRES_DATA_DIR]/postgres-data-backup + rm -rf [POSTGRES_DATA_DIR]/* + +5. Change the PostgreSQL version in `docker-compose.yml` and start the container. Below is an example to change the PostgreSQL version to 16. + + .. code-block:: yaml + + services: + + backendai-half-db: + image: postgres:16-alpine + container_name: backendai-db + +6. Start the container:: + + docker compose -p [DOCKER_COMPOSE_PREFIX] -f [DOCKER_COMPOSE_YAML_FILE] up -d + +7. Restore the database:: + + docker compose -p [DOCKER_COMPOSE_PREFIX] -f [DOCKER_COMPOSE_YAML_FILE] exec -T [DB_SERVICE_NAME] psql -U [DB_USER] -d backend < ./db_backup.bck + +8. Run Backend.AI and test. + +Halfstack Configuration (For Developers) Update +----------------------------------------------- + +This guide uses the default halfstack configuration, such as: +- Service name in `docker-compose.yaml`: backendai-half-db +- Default database user: postgres +- Default database: backend +- `docker-compose` YAML file name: `docker-compose.halfstack.2009.yml` +- `docker-compose` prefix: 2303 + +1. Turn off Backend.AI manager/agent/storage-proxy. +2. Backup the current database:: + + docker compose -p 2303 -f backend.ai-dev/backend.ai/docker-compose.halfstack.2303.yml exec -T backendai-half-db pg_dump -U postgres backend > ./db_backup.bck + +3. Stop all containers:: + + docker compose -f ./backend.ai-dev/backend.ai/docker-compose.halfstack.yml -p 2303 down + +4. Delete the database volume. + + .. note:: + Be careful, this will delete all Backend.AI database! Ensure that your backup can be restored before you start. + + :: + + cp -Rp ./backend.ai-dev/backend.ai/tmp/backend.ai-halfstack/postgres-data ./backend.ai-dev/backend.ai/tmp/backend.ai-halfstack/postgres-data-backup + rm -rf ./backend.ai-dev/backend.ai/tmp/backend.ai-halfstack/postgres-data/* + +5. Change the PostgreSQL version in `docker-compose.yml` and start the container. Below is an example to change the PostgreSQL version to 16. + + .. code-block:: yaml + + services: + + backendai-half-db: + image: postgres:16-alpine + container_name: backendai-db + +6. Start the container:: + + docker compose -p 2303 -f backend.ai-dev/backend.ai/docker-compose.halfstack.2303.yml up -d + +7. Restore the database:: + + docker compose -p 2303 -f backend.ai-dev/backend.ai/docker-compose.halfstack.2303.yml exec -T backendai-half-db psql -U postgres -d backend < ./db_backup.bck + +8. Run Backend.AI and test. + +Additional Information +---------------------- + +- Make sure all backups are verified for integrity before starting the upgrade process. +- Always use the appropriate `docker-compose` YAML file and prefix for your setup. +- For specific configurations or advanced setups, refer to the official Backend.AI documentation or contact support. \ No newline at end of file diff --git a/docs/install/install-from-package/prepare-database.rst b/docs/install/install-from-package/prepare-database.rst index a4678064543..ed42ba4b0fd 100644 --- a/docs/install/install-from-package/prepare-database.rst +++ b/docs/install/install-from-package/prepare-database.rst @@ -21,7 +21,7 @@ refer services: backendai-pg-active: <<: *base - image: postgres:15.1-alpine + image: postgres:16.3-alpine restart: unless-stopped command: > postgres