add images

cluster/bin/fitnessctl

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+
+set -ex
+
+if [[ -z "${KUBE_NAMESPACE}" ]]
+then
+  kubectl_args=""
+else
+  kubectl_args="-n ${KUBE_NAMESPACE}"
+fi
+
+registry="node2.bdcl:5000"
+
+mlflow_image="${registry}/mlflow-webserver:latest"
+flower_image="${registry}/flower:latest"
+fitness_worker_image="${registry}/fitness-worker:latest"
+deploy_files_dir="${base_dir}/deploy/"
+default_cfg_path="${deploy_files_dir}/kube-fitness-workers.yaml"
+docker_files_dir="./cluster/docker/"
+
+
+function build_app() {
+  echo "Building app..."
+
+  poetry export --without-hashes > requirements.txt
+  poetry build
+
+  echo "Finished app building"
+}
+
+function build_images(){
+  echo "Building images..."
+
+  docker build -f "${docker_files_dir}/mlflow-webserver.dockerfile" -t ${mlflow_image} .
+  docker build -f "${docker_files_dir}/flower.dockerfile" -t ${flower_image} .
+  docker build -f "${docker_files_dir}/worker.dockerfile" -t ${fitness_worker_image} .
+
+  echo "Finished images building"
+}
+
+function push_images() {
+  echo "Pushing images..."
+
+  docker push ${mlflow_image}
+  docker push ${flower_image}
+  docker push ${fitness_worker_image}
+
+  echo "Finished pushing images"
+}
+
+function install() {
+  echo "Installing images..."
+
+  build_app
+  build_images
+  push_images
+
+  echo "Finished installing images"
+}
+
+
+
+
+function help() {
+  echo "
+  Supported env variables:
+    KUBE_NAMESPACE - a kubernetes namespace to make actions in
+
+  List of commands.
+    build-app - build the app as a .whl distribution
+    build-images - build all required docker images
+    push-images - push all required docker images to the private registry on node2.bdcl
+    install-images - build-images and push-images
+    help - prints this message
+  "
+}
+
+function main () {
+    cmd="$1"
+
+    if [ -z "${cmd}" ]
+    then
+      echo "No command is provided."
+      help
+      exit 1
+    fi
+
+    shift 1
+
+    echo "Executing command: ${cmd}"
+
+    case "${cmd}" in
+    "build-app")
+        build_app
+        ;;
+
+    "build-images")
+        build_images
+        ;;
+
+    "push-images")
+        push_images
+        ;;
+
+    "install-images")
+        install
+        ;;
+
+    "help")
+        help
+        ;;
+
+    *)
+        echo "Unknown command: ${cmd}"
+        ;;
+
+    esac
+}
+
+main "${@}"

cluster/docker/flower.dockerfile

@@ -0,0 +1,33 @@
+FROM python:alpine
+
+# Get latest root certificates
+RUN apk add --no-cache ca-certificates && update-ca-certificates
+
+# Install the required packages
+RUN pip install --no-cache-dir redis flower==1.0.0
+
+# PYTHONUNBUFFERED: Force stdin, stdout and stderr to be totally unbuffered. (equivalent to `python -u`)
+# PYTHONHASHSEED: Enable hash randomization (equivalent to `python -R`)
+# PYTHONDONTWRITEBYTECODE: Do not write byte files to disk, since we maintain it as readonly. (equivalent to `python -B`)
+ENV PYTHONUNBUFFERED=1 PYTHONHASHSEED=random PYTHONDONTWRITEBYTECODE=1
+
+# Default port
+EXPOSE 5555
+
+ENV FLOWER_DATA_DIR /data
+ENV PYTHONPATH ${FLOWER_DATA_DIR}
+
+WORKDIR $FLOWER_DATA_DIR
+
+# Add a user with an explicit UID/GID and create necessary directories
+RUN set -eux; \
+    addgroup -g 1000 flower; \
+    adduser -u 1000 -G flower flower -D; \
+    mkdir -p "$FLOWER_DATA_DIR"; \
+    chown flower:flower "$FLOWER_DATA_DIR"
+USER flower
+
+VOLUME $FLOWER_DATA_DIR
+
+# for '-A distributed_fitness' see kube_fitness.tasks.make_celery_app
+ENTRYPOINT celery flower --address=0.0.0.0 --port=5555

cluster/docker/mlflow-webserver.dockerfile

@@ -0,0 +1,5 @@
+FROM python:3.8
+
+RUN pip install PyMySQL==0.9.3 psycopg2-binary==2.8.5 protobuf==3.20.1 mlflow[extras]==1.18.0
+
+ENTRYPOINT ["mlflow", "server"]

cluster/docker/worker.dockerfile

@@ -0,0 +1,18 @@
+FROM ubuntu:20.04
+
+RUN apt-get update && apt-get install -y bash python3 python3-pip
+
+#RUN pip3 install  'celery == 4.4.7' 'bigartm == 0.9.2' 'tqdm == 4.50.2' 'numpy == 1.19.2' 'dataclasses-json == 0.5.2'
+
+COPY requirements.txt /tmp/
+
+RUN pip3 install -r  /tmp/requirements.txt
+
+COPY dist/autotm-0.1.0-py3-none-any.whl /tmp
+
+RUN pip3 install /tmp/autotm-0.1.0-py3-none-any.whl
+
+ENTRYPOINT fitness-worker \
+    --concurrency 1 \
+    --queues fitness_tasks \
+    --loglevel INFO