bump storage-initializer to 0.11.2 (#44)

* bump storage-initializer to 0.11.2
* add tests for storage-initializer

storage-initializer/README.md

@@ -0,0 +1,16 @@
+## Testing
+
+### Instructions
+
+Test the storage-initializer by downloading one of the sample kserve models from google storage:
+```
+docker run storage-initializer:<version> gs://kfserving-examples/models/xgboost/iris /work
+```
+
+We should see:
+```
+INFO:root:Successfully copied gs://kfserving-examples/models/xgboost/iris to /work/stuff
+```
+
+In the logs (some other warnings may occur, but they're normal).  To further test, you can also
+shell into the container while it is running and confirm the file is saved. 

storage-initializer/dummy_pyproject.toml

@@ -0,0 +1,9 @@
+[tool.poetry]
+name = "workaround-for-editable-install"
+version = "0.0.1"
+description = ""
+authors = ["none"]
+
+[tool.poetry.dependencies]
+python = ">=3.8,<3.12"
+kserve = { path = "../python/kserve", develop = false, extras = ["storage"] }

storage-initializer/rockcraft.yaml

@@ -1,24 +1,28 @@
-# Based on https://github.com/kserve/kserve/blob/v0.10.0/python/storage-initializer.Dockerfile
+# Based on https://github.com/kserve/kserve/blob/master/python/storage-initializer.Dockerfile
+#
+# See ../CONTRIBUTING.md for more details about the patterns used in this rock.
+# This rock is implemented with some atypical patterns due to the native of the upstream
 name: storage-initializer
 summary: Storage initializer for Kserve deployments
 description: "Kserve storage initializer"
-version: "0.10.0"
+version: "0.11.2"
 license: Apache-2.0
 base: [email protected]
 platforms:
     amd64:
 run-user: _daemon_
 services:
   storage-initializer:
-    summary: "Kserve storage initializer service"
     override: replace
-    command: "/storage-initializer/scripts/initializer-entrypoint [ args ]"
+    summary: "Kserve storage initializer service"
     startup: enabled
+    command: "/storage-initializer/scripts/initializer-entrypoint [ ]"
     on-success: shutdown
     on-failure: shutdown
     working-dir: /work/
 entrypoint-service: storage-initializer
 
+# TODO
 package-repositories:
   - type: apt
     ppa: deadsnakes/ppa
@@ -32,41 +36,72 @@ parts:
       (echo "# os-release" && cat /etc/os-release && echo "# dpkg-query" && dpkg-query -f '${db:Status-Abbrev},${binary:Package},${Version},${source:Package},${Source:Version}\n' -W) > ${CRAFT_PART_INSTALL}/usr/share/rocks/dpkg.query
 
   python:
-    plugin: python
+    plugin: nil
     source: https://github.com/kserve/kserve.git
-    source-subdir: python
-    source-tag: v0.10.0
-    stage-packages:
-    - python3.9-venv
+    source-tag: v0.11.2
     build-packages:
-    - python3.9
-    - python3.9-venv
-    - python3.9-dev
     - gcc
     - libkrb5-dev
     - krb5-config
-    python-packages:
-    - ./kserve
-    - krbcontext==0.10
-    - hdfs~=2.6.0
-    - requests-kerberos==0.14.0
-    build-environment: 
-    - PARTS_PYTHON_INTERPRETER: python3.9
+    overlay-packages:
+    - python3.10
+    # Including python3-pip here means pip also gets primed for the final rock
+    - python3-pip
     override-build: |
-      ln -s python3.9 "${CRAFT_PART_INSTALL}"/usr/bin/python3
-      craftctl default
+      # Populate the build system's python environment with all packages needed for 
+      # the server in the final rock
+      
+      # Setup poetry
+      pip install poetry==1.7.1
+      poetry config virtualenvs.create false 
+
+      # Install the kserve package, this specific server package, and their dependencies.
+      mkdir -p ./python_env_builddir
+      cp -rf $CRAFT_PROJECT_DIR/dummy_pyproject.toml ./python_env_builddir/pyproject.toml
+      (cd python_env_builddir && poetry install --no-interaction --no-root)
+
+      # Promote the packages we've installed from the local env to the primed image
+      mkdir -p $CRAFT_PART_INSTALL/usr/local/lib/python3.10/dist-packages
+      cp -fr /usr/local/lib/python3.10/dist-packages/* $CRAFT_PART_INSTALL/usr/local/lib/python3.10/dist-packages/
+
+      # TODO: why do we need this?
+      mkdir -p $CRAFT_PART_INSTALL/usr/local/share
+      cp -fr /usr/local/share/* $CRAFT_PART_INSTALL/usr/local/share/
 
-  files:
+      # Ensure `python` is an executable command in our primed image by making
+      # a symbolic link
+      mkdir -p $CRAFT_PART_INSTALL/usr/bin/
+      ln -sf /usr/bin/python3.10 $CRAFT_PART_INSTALL/usr/bin/python
+      ln -sf /usr/bin/python3.10 $CRAFT_PART_INSTALL/usr/bin/python3
+
+      # Install additional packages
+      pip install --no-cache-dir krbcontext==0.10 hdfs~=2.6.0 requests-kerberos==0.14.0
+
+      # Copy the storage-initializer scripts
+      chmod +x ./python/storage-initializer/scripts/initializer-entrypoint
+      cp -r ./python/storage-initializer $CRAFT_PART_INSTALL/storage-initializer
+
+  copy-licenses:
     plugin: nil
+    after: [python]
     source: https://github.com/kserve/kserve.git
     source-subdir: python
-    source-tag: v0.10.0
+    source-tag: v0.11.2
+    override-build: |
+      cp -fr third_party/* ${CRAFT_PART_INSTALL}/third_party
+
+  create-work-dir:
+    # Create a directory that the user has write access to.  This is the default
+    # root location for the script
+    # Note: If any other part writes to this dir, here's part must execute after that
+    #       otherwise the other part's operations will clobber the permissions set here
+    plugin: nil
     override-build: |
-      cd python
+      mkdir -p $CRAFT_PART_INSTALL/work
+    permissions:
+      - path: work
+        # 584792 is the _daemon_ user
+        owner: 584792
+        group: 584792
+        mode: "755"
 
-      cp -r third_party $CRAFT_PART_INSTALL/third_party
-      cp -r kserve $CRAFT_PART_INSTALL/kserve
-      cp VERSION $CRAFT_PART_INSTALL/VERSION
-      cp -r storage-initializer $CRAFT_PART_INSTALL/storage-initializer
-      chmod +x $CRAFT_PART_INSTALL/storage-initializer/scripts/initializer-entrypoint
-      mkdir $CRAFT_PART_INSTALL/work

storage-initializer/tests/test_rock.py

@@ -0,0 +1,73 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+import random
+import pytest
+import string
+import subprocess
+
+from charmed_kubeflow_chisme.rock import CheckRock
+
+
+@pytest.fixture()
+def rock_test_env(tmpdir):
+    """Yields a temporary directory and random docker container name, then cleans them up after."""
+    container_name = "".join(
+        [str(i) for i in random.choices(string.ascii_lowercase, k=8)]
+    )
+    yield tmpdir, container_name
+
+    try:
+        subprocess.run(["docker", "rm", container_name])
+    except Exception:
+        pass
+    # tmpdir fixture we use here should clean up the other files for us
+
+
+@pytest.mark.abort_on_fail
+def test_rock(rock_test_env):
+    """Test rock."""
+    temp_dir, container_name = rock_test_env
+    check_rock = CheckRock("rockcraft.yaml")
+    rock_image = check_rock.get_name()
+    rock_version = check_rock.get_version()
+    LOCAL_ROCK_IMAGE = f"{rock_image}:{rock_version}"
+
+    # assert we have the expected files
+    subprocess.run(
+        [
+            "docker",
+            "run",
+            "--entrypoint",
+            "/bin/bash",
+            LOCAL_ROCK_IMAGE,
+            "-c",
+            "ls -la /usr/local/lib/python3.10/dist-packages/kserve",
+        ],
+        check=True,
+    )
+    subprocess.run(
+        [
+            "docker",
+            "run",
+            "--entrypoint",
+            "/bin/bash",
+            LOCAL_ROCK_IMAGE,
+            "-c",
+            "ls -la /third_party",
+        ],
+        check=True,
+    )
+    subprocess.run(
+        [
+            "docker",
+            "run",
+            "--entrypoint",
+            "/bin/bash",
+            LOCAL_ROCK_IMAGE,
+            "-c",
+            "ls -la /storage-initializer/scripts/initializer-entrypoint",
+        ],
+        check=True,
+    )
+

storage-initializer/tox.ini

@@ -0,0 +1,54 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+[tox]
+skipsdist = True
+skip_missing_interpreters = True
+envlist = pack, export-to-docker, sanity, integration
+
+[testenv]
+setenv =
+    PYTHONPATH={toxinidir}
+    PYTHONBREAKPOINT=ipdb.set_trace
+    CHARM_REPO=https://github.com/canonical/kserve-operators.git
+    CHARM_BRANCH=main
+    LOCAL_CHARM_DIR=charm_repo
+
+[testenv:pack]
+passenv = *
+allowlist_externals =
+    rockcraft
+commands =
+    rockcraft pack
+
+[testenv:export-to-docker]
+passenv = *
+allowlist_externals =
+    bash
+    skopeo
+    yq
+commands =
+    # pack rock and export to docker
+    bash -c 'NAME=$(yq eval .name rockcraft.yaml) && \
+             VERSION=$(yq eval .version rockcraft.yaml) && \
+             ARCH=$(yq eval ".platforms | keys | .[0]" rockcraft.yaml) && \
+             ROCK="$\{NAME\}_$\{VERSION\}_$\{ARCH\}.rock" && \
+             DOCKER_IMAGE=$NAME:$VERSION && \
+             echo "Exporting $ROCK to docker as $DOCKER_IMAGE" && \
+             skopeo --insecure-policy copy oci-archive:$ROCK docker-daemon:$DOCKER_IMAGE'
+
+[testenv:sanity]
+passenv = *
+deps =
+    pytest
+    charmed-kubeflow-chisme
+commands =
+    # run rock tests
+    pytest -s -v --tb native --show-capture=all --log-cli-level=INFO {posargs} {toxinidir}/tests
+
+[testenv:integration]
+passenv = *
+allowlist_externals =
+    echo
+commands =
+    # TODO: Implement integration tests here
+    echo "WARNING: This is a placeholder test - no test is implemented here."