From 43e44b3b09088af5cb8abb7ee8ba37cd9ad91703 Mon Sep 17 00:00:00 2001 From: Julio Martinez Date: Tue, 17 Dec 2024 09:00:16 -0800 Subject: [PATCH 01/16] Initial hack at adding copy support --- python-sdk/indexify/cli.py | 130 +++++++++--------- python-sdk/indexify/executor/api_objects.py | 1 + python-sdk/indexify/functions_sdk/image.py | 138 ++++++++++++++++++-- 3 files changed, 196 insertions(+), 73 deletions(-) diff --git a/python-sdk/indexify/cli.py b/python-sdk/indexify/cli.py index c51412f5e..af27ee57e 100644 --- a/python-sdk/indexify/cli.py +++ b/python-sdk/indexify/cli.py @@ -2,7 +2,7 @@ configure_logging_early() - +import httpx import asyncio import os import shutil @@ -161,6 +161,32 @@ def build_image( _create_image(obj, python_sdk_path) +@app.command(help="Build platform images for function names") +def build_platform_image( + workflow_file_path: Optional[str] = typer.Option(help="Path to the executor cache directory"), + image_names: Optional[List[str]] = None): + globals_dict = {} + + # Add the folder in the workflow file path to the current Python path + folder_path = os.path.dirname(workflow_file_path) + if folder_path not in sys.path: + sys.path.append(folder_path) + + try: + exec(open(workflow_file_path).read(), globals_dict) + except FileNotFoundError as e: + raise Exception( + f"Could not find workflow file to execute at: " f"`{workflow_file_path}`" + ) + for _, obj in globals_dict.items(): + if type(obj) and isinstance(obj, Image): + if image_names is None or obj._image_name in image_names: + _create_platform_image(obj) + + + + + @app.command(help="Build default image for indexify") def build_default_image( python_version: Optional[str] = typer.Option( @@ -230,7 +256,6 @@ def executor( image_version=image_version, development_mode=dev, ) - try: asyncio.get_event_loop().run_until_complete(agent.run()) except asyncio.CancelledError: @@ -266,72 +291,55 @@ def function_executor( ).run() -def _create_image(image: Image, python_sdk_path): - console.print( - Text("Creating container for ", style="cyan"), - Text(f"`{image._image_name}`", style="cyan bold"), - ) - _build_image(image=image, python_sdk_path=python_sdk_path) +IMAGE_SERVICE="http://localhost:8000" -def _build_image(image: Image, python_sdk_path: Optional[str] = None): +def _create_platform_image(image: Image): + # TODO: Wire this up to use the proxy with API keys + response = httpx.post(f"{IMAGE_SERVICE}/images", data=image.to_builder_image().model_dump_json()) + if response.status_code != 200: + response.raise_for_status() - try: - import docker - - client = docker.from_env() - client.ping() - except Exception as e: - console.print( - Text("Unable to connect with docker: ", style="red bold"), - Text(f"{e}", style="red"), - ) - exit(-1) - - docker_contents = [ - f"FROM {image._base_image}", - "RUN mkdir -p ~/.indexify", - "RUN touch ~/.indexify/image_name", - f"RUN echo {image._image_name} > ~/.indexify/image_name", - f"RUN echo {image.hash()} > ~/.indexify/image_hash", - "WORKDIR /app", - ] - - docker_contents.extend(["RUN " + i for i in image._run_strs]) - - if python_sdk_path is not None: - logging.info( - f"Building image {image._image_name} with local version of the SDK" - ) - if not os.path.exists(python_sdk_path): - print(f"error: {python_sdk_path} does not exist") - os.exit(1) - docker_contents.append(f"COPY {python_sdk_path} /app/python-sdk") - docker_contents.append("RUN (cd /app/python-sdk && pip install .)") - else: - docker_contents.append(f"RUN pip install indexify=={image._sdk_version}") + image_id = response.json()['id'] - docker_file = "\n".join(docker_contents) + # Get the latest build for this image + latest_build_response = httpx.get(f"{IMAGE_SERVICE}/images/{image_id}/latest_completed_build") + if latest_build_response.status_code == 404: # No completed builds, exist. Starting one now. + logger.info(f"No existing builds found for {image._image_name}") + create_build_response = httpx.put(f"{IMAGE_SERVICE}/images/{image_id}/build") - import docker.api.build + if create_build_response.status_code not in (201, 200): + response.raise_for_status() - docker.api.build.process_dockerfile = lambda dockerfile, path: ( - "Dockerfile", - dockerfile, - ) + latest_build = create_build_response.json() + build_id = latest_build['id'] + + # Block and poll while build completes + logger.info("Waiting for build to complete") + while latest_build['status'] != 'completed': + latest_build_response = httpx.get(f"{IMAGE_SERVICE}/build/{build_id}") + if latest_build_response.status_code != 200: + latest_build_response.raise_for_status() + latest_build = latest_build_response.json() + time.sleep(1) - console.print("Creating image using Dockerfile contents:", style="cyan bold") - print(f"{docker_file}") + logger.info(f"New build is hosted in {latest_build["uri"]}") + elif latest_build_response.status_code == 200: + latest_build = latest_build_response.json() + logger.info(f"Image is already built and hosted on {latest_build['uri']}") + else: + response.raise_for_status() - client = docker.from_env() - image_name = f"{image._image_name}:{image._tag}" - (_image, generator) = client.images.build( - path=".", - dockerfile=docker_file, - tag=image_name, - rm=True, +def _create_image(image: Image, python_sdk_path): + console.print( + Text("Creating container for ", style="cyan"), + Text(f"`{image._image_name}`", style="cyan bold"), ) - for result in generator: - print(result) + _build_image(image=image, python_sdk_path=python_sdk_path) - print(f"built image: {image_name}") + +def _build_image(image: Image, python_sdk_path: Optional[str] = None): + built_image, output = image.build(python_sdk_path=python_sdk_path) + for line in output: + print(line) + print(f"built image: {built_image.tags[0]}") diff --git a/python-sdk/indexify/executor/api_objects.py b/python-sdk/indexify/executor/api_objects.py index 4753de81b..845900653 100644 --- a/python-sdk/indexify/executor/api_objects.py +++ b/python-sdk/indexify/executor/api_objects.py @@ -37,3 +37,4 @@ class TaskResult(BaseModel): executor_id: str task_id: str reducer: bool = False + diff --git a/python-sdk/indexify/functions_sdk/image.py b/python-sdk/indexify/functions_sdk/image.py index 503855469..c91b1fa25 100644 --- a/python-sdk/indexify/functions_sdk/image.py +++ b/python-sdk/indexify/functions_sdk/image.py @@ -1,20 +1,83 @@ import hashlib import importlib import sys +import os +import enum from typing import List, Optional +import pathlib +import logging from pydantic import BaseModel +import docker + + +import docker.api.build + +docker.api.build.process_dockerfile = lambda dockerfile, path: ( + "Dockerfile", + dockerfile, + ) + # Pydantic object for API class ImageInformation(BaseModel): image_name: str - tag: str - base_image: str - run_strs: List[str] + image_hash: str image_url: Optional[str] = "" sdk_version: str +# Version of the builder used by the builder service + +class BuildOpType(enum.Enum): + RUN = "RUN" + COPY = "COPY" + # ADD = "ADD" + # FROM = "FROM" + # WORKDIR = "WORKDIR" + +HASH_BUFF_SIZE=1024**2 + +class BuildOp(BaseModel): + op_type:BuildOpType + args:List[str] + + def hash(self, hash): + match self.op_type: + case BuildOpType.RUN: + hash.update("RUN".encode()) + for a in self.args: hash.update(a.encode()) + + case BuildOpType.COPY: + hash.update("COPY".encode()) + for root, dirs, files in os.walk(self.args[0]): + for file in files: + filename = pathlib.Path(root, file) + with open(filename, "rb") as fp: + data = fp.read(HASH_BUFF_SIZE) + while data: + hash.update(data) + data = fp.read(HASH_BUFF_SIZE) + + case _: + raise ValueError(f"Unsupported build op type {self.op_type}") + + def render(self): + match self.op_type: + case BuildOpType.RUN: + return f"RUN {"".join(self.args)}" + case BuildOpType.COPY: + return f"COPY {self.args[0]} {self.args[1]}" + case _: + raise ValueError(f"Unsupported build op type {self.op_type}") + + +class BuilderImage(BaseModel): + namespace: str + sdk_version:str + name:str + build_ops: List[BuildOp] + hash:str class Image: def __init__(self): @@ -22,7 +85,7 @@ def __init__(self): self._tag = "latest" self._base_image = BASE_IMAGE_NAME self._python_version = LOCAL_PYTHON_VERSION - self._run_strs = [] + self._build_ops = [] # List of ImageOperation self._sdk_version = importlib.metadata.version("indexify") def name(self, image_name): @@ -38,16 +101,68 @@ def base_image(self, base_image): return self def run(self, run_str): - self._run_strs.append(run_str) + self._build_ops.append(BuildOp(op_type=BuildOpType.RUN, args=[run_str])) return self + def copy(self, source:str, dest:str): + self._build_ops.append(BuildOp(op_type=BuildOpType.COPY, args=[source, dest])) + return self + def to_image_information(self): return ImageInformation( image_name=self._image_name, - tag=self._tag, - base_image=self._base_image, - run_strs=self._run_strs, sdk_version=self._sdk_version, + image_hash=self.hash() + ) + + def to_builder_image(self, namespace="default"): + return BuilderImage( + namespace=namespace, + name = self._image_name, + sdk_version=self._sdk_version, + hash=self.hash(), + build_ops=self._build_ops, + ) + + def build(self, python_sdk_path:Optional[str]=None, docker_client=None): + if docker_client is None: + docker_client = docker.from_env() + docker_client.ping() + + docker_contents = [ + f"FROM {self._base_image}", + "RUN mkdir -p ~/.indexify", + "RUN touch ~/.indexify/image_name", + f"RUN echo {self._image_name} > ~/.indexify/image_name", + f"RUN echo {self.hash()} > ~/.indexify/image_hash", + "WORKDIR /app", + ] + + for build_op in self._build_ops: + docker_contents.append(build_op.render()) + + if python_sdk_path is not None: + logging.info( + f"Building image {self._image_name} with local version of the SDK" + ) + if not os.path.exists(python_sdk_path): + print(f"error: {python_sdk_path} does not exist") + os.exit(1) + docker_contents.append(f"COPY {python_sdk_path} /app/python-sdk") + docker_contents.append("RUN (cd /app/python-sdk && pip install .)") + else: + docker_contents.append(f"RUN pip install indexify=={self._sdk_version}") + + docker_file = "\n".join(docker_contents) + print(docker_file) + + image_name = f"{self._image_name}:{self._tag}" + + return docker_client.images.build( + path=".", + dockerfile=docker_file, + tag=image_name, + rm=True, ) def hash(self) -> str: @@ -55,16 +170,16 @@ def hash(self) -> str: self._image_name.encode() ) # Make a hash of the image name hash.update(self._base_image.encode()) - hash.update("".join(self._run_strs).encode()) + for op in self._build_ops: + op.hash(hash) + hash.update(self._sdk_version.encode()) return hash.hexdigest() - LOCAL_PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}" BASE_IMAGE_NAME = f"python:{LOCAL_PYTHON_VERSION}-slim-bookworm" - def GetDefaultPythonImage(python_version: str): return ( Image() @@ -73,5 +188,4 @@ def GetDefaultPythonImage(python_version: str): .tag(python_version) ) - DEFAULT_IMAGE = GetDefaultPythonImage(LOCAL_PYTHON_VERSION) From bbe1b004db2808533759e1149584190ad5318da6 Mon Sep 17 00:00:00 2001 From: Julio Martinez Date: Wed, 18 Dec 2024 11:21:48 -0800 Subject: [PATCH 02/16] Added COPY support to image builder --- python-sdk/indexify/cli.py | 89 +++++++++++++--------- python-sdk/indexify/functions_sdk/image.py | 87 ++++++++++++--------- 2 files changed, 101 insertions(+), 75 deletions(-) diff --git a/python-sdk/indexify/cli.py b/python-sdk/indexify/cli.py index af27ee57e..55cc05c47 100644 --- a/python-sdk/indexify/cli.py +++ b/python-sdk/indexify/cli.py @@ -13,6 +13,7 @@ import time from importlib.metadata import version from typing import Annotated, List, Optional +import tempfile import nanoid import structlog @@ -28,9 +29,8 @@ ) from indexify.function_executor.server import Server as FunctionExecutorServer from indexify.functions_sdk.image import ( - LOCAL_PYTHON_VERSION, GetDefaultPythonImage, - Image, + Image, Build ) from indexify.http_client import IndexifyClient @@ -290,45 +290,61 @@ def function_executor( ), ).run() - - IMAGE_SERVICE="http://localhost:8000" def _create_platform_image(image: Image): - # TODO: Wire this up to use the proxy with API keys - response = httpx.post(f"{IMAGE_SERVICE}/images", data=image.to_builder_image().model_dump_json()) - if response.status_code != 200: - response.raise_for_status() - - image_id = response.json()['id'] - - # Get the latest build for this image - latest_build_response = httpx.get(f"{IMAGE_SERVICE}/images/{image_id}/latest_completed_build") - if latest_build_response.status_code == 404: # No completed builds, exist. Starting one now. - logger.info(f"No existing builds found for {image._image_name}") - create_build_response = httpx.put(f"{IMAGE_SERVICE}/images/{image_id}/build") - - if create_build_response.status_code not in (201, 200): - response.raise_for_status() - - latest_build = create_build_response.json() - build_id = latest_build['id'] - - # Block and poll while build completes - logger.info("Waiting for build to complete") - while latest_build['status'] != 'completed': - latest_build_response = httpx.get(f"{IMAGE_SERVICE}/build/{build_id}") - if latest_build_response.status_code != 200: - latest_build_response.raise_for_status() - latest_build = latest_build_response.json() - time.sleep(1) + fd, context_file = tempfile.mkstemp() + image.build_context(context_file) + client = httpx + + + image_hash = image.hash() + + # Check if the image is built before pushing a new one + builds_response = client.get(f"{IMAGE_SERVICE}/builds", params={"namespace":"default", "image_name":image._image_name, "image_hash":image_hash}) + builds_response.raise_for_status() + matching_builds = [Build.model_validate(b) for b in builds_response.json()] + if not matching_builds: + files = { + "context":open(context_file, "rb") + } + + data = { + "namespace": "default", + "name":image._image_name, + "hash":image_hash + } - logger.info(f"New build is hosted in {latest_build["uri"]}") - elif latest_build_response.status_code == 200: - latest_build = latest_build_response.json() - logger.info(f"Image is already built and hosted on {latest_build['uri']}") + res = client.post(f"{IMAGE_SERVICE}/builds", data=data, files=files) + res.raise_for_status() + + build = Build.model_validate(res.json()) else: - response.raise_for_status() + build = matching_builds[0] + + match build.status: + case "completed": + print(f"image {build.image_name}:{build.image_hash} is already built") + case "ready" | "building": + print(f"waiting for {build.image_name} image to build") + while build.status != 'completed': + res = client.get(f"{IMAGE_SERVICE}/builds/{build.id}") + build = Build.model_validate(res.json()) + time.sleep(5) + + case _: + raise ValueError(f"Unexpected build status {build.status}") + + match build.result: + case "success": + build_duration = build.push_completed_at - build.started_at + print(f"Building completed in {build_duration}; image is stored in {build.uri}") + + case "failed": + print(f"Building failed, please see logs for details") + + case _: + raise ValueError(f"Unexpected build result {build.status}") def _create_image(image: Image, python_sdk_path): console.print( @@ -337,7 +353,6 @@ def _create_image(image: Image, python_sdk_path): ) _build_image(image=image, python_sdk_path=python_sdk_path) - def _build_image(image: Image, python_sdk_path: Optional[str] = None): built_image, output = image.build(python_sdk_path=python_sdk_path) for line in output: diff --git a/python-sdk/indexify/functions_sdk/image.py b/python-sdk/indexify/functions_sdk/image.py index c91b1fa25..9e588f32f 100644 --- a/python-sdk/indexify/functions_sdk/image.py +++ b/python-sdk/indexify/functions_sdk/image.py @@ -2,16 +2,17 @@ import importlib import sys import os -import enum +import tarfile from typing import List, Optional import pathlib import logging +from io import BytesIO from pydantic import BaseModel import docker - +import datetime import docker.api.build docker.api.build.process_dockerfile = lambda dockerfile, path: ( @@ -27,28 +28,19 @@ class ImageInformation(BaseModel): image_url: Optional[str] = "" sdk_version: str -# Version of the builder used by the builder service - -class BuildOpType(enum.Enum): - RUN = "RUN" - COPY = "COPY" - # ADD = "ADD" - # FROM = "FROM" - # WORKDIR = "WORKDIR" - HASH_BUFF_SIZE=1024**2 class BuildOp(BaseModel): - op_type:BuildOpType + op_type:str args:List[str] def hash(self, hash): match self.op_type: - case BuildOpType.RUN: + case "RUN": hash.update("RUN".encode()) for a in self.args: hash.update(a.encode()) - case BuildOpType.COPY: + case "COPY": hash.update("COPY".encode()) for root, dirs, files in os.walk(self.args[0]): for file in files: @@ -64,20 +56,32 @@ def hash(self, hash): def render(self): match self.op_type: - case BuildOpType.RUN: + case "RUN": return f"RUN {"".join(self.args)}" - case BuildOpType.COPY: + case "COPY": return f"COPY {self.args[0]} {self.args[1]}" case _: raise ValueError(f"Unsupported build op type {self.op_type}") -class BuilderImage(BaseModel): +class Build(BaseModel): + """ + Model for talking with the build service. + """ + + id: int | None=None namespace: str - sdk_version:str - name:str - build_ops: List[BuildOp] - hash:str + image_name: str + image_hash: str + status: str | None + result: str | None + + created_at: datetime.datetime | None + started_at: datetime.datetime | None = None + build_completed_at: datetime.datetime | None = None + push_completed_at: datetime.datetime | None = None + uri:str | None = None + class Image: def __init__(self): @@ -101,11 +105,11 @@ def base_image(self, base_image): return self def run(self, run_str): - self._build_ops.append(BuildOp(op_type=BuildOpType.RUN, args=[run_str])) + self._build_ops.append(BuildOp(op_type="RUN", args=[run_str])) return self def copy(self, source:str, dest:str): - self._build_ops.append(BuildOp(op_type=BuildOpType.COPY, args=[source, dest])) + self._build_ops.append(BuildOp(op_type="COPY", args=[source, dest])) return self def to_image_information(self): @@ -114,21 +118,22 @@ def to_image_information(self): sdk_version=self._sdk_version, image_hash=self.hash() ) - - def to_builder_image(self, namespace="default"): - return BuilderImage( - namespace=namespace, - name = self._image_name, - sdk_version=self._sdk_version, - hash=self.hash(), - build_ops=self._build_ops, - ) - def build(self, python_sdk_path:Optional[str]=None, docker_client=None): - if docker_client is None: - docker_client = docker.from_env() - docker_client.ping() - + def build_context(self, filename:str): + with tarfile.open(filename, "w:gz") as tf: + for op in self._build_ops: + if op.op_type == "COPY": + src = op.args[0] + logging.info(f"Adding {src}") + tf.add(src, src) + + dockerfile = self._generate_dockerfile() + tarinfo = tarfile.TarInfo('Dockerfile') + tarinfo.size = len(dockerfile) + + tf.addfile(tarinfo, BytesIO(dockerfile.encode())) + + def _generate_dockerfile(self, python_sdk_path:Optional[str]=None): docker_contents = [ f"FROM {self._base_image}", "RUN mkdir -p ~/.indexify", @@ -154,8 +159,14 @@ def build(self, python_sdk_path:Optional[str]=None, docker_client=None): docker_contents.append(f"RUN pip install indexify=={self._sdk_version}") docker_file = "\n".join(docker_contents) - print(docker_file) + return docker_file + + def build(self, python_sdk_path:Optional[str]=None, docker_client=None): + if docker_client is None: + docker_client = docker.from_env() + docker_client.ping() + docker_file = self._generate_dockerfile(python_sdk_path=python_sdk_path) image_name = f"{self._image_name}:{self._tag}" return docker_client.images.build( From 636594bc4b0faef52f41f92e3c0546e24f8ac5f9 Mon Sep 17 00:00:00 2001 From: Julio Martinez Date: Wed, 18 Dec 2024 13:45:45 -0800 Subject: [PATCH 03/16] Reworked Image to support COPY. - Adds a simple extensible framework for supporting multiple build operations. Right now only RUN and COPY are implemented. - Refactored Dockerfile generation into the Image class. Rewired existing builders to use this. - Re-implemented `build-platform-image` to use new builder service API. --- python-sdk/indexify/cli.py | 57 +++++++++--------- python-sdk/indexify/functions_sdk/image.py | 68 ++++++++++++---------- 2 files changed, 64 insertions(+), 61 deletions(-) diff --git a/python-sdk/indexify/cli.py b/python-sdk/indexify/cli.py index 7673ea428..7cbb88ea8 100644 --- a/python-sdk/indexify/cli.py +++ b/python-sdk/indexify/cli.py @@ -2,19 +2,19 @@ configure_logging_early() -import httpx import asyncio import os import shutil import signal import subprocess import sys +import tempfile import threading import time from importlib.metadata import version from typing import Annotated, List, Optional -import tempfile +import httpx import nanoid import structlog import typer @@ -28,10 +28,7 @@ FunctionExecutorService, ) from indexify.function_executor.server import Server as FunctionExecutorServer -from indexify.functions_sdk.image import ( - GetDefaultPythonImage, - Image, Build -) +from indexify.functions_sdk.image import Build, GetDefaultPythonImage, Image from indexify.http_client import IndexifyClient logger = structlog.get_logger(module=__name__) @@ -163,8 +160,11 @@ def build_image( @app.command(help="Build platform images for function names") def build_platform_image( - workflow_file_path: Optional[str] = typer.Option(help="Path to the executor cache directory"), - image_names: Optional[List[str]] = None): + workflow_file_path: Annotated[str, typer.Argument()], + image_names: Optional[List[str]] = None, + build_service= "https://api.tensorlake.ai/images" +): + globals_dict = {} # Add the folder in the workflow file path to the current Python path @@ -181,10 +181,7 @@ def build_platform_image( for _, obj in globals_dict.items(): if type(obj) and isinstance(obj, Image): if image_names is None or obj._image_name in image_names: - _create_platform_image(obj) - - - + _create_platform_image(obj, build_service) @app.command(help="Build default image for indexify") @@ -290,32 +287,30 @@ def function_executor( ), ).run() -IMAGE_SERVICE="http://localhost:8000" - -def _create_platform_image(image: Image): +def _create_platform_image(image: Image, service_endpoint:str): fd, context_file = tempfile.mkstemp() image.build_context(context_file) client = httpx - image_hash = image.hash() # Check if the image is built before pushing a new one - builds_response = client.get(f"{IMAGE_SERVICE}/builds", params={"namespace":"default", "image_name":image._image_name, "image_hash":image_hash}) + builds_response = client.get( + f"{service_endpoint}/builds", + params={ + "namespace": "default", + "image_name": image._image_name, + "image_hash": image_hash, + }, + ) builds_response.raise_for_status() matching_builds = [Build.model_validate(b) for b in builds_response.json()] if not matching_builds: - files = { - "context":open(context_file, "rb") - } + files = {"context": open(context_file, "rb")} - data = { - "namespace": "default", - "name":image._image_name, - "hash":image_hash - } + data = {"namespace": "default", "name": image._image_name, "hash": image_hash} - res = client.post(f"{IMAGE_SERVICE}/builds", data=data, files=files) + res = client.post(f"{service_endpoint}/builds", data=data, files=files) res.raise_for_status() build = Build.model_validate(res.json()) @@ -327,8 +322,8 @@ def _create_platform_image(image: Image): print(f"image {build.image_name}:{build.image_hash} is already built") case "ready" | "building": print(f"waiting for {build.image_name} image to build") - while build.status != 'completed': - res = client.get(f"{IMAGE_SERVICE}/builds/{build.id}") + while build.status != "completed": + res = client.get(f"{service_endpoint}/builds/{build.id}") build = Build.model_validate(res.json()) time.sleep(5) @@ -338,7 +333,9 @@ def _create_platform_image(image: Image): match build.result: case "success": build_duration = build.push_completed_at - build.started_at - print(f"Building completed in {build_duration}; image is stored in {build.uri}") + print( + f"Building completed in {build_duration}; image is stored in {build.uri}" + ) case "failed": print(f"Building failed, please see logs for details") @@ -346,6 +343,7 @@ def _create_platform_image(image: Image): case _: raise ValueError(f"Unexpected build result {build.status}") + def _create_image(image: Image, python_sdk_path): console.print( Text("Creating container for ", style="cyan"), @@ -353,6 +351,7 @@ def _create_image(image: Image, python_sdk_path): ) _build_image(image=image, python_sdk_path=python_sdk_path) + def _build_image(image: Image, python_sdk_path: Optional[str] = None): built_image, output = image.build(python_sdk_path=python_sdk_path) for line in output: diff --git a/python-sdk/indexify/functions_sdk/image.py b/python-sdk/indexify/functions_sdk/image.py index 9e588f32f..23c6cc43d 100644 --- a/python-sdk/indexify/functions_sdk/image.py +++ b/python-sdk/indexify/functions_sdk/image.py @@ -1,24 +1,22 @@ +import datetime import hashlib import importlib -import sys +import logging import os -import tarfile -from typing import List, Optional import pathlib -import logging +import sys +import tarfile from io import BytesIO - -from pydantic import BaseModel +from typing import List, Optional import docker - -import datetime import docker.api.build +from pydantic import BaseModel docker.api.build.process_dockerfile = lambda dockerfile, path: ( - "Dockerfile", - dockerfile, - ) + "Dockerfile", + dockerfile, +) # Pydantic object for API @@ -28,17 +26,20 @@ class ImageInformation(BaseModel): image_url: Optional[str] = "" sdk_version: str -HASH_BUFF_SIZE=1024**2 + +HASH_BUFF_SIZE = 1024**2 + class BuildOp(BaseModel): - op_type:str - args:List[str] - + op_type: str + args: List[str] + def hash(self, hash): match self.op_type: case "RUN": hash.update("RUN".encode()) - for a in self.args: hash.update(a.encode()) + for a in self.args: + hash.update(a.encode()) case "COPY": hash.update("COPY".encode()) @@ -50,10 +51,10 @@ def hash(self, hash): while data: hash.update(data) data = fp.read(HASH_BUFF_SIZE) - + case _: raise ValueError(f"Unsupported build op type {self.op_type}") - + def render(self): match self.op_type: case "RUN": @@ -69,10 +70,10 @@ class Build(BaseModel): Model for talking with the build service. """ - id: int | None=None + id: int | None = None namespace: str image_name: str - image_hash: str + image_hash: str status: str | None result: str | None @@ -80,7 +81,7 @@ class Build(BaseModel): started_at: datetime.datetime | None = None build_completed_at: datetime.datetime | None = None push_completed_at: datetime.datetime | None = None - uri:str | None = None + uri: str | None = None class Image: @@ -89,7 +90,7 @@ def __init__(self): self._tag = "latest" self._base_image = BASE_IMAGE_NAME self._python_version = LOCAL_PYTHON_VERSION - self._build_ops = [] # List of ImageOperation + self._build_ops = [] # List of ImageOperation self._sdk_version = importlib.metadata.version("indexify") def name(self, image_name): @@ -108,18 +109,18 @@ def run(self, run_str): self._build_ops.append(BuildOp(op_type="RUN", args=[run_str])) return self - def copy(self, source:str, dest:str): + def copy(self, source: str, dest: str): self._build_ops.append(BuildOp(op_type="COPY", args=[source, dest])) return self - + def to_image_information(self): return ImageInformation( image_name=self._image_name, sdk_version=self._sdk_version, - image_hash=self.hash() + image_hash=self.hash(), ) - - def build_context(self, filename:str): + + def build_context(self, filename: str): with tarfile.open(filename, "w:gz") as tf: for op in self._build_ops: if op.op_type == "COPY": @@ -128,12 +129,12 @@ def build_context(self, filename:str): tf.add(src, src) dockerfile = self._generate_dockerfile() - tarinfo = tarfile.TarInfo('Dockerfile') + tarinfo = tarfile.TarInfo("Dockerfile") tarinfo.size = len(dockerfile) tf.addfile(tarinfo, BytesIO(dockerfile.encode())) - - def _generate_dockerfile(self, python_sdk_path:Optional[str]=None): + + def _generate_dockerfile(self, python_sdk_path: Optional[str] = None): docker_contents = [ f"FROM {self._base_image}", "RUN mkdir -p ~/.indexify", @@ -141,7 +142,7 @@ def _generate_dockerfile(self, python_sdk_path:Optional[str]=None): f"RUN echo {self._image_name} > ~/.indexify/image_name", f"RUN echo {self.hash()} > ~/.indexify/image_hash", "WORKDIR /app", - ] + ] for build_op in self._build_ops: docker_contents.append(build_op.render()) @@ -161,7 +162,7 @@ def _generate_dockerfile(self, python_sdk_path:Optional[str]=None): docker_file = "\n".join(docker_contents) return docker_file - def build(self, python_sdk_path:Optional[str]=None, docker_client=None): + def build(self, python_sdk_path: Optional[str] = None, docker_client=None): if docker_client is None: docker_client = docker.from_env() docker_client.ping() @@ -188,9 +189,11 @@ def hash(self) -> str: return hash.hexdigest() + LOCAL_PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}" BASE_IMAGE_NAME = f"python:{LOCAL_PYTHON_VERSION}-slim-bookworm" + def GetDefaultPythonImage(python_version: str): return ( Image() @@ -199,4 +202,5 @@ def GetDefaultPythonImage(python_version: str): .tag(python_version) ) + DEFAULT_IMAGE = GetDefaultPythonImage(LOCAL_PYTHON_VERSION) From e61afef63dc21b772c4378a9c0570334a77dc05d Mon Sep 17 00:00:00 2001 From: Julio Martinez Date: Thu, 19 Dec 2024 14:44:00 -0800 Subject: [PATCH 04/16] Moved docker overrride to build function, fixed the lints. --- python-sdk/indexify/cli.py | 24 +++++++++++++++------ python-sdk/indexify/executor/api_objects.py | 1 - python-sdk/indexify/functions_sdk/image.py | 10 ++++----- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/python-sdk/indexify/cli.py b/python-sdk/indexify/cli.py index 7cbb88ea8..8f8ee3d7c 100644 --- a/python-sdk/indexify/cli.py +++ b/python-sdk/indexify/cli.py @@ -162,9 +162,9 @@ def build_image( def build_platform_image( workflow_file_path: Annotated[str, typer.Argument()], image_names: Optional[List[str]] = None, - build_service= "https://api.tensorlake.ai/images" + build_service="https://api.tensorlake.ai/images/v1", ): - + globals_dict = {} # Add the folder in the workflow file path to the current Python path @@ -287,30 +287,38 @@ def function_executor( ), ).run() -def _create_platform_image(image: Image, service_endpoint:str): + +def _create_platform_image(image: Image, service_endpoint: str): fd, context_file = tempfile.mkstemp() image.build_context(context_file) client = httpx + headers = {} + api_key = os.getenv("TENSORLAKE_API_KEY") + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + image_hash = image.hash() # Check if the image is built before pushing a new one builds_response = client.get( f"{service_endpoint}/builds", params={ - "namespace": "default", "image_name": image._image_name, "image_hash": image_hash, }, + headers=headers, ) builds_response.raise_for_status() matching_builds = [Build.model_validate(b) for b in builds_response.json()] if not matching_builds: files = {"context": open(context_file, "rb")} - data = {"namespace": "default", "name": image._image_name, "hash": image_hash} + data = {"name": image._image_name, "hash": image_hash} - res = client.post(f"{service_endpoint}/builds", data=data, files=files) + res = client.post( + f"{service_endpoint}/builds", data=data, files=files, headers=headers + ) res.raise_for_status() build = Build.model_validate(res.json()) @@ -323,7 +331,9 @@ def _create_platform_image(image: Image, service_endpoint:str): case "ready" | "building": print(f"waiting for {build.image_name} image to build") while build.status != "completed": - res = client.get(f"{service_endpoint}/builds/{build.id}") + res = client.get( + f"{service_endpoint}/builds/{build.id}", headers=headers + ) build = Build.model_validate(res.json()) time.sleep(5) diff --git a/python-sdk/indexify/executor/api_objects.py b/python-sdk/indexify/executor/api_objects.py index 75f8afbb8..7f54fd78b 100644 --- a/python-sdk/indexify/executor/api_objects.py +++ b/python-sdk/indexify/executor/api_objects.py @@ -37,4 +37,3 @@ class TaskResult(BaseModel): executor_id: str task_id: str reducer: bool = False - diff --git a/python-sdk/indexify/functions_sdk/image.py b/python-sdk/indexify/functions_sdk/image.py index 23c6cc43d..c8d84b508 100644 --- a/python-sdk/indexify/functions_sdk/image.py +++ b/python-sdk/indexify/functions_sdk/image.py @@ -13,11 +13,6 @@ import docker.api.build from pydantic import BaseModel -docker.api.build.process_dockerfile = lambda dockerfile, path: ( - "Dockerfile", - dockerfile, -) - # Pydantic object for API class ImageInformation(BaseModel): @@ -170,6 +165,11 @@ def build(self, python_sdk_path: Optional[str] = None, docker_client=None): docker_file = self._generate_dockerfile(python_sdk_path=python_sdk_path) image_name = f"{self._image_name}:{self._tag}" + docker.api.build.process_dockerfile = lambda dockerfile, path: ( + "Dockerfile", + dockerfile, + ) + return docker_client.images.build( path=".", dockerfile=docker_file, From e70fbfaac0287daa400e3d38122ba029710fd092 Mon Sep 17 00:00:00 2001 From: Julio Martinez Date: Fri, 20 Dec 2024 07:59:37 -0800 Subject: [PATCH 05/16] Fix double quoted string --- python-sdk/indexify/functions_sdk/image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-sdk/indexify/functions_sdk/image.py b/python-sdk/indexify/functions_sdk/image.py index c8d84b508..ca73dc554 100644 --- a/python-sdk/indexify/functions_sdk/image.py +++ b/python-sdk/indexify/functions_sdk/image.py @@ -53,7 +53,7 @@ def hash(self, hash): def render(self): match self.op_type: case "RUN": - return f"RUN {"".join(self.args)}" + return f"RUN {''.join(self.args)}" case "COPY": return f"COPY {self.args[0]} {self.args[1]}" case _: From 07b3ce44c7cb5959606c9d78c2d31ca8b65b6aa6 Mon Sep 17 00:00:00 2001 From: Julio Martinez Date: Fri, 20 Dec 2024 11:26:03 -0800 Subject: [PATCH 06/16] Add image_hash to API and mark deprecated fields --- python-sdk/indexify/functions_sdk/image.py | 11 +++--- server/data_model/src/lib.rs | 44 +++++++++++----------- server/src/http_objects.rs | 10 +++-- 3 files changed, 34 insertions(+), 31 deletions(-) diff --git a/python-sdk/indexify/functions_sdk/image.py b/python-sdk/indexify/functions_sdk/image.py index ca73dc554..f983a0499 100644 --- a/python-sdk/indexify/functions_sdk/image.py +++ b/python-sdk/indexify/functions_sdk/image.py @@ -13,18 +13,20 @@ import docker.api.build from pydantic import BaseModel - # Pydantic object for API class ImageInformation(BaseModel): image_name: str image_hash: str image_url: Optional[str] = "" sdk_version: str - - + + # These are deprecated and here for backwards compatibility + run_strs: List[str] | None=[] + tag:str | None="" + base_image:str | None="" + HASH_BUFF_SIZE = 1024**2 - class BuildOp(BaseModel): op_type: str args: List[str] @@ -78,7 +80,6 @@ class Build(BaseModel): push_completed_at: datetime.datetime | None = None uri: str | None = None - class Image: def __init__(self): self._image_name = None diff --git a/server/data_model/src/lib.rs b/server/data_model/src/lib.rs index b86828dcc..ca2eeaa7d 100644 --- a/server/data_model/src/lib.rs +++ b/server/data_model/src/lib.rs @@ -105,23 +105,29 @@ pub struct ImageInformation { impl ImageInformation { pub fn new( image_name: String, + image_hash: String, tag: String, base_image: String, run_strs: Vec, sdk_version: Option, ) -> Self { - let mut image_hasher = Sha256::new(); - image_hasher.update(image_name.clone()); - image_hasher.update(base_image.clone()); - image_hasher.update(run_strs.clone().join("")); - image_hasher.update(sdk_version.clone().unwrap_or("".to_string())); // Igh..... + let mut compat_image_hash: String = "".to_string(); + if image_hash == "" { + // Preserve backwards compatibility with old hash calculation + let mut image_hasher = Sha256::new(); + image_hasher.update(image_name.clone()); + image_hasher.update(base_image.clone()); + image_hasher.update(run_strs.clone().join("")); + image_hasher.update(sdk_version.clone().unwrap_or("".to_string())); // Igh..... + compat_image_hash = format!("{:x}", image_hasher.finalize()) + } ImageInformation { image_name, tag, base_image, run_strs, - image_hash: format!("{:x}", image_hasher.finalize()), + image_hash: compat_image_hash, version: ImageVersion::default(), image_uri: None, sdk_version, @@ -385,12 +391,12 @@ impl ComputeGraph { let mut graph_version: Option = None; - if self.code.sha256_hash != update.code.sha256_hash || - self.runtime_information != update.runtime_information || - self.edges != update.edges || - self.start_fn != update.start_fn || - self.nodes.len() != update.nodes.len() || - self.nodes.iter().any(|(k, v)| { + if self.code.sha256_hash != update.code.sha256_hash + || self.runtime_information != update.runtime_information + || self.edges != update.edges + || self.start_fn != update.start_fn + || self.nodes.len() != update.nodes.len() + || self.nodes.iter().any(|(k, v)| { update .nodes .get(k) @@ -1040,17 +1046,9 @@ mod tests { use std::collections::HashMap; use crate::{ - test_objects::tests::test_compute_fn, - ComputeFn, - ComputeGraph, - ComputeGraphCode, - ComputeGraphVersion, - DynamicEdgeRouter, - ExecutorMetadata, - GraphVersion, - ImageInformation, - Node, - RuntimeInformation, + test_objects::tests::test_compute_fn, ComputeFn, ComputeGraph, ComputeGraphCode, + ComputeGraphVersion, DynamicEdgeRouter, ExecutorMetadata, GraphVersion, ImageInformation, + Node, RuntimeInformation, }; #[test] diff --git a/server/src/http_objects.rs b/server/src/http_objects.rs index 99a3d7147..d311b9936 100644 --- a/server/src/http_objects.rs +++ b/server/src/http_objects.rs @@ -88,9 +88,11 @@ pub struct NamespaceList { #[derive(Clone, Serialize, Deserialize, ToSchema)] pub struct ImageInformation { pub image_name: String, - pub tag: String, - pub base_image: String, - pub run_strs: Vec, + #[serde(default)] + pub image_hash: String, + pub tag: String, // Deprecated + pub base_image: String, // Deprecated + pub run_strs: Vec, // Deprecated pub image_uri: Option, pub sdk_version: Option, } @@ -110,6 +112,7 @@ impl From for data_model::ImageInformation { fn from(value: ImageInformation) -> Self { data_model::ImageInformation::new( value.image_name, + value.image_hash, value.tag, value.base_image, value.run_strs, @@ -122,6 +125,7 @@ impl From for ImageInformation { fn from(value: data_model::ImageInformation) -> ImageInformation { ImageInformation { image_name: value.image_name, + image_hash: value.image_hash, tag: value.tag, base_image: value.base_image, run_strs: value.run_strs, From a41c539ba575b0f57318c444dc8ceacad931f3ff Mon Sep 17 00:00:00 2001 From: Julio Martinez Date: Fri, 20 Dec 2024 11:27:17 -0800 Subject: [PATCH 07/16] Lint for the lint god. --- python-sdk/indexify/functions_sdk/image.py | 14 +++++++----- server/data_model/src/lib.rs | 26 ++++++++++++++-------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/python-sdk/indexify/functions_sdk/image.py b/python-sdk/indexify/functions_sdk/image.py index f983a0499..45a6033e6 100644 --- a/python-sdk/indexify/functions_sdk/image.py +++ b/python-sdk/indexify/functions_sdk/image.py @@ -13,20 +13,23 @@ import docker.api.build from pydantic import BaseModel + # Pydantic object for API class ImageInformation(BaseModel): image_name: str image_hash: str image_url: Optional[str] = "" sdk_version: str - + # These are deprecated and here for backwards compatibility - run_strs: List[str] | None=[] - tag:str | None="" - base_image:str | None="" - + run_strs: List[str] | None = [] + tag: str | None = "" + base_image: str | None = "" + + HASH_BUFF_SIZE = 1024**2 + class BuildOp(BaseModel): op_type: str args: List[str] @@ -80,6 +83,7 @@ class Build(BaseModel): push_completed_at: datetime.datetime | None = None uri: str | None = None + class Image: def __init__(self): self._image_name = None diff --git a/server/data_model/src/lib.rs b/server/data_model/src/lib.rs index ca2eeaa7d..baf1ca8ef 100644 --- a/server/data_model/src/lib.rs +++ b/server/data_model/src/lib.rs @@ -391,12 +391,12 @@ impl ComputeGraph { let mut graph_version: Option = None; - if self.code.sha256_hash != update.code.sha256_hash - || self.runtime_information != update.runtime_information - || self.edges != update.edges - || self.start_fn != update.start_fn - || self.nodes.len() != update.nodes.len() - || self.nodes.iter().any(|(k, v)| { + if self.code.sha256_hash != update.code.sha256_hash || + self.runtime_information != update.runtime_information || + self.edges != update.edges || + self.start_fn != update.start_fn || + self.nodes.len() != update.nodes.len() || + self.nodes.iter().any(|(k, v)| { update .nodes .get(k) @@ -1046,9 +1046,17 @@ mod tests { use std::collections::HashMap; use crate::{ - test_objects::tests::test_compute_fn, ComputeFn, ComputeGraph, ComputeGraphCode, - ComputeGraphVersion, DynamicEdgeRouter, ExecutorMetadata, GraphVersion, ImageInformation, - Node, RuntimeInformation, + test_objects::tests::test_compute_fn, + ComputeFn, + ComputeGraph, + ComputeGraphCode, + ComputeGraphVersion, + DynamicEdgeRouter, + ExecutorMetadata, + GraphVersion, + ImageInformation, + Node, + RuntimeInformation, }; #[test] From 69b0b1e101d01aaacd69f7dd5b8ea26afce11852 Mon Sep 17 00:00:00 2001 From: Julio Martinez Date: Fri, 20 Dec 2024 11:34:06 -0800 Subject: [PATCH 08/16] Remove unused test --- server/data_model/src/lib.rs | 43 ++++++++---------------------------- 1 file changed, 9 insertions(+), 34 deletions(-) diff --git a/server/data_model/src/lib.rs b/server/data_model/src/lib.rs index baf1ca8ef..0a00069c8 100644 --- a/server/data_model/src/lib.rs +++ b/server/data_model/src/lib.rs @@ -391,12 +391,12 @@ impl ComputeGraph { let mut graph_version: Option = None; - if self.code.sha256_hash != update.code.sha256_hash || - self.runtime_information != update.runtime_information || - self.edges != update.edges || - self.start_fn != update.start_fn || - self.nodes.len() != update.nodes.len() || - self.nodes.iter().any(|(k, v)| { + if self.code.sha256_hash != update.code.sha256_hash + || self.runtime_information != update.runtime_information + || self.edges != update.edges + || self.start_fn != update.start_fn + || self.nodes.len() != update.nodes.len() + || self.nodes.iter().any(|(k, v)| { update .nodes .get(k) @@ -1046,36 +1046,11 @@ mod tests { use std::collections::HashMap; use crate::{ - test_objects::tests::test_compute_fn, - ComputeFn, - ComputeGraph, - ComputeGraphCode, - ComputeGraphVersion, - DynamicEdgeRouter, - ExecutorMetadata, - GraphVersion, - ImageInformation, - Node, - RuntimeInformation, + test_objects::tests::test_compute_fn, ComputeFn, ComputeGraph, ComputeGraphCode, + ComputeGraphVersion, DynamicEdgeRouter, ExecutorMetadata, GraphVersion, ImageInformation, + Node, RuntimeInformation, }; - #[test] - fn test_image_hash_consistency() { - let image_info = ImageInformation::new( - "test".to_string(), - "test".to_string(), - "static_base_image".to_string(), - vec!["pip install all_the_things".to_string()], - Some("1.2.3".to_string()), - ); - - assert_eq!( - image_info.image_hash, - "229514da1c19e40fda77e8b4a4990f69ce1ec460f025f4e1367bb2219f6abea1", - "image hash should not change" - ); - } - #[test] fn test_node_matches_executor_scenarios() { fn check( From 88fdb53787778c6ef492a298eeb1d6ebcb9b766a Mon Sep 17 00:00:00 2001 From: Julio Martinez Date: Fri, 20 Dec 2024 11:36:20 -0800 Subject: [PATCH 09/16] Lint --- server/data_model/src/lib.rs | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/server/data_model/src/lib.rs b/server/data_model/src/lib.rs index 0a00069c8..034118e07 100644 --- a/server/data_model/src/lib.rs +++ b/server/data_model/src/lib.rs @@ -391,12 +391,12 @@ impl ComputeGraph { let mut graph_version: Option = None; - if self.code.sha256_hash != update.code.sha256_hash - || self.runtime_information != update.runtime_information - || self.edges != update.edges - || self.start_fn != update.start_fn - || self.nodes.len() != update.nodes.len() - || self.nodes.iter().any(|(k, v)| { + if self.code.sha256_hash != update.code.sha256_hash || + self.runtime_information != update.runtime_information || + self.edges != update.edges || + self.start_fn != update.start_fn || + self.nodes.len() != update.nodes.len() || + self.nodes.iter().any(|(k, v)| { update .nodes .get(k) @@ -1046,9 +1046,17 @@ mod tests { use std::collections::HashMap; use crate::{ - test_objects::tests::test_compute_fn, ComputeFn, ComputeGraph, ComputeGraphCode, - ComputeGraphVersion, DynamicEdgeRouter, ExecutorMetadata, GraphVersion, ImageInformation, - Node, RuntimeInformation, + test_objects::tests::test_compute_fn, + ComputeFn, + ComputeGraph, + ComputeGraphCode, + ComputeGraphVersion, + DynamicEdgeRouter, + ExecutorMetadata, + GraphVersion, + ImageInformation, + Node, + RuntimeInformation, }; #[test] From b17d99ac26048fa6077bbca9d2f9dd914dcdaee8 Mon Sep 17 00:00:00 2001 From: Julio Martinez Date: Fri, 20 Dec 2024 12:04:01 -0800 Subject: [PATCH 10/16] Updated image_hash test to test for copy changes --- python-sdk/tests/test_image_hash.py | 38 +++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/python-sdk/tests/test_image_hash.py b/python-sdk/tests/test_image_hash.py index 25b4def4c..0e3339542 100644 --- a/python-sdk/tests/test_image_hash.py +++ b/python-sdk/tests/test_image_hash.py @@ -1,25 +1,59 @@ import unittest +import os +import shutil from indexify.functions_sdk.image import BASE_IMAGE_NAME, Image -KNOWN_GOOD_HASH = "229514da1c19e40fda77e8b4a4990f69ce1ec460f025f4e1367bb2219f6abea1" +KNOWN_GOOD_HASH = "3f25487a14c4c0e8c9948545e85e17404717a3eebf33c91821b6d8d06cc6d704" class TestImage(unittest.TestCase): + def setUp(self): + os.mkdir("copy_test_dir") + with open("copy_test_dir/test_file", "w") as fp: + fp.write("Test data\n") + + def tearDown(self): + shutil.rmtree("copy_test_dir") + + def test_copy_hash_update(self): + i = self.newTestImage() + + prevHash = i.hash() + + with open("copy_test_dir/test_file", "w+") as fp: + fp.write("Some more test data\n") + + self.assertNotEqual(i.hash(), prevHash) + def test_image_hash(self): i = ( Image() .name("test") .base_image("static_base_image") + .copy("copy_test_dir", "test_dir") .run("pip install all_the_things") .tag("test") ) i._sdk_version = ( - "1.2.3" # This needs to be statc for the hash to be predictable + "1.2.3" # This needs to be static for the hash to be predictable ) self.assertEqual(i.hash(), KNOWN_GOOD_HASH) + def newTestImage(self): + i = ( + Image() + .name("test") + .base_image("static_base_image") + .copy("copy_test_dir", "test_dir") + .run("pip install all_the_things") + .tag("test") + ) + i._sdk_version = ( + "1.2.3" # This needs to be static for the hash to be predictable + ) + return i if __name__ == "__main__": unittest.main() From a54bcb3b75b69ac9662bfc4ae5e1238d341200ce Mon Sep 17 00:00:00 2001 From: Julio Martinez Date: Fri, 20 Dec 2024 12:05:43 -0800 Subject: [PATCH 11/16] Lint --- python-sdk/tests/test_image_hash.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python-sdk/tests/test_image_hash.py b/python-sdk/tests/test_image_hash.py index 0e3339542..9415de114 100644 --- a/python-sdk/tests/test_image_hash.py +++ b/python-sdk/tests/test_image_hash.py @@ -1,6 +1,6 @@ -import unittest import os import shutil +import unittest from indexify.functions_sdk.image import BASE_IMAGE_NAME, Image @@ -12,7 +12,7 @@ def setUp(self): os.mkdir("copy_test_dir") with open("copy_test_dir/test_file", "w") as fp: fp.write("Test data\n") - + def tearDown(self): shutil.rmtree("copy_test_dir") @@ -55,5 +55,6 @@ def newTestImage(self): ) return i + if __name__ == "__main__": unittest.main() From d7ff783730f81e04e562a50eb8f2fdffd53bf457 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julio=20Mart=C3=ADnez?= Date: Fri, 20 Dec 2024 13:12:52 -0800 Subject: [PATCH 12/16] Update server/data_model/src/lib.rs Co-authored-by: Benjamin Boudreau --- server/data_model/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/data_model/src/lib.rs b/server/data_model/src/lib.rs index 034118e07..f506fe52d 100644 --- a/server/data_model/src/lib.rs +++ b/server/data_model/src/lib.rs @@ -111,7 +111,7 @@ impl ImageInformation { run_strs: Vec, sdk_version: Option, ) -> Self { - let mut compat_image_hash: String = "".to_string(); + let mut compat_image_hash: String = image_hash; if image_hash == "" { // Preserve backwards compatibility with old hash calculation let mut image_hasher = Sha256::new(); From e59bb497c338bc6b9f7fa81550b7b7faf585b677 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julio=20Mart=C3=ADnez?= Date: Fri, 20 Dec 2024 13:14:48 -0800 Subject: [PATCH 13/16] Update python-sdk/indexify/functions_sdk/image.py Co-authored-by: Benjamin Boudreau --- python-sdk/indexify/functions_sdk/image.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python-sdk/indexify/functions_sdk/image.py b/python-sdk/indexify/functions_sdk/image.py index 45a6033e6..01bcbfd21 100644 --- a/python-sdk/indexify/functions_sdk/image.py +++ b/python-sdk/indexify/functions_sdk/image.py @@ -138,7 +138,6 @@ def _generate_dockerfile(self, python_sdk_path: Optional[str] = None): docker_contents = [ f"FROM {self._base_image}", "RUN mkdir -p ~/.indexify", - "RUN touch ~/.indexify/image_name", f"RUN echo {self._image_name} > ~/.indexify/image_name", f"RUN echo {self.hash()} > ~/.indexify/image_hash", "WORKDIR /app", From 9af8b9b3a46e4b88eeb9c1438fa913f9063a1c38 Mon Sep 17 00:00:00 2001 From: Julio Martinez Date: Fri, 20 Dec 2024 13:31:02 -0800 Subject: [PATCH 14/16] Move to wait before polling server --- python-sdk/indexify/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-sdk/indexify/cli.py b/python-sdk/indexify/cli.py index 8f8ee3d7c..1b2e0e309 100644 --- a/python-sdk/indexify/cli.py +++ b/python-sdk/indexify/cli.py @@ -331,11 +331,11 @@ def _create_platform_image(image: Image, service_endpoint: str): case "ready" | "building": print(f"waiting for {build.image_name} image to build") while build.status != "completed": + time.sleep(5) res = client.get( f"{service_endpoint}/builds/{build.id}", headers=headers ) build = Build.model_validate(res.json()) - time.sleep(5) case _: raise ValueError(f"Unexpected build status {build.status}") From 7eabab4f0fcdf15366b665042f2508065b7002b9 Mon Sep 17 00:00:00 2001 From: Julio Martinez Date: Fri, 20 Dec 2024 13:34:13 -0800 Subject: [PATCH 15/16] Lint --- server/data_model/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/data_model/src/lib.rs b/server/data_model/src/lib.rs index aa0414015..216fcaa4f 100644 --- a/server/data_model/src/lib.rs +++ b/server/data_model/src/lib.rs @@ -127,7 +127,7 @@ impl ImageInformation { base_image, run_strs, image_hash: compat_image_hash, - version: ::default(), + version: ::default(), image_uri: None, sdk_version, } From 7d8cec29c8c29e98c3c84d09d23a42b15826e8ac Mon Sep 17 00:00:00 2001 From: Julio Martinez Date: Fri, 20 Dec 2024 13:38:35 -0800 Subject: [PATCH 16/16] Fixed build errors and merge droppings --- server/data_model/src/lib.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/data_model/src/lib.rs b/server/data_model/src/lib.rs index 216fcaa4f..137e2baf1 100644 --- a/server/data_model/src/lib.rs +++ b/server/data_model/src/lib.rs @@ -111,7 +111,7 @@ impl ImageInformation { sdk_version: Option, ) -> Self { let mut compat_image_hash: String = image_hash; - if image_hash == "" { + if compat_image_hash == "" { // Preserve backwards compatibility with old hash calculation let mut image_hasher = Sha256::new(); image_hasher.update(image_name.clone()); @@ -127,7 +127,6 @@ impl ImageInformation { base_image, run_strs, image_hash: compat_image_hash, - version: ::default(), image_uri: None, sdk_version, }