diff --git a/docker/sandbox-bundled/manifests/complete-agent.yaml b/docker/sandbox-bundled/manifests/complete-agent.yaml index 761469bd83..ea6321200d 100644 --- a/docker/sandbox-bundled/manifests/complete-agent.yaml +++ b/docker/sandbox-bundled/manifests/complete-agent.yaml @@ -816,7 +816,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: aVh1N3lZb0F1c2l0NHVuRg== + haSharedSecret: ZXlJVkhWYjdIMHhjamZadA== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1413,7 +1413,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: 042e6b21a3852a65952e0701cd9667e53bfef57590eea4d116b261472f29a882 + checksum/secret: 94a4c448ea7ad0892283bc4cfc6c506c83c9c5fe998587f4b2c55194c6a674e3 labels: app: docker-registry release: flyte-sandbox diff --git a/docker/sandbox-bundled/manifests/complete.yaml b/docker/sandbox-bundled/manifests/complete.yaml index e80cc05d20..3437469a1c 100644 --- a/docker/sandbox-bundled/manifests/complete.yaml +++ b/docker/sandbox-bundled/manifests/complete.yaml @@ -798,7 +798,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: Q2EyanRtd1JjWmVKS2tHMw== + haSharedSecret: OW1PbDdRY0t4RllhM3Nybg== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1362,7 +1362,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: 01201329c98a1417f04feeef00dc21e67cf73d99ac9b99486ce5788eca0c282c + checksum/secret: 1f30487909a5b2db21b8f92a734fcb321ab30f01694f4257333026e00d512053 labels: app: docker-registry release: flyte-sandbox diff --git a/docker/sandbox-bundled/manifests/dev.yaml b/docker/sandbox-bundled/manifests/dev.yaml index fe04e4c059..f0e2a866af 100644 --- a/docker/sandbox-bundled/manifests/dev.yaml +++ b/docker/sandbox-bundled/manifests/dev.yaml @@ -499,7 +499,7 @@ metadata: --- apiVersion: v1 data: - haSharedSecret: N3dIemE2TnF1b3l1SWdNTw== + haSharedSecret: MWVqaUwzWDZtUWY4TDdscA== proxyPassword: "" proxyUsername: "" kind: Secret @@ -934,7 +934,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: ca7423805c2fd3a98507c790af575a6e5389f50d6baa09bd8c49cb59c4452340 + checksum/secret: 53219c6f309435a180b4635448e130a2ec19b63b379a881dde73bf8ae957a1ad labels: app: docker-registry release: flyte-sandbox diff --git a/docs/user_guide/customizing_dependencies/imagespec.md b/docs/user_guide/customizing_dependencies/imagespec.md index ccdd52fe28..d9bf8f24bf 100644 --- a/docs/user_guide/customizing_dependencies/imagespec.md +++ b/docs/user_guide/customizing_dependencies/imagespec.md @@ -6,9 +6,13 @@ .. tags:: Containerization, Intermediate ``` -`ImageSpec` is a way to specify how to build a container image without a Dockerfile. The `ImageSpec` by default will be -converted to an [Envd](https://envd.tensorchord.ai/) config, and the [Envd builder](https://github.com/flyteorg/flytekit/blob/master/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py#L12-L34) will build the image for you. However, you can also register your own builder to build -the image using other tools. + +`ImageSpec` allows you to customize the container image for your Flyte tasks without a Dockerfile. +`ImageSpec` speeds up the build process by allowing you to reuse previously downloaded packages from the PyPI and APT caches. + +By default, the `ImageSpec` will be built using the `default` builder associated with Flytekit, but you can register your own builder. + +For example, [flytekitplugins-envd](https://github.com/flyteorg/flytekit/blob/c06ef30518dec2057e554fbed375dfa43b985c60/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py#L25) is another image builder that uses envd to build the ImageSpec. For every {py:class}`flytekit.PythonFunctionTask` task or a task decorated with the `@task` decorator, you can specify rules for binding container images. By default, flytekit binds a single container image, i.e., @@ -16,58 +20,164 @@ the [default Docker image](https://ghcr.io/flyteorg/flytekit), to all tasks. To use the `container_image` parameter available in the {py:func}`flytekit.task` decorator, and pass an `ImageSpec`. -Before building the image, Flytekit checks the container registry first to see if the image already exists. By doing so, it avoids having to rebuild the image over and over again. If the image does not exist, flytekit will build the image before registering the workflow, and replace the image name in the task template with the newly built image name. - -```{note} -To clone and run the example code on this page, see the [Flytesnacks repo][flytesnacks]. -``` - -```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/69dbe4840031a85d79d9ded25f80397c6834752d/examples/customizing_dependencies/customizing_dependencies/image_spec.py -:caption: customizing_dependencies/image_spec.py -:lines: 1-4 -``` +Before building the image, Flytekit checks the container registry to see if the image already exists. +If the image does not exist, +Flytekit will build the image before registering the workflow and replace the image name in the task template with the newly built image name. :::{admonition} Prerequisites :class: important -- Install [flytekitplugins-envd](https://github.com/flyteorg/flytekit/tree/master/plugins/flytekit-envd) to build the `ImageSpec`. -- To build the image on remote machine, check this [doc](https://envd.tensorchord.ai/teams/context.html#start-remote-buildkitd-on-builder-machine). +- Make sure `docker` is running on your local machine. - When using a registry in ImageSpec, `docker login` is required to push the image ::: -You can specify python packages, apt packages, and environment variables in the `ImageSpec`. +## Install Python or APT packages +You can specify Python packages and APT packages in the `ImageSpec`. These specified packages will be added on top of the [default image](https://github.com/flyteorg/flytekit/blob/master/Dockerfile), which can be found in the Flytekit Dockerfile. More specifically, flytekit invokes [DefaultImages.default_image()](https://github.com/flyteorg/flytekit/blob/f2cfef0ec098d4ae8f042ab915b0b30d524092c6/flytekit/configuration/default_images.py#L26-L27) function. -This function determines and returns the default image based on the Python version and flytekit version. For example, if you are using python 3.8 and flytekit 0.16.0, the default image assigned will be `ghcr.io/flyteorg/flytekit:py3.8-1.6.0`. -If desired, you can also override the default image by providing a custom `base_image` parameter when using the `ImageSpec`. - -```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/69dbe4840031a85d79d9ded25f80397c6834752d/examples/customizing_dependencies/customizing_dependencies/image_spec.py -:caption: customizing_dependencies/image_spec.py -:lines: 6-19 -``` +This function determines and returns the default image based on the Python version and flytekit version. +For example, if you are using Python 3.8 and flytekit 1.6.0, the default image assigned will be `ghcr.io/flyteorg/flytekit:py3.8-1.6.0`. :::{important} Replace `ghcr.io/flyteorg` with a container registry you can publish to. To upload the image to the local registry in the demo cluster, indicate the registry as `localhost:30000`. ::: -`is_container` is used to determine whether the task is utilizing the image constructed from the `ImageSpec`. -If the task is indeed using the image built from the `ImageSpec`, it will then import Tensorflow. +```python +from flytekit import ImageSpec + +sklearn_image_spec = ImageSpec( + packages=["scikit-learn", "tensorflow==2.5.0"], + apt_packages=["curl", "wget"], + registry="ghcr.io/flyteorg", +) +``` + +## Install Conda packages +Define the ImageSpec to install packages from a specific conda channel. +```python +image_spec = ImageSpec( + conda_packages=["langchain"], + conda_channels=["conda-forge"], # List of channels to pull packages from. + registry="ghcr.io/flyteorg", +) +``` + +## Use different Python versions in the image +You can specify the Python version in the `ImageSpec` to build the image with a different Python version. + +```python +image_spec = ImageSpec( + packages=["pandas"], + python_version="3.9", + registry="ghcr.io/flyteorg", +) +``` + +## Import modules only in a specify imageSpec environment + +`is_container()` is used to determine whether the task is utilizing the image constructed from the `ImageSpec`. +If the task is indeed using the image built from the `ImageSpec`, it will return true. This approach helps minimize module loading time and prevents unnecessary dependency installation within a single image. -```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/69dbe4840031a85d79d9ded25f80397c6834752d/examples/customizing_dependencies/customizing_dependencies/image_spec.py -:caption: customizing_dependencies/image_spec.py -:lines: 21-22 +In the following example, both `task1` and `task2` will import the `pandas` module. However, `Tensorflow` will only be imported in `task2`. + +```python +from flytekit import ImageSpec, task +import pandas as pd + +pandas_image_spec = ImageSpec( + packages=["pandas"], + registry="ghcr.io/flyteorg", +) + +tensorflow_image_spec = ImageSpec( + packages=["tensorflow", "pandas"], + registry="ghcr.io/flyteorg", +) + +# Return if and only if the task is using the image built from tensorflow_image_spec. +if tensorflow_image_spec.is_container(): + import tensorflow as tf + +@task(container_image=pandas_image_spec) +def task1() -> pd.DataFrame: + return pd.DataFrame({"Name": ["Tom", "Joseph"], "Age": [1, 22]}) + + +@task(container_image=tensorflow_image_spec) +def task2() -> int: + num_gpus = len(tf.config.list_physical_devices('GPU')) + print("Num GPUs Available: ", num_gpus) + return num_gpus +``` + +## Install CUDA in the image +There are few ways to install CUDA in the image. + +### Use Nvidia docker image +CUDA is pre-installed in the Nvidia docker image. You can specify the base image in the `ImageSpec`. +```python +image_spec = ImageSpec( + base_image="nvidia/cuda:12.6.1-cudnn-devel-ubuntu22.04", + packages=["tensorflow", "pandas"], + python_version="3.9", + registry="ghcr.io/flyteorg", +) ``` -To enable tasks to utilize the images built with `ImageSpec`, you can specify the `container_image` parameter for those tasks. +### Install packages from extra index +CUDA can be installed by specifying the `pip_extra_index_url` in the `ImageSpec`. +```python +image_spec = ImageSpec( + name="pytorch-mnist", + packages=["torch", "torchvision", "flytekitplugins-kfpytorch"], + pip_extra_index_url=["https://download.pytorch.org/whl/cu118"], + registry="ghcr.io/flyteorg", +) +``` -```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/69dbe4840031a85d79d9ded25f80397c6834752d/examples/customizing_dependencies/customizing_dependencies/image_spec.py -:caption: customizing_dependencies/image_spec.py -:lines: 27-56 +## Build an image in different architecture +You can specify the platform in the `ImageSpec` to build the image in a different architecture, such as `linux/arm64` or `darwin/arm64`. +```python +image_spec = ImageSpec( + packages=["pandas"], + platform="linux/arm64", + registry="ghcr.io/flyteorg", +) ``` -There exists an option to override the container image by providing an Image Spec YAML file to the `pyflyte run` or `pyflyte register` command. +## Install flytekit from GitHub +When you update the flytekit, you may want to test the changes with your tasks. +You can install the flytekit from a specific commit hash in the `ImageSpec`. + +```python +new_flytekit = "git+https://github.com/flyteorg/flytekit@90a4455c2cc2b3e171dfff69f605f47d48ea1ff1" +new_spark_plugins = f"git+https://github.com/flyteorg/flytekit.git@90a4455c2cc2b3e171dfff69f605f47d48ea1ff1#subdirectory=plugins/flytekit-spark" + +image_spec = ImageSpec( + apt_packages=["git"], + packages=[new_flytekit, new_spark_plugins], + registry="ghcr.io/flyteorg", +) +``` + +## Customize the tag of the image +You can customize the tag of the image by specifying the `tag_format` in the `ImageSpec`. +In the following example, the full qualified image name will be `ghcr.io/flyteorg/my-image:-dev`. + +```python +image_spec = ImageSpec( + name="my-image", + packages=["pandas"], + tag_format="{spec_hash}-dev", + registry="ghcr.io/flyteorg", +) +``` + +## Define ImageSpec in a YAML File + +You can override the container image by providing an ImageSpec YAML file to the `pyflyte run` or `pyflyte register` command. This allows for greater flexibility in specifying a custom container image. For example: ```yaml @@ -85,19 +195,24 @@ env: pyflyte run --remote --image image.yaml image_spec.py wf ``` +## Build the image without registering the workflow + If you only want to build the image without registering the workflow, you can use the `pyflyte build` command. ``` pyflyte build --remote image_spec.py wf ``` -In some cases, you may want to force an image to rebuild, even if the image spec hasn’t changed. If you want to overwrite an existing image, you can pass the `FLYTE_FORCE_PUSH_IMAGE_SPEC=True` to `pyflyte` command or add `force_push()` to the ImageSpec. +## Force push an image + +In some cases, you may want to force an image to rebuild, even if the ImageSpec hasn’t changed. +To overwrite an existing image, pass the `FLYTE_FORCE_PUSH_IMAGE_SPEC=True` to the `pyflyte` command. ```bash FLYTE_FORCE_PUSH_IMAGE_SPEC=True pyflyte run --remote image_spec.py wf ``` -or +You can also force push an image in the Python code by calling the `force_push()` method. ```python image = ImageSpec(registry="ghcr.io/flyteorg", packages=["pandas"]).force_push() diff --git a/docs/user_guide/development_lifecycle/decks.md b/docs/user_guide/development_lifecycle/decks.md index c59cbd1945..366302d49e 100644 --- a/docs/user_guide/development_lifecycle/decks.md +++ b/docs/user_guide/development_lifecycle/decks.md @@ -39,9 +39,9 @@ We create a new deck named `pca` and render Markdown content along with a You can begin by initializing an {ref}`ImageSpec ` object to encompass all the necessary dependencies. This approach automatically triggers a Docker build, alleviating the need for you to manually create a Docker image. -```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/69dbe4840031a85d79d9ded25f80397c6834752d/examples/development_lifecycle/development_lifecycle/decks.py +```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/b431ae399def3a749833fe81c2c291b016cf3213/examples/development_lifecycle/development_lifecycle/decks.py :caption: development_lifecycle/decks.py -:lines: 15-19 +:lines: 15-27 ``` :::{important} @@ -96,9 +96,9 @@ When the task connected with a deck object is executed, these objects employ ren Creates a profile report from a Pandas DataFrame. -```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/69dbe4840031a85d79d9ded25f80397c6834752d/examples/development_lifecycle/development_lifecycle/decks.py +```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/b431ae399def3a749833fe81c2c291b016cf3213/examples/development_lifecycle/development_lifecycle/decks.py :caption: development_lifecycle/decks.py -:lines: 44-51 +:lines: 56-63 ``` :::{figure} https://raw.githubusercontent.com/flyteorg/static-resources/main/flytesnacks/user_guide/flyte_decks_frame_renderer.png @@ -113,9 +113,9 @@ Creates a profile report from a Pandas DataFrame. Renders DataFrame as an HTML table. This renderer doesn't necessitate plugin installation since it's accessible within the flytekit library. -```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/69dbe4840031a85d79d9ded25f80397c6834752d/examples/development_lifecycle/development_lifecycle/decks.py +```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/b431ae399def3a749833fe81c2c291b016cf3213/examples/development_lifecycle/development_lifecycle/decks.py :caption: development_lifecycle/decks.py -:lines: 57-64 +:lines: 69-76 ``` :::{figure} https://raw.githubusercontent.com/flyteorg/static-resources/main/flytesnacks/user_guide/flyte_decks_top_frame_renderer.png @@ -147,9 +147,9 @@ The median (Q2) is indicated by a line within the box. Typically, the whiskers extend to the edges of the box, plus or minus 1.5 times the interquartile range (IQR: Q3-Q1). -```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/69dbe4840031a85d79d9ded25f80397c6834752d/examples/development_lifecycle/development_lifecycle/decks.py +```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/b431ae399def3a749833fe81c2c291b016cf3213/examples/development_lifecycle/development_lifecycle/decks.py :caption: development_lifecycle/decks.py -:lines: 85-91 +:lines: 97-103 ``` :::{figure} https://raw.githubusercontent.com/flyteorg/static-resources/main/flytesnacks/user_guide/flyte_decks_box_renderer.png @@ -162,9 +162,9 @@ plus or minus 1.5 times the interquartile range (IQR: Q3-Q1). Converts a {ref}`FlyteFile ` or `PIL.Image.Image` object into an HTML string, where the image data is encoded as a base64 string. -```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/69dbe4840031a85d79d9ded25f80397c6834752d/examples/development_lifecycle/development_lifecycle/decks.py +```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/b431ae399def3a749833fe81c2c291b016cf3213/examples/development_lifecycle/development_lifecycle/decks.py :caption: development_lifecycle/decks.py -:lines: 97-111 +:lines: 109-123 ``` :::{figure} https://raw.githubusercontent.com/flyteorg/static-resources/main/flytesnacks/user_guide/flyte_decks_image_renderer.png @@ -176,9 +176,9 @@ where the image data is encoded as a base64 string. Converts a Pandas dataframe into an HTML table. -```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/69dbe4840031a85d79d9ded25f80397c6834752d/examples/development_lifecycle/development_lifecycle/decks.py +```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/b431ae399def3a749833fe81c2c291b016cf3213/examples/development_lifecycle/development_lifecycle/decks.py :caption: development_lifecycle/decks.py -:lines: 115-123 +:lines: 127-135 ``` :::{figure} https://raw.githubusercontent.com/flyteorg/static-resources/main/flytesnacks/user_guide/flyte_decks_table_renderer.png diff --git a/flyteadmin/go.mod b/flyteadmin/go.mod index cfc2bfa010..836bc69979 100644 --- a/flyteadmin/go.mod +++ b/flyteadmin/go.mod @@ -48,7 +48,6 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 github.com/wI2L/jsondiff v0.5.0 - github.com/wolfeidau/humanhash v1.1.0 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go.opentelemetry.io/otel v1.24.0 golang.org/x/net v0.27.0 diff --git a/flyteadmin/go.sum b/flyteadmin/go.sum index 31a1714ed7..7c9c02881f 100644 --- a/flyteadmin/go.sum +++ b/flyteadmin/go.sum @@ -1301,8 +1301,6 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/wI2L/jsondiff v0.5.0 h1:RRMTi/mH+R2aXcPe1VYyvGINJqQfC3R+KSEakuU1Ikw= github.com/wI2L/jsondiff v0.5.0/go.mod h1:qqG6hnK0Lsrz2BpIVCxWiK9ItsBCpIZQiv0izJjOZ9s= -github.com/wolfeidau/humanhash v1.1.0 h1:06KgtyyABJGBbrfMONrW7S+b5TTYVyrNB/jss5n7F3E= -github.com/wolfeidau/humanhash v1.1.0/go.mod h1:jkpynR1bfyfkmKEQudIC0osWKynFAoayRjzH9OJdVIg= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= diff --git a/flyteadmin/pkg/async/schedule/aws/workflow_executor.go b/flyteadmin/pkg/async/schedule/aws/workflow_executor.go index 523fdd077e..c4a5d75d14 100644 --- a/flyteadmin/pkg/async/schedule/aws/workflow_executor.go +++ b/flyteadmin/pkg/async/schedule/aws/workflow_executor.go @@ -15,7 +15,7 @@ import ( "github.com/flyteorg/flyte/flyteadmin/pkg/async" scheduleInterfaces "github.com/flyteorg/flyte/flyteadmin/pkg/async/schedule/interfaces" - "github.com/flyteorg/flyte/flyteadmin/pkg/common/naming" + "github.com/flyteorg/flyte/flyteadmin/pkg/common" "github.com/flyteorg/flyte/flyteadmin/pkg/errors" "github.com/flyteorg/flyte/flyteadmin/pkg/manager/interfaces" runtimeInterfaces "github.com/flyteorg/flyte/flyteadmin/pkg/runtime/interfaces" @@ -129,7 +129,7 @@ func generateExecutionName(launchPlan *admin.LaunchPlan, kickoffTime time.Time) Name: launchPlan.Id.Name, }) randomSeed := kickoffTime.UnixNano() + int64(hashedIdentifier) - return naming.GetExecutionName(randomSeed) + return common.GetExecutionName(randomSeed) } func (e *workflowExecutor) formulateExecutionCreateRequest( @@ -207,6 +207,7 @@ func (e *workflowExecutor) run() error { continue } executionRequest := e.formulateExecutionCreateRequest(launchPlan, scheduledWorkflowExecutionRequest.KickoffTime) + ctx = contextutils.WithWorkflowID(ctx, fmt.Sprintf(workflowIdentifierFmt, executionRequest.Project, executionRequest.Domain, executionRequest.Name)) err = e.resolveKickoffTimeArg(scheduledWorkflowExecutionRequest, launchPlan, executionRequest) diff --git a/flyteadmin/pkg/common/executions.go b/flyteadmin/pkg/common/executions.go index 4ac1ec7300..fbb5bdd6bd 100644 --- a/flyteadmin/pkg/common/executions.go +++ b/flyteadmin/pkg/common/executions.go @@ -1,9 +1,22 @@ package common import ( + "fmt" + + "k8s.io/apimachinery/pkg/util/rand" + "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" ) +const ExecutionIDLength = 20 +const ExecutionStringFormat = "a%s" + +/* #nosec */ +func GetExecutionName(seed int64) string { + rand.Seed(seed) + return fmt.Sprintf(ExecutionStringFormat, rand.String(ExecutionIDLength-1)) +} + var terminalExecutionPhases = map[core.WorkflowExecution_Phase]bool{ core.WorkflowExecution_SUCCEEDED: true, core.WorkflowExecution_FAILED: true, diff --git a/flyteadmin/pkg/common/executions_test.go b/flyteadmin/pkg/common/executions_test.go new file mode 100644 index 0000000000..628abd6e9d --- /dev/null +++ b/flyteadmin/pkg/common/executions_test.go @@ -0,0 +1,23 @@ +package common + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +const AllowedExecutionIDStartCharStr = "abcdefghijklmnopqrstuvwxyz" +const AllowedExecutionIDStr = "abcdefghijklmnopqrstuvwxyz1234567890" + +var AllowedExecutionIDStartChars = []rune(AllowedExecutionIDStartCharStr) +var AllowedExecutionIDChars = []rune(AllowedExecutionIDStr) + +func TestGetExecutionName(t *testing.T) { + randString := GetExecutionName(time.Now().UnixNano()) + assert.Len(t, randString, ExecutionIDLength) + assert.Contains(t, AllowedExecutionIDStartChars, rune(randString[0])) + for i := 1; i < len(randString); i++ { + assert.Contains(t, AllowedExecutionIDChars, rune(randString[i])) + } +} diff --git a/flyteadmin/pkg/common/naming/execution_name.go b/flyteadmin/pkg/common/naming/execution_name.go deleted file mode 100644 index 01aa3fe8b6..0000000000 --- a/flyteadmin/pkg/common/naming/execution_name.go +++ /dev/null @@ -1,30 +0,0 @@ -package naming - -import ( - "fmt" - - "github.com/wolfeidau/humanhash" - "k8s.io/apimachinery/pkg/util/rand" - - "github.com/flyteorg/flyte/flyteadmin/pkg/runtime" - runtimeInterfaces "github.com/flyteorg/flyte/flyteadmin/pkg/runtime/interfaces" -) - -const ExecutionIDLength = 20 -const ExecutionIDLengthLimit = 63 -const ExecutionStringFormat = "a%s" - -var configProvider runtimeInterfaces.ApplicationConfiguration = runtime.NewApplicationConfigurationProvider() - -/* #nosec */ -func GetExecutionName(seed int64) string { - rand.Seed(seed) - config := configProvider.GetTopLevelConfig() - if config.FeatureGates.EnableFriendlyNames { - hashKey := []byte(rand.String(ExecutionIDLength)) - // Ignoring the error as it's guaranteed hash key longer than result in this context. - result, _ := humanhash.Humanize(hashKey, 4) - return result - } - return fmt.Sprintf(ExecutionStringFormat, rand.String(ExecutionIDLength-1)) -} diff --git a/flyteadmin/pkg/common/naming/execution_name_test.go b/flyteadmin/pkg/common/naming/execution_name_test.go deleted file mode 100644 index 91f04d3bf1..0000000000 --- a/flyteadmin/pkg/common/naming/execution_name_test.go +++ /dev/null @@ -1,78 +0,0 @@ -package naming - -import ( - "context" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/assert" - - runtimeInterfaces "github.com/flyteorg/flyte/flyteadmin/pkg/runtime/interfaces" - runtimeMocks "github.com/flyteorg/flyte/flyteadmin/pkg/runtime/mocks" - "github.com/flyteorg/flyte/flyteadmin/scheduler/identifier" - "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" -) - -const AllowedExecutionIDAlphabetStr = "abcdefghijklmnopqrstuvwxyz" -const AllowedExecutionIDAlphanumericStr = "abcdefghijklmnopqrstuvwxyz1234567890" -const AllowedExecutionIDFriendlyNameStr = "abcdefghijklmnopqrstuvwxyz-" - -var AllowedExecutionIDAlphabets = []rune(AllowedExecutionIDAlphabetStr) -var AllowedExecutionIDAlphanumerics = []rune(AllowedExecutionIDAlphanumericStr) -var AllowedExecutionIDFriendlyNameChars = []rune(AllowedExecutionIDFriendlyNameStr) - -func TestGetExecutionName(t *testing.T) { - originalConfigProvider := configProvider - defer func() { configProvider = originalConfigProvider }() - - mockConfigProvider := &runtimeMocks.MockApplicationProvider{} - configProvider = mockConfigProvider - - t.Run("general name", func(t *testing.T) { - appConfig := runtimeInterfaces.ApplicationConfig{ - FeatureGates: runtimeInterfaces.FeatureGates{ - EnableFriendlyNames: false, - }, - } - mockConfigProvider.SetTopLevelConfig(appConfig) - - randString := GetExecutionName(time.Now().UnixNano()) - assert.Len(t, randString, ExecutionIDLength) - assert.Contains(t, AllowedExecutionIDAlphabets, rune(randString[0])) - for i := 1; i < len(randString); i++ { - assert.Contains(t, AllowedExecutionIDAlphanumerics, rune(randString[i])) - } - }) - - t.Run("friendly name", func(t *testing.T) { - appConfig := runtimeInterfaces.ApplicationConfig{ - FeatureGates: runtimeInterfaces.FeatureGates{ - EnableFriendlyNames: true, - }, - } - mockConfigProvider.SetTopLevelConfig(appConfig) - - randString := GetExecutionName(time.Now().UnixNano()) - assert.LessOrEqual(t, len(randString), ExecutionIDLengthLimit) - for i := 0; i < len(randString); i++ { - assert.Contains(t, AllowedExecutionIDFriendlyNameChars, rune(randString[i])) - } - hyphenCount := strings.Count(randString, "-") - assert.Equal(t, 3, hyphenCount, "FriendlyName should contain exactly three hyphens") - words := strings.Split(randString, "-") - assert.Equal(t, 4, len(words), "FriendlyName should be split into exactly four words") - }) - - t.Run("deterministic name", func(t *testing.T) { - hashValue := identifier.HashScheduledTimeStamp(context.Background(), &core.Identifier{ - Project: "Project", - Domain: "Domain", - Name: "Name", - Version: "Version", - }, time.Time{}) - - name := GetExecutionName(int64(hashValue)) - assert.Equal(t, name, "carpet-juliet-kentucky-kentucky") - }) -} diff --git a/flyteadmin/pkg/manager/impl/util/shared.go b/flyteadmin/pkg/manager/impl/util/shared.go index ba8fc41760..8402451200 100644 --- a/flyteadmin/pkg/manager/impl/util/shared.go +++ b/flyteadmin/pkg/manager/impl/util/shared.go @@ -8,7 +8,6 @@ import ( "google.golang.org/grpc/codes" "github.com/flyteorg/flyte/flyteadmin/pkg/common" - "github.com/flyteorg/flyte/flyteadmin/pkg/common/naming" "github.com/flyteorg/flyte/flyteadmin/pkg/errors" "github.com/flyteorg/flyte/flyteadmin/pkg/manager/impl/shared" "github.com/flyteorg/flyte/flyteadmin/pkg/manager/impl/validation" @@ -26,7 +25,7 @@ func GetExecutionName(request *admin.ExecutionCreateRequest) string { if request.Name != "" { return request.Name } - return naming.GetExecutionName(time.Now().UnixNano()) + return common.GetExecutionName(time.Now().UnixNano()) } func GetTask(ctx context.Context, repo repoInterfaces.Repository, identifier *core.Identifier) ( diff --git a/flyteadmin/pkg/manager/impl/util/shared_test.go b/flyteadmin/pkg/manager/impl/util/shared_test.go index 114dbebdfb..b9b296971e 100644 --- a/flyteadmin/pkg/manager/impl/util/shared_test.go +++ b/flyteadmin/pkg/manager/impl/util/shared_test.go @@ -12,8 +12,8 @@ import ( "github.com/stretchr/testify/assert" "google.golang.org/grpc/codes" + "github.com/flyteorg/flyte/flyteadmin/pkg/common" commonMocks "github.com/flyteorg/flyte/flyteadmin/pkg/common/mocks" - "github.com/flyteorg/flyte/flyteadmin/pkg/common/naming" flyteAdminErrors "github.com/flyteorg/flyte/flyteadmin/pkg/errors" "github.com/flyteorg/flyte/flyteadmin/pkg/manager/impl/testutils" managerInterfaces "github.com/flyteorg/flyte/flyteadmin/pkg/manager/interfaces" @@ -42,7 +42,7 @@ func TestPopulateExecutionID(t *testing.T) { Domain: "domain", }) assert.NotEmpty(t, name) - assert.Len(t, name, naming.ExecutionIDLength) + assert.Len(t, name, common.ExecutionIDLength) } func TestPopulateExecutionID_ExistingName(t *testing.T) { diff --git a/flyteadmin/pkg/runtime/interfaces/application_configuration.go b/flyteadmin/pkg/runtime/interfaces/application_configuration.go index e3453db0f7..15ed271412 100644 --- a/flyteadmin/pkg/runtime/interfaces/application_configuration.go +++ b/flyteadmin/pkg/runtime/interfaces/application_configuration.go @@ -49,8 +49,7 @@ type PostgresConfig struct { } type FeatureGates struct { - EnableArtifacts bool `json:"enableArtifacts" pflag:",Enable artifacts feature."` - EnableFriendlyNames bool `json:"enableFriendlyNames" pflag:",Enable generation of friendly execution names feature."` + EnableArtifacts bool `json:"enableArtifacts" pflag:",Enable artifacts feature."` } // ApplicationConfig is the base configuration to start admin diff --git a/flyteadmin/scheduler/executor/executor_impl.go b/flyteadmin/scheduler/executor/executor_impl.go index e269d79b2a..5e4a8fcf8e 100644 --- a/flyteadmin/scheduler/executor/executor_impl.go +++ b/flyteadmin/scheduler/executor/executor_impl.go @@ -2,6 +2,7 @@ package executor import ( "context" + "strings" "time" "github.com/prometheus/client_golang/prometheus" @@ -11,7 +12,6 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/retry" - "github.com/flyteorg/flyte/flyteadmin/pkg/common/naming" "github.com/flyteorg/flyte/flyteadmin/scheduler/identifier" "github.com/flyteorg/flyte/flyteadmin/scheduler/repositories/models" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/admin" @@ -54,18 +54,22 @@ func (w *executor) Execute(ctx context.Context, scheduledTime time.Time, s model } // Making the identifier deterministic using the hash of the identifier and scheduled time - hashValue := identifier.HashScheduledTimeStamp(ctx, &core.Identifier{ + executionIdentifier, err := identifier.GetExecutionIdentifier(ctx, &core.Identifier{ Project: s.Project, Domain: s.Domain, Name: s.Name, Version: s.Version, }, scheduledTime) - executionName := naming.GetExecutionName(int64(hashValue)) + if err != nil { + logger.Errorf(ctx, "failed to generate execution identifier for schedule %+v due to %v", s, err) + return err + } + executionRequest := &admin.ExecutionCreateRequest{ Project: s.Project, Domain: s.Domain, - Name: executionName, + Name: "f" + strings.ReplaceAll(executionIdentifier.String(), "-", "")[:19], Spec: &admin.ExecutionSpec{ LaunchPlan: &core.Identifier{ ResourceType: core.ResourceType_LAUNCH_PLAN, @@ -93,7 +97,7 @@ func (w *executor) Execute(ctx context.Context, scheduledTime time.Time, s model // Do maximum of 30 retries on failures with constant backoff factor opts := wait.Backoff{Duration: 3000, Factor: 2.0, Steps: 30} - err := retry.OnError(opts, + err = retry.OnError(opts, func(err error) bool { // For idempotent behavior ignore the AlreadyExists error which happens if we try to schedule a launchplan // for execution at the same time which is already available in admin. diff --git a/flyteadmin/scheduler/identifier/identifier.go b/flyteadmin/scheduler/identifier/identifier.go index caf5c94296..5d386e8652 100644 --- a/flyteadmin/scheduler/identifier/identifier.go +++ b/flyteadmin/scheduler/identifier/identifier.go @@ -34,7 +34,7 @@ func GetScheduleName(ctx context.Context, s models.SchedulableEntity) string { // GetExecutionIdentifier returns UUID using the hashed value of the schedule identifier and the scheduledTime func GetExecutionIdentifier(ctx context.Context, identifier *core.Identifier, scheduledTime time.Time) (uuid.UUID, error) { - hashValue := HashScheduledTimeStamp(ctx, identifier, scheduledTime) + hashValue := hashScheduledTimeStamp(ctx, identifier, scheduledTime) b := make([]byte, 16) binary.LittleEndian.PutUint64(b, hashValue) return uuid.FromBytes(b) @@ -55,8 +55,8 @@ func hashIdentifier(ctx context.Context, identifier *core.Identifier) uint64 { return h.Sum64() } -// HashScheduledTimeStamp return the hash of the identifier and the scheduledTime -func HashScheduledTimeStamp(ctx context.Context, identifier *core.Identifier, scheduledTime time.Time) uint64 { +// hashScheduledTimeStamp return the hash of the identifier and the scheduledTime +func hashScheduledTimeStamp(ctx context.Context, identifier *core.Identifier, scheduledTime time.Time) uint64 { h := fnv.New64() _, err := h.Write([]byte(fmt.Sprintf(executionIDInputsFormat, identifier.Project, identifier.Domain, identifier.Name, identifier.Version, scheduledTime.Unix()))) diff --git a/go.mod b/go.mod index 8c8053def6..6c25974da0 100644 --- a/go.mod +++ b/go.mod @@ -179,7 +179,6 @@ require ( github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/sjson v1.2.5 // indirect github.com/wI2L/jsondiff v0.5.0 // indirect - github.com/wolfeidau/humanhash v1.1.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect diff --git a/go.sum b/go.sum index 63453bbd87..ac7b9f5987 100644 --- a/go.sum +++ b/go.sum @@ -1337,8 +1337,6 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/wI2L/jsondiff v0.5.0 h1:RRMTi/mH+R2aXcPe1VYyvGINJqQfC3R+KSEakuU1Ikw= github.com/wI2L/jsondiff v0.5.0/go.mod h1:qqG6hnK0Lsrz2BpIVCxWiK9ItsBCpIZQiv0izJjOZ9s= -github.com/wolfeidau/humanhash v1.1.0 h1:06KgtyyABJGBbrfMONrW7S+b5TTYVyrNB/jss5n7F3E= -github.com/wolfeidau/humanhash v1.1.0/go.mod h1:jkpynR1bfyfkmKEQudIC0osWKynFAoayRjzH9OJdVIg= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=