From 04716afcf539ba644a59c0b92b91506e4dd12af6 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Sun, 3 Dec 2023 16:59:19 +0530 Subject: [PATCH 1/6] remove hardcoded list of tests Signed-off-by: Samhita Alla --- .github/workflows/end2end.yml | 17 ++++------------- .github/workflows/single-binary.yml | 25 ++++++++----------------- 2 files changed, 12 insertions(+), 30 deletions(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index cee310b195..9e0f1fb9c6 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -79,25 +79,16 @@ jobs: - name: Register P0 tests if: ${{ inputs.priorities == 'P0' }} run: | - for f in \ - basics/basics/hello_world.py \ - basics/basics/workflow.py \ - basics/basics/named_outputs.py \ - advanced_composition/advanced_composition/chain_entities.py \ - advanced_composition/advanced_composition/dynamics.py \ - advanced_composition/advanced_composition/map_task.py \ - advanced_composition/advanced_composition/subworkflows.py \ - data_types_and_io/data_types_and_io/dataclass.py \ - data_types_and_io/data_types_and_io/structured_dataset.py ; + while read -r line; do pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \ register \ --project flytesnacks \ --domain development \ - --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \ + --image cr.flyte.org/flyteorg/flytekit:py3.11-1.10.0 \ --version ${{ env.FLYTESNACKS_VERSION }} \ - flytesnacks/examples/$f; - done + flytesnacks/$line; + done < flyte_tests.txt - name: Register all flytesnacks examples if: ${{ inputs.priorities != 'P0' }} uses: unionai/flyte-register-action@v0.0.2 diff --git a/.github/workflows/single-binary.yml b/.github/workflows/single-binary.yml index 630068da00..927484c865 100644 --- a/.github/workflows/single-binary.yml +++ b/.github/workflows/single-binary.yml @@ -27,7 +27,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v3 with: - go-version: '1.19' + go-version: "1.19" - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: @@ -129,8 +129,8 @@ jobs: sandbox-bundled-functional-tests: runs-on: ubuntu-latest env: - FLYTESNACKS_PRIORITIES: 'P0' - FLYTESNACKS_VERSION: '' + FLYTESNACKS_PRIORITIES: "P0" + FLYTESNACKS_VERSION: "" timeout-minutes: 20 needs: [build-and-push-single-binary-image] steps: @@ -164,7 +164,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.11' + python-version: "3.11" - uses: unionai/flytectl-setup-action@v0.0.1 - name: Setup sandbox run: | @@ -193,25 +193,16 @@ jobs: # ref: ${{ env.FLYTESNACKS_VERSION }} - name: Register specific tests run: | - for f in \ - basics/basics/hello_world.py \ - basics/basics/workflow.py \ - basics/basics/named_outputs.py \ - advanced_composition/advanced_composition/chain_entities.py \ - advanced_composition/advanced_composition/dynamics.py \ - advanced_composition/advanced_composition/map_task.py \ - advanced_composition/advanced_composition/subworkflows.py \ - data_types_and_io/data_types_and_io/dataclass.py \ - data_types_and_io/data_types_and_io/structured_dataset.py ; + while read -r line; do pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \ register \ --project flytesnacks \ --domain development \ - --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \ + --image cr.flyte.org/flyteorg/flytekit:py3.11-1.10.0 \ --version ${{ env.FLYTESNACKS_VERSION }} \ - flytesnacks/examples/$f; - done + flytesnacks/$line; + done < flyte_tests.txt - name: End2End run: | make end2end_execute From 4698045bed7c10749b1e8da7d21d7854fa1c2102 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Wed, 6 Dec 2023 17:40:37 +0530 Subject: [PATCH 2/6] latest version Signed-off-by: Samhita Alla --- .github/workflows/single-binary.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/single-binary.yml b/.github/workflows/single-binary.yml index 927484c865..138e5a238a 100644 --- a/.github/workflows/single-binary.yml +++ b/.github/workflows/single-binary.yml @@ -199,7 +199,7 @@ jobs: register \ --project flytesnacks \ --domain development \ - --image cr.flyte.org/flyteorg/flytekit:py3.11-1.10.0 \ + --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \ --version ${{ env.FLYTESNACKS_VERSION }} \ flytesnacks/$line; done < flyte_tests.txt From e53fb76ba826267b662e7b2fb632c13ebcea11fa Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Wed, 6 Dec 2023 17:51:45 +0530 Subject: [PATCH 3/6] latest version Signed-off-by: Samhita Alla --- .github/workflows/end2end.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 9e0f1fb9c6..15de425846 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -85,7 +85,7 @@ jobs: register \ --project flytesnacks \ --domain development \ - --image cr.flyte.org/flyteorg/flytekit:py3.11-1.10.0 \ + --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \ --version ${{ env.FLYTESNACKS_VERSION }} \ flytesnacks/$line; done < flyte_tests.txt From ddbb2bab155c6008d4537ea45facfcaa5af93a3e Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Thu, 7 Dec 2023 10:33:33 +0530 Subject: [PATCH 4/6] add sensor Signed-off-by: Samhita Alla --- docs/integrations.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/integrations.md b/docs/integrations.md index 34ebfd073d..5a4bbcc909 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -49,13 +49,12 @@ Data is automatically marshalled and unmarshalled in and out of the plugin. User {py:class}`~flytekit.core.base_task.PythonTask` API defined in Flytekit. Flytekit Plugins are lazily loaded and can be released independently like libraries. We follow a convention to name the -plugin like `flytekitplugins-*`, where * indicates the package to be integrated into Flytekit. For example +plugin like `flytekitplugins-*`, where \* indicates the package to be integrated into Flytekit. For example `flytekitplugins-papermill` enables users to author Flytekit tasks using [Papermill](https://papermill.readthedocs.io/en/latest/). You can find the plugins maintained by the core Flyte team [here](https://github.com/flyteorg/flytekit/tree/master/plugins). ::: - ## Native Backend Plugins Native Backend Plugins are the plugins that can be executed without any external service dependencies because the compute is @@ -83,7 +82,6 @@ orchestrated by Flyte itself, within its provisioned Kubernetes clusters. (external_service_backend_plugins)= - ## External Service Backend Plugins As the term suggests, external service backend plugins relies on external services like @@ -138,7 +136,6 @@ Flyte uses Kustomize to generate the the deployment configuration which can be l :::: - ## Custom Container Tasks Because Flyte uses executable docker containers as the smallest unit of compute, you can write custom tasks with the @@ -208,6 +205,7 @@ flytesnacks/examples/athena_plugin/index flytesnacks/examples/aws_batch_plugin/index flytesnacks/examples/hive_plugin/index flytesnacks/examples/mmcloud_plugin/index +flytesnacks/examples/sensor/index flytesnacks/examples/snowflake_plugin/index flytesnacks/examples/databricks_plugin/index flytesnacks/examples/bigquery_plugin/index From 4757dfe73a3a86c248abe32c539802a813e610ce Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Thu, 7 Dec 2023 11:58:49 +0530 Subject: [PATCH 5/6] add flytesnacks to the path Signed-off-by: Samhita Alla --- .github/workflows/single-binary.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/single-binary.yml b/.github/workflows/single-binary.yml index 138e5a238a..5a22cf4594 100644 --- a/.github/workflows/single-binary.yml +++ b/.github/workflows/single-binary.yml @@ -202,7 +202,7 @@ jobs: --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \ --version ${{ env.FLYTESNACKS_VERSION }} \ flytesnacks/$line; - done < flyte_tests.txt + done < flytesnacks/flyte_tests.txt - name: End2End run: | make end2end_execute From 0995777419e8b2a7dc29d52d4fc8a2f3d4e0602a Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Thu, 7 Dec 2023 11:59:52 +0530 Subject: [PATCH 6/6] add flytesnacks to the path Signed-off-by: Samhita Alla --- .github/workflows/end2end.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 15de425846..aff4665fee 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -88,7 +88,7 @@ jobs: --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \ --version ${{ env.FLYTESNACKS_VERSION }} \ flytesnacks/$line; - done < flyte_tests.txt + done < flytesnacks/flyte_tests.txt - name: Register all flytesnacks examples if: ${{ inputs.priorities != 'P0' }} uses: unionai/flyte-register-action@v0.0.2