From 29eb7a018998d420cb437ea3d23d5991ec263dde Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Sun, 10 Mar 2024 05:04:28 -0700 Subject: [PATCH 01/12] Use flytectl demo to run integration tests Signed-off-by: Kevin Su --- .github/workflows/end2end.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index aff4665fee..6a9253b441 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -57,13 +57,13 @@ jobs: cp /tmp/tmp/docker-images-flytecopilot/snapshot-flytecopilot.tar snapshot-flytecopilot.tar cp /tmp/tmp/docker-images-flytepropeller/snapshot-flytepropeller.tar snapshot-flytepropeller.tar flytectl config init - flytectl sandbox start --source=$(pwd) + flytectl demo start --source=$(pwd) - name: Prime docker cache run: | - flytectl sandbox exec -- docker load -i /root/snapshot-datacatalog.tar - flytectl sandbox exec -- docker load -i /root/snapshot-flyteadmin.tar - flytectl sandbox exec -- docker load -i /root/snapshot-flytecopilot.tar - flytectl sandbox exec -- docker load -i /root/snapshot-flytepropeller.tar + flytectl demo exec -- docker load -i /root/snapshot-datacatalog.tar + flytectl demo exec -- docker load -i /root/snapshot-flyteadmin.tar + flytectl demo exec -- docker load -i /root/snapshot-flytecopilot.tar + flytectl demo exec -- docker load -i /root/snapshot-flytepropeller.tar - name: Setup Flytekit run: | python -m pip install --upgrade pip @@ -105,18 +105,18 @@ jobs: # make end2end_execute - name: Upgrade Helm charts run: | - flytectl sandbox exec -- helm repo add flyteorg https://flyteorg.github.io/flyte - flytectl sandbox exec -- helm repo update - flytectl sandbox exec -- helm upgrade flyte -n flyte-core --kubeconfig=/etc/rancher/k3s/k3s.yaml flyteorg/flyte-core -f /flyteorg/share/flyte/values-sandbox.yaml --wait --set datacatalog.image.repository=${{ github.repository_owner }}/datacalog,datacatalog.image.tag=latest + flytectl demo exec -- helm repo add flyteorg https://flyteorg.github.io/flyte + flytectl demo exec -- helm repo update + flytectl demo exec -- helm upgrade flyte -n flyte-core --kubeconfig=/etc/rancher/k3s/k3s.yaml flyteorg/flyte-core -f /flyteorg/share/flyte/values-sandbox.yaml --wait --set datacatalog.image.repository=${{ github.repository_owner }}/datacalog,datacatalog.image.tag=latest # TODO(monorepo): the following commands are not correct. # we have to separate the calls to replace the images into different commands. - # flytectl sandbox exec -- helm upgrade flyte -n flyte-core --kubeconfig=/etc/rancher/k3s/k3s.yaml flyteorg/flyte-core -f /flyteorg/share/flyte/values-sandbox.yaml --wait \ + # flytectl demo exec -- helm upgrade flyte -n flyte-core --kubeconfig=/etc/rancher/k3s/k3s.yaml flyteorg/flyte-core -f /flyteorg/share/flyte/values-sandbox.yaml --wait \ # --set datacatalog.image.repository=${{ github.repository_owner }}/datacalog,datacatalog.image.tag=latest # --set flyteadmin.image.repository=${{ github.repository_owner }}/datacalog,flyteadmin.image.tag=latest # --set flytecopilot.image.repository=${{ github.repository_owner }}/datacalog,flytecopilot.image.tag=latest # --set flytepropeller.image.repository=${{ github.repository_owner }}/datacalog,flytepropeller.image.tag=latest - flytectl sandbox exec -- k3s kubectl get pods -n flyte -oyaml + flytectl demo exec -- k3s kubectl get pods -n flyte -oyaml - name: Post Upgrade Tests env: PRIORITIES: "${{ inputs.priorities }}" From 679fbc7c5f3752bfafd60abebe11c25251e674c8 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 6 May 2024 15:37:11 +0800 Subject: [PATCH 02/12] test Signed-off-by: Kevin Su --- .github/workflows/checks.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 8532179444..20e46840f1 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -76,15 +76,14 @@ jobs: with: component: ${{ matrix.component }} - # TODO(monorepo): these tests are broken. They never test an actual change. - # endtoend: - # name: End2End Test - # needs: [ docker-build ] - # uses: ./.github/workflows/end2end.yml - # with: - # # Reusing the output of the matrix is ok as they are essentially writing the same value (i.e. the directory artifacts are written + run id) - # cache_key: ${{ needs.docker-build.outputs.cache_key }} - # priorities: "P0" + endtoend: + name: End2End Test + needs: [ docker-build ] + uses: ./.github/workflows/end2end.yml + with: + # Reusing the output of the matrix is ok as they are essentially writing the same value (i.e. the directory artifacts are written + run id) + cache_key: ${{ needs.docker-build.outputs.cache_key }} + priorities: "P0" integration: name: Integration Test From 2cffdbeb98cf1064f3a7e497ba693e0d614a2476 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 6 May 2024 15:45:13 +0800 Subject: [PATCH 03/12] test Signed-off-by: Kevin Su --- .github/workflows/end2end.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 6a9253b441..a09a42f9ab 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -18,6 +18,7 @@ jobs: env: FLYTESNACKS_VERSION: "" steps: + - uses: insightsengineering/disk-space-reclaimer@v1 - name: Set latest Flytesnacks release if: ${{ env.FLYTESNACKS_VERSION == '' }} run: | From e3f1602493d5fe60c31ab58d86cac3b745398285 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 6 May 2024 17:20:54 +0800 Subject: [PATCH 04/12] test Signed-off-by: Kevin Su --- .github/workflows/end2end.yml | 46 +------------------------- boilerplate/flyte/end2end/run-tests.py | 17 ++++++++-- 2 files changed, 16 insertions(+), 47 deletions(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index a09a42f9ab..a396f6b9ba 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -31,34 +31,10 @@ jobs: - uses: actions/setup-python@v3 with: python-version: 3.11 - - id: load-docker-cache-datacatalog - uses: actions/cache@v3 - with: - path: /tmp/tmp/docker-images-datacatalog - key: ${{ inputs.cache_key }}-datacatalog - - id: load-docker-cache-flyteadmin - uses: actions/cache@v3 - with: - path: /tmp/tmp/docker-images-flyteadmin - key: ${{ inputs.cache_key }}-flyteadmin - - id: load-docker-cache-flytecopilot - uses: actions/cache@v3 - with: - path: /tmp/tmp/docker-images-flytecopilot - key: ${{ inputs.cache_key }}-flytecopilot - - id: load-docker-cache-flytepropeller - uses: actions/cache@v3 - with: - path: /tmp/tmp/docker-images-flytepropeller - key: ${{ inputs.cache_key }}-flytepropeller - name: Create Sandbox Cluster run: | - cp /tmp/tmp/docker-images-datacatalog/snapshot-datacatalog.tar snapshot-datacatalog.tar - cp /tmp/tmp/docker-images-flyteadmin/snapshot-flyteadmin.tar snapshot-flyteadmin.tar - cp /tmp/tmp/docker-images-flytecopilot/snapshot-flytecopilot.tar snapshot-flytecopilot.tar - cp /tmp/tmp/docker-images-flytepropeller/snapshot-flytepropeller.tar snapshot-flytepropeller.tar flytectl config init - flytectl demo start --source=$(pwd) + flytectl demo start --source=$(pwd) --image ghcr.io/${{ github.repository_owner }}/flyte-sandbox-bundled:latest - name: Prime docker cache run: | flytectl demo exec -- docker load -i /root/snapshot-datacatalog.tar @@ -98,26 +74,6 @@ jobs: project: flytesnacks version: ${{ env.FLYTESNACKS_VERSION }} domain: development - # - name: Pre Upgrade Tests - # if: ${{ github.event.repository.name == 'flyteadmin' }} - # env: - # PRIORITIES: "${{ inputs.priorities }}" - # run: | - # make end2end_execute - - name: Upgrade Helm charts - run: | - flytectl demo exec -- helm repo add flyteorg https://flyteorg.github.io/flyte - flytectl demo exec -- helm repo update - flytectl demo exec -- helm upgrade flyte -n flyte-core --kubeconfig=/etc/rancher/k3s/k3s.yaml flyteorg/flyte-core -f /flyteorg/share/flyte/values-sandbox.yaml --wait --set datacatalog.image.repository=${{ github.repository_owner }}/datacalog,datacatalog.image.tag=latest - # TODO(monorepo): the following commands are not correct. - # we have to separate the calls to replace the images into different commands. - # flytectl demo exec -- helm upgrade flyte -n flyte-core --kubeconfig=/etc/rancher/k3s/k3s.yaml flyteorg/flyte-core -f /flyteorg/share/flyte/values-sandbox.yaml --wait \ - # --set datacatalog.image.repository=${{ github.repository_owner }}/datacalog,datacatalog.image.tag=latest - # --set flyteadmin.image.repository=${{ github.repository_owner }}/datacalog,flyteadmin.image.tag=latest - # --set flytecopilot.image.repository=${{ github.repository_owner }}/datacalog,flytecopilot.image.tag=latest - # --set flytepropeller.image.repository=${{ github.repository_owner }}/datacalog,flytepropeller.image.tag=latest - - flytectl demo exec -- k3s kubectl get pods -n flyte -oyaml - name: Post Upgrade Tests env: PRIORITIES: "${{ inputs.priorities }}" diff --git a/boilerplate/flyte/end2end/run-tests.py b/boilerplate/flyte/end2end/run-tests.py index 905f29079c..88fb09b0c9 100644 --- a/boilerplate/flyte/end2end/run-tests.py +++ b/boilerplate/flyte/end2end/run-tests.py @@ -22,10 +22,12 @@ def execute_workflow( workflow_name, inputs, cluster_pool_name: Optional[str] = None, + wait: bool = False, ): print(f"Fetching workflow={workflow_name} and version={version}") wf = remote.fetch_workflow(name=workflow_name, version=version) - return remote.execute(wf, inputs=inputs, wait=False, cluster_pool=cluster_pool_name) + print(f"Executing workflow={workflow_name} and version={version}") + return remote.execute(wf, inputs=inputs, wait=wait, cluster_pool=cluster_pool_name) def executions_finished( @@ -64,6 +66,7 @@ def schedule_workflow_groups( terminate_workflow_on_failure: bool, parsed_manifest: List[dict], cluster_pool_name: Optional[str] = None, + parallel: bool = True, ) -> Dict[str, bool]: """ Schedule workflows executions for all workflow groups and return True if all executions succeed, otherwise @@ -79,7 +82,7 @@ def schedule_workflow_groups( if workflow_group_item: workflows = workflow_group_item[0]["examples"] executions_by_wfgroup[wf_group] = [ - execute_workflow(remote, tag, workflow[0], workflow[1], cluster_pool_name) + execute_workflow(remote, tag, workflow[0], workflow[1], cluster_pool_name, not parallel) for workflow in workflows ] @@ -135,6 +138,7 @@ def run( test_project_name: str, test_project_domain: str, cluster_pool_name: Optional[str] = None, + parallel: bool = True, ) -> List[Dict[str, str]]: remote = FlyteRemote( Config.auto(config_file=config_file_path), @@ -182,6 +186,7 @@ def run( terminate_workflow_on_failure, parsed_manifest, cluster_pool_name, + parallel ) for workflow_group, succeeded in results_by_wfgroup.items(): @@ -241,6 +246,12 @@ def run( is_flag=False, help="Name of domain in project to run functional tests on", ) +@click.option( + "--parallel", + is_flag=True, + default=False, + help="Run tests in parallel if set to True", +) @click.argument( "cluster_pool_name", required=False, @@ -255,6 +266,7 @@ def cli( terminate_workflow_on_failure, test_project_name, test_project_domain, + parallel, cluster_pool_name, ): print(f"return_non_zero_on_failure={return_non_zero_on_failure}") @@ -266,6 +278,7 @@ def cli( test_project_name, test_project_domain, cluster_pool_name, + parallel ) # Write a json object in its own line describing the result of this run to stdout From 2187b0e56abf9e0bc8f130cf585ba49e6c2dc38f Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 6 May 2024 18:08:19 +0800 Subject: [PATCH 05/12] test Signed-off-by: Kevin Su --- .github/workflows/end2end.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 41ba84247f..98def202f7 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -34,13 +34,7 @@ jobs: - name: Create Sandbox Cluster run: | flytectl config init - flytectl demo start --source=$(pwd) --image ghcr.io/${{ github.repository_owner }}/flyte-sandbox-bundled:latest - - name: Prime docker cache - run: | - flytectl demo exec -- docker load -i /root/snapshot-datacatalog.tar - flytectl demo exec -- docker load -i /root/snapshot-flyteadmin.tar - flytectl demo exec -- docker load -i /root/snapshot-flytecopilot.tar - flytectl demo exec -- docker load -i /root/snapshot-flytepropeller.tar + flytectl demo start --source=$(pwd) --image ghcr.io/${{ github.repository_owner }}/flyte-sandbox-bundled:latest --disable-agent - name: Setup Flytekit run: | python -m pip install --upgrade pip From 785ff3efae1f0159609b16ed5599db6cfaf9de62 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 6 May 2024 18:32:28 +0800 Subject: [PATCH 06/12] test Signed-off-by: Kevin Su --- .github/workflows/end2end.yml | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 98def202f7..7a1aaf4107 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -47,28 +47,14 @@ jobs: repository: flyteorg/flytesnacks path: flytesnacks ref: ${{ env.FLYTESNACKS_VERSION }} - - name: Register P0 tests + - name: Register flytesnacks examples if: ${{ inputs.priorities == 'P0' }} run: | - while read -r line; - do - pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \ - register \ - --project flytesnacks \ - --domain development \ - --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \ - --version ${{ env.FLYTESNACKS_VERSION }} \ - flytesnacks/$line; - done < flytesnacks/flyte_tests.txt - - name: Register all flytesnacks examples - if: ${{ inputs.priorities != 'P0' }} - uses: unionai/flyte-register-action@v0.0.2 - with: - flytesnacks: true - project: flytesnacks - version: ${{ env.FLYTESNACKS_VERSION }} - domain: development - - name: Post Upgrade Tests + flytectl --config ./boilerplate/flyte/end2end/functional-test-config.yaml \ + register examples \ + --project flytesnacks \ + --domain development + - name: Run flytesnacks examples env: PRIORITIES: "${{ inputs.priorities }}" run: | From e260f0053d75dff9856a145476e692e0a13d3137 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 6 May 2024 20:24:12 +0800 Subject: [PATCH 07/12] on push Signed-off-by: Kevin Su --- .github/workflows/checks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ec07b9ea5b..f517974561 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -81,6 +81,7 @@ jobs: endtoend: name: End2End Test needs: [ docker-build ] + if: ${{ github.event_name != 'pull_request' }} uses: ./.github/workflows/end2end.yml with: # Reusing the output of the matrix is ok as they are essentially writing the same value (i.e. the directory artifacts are written + run id) From c659e43e321e2fccf31da262283354867a5bc264 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 6 May 2024 20:53:17 +0800 Subject: [PATCH 08/12] run after build-and-push-sandbox-bundled-image Signed-off-by: Kevin Su --- .github/workflows/checks.yml | 10 ---------- .github/workflows/end2end.yml | 4 ---- .github/workflows/single-binary.yml | 8 ++++++++ 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index f517974561..2849ec06bf 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -78,16 +78,6 @@ jobs: with: component: ${{ matrix.component }} - endtoend: - name: End2End Test - needs: [ docker-build ] - if: ${{ github.event_name != 'pull_request' }} - uses: ./.github/workflows/end2end.yml - with: - # Reusing the output of the matrix is ok as they are essentially writing the same value (i.e. the directory artifacts are written + run id) - cache_key: ${{ needs.docker-build.outputs.cache_key }} - priorities: "P0" - integration: name: Integration Test needs: diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 7a1aaf4107..07a1a0c07d 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -7,10 +7,6 @@ on: description: "Priorities of tests to register (comma-separated)" required: true type: string - cache_key: - description: "Cache key for docker image" - required: true - type: string jobs: endtoend: name: End to End tests diff --git a/.github/workflows/single-binary.yml b/.github/workflows/single-binary.yml index 657160a7a9..02badc3e3a 100644 --- a/.github/workflows/single-binary.yml +++ b/.github/workflows/single-binary.yml @@ -246,3 +246,11 @@ jobs: tags: ${{ steps.image-names.outputs.tags }} build-args: "FLYTE_SANDBOX_VERSION=${{ env.FLYTE_SANDBOX_VERSION }}" push: true + + endtoend: + name: End2End Test + needs: [ build-and-push-sandbox-bundled-image ] + if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} + uses: ./.github/workflows/end2end.yml + with: + priorities: "P0" \ No newline at end of file From faf8fc8153b3e0ac039c300ceab31c54be537985 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Thu, 9 May 2024 19:43:13 +0800 Subject: [PATCH 09/12] test Signed-off-by: Kevin Su --- .github/workflows/end2end.yml | 57 -------------------- .github/workflows/single-binary.yml | 8 --- boilerplate/flyte/end2end/run-tests.py | 75 +++++++++++--------------- 3 files changed, 31 insertions(+), 109 deletions(-) delete mode 100644 .github/workflows/end2end.yml diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml deleted file mode 100644 index 07a1a0c07d..0000000000 --- a/.github/workflows/end2end.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: End to End tests - -on: - workflow_call: - inputs: - priorities: - description: "Priorities of tests to register (comma-separated)" - required: true - type: string -jobs: - endtoend: - name: End to End tests - runs-on: ubuntu-latest - env: - FLYTESNACKS_VERSION: "" - steps: - - uses: insightsengineering/disk-space-reclaimer@v1 - - name: Set latest Flytesnacks release - if: ${{ env.FLYTESNACKS_VERSION == '' }} - run: | - FLYTESNACKS_VERSION="$(curl --silent https://api.github.com/repos/flyteorg/flytesnacks/releases/latest | jq -r .tag_name)" - echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV} - - name: Checkout - uses: actions/checkout@v4 - - uses: unionai/flytectl-setup-action@v0.0.1 - name: Setup flytectl - - uses: actions/setup-python@v3 - with: - python-version: 3.11 - - name: Create Sandbox Cluster - run: | - flytectl config init - flytectl demo start --source=$(pwd) --image ghcr.io/${{ github.repository_owner }}/flyte-sandbox-bundled:latest --disable-agent - - name: Setup Flytekit - run: | - python -m pip install --upgrade pip - pip install flytekit flytekitplugins-deck-standard - pip freeze - - name: Checkout flytesnacks - if: ${{ inputs.priorities == 'P0' }} - uses: actions/checkout@v4 - with: - repository: flyteorg/flytesnacks - path: flytesnacks - ref: ${{ env.FLYTESNACKS_VERSION }} - - name: Register flytesnacks examples - if: ${{ inputs.priorities == 'P0' }} - run: | - flytectl --config ./boilerplate/flyte/end2end/functional-test-config.yaml \ - register examples \ - --project flytesnacks \ - --domain development - - name: Run flytesnacks examples - env: - PRIORITIES: "${{ inputs.priorities }}" - run: | - make end2end_execute diff --git a/.github/workflows/single-binary.yml b/.github/workflows/single-binary.yml index 14f7f3479a..a932c20191 100644 --- a/.github/workflows/single-binary.yml +++ b/.github/workflows/single-binary.yml @@ -246,11 +246,3 @@ jobs: tags: ${{ steps.image-names.outputs.tags }} build-args: "FLYTE_SANDBOX_VERSION=${{ env.FLYTE_SANDBOX_VERSION }}" push: true - - endtoend: - name: End2End Test - needs: [ build-and-push-sandbox-bundled-image ] - if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - uses: ./.github/workflows/end2end.yml - with: - priorities: "P0" \ No newline at end of file diff --git a/boilerplate/flyte/end2end/run-tests.py b/boilerplate/flyte/end2end/run-tests.py index 21c62b9288..b345edb19c 100644 --- a/boilerplate/flyte/end2end/run-tests.py +++ b/boilerplate/flyte/end2end/run-tests.py @@ -17,21 +17,19 @@ def execute_workflow( - remote: FlyteRemote, - version, - workflow_name, - inputs, - cluster_pool_name: Optional[str] = None, - wait: bool = False, + remote: FlyteRemote, + version, + workflow_name, + inputs, + cluster_pool_name: Optional[str] = None, ): print(f"Fetching workflow={workflow_name} and version={version}") wf = remote.fetch_workflow(name=workflow_name, version=version) - print(f"Executing workflow={workflow_name} and version={version}") - return remote.execute(wf, inputs=inputs, wait=wait, cluster_pool=cluster_pool_name) + return remote.execute(wf, inputs=inputs, wait=False, cluster_pool=cluster_pool_name) def executions_finished( - executions_by_wfgroup: Dict[str, List[FlyteWorkflowExecution]] + executions_by_wfgroup: Dict[str, List[FlyteWorkflowExecution]] ) -> bool: for executions in executions_by_wfgroup.values(): if not all([execution.is_done for execution in executions]): @@ -40,7 +38,7 @@ def executions_finished( def sync_executions( - remote: FlyteRemote, executions_by_wfgroup: Dict[str, List[FlyteWorkflowExecution]] + remote: FlyteRemote, executions_by_wfgroup: Dict[str, List[FlyteWorkflowExecution]] ): try: for executions in executions_by_wfgroup.values(): @@ -60,13 +58,12 @@ def report_executions(executions_by_wfgroup: Dict[str, List[FlyteWorkflowExecuti def schedule_workflow_groups( - tag: str, - workflow_groups: List[str], - remote: FlyteRemote, - terminate_workflow_on_failure: bool, - parsed_manifest: List[dict], - cluster_pool_name: Optional[str] = None, - parallel: bool = True, + tag: str, + workflow_groups: List[str], + remote: FlyteRemote, + terminate_workflow_on_failure: bool, + parsed_manifest: List[dict], + cluster_pool_name: Optional[str] = None, ) -> Dict[str, bool]: """ Schedule workflows executions for all workflow groups and return True if all executions succeed, otherwise @@ -84,14 +81,14 @@ def schedule_workflow_groups( if not workflows: continue executions_by_wfgroup[wf_group] = [ - execute_workflow(remote, tag, workflow[0], workflow[1], cluster_pool_name, not parallel) + execute_workflow(remote, tag, workflow[0], workflow[1], cluster_pool_name) for workflow in workflows ] # Wait for all executions to finish attempt = 0 while attempt == 0 or ( - not executions_finished(executions_by_wfgroup) and attempt < MAX_ATTEMPTS + not executions_finished(executions_by_wfgroup) and attempt < MAX_ATTEMPTS ): attempt += 1 print( @@ -133,14 +130,13 @@ def valid(workflow_group, parsed_manifest): def run( - flytesnacks_release_tag: str, - priorities: List[str], - config_file_path, - terminate_workflow_on_failure: bool, - test_project_name: str, - test_project_domain: str, - cluster_pool_name: Optional[str] = None, - parallel: bool = True, + flytesnacks_release_tag: str, + priorities: List[str], + config_file_path, + terminate_workflow_on_failure: bool, + test_project_name: str, + test_project_domain: str, + cluster_pool_name: Optional[str] = None, ) -> List[Dict[str, str]]: remote = FlyteRemote( Config.auto(config_file=config_file_path), @@ -188,7 +184,6 @@ def run( terminate_workflow_on_failure, parsed_manifest, cluster_pool_name, - parallel ) for workflow_group, succeeded in results_by_wfgroup.items(): @@ -248,12 +243,6 @@ def run( is_flag=False, help="Name of domain in project to run functional tests on", ) -@click.option( - "--parallel", - is_flag=True, - default=False, - help="Run tests in parallel if set to True", -) @click.argument( "cluster_pool_name", required=False, @@ -261,15 +250,14 @@ def run( default=None, ) def cli( - flytesnacks_release_tag, - priorities, - config_file, - return_non_zero_on_failure, - terminate_workflow_on_failure, - test_project_name, - test_project_domain, - parallel, - cluster_pool_name, + flytesnacks_release_tag, + priorities, + config_file, + return_non_zero_on_failure, + terminate_workflow_on_failure, + test_project_name, + test_project_domain, + cluster_pool_name, ): print(f"return_non_zero_on_failure={return_non_zero_on_failure}") results = run( @@ -280,7 +268,6 @@ def cli( test_project_name, test_project_domain, cluster_pool_name, - parallel ) # Write a json object in its own line describing the result of this run to stdout From 80bba9570ffd5949212ca9e43a397a23deb57156 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Thu, 9 May 2024 19:44:22 +0800 Subject: [PATCH 10/12] test Signed-off-by: Kevin Su --- boilerplate/flyte/end2end/run-tests.py | 60 +++++++++++++------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/boilerplate/flyte/end2end/run-tests.py b/boilerplate/flyte/end2end/run-tests.py index b345edb19c..e68e54fc14 100644 --- a/boilerplate/flyte/end2end/run-tests.py +++ b/boilerplate/flyte/end2end/run-tests.py @@ -17,11 +17,11 @@ def execute_workflow( - remote: FlyteRemote, - version, - workflow_name, - inputs, - cluster_pool_name: Optional[str] = None, + remote: FlyteRemote, + version, + workflow_name, + inputs, + cluster_pool_name: Optional[str] = None, ): print(f"Fetching workflow={workflow_name} and version={version}") wf = remote.fetch_workflow(name=workflow_name, version=version) @@ -29,7 +29,7 @@ def execute_workflow( def executions_finished( - executions_by_wfgroup: Dict[str, List[FlyteWorkflowExecution]] + executions_by_wfgroup: Dict[str, List[FlyteWorkflowExecution]] ) -> bool: for executions in executions_by_wfgroup.values(): if not all([execution.is_done for execution in executions]): @@ -38,7 +38,7 @@ def executions_finished( def sync_executions( - remote: FlyteRemote, executions_by_wfgroup: Dict[str, List[FlyteWorkflowExecution]] + remote: FlyteRemote, executions_by_wfgroup: Dict[str, List[FlyteWorkflowExecution]] ): try: for executions in executions_by_wfgroup.values(): @@ -58,12 +58,12 @@ def report_executions(executions_by_wfgroup: Dict[str, List[FlyteWorkflowExecuti def schedule_workflow_groups( - tag: str, - workflow_groups: List[str], - remote: FlyteRemote, - terminate_workflow_on_failure: bool, - parsed_manifest: List[dict], - cluster_pool_name: Optional[str] = None, + tag: str, + workflow_groups: List[str], + remote: FlyteRemote, + terminate_workflow_on_failure: bool, + parsed_manifest: List[dict], + cluster_pool_name: Optional[str] = None, ) -> Dict[str, bool]: """ Schedule workflows executions for all workflow groups and return True if all executions succeed, otherwise @@ -88,7 +88,7 @@ def schedule_workflow_groups( # Wait for all executions to finish attempt = 0 while attempt == 0 or ( - not executions_finished(executions_by_wfgroup) and attempt < MAX_ATTEMPTS + not executions_finished(executions_by_wfgroup) and attempt < MAX_ATTEMPTS ): attempt += 1 print( @@ -130,13 +130,13 @@ def valid(workflow_group, parsed_manifest): def run( - flytesnacks_release_tag: str, - priorities: List[str], - config_file_path, - terminate_workflow_on_failure: bool, - test_project_name: str, - test_project_domain: str, - cluster_pool_name: Optional[str] = None, + flytesnacks_release_tag: str, + priorities: List[str], + config_file_path, + terminate_workflow_on_failure: bool, + test_project_name: str, + test_project_domain: str, + cluster_pool_name: Optional[str] = None, ) -> List[Dict[str, str]]: remote = FlyteRemote( Config.auto(config_file=config_file_path), @@ -250,14 +250,14 @@ def run( default=None, ) def cli( - flytesnacks_release_tag, - priorities, - config_file, - return_non_zero_on_failure, - terminate_workflow_on_failure, - test_project_name, - test_project_domain, - cluster_pool_name, + flytesnacks_release_tag, + priorities, + config_file, + return_non_zero_on_failure, + terminate_workflow_on_failure, + test_project_name, + test_project_domain, + cluster_pool_name, ): print(f"return_non_zero_on_failure={return_non_zero_on_failure}") results = run( @@ -281,4 +281,4 @@ def cli( if __name__ == "__main__": - cli() + cli() \ No newline at end of file From 19b1b49b0e2a39eeb4c2ca7a9c0de8547f089615 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Thu, 9 May 2024 19:44:35 +0800 Subject: [PATCH 11/12] test Signed-off-by: Kevin Su --- boilerplate/flyte/end2end/run-tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boilerplate/flyte/end2end/run-tests.py b/boilerplate/flyte/end2end/run-tests.py index e68e54fc14..f6558247a6 100644 --- a/boilerplate/flyte/end2end/run-tests.py +++ b/boilerplate/flyte/end2end/run-tests.py @@ -281,4 +281,4 @@ def cli( if __name__ == "__main__": - cli() \ No newline at end of file + cli() From 83eecb591ae5b48a94dbfd6ea7414f0aeaeaefd2 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Sat, 15 Jun 2024 18:01:16 -0700 Subject: [PATCH 12/12] remove end2end.yml Signed-off-by: Kevin Su --- .github/workflows/end2end.yml | 124 ---------------------------------- 1 file changed, 124 deletions(-) delete mode 100644 .github/workflows/end2end.yml diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml deleted file mode 100644 index d7cf8182c0..0000000000 --- a/.github/workflows/end2end.yml +++ /dev/null @@ -1,124 +0,0 @@ -name: End to End tests - -on: - workflow_call: - inputs: - priorities: - description: "Priorities of tests to register (comma-separated)" - required: true - type: string - cache_key: - description: "Cache key for docker image" - required: true - type: string -jobs: - endtoend: - name: End to End tests - runs-on: ubuntu-latest - env: - FLYTESNACKS_VERSION: "" - steps: - - name: Set latest Flytesnacks release - if: ${{ env.FLYTESNACKS_VERSION == '' }} - run: | - FLYTESNACKS_VERSION="$(curl --silent https://api.github.com/repos/flyteorg/flytesnacks/releases/latest | jq -r .tag_name)" - echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV} - - name: Checkout - uses: actions/checkout@v4 - - uses: unionai/flytectl-setup-action@v0.0.3 - name: Setup flytectl - - uses: actions/setup-python@v3 - with: - python-version: 3.11 - - id: load-docker-cache-datacatalog - uses: actions/cache@v3 - with: - path: /tmp/tmp/docker-images-datacatalog - key: ${{ inputs.cache_key }}-datacatalog - - id: load-docker-cache-flyteadmin - uses: actions/cache@v3 - with: - path: /tmp/tmp/docker-images-flyteadmin - key: ${{ inputs.cache_key }}-flyteadmin - - id: load-docker-cache-flytecopilot - uses: actions/cache@v3 - with: - path: /tmp/tmp/docker-images-flytecopilot - key: ${{ inputs.cache_key }}-flytecopilot - - id: load-docker-cache-flytepropeller - uses: actions/cache@v3 - with: - path: /tmp/tmp/docker-images-flytepropeller - key: ${{ inputs.cache_key }}-flytepropeller - - name: Create Sandbox Cluster - run: | - cp /tmp/tmp/docker-images-datacatalog/snapshot-datacatalog.tar snapshot-datacatalog.tar - cp /tmp/tmp/docker-images-flyteadmin/snapshot-flyteadmin.tar snapshot-flyteadmin.tar - cp /tmp/tmp/docker-images-flytecopilot/snapshot-flytecopilot.tar snapshot-flytecopilot.tar - cp /tmp/tmp/docker-images-flytepropeller/snapshot-flytepropeller.tar snapshot-flytepropeller.tar - flytectl config init - flytectl sandbox start --source=$(pwd) - - name: Prime docker cache - run: | - flytectl sandbox exec -- docker load -i /root/snapshot-datacatalog.tar - flytectl sandbox exec -- docker load -i /root/snapshot-flyteadmin.tar - flytectl sandbox exec -- docker load -i /root/snapshot-flytecopilot.tar - flytectl sandbox exec -- docker load -i /root/snapshot-flytepropeller.tar - - name: Setup Flytekit - run: | - python -m pip install --upgrade pip - pip install flytekit flytekitplugins-deck-standard - pip freeze - - name: Checkout flytesnacks - if: ${{ inputs.priorities == 'P0' }} - uses: actions/checkout@v4 - with: - repository: flyteorg/flytesnacks - path: flytesnacks - ref: ${{ env.FLYTESNACKS_VERSION }} - - name: Register P0 tests - if: ${{ inputs.priorities == 'P0' }} - run: | - while read -r line; - do - pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \ - register \ - --project flytesnacks \ - --domain development \ - --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \ - --version ${{ env.FLYTESNACKS_VERSION }} \ - flytesnacks/$line; - done < flytesnacks/flyte_tests.txt - - name: Register all flytesnacks examples - if: ${{ inputs.priorities != 'P0' }} - uses: unionai/flyte-register-action@v0.0.2 - with: - flytesnacks: true - project: flytesnacks - version: ${{ env.FLYTESNACKS_VERSION }} - domain: development - # - name: Pre Upgrade Tests - # if: ${{ github.event.repository.name == 'flyteadmin' }} - # env: - # PRIORITIES: "${{ inputs.priorities }}" - # run: | - # make end2end_execute - - name: Upgrade Helm charts - run: | - flytectl sandbox exec -- helm repo add flyteorg https://flyteorg.github.io/flyte - flytectl sandbox exec -- helm repo update - flytectl sandbox exec -- helm upgrade flyte -n flyte-core --kubeconfig=/etc/rancher/k3s/k3s.yaml flyteorg/flyte-core -f /flyteorg/share/flyte/values-sandbox.yaml --wait --set datacatalog.image.repository=${{ github.repository_owner }}/datacalog,datacatalog.image.tag=latest - # TODO(monorepo): the following commands are not correct. - # we have to separate the calls to replace the images into different commands. - # flytectl sandbox exec -- helm upgrade flyte -n flyte-core --kubeconfig=/etc/rancher/k3s/k3s.yaml flyteorg/flyte-core -f /flyteorg/share/flyte/values-sandbox.yaml --wait \ - # --set datacatalog.image.repository=${{ github.repository_owner }}/datacalog,datacatalog.image.tag=latest - # --set flyteadmin.image.repository=${{ github.repository_owner }}/datacalog,flyteadmin.image.tag=latest - # --set flytecopilot.image.repository=${{ github.repository_owner }}/datacalog,flytecopilot.image.tag=latest - # --set flytepropeller.image.repository=${{ github.repository_owner }}/datacalog,flytepropeller.image.tag=latest - - flytectl sandbox exec -- k3s kubectl get pods -n flyte -oyaml - - name: Post Upgrade Tests - env: - PRIORITIES: "${{ inputs.priorities }}" - run: | - make end2end_execute