merge master, add isodate to deps

Signed-off-by: Yee Hing Tong <[email protected]>
flyteorg · Dec 26, 2023 · dbff1cc · dbff1cc
2 parents 1b2a831 + ad328a7
commit dbff1cc
Show file tree

Hide file tree

Showing 419 changed files with 11,709 additions and 11,642 deletions.
diff --git a/.github/workflows/build_image.yml b/.github/workflows/build_image.yml
@@ -10,7 +10,7 @@ jobs:
       matrix:
         python-version: [3.9", "3.10", "3.11"]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: "0"
       - name: Set up QEMU
@@ -50,7 +50,7 @@ jobs:
   build-and-push-flyteagent-images-manual:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: "0"
       - name: Set up QEMU

diff --git a/.github/workflows/docs_build.yml b/.github/workflows/docs_build.yml
@@ -7,6 +7,11 @@ on:
   pull_request:
     branches:
       - master
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   docs_warnings:
     name: Docs Warnings

diff --git a/.github/workflows/monodocs_build.yml b/.github/workflows/monodocs_build.yml
@@ -0,0 +1,52 @@
+name: Monodocs Build
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+jobs:
+  docs:
+    name: Monodocs Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Fetch flytekit code
+        uses: actions/checkout@v4
+        with:
+          path: "${{ github.workspace }}/flytekit"
+      - name: Fetch flyte code
+        uses: actions/checkout@v4
+        with:
+          repository: flyteorg/flyte
+          path: "${{ github.workspace }}/flyte"
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          auto-update-conda: true
+          python-version: 3.9
+      - shell: bash -el {0}
+        working-directory: ${{ github.workspace }}/flyte
+        run: |
+          conda install -c conda-forge conda-lock
+          conda-lock install -n monodocs-env monodocs-environment.lock.yaml
+      - shell: bash -el {0}
+        run: |
+          conda activate monodocs-env
+          conda info
+          conda list
+          conda config --show-sources
+          conda config --show
+          printenv | sort
+      - name: Build the documentation
+        working-directory: ${{ github.workspace }}/flyte
+        shell: bash -el {0}
+        env:
+          FLYTEKIT_LOCAL_PATH: ${{ github.workspace }}/flytekit
+        run: |
+          conda activate monodocs-env
+          make -C docs clean html SPHINXOPTS="-W -vvv"
diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml
@@ -9,55 +9,147 @@ on:
 env:
   FLYTE_SDK_LOGGING_LEVEL: 10 # debug
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ["3.8", "3.11"]
+        python-version: ["3.8", "3.11", "3.12"]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Cache pip
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           # This path is specific to Ubuntu
           path: ~/.cache/pip
           # Look to see if there is a cache hit for the corresponding requirements files
           key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
       - name: Install dependencies
         run: |
-          make setup && pip freeze
+          make setup
+          pip uninstall -y pandas
+          pip freeze
+      - name: Run extras unit tests with coverage
+        # Skip this step if running on python 3.12 due to https://github.com/tensorflow/tensorflow/issues/62003
+        # and https://github.com/pytorch/pytorch/issues/110436
+        if: ${{ matrix.python-version != '3.12' }}
+        env:
+          PYTEST_OPTS: -n2
+        run: |
+          make unit_test_extras_codecov
       - name: Test with coverage
+        env:
+          PYTEST_OPTS: -n2
         run: |
           make unit_test_codecov
       - name: Codecov
-        uses: codecov/[email protected].0
+        uses: codecov/[email protected].4
         with:
           fail_ci_if_error: false
+          files: coverage.xml
 
-  build-integration:
+  build-with-pandas:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest ]
+        python-version: [ "3.11" ]
+        pandas: [ "pandas<2.0.0", "pandas>=2.0.0" ]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache pip
+        uses: actions/cache@v3
+        with:
+          # This path is specific to Ubuntu
+          path: ~/.cache/pip
+          # Look to see if there is a cache hit for the corresponding requirements files
+          key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
+      - name: Install dependencies
+        run: |
+          make setup
+          pip install --force-reinstall "${{ matrix.pandas }}"
+          pip freeze
+      - name: Test with coverage
+        env:
+          PYTEST_OPTS: -n2
+        run: |
+          make unit_test_codecov
+      - name: Codecov
+        uses: codecov/[email protected]
+        with:
+          fail_ci_if_error: false
+          files: coverage.xml
+
+  test-serialization:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.8", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache pip
+        uses: actions/cache@v3
+        with:
+          # This path is specific to Ubuntu
+          path: ~/.cache/pip
+          # Look to see if there is a cache hit for the corresponding requirements files
+          key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
+      - name: Install dependencies
+        run: make setup && pip freeze
+      - name: Test with coverage
+        env:
+          PYTEST_OPTS: -n2
+        run: |
+          make test_serialization_codecov
+      - name: Codecov
+        uses: codecov/[email protected]
+        with:
+          fail_ci_if_error: false
+          files: coverage.xml
+
+  integration:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
         # python 3.11 has intermittent issues with the docker build + push step
         # https://github.com/flyteorg/flytekit/actions/runs/5800978835/job/15724237979?pr=1579
-        python-version: ["3.8", "3.11"]
+        python-version: ["3.8", "3.11", "3.12"]
     steps:
-      - uses: actions/checkout@v2
+      - uses: insightsengineering/disk-space-reclaimer@v1
+      # As described in https://github.com/pypa/setuptools_scm/issues/414, SCM needs git history
+      # and tags to work.
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Cache pip
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           # This path is specific to Ubuntu
           path: ~/.cache/pip
@@ -70,14 +162,14 @@ jobs:
       - name: Setup Flyte Sandbox
         run: |
           flytectl demo start
-          flytectl config init
       - name: Build and push to local registry
         run: |
-          docker build . -f Dockerfile.dev -t localhost:30000/flytekit:dev --build-arg PYTHON_VERSION=${{ matrix.python-version }}
+          docker build --push . -f Dockerfile.dev -t localhost:30000/flytekit:dev --build-arg PYTHON_VERSION=${{ matrix.python-version }}
       - name: Integration Test with coverage
         env:
           FLYTEKIT_IMAGE: localhost:30000/flytekit:dev
           FLYTEKIT_CI: 1
+          PYTEST_OPTS: -n2
         run: make integration_test_codecov
       - name: Codecov
         uses: codecov/[email protected]
@@ -93,9 +185,12 @@ jobs:
         python-version: ["3.8", "3.11"]
         plugin-names:
           # Please maintain an alphabetical order in the following list
+          - flytekit-airflow
+          - flytekit-async-fsspec
           - flytekit-aws-athena
           - flytekit-aws-batch
-          - flytekit-aws-sagemaker
+          # TODO: uncomment this when the sagemaker agent is implemented: https://github.com/flyteorg/flyte/issues/4079
+          # - flytekit-aws-sagemaker
           - flytekit-bigquery
           - flytekit-dask
           - flytekit-data-fsspec
@@ -126,6 +221,7 @@ jobs:
           - flytekit-spark
           - flytekit-sqlalchemy
           - flytekit-vaex
+          - flytekit-flyin
           - flytekit-whylogs
         exclude:
           # flytekit-modin depends on ray which does not have a 3.11 wheel yet.
@@ -151,10 +247,6 @@ jobs:
           # https://github.com/numba/numba/issues/8304
           - python-version: 3.11
             plugin-names: "flytekit-mlflow"
-          # pyspark 3.4.0 will support python 3.11, which isn't available yet
-          # Issue tracked in: https://github.com/apache/spark/pull/38987
-          - python-version: 3.11
-            plugin-names: "flytekit-spark"
           # vaex currently doesn't support python 3.11
           - python-version: 3.11
             plugin-names: "flytekit-vaex"
@@ -163,13 +255,13 @@ jobs:
           - python-version: 3.11
             plugin-names: "flytekit-whylogs"
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Cache pip
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           # This path is specific to Ubuntu
           path: ~/.cache/pip
@@ -179,9 +271,9 @@ jobs:
         run: |
           make setup
           cd plugins/${{ matrix.plugin-names }}
-          pip install -r requirements.txt
+          pip install .
           if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi
-          pip install -U https://github.com/flyteorg/flytekit/archive/${{ github.sha }}.zip#egg=flytekit
+          pip install -U $GITHUB_WORKSPACE
           pip freeze
       - name: Test with coverage
         run: |
@@ -201,12 +293,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Fetch the code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: 3.8
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/dev-requirements.in') }}