Separate time partition (#2114)

Signed-off-by: Yee Hing Tong <[email protected]>

.pre-commit-config.yaml

@@ -22,3 +22,9 @@ repos:
     rev: 0.0.9
     hooks:
       - id: check_pdb_hook
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.6
+    hooks:
+      - id: codespell
+        additional_dependencies:
+          - tomli

Dockerfile.dev

@@ -27,7 +27,9 @@ COPY . /flytekit
 # 3. Clean up the apt cache to reduce image size. Reference: https://gist.github.com/marvell/7c812736565928e602c4
 # 4. Create a non-root user 'flytekit' and set appropriate permissions for directories.
 RUN apt-get update && apt-get install build-essential vim libmagic1 git -y \
-    && pip install --no-cache-dir -e /flytekit \
+    && pip install --no-cache-dir -U --pre \
+        flyteidl \
+        -e /flytekit \
         -e /flytekit/plugins/flytekit-k8s-pod \
         -e /flytekit/plugins/flytekit-deck-standard \
         -e /flytekit/plugins/flytekit-flyin \

Makefile

@@ -24,6 +24,7 @@ update_boilerplate:
 
 .PHONY: setup
 setup: install-piptools ## Install requirements
+	pip install flyteidl --pre
 	pip install -r dev-requirements.in
 
 .PHONY: fmt
@@ -43,7 +44,8 @@ lint: ## Run linters
 
 .PHONY: spellcheck
 spellcheck:  ## Runs a spellchecker over all code and documentation
-	codespell -L "te,raison,fo" --skip="./docs/build,./.git"
+	# Configuration is in pyproject.toml
+	codespell
 
 .PHONY: test
 test: lint unit_test

dev-requirements.txt

@@ -10,7 +10,7 @@ absl-py==2.0.0
     # via
     #   tensorboard
     #   tensorflow
-adlfs==2023.10.0
+adlfs==2023.9.0
     # via flytekit
 aiobotocore==2.5.4
     # via s3fs
@@ -24,8 +24,6 @@ aioitertools==0.11.0
     # via aiobotocore
 aiosignal==1.3.1
     # via aiohttp
-appnope==0.1.3
-    # via ipython
 arrow==1.3.0
     # via cookiecutter
 asttokens==2.4.1
@@ -97,6 +95,7 @@ cryptography==41.0.5
     #   azure-storage-blob
     #   msal
     #   pyjwt
+    #   secretstorage
 dataclasses-json==0.5.9
     # via flytekit
 decorator==5.1.1
@@ -224,7 +223,11 @@ jaraco-classes==3.3.0
     #   keyrings-alt
 jedi==0.19.1
     # via ipython
-jinja2==3.1.2
+jeepney==0.8.0
+    # via
+    #   keyring
+    #   secretstorage
+jinja2==3.1.3
     # via cookiecutter
 jmespath==1.0.1
     # via botocore
@@ -297,7 +300,7 @@ nodeenv==1.8.0
     # via pre-commit
 numpy==1.23.5
     # via
-    #   flytekit
+    #   -r dev-requirements.in
     #   h5py
     #   keras-preprocessing
     #   opt-einsum
@@ -321,7 +324,7 @@ packaging==23.2
     #   pytest
     #   setuptools-scm
 pandas==1.5.3
-    # via flytekit
+    # via -r dev-requirements.in
 parso==0.8.3
     # via jedi
 pexpect==4.8.0
@@ -411,6 +414,8 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.7
     # via flytekit
+python-magic==0.4.27 ; platform_system == "Darwin" or platform_system == "Linux"
+    # via -r dev-requirements.in
 python-slugify==8.0.1
     # via cookiecutter
 pytimeparse==1.1.8
@@ -459,6 +464,8 @@ scikit-learn==1.3.2
     # via -r dev-requirements.in
 scipy==1.11.3
     # via scikit-learn
+secretstorage==3.3.3
+    # via keyring
 setuptools-scm==8.0.4
     # via -r dev-requirements.in
 six==1.16.0
@@ -473,8 +480,7 @@ six==1.16.0
     #   python-dateutil
     #   tensorflow
 sortedcontainers==2.4.0
-    # via
-    #   hypothesis
+    # via hypothesis
 stack-data==0.6.3
     # via ipython
 statsd==3.3.0
@@ -485,7 +491,7 @@ tensorboard-data-server==0.6.1
     # via tensorboard
 tensorboard-plugin-wit==1.8.1
     # via tensorboard
-tensorflow==2.8.1 ; python_version < "3.11" and (platform_machine != "arm64" or platform_system != "Darwin")
+tensorflow==2.8.1 ; python_version < "3.12"
     # via -r dev-requirements.in
 tensorflow-estimator==2.8.0
     # via tensorflow
@@ -503,8 +509,8 @@ tomli==2.0.1
     #   coverage
     #   mypy
     #   pytest
-    # . setuptools-scm
-torch==1.12.1 ; python_version >= "3.11" or platform_system != "Windows"
+    #   setuptools-scm
+torch==1.12.1 ; python_version < "3.12"
     # via -r dev-requirements.in
 traitlets==5.13.0
     # via
@@ -530,7 +536,7 @@ typing-extensions==4.8.0
     #   mashumaro
     #   mypy
     #   rich-click
-    # . setuptools-scm
+    #   setuptools-scm
     #   tensorflow
     #   torch
     #   typing-inspect

doc-requirements.in

@@ -11,6 +11,7 @@ sphinx-autoapi
 sphinx-copybutton
 sphinx_fontawesome
 sphinx-panels
+sphinx-reredirects
 sphinxcontrib-youtube==1.2.0
 cryptography
 google-api-core[grpc]
@@ -19,37 +20,8 @@ sphinx-tags
 sphinx-click
 retry
 mashumaro
+pydantic<2.0.0
 
 # Packages for Plugin docs
 # Package name              Plugin needing it
-botocore                    # fsspec
-fsspec                      # fsspec
-google-cloud                # bigquery
-google-cloud-bigquery       # bigquery
-markdown                    # deck
-plotly                      # deck
-pandas_profiling            # deck
-dolt_integrations           # dolt
-great-expectations          # greatexpectations
-datasets                    # huggingface
-kubernetes                  # k8s-pod
-modin                       # modin
-pandera                     # pandera
-papermill                   # papermill
-jupyter                     # papermill
-polars                      # polars
-pyspark                     # spark
-sqlalchemy                  # sqlalchemy
-torch                       # pytorch
-skl2onnx                    # onnxscikitlearn
-tf2onnx                     # onnxtensorflow
-tensorflow                  # onnxtensorflow
-whylogs==1.3.3              # whylogs
-whylabs-client              # whylogs
-ray==2.6.3                  # ray
-scikit-learn                # scikit-learn
-dask[distributed]           # dask
 vaex                        # vaex
-mlflow==2.7.0               # mlflow
-duckdb                      # duckdb
-snowflake-connector-python  # snowflake