Release 0.2.7 (#1126)

RunLLM · Mar 23, 2023 · a11945f · a11945f
1 parent 15a45fa
commit a11945f
Show file tree

Hide file tree

Showing 33 changed files with 243 additions and 161 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,53 @@
 # Changelog
 
+## 0.2.7
+Released on March 22, 2023.
+
+### Key Features
+* [Beta] Aqueduct now has support for on-demand Kubernetes cluster creation and
+    management on AWS. From the Aqueduct UI, you can connect Aqueduct to your
+    AWS account via the cloud integration feature. Once connected, you can use
+    this cloud integration to ask Aqueduct to automatically create a Kubernetes
+    cluster for you. See the documentation
+    [here](https://docs.aqueducthq.com/integrations/on-demand-resources/on-demand-aws-eks-clusters)
+    for how to create an operator that uses on-demand Kubernetes.
+
+### Enhancements
+* Improves error handling to return more detailed error messages from errors
+    occuring during execution.
+* Improves error handling by surfacing errors that occur outside of the
+    execution of an indvidual function as workflow-level errors; these errors
+    could occur for example if a compute system was misconfigured.
+* Improves handling of artifact name conflicts in the Python SDK. Explicitly
+    named artifacts (using either the `outputs` argument to the `@op` decorator
+    or the `.set_name()` function) will immediately flag and prevent conflicts
+    in artifact names. Automatically named artifacts will error if multiple
+    artifacts with the same name are included in a single `publish_flow` call.
+* Displays all compute engines associated with a workflow on the workflows list
+    page.
+* Improves efficiency when previewing large objects on the UI by retrieving a
+    sample of the data instead of the full data object and noting that the
+    displayed data is a sample.
+
+### Bugfixes
+* Fixes bug where an S3 or GCS bucket being used as the Aqueduct artifact store
+    could possibly be deleted from the UI.
+* Fixes bug that caused navigation buttons to be misaligned with other buttons
+    on the action bar on the workflow details page.
+* Fixes bug where navigating to the next most recent run on the workflow
+    details page would not work correctly.
+* Fixes bug pending or errored metric would show as "Unknown" on the UI instead
+    of with the correct status.
+* Fixes bug where warning-level checks were being shown as failures on the
+    workflows list page.
+* Fixes bug where certain DAG layouts would continue to show a layout with
+    overlapping and crossing edges.
+
+### Note
+* The parameterization of SQL queries may have unexpected behavior if you accidentally define a 
+    parameter with the same name twice. The parameter value will be chosen at random in such a case.
+    This bug will be fixed in the next release.
+
 ## 0.2.6
 
 Released on March 14, 2023.

diff --git a/sdk/setup.py b/sdk/setup.py
@@ -18,7 +18,7 @@
 
 setuptools.setup(
     name="aqueduct-sdk",
-    version="0.2.6",
+    version="0.2.7",
     author="Aqueduct, Inc.",
     author_email="[email protected]",
     description="Python SDK for the Aqueduct prediction infrastructure",

diff --git a/src/dockerfiles/Makefile b/src/dockerfiles/Makefile
@@ -1,5 +1,5 @@
 .DEFAULT_GOAL = all
-VERSION = 0.2.6
+VERSION = 0.2.7
 
 # By default, turn off the Makefile practice of printing each command before
 # you run it.

diff --git a/src/dockerfiles/connectors/athena.dockerfile b/src/dockerfiles/connectors/athena.dockerfile
@@ -1,4 +1,4 @@
-FROM aqueducthq/base_connector:0.2.6
+FROM aqueducthq/base_connector:0.2.7
 
 MAINTAINER Aqueduct <[email protected]> version: 0.1
 

diff --git a/src/dockerfiles/connectors/bigquery.dockerfile b/src/dockerfiles/connectors/bigquery.dockerfile
@@ -1,4 +1,4 @@
-FROM aqueducthq/base_connector:0.2.6
+FROM aqueducthq/base_connector:0.2.7
 
 MAINTAINER Aqueduct <[email protected]> version: 0.1
 

diff --git a/src/dockerfiles/connectors/mysql.dockerfile b/src/dockerfiles/connectors/mysql.dockerfile
@@ -1,4 +1,4 @@
-FROM aqueducthq/base_connector:0.2.6
+FROM aqueducthq/base_connector:0.2.7
 
 MAINTAINER Aqueduct <[email protected]> version: 0.1
 

diff --git a/src/dockerfiles/connectors/postgres.dockerfile b/src/dockerfiles/connectors/postgres.dockerfile
@@ -1,4 +1,4 @@
-FROM aqueducthq/base_connector:0.2.6
+FROM aqueducthq/base_connector:0.2.7
 
 MAINTAINER Aqueduct <[email protected]> version: 0.1
 

diff --git a/src/dockerfiles/connectors/s3.dockerfile b/src/dockerfiles/connectors/s3.dockerfile
@@ -1,4 +1,4 @@
-FROM aqueducthq/base_connector:0.2.6
+FROM aqueducthq/base_connector:0.2.7
 
 MAINTAINER Aqueduct <[email protected]> version: 0.1
 

diff --git a/src/dockerfiles/connectors/snowflake.dockerfile b/src/dockerfiles/connectors/snowflake.dockerfile
@@ -1,4 +1,4 @@
-FROM aqueducthq/base_connector:0.2.6
+FROM aqueducthq/base_connector:0.2.7
 
 MAINTAINER Aqueduct <[email protected]> version: 0.1
 

diff --git a/src/dockerfiles/connectors/sqlserver.dockerfile b/src/dockerfiles/connectors/sqlserver.dockerfile
@@ -1,4 +1,4 @@
-FROM aqueducthq/base_connector:0.2.6
+FROM aqueducthq/base_connector:0.2.7
 
 MAINTAINER Aqueduct <[email protected]> version: 0.1
 

diff --git a/src/dockerfiles/function/function310.dockerfile b/src/dockerfiles/function/function310.dockerfile
@@ -16,7 +16,7 @@ pyarrow==7.0.0 \
 boto3==1.18.0 \
 pydantic==1.9.0 \
 scikit_learn==1.0.2 \
-aqueduct-ml==0.2.6
+aqueduct-ml==0.2.7
 
 ENV PYTHONUNBUFFERED 1
 

diff --git a/src/dockerfiles/function/function37.dockerfile b/src/dockerfiles/function/function37.dockerfile
@@ -16,7 +16,7 @@ boto3==1.18.0 \
 pydantic==1.9.0 \
 scikit_learn==1.0.2 \
 typing_extensions==4.3.0 \
-aqueduct-ml==0.2.6
+aqueduct-ml==0.2.7
 
 
 ENV PYTHONUNBUFFERED 1

diff --git a/src/dockerfiles/function/function38.dockerfile b/src/dockerfiles/function/function38.dockerfile
@@ -16,7 +16,7 @@ pyarrow==7.0.0 \
 boto3==1.18.0 \
 pydantic==1.9.0 \
 scikit_learn==1.0.2 \
-aqueduct-ml==0.2.6
+aqueduct-ml==0.2.7
 
 ENV PYTHONUNBUFFERED 1
 

diff --git a/src/dockerfiles/function/function39.dockerfile b/src/dockerfiles/function/function39.dockerfile
@@ -16,7 +16,7 @@ pyarrow==7.0.0 \
 boto3==1.18.0 \
 pydantic==1.9.0 \
 scikit_learn==1.0.2 \
-aqueduct-ml==0.2.6
+aqueduct-ml==0.2.7
 
 ENV PYTHONUNBUFFERED 1
 

diff --git a/src/dockerfiles/gpu/py310_env.yml b/src/dockerfiles/gpu/py310_env.yml
@@ -14,4 +14,4 @@ dependencies:
   - pip:
     - scikit_learn==1.0.2
     - typing_extensions==4.3.0
-    - aqueduct-ml==0.2.6
+    - aqueduct-ml==0.2.7
diff --git a/src/dockerfiles/gpu/py37_env.yml b/src/dockerfiles/gpu/py37_env.yml
@@ -14,4 +14,4 @@ dependencies:
   - pip:
     - scikit_learn==1.0.2
     - typing_extensions==4.3.0
-    - aqueduct-ml==0.2.6
+    - aqueduct-ml==0.2.7
diff --git a/src/dockerfiles/gpu/py38_env.yml b/src/dockerfiles/gpu/py38_env.yml
@@ -14,4 +14,4 @@ dependencies:
   - pip:
     - scikit_learn==1.0.2
     - typing_extensions==4.3.0
-    - aqueduct-ml==0.2.6
+    - aqueduct-ml==0.2.7
diff --git a/src/dockerfiles/gpu/py39_env.yml b/src/dockerfiles/gpu/py39_env.yml
@@ -14,4 +14,4 @@ dependencies:
   - pip:
     - scikit_learn==1.0.2
     - typing_extensions==4.3.0
-    - aqueduct-ml==0.2.6
+    - aqueduct-ml==0.2.7
diff --git a/src/dockerfiles/lambda/function/requirements-37.txt b/src/dockerfiles/lambda/function/requirements-37.txt
@@ -7,4 +7,4 @@ boto3==1.18.0
 pydantic==1.9.0
 scikit_learn==1.0.2
 typing_extensions==4.3.0
-aqueduct-ml==0.2.6
+aqueduct-ml==0.2.7
diff --git a/src/dockerfiles/lambda/function/requirements.txt b/src/dockerfiles/lambda/function/requirements.txt
@@ -6,4 +6,4 @@ pyarrow==7.0.0
 boto3==1.18.0
 pydantic==1.9.0
 scikit_learn==1.0.2
-aqueduct-ml==0.2.6
+aqueduct-ml==0.2.7
diff --git a/src/dockerfiles/lambda/requirements.txt b/src/dockerfiles/lambda/requirements.txt
@@ -6,4 +6,4 @@ pydantic==1.9.0
 pyyaml
 SQLAlchemy==1.4.30
 typing_extensions==4.3.0
-aqueduct-ml==0.2.6
+aqueduct-ml==0.2.7
diff --git a/src/golang/cmd/server/handler/connect_integration.go b/src/golang/cmd/server/handler/connect_integration.go
@@ -30,6 +30,7 @@ import (
 	"github.com/aqueducthq/aqueduct/lib/workflow/operator/connector/auth"
 	"github.com/aqueducthq/aqueduct/lib/workflow/utils"
 	"github.com/google/uuid"
+	"github.com/hashicorp/go-version"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -792,10 +793,21 @@ func ValidatePrerequisites(
 			return http.StatusNotFound, errors.Wrap(err, "terraform executable not found. Please go to https://developer.hashicorp.com/terraform/downloads to install terraform")
 		}
 
-		if _, _, err := lib_utils.RunCmd("aws", []string{"--version"}, "", false); err != nil {
+		awsVersionString, _, err := lib_utils.RunCmd("aws", []string{"--version"}, "", false)
+		if err != nil {
 			return http.StatusNotFound, errors.Wrap(err, "AWS CLI executable not found. Please go to https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html to install AWS CLI")
 		}
 
+		awsVersion, err := version.NewVersion(strings.Split(strings.Split(awsVersionString, " ")[0], "/")[1])
+		if err != nil {
+			return http.StatusUnprocessableEntity, errors.Wrap(err, "Error parsing AWS CLI version")
+		}
+
+		requiredVersion, _ := version.NewVersion("2.11.5")
+		if awsVersion.LessThan(requiredVersion) {
+			return http.StatusUnprocessableEntity, errors.Wrapf(err, "AWS CLI version 2.11.5 and above is required, but you got %s. Please update!", awsVersion.String())
+		}
+
 		if _, _, err := lib_utils.RunCmd("env", []string{"--version"}, "", false); err != nil {
 			return http.StatusNotFound, errors.Wrap(err, "env (GNU coreutils) executable not found.")
 		}

diff --git a/src/golang/go.mod b/src/golang/go.mod
@@ -30,37 +30,35 @@ require (
 	google.golang.org/grpc v1.50.1
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/yaml.v2 v2.4.0
-	k8s.io/api v0.21.3
-	k8s.io/apimachinery v0.21.3
-	k8s.io/client-go v0.21.3
+	k8s.io/api v0.26.3
+	k8s.io/apimachinery v0.26.3
+	k8s.io/client-go v0.26.3
 )
 
 require (
 	cloud.google.com/go v0.105.0 // indirect
 	cloud.google.com/go/compute v1.12.1 // indirect
 	cloud.google.com/go/compute/metadata v0.2.1 // indirect
 	cloud.google.com/go/iam v0.6.0 // indirect
-	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
-	github.com/Azure/go-autorest/autorest v0.11.12 // indirect
-	github.com/Azure/go-autorest/autorest/adal v0.9.5 // indirect
-	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
-	github.com/Azure/go-autorest/logger v0.2.0 // indirect
-	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect
-	github.com/go-logr/logr v0.4.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/jsonreference v0.20.0 // indirect
+	github.com/go-openapi/swag v0.19.14 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/gnostic v0.5.7-v3refs // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
 	github.com/googleapis/gax-go/v2 v2.7.0 // indirect
-	github.com/googleapis/gnostic v0.4.1 // indirect
 	github.com/gorilla/websocket v1.4.2 // indirect
-	github.com/imdario/mergo v0.3.5 // indirect
+	github.com/hashicorp/go-version v1.6.0 // indirect
+	github.com/imdario/mergo v0.3.6 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
 	github.com/jackc/pgconn v1.10.0 // indirect
 	github.com/jackc/pgio v1.0.0 // indirect
@@ -69,9 +67,12 @@ require (
 	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
 	github.com/jackc/pgtype v1.8.1 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/json-iterator/go v1.1.10 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mailru/easyjson v0.7.6 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
@@ -81,20 +82,22 @@ require (
 	go.opencensus.io v0.24.0 // indirect
 	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
-	golang.org/x/net v0.0.0-20221014081412-f15817d10f9b // indirect
-	golang.org/x/sys v0.2.0 // indirect
-	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
-	golang.org/x/text v0.4.0 // indirect
-	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	golang.org/x/net v0.7.0 // indirect
+	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/term v0.5.0 // indirect
+	golang.org/x/text v0.7.0 // indirect
+	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/klog/v2 v2.8.0 // indirect
-	k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
-	sigs.k8s.io/yaml v1.2.0 // indirect
+	k8s.io/klog/v2 v2.80.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
+	k8s.io/utils v0.0.0-20221107191617-1a15be271d1d // indirect
+	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
 )