From ee109d053904bf5efe688c4b5fbaf7fca36d3a46 Mon Sep 17 00:00:00 2001 From: Ryanne Dolan Date: Tue, 14 Jan 2025 12:41:28 -0600 Subject: [PATCH 1/3] Add Engine CRD --- generate-models.sh | 1 + .../k8s/models/V1alpha1Database.java | 2 +- .../k8s/models/V1alpha1DatabaseList.java | 2 +- .../k8s/models/V1alpha1DatabaseSpec.java | 2 +- .../hoptimator/k8s/models/V1alpha1Engine.java | 218 ++++++++++++++++ .../k8s/models/V1alpha1EngineList.java | 195 ++++++++++++++ .../k8s/models/V1alpha1EngineSpec.java | 245 ++++++++++++++++++ .../k8s/models/V1alpha1JobTemplate.java | 2 +- .../k8s/models/V1alpha1JobTemplateList.java | 2 +- .../k8s/models/V1alpha1JobTemplateSpec.java | 2 +- .../k8s/models/V1alpha1Pipeline.java | 2 +- .../k8s/models/V1alpha1PipelineList.java | 2 +- .../k8s/models/V1alpha1PipelineSpec.java | 2 +- .../k8s/models/V1alpha1PipelineStatus.java | 2 +- .../k8s/models/V1alpha1Subscription.java | 2 +- .../k8s/models/V1alpha1SubscriptionList.java | 2 +- .../k8s/models/V1alpha1SubscriptionSpec.java | 2 +- .../models/V1alpha1SubscriptionStatus.java | 2 +- .../k8s/models/V1alpha1TableTemplate.java | 2 +- .../k8s/models/V1alpha1TableTemplateList.java | 2 +- .../k8s/models/V1alpha1TableTemplateSpec.java | 2 +- .../hoptimator/k8s/models/V1alpha1View.java | 2 +- .../k8s/models/V1alpha1ViewList.java | 2 +- .../k8s/models/V1alpha1ViewSpec.java | 2 +- .../src/main/resources/engines.crd.yaml | 64 +++++ 25 files changed, 743 insertions(+), 20 deletions(-) create mode 100644 hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Engine.java create mode 100644 hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1EngineList.java create mode 100644 hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1EngineSpec.java create mode 100644 hoptimator-k8s/src/main/resources/engines.crd.yaml diff --git a/generate-models.sh b/generate-models.sh index 5f7a6ff..16c7152 100755 --- a/generate-models.sh +++ b/generate-models.sh @@ -16,4 +16,5 @@ docker run \ -u "$(pwd)/hoptimator-k8s/src/main/resources/tabletemplates.crd.yaml" \ -u "$(pwd)/hoptimator-k8s/src/main/resources/views.crd.yaml" \ -u "$(pwd)/hoptimator-k8s/src/main/resources/subscriptions.crd.yaml" \ + -u "$(pwd)/hoptimator-k8s/src/main/resources/engines.crd.yaml" \ && echo "done." diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Database.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Database.java index e3360a2..6fc51eb 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Database.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Database.java @@ -30,7 +30,7 @@ * Database metadata. */ @ApiModel(description = "Database metadata.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2024-12-09T16:52:51.758Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") public class V1alpha1Database implements io.kubernetes.client.common.KubernetesObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1DatabaseList.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1DatabaseList.java index 1ca270e..4056015 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1DatabaseList.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1DatabaseList.java @@ -32,7 +32,7 @@ * DatabaseList is a list of Database */ @ApiModel(description = "DatabaseList is a list of Database") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2024-12-09T16:52:51.758Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") public class V1alpha1DatabaseList implements io.kubernetes.client.common.KubernetesListObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1DatabaseSpec.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1DatabaseSpec.java index de71984..788aff5 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1DatabaseSpec.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1DatabaseSpec.java @@ -28,7 +28,7 @@ * Database spec. */ @ApiModel(description = "Database spec.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2024-12-09T16:52:51.758Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") public class V1alpha1DatabaseSpec { /** * SQL dialect the driver expects. diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Engine.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Engine.java new file mode 100644 index 0000000..2e5c679 --- /dev/null +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Engine.java @@ -0,0 +1,218 @@ +/* + * Kubernetes + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: v1.21.1 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +package com.linkedin.hoptimator.k8s.models; + +import java.util.Objects; +import java.util.Arrays; +import com.google.gson.TypeAdapter; +import com.google.gson.annotations.JsonAdapter; +import com.google.gson.annotations.SerializedName; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; +import com.linkedin.hoptimator.k8s.models.V1alpha1EngineSpec; +import io.kubernetes.client.openapi.models.V1ObjectMeta; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import java.io.IOException; + +/** + * Engine metadata. + */ +@ApiModel(description = "Engine metadata.") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +public class V1alpha1Engine implements io.kubernetes.client.common.KubernetesObject { + public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; + @SerializedName(SERIALIZED_NAME_API_VERSION) + private String apiVersion; + + public static final String SERIALIZED_NAME_KIND = "kind"; + @SerializedName(SERIALIZED_NAME_KIND) + private String kind; + + public static final String SERIALIZED_NAME_METADATA = "metadata"; + @SerializedName(SERIALIZED_NAME_METADATA) + private V1ObjectMeta metadata = null; + + public static final String SERIALIZED_NAME_SPEC = "spec"; + @SerializedName(SERIALIZED_NAME_SPEC) + private V1alpha1EngineSpec spec; + + public static final String SERIALIZED_NAME_STATUS = "status"; + @SerializedName(SERIALIZED_NAME_STATUS) + private Object status; + + + public V1alpha1Engine apiVersion(String apiVersion) { + + this.apiVersion = apiVersion; + return this; + } + + /** + * APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + * @return apiVersion + **/ + @javax.annotation.Nullable + @ApiModelProperty(value = "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources") + + public String getApiVersion() { + return apiVersion; + } + + + public void setApiVersion(String apiVersion) { + this.apiVersion = apiVersion; + } + + + public V1alpha1Engine kind(String kind) { + + this.kind = kind; + return this; + } + + /** + * Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + * @return kind + **/ + @javax.annotation.Nullable + @ApiModelProperty(value = "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds") + + public String getKind() { + return kind; + } + + + public void setKind(String kind) { + this.kind = kind; + } + + + public V1alpha1Engine metadata(V1ObjectMeta metadata) { + + this.metadata = metadata; + return this; + } + + /** + * Get metadata + * @return metadata + **/ + @javax.annotation.Nullable + @ApiModelProperty(value = "") + + public V1ObjectMeta getMetadata() { + return metadata; + } + + + public void setMetadata(V1ObjectMeta metadata) { + this.metadata = metadata; + } + + + public V1alpha1Engine spec(V1alpha1EngineSpec spec) { + + this.spec = spec; + return this; + } + + /** + * Get spec + * @return spec + **/ + @javax.annotation.Nullable + @ApiModelProperty(value = "") + + public V1alpha1EngineSpec getSpec() { + return spec; + } + + + public void setSpec(V1alpha1EngineSpec spec) { + this.spec = spec; + } + + + public V1alpha1Engine status(Object status) { + + this.status = status; + return this; + } + + /** + * Get status + * @return status + **/ + @javax.annotation.Nullable + @ApiModelProperty(value = "") + + public Object getStatus() { + return status; + } + + + public void setStatus(Object status) { + this.status = status; + } + + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + V1alpha1Engine v1alpha1Engine = (V1alpha1Engine) o; + return Objects.equals(this.apiVersion, v1alpha1Engine.apiVersion) && + Objects.equals(this.kind, v1alpha1Engine.kind) && + Objects.equals(this.metadata, v1alpha1Engine.metadata) && + Objects.equals(this.spec, v1alpha1Engine.spec) && + Objects.equals(this.status, v1alpha1Engine.status); + } + + @Override + public int hashCode() { + return Objects.hash(apiVersion, kind, metadata, spec, status); + } + + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class V1alpha1Engine {\n"); + sb.append(" apiVersion: ").append(toIndentedString(apiVersion)).append("\n"); + sb.append(" kind: ").append(toIndentedString(kind)).append("\n"); + sb.append(" metadata: ").append(toIndentedString(metadata)).append("\n"); + sb.append(" spec: ").append(toIndentedString(spec)).append("\n"); + sb.append(" status: ").append(toIndentedString(status)).append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } + +} + diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1EngineList.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1EngineList.java new file mode 100644 index 0000000..d573c70 --- /dev/null +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1EngineList.java @@ -0,0 +1,195 @@ +/* + * Kubernetes + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: v1.21.1 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +package com.linkedin.hoptimator.k8s.models; + +import java.util.Objects; +import java.util.Arrays; +import com.google.gson.TypeAdapter; +import com.google.gson.annotations.JsonAdapter; +import com.google.gson.annotations.SerializedName; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; +import com.linkedin.hoptimator.k8s.models.V1alpha1Engine; +import io.kubernetes.client.openapi.models.V1ListMeta; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * EngineList is a list of Engine + */ +@ApiModel(description = "EngineList is a list of Engine") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +public class V1alpha1EngineList implements io.kubernetes.client.common.KubernetesListObject { + public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; + @SerializedName(SERIALIZED_NAME_API_VERSION) + private String apiVersion; + + public static final String SERIALIZED_NAME_ITEMS = "items"; + @SerializedName(SERIALIZED_NAME_ITEMS) + private List items = new ArrayList<>(); + + public static final String SERIALIZED_NAME_KIND = "kind"; + @SerializedName(SERIALIZED_NAME_KIND) + private String kind; + + public static final String SERIALIZED_NAME_METADATA = "metadata"; + @SerializedName(SERIALIZED_NAME_METADATA) + private V1ListMeta metadata = null; + + + public V1alpha1EngineList apiVersion(String apiVersion) { + + this.apiVersion = apiVersion; + return this; + } + + /** + * APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + * @return apiVersion + **/ + @javax.annotation.Nullable + @ApiModelProperty(value = "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources") + + public String getApiVersion() { + return apiVersion; + } + + + public void setApiVersion(String apiVersion) { + this.apiVersion = apiVersion; + } + + + public V1alpha1EngineList items(List items) { + + this.items = items; + return this; + } + + public V1alpha1EngineList addItemsItem(V1alpha1Engine itemsItem) { + this.items.add(itemsItem); + return this; + } + + /** + * List of engines. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md + * @return items + **/ + @ApiModelProperty(required = true, value = "List of engines. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md") + + public List getItems() { + return items; + } + + + public void setItems(List items) { + this.items = items; + } + + + public V1alpha1EngineList kind(String kind) { + + this.kind = kind; + return this; + } + + /** + * Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + * @return kind + **/ + @javax.annotation.Nullable + @ApiModelProperty(value = "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds") + + public String getKind() { + return kind; + } + + + public void setKind(String kind) { + this.kind = kind; + } + + + public V1alpha1EngineList metadata(V1ListMeta metadata) { + + this.metadata = metadata; + return this; + } + + /** + * Get metadata + * @return metadata + **/ + @javax.annotation.Nullable + @ApiModelProperty(value = "") + + public V1ListMeta getMetadata() { + return metadata; + } + + + public void setMetadata(V1ListMeta metadata) { + this.metadata = metadata; + } + + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + V1alpha1EngineList v1alpha1EngineList = (V1alpha1EngineList) o; + return Objects.equals(this.apiVersion, v1alpha1EngineList.apiVersion) && + Objects.equals(this.items, v1alpha1EngineList.items) && + Objects.equals(this.kind, v1alpha1EngineList.kind) && + Objects.equals(this.metadata, v1alpha1EngineList.metadata); + } + + @Override + public int hashCode() { + return Objects.hash(apiVersion, items, kind, metadata); + } + + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class V1alpha1EngineList {\n"); + sb.append(" apiVersion: ").append(toIndentedString(apiVersion)).append("\n"); + sb.append(" items: ").append(toIndentedString(items)).append("\n"); + sb.append(" kind: ").append(toIndentedString(kind)).append("\n"); + sb.append(" metadata: ").append(toIndentedString(metadata)).append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } + +} + diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1EngineSpec.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1EngineSpec.java new file mode 100644 index 0000000..bbf77eb --- /dev/null +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1EngineSpec.java @@ -0,0 +1,245 @@ +/* + * Kubernetes + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: v1.21.1 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +package com.linkedin.hoptimator.k8s.models; + +import java.util.Objects; +import java.util.Arrays; +import com.google.gson.TypeAdapter; +import com.google.gson.annotations.JsonAdapter; +import com.google.gson.annotations.SerializedName; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Engine spec. + */ +@ApiModel(description = "Engine spec.") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +public class V1alpha1EngineSpec { + public static final String SERIALIZED_NAME_DATABASES = "databases"; + @SerializedName(SERIALIZED_NAME_DATABASES) + private List databases = null; + + /** + * SQL dialect the driver expects. + */ + @JsonAdapter(DialectEnum.Adapter.class) + public enum DialectEnum { + ANSI("ANSI"), + + MYSQL("MySQL"), + + CALCITE("Calcite"); + + private String value; + + DialectEnum(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + public static DialectEnum fromValue(String value) { + for (DialectEnum b : DialectEnum.values()) { + if (b.value.equals(value)) { + return b; + } + } + throw new IllegalArgumentException("Unexpected value '" + value + "'"); + } + + public static class Adapter extends TypeAdapter { + @Override + public void write(final JsonWriter jsonWriter, final DialectEnum enumeration) throws IOException { + jsonWriter.value(enumeration.getValue()); + } + + @Override + public DialectEnum read(final JsonReader jsonReader) throws IOException { + String value = jsonReader.nextString(); + return DialectEnum.fromValue(value); + } + } + } + + public static final String SERIALIZED_NAME_DIALECT = "dialect"; + @SerializedName(SERIALIZED_NAME_DIALECT) + private DialectEnum dialect; + + public static final String SERIALIZED_NAME_DRIVER = "driver"; + @SerializedName(SERIALIZED_NAME_DRIVER) + private String driver; + + public static final String SERIALIZED_NAME_URL = "url"; + @SerializedName(SERIALIZED_NAME_URL) + private String url; + + + public V1alpha1EngineSpec databases(List databases) { + + this.databases = databases; + return this; + } + + public V1alpha1EngineSpec addDatabasesItem(String databasesItem) { + if (this.databases == null) { + this.databases = new ArrayList<>(); + } + this.databases.add(databasesItem); + return this; + } + + /** + * Databases this engine supports. If null, supports everything. + * @return databases + **/ + @javax.annotation.Nullable + @ApiModelProperty(value = "Databases this engine supports. If null, supports everything.") + + public List getDatabases() { + return databases; + } + + + public void setDatabases(List databases) { + this.databases = databases; + } + + + public V1alpha1EngineSpec dialect(DialectEnum dialect) { + + this.dialect = dialect; + return this; + } + + /** + * SQL dialect the driver expects. + * @return dialect + **/ + @javax.annotation.Nullable + @ApiModelProperty(value = "SQL dialect the driver expects.") + + public DialectEnum getDialect() { + return dialect; + } + + + public void setDialect(DialectEnum dialect) { + this.dialect = dialect; + } + + + public V1alpha1EngineSpec driver(String driver) { + + this.driver = driver; + return this; + } + + /** + * Fully qualified class name of JDBD driver. + * @return driver + **/ + @javax.annotation.Nullable + @ApiModelProperty(value = "Fully qualified class name of JDBD driver.") + + public String getDriver() { + return driver; + } + + + public void setDriver(String driver) { + this.driver = driver; + } + + + public V1alpha1EngineSpec url(String url) { + + this.url = url; + return this; + } + + /** + * JDBC connection URL + * @return url + **/ + @ApiModelProperty(required = true, value = "JDBC connection URL") + + public String getUrl() { + return url; + } + + + public void setUrl(String url) { + this.url = url; + } + + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + V1alpha1EngineSpec v1alpha1EngineSpec = (V1alpha1EngineSpec) o; + return Objects.equals(this.databases, v1alpha1EngineSpec.databases) && + Objects.equals(this.dialect, v1alpha1EngineSpec.dialect) && + Objects.equals(this.driver, v1alpha1EngineSpec.driver) && + Objects.equals(this.url, v1alpha1EngineSpec.url); + } + + @Override + public int hashCode() { + return Objects.hash(databases, dialect, driver, url); + } + + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class V1alpha1EngineSpec {\n"); + sb.append(" databases: ").append(toIndentedString(databases)).append("\n"); + sb.append(" dialect: ").append(toIndentedString(dialect)).append("\n"); + sb.append(" driver: ").append(toIndentedString(driver)).append("\n"); + sb.append(" url: ").append(toIndentedString(url)).append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } + +} + diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1JobTemplate.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1JobTemplate.java index 1ec14d1..06285be 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1JobTemplate.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1JobTemplate.java @@ -30,7 +30,7 @@ * Template to apply to matching jobs. */ @ApiModel(description = "Template to apply to matching jobs.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2024-12-09T16:52:51.758Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") public class V1alpha1JobTemplate implements io.kubernetes.client.common.KubernetesObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1JobTemplateList.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1JobTemplateList.java index c77dd80..6dee379 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1JobTemplateList.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1JobTemplateList.java @@ -32,7 +32,7 @@ * JobTemplateList is a list of JobTemplate */ @ApiModel(description = "JobTemplateList is a list of JobTemplate") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2024-12-09T16:52:51.758Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") public class V1alpha1JobTemplateList implements io.kubernetes.client.common.KubernetesListObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1JobTemplateSpec.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1JobTemplateSpec.java index 8a37e81..62f7f26 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1JobTemplateSpec.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1JobTemplateSpec.java @@ -30,7 +30,7 @@ * TableTemplate spec. */ @ApiModel(description = "TableTemplate spec.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2024-12-09T16:52:51.758Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") public class V1alpha1JobTemplateSpec { public static final String SERIALIZED_NAME_DATABASES = "databases"; @SerializedName(SERIALIZED_NAME_DATABASES) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Pipeline.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Pipeline.java index b2cdc19..c9d18d6 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Pipeline.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Pipeline.java @@ -31,7 +31,7 @@ * A set of objects that work together to deliver data. */ @ApiModel(description = "A set of objects that work together to deliver data.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2024-12-09T16:52:51.758Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") public class V1alpha1Pipeline implements io.kubernetes.client.common.KubernetesObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1PipelineList.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1PipelineList.java index adcce65..92259d4 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1PipelineList.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1PipelineList.java @@ -32,7 +32,7 @@ * PipelineList is a list of Pipeline */ @ApiModel(description = "PipelineList is a list of Pipeline") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2024-12-09T16:52:51.758Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") public class V1alpha1PipelineList implements io.kubernetes.client.common.KubernetesListObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1PipelineSpec.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1PipelineSpec.java index dc6949d..f9090d1 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1PipelineSpec.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1PipelineSpec.java @@ -28,7 +28,7 @@ * Pipeline spec. */ @ApiModel(description = "Pipeline spec.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2024-12-09T16:52:51.758Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") public class V1alpha1PipelineSpec { public static final String SERIALIZED_NAME_SQL = "sql"; @SerializedName(SERIALIZED_NAME_SQL) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1PipelineStatus.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1PipelineStatus.java index 3cd8a38..6a093d0 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1PipelineStatus.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1PipelineStatus.java @@ -28,7 +28,7 @@ * Pipeline status. */ @ApiModel(description = "Pipeline status.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2024-12-09T16:52:51.758Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") public class V1alpha1PipelineStatus { public static final String SERIALIZED_NAME_FAILED = "failed"; @SerializedName(SERIALIZED_NAME_FAILED) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Subscription.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Subscription.java index b6a39cb..197337d 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Subscription.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Subscription.java @@ -31,7 +31,7 @@ * Hoptimator Subscription */ @ApiModel(description = "Hoptimator Subscription") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2024-12-09T16:52:51.758Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") public class V1alpha1Subscription implements io.kubernetes.client.common.KubernetesObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SubscriptionList.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SubscriptionList.java index 9a7a137..966b4db 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SubscriptionList.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SubscriptionList.java @@ -32,7 +32,7 @@ * SubscriptionList is a list of Subscription */ @ApiModel(description = "SubscriptionList is a list of Subscription") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2024-12-09T16:52:51.758Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") public class V1alpha1SubscriptionList implements io.kubernetes.client.common.KubernetesListObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SubscriptionSpec.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SubscriptionSpec.java index 9944af2..fb107ea 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SubscriptionSpec.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SubscriptionSpec.java @@ -31,7 +31,7 @@ * Subscription spec */ @ApiModel(description = "Subscription spec") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2024-12-09T16:52:51.758Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") public class V1alpha1SubscriptionSpec { public static final String SERIALIZED_NAME_DATABASE = "database"; @SerializedName(SERIALIZED_NAME_DATABASE) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SubscriptionStatus.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SubscriptionStatus.java index d68afbc..8903258 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SubscriptionStatus.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SubscriptionStatus.java @@ -32,7 +32,7 @@ * Filled in by the operator. */ @ApiModel(description = "Filled in by the operator.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2024-12-09T16:52:51.758Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") public class V1alpha1SubscriptionStatus { public static final String SERIALIZED_NAME_ATTRIBUTES = "attributes"; @SerializedName(SERIALIZED_NAME_ATTRIBUTES) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1TableTemplate.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1TableTemplate.java index 02ccf68..dc0020a 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1TableTemplate.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1TableTemplate.java @@ -30,7 +30,7 @@ * Template to apply to matching tables. */ @ApiModel(description = "Template to apply to matching tables.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2024-12-09T16:52:51.758Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") public class V1alpha1TableTemplate implements io.kubernetes.client.common.KubernetesObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1TableTemplateList.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1TableTemplateList.java index 3381edd..49badac 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1TableTemplateList.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1TableTemplateList.java @@ -32,7 +32,7 @@ * TableTemplateList is a list of TableTemplate */ @ApiModel(description = "TableTemplateList is a list of TableTemplate") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2024-12-09T16:52:51.758Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") public class V1alpha1TableTemplateList implements io.kubernetes.client.common.KubernetesListObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1TableTemplateSpec.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1TableTemplateSpec.java index e787d31..68e221a 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1TableTemplateSpec.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1TableTemplateSpec.java @@ -30,7 +30,7 @@ * TableTemplate spec. */ @ApiModel(description = "TableTemplate spec.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2024-12-09T16:52:51.758Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") public class V1alpha1TableTemplateSpec { public static final String SERIALIZED_NAME_CONNECTOR = "connector"; @SerializedName(SERIALIZED_NAME_CONNECTOR) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1View.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1View.java index 6a7cb46..9909777 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1View.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1View.java @@ -30,7 +30,7 @@ * A SQL view. */ @ApiModel(description = "A SQL view.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2024-12-09T16:52:51.758Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") public class V1alpha1View implements io.kubernetes.client.common.KubernetesObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1ViewList.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1ViewList.java index c6afec7..a5b0f7a 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1ViewList.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1ViewList.java @@ -32,7 +32,7 @@ * ViewList is a list of View */ @ApiModel(description = "ViewList is a list of View") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2024-12-09T16:52:51.758Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") public class V1alpha1ViewList implements io.kubernetes.client.common.KubernetesListObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1ViewSpec.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1ViewSpec.java index b106cb1..5faaac0 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1ViewSpec.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1ViewSpec.java @@ -28,7 +28,7 @@ * View spec. */ @ApiModel(description = "View spec.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2024-12-09T16:52:51.758Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") public class V1alpha1ViewSpec { public static final String SERIALIZED_NAME_MATERIALIZED = "materialized"; @SerializedName(SERIALIZED_NAME_MATERIALIZED) diff --git a/hoptimator-k8s/src/main/resources/engines.crd.yaml b/hoptimator-k8s/src/main/resources/engines.crd.yaml new file mode 100644 index 0000000..136b2a9 --- /dev/null +++ b/hoptimator-k8s/src/main/resources/engines.crd.yaml @@ -0,0 +1,64 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: engines.hoptimator.linkedin.com +spec: + group: hoptimator.linkedin.com + names: + kind: Engine + listKind: EngineList + plural: engines + singular: engine + shortNames: + - eng + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + description: Engine metadata. + type: object + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + description: Engine spec. + type: object + properties: + url: + description: JDBC connection URL + type: string + dialect: + description: SQL dialect the driver expects. + type: string + enum: + - ANSI + - MySQL + - Calcite + driver: + description: Fully qualified class name of JDBD driver. + type: string + databases: + description: Databases this engine supports. If null, supports everything. + type: array + items: + type: string + required: + - url + status: + type: object + properties: + subresources: + status: {} + additionalPrinterColumns: + - name: URL + type: string + description: JDBC URL. + jsonPath: .spec.url From 0a81abf69730a178058ed44761a7ee2c03d75fdf Mon Sep 17 00:00:00 2001 From: Ryanne Dolan Date: Tue, 14 Jan 2025 17:57:54 -0600 Subject: [PATCH 2/3] WIP: Add support for external engines --- .../java/com/linkedin/hoptimator/Engine.java | 13 ++ .../hoptimator/k8s/K8sApiEndpoints.java | 5 + .../hoptimator/k8s/K8sDatabaseTable.java | 7 +- .../linkedin/hoptimator/k8s/K8sEngine.java | 40 +++++ .../hoptimator/k8s/K8sEngineTable.java | 91 ++++++++++ .../linkedin/hoptimator/k8s/K8sMetadata.java | 9 +- .../k8s/models/V1alpha1Database.java | 2 +- .../k8s/models/V1alpha1DatabaseList.java | 2 +- .../k8s/models/V1alpha1DatabaseSpec.java | 2 +- .../hoptimator/k8s/models/V1alpha1Engine.java | 2 +- .../k8s/models/V1alpha1EngineList.java | 2 +- .../k8s/models/V1alpha1EngineSpec.java | 6 +- .../k8s/models/V1alpha1JobTemplate.java | 2 +- .../k8s/models/V1alpha1JobTemplateList.java | 2 +- .../k8s/models/V1alpha1JobTemplateSpec.java | 2 +- .../k8s/models/V1alpha1Pipeline.java | 2 +- .../k8s/models/V1alpha1PipelineList.java | 2 +- .../k8s/models/V1alpha1PipelineSpec.java | 2 +- .../k8s/models/V1alpha1PipelineStatus.java | 2 +- .../k8s/models/V1alpha1Subscription.java | 2 +- .../k8s/models/V1alpha1SubscriptionList.java | 2 +- .../k8s/models/V1alpha1SubscriptionSpec.java | 2 +- .../models/V1alpha1SubscriptionStatus.java | 2 +- .../k8s/models/V1alpha1TableTemplate.java | 2 +- .../k8s/models/V1alpha1TableTemplateList.java | 2 +- .../k8s/models/V1alpha1TableTemplateSpec.java | 2 +- .../hoptimator/k8s/models/V1alpha1View.java | 2 +- .../k8s/models/V1alpha1ViewList.java | 2 +- .../k8s/models/V1alpha1ViewSpec.java | 2 +- .../src/main/resources/engines.crd.yaml | 3 +- .../hoptimator/util/HoptimatorJdbcSchema.java | 30 ++-- .../hoptimator/util/planner/EngineRules.java | 164 ++++++++++++++++++ .../planner/HoptimatorJdbcConvention.java | 14 +- 33 files changed, 383 insertions(+), 43 deletions(-) create mode 100644 hoptimator-api/src/main/java/com/linkedin/hoptimator/Engine.java create mode 100644 hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sEngine.java create mode 100644 hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sEngineTable.java create mode 100644 hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/EngineRules.java diff --git a/hoptimator-api/src/main/java/com/linkedin/hoptimator/Engine.java b/hoptimator-api/src/main/java/com/linkedin/hoptimator/Engine.java new file mode 100644 index 0000000..5b3091d --- /dev/null +++ b/hoptimator-api/src/main/java/com/linkedin/hoptimator/Engine.java @@ -0,0 +1,13 @@ +package com.linkedin.hoptimator; + +import javax.sql.DataSource; + +/** An execution engine. */ +public interface Engine { + + String engineName(); + + DataSource dataSource(); + + SqlDialect dialect(); +} diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sApiEndpoints.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sApiEndpoints.java index f2f3f46..02ed347 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sApiEndpoints.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sApiEndpoints.java @@ -9,6 +9,8 @@ import com.linkedin.hoptimator.k8s.models.V1alpha1Database; import com.linkedin.hoptimator.k8s.models.V1alpha1DatabaseList; +import com.linkedin.hoptimator.k8s.models.V1alpha1Engine; +import com.linkedin.hoptimator.k8s.models.V1alpha1EngineList; import com.linkedin.hoptimator.k8s.models.V1alpha1JobTemplate; import com.linkedin.hoptimator.k8s.models.V1alpha1JobTemplateList; import com.linkedin.hoptimator.k8s.models.V1alpha1Pipeline; @@ -34,6 +36,9 @@ public final class K8sApiEndpoints { public static final K8sApiEndpoint DATABASES = new K8sApiEndpoint<>("Database", "hoptimator.linkedin.com", "v1alpha1", "databases", false, V1alpha1Database.class, V1alpha1DatabaseList.class); + public static final K8sApiEndpoint ENGINES = + new K8sApiEndpoint<>("Engine", "hoptimator.linkedin.com", "v1alpha1", "engines", false, + V1alpha1Engine.class, V1alpha1EngineList.class); public static final K8sApiEndpoint PIPELINES = new K8sApiEndpoint<>("Pipeline", "hoptimator.linkedin.com", "v1alpha1", "pipelines", false, V1alpha1Pipeline.class, V1alpha1PipelineList.class); diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sDatabaseTable.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sDatabaseTable.java index 79541bd..cc7f060 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sDatabaseTable.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sDatabaseTable.java @@ -40,14 +40,17 @@ public Row(String name, String url, String schema, String dialect, String driver } // CHECKSTYLE:ON - public K8sDatabaseTable(K8sContext context) { + private final K8sEngineTable engines; + + public K8sDatabaseTable(K8sContext context, K8sEngineTable engines) { super(context, K8sApiEndpoints.DATABASES, Row.class); + this.engines = engines; } public void addDatabases(SchemaPlus parentSchema) { for (Row row : rows()) { parentSchema.add(schemaName(row), - HoptimatorJdbcSchema.create(row.NAME, row.SCHEMA, dataSource(row), parentSchema, dialect(row))); + HoptimatorJdbcSchema.create(row.NAME, row.SCHEMA, dataSource(row), parentSchema, dialect(row), engines.forDatabase(row.NAME))); } } diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sEngine.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sEngine.java new file mode 100644 index 0000000..6cbdeea --- /dev/null +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sEngine.java @@ -0,0 +1,40 @@ +package com.linkedin.hoptimator.k8s; + +import javax.sql.DataSource; + +import com.linkedin.hoptimator.Engine; +import com.linkedin.hoptimator.SqlDialect; + +import org.apache.calcite.adapter.jdbc.JdbcSchema; + + +public class K8sEngine implements Engine { + + private final String name; + private final String url; + private final SqlDialect dialect; + private final String driver; + + public K8sEngine(String name, String url, SqlDialect dialect, String driver) { + this.name = name; + this.url = url; + this.dialect = dialect; + this.driver = driver; + } + + @Override + public String engineName() { + return name; + } + + @Override + public DataSource dataSource() { + // TODO support username, password via Secrets + return JdbcSchema.dataSource(url, driver, null, null); + } + + @Override + public SqlDialect dialect() { + return SqlDialect.FLINK; // TODO fix hardcoded dialect + } +} diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sEngineTable.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sEngineTable.java new file mode 100644 index 0000000..6026308 --- /dev/null +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sEngineTable.java @@ -0,0 +1,91 @@ +package com.linkedin.hoptimator.k8s; + +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.stream.Collectors; +import javax.sql.DataSource; + +import org.apache.calcite.adapter.jdbc.JdbcSchema; +import org.apache.calcite.schema.Schema; + +import io.kubernetes.client.openapi.models.V1ObjectMeta; + +import com.linkedin.hoptimator.Engine; +import com.linkedin.hoptimator.SqlDialect; +import com.linkedin.hoptimator.k8s.models.V1alpha1Engine; +import com.linkedin.hoptimator.k8s.models.V1alpha1EngineList; +import com.linkedin.hoptimator.k8s.models.V1alpha1EngineSpec; +import com.linkedin.hoptimator.util.HoptimatorJdbcSchema; + + +public class K8sEngineTable extends K8sTable { + + // CHECKSTYLE:OFF + public static class Row { + public String NAME; + public String URL; + public String DIALECT; + public String DRIVER; + public String[] DATABASES; + + public Row(String name, String url, String dialect, String driver, String[] databases) { + this.NAME = name; + this.URL = url; + this.DIALECT = dialect; + this.DRIVER = driver; + this.DATABASES = databases; + } + } + // CHECKSTYLE:ON + + public K8sEngineTable(K8sContext context) { + super(context, K8sApiEndpoints.ENGINES, Row.class); + } + + /** Engines supporting a given database. */ + public List forDatabase(String database) { + return rows().stream().filter(x -> x.DATABASES == null + || x.DATABASES.length == 0 + || Arrays.asList(x.DATABASES).contains(database)) + .map(x -> new K8sEngine(x.NAME, x.URL, dialect(x), x.DRIVER)) + .collect(Collectors.toList()); + } + + @Override + public Row toRow(V1alpha1Engine obj) { + return new Row(obj.getMetadata().getName(), obj.getSpec().getUrl(), + Optional.ofNullable(obj.getSpec().getDialect()).map(x -> x.toString()).orElseGet(() -> null), + obj.getSpec().getDriver(), obj.getSpec().getDatabases() != null ? + obj.getSpec().getDatabases().toArray(new String[0]) : null); + } + + @Override + public V1alpha1Engine fromRow(Row row) { + K8sUtils.checkK8sName(row.NAME); + return new V1alpha1Engine().kind(K8sApiEndpoints.ENGINES.kind()) + .apiVersion(K8sApiEndpoints.ENGINES.apiVersion()) + .metadata(new V1ObjectMeta().name(row.NAME)) + .spec(new V1alpha1EngineSpec().url(row.URL) + .dialect(V1alpha1EngineSpec.DialectEnum.fromValue(row.DIALECT)) + .driver(row.DRIVER).databases(Arrays.asList(row.DATABASES))); + } + + private static SqlDialect dialect(Row row) { + if (row.DIALECT == null) { + return SqlDialect.ANSI; + } + switch (row.DIALECT) { + case "Flink": + return SqlDialect.FLINK; + default: + return SqlDialect.valueOf(row.DIALECT); + } + } + + @Override + public Schema.TableType getJdbcTableType() { + return Schema.TableType.SYSTEM_TABLE; + } +} diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sMetadata.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sMetadata.java index e07902c..c27ee67 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sMetadata.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sMetadata.java @@ -12,12 +12,15 @@ public class K8sMetadata extends AbstractSchema { private final Map tableMap = new HashMap<>(); private final K8sDatabaseTable databaseTable; + private final K8sEngineTable engineTable; private final K8sViewTable viewTable; public K8sMetadata(K8sContext context) { - this.databaseTable = new K8sDatabaseTable(context); + this.engineTable = new K8sEngineTable(context); + this.databaseTable = new K8sDatabaseTable(context, engineTable); this.viewTable = new K8sViewTable(context); tableMap.put("DATABASES", databaseTable); + tableMap.put("ENGINES", engineTable); tableMap.put("VIEWS", viewTable); } @@ -25,6 +28,10 @@ public K8sDatabaseTable databaseTable() { return databaseTable; } + public K8sEngineTable engineTable() { + return engineTable; + } + public K8sViewTable viewTable() { return viewTable; } diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Database.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Database.java index 6fc51eb..b8a0762 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Database.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Database.java @@ -30,7 +30,7 @@ * Database metadata. */ @ApiModel(description = "Database metadata.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]") public class V1alpha1Database implements io.kubernetes.client.common.KubernetesObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1DatabaseList.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1DatabaseList.java index 4056015..e5c827d 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1DatabaseList.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1DatabaseList.java @@ -32,7 +32,7 @@ * DatabaseList is a list of Database */ @ApiModel(description = "DatabaseList is a list of Database") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]") public class V1alpha1DatabaseList implements io.kubernetes.client.common.KubernetesListObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1DatabaseSpec.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1DatabaseSpec.java index 788aff5..2cde0ed 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1DatabaseSpec.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1DatabaseSpec.java @@ -28,7 +28,7 @@ * Database spec. */ @ApiModel(description = "Database spec.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]") public class V1alpha1DatabaseSpec { /** * SQL dialect the driver expects. diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Engine.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Engine.java index 2e5c679..bc11f2a 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Engine.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Engine.java @@ -30,7 +30,7 @@ * Engine metadata. */ @ApiModel(description = "Engine metadata.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]") public class V1alpha1Engine implements io.kubernetes.client.common.KubernetesObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1EngineList.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1EngineList.java index d573c70..e38a6d0 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1EngineList.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1EngineList.java @@ -32,7 +32,7 @@ * EngineList is a list of Engine */ @ApiModel(description = "EngineList is a list of Engine") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]") public class V1alpha1EngineList implements io.kubernetes.client.common.KubernetesListObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1EngineSpec.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1EngineSpec.java index bbf77eb..b5bc526 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1EngineSpec.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1EngineSpec.java @@ -30,7 +30,7 @@ * Engine spec. */ @ApiModel(description = "Engine spec.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]") public class V1alpha1EngineSpec { public static final String SERIALIZED_NAME_DATABASES = "databases"; @SerializedName(SERIALIZED_NAME_DATABASES) @@ -43,9 +43,7 @@ public class V1alpha1EngineSpec { public enum DialectEnum { ANSI("ANSI"), - MYSQL("MySQL"), - - CALCITE("Calcite"); + FLINK("Flink"); private String value; diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1JobTemplate.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1JobTemplate.java index 06285be..536bf92 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1JobTemplate.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1JobTemplate.java @@ -30,7 +30,7 @@ * Template to apply to matching jobs. */ @ApiModel(description = "Template to apply to matching jobs.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]") public class V1alpha1JobTemplate implements io.kubernetes.client.common.KubernetesObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1JobTemplateList.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1JobTemplateList.java index 6dee379..83c9ba4 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1JobTemplateList.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1JobTemplateList.java @@ -32,7 +32,7 @@ * JobTemplateList is a list of JobTemplate */ @ApiModel(description = "JobTemplateList is a list of JobTemplate") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]") public class V1alpha1JobTemplateList implements io.kubernetes.client.common.KubernetesListObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1JobTemplateSpec.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1JobTemplateSpec.java index 62f7f26..ec74d18 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1JobTemplateSpec.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1JobTemplateSpec.java @@ -30,7 +30,7 @@ * TableTemplate spec. */ @ApiModel(description = "TableTemplate spec.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]") public class V1alpha1JobTemplateSpec { public static final String SERIALIZED_NAME_DATABASES = "databases"; @SerializedName(SERIALIZED_NAME_DATABASES) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Pipeline.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Pipeline.java index c9d18d6..db77d7a 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Pipeline.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Pipeline.java @@ -31,7 +31,7 @@ * A set of objects that work together to deliver data. */ @ApiModel(description = "A set of objects that work together to deliver data.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]") public class V1alpha1Pipeline implements io.kubernetes.client.common.KubernetesObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1PipelineList.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1PipelineList.java index 92259d4..713349b 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1PipelineList.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1PipelineList.java @@ -32,7 +32,7 @@ * PipelineList is a list of Pipeline */ @ApiModel(description = "PipelineList is a list of Pipeline") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]") public class V1alpha1PipelineList implements io.kubernetes.client.common.KubernetesListObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1PipelineSpec.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1PipelineSpec.java index f9090d1..0710d83 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1PipelineSpec.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1PipelineSpec.java @@ -28,7 +28,7 @@ * Pipeline spec. */ @ApiModel(description = "Pipeline spec.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]") public class V1alpha1PipelineSpec { public static final String SERIALIZED_NAME_SQL = "sql"; @SerializedName(SERIALIZED_NAME_SQL) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1PipelineStatus.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1PipelineStatus.java index 6a093d0..1fbe592 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1PipelineStatus.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1PipelineStatus.java @@ -28,7 +28,7 @@ * Pipeline status. */ @ApiModel(description = "Pipeline status.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]") public class V1alpha1PipelineStatus { public static final String SERIALIZED_NAME_FAILED = "failed"; @SerializedName(SERIALIZED_NAME_FAILED) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Subscription.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Subscription.java index 197337d..c15202a 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Subscription.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1Subscription.java @@ -31,7 +31,7 @@ * Hoptimator Subscription */ @ApiModel(description = "Hoptimator Subscription") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]") public class V1alpha1Subscription implements io.kubernetes.client.common.KubernetesObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SubscriptionList.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SubscriptionList.java index 966b4db..b289842 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SubscriptionList.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SubscriptionList.java @@ -32,7 +32,7 @@ * SubscriptionList is a list of Subscription */ @ApiModel(description = "SubscriptionList is a list of Subscription") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]") public class V1alpha1SubscriptionList implements io.kubernetes.client.common.KubernetesListObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SubscriptionSpec.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SubscriptionSpec.java index fb107ea..299aede 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SubscriptionSpec.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SubscriptionSpec.java @@ -31,7 +31,7 @@ * Subscription spec */ @ApiModel(description = "Subscription spec") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]") public class V1alpha1SubscriptionSpec { public static final String SERIALIZED_NAME_DATABASE = "database"; @SerializedName(SERIALIZED_NAME_DATABASE) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SubscriptionStatus.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SubscriptionStatus.java index 8903258..5e84dc8 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SubscriptionStatus.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SubscriptionStatus.java @@ -32,7 +32,7 @@ * Filled in by the operator. */ @ApiModel(description = "Filled in by the operator.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]") public class V1alpha1SubscriptionStatus { public static final String SERIALIZED_NAME_ATTRIBUTES = "attributes"; @SerializedName(SERIALIZED_NAME_ATTRIBUTES) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1TableTemplate.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1TableTemplate.java index dc0020a..c7370a7 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1TableTemplate.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1TableTemplate.java @@ -30,7 +30,7 @@ * Template to apply to matching tables. */ @ApiModel(description = "Template to apply to matching tables.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]") public class V1alpha1TableTemplate implements io.kubernetes.client.common.KubernetesObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1TableTemplateList.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1TableTemplateList.java index 49badac..cc38012 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1TableTemplateList.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1TableTemplateList.java @@ -32,7 +32,7 @@ * TableTemplateList is a list of TableTemplate */ @ApiModel(description = "TableTemplateList is a list of TableTemplate") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]") public class V1alpha1TableTemplateList implements io.kubernetes.client.common.KubernetesListObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1TableTemplateSpec.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1TableTemplateSpec.java index 68e221a..f7b317a 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1TableTemplateSpec.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1TableTemplateSpec.java @@ -30,7 +30,7 @@ * TableTemplate spec. */ @ApiModel(description = "TableTemplate spec.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]") public class V1alpha1TableTemplateSpec { public static final String SERIALIZED_NAME_CONNECTOR = "connector"; @SerializedName(SERIALIZED_NAME_CONNECTOR) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1View.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1View.java index 9909777..533c7b3 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1View.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1View.java @@ -30,7 +30,7 @@ * A SQL view. */ @ApiModel(description = "A SQL view.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]") public class V1alpha1View implements io.kubernetes.client.common.KubernetesObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1ViewList.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1ViewList.java index a5b0f7a..603535b 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1ViewList.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1ViewList.java @@ -32,7 +32,7 @@ * ViewList is a list of View */ @ApiModel(description = "ViewList is a list of View") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]") public class V1alpha1ViewList implements io.kubernetes.client.common.KubernetesListObject { public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1ViewSpec.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1ViewSpec.java index 5faaac0..fc86e02 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1ViewSpec.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1ViewSpec.java @@ -28,7 +28,7 @@ * View spec. */ @ApiModel(description = "View spec.") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T18:40:21.513Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]") public class V1alpha1ViewSpec { public static final String SERIALIZED_NAME_MATERIALIZED = "materialized"; @SerializedName(SERIALIZED_NAME_MATERIALIZED) diff --git a/hoptimator-k8s/src/main/resources/engines.crd.yaml b/hoptimator-k8s/src/main/resources/engines.crd.yaml index 136b2a9..bea20af 100644 --- a/hoptimator-k8s/src/main/resources/engines.crd.yaml +++ b/hoptimator-k8s/src/main/resources/engines.crd.yaml @@ -40,8 +40,7 @@ spec: type: string enum: - ANSI - - MySQL - - Calcite + - Flink driver: description: Fully qualified class name of JDBD driver. type: string diff --git a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/HoptimatorJdbcSchema.java b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/HoptimatorJdbcSchema.java index c66780c..875e30e 100644 --- a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/HoptimatorJdbcSchema.java +++ b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/HoptimatorJdbcSchema.java @@ -1,5 +1,6 @@ package com.linkedin.hoptimator.util; +import java.util.List; import javax.sql.DataSource; import org.apache.calcite.adapter.jdbc.JdbcSchema; @@ -11,47 +12,54 @@ import org.apache.calcite.sql.SqlDialectFactoryImpl; import com.linkedin.hoptimator.Database; +import com.linkedin.hoptimator.Engine; import com.linkedin.hoptimator.util.planner.HoptimatorJdbcConvention; public class HoptimatorJdbcSchema extends JdbcSchema implements Database { private final String database; + private final List engines; public static HoptimatorJdbcSchema create(String database, String schema, DataSource dataSource, - SchemaPlus parentSchema, SqlDialect dialect) { + SchemaPlus parentSchema, SqlDialect dialect, List engines) { if (dialect == null) { - return new HoptimatorJdbcSchema(database, schema, dataSource, parentSchema); + return new HoptimatorJdbcSchema(database, schema, dataSource, parentSchema, engines); } else { - return new HoptimatorJdbcSchema(database, schema, dataSource, parentSchema, dialect); + return new HoptimatorJdbcSchema(database, schema, dataSource, parentSchema, dialect, engines); } } public HoptimatorJdbcSchema(String database, String schema, DataSource dataSource, SqlDialect dialect, - Expression expression) { - super(dataSource, dialect, new HoptimatorJdbcConvention(dialect, expression, database), + Expression expression, List engines) { + super(dataSource, dialect, new HoptimatorJdbcConvention(dialect, expression, database, engines), null, schema); this.database = database; + this.engines = engines; } public HoptimatorJdbcSchema(String database, String schema, DataSource dataSource, - SchemaPlus parentSchema, SqlDialect dialect) { + SchemaPlus parentSchema, SqlDialect dialect, List engines) { this(database, schema, dataSource, dialect, - Schemas.subSchemaExpression(parentSchema, schema, HoptimatorJdbcSchema.class)); + Schemas.subSchemaExpression(parentSchema, schema, HoptimatorJdbcSchema.class), engines); } public HoptimatorJdbcSchema(String database, String schema, DataSource dataSource, - SchemaPlus parentSchema, SqlDialectFactory dialectFactory) { - this(database, schema, dataSource, parentSchema, createDialect(dialectFactory, dataSource)); + SchemaPlus parentSchema, SqlDialectFactory dialectFactory, List engines) { + this(database, schema, dataSource, parentSchema, createDialect(dialectFactory, dataSource), engines); } public HoptimatorJdbcSchema(String database, String schema, DataSource dataSource, - SchemaPlus parentSchema) { - this(database, schema, dataSource, parentSchema, SqlDialectFactoryImpl.INSTANCE); + SchemaPlus parentSchema, List engines) { + this(database, schema, dataSource, parentSchema, SqlDialectFactoryImpl.INSTANCE, engines); } @Override public String databaseName() { return database; } + + public List engines() { + return engines; + } } diff --git a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/EngineRules.java b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/EngineRules.java new file mode 100644 index 0000000..e3f3399 --- /dev/null +++ b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/EngineRules.java @@ -0,0 +1,164 @@ +package com.linkedin.hoptimator.util.planner; + +import java.util.Collections; + +import org.apache.calcite.adapter.jdbc.JdbcConvention; +import org.apache.calcite.adapter.jdbc.JdbcImplementor; +import org.apache.calcite.adapter.jdbc.JdbcRel; +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptCost; +import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.InvalidRelException; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.convert.ConverterRule; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rel.type.RelDataTypeSystem; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexProgram; +import org.apache.calcite.schema.Table; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.sql.SqlDialect; +import org.apache.calcite.sql.dialect.AnsiSqlDialect; +import org.apache.calcite.sql.dialect.MysqlSqlDialect; +import org.apache.calcite.sql.type.SqlTypeFactoryImpl; + +import com.linkedin.hoptimator.Engine; + + +/** Calling convention using a remote execution engine. */ +public final class EngineRules { + + private final Engine engine; + + public EngineRules(Engine engine) { + this.engine = engine; + } + + public void register(HoptimatorJdbcConvention inTrait, RelOptPlanner planner) { + SqlDialect dialect = dialect(engine); + String name = engine.engineName() + "-" + inTrait.database(); + JdbcConvention outTrait = JdbcConvention.of(dialect, inTrait.expression, name); + + System.out.println("Registering rules for " + name + " using dialect " + dialect.toString()); + planner.addRule(RemoteJoinRule.Config.INSTANCE + .withConversion(Join.class, Convention.NONE, outTrait, "RemoteJoinRule") + .withRuleFactory(RemoteJoinRule::new) + .as(RemoteJoinRule.Config.class) + .toRule(RemoteJoinRule.class)); + planner.addRule(RemoteTableScanRule.Config.INSTANCE + .withConversion(TableScan.class, Convention.NONE, outTrait, "RemoteTableScan") + .withRuleFactory(RemoteTableScanRule::new) + .as(RemoteTableScanRule.Config.class) + .toRule(RemoteTableScanRule.class)); + } + + private class RemoteTableScanRule extends ConverterRule { + + protected RemoteTableScanRule(Config config) { + super(config); + } + + @Override + public RelNode convert(RelNode rel) { + TableScan scan = (TableScan) rel; + RelOptTable relOptTable = scan.getTable(); + RelTraitSet newTraitSet = rel.getTraitSet().replace(getOutTrait()); + return new RemoteTableScan(rel.getCluster(), newTraitSet, relOptTable); + } + } + + private class RemoteTableScan extends TableScan implements JdbcRel { + + public RemoteTableScan(RelOptCluster cluster, RelTraitSet traitSet, RelOptTable table) { + super(cluster, traitSet, Collections.emptyList(), table); + } + + @Override + public JdbcImplementor.Result implement(JdbcImplementor implementor) { + SqlDialect dialect = dialect(engine); + System.out.println("Generating sql in dialect " + dialect.toString()); + JdbcImplementor.Result res = new JdbcImplementor(dialect, new JavaTypeFactoryImpl()).implement(getInput(0)); + System.out.println("Implemented: " + res.toString()); + return res; + } + } + + private class RemoteJoinRule extends ConverterRule { + + protected RemoteJoinRule(Config config) { + super(config); + } + + @Override + public RelNode convert(RelNode rel) { + RelTraitSet newTraitSet = rel.getTraitSet().replace(getOutTrait()); + Join join = (Join) rel; + try { + return new RemoteJoin(rel.getCluster(), newTraitSet, join.getLeft(), join.getRight(), + join.getCondition(), join.getJoinType()); + } catch (InvalidRelException e) { + System.out.println(e); + throw new AssertionError(e); + } + } + } + + private class RemoteJoin extends Join implements JdbcRel { + + protected RemoteJoin(RelOptCluster cluster, RelTraitSet traitSet, RelNode left, + RelNode right, RexNode condition, JoinRelType joinType) + throws InvalidRelException { + super(cluster, traitSet, Collections.emptyList(), left, right, condition, + Collections.emptySet(), joinType); + } + + @Override + public Join copy(RelTraitSet traitSet, RexNode condition, RelNode left, RelNode right, + JoinRelType joinType, boolean semiJoinDone) { + try { + return new RemoteJoin(getCluster(), traitSet, left, right, condition, joinType); + } catch (InvalidRelException e) { + System.out.println(e); + throw new AssertionError(e); + } + } + + @Override + public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { + return super.computeSelfCost(planner, mq).multiplyBy(0); // TODO fix zero cost + } + + @Override + public JdbcImplementor.Result implement(JdbcImplementor implementor) { + SqlDialect dialect = dialect(engine); + System.out.println("Generating sql in dialect " + dialect.toString()); + JdbcImplementor.Result res = new JdbcImplementor(dialect, + new JavaTypeFactoryImpl()).implement(getInput(0)); + System.out.println("implemented (2) " + res.toString()); + return res; + } + } + + private static SqlDialect dialect(Engine engine) { + switch (engine.dialect()) { + case ANSI: + return AnsiSqlDialect.DEFAULT; + case FLINK: + return MysqlSqlDialect.DEFAULT; + default: + throw new IllegalArgumentException("Unknown dialect " + engine.dialect()); + } + } +} diff --git a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcConvention.java b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcConvention.java index 1e61022..23ebbd4 100644 --- a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcConvention.java +++ b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcConvention.java @@ -1,28 +1,40 @@ package com.linkedin.hoptimator.util.planner; +import java.util.List; + import org.apache.calcite.adapter.jdbc.JdbcConvention; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.sql.SqlDialect; +import com.linkedin.hoptimator.Engine; + public class HoptimatorJdbcConvention extends JdbcConvention { private final String database; + private final List engines; - public HoptimatorJdbcConvention(SqlDialect dialect, Expression expression, String name) { + public HoptimatorJdbcConvention(SqlDialect dialect, Expression expression, String name, + List engines) { super(dialect, expression, name); this.database = name; + this.engines = engines; } public String database() { return database; } + public List engines() { + return engines; + } + @Override public void register(RelOptPlanner planner) { super.register(planner); planner.addRule(PipelineRules.PipelineTableScanRule.create(this)); planner.addRule(PipelineRules.PipelineTableModifyRule.create(this)); + engines().forEach(x -> new EngineRules(x).register(this, planner)); } } From ca31dd18b52efb940adc7b4e80275d66f514f7a2 Mon Sep 17 00:00:00 2001 From: Ryanne Dolan Date: Tue, 14 Jan 2025 17:57:54 -0600 Subject: [PATCH 3/3] WIP: Add support for external engines --- README.md | 2 +- deploy/samples/flinkengine.yaml | 8 + gradle/libs.versions.toml | 1 + .../java/com/linkedin/hoptimator/Engine.java | 2 + hoptimator-cli/build.gradle | 1 + .../linkedin/hoptimator/k8s/K8sEngine.java | 11 +- .../hoptimator/k8s/K8sEngineTable.java | 1 - .../hoptimator/util/SimpleDataSource.java | 69 ++++ .../hoptimator/util/planner/EngineRules.java | 57 ++- .../planner/HoptimatorJdbcConvention.java | 1 + .../util/planner/RemoteConvention.java | 21 + .../hoptimator/util/planner/RemoteRel.java | 9 + .../planner/RemoteToEnumerableConverter.java | 390 ++++++++++++++++++ .../RemoteToEnumerableConverterRule.java | 52 +++ 14 files changed, 592 insertions(+), 33 deletions(-) create mode 100644 deploy/samples/flinkengine.yaml create mode 100644 hoptimator-util/src/main/java/com/linkedin/hoptimator/util/SimpleDataSource.java create mode 100644 hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/RemoteConvention.java create mode 100644 hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/RemoteRel.java create mode 100644 hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/RemoteToEnumerableConverter.java create mode 100644 hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/RemoteToEnumerableConverterRule.java diff --git a/README.md b/README.md index dc42b09..bcc3c11 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ Once the Flink deployment pod has STATUS 'Running', you can forward port 8081 an to access the Flink dashboard. ``` - $ kubectl port-forward basic-session-deployment-7b94b98b6b-d6jt5 8081 & + $ kubectl port-forward svc/basic-session-deployment-rest 8081 & ``` See the [Flink SQL Gateway Documentation](https://nightlies.apache.org/flink/flink-docs-release-1.18/docs/dev/table/sql-gateway/overview/) diff --git a/deploy/samples/flinkengine.yaml b/deploy/samples/flinkengine.yaml new file mode 100644 index 0000000..4941f97 --- /dev/null +++ b/deploy/samples/flinkengine.yaml @@ -0,0 +1,8 @@ +apiVersion: hoptimator.linkedin.com/v1alpha1 +kind: Engine +metadata: + name: flink-engine +spec: + url: jdbc:flink://localhost:8083 + dialect: Flink + diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index a1b9d64..14f2cf8 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -8,6 +8,7 @@ flink-clients = "org.apache.flink:flink-clients:1.18.1" flink-connector-base = "org.apache.flink:flink-connector-base:1.18.1" flink-core = "org.apache.flink:flink-core:1.18.1" flink-csv = "org.apache.flink:flink-csv:1.18.1" +flink-jdbc = "org.apache.flink:flink-sql-jdbc-driver-bundle:1.18.1" flink-streaming-java = "org.apache.flink:flink-streaming-java:1.18.1" flink-table-api-java = "org.apache.flink:flink-table-api-java:1.18.1" flink-table-api-java-bridge = "org.apache.flink:flink-table-api-java-bridge:1.18.1" diff --git a/hoptimator-api/src/main/java/com/linkedin/hoptimator/Engine.java b/hoptimator-api/src/main/java/com/linkedin/hoptimator/Engine.java index 5b3091d..c6ddc21 100644 --- a/hoptimator-api/src/main/java/com/linkedin/hoptimator/Engine.java +++ b/hoptimator-api/src/main/java/com/linkedin/hoptimator/Engine.java @@ -10,4 +10,6 @@ public interface Engine { DataSource dataSource(); SqlDialect dialect(); + + String url(); } diff --git a/hoptimator-cli/build.gradle b/hoptimator-cli/build.gradle index 0702e8e..713adc3 100644 --- a/hoptimator-cli/build.gradle +++ b/hoptimator-cli/build.gradle @@ -17,6 +17,7 @@ dependencies { implementation libs.calcite.core implementation libs.sqlline implementation libs.slf4j.simple + implementation libs.flink.jdbc } publishing { diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sEngine.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sEngine.java index 6cbdeea..5ba3df6 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sEngine.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sEngine.java @@ -5,6 +5,8 @@ import com.linkedin.hoptimator.Engine; import com.linkedin.hoptimator.SqlDialect; +import java.util.Objects; + import org.apache.calcite.adapter.jdbc.JdbcSchema; @@ -17,7 +19,7 @@ public class K8sEngine implements Engine { public K8sEngine(String name, String url, SqlDialect dialect, String driver) { this.name = name; - this.url = url; + this.url = Objects.requireNonNull(url, "url"); this.dialect = dialect; this.driver = driver; } @@ -33,8 +35,13 @@ public DataSource dataSource() { return JdbcSchema.dataSource(url, driver, null, null); } + @Override + public String url() { + return url; + } + @Override public SqlDialect dialect() { - return SqlDialect.FLINK; // TODO fix hardcoded dialect + return dialect; } } diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sEngineTable.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sEngineTable.java index 6026308..0b12d7f 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sEngineTable.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sEngineTable.java @@ -5,7 +5,6 @@ import java.util.Locale; import java.util.Optional; import java.util.stream.Collectors; -import javax.sql.DataSource; import org.apache.calcite.adapter.jdbc.JdbcSchema; import org.apache.calcite.schema.Schema; diff --git a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/SimpleDataSource.java b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/SimpleDataSource.java new file mode 100644 index 0000000..b8057a6 --- /dev/null +++ b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/SimpleDataSource.java @@ -0,0 +1,69 @@ +package com.linkedin.hoptimator.util; + +import java.io.PrintWriter; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.logging.Logger; +import javax.sql.DataSource; + + +public class SimpleDataSource implements DataSource { + private static final Logger logger = Logger.getLogger("SimpleDataSource"); + + private String url; + private int loginTimeout = 60; + private PrintWriter printWriter = new PrintWriter(System.out); + + public SimpleDataSource() { + } + + public void setUrl(String url) { + this.url = url; + } + + @Override + public Connection getConnection() throws SQLException { + return DriverManager.getConnection(url); + } + + @Override + public Connection getConnection(String user, String pass) throws SQLException { + return DriverManager.getConnection(url, user, pass); + } + + @Override + public int getLoginTimeout() { + return loginTimeout; + } + + @Override + public void setLoginTimeout(int timeout) { + this.loginTimeout = timeout; + } + + @Override + public PrintWriter getLogWriter() { + return printWriter; + } + + @Override + public void setLogWriter(PrintWriter printWriter) { + this.printWriter = printWriter; + } + + @Override + public Logger getParentLogger() { + return logger; + } + + @Override + public boolean isWrapperFor(Class clazz) throws SQLException { + return false; + } + + @Override + public T unwrap(Class clazz) throws SQLException { + return null; + } +} diff --git a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/EngineRules.java b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/EngineRules.java index e3f3399..453222d 100644 --- a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/EngineRules.java +++ b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/EngineRules.java @@ -4,7 +4,6 @@ import org.apache.calcite.adapter.jdbc.JdbcConvention; import org.apache.calcite.adapter.jdbc.JdbcImplementor; -import org.apache.calcite.adapter.jdbc.JdbcRel; import org.apache.calcite.jdbc.JavaTypeFactoryImpl; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.plan.Convention; @@ -31,9 +30,29 @@ import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.sql.SqlDialect; import org.apache.calcite.sql.dialect.AnsiSqlDialect; +import org.apache.calcite.sql.dialect.CalciteSqlDialect; import org.apache.calcite.sql.dialect.MysqlSqlDialect; import org.apache.calcite.sql.type.SqlTypeFactoryImpl; +import org.apache.calcite.adapter.enumerable.EnumerableConvention; +import org.apache.calcite.adapter.enumerable.EnumerableRel; +import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor; +import org.apache.calcite.adapter.enumerable.JavaRowFormat; +import org.apache.calcite.adapter.enumerable.PhysType; +import org.apache.calcite.adapter.enumerable.PhysTypeImpl; +import org.apache.calcite.linq4j.tree.BlockBuilder; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.plan.ConventionTraitDef; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.convert.ConverterImpl; +import org.apache.calcite.rel.convert.ConverterRule; +import org.apache.calcite.runtime.Hook; +import org.apache.calcite.util.BuiltInMethod; + import com.linkedin.hoptimator.Engine; @@ -47,18 +66,17 @@ public EngineRules(Engine engine) { } public void register(HoptimatorJdbcConvention inTrait, RelOptPlanner planner) { - SqlDialect dialect = dialect(engine); - String name = engine.engineName() + "-" + inTrait.database(); - JdbcConvention outTrait = JdbcConvention.of(dialect, inTrait.expression, name); - - System.out.println("Registering rules for " + name + " using dialect " + dialect.toString()); + String name = engine.engineName() + "-" + inTrait.database(); + RemoteConvention remote = new RemoteConvention(name, engine); + System.out.println("Registering rules for " + engine.engineName()); + planner.addRule(RemoteToEnumerableConverterRule.create(remote)); planner.addRule(RemoteJoinRule.Config.INSTANCE - .withConversion(Join.class, Convention.NONE, outTrait, "RemoteJoinRule") + .withConversion(PipelineRules.PipelineJoin.class, PipelineRel.CONVENTION, remote, "RemoteJoinRule") .withRuleFactory(RemoteJoinRule::new) .as(RemoteJoinRule.Config.class) .toRule(RemoteJoinRule.class)); planner.addRule(RemoteTableScanRule.Config.INSTANCE - .withConversion(TableScan.class, Convention.NONE, outTrait, "RemoteTableScan") + .withConversion(PipelineRules.PipelineTableScan.class, PipelineRel.CONVENTION, remote, "RemoteTableScan") .withRuleFactory(RemoteTableScanRule::new) .as(RemoteTableScanRule.Config.class) .toRule(RemoteTableScanRule.class)); @@ -79,20 +97,11 @@ public RelNode convert(RelNode rel) { } } - private class RemoteTableScan extends TableScan implements JdbcRel { + private class RemoteTableScan extends TableScan implements RemoteRel { public RemoteTableScan(RelOptCluster cluster, RelTraitSet traitSet, RelOptTable table) { super(cluster, traitSet, Collections.emptyList(), table); } - - @Override - public JdbcImplementor.Result implement(JdbcImplementor implementor) { - SqlDialect dialect = dialect(engine); - System.out.println("Generating sql in dialect " + dialect.toString()); - JdbcImplementor.Result res = new JdbcImplementor(dialect, new JavaTypeFactoryImpl()).implement(getInput(0)); - System.out.println("Implemented: " + res.toString()); - return res; - } } private class RemoteJoinRule extends ConverterRule { @@ -115,7 +124,7 @@ public RelNode convert(RelNode rel) { } } - private class RemoteJoin extends Join implements JdbcRel { + private class RemoteJoin extends Join implements RemoteRel { protected RemoteJoin(RelOptCluster cluster, RelTraitSet traitSet, RelNode left, RelNode right, RexNode condition, JoinRelType joinType) @@ -139,16 +148,6 @@ public Join copy(RelTraitSet traitSet, RexNode condition, RelNode left, RelNode public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { return super.computeSelfCost(planner, mq).multiplyBy(0); // TODO fix zero cost } - - @Override - public JdbcImplementor.Result implement(JdbcImplementor implementor) { - SqlDialect dialect = dialect(engine); - System.out.println("Generating sql in dialect " + dialect.toString()); - JdbcImplementor.Result res = new JdbcImplementor(dialect, - new JavaTypeFactoryImpl()).implement(getInput(0)); - System.out.println("implemented (2) " + res.toString()); - return res; - } } private static SqlDialect dialect(Engine engine) { diff --git a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcConvention.java b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcConvention.java index 23ebbd4..4f8e3b4 100644 --- a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcConvention.java +++ b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcConvention.java @@ -35,6 +35,7 @@ public void register(RelOptPlanner planner) { super.register(planner); planner.addRule(PipelineRules.PipelineTableScanRule.create(this)); planner.addRule(PipelineRules.PipelineTableModifyRule.create(this)); + PipelineRules.rules().forEach(x -> planner.addRule(x)); engines().forEach(x -> new EngineRules(x).register(this, planner)); } } diff --git a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/RemoteConvention.java b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/RemoteConvention.java new file mode 100644 index 0000000..4de6955 --- /dev/null +++ b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/RemoteConvention.java @@ -0,0 +1,21 @@ +package com.linkedin.hoptimator.util.planner; + +import com.linkedin.hoptimator.Engine; + +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.plan.Convention; + + +class RemoteConvention extends Convention.Impl { + + private final Engine engine; + + RemoteConvention(String name, Engine engine) { + super(name, RemoteRel.class); + this.engine = engine; + } + + Engine engine() { + return engine; + } +} diff --git a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/RemoteRel.java b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/RemoteRel.java new file mode 100644 index 0000000..b55968b --- /dev/null +++ b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/RemoteRel.java @@ -0,0 +1,9 @@ +package com.linkedin.hoptimator.util.planner; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.plan.Convention; + + +public interface RemoteRel extends RelNode { + +} diff --git a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/RemoteToEnumerableConverter.java b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/RemoteToEnumerableConverter.java new file mode 100644 index 0000000..10903ea --- /dev/null +++ b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/RemoteToEnumerableConverter.java @@ -0,0 +1,390 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * N.B. this file copy-pasted from Apache Calcite with modifications. + */ +package com.linkedin.hoptimator.util.planner; + +import com.linkedin.hoptimator.util.DeploymentService; +import com.linkedin.hoptimator.util.SimpleDataSource; + +import org.apache.calcite.DataContext; +import org.apache.calcite.adapter.enumerable.EnumerableRel; +import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor; +import org.apache.calcite.adapter.enumerable.JavaRowFormat; +import org.apache.calcite.adapter.enumerable.PhysType; +import org.apache.calcite.adapter.enumerable.PhysTypeImpl; +import org.apache.calcite.adapter.enumerable.RexImpTable; +import org.apache.calcite.adapter.java.JavaTypeFactory; +import org.apache.calcite.adapter.jdbc.JdbcConvention; +import org.apache.calcite.adapter.jdbc.JdbcRel; +import org.apache.calcite.config.CalciteSystemProperty; +import org.apache.calcite.linq4j.tree.BlockBuilder; +import org.apache.calcite.linq4j.tree.ConstantExpression; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.linq4j.tree.ParameterExpression; +import org.apache.calcite.linq4j.tree.Primitive; +import org.apache.calcite.linq4j.tree.UnaryExpression; +import org.apache.calcite.plan.ConventionTraitDef; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptCost; +import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.rel.convert.ConverterImpl; +import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.runtime.Hook; +import org.apache.calcite.runtime.SqlFunctions; +import org.apache.calcite.schema.Schemas; +import org.apache.calcite.sql.SqlDialect; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.dialect.MysqlSqlDialect; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.util.SqlString; +import org.apache.calcite.util.BuiltInMethod; + +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.List; +import java.util.TimeZone; +import java.util.stream.Collectors; +import javax.sql.DataSource; + +import static org.apache.calcite.linq4j.Nullness.castNonNull; + +import static java.util.Objects.requireNonNull; + +/** + * Relational expression representing a scan of a table in a JDBC data source. + */ +public class RemoteToEnumerableConverter + extends ConverterImpl + implements EnumerableRel { + protected RemoteToEnumerableConverter( + RelOptCluster cluster, + RelTraitSet traits, + RelNode input) { + super(cluster, ConventionTraitDef.INSTANCE, traits, input); + } + + /** This method modified from upstream */ + private SqlString generateSql(SqlDialect dialect) { + RelRoot root = RelRoot.of(getInput(), SqlKind.SELECT); + try { + PipelineRel.Implementor plan = DeploymentService.plan(root); + return new SqlString(MysqlSqlDialect.DEFAULT, plan.sql().apply(com.linkedin.hoptimator.SqlDialect.FLINK)); // TODO dialect + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override public RelNode copy(RelTraitSet traitSet, List inputs) { + return new RemoteToEnumerableConverter( + getCluster(), traitSet, sole(inputs)); + } + + @Override public @Nullable RelOptCost computeSelfCost(RelOptPlanner planner, + RelMetadataQuery mq) { + RelOptCost cost = super.computeSelfCost(planner, mq); + if (cost == null) { + return null; + } + return cost.multiplyBy(.1); + } + + @Override public Result implement(EnumerableRelImplementor implementor, Prefer pref) { + // Generate: + // ResultSetEnumerable.of(schema.getDataSource(), "select ...") + final BlockBuilder builder0 = new BlockBuilder(false); + final RemoteRel child = (RemoteRel) getInput(); + final PhysType physType = + PhysTypeImpl.of( + implementor.getTypeFactory(), getRowType(), + pref.prefer(JavaRowFormat.CUSTOM)); + final RemoteConvention convention = + (RemoteConvention) requireNonNull(child.getConvention(), + () -> "child.getConvention() is null for " + child); + SqlString sqlString = generateSql(MysqlSqlDialect.DEFAULT); // TODO hard-coded dialect + String sql = sqlString.getSql(); + if (CalciteSystemProperty.DEBUG.value()) { + System.out.println("[" + sql + "]"); + } + Hook.QUERY_PLAN.run(sql); + final Expression sql_ = + builder0.append("sql", Expressions.constant(sql)); + final int fieldCount = getRowType().getFieldCount(); + BlockBuilder builder = new BlockBuilder(); + final ParameterExpression resultSet_ = + Expressions.parameter(Modifier.FINAL, ResultSet.class, + builder.newName("resultSet")); + final SqlDialect.CalendarPolicy calendarPolicy = + MysqlSqlDialect.DEFAULT.getCalendarPolicy(); // TODO hard-coded dialect + final Expression calendar_; + switch (calendarPolicy) { + case LOCAL: + calendar_ = + builder0.append("calendar", + Expressions.call(Calendar.class, "getInstance", + getTimeZoneExpression(implementor))); + break; + default: + calendar_ = null; + } + if (fieldCount == 1) { + final ParameterExpression value_ = + Expressions.parameter(Object.class, builder.newName("value")); + builder.add(Expressions.declare(Modifier.FINAL, value_, null)); + generateGet(implementor, physType, builder, resultSet_, 0, value_, + calendar_, calendarPolicy); + builder.add(Expressions.return_(null, value_)); + } else { + final Expression values_ = + builder.append("values", + Expressions.newArrayBounds(Object.class, 1, + Expressions.constant(fieldCount))); + for (int i = 0; i < fieldCount; i++) { + generateGet(implementor, physType, builder, resultSet_, i, + Expressions.arrayIndex(values_, Expressions.constant(i)), + calendar_, calendarPolicy); + } + builder.add( + Expressions.return_(null, values_)); + } + final ParameterExpression e_ = + Expressions.parameter(SQLException.class, builder.newName("e")); + final Expression rowBuilderFactory_ = + builder0.append("rowBuilderFactory", + Expressions.lambda( + Expressions.block( + Expressions.return_(null, + Expressions.lambda( + Expressions.block( + Expressions.tryCatch( + builder.toBlock(), + Expressions.catch_( + e_, + Expressions.throw_( + Expressions.new_( + RuntimeException.class, + e_)))))))), + resultSet_)); + + String dataSourceUrl = convention.engine().url(); + Expression dataSource = builder0.append("dataSource", + Expressions.new_(SimpleDataSource.class)); + + builder0.add( + Expressions.statement( + Expressions.call(dataSource, "setUrl", Expressions.constant(dataSourceUrl)))); + + final Expression enumerable; + + if (sqlString.getDynamicParameters() != null + && !sqlString.getDynamicParameters().isEmpty()) { + final Expression preparedStatementConsumer_ = + builder0.append("preparedStatementConsumer", + Expressions.call(BuiltInMethod.CREATE_ENRICHER.method, + Expressions.newArrayInit(Integer.class, 1, + toIndexesTableExpression(sqlString)), + DataContext.ROOT)); + + enumerable = + builder0.append("enumerable", + Expressions.call( + BuiltInMethod.RESULT_SET_ENUMERABLE_OF_PREPARED.method, +// Schemas.unwrap(convention.expression(), DataSource.class), + dataSource, + sql_, + rowBuilderFactory_, + preparedStatementConsumer_)); + } else { + enumerable = + builder0.append("enumerable", + Expressions.call( + BuiltInMethod.RESULT_SET_ENUMERABLE_OF.method, +// Schemas.unwrap(convention.expression(), DataSource.class), + dataSource, + sql_, + rowBuilderFactory_)); + } + builder0.add( + Expressions.statement( + Expressions.call(enumerable, + BuiltInMethod.RESULT_SET_ENUMERABLE_SET_TIMEOUT.method, + DataContext.ROOT))); + builder0.add( + Expressions.return_(null, enumerable)); + return implementor.result(physType, builder0.toBlock()); + } + + private static List toIndexesTableExpression(SqlString sqlString) { + return requireNonNull(sqlString.getDynamicParameters(), + () -> "sqlString.getDynamicParameters() is null for " + sqlString).stream() + .map(Expressions::constant) + .collect(Collectors.toList()); + } + + private static UnaryExpression getTimeZoneExpression( + EnumerableRelImplementor implementor) { + return Expressions.convert_( + Expressions.call( + implementor.getRootExpression(), + "get", + Expressions.constant("timeZone")), + TimeZone.class); + } + + private static void generateGet(EnumerableRelImplementor implementor, + PhysType physType, BlockBuilder builder, ParameterExpression resultSet_, + int i, Expression target, @Nullable Expression calendar_, + SqlDialect.CalendarPolicy calendarPolicy) { + final Primitive primitive = Primitive.ofBoxOr(physType.fieldClass(i)); + final RelDataType fieldType = + physType.getRowType().getFieldList().get(i).getType(); + final List dateTimeArgs = new ArrayList<>(); + dateTimeArgs.add(Expressions.constant(i + 1)); + SqlTypeName sqlTypeName = fieldType.getSqlTypeName(); + boolean offset = false; + switch (calendarPolicy) { + case LOCAL: + requireNonNull(calendar_, "calendar_"); + dateTimeArgs.add(calendar_); + break; + case NULL: + // We don't specify a calendar at all, so we don't add an argument and + // instead use the version of the getXXX that doesn't take a Calendar + break; + case DIRECT: + sqlTypeName = SqlTypeName.ANY; + break; + case SHIFT: + switch (sqlTypeName) { + case TIMESTAMP: + case DATE: + offset = true; + break; + default: + break; + } + break; + default: + break; + } + final Expression source; + switch (sqlTypeName) { + case DATE: + case TIME: + case TIMESTAMP: + source = + Expressions.call( + getMethod(sqlTypeName, fieldType.isNullable(), offset), + Expressions.list() + .append( + Expressions.call(resultSet_, + getMethod2(sqlTypeName), dateTimeArgs)) + .appendIf(offset, getTimeZoneExpression(implementor))); + break; + case ARRAY: + final Expression x = + Expressions.convert_( + Expressions.call(resultSet_, jdbcGetMethod(primitive), + Expressions.constant(i + 1)), + java.sql.Array.class); + source = Expressions.call(BuiltInMethod.JDBC_ARRAY_TO_LIST.method, x); + break; + case NULL: + source = RexImpTable.NULL_EXPR; + break; + default: + source = + Expressions.call(resultSet_, jdbcGetMethod(primitive), + Expressions.constant(i + 1)); + } + builder.add( + Expressions.statement( + Expressions.assign( + target, source))); + + // [CALCITE-596] If primitive type columns contain null value, returns null + // object + if (primitive != null) { + builder.add( + Expressions.ifThen( + Expressions.call(resultSet_, "wasNull"), + Expressions.statement( + Expressions.assign(target, + Expressions.constant(null))))); + } + } + + private static Method getMethod(SqlTypeName sqlTypeName, boolean nullable, + boolean offset) { + switch (sqlTypeName) { + case DATE: + return (nullable + ? (offset + ? BuiltInMethod.DATE_TO_INT_OPTIONAL_OFFSET + : BuiltInMethod.DATE_TO_INT_OPTIONAL) + : (offset + ? BuiltInMethod.DATE_TO_INT_OFFSET + : BuiltInMethod.DATE_TO_INT)).method; + case TIME: + return (nullable + ? BuiltInMethod.TIME_TO_INT_OPTIONAL + : BuiltInMethod.TIME_TO_INT).method; + case TIMESTAMP: + return (nullable + ? (offset + ? BuiltInMethod.TIMESTAMP_TO_LONG_OPTIONAL_OFFSET + : BuiltInMethod.TIMESTAMP_TO_LONG_OPTIONAL) + : (offset + ? BuiltInMethod.TIMESTAMP_TO_LONG_OFFSET + : BuiltInMethod.TIMESTAMP_TO_LONG)).method; + default: + throw new AssertionError(sqlTypeName + ":" + nullable); + } + } + + private static Method getMethod2(SqlTypeName sqlTypeName) { + switch (sqlTypeName) { + case DATE: + return BuiltInMethod.RESULT_SET_GET_DATE2.method; + case TIME: + return BuiltInMethod.RESULT_SET_GET_TIME2.method; + case TIMESTAMP: + return BuiltInMethod.RESULT_SET_GET_TIMESTAMP2.method; + default: + throw new AssertionError(sqlTypeName); + } + } + + /** E,g, {@code jdbcGetMethod(int)} returns "getInt". */ + private static String jdbcGetMethod(@Nullable Primitive primitive) { + return primitive == null + ? "getObject" + : "get" + SqlFunctions.initcap(castNonNull(primitive.primitiveName)); + } +} diff --git a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/RemoteToEnumerableConverterRule.java b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/RemoteToEnumerableConverterRule.java new file mode 100644 index 0000000..3804874 --- /dev/null +++ b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/RemoteToEnumerableConverterRule.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * N.B. this file copy-pasted from Apache Calcite with modifications. + */ +package com.linkedin.hoptimator.util.planner; + +import org.apache.calcite.adapter.enumerable.EnumerableConvention; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.convert.ConverterRule; + +import org.checkerframework.checker.nullness.qual.Nullable; + +/** + * Rule to convert a relational expression from + * {@link JdbcConvention} to + * {@link EnumerableConvention}. + */ +public class RemoteToEnumerableConverterRule extends ConverterRule { + /** Creates a RemoteToEnumerableConverterRule. */ + public static RemoteToEnumerableConverterRule create(RemoteConvention inTrait) { + return Config.INSTANCE + .withConversion(RelNode.class, inTrait, EnumerableConvention.INSTANCE, + "RemoteToEnumerableConverterRule") + .withRuleFactory(RemoteToEnumerableConverterRule::new) + .toRule(RemoteToEnumerableConverterRule.class); + } + + /** Called from the Config. */ + protected RemoteToEnumerableConverterRule(Config config) { + super(config); + } + + @Override public @Nullable RelNode convert(RelNode rel) { + RelTraitSet newTraitSet = rel.getTraitSet().replace(getOutTrait()); + return new RemoteToEnumerableConverter(rel.getCluster(), newTraitSet, rel); + } +}