Skip to content

Commit

Permalink
[ADH-5242]: Prepare environment for integration tests for the trino-a…
Browse files Browse the repository at this point in the history 
…db connector (#7)

* ADH-5242
- added starting adb env for integration tests
- fixed bugs
- fixed some integration tests

* ADH-5242
- fixed binding

* ADH-5242
- fixed tests for decimal type with exceeding precision
- fixed unloading data bug

* ADH-5242
- added support adb array type
- added support adb hstore (trino map) type
- fixed types tests

* ADH-5242
- fixed pr comments

* ADH-5429
- fixed trino-adb connector tests
- fixed bugs

* ADH-5429
- fixed formatting
  • Loading branch information
@VitekArkhipov
VitekArkhipov authored Jan 9, 2025
1 parent 0c25a25 commit bef1dc9
Show file tree
Hide file tree
Showing 53 changed files with 4,947 additions and 244 deletions.
22 changes: 22 additions & 0 deletions plugin/trino-adb/README.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Trino Adb connector

## Build

```shell
mvn clean install -DskipTests=true
```

## Build adb image before start integration tests

Command will run [build_docker_images.sh](src/test/env/scripts/build_docker_images.sh) script which create and push adb
docker image

```shell
mvn exec:exec -P test package
```

## Run tests

```shell
mvn clean install
```
216 changes: 215 additions & 1 deletion plugin/trino-adb/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,21 @@
<description>Trino - Adb connector</description>

<dependencies>
<dependency>
<groupId>com.google.errorprone</groupId>
<artifactId>error_prone_annotations</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<exclusions>
<exclusion>
<groupId>com.google.errorprone</groupId>
<artifactId>error_prone_annotations</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
Expand Down Expand Up @@ -60,6 +70,12 @@
<groupId>io.airlift</groupId>
<artifactId>http-server</artifactId>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>jakarta.annotation</groupId>
<artifactId>jakarta.annotation-api</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
Expand Down Expand Up @@ -158,6 +174,25 @@
</exclusions>
</dependency>

<dependency>
<groupId>io.trino</groupId>
<artifactId>trino-main</artifactId>
<exclusions>
<exclusion>
<groupId>io.arenadata</groupId>
<artifactId>trino-catalog-manager</artifactId>
</exclusion>
<exclusion>
<groupId>org.jetbrains</groupId>
<artifactId>annotations</artifactId>
</exclusion>
<exclusion>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>io.trino</groupId>
<artifactId>trino-matching</artifactId>
Expand All @@ -170,6 +205,12 @@
<scope>compile</scope>
</dependency>

<dependency>
<groupId>jakarta.annotation</groupId>
<artifactId>jakarta.annotation-api</artifactId>
<scope>compile</scope>
</dependency>

<dependency>
<groupId>jakarta.servlet</groupId>
<artifactId>jakarta.servlet-api</artifactId>
Expand Down Expand Up @@ -253,5 +294,178 @@
<artifactId>log-manager</artifactId>
<scope>runtime</scope>
</dependency>

<dependency>
<groupId>com.github.docker-java</groupId>
<artifactId>docker-java-api</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>io.airlift</groupId>
<artifactId>junit-extensions</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>io.airlift</groupId>
<artifactId>testing</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>io.trino</groupId>
<artifactId>trino-base-jdbc</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>

<dependency>
<groupId>io.trino</groupId>
<artifactId>trino-main</artifactId>
<type>test-jar</type>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>io.arenadata</groupId>
<artifactId>trino-catalog-manager</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>io.trino</groupId>
<artifactId>trino-plugin-toolkit</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>

<dependency>
<groupId>io.trino</groupId>
<artifactId>trino-testing</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>io.airlift</groupId>
<artifactId>testing</artifactId>
</exclusion>
<exclusion>
<groupId>io.trino</groupId>
<artifactId>trino-main</artifactId>
</exclusion>
<exclusion>
<groupId>io.trino</groupId>
<artifactId>trino-tpch</artifactId>
</exclusion>
<exclusion>
<groupId>io.trino.tpch</groupId>
<artifactId>tpch</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>io.trino</groupId>
<artifactId>trino-tpch</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>io.trino.tpch</groupId>
<artifactId>tpch</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jetbrains</groupId>
<artifactId>annotations</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>com.github.docker-java</groupId>
<artifactId>docker-java-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.jetbrains</groupId>
<artifactId>annotations</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<build>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<configuration>
<ignoredNonTestScopedDependencies>
<ignoredNonTestScopedDependency>com.google.errorprone:error_prone_annotations</ignoredNonTestScopedDependency>
</ignoredNonTestScopedDependencies>
</configuration>
</plugin>
</plugins>
</pluginManagement>
</build>
<profiles>
<profile>
<id>test</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>3.4.1</version>
<configuration>
<executable>/bin/bash</executable>
<arguments>
<argument>${project.basedir}/src/test/env/scripts/build_docker_images.sh</argument>
</arguments>
</configuration>
<executions>
<execution>
<id>build-adb-docker-image</id>
<goals>
<goal>exec</goal>
</goals>
<phase>pre-integration-test</phase>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
9 changes: 8 additions & 1 deletion plugin/trino-adb/src/main/java/io/trino/plugin/adb/TypeUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,12 @@

public final class TypeUtil
{
public static final String ARRAY_TYPE_ELEMENT_DELIMITER = ",";
public static final String MAP_TYPE_VALUE_SEPARATOR = "=>";
public static final String MAP_TYPE_NULL_VALUE = "NULL";
public static final String MAP_TYPE_ENTRY_SEPARATOR = ", ";
public static final char MAP_TYPE_VALUE_QUOTE = '"';
public static final int POSTGRESQL_MAX_SUPPORTED_TIMESTAMP_PRECISION = 6;
public static final DateTimeFormatter TIMESTAMP_TYPE_FORMATTER = new DateTimeFormatterBuilder()
.appendValue(ChronoField.YEAR_OF_ERA, 4, 9, SignStyle.NORMAL)
.parseLenient()
Expand Down Expand Up @@ -131,7 +137,8 @@ public static Object[] getJdbcObjectArray(ConnectorSession session, Type element
return valuesArray;
}

public static String getArrayElementPgTypeName(ConnectorSession session, DataTypeMapper typeMapper, Type elementType)
public static String getArrayElementPgTypeName(ConnectorSession session, DataTypeMapper typeMapper,
Type elementType)
{
if (DOUBLE.equals(elementType)) {
return "float8";
Expand Down
1 change: 1 addition & 0 deletions plugin/trino-adb/src/main/java/io/trino/plugin/adb/connector/AdbClientModule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ protected void setup(Binder binder)
binder.bind(JdbcClient.class).annotatedWith(ForBaseJdbc.class).to(AdbSqlClient.class).in(Scopes.SINGLETON);
ConfigBinder.configBinder(binder).bindConfig(AdbCreateTableStorageConfig.class);
ConfigBinder.configBinder(binder).bindConfig(JdbcStatisticsConfig.class);
ConfigBinder.configBinder(binder).bindConfig(AdbPushdownConfig.class);
JdbcModule.bindSessionPropertiesProvider(binder, AdbSessionProperties.class);
JdbcModule.bindSessionPropertiesProvider(binder, AdbPushdownSessionProperties.class);
JdbcModule.bindTablePropertiesProvider(binder, AdbTableProperties.class);
Expand Down
14 changes: 14 additions & 0 deletions plugin/trino-adb/src/main/java/io/trino/plugin/adb/connector/AdbPushdownConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

public class AdbPushdownConfig
{
private boolean pushdownEnums;
private boolean pushdownLiterals = true;
private boolean pushdownDecimalArithmetics = true;
private boolean pushdownDoubleArithmetics = true;
Expand All @@ -30,6 +31,19 @@ public class AdbPushdownConfig
private boolean pushdownFunctionUpper = true;
private boolean pushdownFunctionLower = true;

public boolean isPushdownEnums()
{
return this.pushdownEnums;
}

@Config("adb.pushdown.enums")
@ConfigDescription("Whether to pushdown enums (not safe in the general case because Postgres/Adb enum comparison in based on ordinals and not lexicographical order)")
public AdbPushdownConfig setPushdownEnums(boolean pushdownEnums)
{
this.pushdownEnums = pushdownEnums;
return this;
}

public boolean isPushdownLiterals()
{
return this.pushdownLiterals;
Expand Down
11 changes: 11 additions & 0 deletions ...n/trino-adb/src/main/java/io/trino/plugin/adb/connector/AdbPushdownSessionProperties.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
public class AdbPushdownSessionProperties
implements SessionPropertiesProvider
{
public static final String PUSHDOWN_ENUMS = "pushdown_enums";
public static final String PUSHDOWN_LITERALS = "pushdown_literals";
public static final String PUSHDOWN_DECIMAL_ARITHMETICS = "pushdown_decimal_arithmetics";
public static final String PUSHDOWN_DOUBLE_ARITHMETICS = "pushdown_double_arithmetics";
Expand All @@ -42,6 +43,11 @@ public AdbPushdownSessionProperties(AdbPushdownConfig config)
{
ImmutableList.Builder<PropertyMetadata<?>> builder = ImmutableList.builder();
this.sessionProperties = builder
.add(PropertyMetadata.booleanProperty(
PUSHDOWN_ENUMS,
"Whether to pushdown enums (not safe in the general case because Postgres/Greenplum enum comparison in based on ordinals and not lexicographical order)",
config.isPushdownEnums(),
false))
.add(PropertyMetadata.booleanProperty(
PUSHDOWN_LITERALS,
"Whether to pushdown BOOLEAN, CHAR, REAL and DOUBLE literals, as well as literals with NULL values",
Expand Down Expand Up @@ -99,6 +105,11 @@ public List<PropertyMetadata<?>> getSessionProperties()
return this.sessionProperties;
}

public static boolean isPushdownEnums(ConnectorSession session)
{
return session.getProperty(PUSHDOWN_ENUMS, Boolean.class);
}

public static boolean isPushdownLiterals(ConnectorSession session)
{
return session.getProperty(PUSHDOWN_LITERALS, Boolean.class);
Expand Down
2 changes: 1 addition & 1 deletion plugin/trino-adb/src/main/java/io/trino/plugin/adb/connector/AdbSessionProperties.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
public class AdbSessionProperties
implements SessionPropertiesProvider
{
private static final String ARRAY_MAPPING_PROPERTY = "array_mapping";
public static final String ARRAY_MAPPING_PROPERTY = "array_mapping";
private static final String MAX_SCAN_PARALLELISM_PROPERTY = "max_scan_parallelism";
private static final String GPFDIST_RETRY_TIMEOUT_PROPERTY = "gpfdist_retry_timeout";
private static final String ENABLE_STRING_PUSHDOWN_WITH_COLLATE_PROPERTY = "enable_string_pushdown_with_collate";
Expand Down
Loading

0 comments on commit bef1dc9

Please sign in to comment.