diff --git a/build.gradle.kts b/build.gradle.kts index 465f194a12..6a2bf6fcc9 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -176,6 +176,8 @@ allprojects { plugins.withId("java-library") { dependencies { "implementation"(platform(project(":bom"))) + // Add the locally bundled LookerSDK fat jar + "implementation"(files("../libs/looker-kotlin-sdk-a48011f.jar")) } } if (!skipAutostyle) { diff --git a/core/src/main/java/org/apache/calcite/avatica/AvaticaConnection.java b/core/src/main/java/org/apache/calcite/avatica/AvaticaConnection.java index f740d85ffb..7afa8232a8 100644 --- a/core/src/main/java/org/apache/calcite/avatica/AvaticaConnection.java +++ b/core/src/main/java/org/apache/calcite/avatica/AvaticaConnection.java @@ -25,6 +25,7 @@ import org.apache.calcite.avatica.remote.Service.ErrorResponse; import org.apache.calcite.avatica.remote.Service.OpenConnectionRequest; import org.apache.calcite.avatica.remote.TypedValue; +import org.apache.calcite.avatica.remote.looker.LookerRemoteMeta; import org.apache.calcite.avatica.util.ArrayFactoryImpl; import java.sql.Array; @@ -691,9 +692,15 @@ protected ResultSet createResultSet(Meta.MetaResultSet metaResultSet, QueryState // These are all the metadata operations, no updates ResultSet resultSet = executeQueryInternal(statement, metaResultSet.signature.sanitize(), metaResultSet.firstFrame, state, false); - if (metaResultSet.ownStatement) { + + // TODO (b/300522680): All metadata responses from the Calcite adapter are being treated as + // "ownStatement" since there is no Avatica server to handle the result set response. We get + // the raw response from CalciteMetaImpl which correctly sets `closeOnCompletion`. We don't + // want that behavior here since metadata statements should remain open for multiple requests. + if (metaResultSet.ownStatement && !(meta instanceof LookerRemoteMeta)) { resultSet.getStatement().closeOnCompletion(); } + return resultSet; } diff --git a/core/src/main/java/org/apache/calcite/avatica/Helper.java b/core/src/main/java/org/apache/calcite/avatica/Helper.java index fe716db541..c64c93fde8 100644 --- a/core/src/main/java/org/apache/calcite/avatica/Helper.java +++ b/core/src/main/java/org/apache/calcite/avatica/Helper.java @@ -53,7 +53,9 @@ public SQLException createException(String message, String sql, Exception e) { return new AvaticaSqlException(message, rte.getSqlState(), rte.getErrorCode(), rte.getServerExceptions(), serverAddress); } - return new SQLException(message, e); + // TODO b/300529001: Tableau only surfaces the message to users but the important details + // are often in the main exception so present that as well. + return new SQLException(message + ": " + e.getMessage(), e); } public SQLException createException(String message) { diff --git a/core/src/main/java/org/apache/calcite/avatica/remote/RemoteMeta.java b/core/src/main/java/org/apache/calcite/avatica/remote/RemoteMeta.java index ddb6d106b4..29d283b8be 100644 --- a/core/src/main/java/org/apache/calcite/avatica/remote/RemoteMeta.java +++ b/core/src/main/java/org/apache/calcite/avatica/remote/RemoteMeta.java @@ -39,12 +39,12 @@ * Implementation of {@link org.apache.calcite.avatica.Meta} for the remote * driver. */ -class RemoteMeta extends MetaImpl { +public class RemoteMeta extends MetaImpl { final Service service; final Map propsMap = new HashMap<>(); private Map databaseProperties; - RemoteMeta(AvaticaConnection connection, Service service) { + protected RemoteMeta(AvaticaConnection connection, Service service) { super(connection); this.service = service; } diff --git a/core/src/main/java/org/apache/calcite/avatica/remote/looker/LookerConnectionProperty.java b/core/src/main/java/org/apache/calcite/avatica/remote/looker/LookerConnectionProperty.java new file mode 100644 index 0000000000..489e106ee0 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/avatica/remote/looker/LookerConnectionProperty.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.avatica.remote.looker; + +import org.apache.calcite.avatica.ConnectionConfigImpl.PropEnv; +import org.apache.calcite.avatica.ConnectionProperty; + +import java.util.Properties; + +/** TODO: implement Looker specific connection properties */ +public class LookerConnectionProperty implements ConnectionProperty { + + @Override + public String name() { + return null; + } + + @Override + public String camelName() { + return null; + } + + @Override + public Object defaultValue() { + return null; + } + + @Override + public Type type() { + return null; + } + + @Override + public PropEnv wrap(Properties properties) { + return null; + } + + @Override + public boolean required() { + return false; + } + + @Override + public Class valueClass() { + return null; + } +} diff --git a/core/src/main/java/org/apache/calcite/avatica/remote/looker/LookerDriver.java b/core/src/main/java/org/apache/calcite/avatica/remote/looker/LookerDriver.java new file mode 100644 index 0000000000..1d0e7a9f2a --- /dev/null +++ b/core/src/main/java/org/apache/calcite/avatica/remote/looker/LookerDriver.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.avatica.remote.looker; + +import org.apache.calcite.avatica.AvaticaConnection; +import org.apache.calcite.avatica.ConnectStringParser; +import org.apache.calcite.avatica.DriverVersion; +import org.apache.calcite.avatica.Meta; +import org.apache.calcite.avatica.UnregisteredDriver; +import org.apache.calcite.avatica.remote.Service; + +import com.looker.sdk.LookerSDK; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Properties; + +/** + * JDBC Driver for Looker's SQL Interface. Communicates with a Looker instance via + * {@link LookerSDK}. Backed by Looker-specific {@link LookerRemoteMeta} and + * {@link LookerRemoteService}. + * + * Use 'jdbc:looker' as the protocol to select this over the default remote Avatica driver. + */ +public class LookerDriver extends UnregisteredDriver { + + static { + new LookerDriver().register(); + } + + public LookerDriver() { + super(); + } + + public static final String CONNECT_STRING_PREFIX = "jdbc:looker:"; + + @Override + protected DriverVersion createDriverVersion() { + return DriverVersion.load( + org.apache.calcite.avatica.remote.Driver.class, + "org-apache-calcite-jdbc.properties", + "Looker JDBC Driver", + "unknown version", + "Looker", + "unknown version"); + } + + @Override + protected String getConnectStringPrefix() { + return CONNECT_STRING_PREFIX; + } + + @Override + public Meta createMeta(AvaticaConnection connection) { + final Service service = new LookerRemoteService(); + connection.setService(service); + return new LookerRemoteMeta(connection, service); + } + + @Override public Connection connect(String url, Properties info) + throws SQLException { + AvaticaConnection conn = (AvaticaConnection) super.connect(url, info); + + if (conn == null) { + // the URL did not match Looker's JDBC connection string prefix + return null; + } + + // the `looker` driver should always have a matching Service + Service service = conn.getService(); + assert service instanceof LookerRemoteService; + + // puts all additional url params into properties + Properties properties = ConnectStringParser.parse(url, info); + + // create and set LookerSDK for the connection + LookerSDK sdk = LookerSdkFactory.createSdk(conn.config().url(), properties); + ((LookerRemoteService) service).setSdk(sdk); + return conn; + } + +} diff --git a/core/src/main/java/org/apache/calcite/avatica/remote/looker/LookerFrameEnvelope.java b/core/src/main/java/org/apache/calcite/avatica/remote/looker/LookerFrameEnvelope.java new file mode 100644 index 0000000000..1795d2e60d --- /dev/null +++ b/core/src/main/java/org/apache/calcite/avatica/remote/looker/LookerFrameEnvelope.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.avatica.remote.looker; + +import org.apache.calcite.avatica.Meta.Frame; + +/** + * Wrapper for either a {@link Frame} or {@link Exception}. Allows for + * {@link LookerRemoteMeta#stmtQueueMap} to hold complete Frames and present exceptions when + * consumers are ready to encounter them. + */ +public class LookerFrameEnvelope { + + private final Frame frame; + private final Exception exception; + + private LookerFrameEnvelope(/*@Nullable*/ Frame frame, /*@Nullable*/ Exception exception) { + this.frame = frame; + this.exception = exception; + } + + public Frame getFrame() { + return this.frame; + } + + public Exception getException() { + return this.exception; + } + + /** + * Constructs a LookerFrameEnvelope with a {@link Frame}. + */ + public static LookerFrameEnvelope ok(long offset, boolean done, Iterable rows) { + Frame frame = new Frame(offset, done, rows); + return new LookerFrameEnvelope(frame, null); + } + + /** + * Constructs a LookerFrameEnvelope to hold an exception + */ + public static LookerFrameEnvelope error(Exception e) { + return new LookerFrameEnvelope(null, e); + } + + /** + * Whether this LookerFrameEnvelope holds an exception. If true, the envelope holds no {@link Frame}. + */ + public boolean hasException() { + return this.exception != null; + } +} diff --git a/core/src/main/java/org/apache/calcite/avatica/remote/looker/LookerRemoteMeta.java b/core/src/main/java/org/apache/calcite/avatica/remote/looker/LookerRemoteMeta.java new file mode 100644 index 0000000000..62df7bdd05 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/avatica/remote/looker/LookerRemoteMeta.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.avatica.remote.looker; + +import org.apache.calcite.avatica.AvaticaConnection; +import org.apache.calcite.avatica.AvaticaStatement; +import org.apache.calcite.avatica.Meta; +import org.apache.calcite.avatica.MissingResultsException; +import org.apache.calcite.avatica.NoSuchStatementException; +import org.apache.calcite.avatica.QueryState; +import org.apache.calcite.avatica.remote.JsonService; +import org.apache.calcite.avatica.remote.RemoteMeta; +import org.apache.calcite.avatica.remote.Service; +import org.apache.calcite.avatica.remote.Service.PrepareAndExecuteRequest; +import org.apache.calcite.avatica.remote.Service.PrepareRequest; +import org.apache.calcite.avatica.remote.TypedValue; + +import com.looker.rtl.AuthSession; +import com.looker.rtl.Transport; +import com.looker.sdk.LookerSDK; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.sql.SQLException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.stream.Collectors; +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; + +/** + * Implementation of Meta that works in tandem with {@link LookerRemoteService} to stream results + * from the Looker SDK. + */ +public class LookerRemoteMeta extends RemoteMeta implements Meta { + + private final LookerRemoteService lookerService; + + public LookerRemoteMeta(AvaticaConnection connection, Service service) { + super(connection, service); + // this class _must_ be backed by a LookerRemoteService + assert LookerRemoteService.class.isAssignableFrom(service.getClass()); + lookerService = (LookerRemoteService) service; + } + + /** + * Default queue size. Could probably be more or less. 10 chosen for now. + */ + private static final int DEFAULT_FRAME_QUEUE_SIZE = 10; + + /** + * Returns authenticated LookerSDK from LookerRemoteService. + */ + private LookerSDK getSdk() { + return lookerService.sdk; + } + + /** + * A single meta can have multiple running statements. This map keeps track of + * {@code FrameEnvelopes}s that belong to a running statement. See {@link #prepareStreamingThread} + * for more details. + */ + final ConcurrentMap> stmtQueueMap = + new ConcurrentHashMap<>(); + + /** + * An initially empty frame specific to Looker result sets. {@link #sqlInterfaceQueryId} is used + * to begin a streaming query. + */ + static class LookerFrame extends Frame { + + /** + * A unique ID for the current SQL statement to run. Prepared and set during + * {@link LookerRemoteService#apply(PrepareAndExecuteRequest)} or + * {@link LookerRemoteService#apply(PrepareRequest)}. This is distinct from a statement ID. + * Multiple statements may execute the same query ID. + */ + public final Long sqlInterfaceQueryId; + + LookerFrame(long offset, boolean done, Iterable rows, Long statementId) { + super(offset, done, rows); + this.sqlInterfaceQueryId = statementId; + } + + /** + * Creates a {@code LookerFrame} for the query id. + * + * @param sqlInterfaceQueryId id for the prepared statement generated by a Looker instance. + * @return the {@code firstFrame} for the result set. + */ + public static final LookerFrame create(Long sqlInterfaceQueryId) { + return new LookerFrame(0, false, Collections.emptyList(), sqlInterfaceQueryId); + } + } + + /** + * Helper to disable any SSL and hostname verification when `verifySSL` is false. + */ + private void trustAllHosts(HttpsURLConnection connection) { + // Create a trust manager that does not validate certificate chains + TrustManager[] trustAllCerts = new TrustManager[]{ + new X509TrustManager() { + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return null; + } + + public void checkClientTrusted(java.security.cert.X509Certificate[] certs, + String authType) { + } + + public void checkServerTrusted(java.security.cert.X509Certificate[] certs, + String authType) { + } + } + }; + // Create all-trusting host name verifier + HostnameVerifier trustAllHostNames = (hostname, session) -> true; + try { + SSLContext sc = SSLContext.getInstance("SSL"); + sc.init(null, trustAllCerts, new java.security.SecureRandom()); + connection.setSSLSocketFactory(sc.getSocketFactory()); + connection.setHostnameVerifier(trustAllHostNames); + } catch (NoSuchAlgorithmException | KeyManagementException e) { + throw new RuntimeException(e); + } + } + + /** + * A regrettable necessity. The Kotlin SDK relies on an outdated Ktor HTTP client based on Kotlin + * Coroutines which are difficult to work with in Java. Here we make a HTTP client to handle the + * request and input stream ourselves. This adds complexity that would normally be handled by the + * SDK. We should revisit this once the SDK has built-in streams. + * + * TODO https://github.com/looker-open-source/sdk-codegen/issues/1341: + * Add streaming support to the Kotlin SDK. + */ + protected InputStream makeRunQueryRequest(String url) throws IOException, SQLException { + AuthSession authSession = getSdk().getAuthSession(); + Transport sdkTransport = authSession.getTransport(); + + // makes a proper URL from the API endpoint path as the SDK would. + String endpoint = sdkTransport.makeUrl(url, Collections.emptyMap(), null); + URL httpsUrl = new URL(endpoint); + HttpsURLConnection connection = (HttpsURLConnection) httpsUrl.openConnection(); + + // WARNING: You should only set `verifySSL=false` for local/dev instances!! + if (!sdkTransport.getOptions().getVerifySSL()) { + trustAllHosts(connection); + } + + // convert timeout seconds to milliseconds + int timeout = sdkTransport.getOptions().getTimeout() * 1000; + connection.setReadTimeout(timeout); + + connection.setRequestMethod("GET"); + connection.setRequestProperty("Accept", "application/json"); + connection.setDoOutput(true); + + // set the auth header as the SDK would + connection.setRequestProperty("Authorization", + "token " + authSession.getAuthToken().getAccessToken()); + + // copy the headers from the authenticated SDK. + authSession.getApiSettings().getHeaders() + .forEach((header, value) -> connection.setRequestProperty(header, value)); + + int responseCode = connection.getResponseCode(); + if (responseCode == 200) { + // return the input stream to parse from. + return connection.getInputStream(); + } else { + InputStream errorIn = connection.getErrorStream(); + String errorMessage = + new BufferedReader(new InputStreamReader(errorIn, StandardCharsets.UTF_8)) + .lines() + .collect(Collectors.joining("\n")); + HashMap errorMap = JsonService.MAPPER.readValue(errorMessage, HashMap.class); + throw new SQLException("Looker generated SQL failed to execute. " + + errorMap.getOrDefault("message", "")); + } + } + + /** + * Prepares a thread to stream a query response into a series of {@link LookerFrameEnvelope}s. + */ + protected Thread prepareStreamingThread(String baseUrl, Signature signature, int fetchSize, + BlockingQueue frameQueue) throws IOException, SQLException { + + InputStream in = makeRunQueryRequest(baseUrl); + LookerResponseParser parser = new LookerResponseParser(frameQueue); + + return new Thread(() -> parser.parseStream(in, signature, fetchSize)); + } + + @Override + public Frame fetch(final StatementHandle h, final long offset, final int fetchMaxRowCount) + throws NoSuchStatementException, MissingResultsException { + // If this statement was initiated as a LookerFrame then it will have an entry in the queue map + if (stmtQueueMap.containsKey(h.id)) { + try { + BlockingQueue queue = stmtQueueMap.get(h.id); + + // `take` blocks until there is an entry in the queue + LookerFrameEnvelope nextEnvelope = queue.take(); + + // remove the statement from the map if it has an exception, or it is the last frame + if (nextEnvelope.hasException()) { + stmtQueueMap.remove(h.id); + throw new RuntimeException(nextEnvelope.getException()); + } else if (nextEnvelope.getFrame().done) { + stmtQueueMap.remove(h.id); + } + return nextEnvelope.getFrame(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + // not a streaming query - default to RemoteMeta behavior + return super.fetch(h, offset, fetchMaxRowCount); + } + + /** + * Creates a streaming iterable that parses a JSON {@link InputStream} into a series of + * {@link LookerFrameEnvelope}s. + */ + @Override + public Iterable createIterable(StatementHandle h, QueryState state, Signature signature, + List parameters, Frame firstFrame) { + // If this a LookerFrame, then we must be targeting the sql_interface APIs + if (LookerFrame.class.isAssignableFrom(firstFrame.getClass())) { + try { + // generate the endpoint URL to begin the request + LookerFrame lookerFrame = (LookerFrame) firstFrame; + String url = LookerSdkFactory.queryEndpoint(lookerFrame.sqlInterfaceQueryId); + + // grab the statement + AvaticaStatement stmt = connection.statementMap.get(h.id); + if (null == stmt) { + throw new NoSuchStatementException(h); + } + + // setup queue to place complete frames + BlockingQueue frameQueue = new ArrayBlockingQueue( + DEFAULT_FRAME_QUEUE_SIZE); + + // update map so this statement is associated with a queue + stmtQueueMap.put(stmt.handle.id, frameQueue); + + // init and start a new thread to stream from a Looker instance and populate the frameQueue + prepareStreamingThread(url, signature, stmt.getFetchSize(), frameQueue).start(); + } catch (SQLException | NoSuchStatementException | IOException e) { + throw new RuntimeException(e); + } + } + // always return a FetchIterable - we'll check in LookerRemoteMeta#fetch for any enqueued Frames + return super.createIterable(h, state, signature, parameters, firstFrame); + } +} diff --git a/core/src/main/java/org/apache/calcite/avatica/remote/looker/LookerRemoteService.java b/core/src/main/java/org/apache/calcite/avatica/remote/looker/LookerRemoteService.java new file mode 100644 index 0000000000..7a62a2b3c6 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/avatica/remote/looker/LookerRemoteService.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.avatica.remote.looker; + +import org.apache.calcite.avatica.Meta.Signature; +import org.apache.calcite.avatica.Meta.StatementHandle; +import org.apache.calcite.avatica.remote.JsonService; +import org.apache.calcite.avatica.remote.looker.LookerRemoteMeta.LookerFrame; + +import com.looker.sdk.LookerSDK; +import com.looker.sdk.SqlInterfaceQuery; +import com.looker.sdk.SqlInterfaceQueryMetadata; +import com.looker.sdk.WriteSqlInterfaceQueryCreate; + +import java.io.IOException; +import java.util.Arrays; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.calcite.avatica.remote.looker.LookerSdkFactory.safeSdkCall; + +/** + * Implementation of {@link org.apache.calcite.avatica.remote.Service} that uses the Looker SDK to + * send Avatica request/responses to a Looker instance via JSON. + */ +public class LookerRemoteService extends JsonService { + + /** + * Keep track of next statement ID. This is similar to `CalciteMetaImpl` in that it only + * increments. Statements are scoped to connections, so it is unlikely we will ever encounter + * overflow. If we do, the client must make a new connection. + */ + private final AtomicInteger statementCounter = new AtomicInteger(0); + + /** + * These are statements that have been prepared but not yet executed. We need to know the query id + * to run on the Looker instance when a client decides to `execute` them. Multiple statements can + * run the same query, so it is important to index on the statement id rather than query id. + */ + final ConcurrentMap preparedStmtToQueryMap = + new ConcurrentHashMap<>(); + + /** + * The authenticated SDK used to communicate with a Looker instance. + */ + public LookerSDK sdk; + + void setSdk(LookerSDK sdk) { + this.sdk = sdk; + } + + void checkSdk() { + assert null != sdk : "No authenticated SDK for this connection!"; + } + + private class SqlQueryWithSignature { + + SqlInterfaceQuery query; + Signature signature; + + SqlQueryWithSignature(SqlInterfaceQuery query) { + this.query = query; + + try { + this.signature = JsonService.MAPPER.readValue(query.getSignature(), Signature.class); + } catch (IOException e) { + throw handle(e); + } + } + } + + /** + * Helper method to create a {@link ExecuteResponse} for this request. Since we are using the + * Looker SDK we need to create this response client side. + */ + ExecuteResponse lookerExecuteResponse(String connectionId, int statementId, Signature signature, + LookerFrame lookerFrame) { + ResultSetResponse rs = new ResultSetResponse(connectionId, statementId, false, signature, + lookerFrame, -1, null); + return new ExecuteResponse(Arrays.asList(new ResultSetResponse[]{rs}), false, null); + } + + private SqlQueryWithSignature prepareQuery(String sql) { + checkSdk(); + WriteSqlInterfaceQueryCreate queryRequest = new WriteSqlInterfaceQueryCreate( + sql, /*jdbcClient=*/true); + SqlInterfaceQuery preparedQuery = safeSdkCall( + () -> sdk.create_sql_interface_query(queryRequest)); + + return new SqlQueryWithSignature(preparedQuery); + } + + /** + * Handles all non-overridden {@code apply} methods. + * + * Calls the {@code sql_interface_metadata} endpoint of the instance which behaves similarly to a + * standard Avatica server. + */ + @Override + public String apply(String request) { + checkSdk(); + SqlInterfaceQueryMetadata response = safeSdkCall(() -> sdk.sql_interface_metadata(request)); + + return response.getResults(); + } + + /** + * Handles CreateStatementRequest. We want to control the statement id since Looker instances do + * not keep track of the statement. + */ + @Override + public CreateStatementResponse apply(CreateStatementRequest request) { + return new CreateStatementResponse(request.connectionId, statementCounter.getAndIncrement(), + /*rpcMetadata=*/ null); + } + + /** + * Handles PrepareRequests by preparing a query via {@link LookerSDK#create_sql_query} whose + * response contains a query id. This id is used to execute the query via + * {@link LookerSDK#run_sql_query} with the 'json_bi' format. + * + * @param request the base Avatica request to convert into a Looker SDK call. + * @return a {@link PrepareResponse} containing a new {@link StatementHandle}. + */ + @Override + public PrepareResponse apply(PrepareRequest request) { + checkSdk(); + int currentStatementId = statementCounter.getAndIncrement(); + + SqlQueryWithSignature preparedQuery = prepareQuery(request.sql); + StatementHandle stmt = new StatementHandle(request.connectionId, currentStatementId, + preparedQuery.signature); + preparedStmtToQueryMap.put(currentStatementId, preparedQuery); + + return new PrepareResponse(stmt, null); + } + + /** + * Handles ExecuteRequests by setting up a {@link LookerFrame} to stream the response. + * + * @param request the base Avatica request. Used to locate the query to run in the statement map. + * @return a {@link ExecuteResponse} containing a prepared {@link LookerFrame}. + */ + @Override + public ExecuteResponse apply(ExecuteRequest request) { + checkSdk(); + SqlQueryWithSignature preparedQuery = preparedStmtToQueryMap.get(request.statementHandle.id); + + return lookerExecuteResponse(request.statementHandle.connectionId, request.statementHandle.id, + preparedQuery.signature, LookerFrame.create(preparedQuery.query.getId())); + } + + /** + * Handles PrepareAndExecuteRequests by preparing a query via {@link LookerSDK#create_sql_query} + * whose response contains a query id. This id is used to execute the query via + * {@link LookerSDK#run_sql_query} with the 'json_bi' format. + * + * @param request the base Avatica request to convert into a Looker SDK call. + * @return a {@link ExecuteResponse} containing a prepared {@link LookerFrame}. + */ + @Override + public ExecuteResponse apply(PrepareAndExecuteRequest request) { + checkSdk(); + SqlQueryWithSignature preparedQuery = prepareQuery(request.sql); + + return lookerExecuteResponse(request.connectionId, request.statementId, preparedQuery.signature, + LookerFrame.create(preparedQuery.query.getId())); + } + + /** + * If the statement is closed, clean up the prepared statement map + */ + @Override + public CloseStatementResponse apply(CloseStatementRequest request) { + preparedStmtToQueryMap.remove(request.statementId); + return super.apply(request); + } +} diff --git a/core/src/main/java/org/apache/calcite/avatica/remote/looker/LookerResponseParser.java b/core/src/main/java/org/apache/calcite/avatica/remote/looker/LookerResponseParser.java new file mode 100644 index 0000000000..19416d4558 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/avatica/remote/looker/LookerResponseParser.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.avatica.remote.looker; + +import org.apache.calcite.avatica.ColumnMetaData; +import org.apache.calcite.avatica.ColumnMetaData.Rep; +import org.apache.calcite.avatica.Meta.Frame; +import org.apache.calcite.avatica.Meta.Signature; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; + +import java.io.IOException; +import java.io.InputStream; +import java.sql.Types; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; + +import static com.fasterxml.jackson.core.JsonToken.VALUE_NULL; + +public class LookerResponseParser { + + /** + * Constants used in JSON parsing + */ + private static final String ROWS_KEY = "rows"; + private static final String VALUE_KEY = "value"; + + private final BlockingQueue queue; + + public LookerResponseParser(BlockingQueue queue) { + assert queue != null : "null queue!"; + + this.queue = queue; + } + + /** + * Calls the correct method to read the current value on the stream. The {@code get} methods do + * not advance the current token, so they can be called multiple times without changing the state + * of the parser. + * + * @param columnMetaData the {@link ColumnMetaData} for this value. It is important to use the + * {@link Rep} rather than the type name since Avatica represents most datetime values as + * milliseconds since epoch via {@code long}s or {@code int}s. + * @param parser a JsonParser whose current token is a value from the JSON response. Callers + * must ensure that the parser is ready to consume a value token. This method does not change + * the state of the parser. + * @return the parsed value. + */ + static Object deserializeValue(JsonParser parser, ColumnMetaData columnMetaData) + throws IOException { + // don't attempt to parse null values + if (parser.currentToken() == VALUE_NULL) { + return null; + } + switch (columnMetaData.type.rep) { + case PRIMITIVE_BOOLEAN: + case BOOLEAN: + return parser.getBooleanValue(); + case PRIMITIVE_BYTE: + case BYTE: + return parser.getByteValue(); + case STRING: + return parser.getValueAsString(); + case PRIMITIVE_SHORT: + case SHORT: + return parser.getShortValue(); + case PRIMITIVE_INT: + case INTEGER: + return parser.getIntValue(); + case PRIMITIVE_LONG: + case LONG: + return parser.getLongValue(); + case PRIMITIVE_FLOAT: + case FLOAT: + return parser.getFloatValue(); + case PRIMITIVE_DOUBLE: + case DOUBLE: + return parser.getDoubleValue(); + case NUMBER: + // NUMBER is represented as BigDecimal + return parser.getDecimalValue(); + // TODO: MEASURE types are appearing as Objects. This should have been solved by CALCITE-5549. + // Be sure that the signature of a prepared query matches the metadata we see from JDBC. + case OBJECT: + switch (columnMetaData.type.id) { + case Types.INTEGER: + return parser.getIntValue(); + case Types.BIGINT: + return parser.getBigIntegerValue(); + case Types.DOUBLE: + return parser.getDoubleValue(); + case Types.DECIMAL: + case Types.NUMERIC: + return parser.getDecimalValue(); + } + default: + throw new IOException("Unable to parse " + columnMetaData.type.rep + " from stream!"); + } + } + + private void seekToRows(JsonParser parser) throws IOException { + while (parser.nextToken() != null && !ROWS_KEY.equals(parser.currentName())) { + // move position to start of `rows` + } + } + + private void seekToValue(JsonParser parser) throws IOException { + while (parser.nextToken() != null && !VALUE_KEY.equals(parser.currentName())) { + // seeking to `value` key for the field e.g. `"rows": [{"field_1": {"value": 123 }}] + } + // now move to the actual value + parser.nextToken(); + } + + private void putExceptionOrFail(Exception e) { + try { + // `put` blocks until there is room on the queue but needs a catch + queue.put(LookerFrameEnvelope.error(e)); + } catch (InterruptedException ex) { + throw new RuntimeException(ex); + } + } + + /** + * Takes an input stream from a Looker query request and parses it into a series of + * {@link LookerFrameEnvelope}s. Each LookerFrameEnvelope is enqueued in the {@link #queue} of the parser. + * + * @param in the {@link InputStream} to parse. + * @param signature the {@link Signature} for the statement. Needed to access column metadata + * @param fetchSize the number of rows to populate per {@link Frame} + */ + public void parseStream(InputStream in, Signature signature, int fetchSize) { + assert in != null : "null InputStream!"; + + try { + + JsonParser parser = new JsonFactory().createParser(in); + + int currentOffset = 0; + + while (parser.nextToken() != null) { + if (currentOffset == 0) { + // TODO: Handle `metadata`. We are currently ignoring it and seeking to `rows` array + seekToRows(parser); + } + + int rowsRead = 0; + List rows = new ArrayList<>(); + + while (rowsRead < fetchSize) { + List columnValues = new ArrayList<>(); + + for (int i = 0; i < signature.columns.size(); i++) { + seekToValue(parser); + + if (parser.isClosed()) { + // the stream is closed - all rows should be accounted for + currentOffset += rowsRead; + queue.put(LookerFrameEnvelope.ok(currentOffset, /*done=*/true, rows)); + return; + } + + // add the value to the column list + columnValues.add(deserializeValue(parser, signature.columns.get(i))); + } + + // Meta.CursorFactory#deduce will select an OBJECT cursor if there is only a single + // column in the signature. This is intended behavior. Since the rows of a frame are + // simply an `Object` we could get illegal casts from an ArrayList to the underlying Rep + // type. Discard the wrapping ArrayList for the single value to avoid this issue. + Object row = signature.columns.size() == 1 ? columnValues.get(0) : columnValues; + + rows.add(row); + rowsRead++; + } + // we fetched the allowed number of rows so add the complete frame to the queue + currentOffset += rowsRead; + queue.put(LookerFrameEnvelope.ok(currentOffset, /*done=*/false, rows)); + } + } catch (Exception e) { + // enqueue the first exception encountered + putExceptionOrFail(e); + } + } +} diff --git a/core/src/main/java/org/apache/calcite/avatica/remote/looker/LookerSdkFactory.java b/core/src/main/java/org/apache/calcite/avatica/remote/looker/LookerSdkFactory.java new file mode 100644 index 0000000000..d1e10c81ed --- /dev/null +++ b/core/src/main/java/org/apache/calcite/avatica/remote/looker/LookerSdkFactory.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.avatica.remote.looker; + +import com.looker.rtl.AuthSession; +import com.looker.rtl.AuthToken; +import com.looker.rtl.ConfigurationProvider; +import com.looker.rtl.SDKErrorInfo; +import com.looker.rtl.SDKResponse; +import com.looker.rtl.Transport; +import com.looker.rtl.TransportKt; +import com.looker.sdk.ApiSettings; +import com.looker.sdk.LookerSDK; + +import java.sql.SQLException; +import java.sql.SQLInvalidAuthorizationSpecException; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; + +import static com.looker.rtl.TransportKt.ok; +import static com.looker.rtl.TransportKt.parseSDKError; + +import static org.apache.calcite.avatica.remote.Service.OpenConnectionRequest.serializeProperties; + +/** + * Utility class for generating, authenticating, and calling {@link LookerSDK}s. + */ +public class LookerSdkFactory { + + private LookerSdkFactory() { + } + + private static final String RESULT_FORMAT = "json_bi"; + private static final String QUERY_ENDPOINT = "/api/4.0/sql_interface_queries/%s/run/%s"; + + /** + * 1 hour in seconds. This is not configurable. + */ + private static final Long SESSION_LENGTH = 3600L; + + + /** + * Simple interface to wrap SDK calls + */ + public interface LookerSDKCall { + SDKResponse call(); + } + + /** + * Makes the API endpoint to run a previously made query. + */ + public static String queryEndpoint(Long id) { + return String.format(Locale.ROOT, QUERY_ENDPOINT, TransportKt.encodeParam(id), RESULT_FORMAT); + } + + /** + * Makes the SDK call and throws any errors as runtime exceptions + */ + public static T safeSdkCall(LookerSDKCall sdkCall) { + try { + return ok(sdkCall.call()); + } catch (Error e) { + SDKErrorInfo error = parseSDKError(e.toString()); + // TODO: Get full errors from error.errors array + throw new RuntimeException(error.getMessage(), e); + } + } + + private static boolean hasApiCreds(Map props) { + return props.containsKey("user") && props.containsKey("password"); + } + + private static boolean hasAuthToken(Map props) { + return props.containsKey("token"); + } + + /** + * Creates a {@link AuthSession} to a Looker instance. + *

If {@code client_id} and {@code client_secret} are provided in {@code props} then + * {@link AuthSession#login} is called on the session. Otherwise, if {@code token} is provided + * then its value is set as the auth token in the HTTP header for all requests for the session. + * + * @param url the URL of the Looker instance. + * @param props map of properties for the session. + */ + private static AuthSession createAuthSession(String url, Map props) + throws SQLException { + Map apiConfig = new HashMap<>(); + apiConfig.put("base_url", url); + apiConfig.put("timeout", props.get(props.getOrDefault("timeout", "120"))); + apiConfig.put("verify_ssl", props.get("verifySSL")); + + boolean apiLogin = hasApiCreds(props); + boolean authToken = hasAuthToken(props); + + if (apiLogin && authToken) { + throw new SQLInvalidAuthorizationSpecException("Invalid connection params.\n" + + "Cannot provide both API3 credentials and an access token"); + } + if (!apiLogin && !authToken) { + throw new SQLInvalidAuthorizationSpecException( + "Invalid connection params.\n" + "Missing either API3 credentials or access token"); + } + + if (apiLogin) { + apiConfig.put("client_id", props.get("user")); + apiConfig.put("client_secret", props.get("password")); + } + + ConfigurationProvider finalizedConfig = ApiSettings.fromMap(apiConfig); + + String userAgent = props.get("userAgent"); + if (userAgent != null) { + Map headers = finalizedConfig.getHeaders(); + headers.put("User-Agent", props.get("userAgent")); + finalizedConfig.setHeaders(headers); + } + + AuthSession session = new AuthSession(finalizedConfig, new Transport(finalizedConfig)); + + // need to log in if client_id and client_secret are used + if (apiLogin) { + // empty string means no sudo - we won't support this + session.login(""); + } else if (authToken) { + // set the auth token if one was supplied from the OAuth flow + session.setAuthToken( + new AuthToken(props.get("token"), "Bearer", SESSION_LENGTH, null)); + } + + return session; + } + + /** + * Creates an authenticated {@link LookerSDK}. + * + * @param url the URL of the Looker instance. + * @param props map of properties for the session. + */ + public static LookerSDK createSdk(String url, Properties props) throws SQLException { + Map stringProps = serializeProperties(props); + AuthSession session = createAuthSession(url, stringProps); + return new LookerSDK(session); + } +} diff --git a/core/src/main/java/org/apache/calcite/avatica/remote/looker/package-info.java b/core/src/main/java/org/apache/calcite/avatica/remote/looker/package-info.java new file mode 100644 index 0000000000..46b65a24cb --- /dev/null +++ b/core/src/main/java/org/apache/calcite/avatica/remote/looker/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Looker specific implementations for JDBC remote procedure calls. + */ +package org.apache.calcite.avatica.remote.looker; + +// End package-info.java diff --git a/core/src/main/resources/META-INF/services/java.sql.Driver b/core/src/main/resources/META-INF/services/java.sql.Driver index beb1ac04ea..7128d91399 100644 --- a/core/src/main/resources/META-INF/services/java.sql.Driver +++ b/core/src/main/resources/META-INF/services/java.sql.Driver @@ -1 +1,2 @@ org.apache.calcite.avatica.remote.Driver +org.apache.calcite.avatica.remote.looker.LookerDriver diff --git a/core/src/test/java/org/apache/calcite/avatica/remote/looker/LookerDriverTest.java b/core/src/test/java/org/apache/calcite/avatica/remote/looker/LookerDriverTest.java new file mode 100644 index 0000000000..6b77390cd1 --- /dev/null +++ b/core/src/test/java/org/apache/calcite/avatica/remote/looker/LookerDriverTest.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.avatica.remote.looker; + +import org.apache.calcite.avatica.AvaticaConnection; + +import org.junit.Assert; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.Driver; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.SQLInvalidAuthorizationSpecException; +import java.sql.Types; +import java.util.Properties; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class LookerDriverTest { + + static final String TEST_SQL = + "SELECT " + "`users.created_date`, `users.created_year`, `users.created_time`, `users.name`\n" + + "`users.age`, `users.is45or30`, SUM(`users.age`), AGGREGATE(`users.average_age`)\n" + + "FROM example.users\n" + "GROUP BY 1, 2, 3, 4, 5, 6"; + + @Test + public void driverIsRegistered() throws SQLException { + Driver driver = DriverManager.getDriver("jdbc:looker:url=foobar.com"); + + assertThat(driver, is(instanceOf(LookerDriver.class))); + } + + @Test + public void driverThrowsAuthExceptionForBlankProperties() throws SQLException { + Properties props = new Properties(); + try { + Driver driver = DriverManager.getDriver("jdbc:looker:url=foobar.com"); + driver.connect("jdbc:looker:url=foobar.com", props); + + fail("Should have thrown an auth exception!"); + } catch (SQLInvalidAuthorizationSpecException e) { + assertThat(e.getMessage(), + is("Invalid connection params.\nMissing either API3 credentials" + " or access token")); + } + } + + @Test + public void createsAvaticaConnections() throws SQLException { + Properties props = new Properties(); + props.put("token", "foobar"); + + Driver driver = DriverManager.getDriver("jdbc:looker:url=foobar.com"); + Connection connection = driver.connect("jdbc:looker:url=foobar.com", props); + + assertThat(connection, is(instanceOf(AvaticaConnection.class))); + } + + @Test + public void failsGracefullyOnBadJsonResponse() throws SQLException { + // 250 chars is enough to start the stream, but we'll eventually hit an incomplete JSON object + String incompleteJson = LookerTestCommon.stubbedJsonResults.substring(0, 250); + Driver driver = new StubbedLookerDriver().withStubbedResponse(LookerTestCommon.stubbedSignature, + incompleteJson); + Connection connection = driver.connect(LookerTestCommon.getUrl(), + LookerTestCommon.getBaseProps()); + + try { + connection.createStatement().executeQuery(TEST_SQL); + + fail("Should have thrown an exception during stream parsing!"); + } catch (SQLException e) { + + Assert.assertThat(e.getMessage(), + containsString("Error while executing SQL \"" + TEST_SQL + "\"")); + + assertNotNull(e.getCause()); + Assert.assertThat(e.getCause().getMessage(), containsString("Unexpected end-of-input")); + } + } + + @Test + public void resultsCanBeParsedIntoResultSet() throws SQLException { + // Set up the driver with some pre-recorded responses from Looker. These are large JSON strings + // so have been placed in the LookerTestCommon class to make this test easier to read. + Driver driver = new StubbedLookerDriver().withStubbedResponse(LookerTestCommon.stubbedSignature, + LookerTestCommon.stubbedJsonResults); + Connection connection = driver.connect(LookerTestCommon.getUrl(), + LookerTestCommon.getBaseProps()); + + ResultSet test = connection.createStatement().executeQuery(TEST_SQL); + ResultSetMetaData rsMetaData = test.getMetaData(); + + // verify column types + Assert.assertThat(rsMetaData.getColumnCount(), is(8)); + + // DATE + Assert.assertThat(rsMetaData.getColumnType(1), is(Types.DATE)); + Assert.assertThat(rsMetaData.getColumnName(1), is("users.created_date")); + + // YEAR + Assert.assertThat(rsMetaData.getColumnType(2), is(Types.INTEGER)); + Assert.assertThat(rsMetaData.getColumnName(2), is("users.created_year")); + + // TIMESTAMP + Assert.assertThat(rsMetaData.getColumnType(3), is(Types.TIMESTAMP)); + Assert.assertThat(rsMetaData.getColumnName(3), is("users.created_time")); + + // STRING + Assert.assertThat(rsMetaData.getColumnType(4), is(Types.VARCHAR)); + Assert.assertThat(rsMetaData.getColumnName(4), is("users.name")); + + // DOUBLE dimension + Assert.assertThat(rsMetaData.getColumnType(5), is(Types.DOUBLE)); + Assert.assertThat(rsMetaData.getColumnName(5), is("users.age")); + + // BOOLEAN + Assert.assertThat(rsMetaData.getColumnType(6), is(Types.BOOLEAN)); + Assert.assertThat(rsMetaData.getColumnName(6), is("users.is45or30")); + + // DOUBLE custom measure + Assert.assertThat(rsMetaData.getColumnType(7), is(Types.DOUBLE)); + Assert.assertThat(rsMetaData.getColumnName(7), is("EXPR$6")); + + // DOUBLE LookML measure + Assert.assertThat(rsMetaData.getColumnType(8), is(Types.DOUBLE)); + // TODO: investigate why measures are not being aliased as their LookML field name + Assert.assertThat(rsMetaData.getColumnName(8), is("EXPR$7")); + + // verify every row can be fetched with the appropriate getter method + while (test.next()) { + assertNotNull(test.getDate(1)); + assertNotNull(test.getInt(2)); + assertNotNull(test.getTimestamp(3)); + assertNotNull(test.getString(4)); + assertNotNull(test.getDouble(5)); + assertNotNull(test.getBoolean(6)); + assertNotNull(test.getDouble(7)); + assertNotNull(test.getDouble(8)); + } + } + + @Test + public void canUsePreparedStatement() throws SQLException { + Driver driver = new StubbedLookerDriver().withStubbedResponse(LookerTestCommon.stubbedSignature, + LookerTestCommon.stubbedJsonResults); + Connection connection = driver.connect(LookerTestCommon.getUrl(), + LookerTestCommon.getBaseProps()); + + // This time use a prepared statement + PreparedStatement prepareStatement = connection.prepareStatement(TEST_SQL); + ResultSetMetaData metaData = prepareStatement.getMetaData(); + + // verify column types are accessible prior to execution + Assert.assertThat(metaData.getColumnCount(), is(8)); + + // DATE + Assert.assertThat(metaData.getColumnType(1), is(Types.DATE)); + Assert.assertThat(metaData.getColumnName(1), is("users.created_date")); + + // YEAR + Assert.assertThat(metaData.getColumnType(2), is(Types.INTEGER)); + Assert.assertThat(metaData.getColumnName(2), is("users.created_year")); + + // TIMESTAMP + Assert.assertThat(metaData.getColumnType(3), is(Types.TIMESTAMP)); + Assert.assertThat(metaData.getColumnName(3), is("users.created_time")); + + // STRING + Assert.assertThat(metaData.getColumnType(4), is(Types.VARCHAR)); + Assert.assertThat(metaData.getColumnName(4), is("users.name")); + + // DOUBLE dimension + Assert.assertThat(metaData.getColumnType(5), is(Types.DOUBLE)); + Assert.assertThat(metaData.getColumnName(5), is("users.age")); + + // BOOLEAN + Assert.assertThat(metaData.getColumnType(6), is(Types.BOOLEAN)); + Assert.assertThat(metaData.getColumnName(6), is("users.is45or30")); + + // DOUBLE custom measure + Assert.assertThat(metaData.getColumnType(7), is(Types.DOUBLE)); + Assert.assertThat(metaData.getColumnName(7), is("EXPR$6")); + + // DOUBLE LookML measure + Assert.assertThat(metaData.getColumnType(8), is(Types.DOUBLE)); + // TODO: investigate why measures are not being aliased as their LookML field name + Assert.assertThat(metaData.getColumnName(8), is("EXPR$7")); + + // verify execution on a prepared statement works + assertTrue(prepareStatement.execute()); + ResultSet results = prepareStatement.getResultSet(); + + while (results.next()) { + assertNotNull(results.getDate(1)); + assertNotNull(results.getInt(2)); + assertNotNull(results.getTimestamp(3)); + assertNotNull(results.getString(4)); + assertNotNull(results.getDouble(5)); + assertNotNull(results.getBoolean(6)); + assertNotNull(results.getDouble(7)); + assertNotNull(results.getDouble(8)); + } + } +} diff --git a/core/src/test/java/org/apache/calcite/avatica/remote/looker/LookerResponseParserTest.java b/core/src/test/java/org/apache/calcite/avatica/remote/looker/LookerResponseParserTest.java new file mode 100644 index 0000000000..e8478d9ebe --- /dev/null +++ b/core/src/test/java/org/apache/calcite/avatica/remote/looker/LookerResponseParserTest.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.avatica.remote.looker; + +import org.apache.calcite.avatica.ColumnMetaData; +import org.apache.calcite.avatica.ColumnMetaData.AvaticaType; +import org.apache.calcite.avatica.ColumnMetaData.Rep; +import org.apache.calcite.avatica.util.ByteString; +import org.apache.calcite.avatica.util.StructImpl; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigDecimal; +import java.nio.charset.Charset; +import java.sql.Array; +import java.sql.Date; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +public class LookerResponseParserTest { + + private static Map supportedRepValues; + private static Map unsupportedRepValues; + + private static ObjectMapper mapper = new ObjectMapper(); + + static { + Map buildingMap = new HashMap(); + // Primitive types + buildingMap.put(Rep.PRIMITIVE_BOOLEAN, true); + buildingMap.put(Rep.PRIMITIVE_BYTE, (byte) 10); + buildingMap.put(Rep.PRIMITIVE_SHORT, (short) 5); + buildingMap.put(Rep.PRIMITIVE_INT, 100); + buildingMap.put(Rep.PRIMITIVE_LONG, (long) 10000); + buildingMap.put(Rep.PRIMITIVE_FLOAT, (float) 1.99); + buildingMap.put(Rep.PRIMITIVE_DOUBLE, 1.99); + // Non-Primitive types + buildingMap.put(Rep.BOOLEAN, true); + buildingMap.put(Rep.BYTE, new Byte((byte) 10)); + buildingMap.put(Rep.SHORT, new Short((short) 10)); + buildingMap.put(Rep.INTEGER, new Integer(100)); + buildingMap.put(Rep.LONG, new Long(10000)); + buildingMap.put(Rep.FLOAT, new Float(1.99)); + buildingMap.put(Rep.DOUBLE, new Double(1.99)); + buildingMap.put(Rep.STRING, "hello"); + buildingMap.put(Rep.NUMBER, new BigDecimal(1000000)); + // TODO: We shouldn't need to support OBJECT but MEASUREs are appearing as generic objects in + // the signature + buildingMap.put(Rep.OBJECT, 1000); + supportedRepValues = new HashMap(buildingMap); + buildingMap.clear(); + + // Unsupported datetime types + buildingMap.put(Rep.JAVA_SQL_TIME, new Time(1000000)); + buildingMap.put(Rep.JAVA_SQL_DATE, new Date(1000000)); + buildingMap.put(Rep.JAVA_SQL_TIMESTAMP, new Timestamp(100000)); + buildingMap.put(Rep.JAVA_UTIL_DATE, new java.util.Date(1000000)); + // Unsupported object types + buildingMap.put(Rep.ARRAY, new Array[]{}); + buildingMap.put(Rep.BYTE_STRING, new ByteString(new byte[]{'h', 'e', 'l', 'l', 'o'})); + buildingMap.put(Rep.PRIMITIVE_CHAR, 'c'); + buildingMap.put(Rep.CHARACTER, new Character('c')); + buildingMap.put(Rep.MULTISET, new ArrayList()); + buildingMap.put(Rep.STRUCT, new StructImpl(new ArrayList())); + unsupportedRepValues = new HashMap(buildingMap); + buildingMap.clear(); + } + + private JsonParser makeTestParserFromValue(Object value) throws IOException { + String template = "{ \"value\": %s }"; + try { + String valAsJson = mapper.writeValueAsString(value); + String testInput = String.format(Locale.ROOT, template, valAsJson); + + // start input stream and move to `value` + InputStream in = new ByteArrayInputStream(testInput.getBytes(Charset.defaultCharset())); + JsonParser jp = new JsonFactory().createParser(in); + jp.nextFieldName(); // move to "value:" key + jp.nextValue(); // move to value itself + + return jp; + } catch (IOException e) { + throw e; + } + } + + private ColumnMetaData makeDummyMetadata(Rep rep) { + // MEASUREs appear as Objects but typeId is the underlying data type (usually int or double) + // See relevant TODO in LookerRemoteMeta#deserializeValue + int typeId = rep == Rep.OBJECT ? 4 : rep.typeId; + AvaticaType type = new AvaticaType(typeId, rep.name(), rep); + + return ColumnMetaData.dummy(type, false); + } + + @Test + public void deserializeValueTestingIsExhaustive() { + HashMap allMap = new HashMap(); + allMap.putAll(supportedRepValues); + allMap.putAll(unsupportedRepValues); + + Arrays.stream(Rep.values()).forEach(val -> assertNotNull(allMap.get(val))); + } + + @Test + public void deserializeValueThrowsErrorOnUnsupportedType() { + unsupportedRepValues.forEach((rep, value) -> { + try { + JsonParser parser = makeTestParserFromValue(value); + + // should throw an IOException + LookerResponseParser.deserializeValue(parser, makeDummyMetadata(rep)); + fail("Should have thrown an IOException!"); + + } catch (IOException e) { + assertThat(e.getMessage(), is("Unable to parse " + rep.name() + " from stream!")); + } + }); + } + + @Test + public void deserializeValueWorksForSupportedTypes() { + supportedRepValues.forEach((rep, value) -> { + try { + JsonParser parser = makeTestParserFromValue(value); + Object deserializedValue = LookerResponseParser.deserializeValue(parser, + makeDummyMetadata(rep)); + + assertThat(value, is(equalTo(deserializedValue))); + + } catch (IOException e) { + fail(e.getMessage()); + } + }); + } + + @Test + public void returnsNullIfValueIsNull() { + try { + JsonParser parser = makeTestParserFromValue(null); + Object deserializedValue = LookerResponseParser.deserializeValue(parser, + makeDummyMetadata(Rep.DOUBLE)); + + assertNull(deserializedValue); + + } catch (IOException e) { + fail("Should not throw an exception!"); + } + } +} diff --git a/core/src/test/java/org/apache/calcite/avatica/remote/looker/LookerTestCommon.java b/core/src/test/java/org/apache/calcite/avatica/remote/looker/LookerTestCommon.java new file mode 100644 index 0000000000..be7f61ab75 --- /dev/null +++ b/core/src/test/java/org/apache/calcite/avatica/remote/looker/LookerTestCommon.java @@ -0,0 +1,500 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.avatica.remote.looker; + + +import java.util.Properties; + +public class LookerTestCommon { + private LookerTestCommon() { + } + + private static final String BASE_URL = "https://localhost:19999"; + private static final String CLIENT_ID = "f6qG2zPw464yStBrJwrT"; + private static final String CLIENT_SECRET = "KQTpGMPp5mWRQy2Mgrs4SdQT"; + private static final String URL = "jdbc:looker:url=" + BASE_URL; + + private static final Properties BASE_PROPS = new Properties(); + + static final String USER_AGENT = ";userAgent=FooBarInc"; + + static { + BASE_PROPS.put("user", CLIENT_ID); + BASE_PROPS.put("password", CLIENT_SECRET); + BASE_PROPS.put("verifySSL", "false"); + } + + public static String getUrl() { + return URL; + } + + public static Properties getBaseProps() { + return BASE_PROPS; + } + + /** + * Calcite signature for the query: + * + * SELECT `users.created_date`, `users.created_year`, `users.created_time`, `users.name`, + * `users.age`, `users.is45or30`, SUM(`users.age`), AGGREGATE(`users.average_age`) FROM + * example.users GROUP BY 1, 2, 3, 4, 5, 6 + */ + public static String stubbedSignature = "{\"columns\":[{\"ordinal\":0,\"autoIncrement\":false," + + "\"caseSensitive\":true,\"searchable\":false,\"currency\":false,\"nullable\":1," + + "\"signed\":true,\"displaySize\":0,\"label\":\"users.created_date\"," + + "\"columnName\":\"users.created_date\",\"schemaName\":\"example\",\"precision\":0," + + "\"scale\":0,\"tableName\":\"users\",\"catalogName\":null,\"type\":{\"type\":\"scalar\"," + + "\"id\":91,\"name\":\"DATE\",\"rep\":\"INTEGER\"},\"readOnly\":true,\"writable\":false," + + "\"definitelyWritable\":false,\"columnClassName\":\"java.sql.Date\"},{\"ordinal\":1," + + "\"autoIncrement\":false,\"caseSensitive\":true,\"searchable\":false,\"currency\":false," + + "\"nullable\":1,\"signed\":true,\"displaySize\":10,\"label\":\"users.created_year\"," + + "\"columnName\":\"users.created_year\",\"schemaName\":\"example\",\"precision\":10," + + "\"scale\":0,\"tableName\":\"users\",\"catalogName\":null,\"type\":{\"type\":\"scalar\"," + + "\"id\":4,\"name\":\"INTEGER\",\"rep\":\"INTEGER\"},\"readOnly\":true,\"writable\":false," + + "\"definitelyWritable\":false,\"columnClassName\":\"java.lang.Integer\"},{\"ordinal\":2," + + "\"autoIncrement\":false,\"caseSensitive\":true,\"searchable\":false,\"currency\":false," + + "\"nullable\":1,\"signed\":true,\"displaySize\":0,\"label\":\"users.created_time\"," + + "\"columnName\":\"users.created_time\",\"schemaName\":\"example\",\"precision\":0," + + "\"scale\":0,\"tableName\":\"users\",\"catalogName\":null,\"type\":{\"type\":\"scalar\"," + + "\"id\":93,\"name\":\"TIMESTAMP\",\"rep\":\"LONG\"},\"readOnly\":true,\"writable\":false," + + "\"definitelyWritable\":false,\"columnClassName\":\"java.sql.Timestamp\"},{\"ordinal\":3," + + "\"autoIncrement\":false,\"caseSensitive\":true,\"searchable\":false,\"currency\":false," + + "\"nullable\":1,\"signed\":true,\"displaySize\":-1,\"label\":\"users.name\"," + + "\"columnName\":\"users.name\",\"schemaName\":\"example\",\"precision\":0,\"scale\":0," + + "\"tableName\":\"users\",\"catalogName\":null,\"type\":{\"type\":\"scalar\",\"id\":12," + + "\"name\":\"VARCHAR\",\"rep\":\"STRING\"},\"readOnly\":true,\"writable\":false," + + "\"definitelyWritable\":false,\"columnClassName\":\"java.lang.String\"},{\"ordinal\":4," + + "\"autoIncrement\":false,\"caseSensitive\":true,\"searchable\":false,\"currency\":false," + + "\"nullable\":1,\"signed\":true,\"displaySize\":15,\"label\":\"users.age\"," + + "\"columnName\":\"users.age\",\"schemaName\":\"example\",\"precision\":15,\"scale\":0," + + "\"tableName\":\"users\",\"catalogName\":null,\"type\":{\"type\":\"scalar\",\"id\":8," + + "\"name\":\"DOUBLE\",\"rep\":\"DOUBLE\"},\"readOnly\":true,\"writable\":false," + + "\"definitelyWritable\":false,\"columnClassName\":\"java.lang.Double\"},{\"ordinal\":5," + + "\"autoIncrement\":false,\"caseSensitive\":true,\"searchable\":false,\"currency\":false," + + "\"nullable\":1,\"signed\":true,\"displaySize\":1,\"label\":\"users.is45or30\"," + + "\"columnName\":\"users.is45or30\",\"schemaName\":\"example\",\"precision\":1," + + "\"scale\":0,\"tableName\":\"users\",\"catalogName\":null,\"type\":{\"type\":\"scalar\"," + + "\"id\":16,\"name\":\"BOOLEAN\",\"rep\":\"BOOLEAN\"},\"readOnly\":true," + + "\"writable\":false,\"definitelyWritable\":false,\"columnClassName\":\"java.lang" + + ".Boolean\"},{\"ordinal\":6,\"autoIncrement\":false,\"caseSensitive\":true," + + "\"searchable\":false,\"currency\":false,\"nullable\":1,\"signed\":true," + + "\"displaySize\":15,\"label\":\"EXPR$6\",\"columnName\":\"EXPR$6\",\"schemaName\":null," + + "\"precision\":15,\"scale\":0,\"tableName\":null,\"catalogName\":null," + + "\"type\":{\"type\":\"scalar\",\"id\":8,\"name\":\"DOUBLE\",\"rep\":\"DOUBLE\"}," + + "\"readOnly\":true,\"writable\":false,\"definitelyWritable\":false," + + "\"columnClassName\":\"java.lang.Double\"},{\"ordinal\":7,\"autoIncrement\":false," + + "\"caseSensitive\":true,\"searchable\":false,\"currency\":false,\"nullable\":0," + + "\"signed\":true,\"displaySize\":-1,\"label\":\"EXPR$7\",\"columnName\":\"EXPR$7\"," + + "\"schemaName\":null,\"precision\":0,\"scale\":0,\"tableName\":null,\"catalogName\":null," + + "\"type\":{\"type\":\"scalar\",\"id\":8,\"name\":\"MEASURE\",\"rep\":\"OBJECT\"}," + + "\"readOnly\":true,\"writable\":false,\"definitelyWritable\":false," + + "\"columnClassName\":\"java.lang.Double\"}],\"sql\":\"select `users.created_date`,`users" + + ".created_year`, `users.created_time`, `users.name`, `users.age`, `users.is45or30`, SUM" + + "(`users.age`), AGGREGATE(`users.average_age`) from example.users GROUP BY 1, 2, 3, 4, 5," + + " 6\",\"parameters\":[],\"cursorFactory\":{\"style\":\"ARRAY\",\"clazz\":null," + + "\"fieldNames\":null},\"statementType\":\"SELECT\"}"; + + /** + * This is a real response from Looker for the following query in `json_bi` format: + * + * SELECT `users.created_date`, `users.created_year`, `users.created_time`, `users.name`, + * `users.age`, `users.is45or30`, SUM(`users.age`), AGGREGATE(`users.average_age`) FROM + * example.users GROUP BY 1, 2, 3, 4, 5, 6 + */ + public static String stubbedJsonResults = "{\"metadata\":{\"columns_truncated\":false," + + "\"has_totals\":false,\"has_subtotals\":false,\"fields\":{\"dimensions\":[{\"sql\":\"{% " + + "if _dialect._name contains 'druid' %} TIME_PARSE(${TABLE}.created_at, 'yyyy-MM-dd " + + "HH:mm:ss') {% else %} ${TABLE}.created_at {% endif %}\\n \",\"view\":\"users\"," + + "\"dimension_group\":\"users.created\",\"field_group_label\":\"Created Date\"," + + "\"category\":\"dimension\",\"name\":\"users.created_date\",\"type\":\"date_date\"," + + "\"view_label\":\"Users\",\"label\":\"Users Created Date\"," + + "\"field_group_variant\":\"Date\",\"hidden\":false,\"description\":null},{\"sql\":\"{% if" + + " _dialect._name contains 'druid' %} TIME_PARSE(${TABLE}.created_at, 'yyyy-MM-dd " + + "HH:mm:ss') {% else %} ${TABLE}.created_at {% endif %}\\n \",\"view\":\"users\"," + + "\"dimension_group\":\"users.created\",\"field_group_label\":\"Created Date\"," + + "\"category\":\"dimension\",\"name\":\"users.created_year\",\"type\":\"date_year\"," + + "\"view_label\":\"Users\",\"label\":\"Users Created Year\"," + + "\"field_group_variant\":\"Year\",\"hidden\":false,\"description\":null},{\"sql\":\"{% if" + + " _dialect._name contains 'druid' %} TIME_PARSE(${TABLE}.created_at, 'yyyy-MM-dd " + + "HH:mm:ss') {% else %} ${TABLE}.created_at {% endif %}\\n \",\"view\":\"users\"," + + "\"dimension_group\":\"users.created\",\"field_group_label\":\"Created Date\"," + + "\"category\":\"dimension\",\"name\":\"users.created_time\",\"type\":\"date_time\"," + + "\"view_label\":\"Users\",\"label\":\"Users Created Time\"," + + "\"field_group_variant\":\"Time\",\"hidden\":false,\"description\":null},{\"sql\":\"users" + + ".name\",\"view\":\"users\",\"dimension_group\":null,\"field_group_label\":null," + + "\"category\":\"dimension\",\"name\":\"users.name\",\"type\":\"string\"," + + "\"view_label\":\"Users\",\"label\":\"Users Name\",\"field_group_variant\":\"Name\"," + + "\"hidden\":false,\"description\":null},{\"sql\":\"users.age\",\"view\":\"users\"," + + "\"dimension_group\":null,\"field_group_label\":null,\"category\":\"dimension\"," + + "\"name\":\"users.age\",\"type\":\"number\",\"view_label\":\"Users\",\"label\":\"Users " + + "Age\",\"field_group_variant\":\"Age\",\"hidden\":false,\"description\":null}," + + "{\"sql\":\"${age} = 45 OR ${age} = 30 \",\"view\":\"users\",\"dimension_group\":null," + + "\"field_group_label\":null,\"category\":\"dimension\",\"name\":\"users.is45or30\"," + + "\"type\":\"yesno\",\"view_label\":\"Users\",\"label\":\"Users Is45or30 (Yes / No)\"," + + "\"field_group_variant\":\"Is45or30 (Yes / No)\",\"hidden\":false,\"description\":null}]," + + "\"measures\":[{\"sql\":null,\"view\":\"users\",\"dimension_group\":null," + + "\"field_group_label\":null,\"category\":\"measure\",\"name\":\"EXPR__6\"," + + "\"type\":\"sum\",\"view_label\":null,\"label\":\"Users EXPR 6\"," + + "\"field_group_variant\":\"EXPR 6\",\"hidden\":false,\"description\":null}," + + "{\"sql\":\"${age} \",\"view\":\"users\",\"dimension_group\":null," + + "\"field_group_label\":null,\"category\":\"measure\",\"name\":\"users.average_age\"," + + "\"type\":\"average\",\"view_label\":\"Users\",\"label\":\"Users Average Age\"," + + "\"field_group_variant\":\"Average Age\",\"hidden\":false,\"description\":null}]," + + "\"pivots\":[]},\"pivots\":[],\"filters\":{}," + + "\"bigquery_metadata\":{\"total_bytes_processed\":0,\"backend_cache_hit\":true}}," + + "\"rows\":[{\"users.created_date\":{\"value\":15313},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1323114410000}," + + "\"users.name\":{\"value\":\"Reinoehl Augustinus\"},\"users.age\":{\"value\":94.0}," + + "\"users.is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":94.0},\"users" + + ".average_age\":{\"value\":94.0}},{\"users.created_date\":{\"value\":15251},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1317720158000}," + + "\"users.name\":{\"value\":\"Roming Raubenheimer\"},\"users.age\":{\"value\":92.0}," + + "\"users.is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":92.0},\"users" + + ".average_age\":{\"value\":92.0}},{\"users.created_date\":{\"value\":15243},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1317078357000}," + + "\"users.name\":{\"value\":\"Slazac Sharanya\"},\"users.age\":{\"value\":83.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":83.0},\"users" + + ".average_age\":{\"value\":83.0}},{\"users.created_date\":{\"value\":15238},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1316597408000}," + + "\"users.name\":{\"value\":\"Hunziker Rassmann\"},\"users.age\":{\"value\":40.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":40.0},\"users" + + ".average_age\":{\"value\":40.0}},{\"users.created_date\":{\"value\":15217},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1314793506000}," + + "\"users.name\":{\"value\":\"Farissa Ivon\"},\"users.age\":{\"value\":73.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":73.0},\"users" + + ".average_age\":{\"value\":73.0}},{\"users.created_date\":{\"value\":15212},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1314354877000}," + + "\"users.name\":{\"value\":\"Richter Hodinski\"},\"users.age\":{\"value\":53.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":53.0},\"users" + + ".average_age\":{\"value\":53.0}},{\"users.created_date\":{\"value\":15209},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1314096853000}," + + "\"users.name\":{\"value\":\"Augustinus Ziegfried\"},\"users.age\":{\"value\":27.0}," + + "\"users.is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":27.0},\"users" + + ".average_age\":{\"value\":27.0}},{\"users.created_date\":{\"value\":15203},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1313584363000}," + + "\"users.name\":{\"value\":\"Rocco Lincks\"},\"users.age\":{\"value\":93.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":93.0},\"users" + + ".average_age\":{\"value\":93.0}},{\"users.created_date\":{\"value\":15195},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1312894310000}," + + "\"users.name\":{\"value\":\"Brodner Guntermann\"},\"users.age\":{\"value\":56.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":56.0},\"users" + + ".average_age\":{\"value\":56.0}},{\"users.created_date\":{\"value\":15191},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1312541289000}," + + "\"users.name\":{\"value\":\"Mart Schwetlik\"},\"users.age\":{\"value\":31.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":31.0},\"users" + + ".average_age\":{\"value\":31.0}},{\"users.created_date\":{\"value\":15190},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1312454576000}," + + "\"users.name\":{\"value\":\"Neide Wirnseer\"},\"users.age\":{\"value\":71.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":71.0},\"users" + + ".average_age\":{\"value\":71.0}},{\"users.created_date\":{\"value\":15190},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1312452217000}," + + "\"users.name\":{\"value\":\"Herschal Hemsing\"},\"users.age\":{\"value\":27.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":27.0},\"users" + + ".average_age\":{\"value\":27.0}},{\"users.created_date\":{\"value\":15188},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1312310975000}," + + "\"users.name\":{\"value\":\"Faier Oona\"},\"users.age\":{\"value\":81.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":81.0},\"users" + + ".average_age\":{\"value\":81.0}},{\"users.created_date\":{\"value\":15180},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1311584216000}," + + "\"users.name\":{\"value\":\"Adolf Nikolai\"},\"users.age\":{\"value\":89.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":89.0},\"users" + + ".average_age\":{\"value\":89.0}},{\"users.created_date\":{\"value\":15180},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1311582054000}," + + "\"users.name\":{\"value\":\"Denhardt Alisha\"},\"users.age\":{\"value\":87.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":87.0},\"users" + + ".average_age\":{\"value\":87.0}},{\"users.created_date\":{\"value\":15177},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1311338969000}," + + "\"users.name\":{\"value\":\"Kusenberg Odette\"},\"users.age\":{\"value\":78.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":78.0},\"users" + + ".average_age\":{\"value\":78.0}},{\"users.created_date\":{\"value\":15170},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1310721868000}," + + "\"users.name\":{\"value\":\"Geoff Ziegfried\"},\"users.age\":{\"value\":83.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":83.0},\"users" + + ".average_age\":{\"value\":83.0}},{\"users.created_date\":{\"value\":15167},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1310464594000}," + + "\"users.name\":{\"value\":\"Raier Kloble\"},\"users.age\":{\"value\":62.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":62.0},\"users" + + ".average_age\":{\"value\":62.0}},{\"users.created_date\":{\"value\":15166},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1310369694000}," + + "\"users.name\":{\"value\":\"Schomi Ernst\"},\"users.age\":{\"value\":68.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":68.0},\"users" + + ".average_age\":{\"value\":68.0}},{\"users.created_date\":{\"value\":15163},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1310122201000}," + + "\"users.name\":{\"value\":\"Gais Tsoklan\"},\"users.age\":{\"value\":28.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":28.0},\"users" + + ".average_age\":{\"value\":28.0}},{\"users.created_date\":{\"value\":15156},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1309513852000}," + + "\"users.name\":{\"value\":\"Bilt Bartel\"},\"users.age\":{\"value\":67.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":67.0},\"users" + + ".average_age\":{\"value\":67.0}},{\"users.created_date\":{\"value\":15153},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1309255834000}," + + "\"users.name\":{\"value\":\"Pfenic Mangulabnan\"},\"users.age\":{\"value\":96.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":96.0},\"users" + + ".average_age\":{\"value\":96.0}},{\"users.created_date\":{\"value\":15145},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1308568351000}," + + "\"users.name\":{\"value\":\"Rassmann Warenius\"},\"users.age\":{\"value\":95.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":95.0},\"users" + + ".average_age\":{\"value\":95.0}},{\"users.created_date\":{\"value\":15145},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1308568376000}," + + "\"users.name\":{\"value\":\"Schipman Torge\"},\"users.age\":{\"value\":28.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":28.0},\"users" + + ".average_age\":{\"value\":28.0}},{\"users.created_date\":{\"value\":15139},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1308045651000}," + + "\"users.name\":{\"value\":\"Finarolli Doetschmann\"},\"users.age\":{\"value\":22.0}," + + "\"users.is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":22.0},\"users" + + ".average_age\":{\"value\":22.0}},{\"users.created_date\":{\"value\":15139},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1308041280000}," + + "\"users.name\":{\"value\":\"Schorgmayer Swinner\"},\"users.age\":{\"value\":29.0}," + + "\"users.is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":29.0},\"users" + + ".average_age\":{\"value\":29.0}},{\"users.created_date\":{\"value\":15139},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1308043265000}," + + "\"users.name\":{\"value\":\"Ziegfried Schwetlik\"},\"users.age\":{\"value\":88.0}," + + "\"users.is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":88.0},\"users" + + ".average_age\":{\"value\":88.0}},{\"users.created_date\":{\"value\":15121},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1306483063000}," + + "\"users.name\":{\"value\":\"Bastian Orzesdek\"},\"users.age\":{\"value\":34.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":34.0},\"users" + + ".average_age\":{\"value\":34.0}},{\"users.created_date\":{\"value\":15091},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1303892897000}," + + "\"users.name\":{\"value\":\"Ulner Nixey\"},\"users.age\":{\"value\":39.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":39.0},\"users" + + ".average_age\":{\"value\":39.0}},{\"users.created_date\":{\"value\":15089},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1303724778000}," + + "\"users.name\":{\"value\":\"Ranthum Motsenbocker\"},\"users.age\":{\"value\":35.0}," + + "\"users.is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":35.0},\"users" + + ".average_age\":{\"value\":35.0}},{\"users.created_date\":{\"value\":15082},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1303121526000}," + + "\"users.name\":{\"value\":\"Tarnoczi Oona\"},\"users.age\":{\"value\":60.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":60.0},\"users" + + ".average_age\":{\"value\":60.0}},{\"users.created_date\":{\"value\":15070},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1302083363000}," + + "\"users.name\":{\"value\":\"Goetti Kohlstruck\"},\"users.age\":{\"value\":73.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":73.0},\"users" + + ".average_age\":{\"value\":73.0}},{\"users.created_date\":{\"value\":15065},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1301652628000}," + + "\"users.name\":{\"value\":\"Litterst Manno\"},\"users.age\":{\"value\":91.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":91.0},\"users" + + ".average_age\":{\"value\":91.0}},{\"users.created_date\":{\"value\":15053},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1300648324000}," + + "\"users.name\":{\"value\":\"Majcei Cullop\"},\"users.age\":{\"value\":23.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":23.0},\"users" + + ".average_age\":{\"value\":23.0}},{\"users.created_date\":{\"value\":15048},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1300193792000}," + + "\"users.name\":{\"value\":\"Klein Reifschneider\"},\"users.age\":{\"value\":29.0}," + + "\"users.is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":29.0},\"users" + + ".average_age\":{\"value\":29.0}},{\"users.created_date\":{\"value\":15048},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1300200898000}," + + "\"users.name\":{\"value\":\"Diana Briles\"},\"users.age\":{\"value\":77.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":77.0},\"users" + + ".average_age\":{\"value\":77.0}},{\"users.created_date\":{\"value\":15008},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1296731354000}," + + "\"users.name\":{\"value\":\"Jetzer Paschold\"},\"users.age\":{\"value\":82.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":82.0},\"users" + + ".average_age\":{\"value\":82.0}},{\"users.created_date\":{\"value\":14975},\"users" + + ".created_year\":{\"value\":2011},\"users.created_time\":{\"value\":1293842818000}," + + "\"users.name\":{\"value\":\"Nikolai Lingenfelter\"},\"users.age\":{\"value\":69.0}," + + "\"users.is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":69.0},\"users" + + ".average_age\":{\"value\":69.0}},{\"users.created_date\":{\"value\":14945},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1291303917000}," + + "\"users.name\":{\"value\":\"Derrick Hofer\"},\"users.age\":{\"value\":47.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":47.0},\"users" + + ".average_age\":{\"value\":47.0}},{\"users.created_date\":{\"value\":14908},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1288061375000}," + + "\"users.name\":{\"value\":\"Doela Warenius\"},\"users.age\":{\"value\":57.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":57.0},\"users" + + ".average_age\":{\"value\":57.0}},{\"users.created_date\":{\"value\":14902},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1287601958000}," + + "\"users.name\":{\"value\":\"Sacsu Wolf\"},\"users.age\":{\"value\":68.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":68.0},\"users" + + ".average_age\":{\"value\":68.0}},{\"users.created_date\":{\"value\":14894},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1286924692000}," + + "\"users.name\":{\"value\":\"Kelting Oshanny\"},\"users.age\":{\"value\":35.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":35.0},\"users" + + ".average_age\":{\"value\":35.0}},{\"users.created_date\":{\"value\":14893},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1286788336000}," + + "\"users.name\":{\"value\":\"Kraemer Franc\"},\"users.age\":{\"value\":87.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":87.0},\"users" + + ".average_age\":{\"value\":87.0}},{\"users.created_date\":{\"value\":14889},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1286439797000}," + + "\"users.name\":{\"value\":\"Metz Riegerix\"},\"users.age\":{\"value\":58.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":58.0},\"users" + + ".average_age\":{\"value\":58.0}},{\"users.created_date\":{\"value\":14889},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1286471274000}," + + "\"users.name\":{\"value\":\"Warnk Pelger\"},\"users.age\":{\"value\":49.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":49.0},\"users" + + ".average_age\":{\"value\":49.0}},{\"users.created_date\":{\"value\":14874},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1285148625000}," + + "\"users.name\":{\"value\":\"Maurer Sieber\"},\"users.age\":{\"value\":67.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":67.0},\"users" + + ".average_age\":{\"value\":67.0}},{\"users.created_date\":{\"value\":14868},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1284629978000}," + + "\"users.name\":{\"value\":\"Sharanya Irma\"},\"users.age\":{\"value\":31.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":31.0},\"users" + + ".average_age\":{\"value\":31.0}},{\"users.created_date\":{\"value\":14860},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1283935508000}," + + "\"users.name\":{\"value\":\"Ferderber Lingenfelter\"},\"users.age\":{\"value\":22.0}," + + "\"users.is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":22.0},\"users" + + ".average_age\":{\"value\":22.0}},{\"users.created_date\":{\"value\":14859},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1283853687000}," + + "\"users.name\":{\"value\":\"Gaensli Magdalene\"},\"users.age\":{\"value\":22.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":22.0},\"users" + + ".average_age\":{\"value\":22.0}},{\"users.created_date\":{\"value\":14858},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1283816749000}," + + "\"users.name\":{\"value\":\"Justinovia Oresnik\"},\"users.age\":{\"value\":25.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":25.0},\"users" + + ".average_age\":{\"value\":25.0}},{\"users.created_date\":{\"value\":14854},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1283419292000}," + + "\"users.name\":{\"value\":\"Unta Atoulf\"},\"users.age\":{\"value\":48.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":48.0},\"users" + + ".average_age\":{\"value\":48.0}},{\"users.created_date\":{\"value\":14853},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1283330361000}," + + "\"users.name\":{\"value\":\"Floidea Juric\"},\"users.age\":{\"value\":18.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":18.0},\"users" + + ".average_age\":{\"value\":18.0}},{\"users.created_date\":{\"value\":14851},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1283157151000}," + + "\"users.name\":{\"value\":\"Yuo-jie Schmalisch\"},\"users.age\":{\"value\":51.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":51.0},\"users" + + ".average_age\":{\"value\":51.0}},{\"users.created_date\":{\"value\":14847},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1282816345000}," + + "\"users.name\":{\"value\":\"Boholt Staut\"},\"users.age\":{\"value\":56.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":56.0},\"users" + + ".average_age\":{\"value\":56.0}},{\"users.created_date\":{\"value\":14847},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1282812014000}," + + "\"users.name\":{\"value\":\"Slazac Wertschnig\"},\"users.age\":{\"value\":89.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":89.0},\"users" + + ".average_age\":{\"value\":89.0}},{\"users.created_date\":{\"value\":14847},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1282816833000}," + + "\"users.name\":{\"value\":\"Ferencik Roland\"},\"users.age\":{\"value\":42.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":42.0},\"users" + + ".average_age\":{\"value\":42.0}},{\"users.created_date\":{\"value\":14846},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1282727309000}," + + "\"users.name\":{\"value\":\"Maximilian Brechtelsbauer\"},\"users.age\":{\"value\":59.0}," + + "\"users.is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":59.0},\"users" + + ".average_age\":{\"value\":59.0}},{\"users.created_date\":{\"value\":14846},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1282729314000}," + + "\"users.name\":{\"value\":\"Robel Niehaus\"},\"users.age\":{\"value\":32.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":32.0},\"users" + + ".average_age\":{\"value\":32.0}},{\"users.created_date\":{\"value\":14846},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1282725622000}," + + "\"users.name\":{\"value\":\"Hartmut Schoppenhorst\"},\"users.age\":{\"value\":72.0}," + + "\"users.is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":72.0},\"users" + + ".average_age\":{\"value\":72.0}},{\"users.created_date\":{\"value\":14845},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1282638341000}," + + "\"users.name\":{\"value\":\"Sicoe Dietel\"},\"users.age\":{\"value\":61.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":61.0},\"users" + + ".average_age\":{\"value\":61.0}},{\"users.created_date\":{\"value\":14845},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1282643205000}," + + "\"users.name\":{\"value\":\"Chernbach Kaddoura\"},\"users.age\":{\"value\":45.0},\"users" + + ".is45or30\":{\"value\":true},\"EXPR__6\":{\"value\":45.0},\"users" + + ".average_age\":{\"value\":45.0}},{\"users.created_date\":{\"value\":14845},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1282638263000}," + + "\"users.name\":{\"value\":\"Faier Wendt\"},\"users.age\":{\"value\":76.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":76.0},\"users" + + ".average_age\":{\"value\":76.0}},{\"users.created_date\":{\"value\":14845},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1282639897000}," + + "\"users.name\":{\"value\":\"Oresnek Schaller\"},\"users.age\":{\"value\":30.0},\"users" + + ".is45or30\":{\"value\":true},\"EXPR__6\":{\"value\":30.0},\"users" + + ".average_age\":{\"value\":30.0}},{\"users.created_date\":{\"value\":14844},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1282557023000}," + + "\"users.name\":{\"value\":\"Gaensli Niehaus\"},\"users.age\":{\"value\":40.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":40.0},\"users" + + ".average_age\":{\"value\":40.0}},{\"users.created_date\":{\"value\":14844},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1282560871000}," + + "\"users.name\":{\"value\":\"Meurer Neininger\"},\"users.age\":{\"value\":78.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":78.0},\"users" + + ".average_age\":{\"value\":78.0}},{\"users.created_date\":{\"value\":14844},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1282557593000}," + + "\"users.name\":{\"value\":\"Martell Bernard\"},\"users.age\":{\"value\":62.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":62.0},\"users" + + ".average_age\":{\"value\":62.0}},{\"users.created_date\":{\"value\":14844},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1282557121000}," + + "\"users.name\":{\"value\":\"Iurist Warnk\"},\"users.age\":{\"value\":75.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":75.0},\"users" + + ".average_age\":{\"value\":75.0}},{\"users.created_date\":{\"value\":14844},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1282567572000}," + + "\"users.name\":{\"value\":\"Frey Reifschneider\"},\"users.age\":{\"value\":67.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":67.0},\"users" + + ".average_age\":{\"value\":67.0}},{\"users.created_date\":{\"value\":14844},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1282555404000}," + + "\"users.name\":{\"value\":\"Raphael Shaprisha\"},\"users.age\":{\"value\":19.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":19.0},\"users" + + ".average_age\":{\"value\":19.0}},{\"users.created_date\":{\"value\":14844},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1282556522000}," + + "\"users.name\":{\"value\":\"Hektor Doela\"},\"users.age\":{\"value\":57.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":57.0},\"users" + + ".average_age\":{\"value\":57.0}},{\"users.created_date\":{\"value\":14844},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1282557474000}," + + "\"users.name\":{\"value\":\"Pelger Kroll\"},\"users.age\":{\"value\":37.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":37.0},\"users" + + ".average_age\":{\"value\":37.0}},{\"users.created_date\":{\"value\":14844},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1282560000000}," + + "\"users.name\":{\"value\":\"Lambert Wertschnig\"},\"users.age\":{\"value\":90.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":90.0},\"users" + + ".average_age\":{\"value\":90.0}},{\"users.created_date\":{\"value\":14844},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1282559280000}," + + "\"users.name\":{\"value\":\"Muzzleman Ruppert\"},\"users.age\":{\"value\":78.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":78.0},\"users" + + ".average_age\":{\"value\":78.0}},{\"users.created_date\":{\"value\":14844},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1282560244000}," + + "\"users.name\":{\"value\":\"Gumpla Nixey\"},\"users.age\":{\"value\":18.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":18.0},\"users" + + ".average_age\":{\"value\":18.0}},{\"users.created_date\":{\"value\":14837},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1281952766000}," + + "\"users.name\":{\"value\":\"Augustinus Mathilda\"},\"users.age\":{\"value\":51.0}," + + "\"users.is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":51.0},\"users" + + ".average_age\":{\"value\":51.0}},{\"users.created_date\":{\"value\":14834},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1281686134000}," + + "\"users.name\":{\"value\":\"Musbah Fernando\"},\"users.age\":{\"value\":26.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":26.0},\"users" + + ".average_age\":{\"value\":26.0}},{\"users.created_date\":{\"value\":14834},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1281685698000}," + + "\"users.name\":{\"value\":\"Ager Mathilda\"},\"users.age\":{\"value\":70.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":70.0},\"users" + + ".average_age\":{\"value\":70.0}},{\"users.created_date\":{\"value\":14833},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1281632163000}," + + "\"users.name\":{\"value\":\"Gais Lohkamp\"},\"users.age\":{\"value\":30.0},\"users" + + ".is45or30\":{\"value\":true},\"EXPR__6\":{\"value\":30.0},\"users" + + ".average_age\":{\"value\":30.0}},{\"users.created_date\":{\"value\":14833},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1281635580000}," + + "\"users.name\":{\"value\":\"Rautu Kroll\"},\"users.age\":{\"value\":70.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":70.0},\"users" + + ".average_age\":{\"value\":70.0}},{\"users.created_date\":{\"value\":14833},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1281632646000}," + + "\"users.name\":{\"value\":\"Naiux Torge\"},\"users.age\":{\"value\":84.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":84.0},\"users" + + ".average_age\":{\"value\":84.0}},{\"users.created_date\":{\"value\":14833},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1281632943000}," + + "\"users.name\":{\"value\":\"Bors Lohkamp\"},\"users.age\":{\"value\":77.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":77.0},\"users" + + ".average_age\":{\"value\":77.0}},{\"users.created_date\":{\"value\":14833},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1281631587000}," + + "\"users.name\":{\"value\":\"Benzmiller Reifschneider\"},\"users.age\":{\"value\":24.0}," + + "\"users.is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":24.0},\"users" + + ".average_age\":{\"value\":24.0}},{\"users.created_date\":{\"value\":14833},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1281637134000}," + + "\"users.name\":{\"value\":\"Schwaller Ferderber\"},\"users.age\":{\"value\":62.0}," + + "\"users.is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":62.0},\"users" + + ".average_age\":{\"value\":62.0}},{\"users.created_date\":{\"value\":14833},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1281631812000}," + + "\"users.name\":{\"value\":\"Voigt Isoldas\"},\"users.age\":{\"value\":22.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":22.0},\"users" + + ".average_age\":{\"value\":22.0}},{\"users.created_date\":{\"value\":14832},\"users" + + ".created_year\":{\"value\":2010},\"users.created_time\":{\"value\":1281545389000}," + + "\"users.name\":{\"value\":\"Obeldobel Kyler\"},\"users.age\":{\"value\":56.0},\"users" + + ".is45or30\":{\"value\":false},\"EXPR__6\":{\"value\":56.0},\"users" + + ".average_age\":{\"value\":56.0}}]}"; +} diff --git a/core/src/test/java/org/apache/calcite/avatica/remote/looker/StubbedLookerDriver.java b/core/src/test/java/org/apache/calcite/avatica/remote/looker/StubbedLookerDriver.java new file mode 100644 index 0000000000..f05f20c63e --- /dev/null +++ b/core/src/test/java/org/apache/calcite/avatica/remote/looker/StubbedLookerDriver.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.avatica.remote.looker; + +import org.apache.calcite.avatica.AvaticaConnection; +import org.apache.calcite.avatica.ConnectStringParser; +import org.apache.calcite.avatica.Meta; +import org.apache.calcite.avatica.Meta.Signature; +import org.apache.calcite.avatica.Meta.StatementHandle; +import org.apache.calcite.avatica.remote.JsonService; +import org.apache.calcite.avatica.remote.Service; +import org.apache.calcite.avatica.remote.looker.LookerRemoteMeta.LookerFrame; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Properties; + +import static org.junit.Assert.assertNotNull; + +/** + * A testable Looker driver without requiring access to a Looker instance. + * + * {@link #withStubbedResponse} must be called before creating a connection. + */ +public class StubbedLookerDriver extends LookerDriver { + + String stubbedSignature; + String stubbedResponse; + + /** + * Sets stubbed responses for the test. The signature must match the stubbed response. + * + * @param signature {@link Signature} as a JSON string. + * @param response a JSON response identical to one returned by a Looker API call to + * {@code GET /sql_interface_queries/:id/run/json_bi}. + * @return the driver with a stubbed {@link Service} and {@link Meta}. + */ + public LookerDriver withStubbedResponse(String signature, String response) { + this.stubbedSignature = signature; + this.stubbedResponse = response; + + return this; + } + + @Override + public Meta createMeta(AvaticaConnection connection) { + assertNotNull(stubbedSignature); + assertNotNull(stubbedResponse); + + final Service service = new StubbedLookerRemoteService(stubbedSignature); + connection.setService(service); + + return new StubbedLookerRemoteMeta(connection, service, stubbedResponse); + } + + @Override + public Connection connect(String url, Properties info) throws SQLException { + if (!acceptsURL(url)) { + return null; + } + + final String prefix = getConnectStringPrefix(); + assert url.startsWith(prefix); + final String urlSuffix = url.substring(prefix.length()); + final Properties info2 = ConnectStringParser.parse(urlSuffix, info); + final AvaticaConnection connection = factory.newConnection(this, factory, url, info2); + handler.onConnectionInit(connection); + + return connection; + } + + public class StubbedLookerRemoteMeta extends LookerRemoteMeta { + + String stubbedResponse; + + StubbedLookerRemoteMeta(AvaticaConnection connection, Service service, String testResponse) { + super(connection, service); + this.stubbedResponse = testResponse; + } + + @Override + protected InputStream makeRunQueryRequest(String url) { + return new ByteArrayInputStream(stubbedResponse.getBytes(StandardCharsets.UTF_8)); + } + } + + public class StubbedLookerRemoteService extends LookerRemoteService { + + private String stubbedSignature; + + StubbedLookerRemoteService(String signature) { + super(); + this.stubbedSignature = signature; + } + + @Override + public ConnectionSyncResponse apply(ConnectionSyncRequest request) { + try { + // value does not matter for this stub class but needed by the connection + return decode("{\"response\": \"connectionSync\"}", ConnectionSyncResponse.class); + } catch (IOException e) { + throw handle(e); + } + } + + @Override + public CreateStatementResponse apply(CreateStatementRequest request) { + try { + // value does not matter for this stub class but needed by the connection + return decode("{\"response\": \"createStatement\"}", CreateStatementResponse.class); + } catch (IOException e) { + throw handle(e); + } + } + + @Override + public PrepareResponse apply(PrepareRequest request) { + try { + Signature signature = JsonService.MAPPER.readValue(stubbedSignature, Signature.class); + StatementHandle statementHandle = new StatementHandle(request.connectionId, 1, signature); + return new PrepareResponse(statementHandle, null); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public ExecuteResponse apply(ExecuteRequest request) { + PrepareAndExecuteRequest req = new PrepareAndExecuteRequest( + request.statementHandle.connectionId, request.statementHandle.id, null, -1); + return apply(req); + } + + @Override + public ExecuteResponse apply(PrepareAndExecuteRequest request) { + try { + Signature signature = JsonService.MAPPER.readValue(stubbedSignature, Signature.class); + return lookerExecuteResponse(request.connectionId, request.statementId, signature, + LookerFrame.create(1L)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } +} diff --git a/libs/looker-kotlin-sdk-a48011f.jar b/libs/looker-kotlin-sdk-a48011f.jar new file mode 100644 index 0000000000..d30e71faa6 Binary files /dev/null and b/libs/looker-kotlin-sdk-a48011f.jar differ