Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use recommended semantics for getObject of date/time types #181

Merged
merged 1 commit into from
Dec 11, 2023
Merged

Use recommended semantics for getObject of date/time types #181

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
160 changes: 52 additions & 108 deletions src/main/java/net/starschema/clouddb/jdbc/BQForwardOnlyResultSet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,6 @@
import java.sql.Statement;
import java.sql.Time;
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.*;
import javax.annotation.Nullable;
import org.slf4j.Logger;
Expand Down Expand Up @@ -127,9 +123,6 @@ public class BQForwardOnlyResultSet implements java.sql.ResultSet {
*/
private int Cursor = -1;

private final DateTimeFormatter TIMESTAMP_FORMATTER =
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS").withZone(ZoneId.of("UTC"));

public BQForwardOnlyResultSet(
Bigquery bigquery,
String projectId,
Expand Down Expand Up @@ -248,7 +241,7 @@ public BQForwardOnlyResultSet(
*/
public Object getObject(int columnIndex) throws SQLException {

String result = getString(columnIndex, false);
String result = rawString(columnIndex);

if (this.wasNull()) {
return null;
Expand All @@ -274,16 +267,26 @@ public Object getObject(int columnIndex) throws SQLException {
if (Columntype.equals("INTEGER")) {
return toLong(result);
}
if (Columntype.equals("DATETIME")) {
// A BigQuery DATETIME is essentially defined by its string representation;
// the "clock-calendar parameters" comprising year, month, day, hour, minute, etc.
// On the other hand, a [java.sql.Timestamp] object is defined as a global instant, similar
// to BQ TIMESTAMP. It has a [toString] method that interprets that instant in the system
// default time zone. Thus, in order to produce a [Timestamp] object whose [toString] method
// has the correct result, we must adjust the value of the instant according to the system
// default time zone (passing a null Calendar uses the system default).
return DateTimeUtils.parseDateTime(result, null);
}
if (Columntype.equals("TIMESTAMP")) {
return toTimestamp(result, null);
// A BigQuery TIMESTAMP is defined as a global instant in time, so when we create the
// [java.sql.Timestamp] object to represent it, we must not make any time zone adjustment.
return DateTimeUtils.parseTimestamp(result);
}
if (Columntype.equals("DATE")) {
return toDate(result, null);
return DateTimeUtils.parseDate(result, null);
}
if (Columntype.equals("DATETIME")) {
// Date time represents a "clock face" time and so should NOT be processed into an actual
// time
return result;
if (Columntype.equals("TIME")) {
return DateTimeUtils.parseTime(result, null);
}
if (Columntype.equals("NUMERIC")) {
return toBigDecimal(result);
Expand Down Expand Up @@ -321,71 +324,13 @@ private Long toLong(String value) throws SQLException {
}
}

/** Parse date/time types */
private Timestamp toTimestamp(String value, Calendar cal) throws SQLException {
try {
long dbValue =
new BigDecimal(value)
.movePointRight(3)
.longValue(); // movePointRight(3) = *1000 (done before rounding) - from seconds
// (BigQuery specifications) to milliseconds (required by java).
// Precision under millisecond is discarded (BigQuery supports
// micro-seconds)
if (cal == null) {
cal =
Calendar.getInstance(
TimeZone.getTimeZone(
"UTC")); // The time zone of the server that host the JVM should NOT impact the
// results. Use UTC calendar instead (which wont apply any correction,
// whatever the time zone of the data)
}

Calendar dbCal = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
dbCal.setTimeInMillis(dbValue);
cal.set(Calendar.YEAR, dbCal.get(Calendar.YEAR));
cal.set(Calendar.MONTH, dbCal.get(Calendar.MONTH));
cal.set(Calendar.DAY_OF_MONTH, dbCal.get(Calendar.DAY_OF_MONTH));
cal.set(Calendar.HOUR_OF_DAY, dbCal.get(Calendar.HOUR_OF_DAY));
cal.set(Calendar.MINUTE, dbCal.get(Calendar.MINUTE));
cal.set(Calendar.SECOND, dbCal.get(Calendar.SECOND));
cal.set(Calendar.MILLISECOND, dbCal.get(Calendar.MILLISECOND));
return new Timestamp(cal.getTime().getTime());
} catch (NumberFormatException e) {
// before giving up, check to see if we've been given a 'time' value without a
// date, e.g. from current_time(), and if we have, try to parse it
try {
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String dateStringFromValue = ("1970-01-01 " + value).substring(0, 19);
java.util.Date parsedDate = dateFormat.parse(dateStringFromValue);
return new java.sql.Timestamp(parsedDate.getTime());
} catch (Exception e2) {
throw new BQSQLException(e);
}
}
}

// Secondary converters

/** Parse integral or floating types with (virtually) infinite precision */
private BigDecimal toBigDecimal(String value) {
return new BigDecimal(value);
}

static Date toDate(String value, Calendar cal) throws SQLException {
// Dates in BigQuery come back in the YYYY-MM-DD format
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
try {
java.util.Date date = sdf.parse(value);
return new java.sql.Date(date.getTime());
} catch (java.text.ParseException e) {
throw new BQSQLException(e);
}
}

private Time toTime(String value, Calendar cal) throws SQLException {
return new java.sql.Time(toTimestamp(value, cal).getTime());
}

@Override
/**
* Returns the current rows Data at the given index as String
Expand All @@ -396,10 +341,18 @@ private Time toTime(String value, Calendar cal) throws SQLException {
* is unsupported
*/
public String getString(int columnIndex) throws SQLException {
return getString(columnIndex, true);
String rawString = rawString(columnIndex);

BQResultsetMetaData metadata = getBQResultsetMetaData();
if (metadata.getColumnTypeName(columnIndex).equals("TIMESTAMP")
&& !metadata.getColumnMode(columnIndex).equals("REPEATED")) {
return DateTimeUtils.formatTimestamp(rawString);
}

return rawString;
}

private String getString(int columnIndex, boolean formatTimestamps) throws SQLException {
private String rawString(int columnIndex) throws SQLException {
// to make the logfiles smaller!
// logger.debug("Function call getString columnIndex is: " + String.valueOf(columnIndex));
this.closestrm();
Expand All @@ -410,21 +363,21 @@ private String getString(int columnIndex, boolean formatTimestamps) throws SQLEx
throw new BQSQLException("ColumnIndex is not valid");
}

if (this.rowsofResult == null) throw new BQSQLException("Invalid position!");
if (this.rowsofResult == null) {
throw new BQSQLException("Invalid position!");
}

// Since the results came from BigQuery's JSON API,
// the only types we'll ever see for resultObject are JSON-supported types,
// i.e. strings, numbers, arrays, booleans, or null.
// Many standard Java types, like timestamps, will be represented as strings.
Object resultObject = this.rowsofResult.get(this.Cursor).getF().get(columnIndex - 1).getV();

if (Data.isNull(resultObject)) {
this.wasnull = true;
return null;
}
this.wasnull = false;
if (!this.getBQResultsetMetaData().getColumnMode(columnIndex).equals("REPEATED")
&& formatTimestamps
&& getMetaData().getColumnTypeName(columnIndex).equals("TIMESTAMP")) {
Instant instant =
Instant.ofEpochMilli(
(new BigDecimal((String) resultObject).movePointRight(3)).longValue());
return TIMESTAMP_FORMATTER.format(instant);
}
if (resultObject instanceof List || resultObject instanceof Map) {
Object resultTransformedWithSchema =
smartTransformResult(resultObject, schema.getFields().get(columnIndex - 1));
Expand Down Expand Up @@ -913,11 +866,7 @@ public String getCursorName() throws SQLException {
/** {@inheritDoc} */
@Override
public Date getDate(int columnIndex) throws SQLException {
String value = this.getString(columnIndex);
if (this.wasNull()) {
return null;
}
return toDate(value, null);
return getDate(columnIndex, null);
}

/** {@inheritDoc} */
Expand All @@ -927,14 +876,13 @@ public Date getDate(int columnIndex, Calendar cal) throws SQLException {
if (this.wasNull()) {
return null;
}
return toDate(value, cal);
return DateTimeUtils.parseDate(value, cal);
}

/** {@inheritDoc} */
@Override
public Date getDate(String columnLabel) throws SQLException {
int columnIndex = this.findColumn(columnLabel);
return this.getDate(columnIndex);
return getDate(columnLabel, null);
}

/** {@inheritDoc} */
Expand Down Expand Up @@ -1336,11 +1284,7 @@ public String getString(String columnLabel) throws SQLException {
/** {@inheritDoc} */
@Override
public Time getTime(int columnIndex) throws SQLException {
String value = this.getString(columnIndex);
if (this.wasNull()) {
return null;
}
return toTime(value, null);
return getTime(columnIndex, null);
}

/** {@inheritDoc} */
Expand All @@ -1350,14 +1294,13 @@ public Time getTime(int columnIndex, Calendar cal) throws SQLException {
if (this.wasNull()) {
return null;
}
return toTime(value, cal);
return DateTimeUtils.parseTime(value, cal);
}

/** {@inheritDoc} */
@Override
public Time getTime(String columnLabel) throws SQLException {
int columnIndex = this.findColumn(columnLabel);
return this.getTime(columnIndex);
return getTime(columnLabel, null);
}

/** {@inheritDoc} */
Expand All @@ -1370,28 +1313,29 @@ public Time getTime(String columnLabel, Calendar cal) throws SQLException {
/** {@inheritDoc} */
@Override
public Timestamp getTimestamp(int columnIndex) throws SQLException {
String value = this.getString(columnIndex);
if (this.wasNull()) {
return null;
}
return toTimestamp(value, null);
return getTimestamp(columnIndex, null);
}

/** {@inheritDoc} */
@Override
public Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException {
String value = this.getString(columnIndex);
String value = rawString(columnIndex);
if (this.wasNull()) {
return null;
}
return toTimestamp(value, cal);
// Both the TIMESTAMP and DATETIME objects support JDBC's getTimestamp method.
// DATETIME in BigQuery is analogous to TIMESTAMP in ISO SQL.
// TIMESTAMP in BigQuery is analogous to TIMESTAMP WITH LOCAL TIME ZONE in ISO SQL.
String columnType = this.schema.getFields().get(columnIndex - 1).getType();
return "TIMESTAMP".equals(columnType)
? DateTimeUtils.parseTimestamp(value)
: DateTimeUtils.parseDateTime(value, cal);
}

/** {@inheritDoc} */
@Override
public Timestamp getTimestamp(String columnLabel) throws SQLException {
int columnIndex = this.findColumn(columnLabel);
return this.getTimestamp(columnIndex);
return getTimestamp(columnLabel, null);
}

/** {@inheritDoc} */
Expand Down
31 changes: 19 additions & 12 deletions src/main/java/net/starschema/clouddb/jdbc/BQScrollableResultSet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,6 @@
*/
package net.starschema.clouddb.jdbc;

import static net.starschema.clouddb.jdbc.BQForwardOnlyResultSet.toDate;

import com.google.api.client.util.Data;
import com.google.api.services.bigquery.model.BiEngineReason;
import com.google.api.services.bigquery.model.GetQueryResultsResponse;
Expand All @@ -36,7 +34,6 @@
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Timestamp;
import java.util.List;
import javax.annotation.Nullable;

Expand Down Expand Up @@ -204,21 +201,31 @@ public Object getObject(int columnIndex) throws SQLException {
if (Columntype.equals("INTEGER")) {
return Long.parseLong(result);
}
if (Columntype.equals("DATETIME")) {
// A BigQuery DATETIME is essentially defined by its string representation;
// the "clock-calendar parameters" comprising year, month, day, hour, minute, etc.
// On the other hand, a [java.sql.Timestamp] object is defined as a global instant,
// similar to BQ TIMESTAMP. It has a [toString] method that interprets that instant in the
// system default time zone. Thus, in order to produce a [Timestamp] object whose
// [toString] method has the correct result, we must adjust the value of the instant
// according to the system default time zone (passing a null Calendar uses the system
// default).
return DateTimeUtils.parseDateTime(result, null);
}
if (Columntype.equals("TIMESTAMP")) {
long val = new BigDecimal(result).longValue() * 1000;
return new Timestamp(val);
// A BigQuery TIMESTAMP is defined as a global instant in time, so when we create the
// [java.sql.Timestamp] object to represent it, we must not make any time zone adjustment.
return DateTimeUtils.parseTimestamp(result);
}
if (Columntype.equals("DATETIME")) {
// Date time represents a "clock face" time and so should NOT be processed into an actual
// time
return result;
if (Columntype.equals("DATE")) {
return DateTimeUtils.parseDate(result, null);
}
if (Columntype.equals("TIME")) {
return DateTimeUtils.parseTime(result, null);
}
if (Columntype.equals("NUMERIC")) {
return new BigDecimal(result);
}
if (Columntype.equals("DATE")) {
return toDate(result, null);
}
throw new BQSQLException("Unsupported Type");
} catch (NumberFormatException e) {
throw new BQSQLException(e);
Expand Down
Loading
Loading