Skip to content
This repository has been archived by the owner on Jun 7, 2021. It is now read-only.

[TRAFODION-3183] fetch huge data give rise to core #1694

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+87 −49
Open

[TRAFODION-3183] fetch huge data give rise to core #1694

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 51 additions & 30 deletions core/conn/jdbcT4/src/main/java/org/trafodion/jdbc/t4/TrafT4ResultSet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,16 @@
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.LogRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

// ----------------------------------------------------------------------------
// This class partially implements the result set class as defined in
// java.sql.ResultSet.
// ----------------------------------------------------------------------------
public class TrafT4ResultSet extends TrafT4Handle implements java.sql.ResultSet {

private static final Logger LOG = LoggerFactory.getLogger(TrafT4ResultSet.class);
private static final long FETCH_BYTES_LIMIT = 1024 * 1024 * 1024;
// java.sql.ResultSet interface methods
public boolean absolute(int row) throws SQLException {
if (connection_.props_.t4Logger_.isLoggable(Level.FINE) == true) {
Expand Down Expand Up @@ -2772,7 +2775,7 @@ public boolean next() throws SQLException {
maxRows = 0;
queryTimeout = 0;
}

setFetchSizeIfExceedLimit();
if (maxRows == 0 || maxRows > totalRowsFetched_ + fetchSize_) {
maxRowCnt = fetchSize_;
} else {
Expand Down Expand Up @@ -2973,34 +2976,52 @@ public void setFetchDirection(int direction) throws SQLException {
}
}

public void setFetchSize(int rows) throws SQLException {
if (connection_.props_.t4Logger_.isLoggable(Level.FINE) == true) {
Object p[] = T4LoggingUtilities.makeParams(connection_.props_, rows);
connection_.props_.t4Logger_.logp(Level.FINE, "TrafT4ResultSet", "setFetchSize", "", p);
}
if (connection_.props_.getLogWriter() != null) {
LogRecord lr = new LogRecord(Level.FINE, "");
Object p[] = T4LoggingUtilities.makeParams(connection_.props_, rows);
lr.setParameters(p);
lr.setSourceClassName("TrafT4ResultSet");
lr.setSourceMethodName("setFetchSize");
T4LogFormatter lf = new T4LogFormatter();
String temp = lf.format(lr);
connection_.props_.getLogWriter().println(temp);
}
if (isClosed_) {
throw TrafT4Messages.createSQLException(connection_.props_, connection_.getLocale(), "invalid_cursor_state",
null);
}
if (rows < 0) {
throw TrafT4Messages.createSQLException(connection_.props_, connection_.getLocale(), "invalid_fetch_size",
null);
} else if (rows == 0) {
fetchSize_ = DEFAULT_FETCH_SIZE;
} else {
fetchSize_ = rows;
}
}
public void setFetchSize(int rows) throws SQLException {
if (connection_.props_.t4Logger_.isLoggable(Level.FINE) == true) {
Object p[] = T4LoggingUtilities.makeParams(connection_.props_, rows);
connection_.props_.t4Logger_.logp(Level.FINE, "TrafT4ResultSet", "setFetchSize", "", p);
}
if (connection_.props_.getLogWriter() != null) {
LogRecord lr = new LogRecord(Level.FINE, "");
Object p[] = T4LoggingUtilities.makeParams(connection_.props_, rows);
lr.setParameters(p);
lr.setSourceClassName("TrafT4ResultSet");
lr.setSourceMethodName("setFetchSize");
T4LogFormatter lf = new T4LogFormatter();
String temp = lf.format(lr);
connection_.props_.getLogWriter().println(temp);
}
if (isClosed_) {
throw TrafT4Messages.createSQLException(connection_.props_, connection_.getLocale(),
"invalid_cursor_state", null);
}
if (rows < 0) {
throw TrafT4Messages.createSQLException(connection_.props_, connection_.getLocale(),
"invalid_fetch_size", null);
} else if (rows == 0) {
fetchSize_ = DEFAULT_FETCH_SIZE;
} else {
fetchSize_ = rows;
}

setFetchSizeIfExceedLimit();
}

/**
* if (row width) * (fetch rows) too large, there will have core in server side. once fetch
* bytes bigger than 1GB, divide it into several times to fetch, each time fetch bytes less than
* 1GB.
*/
private void setFetchSizeIfExceedLimit() {
if (outputDesc_ != null && outputDesc_[0] != null) {
long rowLength = outputDesc_[0].rowLength_;
long fetchBytes = rowLength * fetchSize_;
if (fetchBytes >= FETCH_BYTES_LIMIT) {
fetchSize_ = (int) Math.ceil(FETCH_BYTES_LIMIT / (double) rowLength);
LOG.trace("Fetch size exceed limit, change it to <{}>.", fetchSize_);
}
}
}

public void updateArray(int columnIndex, Array x) throws SQLException {
if (connection_.props_.t4Logger_.isLoggable(Level.FINE) == true) {
Expand Down
18 changes: 18 additions & 0 deletions dcs/src/test/jdbc_test/pom.xml.template
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,5 +64,23 @@
<scope>system</scope>
<systemPath>MY_JDBC_SYS_PATH</systemPath>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.5</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.5</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>
37 changes: 18 additions & 19 deletions tests/phx/pom.xml.template
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,24 +174,6 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.5</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.5</version>
<scope>test</scope>
</dependency>

<!-- Trafodion specific test dependencies -->

Expand Down Expand Up @@ -247,7 +229,24 @@
<version>2.6</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.5</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.5</version>
<scope>test</scope>
</dependency>
</dependencies>

<build>
Expand Down