Remove teradata14 inheritance (#191)

google · Apr 11, 2023 · b656fe4 · b656fe4
1 parent 2319454
commit b656fe4
Show file tree

Hide file tree

Showing 6 changed files with 134 additions and 31 deletions.
diff --git a/.../edwmigration/dumper/application/dumper/connector/teradata/AbstractTeradataConnector.java b/.../edwmigration/dumper/application/dumper/connector/teradata/AbstractTeradataConnector.java
@@ -16,6 +16,7 @@
  */
 package com.google.edwmigration.dumper.application.dumper.connector.teradata;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.io.ByteSink;
 import com.google.edwmigration.dumper.application.dumper.ConnectorArguments;
@@ -36,7 +37,11 @@
 import java.sql.Driver;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
 import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import javax.annotation.Nonnull;
 import javax.sql.DataSource;
 import org.slf4j.Logger;
@@ -76,6 +81,10 @@ public abstract class AbstractTeradataConnector extends AbstractJdbcConnector {
   private static final Logger LOG = LoggerFactory.getLogger(AbstractTeradataConnector.class);
 
   public static final int OPT_PORT_DEFAULT = 1025;
+  protected static final DateTimeFormatter SQL_FORMAT =
+      DateTimeFormatter.ISO_OFFSET_DATE_TIME.withZone(ZoneOffset.UTC);
+  @VisibleForTesting /* pp */ static final String DEF_LOG_TABLE = "dbc.DBQLogTbl";
+  @VisibleForTesting /* pp */ static final String DEF_QUERY_TABLE = "dbc.DBQLSQLTbl";
 
   protected static class TeradataJdbcSelectTask extends JdbcSelectTask {
 
@@ -132,6 +141,19 @@ protected Void doInConnection(
     }
   }
 
+  /** This is shared between all instances of TeradataLogsJdbcTask. */
+  protected static class SharedState {
+    /**
+     * Whether a particular expression is valid against the particular target Teradata version. This
+     * is a concurrent Map of immutable objects, so is threadsafe overall.
+     */
+    protected final ConcurrentMap<String, Boolean> expressionValidity = new ConcurrentHashMap<>();
+  }
+
+  protected static boolean isQueryTable(@Nonnull String expression) {
+    return expression.startsWith("ST.");
+  }
+
   /* pp */ AbstractTeradataConnector(@Nonnull String name) {
     super(name);
   }

diff --git a/...le/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java b/...le/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java
@@ -18,6 +18,9 @@
 
 import com.google.auto.service.AutoService;
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicates;
+import com.google.common.io.ByteSink;
 import com.google.edwmigration.dumper.application.dumper.ConnectorArguments;
 import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
 import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentQueryLogDays;
@@ -27,27 +30,38 @@
 import com.google.edwmigration.dumper.application.dumper.connector.LogsConnector;
 import com.google.edwmigration.dumper.application.dumper.connector.ZonedInterval;
 import com.google.edwmigration.dumper.application.dumper.connector.ZonedIntervalIterable;
+import com.google.edwmigration.dumper.application.dumper.handle.JdbcHandle;
+import com.google.edwmigration.dumper.application.dumper.task.AbstractJdbcTask;
 import com.google.edwmigration.dumper.application.dumper.task.DumpMetadataTask;
 import com.google.edwmigration.dumper.application.dumper.task.FormatTask;
 import com.google.edwmigration.dumper.application.dumper.task.Task;
+import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
 import com.google.edwmigration.dumper.plugin.ext.jdk.annotation.Description;
 import com.google.edwmigration.dumper.plugin.lib.dumper.spi.TeradataLogsDumpFormat;
+import java.sql.Connection;
+import java.sql.SQLException;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.function.Function;
 import java.util.function.Predicate;
 import javax.annotation.Nonnull;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.dao.DataAccessException;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.jdbc.core.ResultSetExtractor;
 
 /** */
 @AutoService({Connector.class, LogsConnector.class})
 @Description("Dumps logs from Teradata version <=14.")
 @RespectsArgumentQueryLogDays
 @RespectsArgumentQueryLogStart
 @RespectsArgumentQueryLogEnd
-public class Teradata14LogsConnector extends TeradataLogsConnector {
+public class Teradata14LogsConnector extends AbstractTeradataConnector
+    implements LogsConnector, TeradataLogsDumpFormat {
 
   private static final Logger LOG = LoggerFactory.getLogger(Teradata14LogsConnector.class);
 
@@ -70,7 +84,95 @@ public Teradata14LogsConnector() {
     super("teradata14-logs");
   }
 
-  private static class LSqlQueryFactory extends TeradataLogsJdbcTask {
+  private abstract static class Teradata14LogsJdbcTask extends AbstractJdbcTask<Void> {
+
+    protected static String EXPRESSION_VALIDITY_QUERY = "SELECT TOP 1 %s FROM %s";
+
+    protected final SharedState state;
+    protected final String logTable;
+    protected final String queryTable;
+    protected final List<String> conditions;
+    protected final ZonedInterval interval;
+    protected final List<String> orderBy;
+
+    public Teradata14LogsJdbcTask(
+        @Nonnull String targetPath,
+        SharedState state,
+        String logTable,
+        String queryTable,
+        List<String> conditions,
+        ZonedInterval interval) {
+      this(targetPath, state, logTable, queryTable, conditions, interval, Collections.emptyList());
+    }
+
+    protected Teradata14LogsJdbcTask(
+        @Nonnull String targetPath,
+        SharedState state,
+        String logTable,
+        String queryTable,
+        List<String> conditions,
+        ZonedInterval interval,
+        List<String> orderBy) {
+      super(targetPath);
+      this.state = Preconditions.checkNotNull(state, "SharedState was null.");
+      this.logTable = logTable;
+      this.queryTable = queryTable;
+      this.conditions = conditions;
+      this.interval = interval;
+      this.orderBy = orderBy;
+    }
+
+    @Override
+    protected Void doInConnection(
+        TaskRunContext context, JdbcHandle jdbcHandle, ByteSink sink, Connection connection)
+        throws SQLException {
+      String sql = getSql(jdbcHandle);
+      ResultSetExtractor<Void> rse = newCsvResultSetExtractor(sink, -1);
+      return doSelect(connection, rse, sql);
+    }
+
+    @Nonnull
+    private String getSql(@Nonnull JdbcHandle handle) {
+      Function<String, Boolean> validator =
+          expression -> isValid(handle.getJdbcTemplate(), expression);
+      Predicate<String> predicate =
+          expression -> state.expressionValidity.computeIfAbsent(expression, validator);
+      String sql = getSql(predicate);
+      // LOG.debug("SQL is " + sql);
+      return sql;
+    }
+
+    @Nonnull
+    protected abstract String getSql(@Nonnull Predicate<? super String> predicate);
+
+    /**
+     * Runs a test query to check whether a given projection expression is legal on this Teradata
+     * instance.
+     */
+    @Nonnull
+    private Boolean isValid(@Nonnull JdbcTemplate template, @Nonnull String expression) {
+      String table = isQueryTable(expression) ? queryTable + " ST" : logTable + " L";
+      String sql = String.format(EXPRESSION_VALIDITY_QUERY, expression, table);
+      LOG.info("Checking legality of projection expression '{}' using query: {}", expression, sql);
+      try {
+        template.query(sql, rs -> {});
+        return Boolean.TRUE;
+      } catch (DataAccessException e) {
+        LOG.info(
+            "Attribute '{}' is absent, will use NULL in projection: {}",
+            expression,
+            e.getMessage());
+        return Boolean.FALSE;
+      }
+    }
+
+    @Override
+    public String toString() {
+      return getSql(Predicates.alwaysTrue());
+    }
+  }
+
+  private static class LSqlQueryFactory extends Teradata14LogsJdbcTask {
 
     public LSqlQueryFactory(
         String targetPath,
@@ -84,7 +186,7 @@ public LSqlQueryFactory(
 
     @Override
     @Nonnull
-    String getSql(@Nonnull Predicate<? super String> predicate) {
+    protected String getSql(@Nonnull Predicate<? super String> predicate) {
       StringBuilder buf = new StringBuilder("SELECT ");
 
       String separator = "";
@@ -115,7 +217,7 @@ String getSql(@Nonnull Predicate<? super String> predicate) {
     }
   }
 
-  private static class LogQueryFactory extends TeradataLogsJdbcTask {
+  private static class LogQueryFactory extends Teradata14LogsJdbcTask {
 
     public LogQueryFactory(
         String targetPath,
@@ -129,7 +231,7 @@ public LogQueryFactory(
 
     @Override
     @Nonnull
-    String getSql(@Nonnull Predicate<? super String> predicate) {
+    protected String getSql(@Nonnull Predicate<? super String> predicate) {
       StringBuilder buf = new StringBuilder("SELECT ");
 
       String separator = "";

diff --git a/...igration/dumper/application/dumper/connector/teradata/TeradataAssessmentLogsJdbcTask.java b/...igration/dumper/application/dumper/connector/teradata/TeradataAssessmentLogsJdbcTask.java
@@ -19,7 +19,7 @@
 import static com.google.edwmigration.dumper.application.dumper.connector.teradata.TeradataUtils.createTimestampExpression;
 
 import com.google.edwmigration.dumper.application.dumper.connector.ZonedInterval;
-import com.google.edwmigration.dumper.application.dumper.connector.teradata.TeradataLogsConnector.SharedState;
+import com.google.edwmigration.dumper.application.dumper.connector.teradata.AbstractTeradataConnector.SharedState;
 import java.util.List;
 import java.util.function.Predicate;
 import javax.annotation.Nonnull;

diff --git a/...ogle/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnector.java b/...ogle/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnector.java
@@ -19,7 +19,6 @@
 import static com.google.common.collect.ImmutableList.toImmutableList;
 
 import com.google.auto.service.AutoService;
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.edwmigration.dumper.application.dumper.ConnectorArguments;
 import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
@@ -41,8 +40,6 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
 import javax.annotation.Nonnull;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
@@ -62,32 +59,14 @@ public class TeradataLogsConnector extends AbstractTeradataConnector
     implements LogsConnector, TeradataLogsDumpFormat {
 
   private static final Logger LOG = LoggerFactory.getLogger(TeradataLogsConnector.class);
-
-  @VisibleForTesting /* pp */ static final String DEF_LOG_TABLE = "dbc.DBQLogTbl";
-  /* pp */ static final String ASSESSMENT_DEF_LOG_TABLE = "dbc.QryLogV";
-  @VisibleForTesting /* pp */ static final String DEF_QUERY_TABLE = "dbc.DBQLSQLTbl";
+  private static final String ASSESSMENT_DEF_LOG_TABLE = "dbc.QryLogV";
 
   private static final String DEF_UTILITY_TABLE = "dbc.DBQLUtilityTbl";
 
   public TeradataLogsConnector() {
     super("teradata-logs");
   }
 
-  // to proxy for Terdata14LogsConnector
-  protected TeradataLogsConnector(@Nonnull String name) {
-    super(name);
-  }
-
-  /** This is shared between all instances of TeradataLogsJdbcTask. */
-  protected static class SharedState {
-
-    /**
-     * Whether a particular expression is valid against the particular target Teradata version. This
-     * is a concurrent Map of immutable objects, so is threadsafe overall.
-     */
-    protected final ConcurrentMap<String, Boolean> expressionValidity = new ConcurrentHashMap<>();
-  }
-
   private ImmutableList<TeradataJdbcSelectTask> createTimeSeriesTasks(ZonedInterval interval) {
     return ImmutableList.of("ResUsageScpu", "ResUsageSpma").stream()
         .map(

diff --git a/...oogle/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java b/...oogle/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java
@@ -24,7 +24,7 @@
 import com.google.common.base.Predicates;
 import com.google.common.io.ByteSink;
 import com.google.edwmigration.dumper.application.dumper.connector.ZonedInterval;
-import com.google.edwmigration.dumper.application.dumper.connector.teradata.TeradataLogsConnector.SharedState;
+import com.google.edwmigration.dumper.application.dumper.connector.teradata.AbstractTeradataConnector.SharedState;
 import com.google.edwmigration.dumper.application.dumper.handle.JdbcHandle;
 import com.google.edwmigration.dumper.application.dumper.task.AbstractJdbcTask;
 import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;
@@ -82,7 +82,7 @@ public class TeradataLogsJdbcTask extends AbstractJdbcTask<Void> {
       };
 
   private static final Logger LOG = LoggerFactory.getLogger(TeradataLogsConnector.class);
-  @VisibleForTesting public static String EXPRESSION_VALIDITY_QUERY = "SELECT TOP 1 %s FROM %s";
+  @VisibleForTesting /* pp */ static String EXPRESSION_VALIDITY_QUERY = "SELECT TOP 1 %s FROM %s";
   protected final SharedState state;
   protected final String logTable;
   protected final String queryTable;

diff --git a/...dwmigration/dumper/application/dumper/connector/teradata/TeradataUtilityLogsJdbcTask.java b/...dwmigration/dumper/application/dumper/connector/teradata/TeradataUtilityLogsJdbcTask.java
@@ -22,7 +22,7 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.io.ByteSink;
 import com.google.edwmigration.dumper.application.dumper.connector.ZonedInterval;
-import com.google.edwmigration.dumper.application.dumper.connector.teradata.TeradataLogsConnector.SharedState;
+import com.google.edwmigration.dumper.application.dumper.connector.teradata.AbstractTeradataConnector.SharedState;
 import com.google.edwmigration.dumper.application.dumper.handle.JdbcHandle;
 import com.google.edwmigration.dumper.application.dumper.task.AbstractJdbcTask;
 import com.google.edwmigration.dumper.application.dumper.task.TaskRunContext;