add: 实现 querySql 属性拼装；

Kyofin · Aug 3, 2019 · 394a45d · 394a45d
1 parent 1809910
commit 394a45d
Show file tree

Hide file tree

Showing 12 changed files with 112 additions and 7 deletions.
diff --git a/datax-web/src/main/java/com/wugui/dataxweb/controller/JdbcDatasourceQueryController.java b/datax-web/src/main/java/com/wugui/dataxweb/controller/JdbcDatasourceQueryController.java
@@ -53,4 +53,16 @@ public R<List<String>> getColumns(Long datasourceId, String tableName) {
         return success(jdbcDatasourceQueryService.getColumns(datasourceId, tableName));
     }
 
+    /**
+     * 根据数据源id和sql语句获取所有字段
+     *
+     * @param datasourceId 数据源id
+     * @param querySql     表名
+     * @return
+     */
+    @GetMapping("/getColumnsByQuerySql")
+    @ApiOperation("根据数据源id和sql语句获取所有字段")
+    public R<List<String>> getColumnsByQuerySql(Long datasourceId, String querySql) {
+        return success(jdbcDatasourceQueryService.getColumnsByQuerySql(datasourceId, querySql));
+    }
 }
diff --git a/datax-web/src/main/java/com/wugui/dataxweb/dto/DataxJsonDto.java b/datax-web/src/main/java/com/wugui/dataxweb/dto/DataxJsonDto.java
@@ -32,4 +32,6 @@ public class DataxJsonDto implements Serializable {
 
     private String whereParams;
 
+    private String querySql;
+
 }
diff --git a/datax-web/src/main/java/com/wugui/dataxweb/service/JdbcDatasourceQueryService.java b/datax-web/src/main/java/com/wugui/dataxweb/service/JdbcDatasourceQueryService.java
@@ -29,4 +29,12 @@ public interface JdbcDatasourceQueryService {
      */
     List<String> getColumns(Long id, String tableName);
 
+    /**
+     * 根据 sql 语句获取字段
+     *
+     * @param datasourceId
+     * @param querySql
+     * @return
+     */
+    List<String> getColumnsByQuerySql(Long datasourceId, String querySql);
 }
diff --git a/datax-web/src/main/java/com/wugui/dataxweb/service/impl/DataxJsonServiceImpl.java b/datax-web/src/main/java/com/wugui/dataxweb/service/impl/DataxJsonServiceImpl.java
@@ -34,6 +34,8 @@ public String buildJobJson(DataxJsonDto dataxJsonDto) {
         // reader
         JobJdbcDatasource readerDatasource = jobJdbcDatasourceService.getById(dataxJsonDto.getReaderDatasourceId());
 
+        //querySql
+        dataxJsonHelper.setQuerySql(dataxJsonDto.getQuerySql());
         //where
         if (StrUtil.isNotBlank(dataxJsonDto.getWhereParams())) {
             dataxJsonHelper.addWhereParams(dataxJsonDto.getWhereParams());

diff --git a/datax-web/src/main/java/com/wugui/dataxweb/service/impl/JdbcDatasourceQueryServiceImpl.java b/datax-web/src/main/java/com/wugui/dataxweb/service/impl/JdbcDatasourceQueryServiceImpl.java
@@ -49,4 +49,16 @@ public List<String> getColumns(Long id, String tableName) {
         BaseQueryTool queryTool = QueryToolFactory.getByDbType(jdbcDatasource);
         return queryTool.getColumnNames(tableName);
     }
+
+    @Override
+    public List<String> getColumnsByQuerySql(Long datasourceId, String querySql) {
+        //获取数据源对象
+        JobJdbcDatasource jdbcDatasource = jobJdbcDatasourceService.getById(datasourceId);
+        //queryTool组装
+        if (ObjectUtil.isNull(jdbcDatasource)) {
+            return Lists.newArrayList();
+        }
+        BaseQueryTool queryTool = QueryToolFactory.getByDbType(jdbcDatasource);
+        return queryTool.getColumnsByQuerySql(querySql);
+    }
 }
diff --git a/datax-web/src/main/java/com/wugui/tool/datax/DataxJsonHelper.java b/datax-web/src/main/java/com/wugui/tool/datax/DataxJsonHelper.java
@@ -61,6 +61,8 @@ public class DataxJsonHelper implements DataxJsonInterface {
      */
     private List<String> writerColumns;
 
+    private String querySql;
+
     private BaseDataxPlugin readerPlugin;
 
     private BaseDataxPlugin writerPlugin;
@@ -143,6 +145,7 @@ public Map<String, Object> buildReader() {
         dataxPluginPojo.setJdbcDatasource(readerDatasource);
         dataxPluginPojo.setTables(readerTables);
         dataxPluginPojo.setColumns(readerColumns);
+        dataxPluginPojo.setQuerySql(querySql);
 
         return readerPlugin.build(dataxPluginPojo);
     }
@@ -164,4 +167,8 @@ public void setWriterPlugin(BaseDataxPlugin writerPlugin) {
     public void addWhereParams(String params) {
         extraParams.put("where", params);
     }
+
+    public void setQuerySql(String querySql) {
+        this.querySql = querySql;
+    }
 }
diff --git a/datax-web/src/main/java/com/wugui/tool/datax/reader/BaseReaderPlugin.java b/datax-web/src/main/java/com/wugui/tool/datax/reader/BaseReaderPlugin.java
@@ -1,5 +1,6 @@
 package com.wugui.tool.datax.reader;
 
+import cn.hutool.core.util.StrUtil;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Maps;
 import com.wugui.dataxweb.entity.JobJdbcDatasource;
@@ -30,12 +31,17 @@ public Map<String, Object> build(DataxPluginPojo dataxPluginPojo) {
         JobJdbcDatasource jobJdbcDatasource = dataxPluginPojo.getJdbcDatasource();
         parameterObj.put("username", jobJdbcDatasource.getJdbcUsername());
         parameterObj.put("password", jobJdbcDatasource.getJdbcPassword());
-        //列表
-        parameterObj.put("column", dataxPluginPojo.getColumns());
 
-        //判断是否有where
-        if (extraParams.containsKey("where")) {
-            parameterObj.put("where", extraParams.get("where"));
+        //判断是否是 querySql
+        if (StrUtil.isNotBlank(dataxPluginPojo.getQuerySql())) {
+            parameterObj.put("querySql", dataxPluginPojo.getQuerySql());
+        } else {
+            //列表
+            parameterObj.put("column", dataxPluginPojo.getColumns());
+            //判断是否有where
+            if (extraParams.containsKey("where")) {
+                parameterObj.put("where", extraParams.get("where"));
+            }
         }
 
         Map<String, Object> connectionObj = Maps.newLinkedHashMap();

diff --git a/datax-web/src/main/java/com/wugui/tool/pojo/DataxPluginPojo.java b/datax-web/src/main/java/com/wugui/tool/pojo/DataxPluginPojo.java
@@ -31,4 +31,8 @@ public class DataxPluginPojo {
      */
     private JobJdbcDatasource jdbcDatasource;
 
+    /**
+     * querySql 属性，如果指定了，则优先于columns参数
+     */
+    private String querySql;
 }
diff --git a/datax-web/src/main/java/com/wugui/tool/query/BaseQueryTool.java b/datax-web/src/main/java/com/wugui/tool/query/BaseQueryTool.java
@@ -312,4 +312,45 @@ protected String getSQLQueryTables() {
         return sqlBuilder.getSQLQueryTables();
     }
 
+    @Override
+    public List<String> getColumnsByQuerySql(String querySql) {
+
+        List<String> res = Lists.newArrayList();
+        Statement stmt = null;
+        try {
+            String sql = "";
+            //拼装sql语句，在后面加上 where 1=0 即可
+            sql = querySql.concat(" where 1=0");
+            //判断是否已有where，如果是，则加 and 1=0
+            //从最后一个 ) 开始找 where，或者整个语句找
+            if (querySql.indexOf(")") != -1) {
+                if (querySql.substring(querySql.indexOf(")")).contains("where")) {
+                    sql = querySql.concat(" and 1=0");
+                }
+            } else {
+                if (querySql.contains("where")) {
+                    sql = querySql.concat(" and 1=0");
+                }
+            }
+
+            logger.info("querySql: {}", sql);
+
+            //获取所有字段
+            stmt = connection.createStatement();
+            ResultSet resultSet = stmt.executeQuery(sql);
+            ResultSetMetaData metaData = resultSet.getMetaData();
+
+            int columnCount = metaData.getColumnCount();
+            for (int i = 1; i <= columnCount; i++) {
+                res.add(metaData.getColumnName(i));
+            }
+//            logger.info("res: ");
+//            res.forEach(e -> logger.info(e.toString()));
+        } catch (SQLException e) {
+            e.printStackTrace();
+        } finally {
+            JdbcUtils.close(stmt);
+        }
+        return res;
+    }
 }
diff --git a/datax-web/src/main/java/com/wugui/tool/query/QueryToolInterface.java b/datax-web/src/main/java/com/wugui/tool/query/QueryToolInterface.java
@@ -60,4 +60,6 @@ public interface QueryToolInterface {
      * @return2
      */
     public List<String> getTableNames();
+
+    public List<String> getColumnsByQuerySql(String querySql);
 }
diff --git a/datax-web/src/test/java/com/wugui/tool/datax/DataxJsonHelperTest.java b/datax-web/src/test/java/com/wugui/tool/datax/DataxJsonHelperTest.java
@@ -16,7 +16,7 @@ private JobJdbcDatasource getReaderDatasource() {
         readerDatasource.setDatasourceName("z01_mysql_3306");
         readerDatasource.setJdbcUsername("root");
         readerDatasource.setJdbcPassword("root");
-        readerDatasource.setJdbcUrl("jdbc:mysql://z01:3306/datax_web?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8");
+        readerDatasource.setJdbcUrl("jdbc:mysql://localhost:3306/datax_web?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8");
         readerDatasource.setJdbcDriverClass("com.mysql.jdbc.Driver");
         return readerDatasource;
     }
@@ -69,6 +69,7 @@ public void buildReader() {
         DataxJsonHelper dataxJsonHelper = new DataxJsonHelper();
         dataxJsonHelper.initReader(getReaderDatasource(), ImmutableList.of("datax_plugin"), ImmutableList.of("id"));
 //        dataxJsonHelper.addWhereParams("1=1");
+        dataxJsonHelper.setQuerySql("select 1");
         Map<String, Object> reader = dataxJsonHelper.buildReader();
         System.out.println(JSONUtils.formatJson(reader));
     }

diff --git a/datax-web/src/test/java/com/wugui/tool/query/MySQLQueryToolTest.java b/datax-web/src/test/java/com/wugui/tool/query/MySQLQueryToolTest.java
@@ -24,7 +24,7 @@ private void genMysqlDemo() {
         jdbcDatasource.setDatasourceName("z01_mysql_3306");
         jdbcDatasource.setJdbcUsername("root");
         jdbcDatasource.setJdbcPassword("root");
-        jdbcDatasource.setJdbcUrl("jdbc:mysql://z01:3306/datax_web?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8");
+        jdbcDatasource.setJdbcUrl("jdbc:mysql://localhost:3306/datax_web?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8");
         jdbcDatasource.setJdbcDriverClass("com.mysql.jdbc.Driver");
     }
 
@@ -39,4 +39,12 @@ public void getColumnNames() {
         List<String> columns = queryTool.getColumnNames("datax_plugin");
         log.info(columns.toString());
     }
+
+    @Test
+    public void getColumnsByQuerySql() {
+        String querySql = "select l.log_file_path, c.name, c.job_group\n" +
+                "from job_log l left join job_config c on c.id = l.job_id where l.status = 1";
+        List<String> columns = queryTool.getColumnsByQuerySql(querySql);
+        log.info(columns.toString());
+    }
 }
Original file line number	Diff line number	Diff line change
Expand Up		@@ -32,4 +32,6 @@ public class DataxJsonDto implements Serializable {

		private String whereParams;

		private String querySql;

		}