From 394a45d05d5760341d4c91169952d6e8411d745a Mon Sep 17 00:00:00 2001 From: zhouhongfa Date: Sat, 3 Aug 2019 12:00:33 +0800 Subject: [PATCH] =?UTF-8?q?add:=20=E5=AE=9E=E7=8E=B0=20querySql=20?= =?UTF-8?q?=E5=B1=9E=E6=80=A7=E6=8B=BC=E8=A3=85=EF=BC=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../JdbcDatasourceQueryController.java | 12 ++++++ .../com/wugui/dataxweb/dto/DataxJsonDto.java | 2 + .../service/JdbcDatasourceQueryService.java | 8 ++++ .../service/impl/DataxJsonServiceImpl.java | 2 + .../impl/JdbcDatasourceQueryServiceImpl.java | 12 ++++++ .../com/wugui/tool/datax/DataxJsonHelper.java | 7 ++++ .../tool/datax/reader/BaseReaderPlugin.java | 16 +++++--- .../com/wugui/tool/pojo/DataxPluginPojo.java | 4 ++ .../com/wugui/tool/query/BaseQueryTool.java | 41 +++++++++++++++++++ .../wugui/tool/query/QueryToolInterface.java | 2 + .../wugui/tool/datax/DataxJsonHelperTest.java | 3 +- .../wugui/tool/query/MySQLQueryToolTest.java | 10 ++++- 12 files changed, 112 insertions(+), 7 deletions(-) diff --git a/datax-web/src/main/java/com/wugui/dataxweb/controller/JdbcDatasourceQueryController.java b/datax-web/src/main/java/com/wugui/dataxweb/controller/JdbcDatasourceQueryController.java index 42e318f..26ded2d 100644 --- a/datax-web/src/main/java/com/wugui/dataxweb/controller/JdbcDatasourceQueryController.java +++ b/datax-web/src/main/java/com/wugui/dataxweb/controller/JdbcDatasourceQueryController.java @@ -53,4 +53,16 @@ public R> getColumns(Long datasourceId, String tableName) { return success(jdbcDatasourceQueryService.getColumns(datasourceId, tableName)); } + /** + * 根据数据源id和sql语句获取所有字段 + * + * @param datasourceId 数据源id + * @param querySql 表名 + * @return + */ + @GetMapping("/getColumnsByQuerySql") + @ApiOperation("根据数据源id和sql语句获取所有字段") + public R> getColumnsByQuerySql(Long datasourceId, String querySql) { + return success(jdbcDatasourceQueryService.getColumnsByQuerySql(datasourceId, querySql)); + } } diff --git a/datax-web/src/main/java/com/wugui/dataxweb/dto/DataxJsonDto.java b/datax-web/src/main/java/com/wugui/dataxweb/dto/DataxJsonDto.java index 9fa355f..2967beb 100644 --- a/datax-web/src/main/java/com/wugui/dataxweb/dto/DataxJsonDto.java +++ b/datax-web/src/main/java/com/wugui/dataxweb/dto/DataxJsonDto.java @@ -32,4 +32,6 @@ public class DataxJsonDto implements Serializable { private String whereParams; + private String querySql; + } diff --git a/datax-web/src/main/java/com/wugui/dataxweb/service/JdbcDatasourceQueryService.java b/datax-web/src/main/java/com/wugui/dataxweb/service/JdbcDatasourceQueryService.java index 102c64f..c555e25 100644 --- a/datax-web/src/main/java/com/wugui/dataxweb/service/JdbcDatasourceQueryService.java +++ b/datax-web/src/main/java/com/wugui/dataxweb/service/JdbcDatasourceQueryService.java @@ -29,4 +29,12 @@ public interface JdbcDatasourceQueryService { */ List getColumns(Long id, String tableName); + /** + * 根据 sql 语句获取字段 + * + * @param datasourceId + * @param querySql + * @return + */ + List getColumnsByQuerySql(Long datasourceId, String querySql); } diff --git a/datax-web/src/main/java/com/wugui/dataxweb/service/impl/DataxJsonServiceImpl.java b/datax-web/src/main/java/com/wugui/dataxweb/service/impl/DataxJsonServiceImpl.java index 0763afd..5378e46 100644 --- a/datax-web/src/main/java/com/wugui/dataxweb/service/impl/DataxJsonServiceImpl.java +++ b/datax-web/src/main/java/com/wugui/dataxweb/service/impl/DataxJsonServiceImpl.java @@ -34,6 +34,8 @@ public String buildJobJson(DataxJsonDto dataxJsonDto) { // reader JobJdbcDatasource readerDatasource = jobJdbcDatasourceService.getById(dataxJsonDto.getReaderDatasourceId()); + //querySql + dataxJsonHelper.setQuerySql(dataxJsonDto.getQuerySql()); //where if (StrUtil.isNotBlank(dataxJsonDto.getWhereParams())) { dataxJsonHelper.addWhereParams(dataxJsonDto.getWhereParams()); diff --git a/datax-web/src/main/java/com/wugui/dataxweb/service/impl/JdbcDatasourceQueryServiceImpl.java b/datax-web/src/main/java/com/wugui/dataxweb/service/impl/JdbcDatasourceQueryServiceImpl.java index 3a26f0a..0764bbb 100644 --- a/datax-web/src/main/java/com/wugui/dataxweb/service/impl/JdbcDatasourceQueryServiceImpl.java +++ b/datax-web/src/main/java/com/wugui/dataxweb/service/impl/JdbcDatasourceQueryServiceImpl.java @@ -49,4 +49,16 @@ public List getColumns(Long id, String tableName) { BaseQueryTool queryTool = QueryToolFactory.getByDbType(jdbcDatasource); return queryTool.getColumnNames(tableName); } + + @Override + public List getColumnsByQuerySql(Long datasourceId, String querySql) { + //获取数据源对象 + JobJdbcDatasource jdbcDatasource = jobJdbcDatasourceService.getById(datasourceId); + //queryTool组装 + if (ObjectUtil.isNull(jdbcDatasource)) { + return Lists.newArrayList(); + } + BaseQueryTool queryTool = QueryToolFactory.getByDbType(jdbcDatasource); + return queryTool.getColumnsByQuerySql(querySql); + } } diff --git a/datax-web/src/main/java/com/wugui/tool/datax/DataxJsonHelper.java b/datax-web/src/main/java/com/wugui/tool/datax/DataxJsonHelper.java index b743776..8443f18 100644 --- a/datax-web/src/main/java/com/wugui/tool/datax/DataxJsonHelper.java +++ b/datax-web/src/main/java/com/wugui/tool/datax/DataxJsonHelper.java @@ -61,6 +61,8 @@ public class DataxJsonHelper implements DataxJsonInterface { */ private List writerColumns; + private String querySql; + private BaseDataxPlugin readerPlugin; private BaseDataxPlugin writerPlugin; @@ -143,6 +145,7 @@ public Map buildReader() { dataxPluginPojo.setJdbcDatasource(readerDatasource); dataxPluginPojo.setTables(readerTables); dataxPluginPojo.setColumns(readerColumns); + dataxPluginPojo.setQuerySql(querySql); return readerPlugin.build(dataxPluginPojo); } @@ -164,4 +167,8 @@ public void setWriterPlugin(BaseDataxPlugin writerPlugin) { public void addWhereParams(String params) { extraParams.put("where", params); } + + public void setQuerySql(String querySql) { + this.querySql = querySql; + } } diff --git a/datax-web/src/main/java/com/wugui/tool/datax/reader/BaseReaderPlugin.java b/datax-web/src/main/java/com/wugui/tool/datax/reader/BaseReaderPlugin.java index 5f17d09..e479aae 100644 --- a/datax-web/src/main/java/com/wugui/tool/datax/reader/BaseReaderPlugin.java +++ b/datax-web/src/main/java/com/wugui/tool/datax/reader/BaseReaderPlugin.java @@ -1,5 +1,6 @@ package com.wugui.tool.datax.reader; +import cn.hutool.core.util.StrUtil; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import com.wugui.dataxweb.entity.JobJdbcDatasource; @@ -30,12 +31,17 @@ public Map build(DataxPluginPojo dataxPluginPojo) { JobJdbcDatasource jobJdbcDatasource = dataxPluginPojo.getJdbcDatasource(); parameterObj.put("username", jobJdbcDatasource.getJdbcUsername()); parameterObj.put("password", jobJdbcDatasource.getJdbcPassword()); - //列表 - parameterObj.put("column", dataxPluginPojo.getColumns()); - //判断是否有where - if (extraParams.containsKey("where")) { - parameterObj.put("where", extraParams.get("where")); + //判断是否是 querySql + if (StrUtil.isNotBlank(dataxPluginPojo.getQuerySql())) { + parameterObj.put("querySql", dataxPluginPojo.getQuerySql()); + } else { + //列表 + parameterObj.put("column", dataxPluginPojo.getColumns()); + //判断是否有where + if (extraParams.containsKey("where")) { + parameterObj.put("where", extraParams.get("where")); + } } Map connectionObj = Maps.newLinkedHashMap(); diff --git a/datax-web/src/main/java/com/wugui/tool/pojo/DataxPluginPojo.java b/datax-web/src/main/java/com/wugui/tool/pojo/DataxPluginPojo.java index c1d2f81..712d9e2 100644 --- a/datax-web/src/main/java/com/wugui/tool/pojo/DataxPluginPojo.java +++ b/datax-web/src/main/java/com/wugui/tool/pojo/DataxPluginPojo.java @@ -31,4 +31,8 @@ public class DataxPluginPojo { */ private JobJdbcDatasource jdbcDatasource; + /** + * querySql 属性，如果指定了，则优先于columns参数 + */ + private String querySql; } diff --git a/datax-web/src/main/java/com/wugui/tool/query/BaseQueryTool.java b/datax-web/src/main/java/com/wugui/tool/query/BaseQueryTool.java index 992f87e..34ac6a2 100644 --- a/datax-web/src/main/java/com/wugui/tool/query/BaseQueryTool.java +++ b/datax-web/src/main/java/com/wugui/tool/query/BaseQueryTool.java @@ -312,4 +312,45 @@ protected String getSQLQueryTables() { return sqlBuilder.getSQLQueryTables(); } + @Override + public List getColumnsByQuerySql(String querySql) { + + List res = Lists.newArrayList(); + Statement stmt = null; + try { + String sql = ""; + //拼装sql语句，在后面加上 where 1=0 即可 + sql = querySql.concat(" where 1=0"); + //判断是否已有where，如果是，则加 and 1=0 + //从最后一个 ) 开始找 where，或者整个语句找 + if (querySql.indexOf(")") != -1) { + if (querySql.substring(querySql.indexOf(")")).contains("where")) { + sql = querySql.concat(" and 1=0"); + } + } else { + if (querySql.contains("where")) { + sql = querySql.concat(" and 1=0"); + } + } + + logger.info("querySql: {}", sql); + + //获取所有字段 + stmt = connection.createStatement(); + ResultSet resultSet = stmt.executeQuery(sql); + ResultSetMetaData metaData = resultSet.getMetaData(); + + int columnCount = metaData.getColumnCount(); + for (int i = 1; i <= columnCount; i++) { + res.add(metaData.getColumnName(i)); + } +// logger.info("res: "); +// res.forEach(e -> logger.info(e.toString())); + } catch (SQLException e) { + e.printStackTrace(); + } finally { + JdbcUtils.close(stmt); + } + return res; + } } diff --git a/datax-web/src/main/java/com/wugui/tool/query/QueryToolInterface.java b/datax-web/src/main/java/com/wugui/tool/query/QueryToolInterface.java index 2efee15..056bb34 100644 --- a/datax-web/src/main/java/com/wugui/tool/query/QueryToolInterface.java +++ b/datax-web/src/main/java/com/wugui/tool/query/QueryToolInterface.java @@ -60,4 +60,6 @@ public interface QueryToolInterface { * @return2 */ public List getTableNames(); + + public List getColumnsByQuerySql(String querySql); } diff --git a/datax-web/src/test/java/com/wugui/tool/datax/DataxJsonHelperTest.java b/datax-web/src/test/java/com/wugui/tool/datax/DataxJsonHelperTest.java index 6bfe3a0..dd38f3d 100644 --- a/datax-web/src/test/java/com/wugui/tool/datax/DataxJsonHelperTest.java +++ b/datax-web/src/test/java/com/wugui/tool/datax/DataxJsonHelperTest.java @@ -16,7 +16,7 @@ private JobJdbcDatasource getReaderDatasource() { readerDatasource.setDatasourceName("z01_mysql_3306"); readerDatasource.setJdbcUsername("root"); readerDatasource.setJdbcPassword("root"); - readerDatasource.setJdbcUrl("jdbc:mysql://z01:3306/datax_web?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8"); + readerDatasource.setJdbcUrl("jdbc:mysql://localhost:3306/datax_web?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8"); readerDatasource.setJdbcDriverClass("com.mysql.jdbc.Driver"); return readerDatasource; } @@ -69,6 +69,7 @@ public void buildReader() { DataxJsonHelper dataxJsonHelper = new DataxJsonHelper(); dataxJsonHelper.initReader(getReaderDatasource(), ImmutableList.of("datax_plugin"), ImmutableList.of("id")); // dataxJsonHelper.addWhereParams("1=1"); + dataxJsonHelper.setQuerySql("select 1"); Map reader = dataxJsonHelper.buildReader(); System.out.println(JSONUtils.formatJson(reader)); } diff --git a/datax-web/src/test/java/com/wugui/tool/query/MySQLQueryToolTest.java b/datax-web/src/test/java/com/wugui/tool/query/MySQLQueryToolTest.java index 4c886a3..3da99d6 100644 --- a/datax-web/src/test/java/com/wugui/tool/query/MySQLQueryToolTest.java +++ b/datax-web/src/test/java/com/wugui/tool/query/MySQLQueryToolTest.java @@ -24,7 +24,7 @@ private void genMysqlDemo() { jdbcDatasource.setDatasourceName("z01_mysql_3306"); jdbcDatasource.setJdbcUsername("root"); jdbcDatasource.setJdbcPassword("root"); - jdbcDatasource.setJdbcUrl("jdbc:mysql://z01:3306/datax_web?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8"); + jdbcDatasource.setJdbcUrl("jdbc:mysql://localhost:3306/datax_web?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8"); jdbcDatasource.setJdbcDriverClass("com.mysql.jdbc.Driver"); } @@ -39,4 +39,12 @@ public void getColumnNames() { List columns = queryTool.getColumnNames("datax_plugin"); log.info(columns.toString()); } + + @Test + public void getColumnsByQuerySql() { + String querySql = "select l.log_file_path, c.name, c.job_group\n" + + "from job_log l left join job_config c on c.id = l.job_id where l.status = 1"; + List columns = queryTool.getColumnsByQuerySql(querySql); + log.info(columns.toString()); + } } \ No newline at end of file