From bc5f1f700bf00abb51672acbfe5ac70b6e9384bc Mon Sep 17 00:00:00 2001 From: "xyqiao@vip.qq.com" Date: Sun, 6 Sep 2020 22:03:41 +0800 Subject: [PATCH] Fixing sensitive information --- .../schedulis/common/utils/HttpUtils.java | 47 -------- .../test/java/azkaban/db/AzDBTestUtility.java | 6 +- .../datacheck/test/DatacheckTest.java | 109 ------------------ .../schedulis/jobtype/util/EventChecker.java | 6 +- ...adoopJobUtilsFindApplicationIdFromLog.java | 2 - bin/upgrade/upgrade.sh | 4 +- 6 files changed, 8 insertions(+), 166 deletions(-) delete mode 100644 azkaban-jobtype/src/main/java/com/webank/wedatasphere/schedulis/datacheck/test/DatacheckTest.java diff --git a/azkaban-common/src/main/java/com/webank/wedatasphere/schedulis/common/utils/HttpUtils.java b/azkaban-common/src/main/java/com/webank/wedatasphere/schedulis/common/utils/HttpUtils.java index ce2a346..094e293 100755 --- a/azkaban-common/src/main/java/com/webank/wedatasphere/schedulis/common/utils/HttpUtils.java +++ b/azkaban-common/src/main/java/com/webank/wedatasphere/schedulis/common/utils/HttpUtils.java @@ -333,53 +333,6 @@ public static String getLinuxLocalIp(Logger log) { public static void main(String[] args) { - Map resultMap = new HashMap(); - String maskUrl = "http://10.255.10.90:8087/api/v1/mask-status?"; - - RequestBody requestBody = new FormBody.Builder() - .add("targetDb", "bdp_test_ods_mask") - .add("targetTable", "ccpd_dump") - .add("partition", "dcn_id=UA0/type_id=6042/ip=10.240.228.31/ds=2015-04-06") - .build(); - - FormBody.Builder formBuilder = new FormBody.Builder(); - - Map map = new HashMap<>(); - map.put("targetDb", "bdp_test_ods_mask"); - map.put("targetTable", "ccpd_dump"); - //map.put("partition", "ds=20180925"); - map.put("partition", "ds\\=20180925"); - - //String params = gson.toJson(map); - MediaType JSON = MediaType.parse("application/json; charset=utf-8"); - - Iterator iterator = map.keySet().iterator(); - String key = ""; - while(iterator.hasNext()){ - key = iterator.next().toString(); - formBuilder.add(key, map.get(key)); - } - - RequestBody requestBody2 = formBuilder.build(); - - Properties props = new Properties(); -// props.put(DataChecker.MASK_APP_ID, "wtss"); -// props.put(DataChecker.MASK_APP_TOKEN, "20a0ccdfc0"); -// -// Map dataMap = HttpUtils.initSelectParams(props); -// -// try { -// String result = HttpUtils.httpClientHandle(maskUrl, requestBody, dataMap); -// System.out.println(result); -// Map resulMap = HttpUtils.getReturnMap(result); -// if("200".equals(resulMap.get("code"))){ -// System.out.println("数据查找成功变更datacheck状态"); -// } -// } catch (Exception e) { -// e.printStackTrace(); -// System.out.println(e.getMessage()); -// } - } diff --git a/azkaban-db/src/test/java/azkaban/db/AzDBTestUtility.java b/azkaban-db/src/test/java/azkaban/db/AzDBTestUtility.java index d2a9933..9c1d301 100644 --- a/azkaban-db/src/test/java/azkaban/db/AzDBTestUtility.java +++ b/azkaban-db/src/test/java/azkaban/db/AzDBTestUtility.java @@ -56,11 +56,11 @@ public static class EmbeddedMysqlDataSource extends AzkabanDataSource { public EmbeddedMysqlDataSource() { super(); - final String url = "jdbc:mysql://10.255.4.29:8504/bdp_scheduler_01?useUnicode=true&characterEncoding=UTF-8"; + final String url = "jdbc:mysql://localhost:port/bdp_scheduler_01?useUnicode=true&characterEncoding=UTF-8"; setDriverClassName("com.mysql.jdbc.Driver"); setUrl(url); - setUsername("bdpscheduler"); - setPassword("bdp@2017"); + setUsername("username"); + setPassword("password"); } @Override diff --git a/azkaban-jobtype/src/main/java/com/webank/wedatasphere/schedulis/datacheck/test/DatacheckTest.java b/azkaban-jobtype/src/main/java/com/webank/wedatasphere/schedulis/datacheck/test/DatacheckTest.java deleted file mode 100644 index 9004eb2..0000000 --- a/azkaban-jobtype/src/main/java/com/webank/wedatasphere/schedulis/datacheck/test/DatacheckTest.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright 2020 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.webank.wedatasphere.schedulis.datacheck.test; - -import java.util.Base64; - -public class DatacheckTest { - - public static void main(String[] args){ - -// Properties p = new Properties(); -// p.setProperty("source.type", "bdp"); -// p.setProperty("data.object", "ad_audit_ods_mask.t_amis_plan"); -// -// p.setProperty("source.type.01", "bdp"); -// p.setProperty("data.object.01", "ad_audit_ods_mask.t_amis_plan01"); -// -// p.setProperty("source.type.02", "bdp"); -// p.setProperty("data.object.02", "ad_audit_ods_mask.t_amis_plan02"); -// -// p.setProperty("source.type.1", "bdp"); -// p.setProperty("data.object.1", "ad_audit_ods_mask.t_amis_plan1"); -// -// p.setProperty("source.type.2", "bdp"); -// p.setProperty("data.object.2", "ad_audit_ods_mask.t_amis_plan2"); - - -// p.setProperty("source.type", "bdp"); -// p.setProperty("data.object", "ad_audit_ods_mask.t_amis_plan"); - -// p.setProperty("source.type.01", "bdp"); -// p.setProperty("data.object.01", "ad_audit_ods_mask.t_amis_plan"); - -// p.setProperty("source.type.02", "job"); -// p.setProperty("data.object.2", "ad_report_ods_mask.rep_wepower_trans_sum"); -// p.setProperty("source.type", "job"); -// p.setProperty("data.object", "tpcds_text_10.store_sales"); -// p.setProperty("source.type.01", "job"); -// p.setProperty("data.object.01", "tpcds_text_10.store_sales"); -// p.setProperty("source.type.2", "job"); -// p.setProperty("data.object.2", "tpcds_text_10.store_sales"); -// p.setProperty("data.object.2", "cf_ds_ods_mask.ods_rcs_online_clause"); -// p.setProperty("data.object.3", "cf_ds_ods_mask.rcs_faq_tb_record{ds=20170911}"); -// p.setProperty("data.object.4", "cf_ds_ods_mask.ecif_product_info{ds=20170317}"); -// p.setProperty("data.object.5", "ccpd_dfqg1_safe.tbt_tm_acct_conf"); -// p.setProperty("data.object.6", "dbpd_aml_dm_safe.t07_case_stcr_ky_uh_hbase_20150608_bak"); -// p.setProperty("data.object.7", "hduser03ddb.report_jkc_cooperation_compensatory_detail_day"); -// p.setProperty("data.object.8", "cf_ds_ods_mask.ods_rcs_online_meanless_words"); -// p.setProperty("data.object.9", "cf_ds_ods_mask.ecif_contact_info{ds=20170913}"); -// p.setProperty("data.object.9", "ad_report_ods_mask.rep_wepower_trans_sum"); -// p.setProperty("data.object.10", "majunweidb3.mjw_pt2"); -// -// p.setProperty("bdp.datachecker.jdo.option.name", "bdp"); -// p.setProperty("job.datachecker.jdo.option.url", "jdbc:mysql://10.107.108.111:3306/metastore"); -// p.setProperty("job.datachecker.jdo.option.username", "hive"); -// p.setProperty("job.datachecker.jdo.option.password", "aGl2ZSMyMDE4QDA3Cg=="); -// -// p.setProperty("job.datachecker.jdo.option.name", "job_"); -// p.setProperty("job.datachecker.jdo.option.url", "jdbc:mysql://10.107.108.111:3306/metastore"); -// p.setProperty("job.datachecker.jdo.option.username", "hive"); -// p.setProperty("job.datachecker.jdo.option.password", "aGl2ZSMyMDE4QDA3Cg=="); -// -// p.setProperty("wait.time", "1"); -// p.setProperty("query.frequency", "60"); - -// DataChecker dc = new DataChecker("test", p); -// -// List proList = dc.handleSeparationProperties(p); -// -// for(Properties stp : proList){ -// System.out.println(stp.toString()); -// } - -// DataChecker dc2 = new DataChecker("test", p); -// dc2.run(); - - long s = 20140506L; - System.out.println((s+"").length()); - try { - final Base64.Decoder decoder = Base64.getDecoder(); - final Base64.Encoder encoder = Base64.getEncoder(); - final String text = "wtssat@2019"; - final byte[] textByte = text.getBytes("UTF-8"); - //编码 - final String encodedText = encoder.encodeToString(textByte); - System.out.println(encodedText); - //解码 - System.out.println(new String(decoder.decode("aGl2ZSMyMDE4QDA3Cg=="), "UTF-8")); - }catch(Exception e){ - - } - } - - -} diff --git a/azkaban-jobtype/src/main/java/com/webank/wedatasphere/schedulis/jobtype/util/EventChecker.java b/azkaban-jobtype/src/main/java/com/webank/wedatasphere/schedulis/jobtype/util/EventChecker.java index 10100fa..52346b8 100644 --- a/azkaban-jobtype/src/main/java/com/webank/wedatasphere/schedulis/jobtype/util/EventChecker.java +++ b/azkaban-jobtype/src/main/java/com/webank/wedatasphere/schedulis/jobtype/util/EventChecker.java @@ -225,9 +225,9 @@ public static void main(String[] args) { // p.put("msg.body","msg.body"); p.put("msg.eventchecker.jdo.option.name","msg"); - p.put("msg.eventchecker.jdo.option.url","jdbc:mysql://10.255.0.76:3306/wtss_qyh_test?useUnicode=true&characterEncoding=UTF-8"); - p.put("msg.eventchecker.jdo.option.username","root"); - p.put("msg.eventchecker.jdo.option.password","YmRwI3Jvb3RAMjAxOQ=="); + p.put("msg.eventchecker.jdo.option.url","jdbc:mysql://locahost:port/wtss_qyh_test?useUnicode=true&characterEncoding=UTF-8"); + p.put("msg.eventchecker.jdo.option.username","username"); + p.put("msg.eventchecker.jdo.option.password","password"); EventChecker ec = new EventChecker("AA",p); ec.run(); diff --git a/azkaban-jobtype/src/test/java/azkaban/jobtype/TestHadoopJobUtilsFindApplicationIdFromLog.java b/azkaban-jobtype/src/test/java/azkaban/jobtype/TestHadoopJobUtilsFindApplicationIdFromLog.java index 4025b10..fcedfd1 100644 --- a/azkaban-jobtype/src/test/java/azkaban/jobtype/TestHadoopJobUtilsFindApplicationIdFromLog.java +++ b/azkaban-jobtype/src/test/java/azkaban/jobtype/TestHadoopJobUtilsFindApplicationIdFromLog.java @@ -108,8 +108,6 @@ public void testLogWithMultipleApplicationIdsAppearingMultipleTimes() throws IOE bw.write("28-08-2015 12:29:38 PDT Training_clickSelectFeatures INFO - INFO The url to track the job: http://eat1-nertzwp02.grid.linkedin.com:8080/proxy/application_1440264346270_3088/\n"); bw.write("28-08-2015 12:29:38 PDT Training_clickSelectFeatures INFO - INFO See http://eat1-nertzwp02.grid.linkedin.com:8080/proxy/application_1440264346270_3088/ for details.\n"); bw.write("28-08-2015 12:29:38 PDT Training_clickSelectFeatures INFO - INFO Running job: job_1440264346270_3088\n"); - bw.write("28-08-2015 12:30:21 PDT Training_clickSelectFeatures INFO - INFO Closing idle connection Socket[addr=eat1-hcl5481.grid.linkedin.com/172.20.138.228,port=42492,localport=42382] to server eat1-hcl5481.grid.linkedin.com/172.20.138.228:42492\n"); - bw.write("28-08-2015 12:30:37 PDT Training_clickSelectFeatures INFO - INFO Closing idle connection Socket[addr=eat1-nertznn01.grid.linkedin.com/172.20.158.57,port=9000,localport=30453] to server eat1-nertznn01.grid.linkedin.com/172.20.158.57:9000\n"); bw.write("28-08-2015 12:31:09 PDT Training_clickSelectFeatures INFO - INFO Job job_1440264346270_3088 running in uber mode : false\n"); bw.close(); diff --git a/bin/upgrade/upgrade.sh b/bin/upgrade/upgrade.sh index 92aa512..7fb31af 100644 --- a/bin/upgrade/upgrade.sh +++ b/bin/upgrade/upgrade.sh @@ -11,8 +11,8 @@ v_hosts=/data/change/client/hdp_client.hosts${UUID} echo >$v_hosts cat >$v_hosts <