Add temp_dir option to support store remote nodes temporary result fi…

…les (#405)

* update version to 2.4.0

* Remove duplicate scripts

* The configuration can be passed entirely through parameters

* fix

* fix

* analyze index space add --config option

* fix

* Support querying nodes through the tenant connection string and completing the node config

* ifx

* fix

* fix

* fix

* fix

core.py

@@ -157,6 +157,12 @@ def set_offline_context(self, handler_name, namespace):
     def update_obcluster_nodes(self, config):
         config_data = config.config_data
         cluster_config = config.config_data["obcluster"]
+        lst = Util.get_option(self.options, 'config')
+        if lst:
+            if any(item.startswith('obcluster.servers.nodes') for item in lst):
+                return
+            else:
+                self.stdio.verbose("You have already provided node information, so there is no need to query node information from the sys tenant")
         ob_cluster = {"db_host": cluster_config["db_host"], "db_port": cluster_config["db_port"], "tenant_sys": {"user": cluster_config["tenant_sys"]["user"], "password": cluster_config["tenant_sys"]["password"]}}
         if config_data['obcluster'] and config_data['obcluster']['servers'] and config_data['obcluster']['servers']['nodes']:
             return

diag_cmd.py

@@ -410,6 +410,7 @@ def __init__(self):
         self.parser.add_option('--grep', action="append", type='string', help="specify keywords constrain")
         self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false")
         self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
+        self.parser.add_option('--temp_dir', type='string', help='the dir for temporarily storing files on nodes', default='/tmp')
         self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml'))
         self.parser.add_option('--config', action="append", type="string", help='config options Format: --config key=value')
 
@@ -433,6 +434,7 @@ def __init__(self):
         self.parser.add_option('--grep', action="append", type='string', help="specify keywords constrain")
         self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false")
         self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
+        self.parser.add_option('--temp_dir', type='string', help='the dir for temporarily storing files on nodes', default='/tmp')
         self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml'))
         self.parser.add_option('--config', action="append", type="string", help='config options Format: --config key=value')
 
@@ -662,6 +664,7 @@ def __init__(self):
         self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: <n> <m|h|d>. example: 1h.", default='30m')
         self.parser.add_option('--env', type='string', help='env, eg: "{env1=xxx, env2=xxx}"')
         self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
+        self.parser.add_option('--temp_dir', type='string', help='the dir for temporarily storing files on nodes', default='/tmp')
         self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml'))
         self.parser.add_option('--config', action="append", type="string", help='config options Format: --config key=value')
 
@@ -728,6 +731,7 @@ def __init__(self):
         self.parser.add_option('--files', action="append", type='string', help="specify files")
         self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
         self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: <n> <m|h|d>. example: 1h.", default='30m')
+        self.parser.add_option('--temp_dir', type='string', help='the dir for temporarily storing files on nodes', default='/tmp')
         self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml'))
         self.parser.add_option('--config', action="append", type="string", help='config options Format: --config key=value')
 
@@ -754,6 +758,7 @@ def __init__(self):
         self.parser.add_option('--recursion', type='string', help="Maximum number of recursion", default=8)
         self.parser.add_option('--output', type='string', help="Print the result to the maximum output line on the screen", default=60)
         self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
+        self.parser.add_option('--temp_dir', type='string', help='the dir for temporarily storing files on nodes', default='/tmp')
         self.parser.add_option('-c', type='string', help='obdiag custom config', default=os.path.expanduser('~/.obdiag/config.yml'))
         self.parser.add_option('--config', action="append", type="string", help='config options Format: --config key=value')
 

handler/analyzer/analyze_flt_trace.py

@@ -62,6 +62,7 @@ def init_option(self):
         top_option = Util.get_option(options, 'top')
         recursion_option = Util.get_option(options, 'recursion')
         output_option = Util.get_option(options, 'output')
+        temp_dir_option = Util.get_option(options, 'temp_dir')
         if store_dir_option is not None:
             if not os.path.exists(os.path.abspath(store_dir_option)):
                 self.stdio.warn('Warning: args --store_dir [{0}] incorrect: No such directory, Now create it'.format(os.path.abspath(store_dir_option)))
@@ -82,6 +83,8 @@ def init_option(self):
             self.max_recursion = int(recursion_option)
         if output_option:
             self.output = int(output_option)
+        if temp_dir_option:
+            self.gather_ob_log_temporary_dir = temp_dir_option
         return True
 
     def handle(self):
@@ -151,7 +154,7 @@ def __handle_from_node(self, node, old_files, local_store_parent_dir):
             return resp, node_files
         if not ssh_failed:
             gather_dir_name = "trace_merged_cache"
-            gather_dir_full_path = "{0}/{1}".format("/tmp", gather_dir_name)
+            gather_dir_full_path = "{0}/{1}".format(self.gather_ob_log_temporary_dir, gather_dir_name)
             mkdir(ssh_client, gather_dir_full_path, self.stdio)
             if self.is_ssh:
                 self.__get_online_log_file(ssh_client, node, gather_dir_full_path, local_store_dir)

handler/analyzer/analyze_log.py

@@ -78,6 +78,7 @@ def init_option(self):
         scope_option = Util.get_option(options, 'scope')
         log_level_option = Util.get_option(options, 'log_level')
         files_option = Util.get_option(options, 'files')
+        temp_dir_option = Util.get_option(options, 'temp_dir')
         if files_option:
             self.is_ssh = False
             self.directly_analyze_files = True
@@ -121,6 +122,8 @@ def init_option(self):
             self.scope = scope_option
         if log_level_option:
             self.log_level = OBLogLevel().get_log_level(scope_option)
+        if temp_dir_option:
+            self.gather_ob_log_temporary_dir = temp_dir_option
         return True
 
     def handle(self):
@@ -191,7 +194,7 @@ def __handle_from_node(self, node, local_store_parent_dir):
         from_datetime_timestamp = TimeUtils.timestamp_to_filename_time(TimeUtils.datetime_to_timestamp(self.from_time_str))
         to_datetime_timestamp = TimeUtils.timestamp_to_filename_time(TimeUtils.datetime_to_timestamp(self.to_time_str))
         gather_dir_name = "ob_log_{0}_{1}_{2}".format(ssh_client.get_name(), from_datetime_timestamp, to_datetime_timestamp)
-        gather_dir_full_path = "{0}/{1}".format("/tmp", gather_dir_name)
+        gather_dir_full_path = "{0}/{1}".format(self.gather_ob_log_temporary_dir, gather_dir_name)
         mkdir(ssh_client, gather_dir_full_path)
 
         log_list, resp = self.__handle_log_list(ssh_client, node, resp)

handler/gather/gather_log.py

@@ -79,6 +79,7 @@ def init_option(self):
         grep_option = Util.get_option(options, 'grep')
         scope_option = Util.get_option(options, 'scope')
         encrypt_option = Util.get_option(options, 'encrypt')
+        temp_dir_option = Util.get_option(options, 'temp_dir')
         if self.context.get_variable("gather_from", None):
             from_option = self.context.get_variable("gather_from")
         if self.context.get_variable("gather_to", None):
@@ -91,6 +92,8 @@ def init_option(self):
             scope_option = self.context.get_variable("gather_scope")
         if self.context.get_variable("gather_grep", None):
             grep_option = self.context.get_variable("gather_grep")
+        if self.context.get_variable("temp_dir", None):
+            temp_dir_option = self.context.get_variable("temp_dir")
         if from_option is not None and to_option is not None:
             try:
                 from_timestamp = TimeUtils.parse_time_str(from_option)
@@ -128,6 +131,8 @@ def init_option(self):
             self.zip_encrypt = True
         if grep_option:
             self.grep_options = grep_option
+        if temp_dir_option:
+            self.gather_ob_log_temporary_dir = temp_dir_option
         return True
 
     def handle(self):
@@ -187,7 +192,7 @@ def __handle_from_node(self, pack_dir_this_command, node):
         from_datetime_timestamp = TimeUtils.timestamp_to_filename_time(TimeUtils.datetime_to_timestamp(self.from_time_str))
         to_datetime_timestamp = TimeUtils.timestamp_to_filename_time(TimeUtils.datetime_to_timestamp(self.to_time_str))
         gather_dir_name = "ob_log_{0}_{1}_{2}".format(ssh_client.get_name(), from_datetime_timestamp, to_datetime_timestamp)
-        gather_dir_full_path = "{0}/{1}".format("/tmp", gather_dir_name)
+        gather_dir_full_path = "{0}/{1}".format(self.gather_ob_log_temporary_dir, gather_dir_name)
         mkdir(ssh_client, gather_dir_full_path, self.stdio)
 
         log_list, resp = self.__handle_log_list(ssh_client, node, resp)
@@ -319,7 +324,7 @@ def __pharse_log(self, ssh_client, home_path, log_name, gather_path):
             self.stdio.verbose('grep files, run cmd = [{0}]'.format(grep_cmd))
             ssh_client.exec_cmd(grep_cmd)
         else:
-            cp_cmd = "cp {log_dir}/{log_name} {gather_path}/{log_name} ".format(gather_path=gather_path, log_name=log_name, log_dir=log_path)
+            cp_cmd = "cp -p {log_dir}/{log_name} {gather_path}/{log_name} ".format(gather_path=gather_path, log_name=log_name, log_dir=log_path)
             self.stdio.verbose('copy files, run cmd = [{0}]'.format(cp_cmd))
             ssh_client.exec_cmd(cp_cmd)
 

handler/gather/gather_obproxy_log.py

@@ -81,6 +81,7 @@ def init_option(self):
         grep_option = Util.get_option(options, 'grep')
         encrypt_option = Util.get_option(options, 'encrypt')
         scope_option = Util.get_option(options, 'scope')
+        temp_dir_option = Util.get_option(options, 'temp_dir')
         if self.context.get_variable("gather_from", None):
             from_option = self.context.get_variable("gather_from")
         if self.context.get_variable("gather_to", None):
@@ -93,7 +94,8 @@ def init_option(self):
             scope_option = self.context.get_variable("gather_scope")
         if self.context.get_variable("gather_grep", None):
             grep_option = self.context.get_variable("gather_grep")
-
+        if self.context.get_variable("temp_dir", None):
+            temp_dir_option = self.context.get_variable("temp_dir")
         if from_option is not None and to_option is not None:
             try:
                 from_timestamp = TimeUtils.parse_time_str(from_option)
@@ -131,6 +133,8 @@ def init_option(self):
             self.zip_encrypt = True
         if grep_option:
             self.grep_args = grep_option
+        if temp_dir_option:
+            self.gather_log_temporary_dir = temp_dir_option
         self.stdio.verbose("grep_args:{0}".format(grep_option))
         return True
 
@@ -195,7 +199,7 @@ def __handle_from_node(self, node, pack_dir_this_command):
             from_datetime_timestamp = TimeUtils.timestamp_to_filename_time(TimeUtils.datetime_to_timestamp(self.from_time_str))
             to_datetime_timestamp = TimeUtils.timestamp_to_filename_time(TimeUtils.datetime_to_timestamp(self.to_time_str))
             gather_dir_name = "obproxy_log_{0}_{1}_{2}".format(ssh_client.get_name(), from_datetime_timestamp, to_datetime_timestamp)
-            gather_dir_full_path = "{0}/{1}".format("/tmp", gather_dir_name)
+            gather_dir_full_path = "{0}/{1}".format(self.gather_log_temporary_dir, gather_dir_name)
             mkdir(ssh_client, gather_dir_full_path, self.stdio)
 
             log_list, resp = self.__handle_log_list(ssh_client, node, resp)

handler/gather/gather_scenes.py

@@ -49,6 +49,7 @@ def __init__(self, context, gather_pack_dir='./', tasks_base_path="~/.obdiag/gat
         self.task_type = task_type
         self.variables = {}
         self.is_inner = is_inner
+        self.temp_dir = '/tmp'
         if self.context.get_variable("gather_timestamp", None):
             self.gather_timestamp = self.context.get_variable("gather_timestamp")
         else:
@@ -70,6 +71,7 @@ def handle(self):
         if not self.init_config():
             self.stdio.error('init config failed')
             return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="init config failed")
+        self.context.set_variable('temp_dir', self.temp_dir)
         self.__init_variables()
         self.__init_report_path()
         self.__init_task_names()
@@ -182,6 +184,7 @@ def init_option(self):
         store_dir_option = Util.get_option(options, 'store_dir')
         env_option = Util.get_option(options, 'env')
         scene_option = Util.get_option(options, 'scene')
+        temp_dir_option = Util.get_option(options, 'temp_dir')
         if from_option is not None and to_option is not None:
             try:
                 from_timestamp = TimeUtils.parse_time_str(from_option)
@@ -218,6 +221,8 @@ def init_option(self):
         if env_option:
             env_dict = StringUtils.parse_env(env_option)
             self.env = env_dict
+        if temp_dir_option:
+            self.temp_dir = temp_dir_option
         return True
 
     def __get_sql_result(self):