Merge pull request #551 from wayyoungboy/v3.0-dev

support GatherComponentLogHandler

common/ssh_client/local_client.py

@@ -78,4 +78,4 @@ def get_name(self):
         return "local"
 
     def get_ip(self):
-        return self.client.get_ip()
+        return "127.0.0.1"

common/ssh_client/remote_client.py

@@ -151,3 +151,6 @@ def ssh_invoke_shell_switch_user(self, new_user, cmd, time_out):
 
     def get_name(self):
         return "remote_{0}".format(self.host_ip)
+
+    def get_ip(self):
+        return self.host_ip

common/ssh_client/ssh.py

@@ -84,6 +84,7 @@ def exec_cmd(self, cmd):
         return self.client.exec_cmd(cmd).strip()
 
     def download(self, remote_path, local_path):
+        self.stdio.verbose("download file: {} to {}".format(remote_path, local_path))
         return self.client.download(remote_path, local_path)
 
     def upload(self, remote_path, local_path):

common/tool.py

@@ -31,6 +31,7 @@
 import json
 import hashlib
 import datetime
+import uuid
 import tabulate
 import tarfile
 import socket
@@ -660,7 +661,7 @@ def write_append(filename, result, stdio=None):
             fileobj.write(u'{}'.format(result))
 
     def tar_gz_to_zip(temp_dir, tar_gz_file, output_zip, password, stdio):
-        extract_dir = os.path.join(temp_dir, 'extracted_files')
+        extract_dir = os.path.join(temp_dir, 'extracted_files_{0}'.format(str(uuid.uuid4())[:6]))
 
         try:
             # 1. Extract the tar.gz file
@@ -701,11 +702,6 @@ def tar_gz_to_zip(temp_dir, tar_gz_file, output_zip, password, stdio):
             if os.path.exists(extract_dir):
                 shutil.rmtree(extract_dir)
             return False
-        except pyminizip.compress_error as ce:
-            stdio.exception("compression error: {0}".format(ce))
-            if os.path.exists(extract_dir):
-                shutil.rmtree(extract_dir)
-            return False
         except Exception as e:
             stdio.exception("an error occurred: {0}".format(e))
             if os.path.exists(extract_dir):

core.py

@@ -24,6 +24,7 @@
 
 from common.ssh_client.remote_client import dis_rsa_algorithms
 from handler.gather.gather_ash_report import GatherAshReportHandler
+from handler.gather.gather_component_log import GatherComponentLogHandler
 from handler.rca.rca_handler import RCAHandler
 from handler.rca.rca_list import RcaScenesListHandler
 from common.ssh import SshClient, SshConfig
@@ -39,9 +40,7 @@
 from handler.analyzer.analyze_index_space import AnalyzeIndexSpaceHandler
 from handler.checker.check_handler import CheckHandler
 from handler.checker.check_list import CheckListHandler
-from handler.gather.gather_log import GatherLogHandler
 from handler.gather.gather_awr import GatherAwrHandler
-from handler.gather.gather_obproxy_log import GatherObProxyLogHandler
 from handler.gather.gather_sysstat import GatherOsInfoHandler
 from handler.gather.gather_obstack2 import GatherObstack2Handler
 from handler.gather.gather_obadmin import GatherObAdminHandler
@@ -245,10 +244,24 @@ def gather_function(self, function_type, opt):
             self.stdio.print("{0} start ...".format(function_type))
             self.update_obcluster_nodes(config)
             self.set_context(function_type, 'gather', config)
+            options = self.context.options
             timestamp = TimeUtils.get_current_us_timestamp()
             self.context.set_variable('gather_timestamp', timestamp)
             if function_type == 'gather_log':
-                handler = GatherLogHandler(self.context)
+                handler = GatherComponentLogHandler()
+                handler.init(
+                    self.context,
+                    target="observer",
+                    from_option=Util.get_option(options, 'from'),
+                    to_option=Util.get_option(options, 'to'),
+                    since=Util.get_option(options, 'since'),
+                    scope=Util.get_option(options, 'scope'),
+                    grep=Util.get_option(options, 'grep'),
+                    encrypt=Util.get_option(options, 'encrypt'),
+                    store_dir=Util.get_option(options, 'store_dir'),
+                    temp_dir=Util.get_option(options, 'temp_dir'),
+                    redact=Util.get_option(options, 'redact'),
+                )
                 return handler.handle()
             elif function_type == 'gather_awr':
                 handler = GatherAwrHandler(self.context)
@@ -277,9 +290,35 @@ def gather_function(self, function_type, opt):
                 handler_stack.handle()
                 handler_perf = GatherPerfHandler(self.context)
                 handler_perf.handle()
-                handler_log = GatherLogHandler(self.context)
-                handler_log.handle()
-                handler_obproxy = GatherObProxyLogHandler(self.context)
+                handler_observer_log = GatherComponentLogHandler()
+                handler_observer_log.init(
+                    self.context,
+                    target="observer",
+                    from_option=Util.get_option(options, 'from'),
+                    to_option=Util.get_option(options, 'to'),
+                    since=Util.get_option(options, 'since'),
+                    scope=Util.get_option(options, 'scope'),
+                    grep=Util.get_option(options, 'grep'),
+                    encrypt=Util.get_option(options, 'encrypt'),
+                    store_dir=Util.get_option(options, 'store_dir'),
+                    temp_dir=Util.get_option(options, 'temp_dir'),
+                    redact=Util.get_option(options, 'redact'),
+                )
+                handler_observer_log.handle()
+                handler_obproxy = GatherComponentLogHandler()
+                handler_obproxy.init(
+                    self.context,
+                    target="obproxy",
+                    from_option=Util.get_option(options, 'from'),
+                    to_option=Util.get_option(options, 'to'),
+                    since=Util.get_option(options, 'since'),
+                    scope=Util.get_option(options, 'scope'),
+                    grep=Util.get_option(options, 'grep'),
+                    encrypt=Util.get_option(options, 'encrypt'),
+                    store_dir=Util.get_option(options, 'store_dir'),
+                    temp_dir=Util.get_option(options, 'temp_dir'),
+                    redact=Util.get_option(options, 'redact'),
+                )
                 return handler_obproxy.handle()
             elif function_type == 'gather_sysstat':
                 handler = GatherOsInfoHandler(self.context)
@@ -310,7 +349,21 @@ def gather_obproxy_log(self, opt):
             return ObdiagResult(ObdiagResult.INPUT_ERROR_CODE, error_data='No such custum config')
         else:
             self.set_context_skip_cluster_conn('gather_obproxy_log', 'gather', config)
-            handler = GatherObProxyLogHandler(self.context)
+            options = self.context.options
+            handler = GatherComponentLogHandler()
+            handler.init(
+                self.context,
+                target="obproxy",
+                from_option=Util.get_option(options, 'from'),
+                to_option=Util.get_option(options, 'to'),
+                since=Util.get_option(options, 'since'),
+                scope=Util.get_option(options, 'scope'),
+                grep=Util.get_option(options, 'grep'),
+                encrypt=Util.get_option(options, 'encrypt'),
+                store_dir=Util.get_option(options, 'store_dir'),
+                temp_dir=Util.get_option(options, 'temp_dir'),
+                redact=Util.get_option(options, 'redact'),
+            )
             return handler.handle()
 
     def gather_scenes_list(self, opt):
@@ -401,31 +454,34 @@ def check(self, opts):
             self._call_stdio('error', 'No such custum config')
             return ObdiagResult(ObdiagResult.INPUT_ERROR_CODE, error_data='No such custum config')
         else:
-            self.stdio.print("check start ...")
-            self.update_obcluster_nodes(config)
-            self.set_context('check', 'check', config)
-            obproxy_check_handler = None
-            observer_check_handler = None
-            result_data = {}
-            if self.context.obproxy_config.get("servers") is not None and len(self.context.obproxy_config.get("servers")) > 0:
-                obproxy_check_handler = CheckHandler(self.context, check_target_type="obproxy")
-                obproxy_check_handler.handle()
-                obproxy_result = obproxy_check_handler.execute()
-                result_data['obproxy'] = obproxy_result
-            if self.context.cluster_config.get("servers") is not None and len(self.context.cluster_config.get("servers")) > 0:
-                observer_check_handler = CheckHandler(self.context, check_target_type="observer")
-                observer_check_handler.handle()
-                observer_result = observer_check_handler.execute()
-                result_data['observer'] = observer_result
-            if obproxy_check_handler is not None:
-                obproxy_report_path = os.path.expanduser(obproxy_check_handler.report.get_report_path())
-                if os.path.exists(obproxy_report_path):
-                    self.stdio.print("Check obproxy finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0} ".format(obproxy_check_handler.report.get_report_path()) + Style.RESET_ALL + "'")
-            if observer_check_handler is not None:
-                observer_report_path = os.path.expanduser(observer_check_handler.report.get_report_path())
-                if os.path.exists(observer_report_path):
-                    self.stdio.print("Check observer finished. For more details, please run cmd'" + Fore.YELLOW + " cat {0} ".format(observer_check_handler.report.get_report_path()) + Style.RESET_ALL + "'")
-            return ObdiagResult(ObdiagResult.SUCCESS_CODE, data=result_data)
+            try:
+                self.stdio.print("check start ...")
+                self.update_obcluster_nodes(config)
+                self.set_context('check', 'check', config)
+                obproxy_check_handler = None
+                observer_check_handler = None
+                result_data = {}
+                if self.context.obproxy_config.get("servers") is not None and len(self.context.obproxy_config.get("servers")) > 0:
+                    obproxy_check_handler = CheckHandler(self.context, check_target_type="obproxy")
+                    obproxy_result = obproxy_check_handler.handle()
+                    result_data['obproxy'] = obproxy_result
+                if self.context.cluster_config.get("servers") is not None and len(self.context.cluster_config.get("servers")) > 0:
+                    observer_check_handler = CheckHandler(self.context, check_target_type="observer")
+                    observer_result = observer_check_handler.handle()
+                    result_data['observer'] = observer_result
+                if obproxy_check_handler is not None:
+                    obproxy_report_path = os.path.expanduser(obproxy_check_handler.report.get_report_path())
+                    if os.path.exists(obproxy_report_path):
+                        self.stdio.print("Check obproxy finished. For more details, please run cmd '" + Fore.YELLOW + " cat {0} ".format(obproxy_check_handler.report.get_report_path()) + Style.RESET_ALL + "'")
+                if observer_check_handler is not None:
+                    observer_report_path = os.path.expanduser(observer_check_handler.report.get_report_path())
+                    if os.path.exists(observer_report_path):
+                        self.stdio.print("Check observer finished. For more details, please run cmd'" + Fore.YELLOW + " cat {0} ".format(observer_check_handler.report.get_report_path()) + Style.RESET_ALL + "'")
+                return ObdiagResult(ObdiagResult.SUCCESS_CODE, data=result_data)
+            except Exception as e:
+                self.stdio.error("check Exception: {0}".format(e))
+                self.stdio.verbose(traceback.format_exc())
+                return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data="check Exception: {0}".format(e))
 
     def check_list(self, opts):
         config = self.config_manager

handler/checker/check_handler.py

@@ -30,7 +30,6 @@
 from handler.checker.check_task import TaskBase
 import re
 from common.tool import Util
-from common.tool import YamlUtils
 from common.tool import StringUtils
 
 
@@ -171,11 +170,11 @@ def handle(self):
                             new_tasks[task_name] = task_value
                     self.tasks = new_tasks
             self.stdio.verbose("tasks is {0}".format(self.tasks.keys()))
-            return True
+            return self.__execute()
         except Exception as e:
             self.stdio.error("Get package tasks failed. Error info is {0}".format(e))
             self.stdio.verbose(traceback.format_exc())
-            return False
+            raise CheckException("Internal error :{0}".format(e))
 
     # get all tasks
     def get_all_tasks(self):
@@ -213,7 +212,7 @@ def get_package_tasks(self, package_name):
         return packege_tasks[package_name].get("tasks")
 
     # execute task
-    def execute_one(self, task_name):
+    def __execute_one(self, task_name):
         try:
             self.stdio.verbose("execute tasks is {0}".format(task_name))
             # Verify if the version is within a reasonable range
@@ -236,22 +235,21 @@ def execute_one(self, task_name):
             self.stdio.error("execute_one Exception : {0}".format(e))
             raise CheckException("execute_one Exception : {0}".format(e))
 
-    def execute(self):
+    def __execute(self):
         try:
             self.stdio.verbose("execute_all_tasks. the number of tasks is {0} ,tasks is {1}".format(len(self.tasks.keys()), self.tasks.keys()))
             self.report = CheckReport(self.context, export_report_path=self.export_report_path, export_report_type=self.export_report_type, report_target=self.check_target_type)
             # one of tasks to execute
             for task in self.tasks:
-                t_report = self.execute_one(task)
+                t_report = self.__execute_one(task)
                 self.report.add_task_report(t_report)
             self.report.export_report()
             return self.report.report_tobeMap()
         except CheckrReportException as e:
             self.stdio.error("Report error :{0}".format(e))
-            self.stdio.verbose(traceback.format_exc())
+            raise CheckException("Report error :{0}".format(e))
         except Exception as e:
-            self.stdio.error("Internal error :{0}".format(e))
-            self.stdio.verbose(traceback.format_exc())
+            raise CheckException("Internal error :{0}".format(e))
 
 
 class checkOBConnectorPool:

handler/checker/check_list.py

@@ -82,4 +82,5 @@ def handle(self):
                     Util.print_scene(cases_map, stdio=self.stdio)
             return ObdiagResult(ObdiagResult.SUCCESS_CODE, data=result_map)
         except Exception as e:
+
             return ObdiagResult(ObdiagResult.SERVER_ERROR_CODE, error_data=str(e))