Merge pull request #821 from fkie-cad/database-migration

Database migration

INSTALL.md

@@ -77,8 +77,8 @@ save some time when you already have the images.
 The three components db, backend and frontend can be installed independently to create a distributed installation.
 
 The two worker components (frontend, backend) communicate exclusively through the database. The database in turn does not needed any knowledge of its place in the network, other than on which **ip:port** combination the database server has to be hosted.
-The main.cfg on the frontend system has to be altered so that the values of `data_storage.mongo_server` and `data_storage.mongo_port` match the **ip:port** for the database.
-The same has to be done for the backend. In addition, since the raw firmware and file binaries are stored in the backend, the `data_storage.firmware_file_storage_directory` has to be created (by default `/media/data/fact_fw_data`).
+The main.cfg on the frontend system has to be altered so that the values of `data-storage.mongo-server` and `data-storage.mongo-port` match the **ip:port** for the database.
+The same has to be done for the backend. In addition, since the raw firmware and file binaries are stored in the backend, the `data-storage.firmware-file-storage-directory` has to be created (by default `/media/data/fact_fw_data`).
 On the database system, the `mongod.conf` has to be given the correct `net.bindIp` and `net.port`. In addition the path in `storage.dbPath` of the `mongod.conf` has to be created. 
 
 ## Installation with Nginx (**--nginx**)

README.md

@@ -28,6 +28,7 @@ Minimal | Recommended | Software
 It is possible to install FACT on any Linux distribution, but the installer is limited to
 - Ubuntu 18.04 (with Python >3.6)
 - Ubuntu 20.04 (stable)
+- Ubuntu 22.04 (stable)
 - Debian 10 (stable)
 - Kali (experimental)
 

docsrc/conf.py

@@ -72,6 +72,7 @@
     'pymongo',
     'requests',
     'si_prefix',
+    'sqlalchemy',
     'ssdeep',
     'tlsh',
     'werkzeug',

docsrc/index.rst

@@ -16,6 +16,7 @@ Contents
    :maxdepth: 1
 
    main
+   migration
 
 
 .. toctree::

docsrc/migration.rst

@@ -0,0 +1,22 @@
+Upgrading FACT from 3 to 4
+==========================
+
+With the release of FACT 4.0, the database was switched from MongoDB to PostgreSQL.
+To install all dependencies, simply rerun the installation::
+
+    $ python3 src/install.py
+
+Existing analysis and comparison results from your old FACT installation have to be migrated to the new database.
+First you need to start the database::
+
+    $ mongod --config config/mongod.conf
+
+Then you can start the migration script::
+
+    $ python3 src/migrate_db_to_postgresql.py
+
+After this, you should be able to start FACT normally and should find your old data in the new database.
+When the migration is complete, FACT does not use MongoDB anymore and you may want to uninstall it::
+
+    $ python3 -m pip uninstall pymongo
+    $ sudo apt remove mongodb # or mongodb-org depending on which version is installed

docsrc/modules/helperFunctions.data_conversion.rst

@@ -0,0 +1,7 @@
+helperFunctions.data_conversion module
+======================================
+
+.. automodule:: helperFunctions.data_conversion
+   :members:
+   :undoc-members:
+   :show-inheritance:

docsrc/modules/helperFunctions.rst

@@ -6,23 +6,23 @@ helperFunctions
 
    helperFunctions.compare_sets
    helperFunctions.config
+   helperFunctions.data_conversion
    helperFunctions.database
    helperFunctions.docker
    helperFunctions.fileSystem
    helperFunctions.hash
    helperFunctions.install
    helperFunctions.logging
    helperFunctions.merge_generators
-   helperFunctions.mongo_config_parser
-   helperFunctions.mongo_task_conversion
    helperFunctions.object_conversion
-   helperFunctions.object_storage
    helperFunctions.pdf
    helperFunctions.plugin
    helperFunctions.process
    helperFunctions.program_setup
    helperFunctions.tag
+   helperFunctions.task_conversion
    helperFunctions.uid
+   helperFunctions.virtual_file_path
    helperFunctions.web_interface
    helperFunctions.yara_binary_search
 

docsrc/modules/helperFunctions.mongo_task_conversion.rst → docsrc/modules/helperFunctions.task_conversion.rst

@@ -1,7 +1,7 @@
-helperFunctions.mongo_task_conversion module
+helperFunctions.task_conversion module
 ============================================
 
-.. automodule:: helperFunctions.mongo_task_conversion
+.. automodule:: helperFunctions.task_conversion
    :members:
    :undoc-members:
    :show-inheritance:

docsrc/modules/helperFunctions.virtual_file_path.rst

@@ -0,0 +1,7 @@
+helperFunctions.virtual_file_path module
+========================================
+
+.. automodule:: helperFunctions.virtual_file_path
+   :members:
+   :undoc-members:
+   :show-inheritance:

src/analysis/PluginBase.py

@@ -4,7 +4,7 @@
 from time import time
 
 from helperFunctions.process import (
-    ExceptionSafeProcess, check_worker_exceptions, start_single_worker, terminate_process_and_children
+    ExceptionSafeProcess, check_worker_exceptions, start_single_worker, stop_processes, terminate_process_and_children
 )
 from helperFunctions.tag import TagColor
 from objects.file import FileObject
@@ -20,32 +20,50 @@ def __init__(self, *args, plugin=None):
 class AnalysisBasePlugin(BasePlugin):  # pylint: disable=too-many-instance-attributes
     '''
     This is the base plugin. All plugins should be subclass of this.
-    recursive flag: If True (default) recursively analyze included files
     '''
-    VERSION = 'not set'
-    SYSTEM_VERSION = None
 
-    timeout = None
+    # must be set by the plugin:
+    FILE = None
+    NAME = None
+    DESCRIPTION = None
+    VERSION = None
+
+    # can be set by the plugin:
+    RECURSIVE = True  # If `True` (default) recursively analyze included files
+    TIMEOUT = 300
+    SYSTEM_VERSION = None
+    MIME_BLACKLIST = []
+    MIME_WHITELIST = []
 
-    def __init__(self, plugin_administrator, config=None, recursive=True, no_multithread=False, timeout=300, offline_testing=False, plugin_path=None):  # pylint: disable=too-many-arguments
-        super().__init__(plugin_administrator, config=config, plugin_path=plugin_path)
+    def __init__(self, plugin_administrator, config=None, no_multithread=False, offline_testing=False, view_updater=None):
+        super().__init__(plugin_administrator, config=config, plugin_path=self.FILE, view_updater=view_updater)
+        self._check_plugin_attributes()
         self.check_config(no_multithread)
-        self.recursive = recursive
+        self.additional_setup()
         self.in_queue = Queue()
         self.out_queue = Queue()
         self.stop_condition = Value('i', 0)
         self.workers = []
         self.thread_count = int(self.config[self.NAME]['threads'])
         self.active = [Value('i', 0) for _ in range(self.thread_count)]
-        if self.timeout is None:
-            self.timeout = timeout
         self.register_plugin()
         if not offline_testing:
             self.start_worker()
 
+    def additional_setup(self):
+        '''
+        This function can be implemented by the plugin to do initialization
+        '''
+        pass
+
+    def _check_plugin_attributes(self):
+        for attribute in ['FILE', 'NAME', 'VERSION']:
+            if getattr(self, attribute, None) is None:
+                raise PluginInitException(f'Plugin {self.NAME} is missing {attribute} in configuration')
+
     def add_job(self, fw_object: FileObject):
         if self._dependencies_are_unfulfilled(fw_object):
-            logging.error('{}: dependencies of plugin {} not fulfilled'.format(fw_object.uid, self.NAME))
+            logging.error(f'{fw_object.uid}: dependencies of plugin {self.NAME} not fulfilled')
         elif self._analysis_depth_not_reached_yet(fw_object):
             self.in_queue.put(fw_object)
             return
@@ -57,7 +75,7 @@ def _dependencies_are_unfulfilled(self, fw_object: FileObject):
         return any(dep not in fw_object.processed_analysis for dep in self.DEPENDENCIES)
 
     def _analysis_depth_not_reached_yet(self, fo):
-        return self.recursive or fo.depth == 0
+        return self.RECURSIVE or fo.depth == 0
 
     def process_object(self, file_object):  # pylint: disable=no-self-use
         '''
@@ -76,12 +94,11 @@ def _add_plugin_version_and_timestamp_to_analysis_result(self, fo):  # pylint: d
 
     def shutdown(self):
         '''
-        This function can be called to shutdown all working threads
+        This function can be called to shut down all working threads
         '''
         logging.debug('Shutting down...')
         self.stop_condition.value = 1
-        for process in self.workers:
-            process.join()
+        stop_processes(self.workers)
         self.in_queue.close()
         self.out_queue.close()
 
@@ -116,7 +133,7 @@ def check_config(self, no_multithread):
     def start_worker(self):
         for process_index in range(self.thread_count):
             self.workers.append(start_single_worker(process_index, 'Analysis', self.worker))
-        logging.debug('{}: {} worker threads started'.format(self.NAME, len(self.workers)))
+        logging.debug(f'{self.NAME}: {len(self.workers)} worker threads started')
 
     def process_next_object(self, task, result):
         task.processed_analysis.update({self.NAME: {}})
@@ -132,34 +149,34 @@ def worker_processing_with_timeout(self, worker_id, next_task):
         result = manager.list()
         process = ExceptionSafeProcess(target=self.process_next_object, args=(next_task, result))
         process.start()
-        process.join(timeout=self.timeout)
+        process.join(timeout=self.TIMEOUT)
         if self.timeout_happened(process):
             self._handle_failed_analysis(next_task, process, worker_id, 'Timeout')
         elif process.exception:
             self._handle_failed_analysis(next_task, process, worker_id, 'Exception')
         else:
             self.out_queue.put(result.pop())
-            logging.debug('Worker {}: Finished {} analysis on {}'.format(worker_id, self.NAME, next_task.uid))
+            logging.debug(f'Worker {worker_id}: Finished {self.NAME} analysis on {next_task.uid}')
 
     def _handle_failed_analysis(self, fw_object, process, worker_id, cause: str):
         terminate_process_and_children(process)
-        fw_object.analysis_exception = (self.NAME, '{} occurred during analysis'.format(cause))
-        logging.error('Worker {}: {} during analysis {} on {}'.format(worker_id, cause, self.NAME, fw_object.uid))
+        fw_object.analysis_exception = (self.NAME, f'{cause} occurred during analysis')
+        logging.error(f'Worker {worker_id}: {cause} during analysis {self.NAME} on {fw_object.uid}')
         self.out_queue.put(fw_object)
 
     def worker(self, worker_id):
         while self.stop_condition.value == 0:
             try:
-                next_task = self.in_queue.get(timeout=float(self.config['ExpertSettings']['block_delay']))
-                logging.debug('Worker {}: Begin {} analysis on {}'.format(worker_id, self.NAME, next_task.uid))
+                next_task = self.in_queue.get(timeout=float(self.config['expert-settings']['block-delay']))
+                logging.debug(f'Worker {worker_id}: Begin {self.NAME} analysis on {next_task.uid}')
             except Empty:
                 self.active[worker_id].value = 0
             else:
                 self.active[worker_id].value = 1
                 next_task.processed_analysis.update({self.NAME: {}})
                 self.worker_processing_with_timeout(worker_id, next_task)
 
-        logging.debug('worker {} stopped'.format(worker_id))
+        logging.debug(f'worker {worker_id} stopped')
 
     def check_exceptions(self):
         return check_worker_exceptions(self.workers, 'Analysis', self.config, self.worker)

src/analysis/YaraPluginBase.py

@@ -3,6 +3,7 @@
 import re
 import subprocess
 from pathlib import Path
+from typing import Dict
 
 from analysis.PluginBase import AnalysisBasePlugin, PluginInitException
 from helperFunctions.fileSystem import get_src_dir
@@ -15,19 +16,20 @@ class YaraBasePlugin(AnalysisBasePlugin):
     NAME = 'Yara_Base_Plugin'
     DESCRIPTION = 'this is a Yara plugin'
     VERSION = '0.0'
+    FILE = None
 
-    def __init__(self, plugin_administrator, config=None, recursive=True, plugin_path=None):
+    def __init__(self, plugin_administrator, config=None, view_updater=None):
         '''
         recursive flag: If True recursively analyze included files
         propagate flag: If True add analysis result of child to parent object
         '''
         self.config = config
-        self.signature_path = self._get_signature_file(plugin_path) if plugin_path else None
+        self.signature_path = self._get_signature_file(self.FILE) if self.FILE else None
         if self.signature_path and not Path(self.signature_path).exists():
             logging.error(f'Signature file {self.signature_path} not found. Did you run "compile_yara_signatures.py"?')
             raise PluginInitException(plugin=self)
         self.SYSTEM_VERSION = self.get_yara_system_version()  # pylint: disable=invalid-name
-        super().__init__(plugin_administrator, config=config, recursive=recursive, plugin_path=plugin_path)
+        super().__init__(plugin_administrator, config=config, view_updater=view_updater)
 
     def get_yara_system_version(self):
         with subprocess.Popen(['yara', '--version'], stdout=subprocess.PIPE) as process:
@@ -62,7 +64,7 @@ def _get_signature_file(self, plugin_path):
 
     @staticmethod
     def _parse_yara_output(output):
-        resulting_matches = dict()
+        resulting_matches = {}
 
         match_blocks, rules = _split_output_in_rules_and_matches(output)
 
@@ -88,23 +90,18 @@ def _split_output_in_rules_and_matches(output):
     return match_blocks, rules
 
 
-def _append_match_to_result(match, resulting_matches, rule):
+def _append_match_to_result(match, resulting_matches: Dict[str, dict], rule):
     rule_name, meta_string, _, _ = rule
     _, offset, matched_tag, matched_string = match
-
-    meta_dict = _parse_meta_data(meta_string)
-
-    this_match = resulting_matches[rule_name] if rule_name in resulting_matches else dict(rule=rule_name, matches=True, strings=list(), meta=meta_dict)
-
-    this_match['strings'].append((int(offset, 16), matched_tag, matched_string.encode()))
-    resulting_matches[rule_name] = this_match
+    resulting_matches.setdefault(rule_name, dict(rule=rule_name, matches=True, strings=[], meta=_parse_meta_data(meta_string)))
+    resulting_matches[rule_name]['strings'].append((int(offset, 16), matched_tag, matched_string))
 
 
 def _parse_meta_data(meta_data_string):
     '''
     Will be of form 'item0=lowercaseboolean0,item1="value1",item2=value2,..'
     '''
-    meta_data = dict()
+    meta_data = {}
     for item in meta_data_string.split(','):
         if '=' in item:
             key, value = item.split('=', maxsplit=1)

src/check_signatures.py

@@ -28,9 +28,9 @@
 
 
 def _setup_argparser():
-    parser = argparse.ArgumentParser(description='{} - {}'.format(PROGRAM_NAME, PROGRAM_DESCRIPTION))
+    parser = argparse.ArgumentParser(description=f'{PROGRAM_NAME} - {PROGRAM_DESCRIPTION}')
     parser.add_argument('-V', '--version', action='version',
-                        version='{} {}'.format(PROGRAM_NAME, PROGRAM_VERSION))
+                        version=f'{PROGRAM_NAME} {PROGRAM_VERSION}')
     parser.add_argument('test_file', help='File containing the list of signatures')
     parser.add_argument('--yara_path', help='File or Folder containing yara signatures (Extension .yara mandatory)', default='software_signatures/')
     return parser.parse_args()
@@ -54,6 +54,6 @@ def _setup_logging():
     sig_tester = SignatureTesting()
     diff = sig_tester.check(args.yara_path, args.test_file)
     if diff:
-        logging.error('Missing yara signatures for: {}'.format(diff))
+        logging.error(f'Missing yara signatures for: {diff}')
     else:
         logging.info('Found all strings')

src/compare/PluginBase.py

@@ -10,8 +10,11 @@ class CompareBasePlugin(BasePlugin):
     This is the compare plug-in base class. All compare plug-ins should be derived from this class.
     '''
 
-    def __init__(self, plugin_administrator, config=None, db_interface=None, plugin_path=None):
-        super().__init__(plugin_administrator, config=config, plugin_path=plugin_path)
+    # must be set by the plugin:
+    FILE = None
+
+    def __init__(self, plugin_administrator, config=None, db_interface=None, view_updater=None):
+        super().__init__(plugin_administrator, config=config, plugin_path=self.FILE, view_updater=view_updater)
         self.database = db_interface
         self.register_plugin()
 
@@ -30,7 +33,7 @@ def compare(self, fo_list):
         '''
         missing_deps = _get_unmatched_dependencies(fo_list, self.DEPENDENCIES)
         if len(missing_deps) > 0:
-            return {'Compare Skipped': {'all': 'Required analysis not present: {}'.format(', '.join(missing_deps))}}
+            return {'Compare Skipped': {'all': f"Required analysis not present: {', '.join(missing_deps)}"}}
         return self.compare_function(fo_list)