Merge pull request #422 from CIRCL/misp_modules

Misp modules

OVERVIEW.md

@@ -97,12 +97,24 @@ Redis and ARDB overview
 | ------ | ------ | ------ |
 | ail:all_role | **role** | **int, role priority (1=admin)** |
 
+##### MISP Modules:
+
+| Set Key | Value |
+| ------ | ------ |
+| enabled_misp_modules | **module name** |
+
+| Key | Value |
+| ------ | ------ |
+| misp_module:**module name** | **module dict** |
+
 ##### Item Import:
 | Key | Value |
 | ------ | ------ |
 | **uuid**:isfile   | **boolean** |
 | **uuid**:paste_content | **item_content** |
 
+## DB2 - TermFreq:
+
 | Set Key | Value |
 | ------ | ------ |
 | submitted:uuid | **uuid** |

bin/lib/MispModules.py

@@ -0,0 +1,125 @@
+#!/usr/bin/python3
+
+import os
+import json
+import redis
+import requests
+import configparser
+
+misp_module_url = 'http://localhost:6666'
+
+default_config_path = os.path.join(os.environ['AIL_HOME'], 'configs', 'misp_modules.cfg')
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+
+config_loader = ConfigLoader.ConfigLoader()
+r_serv = config_loader.get_redis_conn("ARDB_DB")
+config_loader = None
+
+def init_config(config_path=default_config_path):
+    config = configparser.ConfigParser()
+    if os.path.isfile(config_path):
+        config.read(config_path)
+    else:
+        config.add_section('misp_modules')
+        config.set('misp_modules', 'url', 'http://localhost')
+        config.set('misp_modules', 'port', '6666')
+    return config
+
+def init_module_config(module_json, config, config_path=default_config_path):
+    if 'config' in module_json['meta']:
+        if module_json['meta']['config']:
+            if module_json['name'] not in config:
+                config.add_section(module_json['name'])
+            for config_var in module_json['meta']['config']:
+                if config_var not in config[module_json['name']]:
+                    config.set(module_json['name'], config_var, '')
+    return config
+
+def load_modules_list():
+    req = requests.get('{}/modules'.format(misp_module_url))
+    if req.status_code == 200:
+        all_misp_modules = req.json()
+        all_modules = []
+        for module_json in all_misp_modules:
+
+            #filter module-types
+            if 'hover' in module_json['meta']['module-type'] or 'expansion' in module_json['meta']['module-type']:
+                all_modules.append(module_json)
+
+            # # TODO: handle import/export modules
+
+        config = init_config()
+        r_serv.delete('misp_modules')
+        for module_json in all_modules:
+            config = init_module_config(module_json, config, config_path=default_config_path)
+            r_serv.hset('misp_modules', module_json['name'], json.dumps(module_json))
+
+        with open(default_config_path, 'w') as f:
+            config.write(f)
+
+    else:
+        print('Error: Module service not reachable.')
+
+
+def build_config_json(module_name):
+    misp_module_config = configparser.ConfigParser()
+    misp_module_config.read(default_config_path)
+    dict_config = {}
+    if module_name in misp_module_config:
+        for config_key in misp_module_config[module_name]:
+            config_value = misp_module_config[module_name][config_key]
+            if config_value:
+                dict_config[config_key] = config_value
+    return dict_config
+
+def build_enrichment_request_json(module_name, var_name, var_value):
+    # # TODO: add error handler
+    request_dict = {'module': module_name, var_name: var_value}
+    # add config
+    config_json = build_config_json(module_name)
+    if config_json:
+        request_dict['config'] = config_json
+    return json.dumps(request_dict)
+
+def misp_module_enrichment_request(misp_module_url, misp_module_port, request_content):
+    # # TODO: check if module is enabled
+    endpoint_url = '{}:{}/query'.format(misp_module_url, misp_module_port)
+    req = requests.post(endpoint_url, headers={'Content-Type': 'application/json'}, data=request_content)
+    if req.status_code == 200:
+        response = req.json()
+        if response:
+            return parse_module_enrichment_response(response)
+    else:
+        print('error: {} Enrichment service not reachable.'.format(req.status_code,))
+        return ''
+
+def parse_module_enrichment_response(misp_module_response):
+    print(misp_module_response)
+    response_values = []
+    if 'results' in misp_module_response:
+        # # TODO: handle misp_format (Attribute, Object, Tags)
+        response_types = []
+        for result in misp_module_response['results']:
+            # get all types
+            for resp_type in result['types']:
+                response_types.append(resp_type)
+            # get all values
+            for resp_value in result['values']:
+                response_values.append(resp_value)
+        # TODO: handle / verify / use response types
+        #print(response_types)
+    return response_values
+
+if __name__ == "__main__":
+
+    load_modules_list()
+
+    misp_module_url = 'http://localhost'
+    misp_module_port = 6666
+
+    bitcoin_address = 'bitcoin_address'
+    test_content = build_enrichment_request_json('btc_steroids', 'btc', bitcoin_address)
+    print(test_content)
+    misp_module_enrichment_request(misp_module_url, misp_module_port, test_content)

update/v2.5/Update.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import re
+import sys
+import time
+import redis
+import datetime
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+
+new_version = 'v2.5'
+
+if __name__ == '__main__':
+
+    start_deb = time.time()
+
+    config_loader = ConfigLoader.ConfigLoader()
+    r_serv = config_loader.get_redis_conn("ARDB_DB")
+    config_loader = None
+
+    r_serv.zadd('ail:all_role', 3, 'user')
+    r_serv.zadd('ail:all_role', 4, 'user_no_api')
+    r_serv.zadd('ail:all_role', 5, 'read_only')
+
+    for user in r_serv.hkeys(user:all):
+        r_serv.sadd('user_role:user', user)
+        r_serv.sadd('user_role:user_no_api', user)
+        r_serv.sadd('user_role:read_only', user)
+
+    #Set current ail version
+    r_serv.set('ail:version', new_version)
+
+    #Set current ail version
+    r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))

update/v2.5/Update.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
+
+export PATH=$AIL_HOME:$PATH
+export PATH=$AIL_REDIS:$PATH
+export PATH=$AIL_ARDB:$PATH
+export PATH=$AIL_BIN:$PATH
+export PATH=$AIL_FLASK:$PATH
+
+GREEN="\\033[1;32m"
+DEFAULT="\\033[0;39m"
+
+echo -e $GREEN"Shutting down AIL ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -ks
+wait
+
+bash ${AIL_BIN}/LAUNCH.sh -lav &
+wait
+echo ""
+
+echo ""
+echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
+echo ""
+python ${AIL_HOME}/update/v2.5/Update.py
+wait
+echo ""
+echo ""
+
+echo ""
+echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -ks
+wait
+
+exit 0

var/www/blueprints/correlation.py

@@ -18,7 +18,7 @@
 
 # Import Role_Manager
 from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity
-from Role_Manager import login_admin, login_analyst
+from Role_Manager import login_admin, login_analyst, login_read_only
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
 import Correlate_object
@@ -121,7 +121,7 @@ def get_card_metadata(object_type, correlation_id, type_id=None, expand_card=Fal
 # ============= ROUTES ==============
 @correlation.route('/correlation/show_correlation', methods=['GET', 'POST']) # GET + POST
 @login_required
-@login_analyst
+@login_read_only
 def show_correlation():
     if request.method == 'POST':
         object_type = request.form.get('object_type')
@@ -197,7 +197,7 @@ def show_correlation():
 
 @correlation.route('/correlation/graph_node_json')
 @login_required
-@login_analyst
+@login_read_only
 def graph_node_json(): # # TODO: use post
     correlation_id = request.args.get('correlation_id')
     type_id = request.args.get('type_id')

var/www/blueprints/crawler_splash.py

@@ -18,7 +18,7 @@
 
 # Import Role_Manager
 from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity
-from Role_Manager import login_admin, login_analyst
+from Role_Manager import login_admin, login_analyst, login_read_only
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
 import Tag
@@ -47,7 +47,7 @@ def api_validator(api_response):
 # add route : /crawlers/show_domain
 @crawler_splash.route('/crawlers/showDomain')
 @login_required
-@login_analyst
+@login_read_only
 def showDomain():
     domain_name = request.args.get('domain')
     epoch = request.args.get('epoch')

var/www/create_default_user.py

@@ -23,6 +23,9 @@
     if not r_serv.exists('ail:all_role'):
         r_serv.zadd('ail:all_role', 1, 'admin')
         r_serv.zadd('ail:all_role', 2, 'analyst')
+        r_serv.zadd('ail:all_role', 3, 'user')
+        r_serv.zadd('ail:all_role', 4, 'user_no_api')
+        r_serv.zadd('ail:all_role', 5, 'read_only')
 
     username = '[email protected]'
     password = gen_password()

var/www/modules/Role_Manager.py

@@ -16,7 +16,7 @@
 from flask import request, make_response, current_app
 
 login_manager = LoginManager()
-login_manager.login_view = 'role'
+login_manager.login_view = 'root.role'
 
 # CONFIG #
 config_loader = ConfigLoader.ConfigLoader()
@@ -68,7 +68,35 @@ def decorated_view(*args, **kwargs):
         return func(*args, **kwargs)
     return decorated_view
 
+def login_user(func):
+    @wraps(func)
+    def decorated_view(*args, **kwargs):
+        if not current_user.is_authenticated:
+            return login_manager.unauthorized()
+        elif (not current_user.is_in_role('user')):
+            return login_manager.unauthorized()
+        return func(*args, **kwargs)
+    return decorated_view
 
+def login_user_no_api(func):
+    @wraps(func)
+    def decorated_view(*args, **kwargs):
+        if not current_user.is_authenticated:
+            return login_manager.unauthorized()
+        elif (not current_user.is_in_role('user_no_api')):
+            return login_manager.unauthorized()
+        return func(*args, **kwargs)
+    return decorated_view
+
+def login_read_only(func):
+    @wraps(func)
+    def decorated_view(*args, **kwargs):
+        if not current_user.is_authenticated:
+            return login_manager.unauthorized()
+        elif (not current_user.is_in_role('read_only')):
+            return login_manager.unauthorized()
+        return func(*args, **kwargs)
+    return decorated_view
 
 ###############################################################
 ###############################################################
@@ -107,11 +135,15 @@ def create_user_db(username_id , password, default=False, role=None, update=Fals
     # create user token
     generate_new_token(username_id)
 
+    if not role:
+        role = 'read_only'
+
     if update:
         r_serv_db.hdel('user_metadata:{}'.format(username_id), 'change_passwd')
         # remove default user password file
         if username_id=='[email protected]':
             os.remove(default_passwd_file)
+        r_serv_db.hset('user:all', username_id, password_hash)
     else:
         if default:
             r_serv_db.hset('user_metadata:{}'.format(username_id), 'change_passwd', True)
@@ -121,7 +153,7 @@ def create_user_db(username_id , password, default=False, role=None, update=Fals
                     r_serv_db.sadd('user_role:{}'.format(role_to_add), username_id)
                 r_serv_db.hset('user_metadata:{}'.format(username_id), 'role', role)
 
-    r_serv_db.hset('user:all', username_id, password_hash)
+        r_serv_db.hset('user:all', username_id, password_hash)
 
 def edit_user_db(user_id, role, password=None):
     if password: