chg: [UI show Item] refactoring + bootstrap 4 migration

bin/RegexTracker.py

@@ -23,7 +23,7 @@
 import Tracker
 import regex_helper
 
-full_item_url = "/showsavedpaste/?paste="
+full_item_url = "/object/item?id="
 mail_body_template = "AIL Framework,\nNew occurrence for term tracked regex: {}\nitem id: {}\nurl: {}{}"
 
 dict_regex_tracked = Term.get_regex_tracked_words_dict()

bin/TermTrackerMod.py

@@ -20,7 +20,7 @@
 
 from lib import Tracker
 
-full_item_url = "/showsavedpaste/?paste="
+full_item_url = "/object/item/?id="
 
 mail_body_template = "AIL Framework,\nNew occurrence for term tracked term: {}\nitem id: {}\nurl: {}{}"
 

bin/export/Export.py

@@ -9,6 +9,24 @@
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
 import ConfigLoader
 
+sys.path.append('../../configs/keys')
+try:
+    from thehive4py.api import TheHiveApi
+    import thehive4py.exceptions
+    from theHiveKEYS import the_hive_url, the_hive_key, the_hive_verifycert
+    if the_hive_url == '':
+        is_hive_connected = False
+    else:
+        is_hive_connected = TheHiveApi(the_hive_url, the_hive_key, cert=the_hive_verifycert)
+except:
+    is_hive_connected = False
+if is_hive_connected != False:
+    try:
+        is_hive_connected.get_alert(0)
+        is_hive_connected = True
+    except thehive4py.exceptions.AlertException:
+        is_hive_connected = False
+
 ## LOAD CONFIG ##
 config_loader = ConfigLoader.ConfigLoader()
 r_serv_cache = config_loader.get_redis_conn("Redis_Cache")
@@ -37,6 +55,16 @@ def load_tags_to_export_in_cache():
         # save combinaison of tags in cache
         pass
 
+def is_hive_connected(): # # TODO: REFRACTOR, put in cache (with retry)
+    return is_hive_connected
+
+def get_item_hive_cases(item_id):
+    hive_case = r_serv_metadata.get('hive_cases:{}'.format(item_id))
+    if hive_case:
+        hive_case = the_hive_url + '/index.html#/case/{}/details'.format(hive_case)
+    return hive_case
+
+
 ###########################################################
 # # set default
 # if r_serv_db.get('hive:auto-alerts') is None:

bin/lib/Correlate_object.py

@@ -223,8 +223,8 @@ def get_item_url(correlation_name, value, correlation_type=None):
         endpoint = 'crawler_splash.showDomain'
         url = url_for(endpoint, domain=value)
     elif correlation_name == 'item':
-        endpoint = 'showsavedpastes.showsavedpaste'
-        url = url_for(endpoint, paste=value)
+        endpoint = 'objects_item.showItem'
+        url = url_for(endpoint, id=value)
     elif correlation_name == 'paste':                   ### # TODO:  remove me
         endpoint = 'showsavedpastes.showsavedpaste'
         url = url_for(endpoint, paste=value)

bin/lib/item_basic.py

@@ -113,9 +113,6 @@ def get_item_parent(item_id):
 def get_item_children(item_id):
     return list(r_serv_metadata.smembers('paste_children:{}'.format(item_id)))
 
-def add_item_parent(item_parent, item_id):
-    return item_basic.add_item_parent(item_parent, item_id)
-
 # # TODO:  handle domain last origin in domain lib
 def _delete_node(item_id):
     # only if item isn't deleted

bin/packages/Item.py

@@ -4,6 +4,7 @@
 import os
 import sys
 import redis
+import html2text
 
 from io import BytesIO
 
@@ -59,6 +60,9 @@ def get_item_basename(item_id):
 def get_item_size(item_id):
     return round(os.path.getsize(os.path.join(PASTES_FOLDER, item_id))/1024.0, 2)
 
+def get_item_encoding(item_id):
+    return None
+
 def get_lines_info(item_id, item_content=None):
     if not item_content:
         item_content = get_item_content(item_id)
@@ -73,9 +77,37 @@ def get_lines_info(item_id, item_content=None):
     return {'nb': nb_line, 'max_length': max_length}
 
 
+def get_item_metadata(item_id, item_content=None):
+    ## TODO: FIXME ##performance
+    # encoding
+    # language
+    # lines info
+
+    item_metadata = {}
+    item_metadata['date'] = get_item_date(item_id, add_separator=True)
+    item_metadata['source'] = get_source(item_id)
+    item_metadata['size'] = get_item_size(item_id)
+    item_metadata['encoding'] = get_item_encoding(item_id)
+    item_metadata['lines'] = get_lines_info(item_id, item_content=item_content)
+
+    return item_metadata
+
+def get_item_parent(item_id):
+    return item_basic.get_item_parent(item_id)
+
+def add_item_parent(item_parent, item_id):
+    return item_basic.add_item_parent(item_parent, item_id)
+
 def get_item_content(item_id):
     return item_basic.get_item_content(item_id)
 
+def get_item_content_html2text(item_id, item_content=None):
+    if not item_content:
+        item_content = get_item_content(item_id)
+    h = html2text.HTML2Text()
+    h.ignore_links = False
+    return h.handle(item_content)
+
 # API
 def get_item(request_dict):
     if not request_dict:
@@ -257,6 +289,18 @@ def get_item_list_desc(list_item_id):
 def is_crawled(item_id):
     return item_basic.is_crawled(item_id)
 
+def get_crawler_matadata(item_id, ltags=None):
+    dict_crawler = {}
+    if is_crawled(item_id):
+        dict_crawler['domain'] = get_item_domain(item_id)
+        if not ltags:
+            ltags = Tag.get_obj_tag(item_id)
+        dict_crawler['is_tags_safe'] = Tag.is_tags_safe(ltags)
+        dict_crawler['url'] = get_item_link(item_id)
+        dict_crawler['screenshot'] = get_item_screenshot(item_id)
+        dict_crawler['har'] = get_item_har_name(item_id)
+    return dict_crawler
+
 def is_onion(item_id):
     is_onion = False
     if len(is_onion) > 62:
@@ -293,7 +337,7 @@ def get_item_screenshot(item_id):
     return ''
 
 def get_item_har_name(item_id):
-    os.path.join(screenshot_directory, item_id) + '.json'
+    har_path = os.path.join(screenshot_directory, item_id) + '.json'
     if os.path.isfile(har_path):
         return har_path
     else:
@@ -322,6 +366,24 @@ def get_item_duplicate(item_id, r_list=True):
             return []
     return res
 
+def get_item_nb_duplicates(item_id):
+    return r_serv_metadata.scard('dup:{}'.format(item_id))
+
+def get_item_duplicates_dict(item_id):
+    dict_duplicates = {}
+    for duplicate in get_item_duplicate(item_id):
+        duplicate = duplicate[1:-1].replace('\'', '').replace(' ', '').split(',')
+        duplicate_id = duplicate[1]
+        if not duplicate_id in dict_duplicates:
+            dict_duplicates[duplicate_id] = {'date': get_item_date(duplicate_id, add_separator=True), 'algo': {}}
+        algo = duplicate[0]
+        if algo == 'tlsh':
+            similarity = 100 - int(duplicate[2])
+        else:
+            similarity = int(duplicate[2])
+        dict_duplicates[duplicate_id]['algo'][algo] = similarity
+    return dict_duplicates
+
 def add_item_duplicate(item_id, l_dup):
     for item_dup in l_dup:
         r_serv_metadata.sadd('dup:{}'.format(item_dup), item_id)

bin/trackers/Tracker_Yara.py

@@ -24,7 +24,7 @@
 import item_basic
 
 
-full_item_url = "/showsavedpaste/?paste="
+full_item_url = "/object/item?id="
 mail_body_template = "AIL Framework,\nNew YARA match: {}\nitem id: {}\nurl: {}{}"
 
 last_refresh = time.time()

requirements.txt

@@ -22,6 +22,7 @@ textblob
 #Tokeniser
 nltk
 
+html2text
 yara-python
 
 #Crawler

update/v3.3/Update.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+import time
+import redis
+import argparse
+import datetime
+import configparser
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+
+new_version = 'v3.3'
+
+if __name__ == '__main__':
+
+    start_deb = time.time()
+
+    config_loader = ConfigLoader.ConfigLoader()
+    r_serv_db = config_loader.get_redis_conn("ARDB_DB")
+    config_loader = None
+
+    #Set current ail version
+    r_serv_db.set('ail:version', new_version)
+
+    #Set current ail version
+    r_serv_db.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))

update/v3.3/Update.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
+
+export PATH=$AIL_HOME:$PATH
+export PATH=$AIL_REDIS:$PATH
+export PATH=$AIL_ARDB:$PATH
+export PATH=$AIL_BIN:$PATH
+export PATH=$AIL_FLASK:$PATH
+
+GREEN="\\033[1;32m"
+DEFAULT="\\033[0;39m"
+
+echo -e $GREEN"Shutting down AIL ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -ks
+wait
+
+bash ${AIL_BIN}/LAUNCH.sh -ldbv &
+wait
+echo ""
+
+# SUBMODULES #
+git submodule update
+
+# echo ""
+# echo -e $GREEN"installing KVORCKS ..."$DEFAULT
+# cd ${AIL_HOME}
+# test ! -d kvrocks/ && git clone https://github.com/bitleak/kvrocks.git
+# pushd kvrocks/
+# make -j4
+# popd
+
+echo -e $GREEN"Installing html2text ..."$DEFAULT
+pip3 install html2text
+
+echo ""
+echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
+echo ""
+python ${AIL_HOME}/update/v3.3/Update.py
+wait
+echo ""
+echo ""
+
+
+echo ""
+echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -ks
+wait
+
+exit 0

var/www/Flask_server.py

@@ -42,6 +42,8 @@
 from blueprints.correlation import correlation
 from blueprints.tags_ui import tags_ui
 from blueprints.import_export import import_export
+from blueprints.objects_item import objects_item
+from blueprints.old_endpoints import old_endpoints
 
 
 Flask_dir = os.environ['AIL_FLASK']
@@ -97,6 +99,8 @@
 app.register_blueprint(correlation, url_prefix=baseUrl)
 app.register_blueprint(tags_ui, url_prefix=baseUrl)
 app.register_blueprint(import_export, url_prefix=baseUrl)
+app.register_blueprint(objects_item, url_prefix=baseUrl)
+app.register_blueprint(old_endpoints, url_prefix=baseUrl)
 # =========       =========#
 
 # ========= Cookie name ========