Merge pull request #206 from CIRCL/tags

Addition of tagging system

.gitignore

@@ -7,10 +7,15 @@
 AILENV
 redis-leveldb
 redis
+ardb
+faup
+tlsh
 Blooms
 LEVEL_DB_DATA
 PASTES
-bin/indexdir/
+BASE64
+DATA_ARDB
+indexdir/
 logs/
 
 # Webstuff
@@ -20,12 +25,14 @@ var/www/static/
 !var/www/static/js/moduleTrending.js
 !var/www/static/js/plot-graph.js
 !var/www/static/js/trendingchart.js
-var/www/templates/header.html # auto-generated
+var/www/templates/header.html
 
 # Local config
 bin/packages/config.cfg
 
 # installed files
 nltk_data/
 doc/all_modules.txt
-doc/module-data-flow.png # auto-generated
+# auto generated
+doc/module-data-flow.png
+doc/data-flow.png

README.md

@@ -35,6 +35,7 @@ Features
 * Detect Amazon AWS and Google API keys
 * Detect Bitcoin address and Bitcoin private keys
 * Detect private keys and certificate
+* Tagging system with [MISP Galaxy](https://github.com/MISP/misp-galaxy) and [MISP Taxonomies](https://github.com/MISP/misp-taxonomies) tags
 
 Installation
 ------------
@@ -144,6 +145,11 @@ Browsing
 
 ![Browse-Pastes](./doc/screenshots/browse-important.png?raw=true "AIL framework browseImportantPastes")
 
+Tagging system
+--------
+
+![Tags](./doc/screenshots/tags.png?raw=true "AIL framework tags")
+
 Sentiment analysis
 ------------------
 

bin/ApiKey.py

@@ -41,15 +41,22 @@ def search_api_key(message):
             print(to_print)
             publisher.warning('{}Checked {} found Google API Key;{}'.format(
                 to_print, len(google_api_key), paste.p_path))
+            msg = 'infoleak:automatic-detection="google-api-key";{}'.format(filename)
+            p.populate_set_out(msg, 'Tags')
 
         if(len(aws_access_key) > 0 or len(aws_secret_key) > 0):
             print('found AWS key')
             print(to_print)
             total = len(aws_access_key) + len(aws_secret_key)
             publisher.warning('{}Checked {} found AWS Key;{}'.format(
                 to_print, total, paste.p_path))
+            msg = 'infoleak:automatic-detection="aws-key";{}'.format(filename)
+            p.populate_set_out(msg, 'Tags')
 
 
+        msg = 'infoleak:automatic-detection="api-key";{}'.format(filename)
+        p.populate_set_out(msg, 'Tags')
+
         msg = 'apikey;{}'.format(filename)
         p.populate_set_out(msg, 'alertHandler')
         #Send to duplicate

bin/Base64.py

@@ -65,6 +65,9 @@ def search_base64(content, message):
         msg = ('base64;{}'.format(message))
         p.populate_set_out( msg, 'alertHandler')
 
+        msg = 'infoleak:automatic-detection="base64";{}'.format(message)
+        p.populate_set_out(msg, 'Tags')
+
 def save_base64_as_file(decode, type, hash, json_data):
 
     filename_b64 = os.path.join(os.environ['AIL_HOME'],

bin/Bitcoin.py

@@ -63,7 +63,14 @@ def search_key(content, message, paste):
             publisher.warning(to_print)
             msg = ('bitcoin;{}'.format(message))
             p.populate_set_out( msg, 'alertHandler')
+
+            msg = 'infoleak:automatic-detection="bitcoin-address";{}'.format(message)
+            p.populate_set_out(msg, 'Tags')
+
             if(key):
+                msg = 'infoleak:automatic-detection="bitcoin-private-key";{}'.format(message)
+                p.populate_set_out(msg, 'Tags')
+
                 to_print = 'Bitcoin;{};{};{};'.format(paste.p_source, paste.p_date,
                                                     paste.p_name)
                 publisher.warning('{}Detected {} Bitcoin private key;{}'.format(

bin/Credential.py

@@ -105,6 +105,9 @@
             msg = 'credential;{}'.format(filepath)
             p.populate_set_out(msg, 'alertHandler')
 
+            msg = 'infoleak:automatic-detection="credential";{}'.format(filepath)
+            p.populate_set_out(msg, 'Tags')
+
             #Put in form, count occurences, then send to moduleStats
             creds_sites = {}
             site_occurence = re.findall(regex_site_for_stats, content)

bin/CreditCards.py

@@ -85,6 +85,9 @@
                     #send to Browse_warning_paste
                     msg = 'creditcard;{}'.format(filename)
                     p.populate_set_out(msg, 'alertHandler')
+
+                    msg = 'infoleak:automatic-detection="credit-card";{}'.format(filename)
+                    p.populate_set_out(msg, 'Tags')
                 else:
                     publisher.info('{}CreditCard related;{}'.format(to_print, paste.p_path))
         else:

bin/Cve.py

@@ -34,6 +34,9 @@ def search_cve(message):
         #send to Browse_warning_paste
         msg = 'cve;{}'.format(filepath)
         p.populate_set_out(msg, 'alertHandler')
+
+        msg = 'infoleak:automatic-detection="cve";{}'.format(filepath)
+        p.populate_set_out(msg, 'Tags')
         #Send to duplicate
         p.populate_set_out(filepath, 'Duplicate')
 

bin/Keys.py

@@ -28,47 +28,76 @@ def search_key(paste):
     if '-----BEGIN PGP MESSAGE-----' in content:
         publisher.warning('{} has a PGP enc message'.format(paste.p_name))
 
+        msg = 'infoleak:automatic-detection="pgp-message";{}'.format(message)
+        p.populate_set_out(msg, 'Tags')
         find = True
 
     if '-----BEGIN CERTIFICATE-----' in content:
         publisher.warning('{} has a certificate message'.format(paste.p_name))
+
+        msg = 'infoleak:automatic-detection="certificate";{}'.format(message)
+        p.populate_set_out(msg, 'Tags')
         find = True
 
     if '-----BEGIN RSA PRIVATE KEY-----' in content:
         publisher.warning('{} has a RSA private key message'.format(paste.p_name))
         print('rsa private key message found')
+
+        msg = 'infoleak:automatic-detection="rsa-private-key";{}'.format(message)
+        p.populate_set_out(msg, 'Tags')
         find = True
 
     if '-----BEGIN PRIVATE KEY-----' in content:
         publisher.warning('{} has a private key message'.format(paste.p_name))
         print('private key message found')
+
+        msg = 'infoleak:automatic-detection="private-key";{}'.format(message)
+        p.populate_set_out(msg, 'Tags')
         find = True
 
     if '-----BEGIN ENCRYPTED PRIVATE KEY-----' in content:
         publisher.warning('{} has an encrypted private key message'.format(paste.p_name))
         print('encrypted private key message found')
+
+        msg = 'infoleak:automatic-detection="encrypted-private-key";{}'.format(message)
+        p.populate_set_out(msg, 'Tags')
         find = True
 
     if '-----BEGIN OPENSSH PRIVATE KEY-----' in content:
         publisher.warning('{} has an openssh private key message'.format(paste.p_name))
         print('openssh private key message found')
+
+        msg = 'infoleak:automatic-detection="private-ssh-key";{}'.format(message)
+        p.populate_set_out(msg, 'Tags')
         find = True
 
     if '-----BEGIN OpenVPN Static key V1-----' in content:
         publisher.warning('{} has an openssh private key message'.format(paste.p_name))
         print('OpenVPN Static key message found')
+
+        msg = 'infoleak:automatic-detection="vpn-static-key";{}'.format(message)
+        p.populate_set_out(msg, 'Tags')
         find = True
 
     if '-----BEGIN DSA PRIVATE KEY-----' in content:
         publisher.warning('{} has a dsa private key message'.format(paste.p_name))
+
+        msg = 'infoleak:automatic-detection="dsa-private-key";{}'.format(message)
+        p.populate_set_out(msg, 'Tags')
         find = True
 
     if '-----BEGIN EC PRIVATE KEY-----' in content:
         publisher.warning('{} has an ec private key message'.format(paste.p_name))
+
+        msg = 'infoleak:automatic-detection="ec-private-key";{}'.format(message)
+        p.populate_set_out(msg, 'Tags')
         find = True
 
     if '-----BEGIN PGP PRIVATE KEY BLOCK-----' in content:
         publisher.warning('{} has a pgp private key block message'.format(paste.p_name))
+
+        msg = 'infoleak:automatic-detection="pgp-private-key";{}'.format(message)
+        p.populate_set_out(msg, 'Tags')
         find = True
 
     if find :

bin/LAUNCH.sh

@@ -160,6 +160,8 @@ function launching_scripts {
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "alertHandler" bash -c './alertHandler.py; read x'
     sleep 0.1
+    screen -S "Script_AIL" -X screen -t "Tags" bash -c './Tags.py; read x'
+    sleep 0.1
     screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'
 
 }
@@ -228,7 +230,7 @@ islogged=`screen -ls | egrep '[0-9]+.Logging_AIL' | cut -d. -f1`
 isqueued=`screen -ls | egrep '[0-9]+.Queue_AIL' | cut -d. -f1`
 isscripted=`screen -ls | egrep '[0-9]+.Script_AIL' | cut -d. -f1`
 
-options=("Redis" "Ardb" "Logs" "Queues" "Scripts" "Killall" "Shutdown" "Update-config")
+options=("Redis" "Ardb" "Logs" "Queues" "Scripts" "Killall" "Shutdown" "Update-config" "Update-thirdparty")
 
 menu() {
     echo "What do you want to Launch?:"
@@ -328,6 +330,17 @@ for i in ${!options[@]}; do
                     echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
                 fi
                 ;;
+            Update-thirdparty)
+                echo -e "\t* Updating thirdparty..."
+                bash -c "(cd ../var/www && ./update_thirdparty.sh)"
+                exitStatus=$?
+                if [ $exitStatus -ge 1 ]; then
+                    echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
+                    exit
+                else
+                    echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
+                fi
+                ;;
         esac
     fi
 done

bin/Mail.py

@@ -76,6 +76,9 @@
                         p.populate_set_out(filename, 'Duplicate')
                         p.populate_set_out('mail;{}'.format(filename), 'alertHandler')
 
+                        msg = 'infoleak:automatic-detection="mail";{}'.format(filename)
+                        p.populate_set_out(msg, 'Tags')
+
                     else:
                         publisher.info(to_print)
                 #Send to ModuleStats

bin/Onion.py

@@ -152,6 +152,9 @@ def fetch(p, r_cache, urls, domains, path):
                     for url in fetch(p, r_cache, urls, domains_list, path):
                         publisher.info('{}Checked {};{}'.format(to_print, url, PST.p_path))
                         p.populate_set_out('onion;{}'.format(PST.p_path), 'alertHandler')
+
+                        msg = 'infoleak:automatic-detection="onion";{}'.format(PST.p_path)
+                        p.populate_set_out(msg, 'Tags')
                 else:
                     publisher.info('{}Onion related;{}'.format(to_print, PST.p_path))
 

bin/Phone.py

@@ -36,6 +36,10 @@ def search_phone(message):
         msg = 'phone;{}'.format(message)
         p.populate_set_out(msg, 'alertHandler')
         #Send to duplicate
+
+        msg = 'infoleak:automatic-detection="phone-number";{}'.format(message)
+        p.populate_set_out(msg, 'Tags')
+
         p.populate_set_out(message, 'Duplicate')
         stats = {}
         for phone_number in results:

bin/SQLInjectionDetection.py

@@ -82,6 +82,9 @@ def analyse(url, path):
             p.populate_set_out(path, 'Duplicate')
             #send to Browse_warning_paste
             p.populate_set_out('sqlinjection;{}'.format(path), 'alertHandler')
+
+            msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path)
+            p.populate_set_out(msg, 'Tags')
         else:
             print("Potential SQL injection:")
             print(urllib.request.unquote(url))

bin/Tags.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+"""
+The Tags Module
+================================
+
+This module create tags.
+
+"""
+import redis
+
+import time
+
+from pubsublogger import publisher
+from Helper import Process
+from packages import Paste
+
+if __name__ == '__main__':
+
+    # Port of the redis instance used by pubsublogger
+    publisher.port = 6380
+    # Script is the default channel used for the modules.
+    publisher.channel = 'Script'
+
+    # Section name in bin/packages/modules.cfg
+    config_section = 'Tags'
+
+    # Setup the I/O queues
+    p = Process(config_section)
+
+    server = redis.StrictRedis(
+                host=p.config.get("ARDB_Tags", "host"),
+                port=p.config.get("ARDB_Tags", "port"),
+                db=p.config.get("ARDB_Tags", "db"),
+                decode_responses=True)
+
+    server_metadata = redis.StrictRedis(
+                host=p.config.get("ARDB_Metadata", "host"),
+                port=p.config.get("ARDB_Metadata", "port"),
+                db=p.config.get("ARDB_Metadata", "db"),
+                decode_responses=True)
+
+    # Sent to the logging a description of the module
+    publisher.info("Tags module started")
+
+    # Endless loop getting messages from the input queue
+    while True:
+        # Get one message from the input queue
+        message = p.get_from_set()
+
+        if message is None:
+            publisher.debug("{} queue is empty, waiting 10s".format(config_section))
+            time.sleep(10)
+            continue
+
+        else:
+            tag, path = message.split(';')
+            # add the tag to the tags word_list
+            res = server.sadd('list_tags', tag)
+            if res == 1:
+                print("new tags added : {}".format(tag))
+            # add the path to the tag set
+            res = server.sadd(tag, path)
+            if res == 1:
+                print("new paste: {}".format(path))
+                print("   tagged: {}".format(tag))
+            server_metadata.sadd('tag:'+path, tag)

bin/packages/Paste.py

@@ -340,7 +340,7 @@ def save_others_pastes_attribute_duplicate(self, list_value):
         Save a new duplicate on others pastes
         """
         for hash_type, path, percent, date in list_value:
-            to_add = [hash_type, self.p_path, percent, date]
+            to_add = (hash_type, self.p_path, percent, date)
             self.store_duplicate.sadd('dup:'+path,to_add)
 
     def _get_from_redis(self, r_serv):