From 849977f0c28142c6d445ca04fc3acd2a08127d29 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 9 Dec 2024 13:54:54 +0100 Subject: [PATCH] chg: [pubsublogger] remove old redis_logger --- bin/LAUNCH.sh | 6 +--- bin/core/Sync_module.py | 1 - bin/modules/ApiKey.py | 16 ++++------- bin/modules/Categ.py | 5 +--- bin/modules/Credential.py | 22 +++++---------- bin/modules/CreditCards.py | 17 ++++------- bin/modules/Cryptocurrencies.py | 10 +------ bin/modules/CveModule.py | 5 +--- bin/modules/DomClassifier.py | 10 ++----- bin/modules/Duplicates.py | 5 ++-- bin/modules/Global.py | 1 - bin/modules/IPAddress.py | 9 ++---- bin/modules/Iban.py | 21 ++++++-------- bin/modules/Keys.py | 29 +++++++++---------- bin/modules/LibInjection.py | 10 ++----- bin/modules/MISP_Thehive_Auto_Push.py | 3 +- bin/modules/Mail.py | 18 ++++++------ bin/modules/Onion.py | 17 +++++------ bin/modules/Phone.py | 6 ++-- bin/modules/SQLInjectionDetection.py | 17 ++++------- bin/modules/SentimentAnalysis.py | 4 +-- bin/modules/SubmitPaste.py | 39 +++++++++++++------------- bin/modules/Urls.py | 2 -- bin/modules/abstract_module.py | 10 ------- bin/trackers/Retro_Hunt.py | 12 +++----- bin/trackers/Tracker_Regex.py | 4 +-- bin/trackers/Tracker_Term.py | 7 ++--- bin/trackers/Tracker_Typo_Squatting.py | 4 +-- bin/trackers/Tracker_Yara.py | 5 +--- requirements.txt | 3 -- var/www/Flask_config.py | 8 ------ var/www/blueprints/ui_submit.py | 27 ------------------ 32 files changed, 112 insertions(+), 241 deletions(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 6c64fea2..0c0d3103 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -615,14 +615,13 @@ function launch_all { update; launch_redis; launch_kvrocks; - launch_logs; launch_scripts; launch_flask; } function menu_display { - options=("Redis" "Kvrocks" "Logs" "Scripts" "Flask" "Killall" "Update" "Update-config" "Update-thirdparty") + options=("Redis" "Kvrocks" "Scripts" "Flask" "Killall" "Update" "Update-config" "Update-thirdparty") menu() { echo "What do you want to Launch?:" @@ -653,9 +652,6 @@ function menu_display { Kvrocks) launch_kvrocks; ;; - Logs) - launch_logs; - ;; Scripts) launch_scripts; ;; diff --git a/bin/core/Sync_module.py b/bin/core/Sync_module.py index 6dc89672..cd39f8a4 100755 --- a/bin/core/Sync_module.py +++ b/bin/core/Sync_module.py @@ -88,7 +88,6 @@ def run(self): if self.last_refresh_queues < time.time(): timeout_processed_objs() self.last_refresh_queues = time.time() + 120 - self.redis_logger.debug('Timeout queues') # print('Timeout queues') # Get one message (paste) from the QueueIn (copy of Redis_Global publish) diff --git a/bin/modules/ApiKey.py b/bin/modules/ApiKey.py index 2a9d11f8..7f75c14a 100755 --- a/bin/modules/ApiKey.py +++ b/bin/modules/ApiKey.py @@ -20,7 +20,6 @@ # Import Project packages ################################## from modules.abstract_module import AbstractModule -from lib.objects.Items import Item class ApiKey(AbstractModule): """ApiKey module for AIL framework""" @@ -48,20 +47,19 @@ def __init__(self): def compute(self, message, r_result=False): score = message - item = self.get_obj() - item_content = item.get_content() + obj = self.get_obj() + content = obj.get_content() - google_api_key = self.regex_findall(self.re_google_api_key, item.get_id(), item_content, r_set=True) - aws_access_key = self.regex_findall(self.re_aws_access_key, item.get_id(), item_content, r_set=True) + google_api_key = self.regex_findall(self.re_google_api_key, obj.get_id(), content, r_set=True) + aws_access_key = self.regex_findall(self.re_aws_access_key, obj.get_id(), content, r_set=True) if aws_access_key: - aws_secret_key = self.regex_findall(self.re_aws_secret_key, item.get_id(), item_content, r_set=True) + aws_secret_key = self.regex_findall(self.re_aws_secret_key, obj.get_id(), content, r_set=True) if aws_access_key or google_api_key: - to_print = f'ApiKey;{item.get_source()};{item.get_date()};{item.get_basename()};' + to_print = obj.get_global_id() if google_api_key: print(f'found google api key: {to_print}') - self.redis_logger.warning(f'{to_print}Checked {len(google_api_key)} found Google API Key;{self.obj.get_global_id()}') tag = 'infoleak:automatic-detection="google-api-key"' self.add_message_to_queue(message=tag, queue='Tags') @@ -69,10 +67,8 @@ def compute(self, message, r_result=False): # # TODO: # FIXME: AWS regex/validate/sanitize KEY + SECRET KEY if aws_access_key: print(f'found AWS key: {to_print}') - self.redis_logger.warning(f'{to_print}Checked {len(aws_access_key)} found AWS Key;{self.obj.get_global_id()}') if aws_secret_key: print(f'found AWS secret key') - self.redis_logger.warning(f'{to_print}Checked {len(aws_secret_key)} found AWS secret Key;{self.obj.get_global_id()}') tag = 'infoleak:automatic-detection="aws-key"' self.add_message_to_queue(message=tag, queue='Tags') diff --git a/bin/modules/Categ.py b/bin/modules/Categ.py index 3cfdadb7..e73b429f 100755 --- a/bin/modules/Categ.py +++ b/bin/modules/Categ.py @@ -65,7 +65,7 @@ def __init__(self, categ_files_dir=os.path.join(os.environ['AIL_HOME'], 'files') self.matchingThreshold = config_loader.get_config_int("Categ", "matchingThreshold") self.reload_categ_words() - self.redis_logger.info("Script Categ started") + self.logger.info("Script Categ started") # # TODO: trigger reload on change ( save last reload time, ...) def reload_categ_words(self): @@ -103,9 +103,6 @@ def compute(self, message, r_result=False): print(msg, categ) self.add_message_to_queue(message=msg, queue=categ) - self.redis_logger.debug( - f'Categ;{obj.get_source()};{obj.get_date()};{obj.get_basename()};Detected {lenfound} as {categ};{obj.get_id()}') - if r_result: return categ_found diff --git a/bin/modules/Credential.py b/bin/modules/Credential.py index 6d0df6a7..60de7d3b 100755 --- a/bin/modules/Credential.py +++ b/bin/modules/Credential.py @@ -29,7 +29,6 @@ import os import sys import time -from datetime import datetime from pyfaup.faup import Faup sys.path.append(os.environ['AIL_BIN']) @@ -37,9 +36,7 @@ # Import Project packages ################################## from modules.abstract_module import AbstractModule -from lib.objects.Items import Item from lib import ConfigLoader -# from lib import Statistics class Credential(AbstractModule): @@ -80,40 +77,36 @@ def __init__(self): self.pending_seconds = 10 # Send module state to logs - self.redis_logger.info(f"Module {self.module_name} initialized") + self.logger.info(f"Module {self.module_name} initialized") def compute(self, message): - count = message - item = self.get_obj() + obj = self.get_obj() - item_content = item.get_content() + content = obj.get_content() # TODO: USE SETS # Extract all credentials - all_credentials = self.regex_findall(self.regex_cred, item.get_id(), item_content) + all_credentials = self.regex_findall(self.regex_cred, obj.get_id(), content) if all_credentials: nb_cred = len(all_credentials) message = f'Checked {nb_cred} credentials found.' - all_sites = self.regex_findall(self.regex_web, item.get_id(), item_content, r_set=True) + all_sites = self.regex_findall(self.regex_web, obj.get_id(), content, r_set=True) if all_sites: discovered_sites = ', '.join(all_sites) message += f' Related websites: {discovered_sites}' print(message) - to_print = f'Credential;{item.get_source()};{item.get_date()};{item.get_basename()};{message};{self.obj.get_global_id()}' - # num of creds above threshold, publish an alert if nb_cred > self.criticalNumberToAlert: print(f"========> Found more than 10 credentials in this file : {self.obj.get_global_id()}") - self.redis_logger.warning(to_print) tag = 'infoleak:automatic-detection="credential"' self.add_message_to_queue(message=tag, queue='Tags') - site_occurrence = self.regex_findall(self.regex_site_for_stats, item.get_id(), item_content) + site_occurrence = self.regex_findall(self.regex_site_for_stats, obj.get_id(), content) creds_sites = {} @@ -162,8 +155,7 @@ def compute(self, message): # for tld in nb_tlds: # Statistics.add_module_tld_stats_by_date('credential', date, tld, nb_tlds[tld]) else: - self.redis_logger.info(to_print) - print(f'found {nb_cred} credentials') + print(f'found {nb_cred} credentials {self.obj.get_global_id()}') # # TODO: # FIXME: TEMP DESABLE # # For searching credential in termFreq diff --git a/bin/modules/CreditCards.py b/bin/modules/CreditCards.py index bd0efe3b..d6abea0f 100755 --- a/bin/modules/CreditCards.py +++ b/bin/modules/CreditCards.py @@ -7,7 +7,7 @@ This module is consuming the Redis-list created by the Categ module. -It apply credit card regexes on item content and warn if a valid card number is found. +It apply credit card regexes on object content and warn if a valid card number is found. """ @@ -23,7 +23,6 @@ # Import Project packages ################################## from modules.abstract_module import AbstractModule -from lib.objects.Items import Item from packages import lib_refine class CreditCards(AbstractModule): @@ -68,10 +67,9 @@ def extract(self, obj, content, tag): return extracted def compute(self, message, r_result=False): - score = message - item = self.get_obj() - content = item.get_content() - all_cards = self.regex_findall(self.regex, item.id, content) + obj = self.get_obj() + content = obj.get_content() + all_cards = self.regex_findall(self.regex, obj.id, content) if len(all_cards) > 0: # self.logger.debug(f'All matching {all_cards}') @@ -84,11 +82,8 @@ def compute(self, message, r_result=False): creditcard_set.add(valid_card) # print(creditcard_set) - to_print = f'CreditCard;{item.get_source()};{item.get_date()};{item.get_basename()};' if creditcard_set: - mess = f'{to_print}Checked {len(creditcard_set)} valid number(s);{self.obj.get_global_id()}' - print(mess) - self.redis_logger.warning(mess) + print(f'{len(creditcard_set)} valid number(s);{self.obj.get_global_id()}') tag = 'infoleak:automatic-detection="credit-card"' self.add_message_to_queue(message=tag, queue='Tags') @@ -96,7 +91,7 @@ def compute(self, message, r_result=False): if r_result: return creditcard_set else: - self.redis_logger.info(f'{to_print}CreditCard related;{self.obj.get_global_id()}') + print(f'CreditCard related;{self.obj.get_global_id()}') if __name__ == '__main__': diff --git a/bin/modules/Cryptocurrencies.py b/bin/modules/Cryptocurrencies.py index cf36ed81..8b5607b8 100755 --- a/bin/modules/Cryptocurrencies.py +++ b/bin/modules/Cryptocurrencies.py @@ -28,7 +28,6 @@ ################################## from modules.abstract_module import AbstractModule from lib.objects.CryptoCurrencies import CryptoCurrency -from lib.objects.Items import Item ################################## ################################## @@ -151,17 +150,10 @@ def compute(self, message): # debug print(private_keys) - to_print = 'Cryptocurrency;{};{};{};'.format(item.get_source(), - item.get_date(), - item.get_basename()) - self.redis_logger.warning('{}Detected {} {} private key;{}'.format( - to_print, len(private_keys), currency['name'], self.obj.get_global_id())) else: private_keys = [] - to_print = f"{currency['name']} found: {len(addresses)} address and {len(private_keys)} private Keys" - print(to_print) - self.redis_logger.warning(to_print) + print(f"{currency['name']} found: {len(addresses)} address and {len(private_keys)} private Keys {self.obj.get_global_id()}") if __name__ == '__main__': diff --git a/bin/modules/CveModule.py b/bin/modules/CveModule.py index 1387686b..cb1a4c1c 100755 --- a/bin/modules/CveModule.py +++ b/bin/modules/CveModule.py @@ -44,7 +44,6 @@ def __init__(self): self.logger.info(f'Module {self.module_name} initialized') def compute(self, message): - count = message item = self.get_obj() item_id = item.get_id() @@ -56,9 +55,7 @@ def compute(self, message): cve = Cves.Cve(cve_id) cve.add(date, item) - warning = f'{self.obj.get_global_id()} contains CVEs {cves}' - print(warning) - self.redis_logger.warning(warning) + print(f'{self.obj.get_global_id()} contains CVEs {cves}') tag = 'infoleak:automatic-detection="cve"' # Send to Tags Queue diff --git a/bin/modules/DomClassifier.py b/bin/modules/DomClassifier.py index fa0a232f..45773dcf 100755 --- a/bin/modules/DomClassifier.py +++ b/bin/modules/DomClassifier.py @@ -58,10 +58,6 @@ def __init__(self): def compute(self, message, r_result=False): host = message - item = self.get_obj() - item_basename = item.get_basename() - item_date = item.get_date() - item_source = item.get_source() try: self.dom_classifier.text(rawtext=host) @@ -82,19 +78,19 @@ def compute(self, message, r_result=False): localizeddomains = self.dom_classifier.include(expression=self.cc_tld) if localizeddomains: print(localizeddomains) - self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc_tld};{self.obj.get_global_id()}") + self.logger.info(f"{localizeddomains} located in {self.cc_tld};{self.obj.get_global_id()}") if self.cc: localizeddomains = self.dom_classifier.localizedomain(cc=self.cc) if localizeddomains: print(localizeddomains) - self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc};{self.obj.get_global_id()}") + self.logger.info(f"{localizeddomains} located in {self.cc};{self.obj.get_global_id()}") if r_result: return self.dom_classifier.vdomain except IOError as err: - self.redis_logger.error(f"Duplicate;{item_source};{item_date};{item_basename};CRC Checksum Failed") + self.logger.error(f"{self.obj.get_global_id()};CRC Checksum Failed") raise Exception(f"CRC Checksum Failed on: {self.obj.get_global_id()}") diff --git a/bin/modules/Duplicates.py b/bin/modules/Duplicates.py index edee6025..553fc326 100755 --- a/bin/modules/Duplicates.py +++ b/bin/modules/Duplicates.py @@ -27,7 +27,6 @@ from modules.abstract_module import AbstractModule from lib.ConfigLoader import ConfigLoader from lib import Duplicate -from lib.objects.Items import Item class Duplicates(AbstractModule): @@ -92,11 +91,11 @@ def compute(self, message): Duplicate.save_object_hash(algo, curr_date_ymonth, self.algos[algo]['hash'], item.get_id()) if nb_duplicates: - self.redis_logger.info(f'Duplicate;{item.get_source()};{item.get_date()};{item.get_basename()};Detected {nb_duplicates};{self.obj.get_global_id()}') + self.logger.info(f'Duplicates {nb_duplicates};{self.obj.get_global_id()}') y = time.time() print(f'{self.obj.get_global_id()} Processed in {y-x} sec') - # self.redis_logger.debug('{}Processed in {} sec'.format(to_print, y-x)) + # self.logger.debug('{}Processed in {} sec'.format(to_print, y-x)) if __name__ == "__main__": diff --git a/bin/modules/Global.py b/bin/modules/Global.py index f5bd5d30..71b00ad4 100755 --- a/bin/modules/Global.py +++ b/bin/modules/Global.py @@ -74,7 +74,6 @@ def computeNone(self): if int(difftime) > 30: to_print = f'Global; ; ; ;glob Processed {self.processed_item} item(s) in {difftime} s' print(to_print) - self.redis_logger.debug(to_print) self.time_last_stats = time.time() self.processed_item = 0 diff --git a/bin/modules/IPAddress.py b/bin/modules/IPAddress.py index 8899ca43..281d553a 100755 --- a/bin/modules/IPAddress.py +++ b/bin/modules/IPAddress.py @@ -25,7 +25,6 @@ ################################## from modules.abstract_module import AbstractModule from lib.ConfigLoader import ConfigLoader -from lib.objects.Items import Item from lib import regex_helper # TODO REWRITE ME -> PERF + IPV6 + Tracker ? @@ -66,11 +65,11 @@ def compute(self, message, r_result=False): if not self.ip_networks: return None - item = self.get_obj() - content = item.get_content() + obj = self.get_obj() + content = obj.get_content() # list of the regex results in the Item - results = self.regex_findall(self.re_ipv4, item.get_id(), content) + results = self.regex_findall(self.re_ipv4, obj.get_id(), content) results = set(results) matching_ips = [] for ip in results: @@ -83,7 +82,6 @@ def compute(self, message, r_result=False): if len(matching_ips) > 0: self.logger.info(f'{self.obj.get_global_id()} contains {len(matching_ips)} IPs') - self.redis_logger.warning(f'{self.obj.get_global_id()} contains IPs') # Tag message with IP tag = 'infoleak:automatic-detection="ip"' @@ -93,4 +91,3 @@ def compute(self, message, r_result=False): if __name__ == "__main__": module = IPAddress() module.run() - # module.compute('submitted/2023/05/15/submitted_8a6136c2-c7f2-4c9e-8f29-e1a62315b482.gz') diff --git a/bin/modules/Iban.py b/bin/modules/Iban.py index 8c0fafb0..1b83e11e 100755 --- a/bin/modules/Iban.py +++ b/bin/modules/Iban.py @@ -24,8 +24,7 @@ # Import Project packages ################################## from modules.abstract_module import AbstractModule -from lib.objects.Items import Item -from lib.ConfigLoader import ConfigLoader +# from lib.ConfigLoader import ConfigLoader # from lib import Statistics class Iban(AbstractModule): @@ -40,7 +39,7 @@ class Iban(AbstractModule): def __init__(self, queue=True): super(Iban, self).__init__(queue=queue) - # Waiting time in secondes between to message proccessed + # Waiting time in seconds between to message processed self.pending_seconds = 10 self.regex_timeout = 30 @@ -49,7 +48,7 @@ def __init__(self, queue=True): self.iban_regex_verify = re.compile(r'^([A-Z]{2})([0-9]{2})([A-Z0-9]{9,30})$') # Send module state to logs - self.redis_logger.info(f'Module {self.module_name} initialized') + self.logger.info(f'Module {self.module_name} initialized') def get_iban_number(self, iban): return (iban[4:] + iban[:4]).translate(Iban.LETTERS_IBAN) @@ -73,29 +72,27 @@ def extract(self, obj, content, tag): return extracted def compute(self, message): - item = self.get_obj() - item_id = item.get_id() + obj = self.get_obj() + obj_id = obj.get_id() - ibans = self.regex_findall(self.iban_regex, item_id, item.get_content()) + ibans = self.regex_findall(self.iban_regex, obj_id, obj.get_content()) if ibans: valid_ibans = set() for iban in ibans: iban = iban[1:-1].replace("'", "").split(',') iban = iban[0]+iban[1]+iban[2] iban = ''.join(e for e in iban if e.isalnum()) - if self.regex_findall(self.iban_regex_verify, item_id, iban): + if self.regex_findall(self.iban_regex_verify, obj_id, iban): print(f'checking {iban}') if self.is_valid_iban(iban): valid_ibans.add(iban) if valid_ibans: - print(f'{valid_ibans} ibans {item_id}') - date = datetime.datetime.now().strftime("%Y%m") + print(f'{valid_ibans} ibans {self.obj.get_global_id()}') + # date = datetime.datetime.now().strftime("%Y%m") # for iban in valid_ibans: # Statistics.add_module_tld_stats_by_date('iban', date, iban[0:2], 1) - to_print = f'Iban;{item.get_source()};{item.get_date()};{item.get_basename()};' - self.redis_logger.warning(f'{to_print}Checked found {len(valid_ibans)} IBAN;{self.obj.get_global_id()}') # Tags tag = 'infoleak:automatic-detection="iban"' self.add_message_to_queue(message=tag, queue='Tags') diff --git a/bin/modules/Keys.py b/bin/modules/Keys.py index 2fdb96f8..45f4335e 100755 --- a/bin/modules/Keys.py +++ b/bin/modules/Keys.py @@ -24,7 +24,6 @@ # Import Project packages ################################## from modules.abstract_module import AbstractModule -from lib.objects.Items import Item class KeyEnum(Enum): @@ -56,14 +55,14 @@ def __init__(self): self.pending_seconds = 1 def compute(self, message): - item = self.get_obj() - content = item.get_content() + obj = self.get_obj() + content = obj.get_content() # find = False get_pgp_content = False if KeyEnum.PGP_MESSAGE.value in content: - self.redis_logger.warning(f'{self.obj.get_global_id()} has a PGP enc message') + print(f'{self.obj.get_global_id()} has a PGP enc message') tag = 'infoleak:automatic-detection="pgp-message"' self.add_message_to_queue(message=tag, queue='Tags') @@ -81,21 +80,21 @@ def compute(self, message): get_pgp_content = True if KeyEnum.PGP_PRIVATE_KEY_BLOCK.value in content: - self.redis_logger.warning(f'{self.obj.get_global_id()} has a pgp private key block message') + print(f'{self.obj.get_global_id()} has a pgp private key block message') tag = 'infoleak:automatic-detection="pgp-private-key"' self.add_message_to_queue(message=tag, queue='Tags') get_pgp_content = True if KeyEnum.CERTIFICATE.value in content: - self.redis_logger.warning(f'{self.obj.get_global_id()} has a certificate message') + print(f'{self.obj.get_global_id()} has a certificate message') tag = 'infoleak:automatic-detection="certificate"' self.add_message_to_queue(message=tag, queue='Tags') # find = True if KeyEnum.RSA_PRIVATE_KEY.value in content: - self.redis_logger.warning(f'{self.obj.get_global_id()} has a RSA private key message') + print(f'{self.obj.get_global_id()} has a RSA private key message') print('rsa private key message found') tag = 'infoleak:automatic-detection="rsa-private-key"' @@ -103,7 +102,7 @@ def compute(self, message): # find = True if KeyEnum.PRIVATE_KEY.value in content: - self.redis_logger.warning(f'{self.obj.get_global_id()} has a private key message') + print(f'{self.obj.get_global_id()} has a private key message') print('private key message found') tag = 'infoleak:automatic-detection="private-key"' @@ -111,7 +110,7 @@ def compute(self, message): # find = True if KeyEnum.ENCRYPTED_PRIVATE_KEY.value in content: - self.redis_logger.warning(f'{self.obj.get_global_id()} has an encrypted private key message') + print(f'{self.obj.get_global_id()} has an encrypted private key message') print('encrypted private key message found') tag = 'infoleak:automatic-detection="encrypted-private-key"' @@ -119,7 +118,7 @@ def compute(self, message): # find = True if KeyEnum.OPENSSH_PRIVATE_KEY.value in content: - self.redis_logger.warning(f'{self.obj.get_global_id()} has an openssh private key message') + print(f'{self.obj.get_global_id()} has an openssh private key message') print('openssh private key message found') tag = 'infoleak:automatic-detection="private-ssh-key"' @@ -127,7 +126,7 @@ def compute(self, message): # find = True if KeyEnum.SSH2_ENCRYPTED_PRIVATE_KEY.value in content: - self.redis_logger.warning(f'{self.obj.get_global_id()} has an ssh2 private key message') + print(f'{self.obj.get_global_id()} has an ssh2 private key message') print('SSH2 private key message found') tag = 'infoleak:automatic-detection="private-ssh-key"' @@ -135,7 +134,7 @@ def compute(self, message): # find = True if KeyEnum.OPENVPN_STATIC_KEY_V1.value in content: - self.redis_logger.warning(f'{self.obj.get_global_id()} has an openssh private key message') + print(f'{self.obj.get_global_id()} has an openssh private key message') print('OpenVPN Static key message found') tag = 'infoleak:automatic-detection="vpn-static-key"' @@ -143,21 +142,21 @@ def compute(self, message): # find = True if KeyEnum.DSA_PRIVATE_KEY.value in content: - self.redis_logger.warning(f'{self.obj.get_global_id()} has a dsa private key message') + print(f'{self.obj.get_global_id()} has a dsa private key message') tag = 'infoleak:automatic-detection="dsa-private-key"' self.add_message_to_queue(message=tag, queue='Tags') # find = True if KeyEnum.EC_PRIVATE_KEY.value in content: - self.redis_logger.warning(f'{self.obj.get_global_id()} has an ec private key message') + print(f'{self.obj.get_global_id()} has an ec private key message') tag = 'infoleak:automatic-detection="ec-private-key"' self.add_message_to_queue(message=tag, queue='Tags') # find = True if KeyEnum.PUBLIC_KEY.value in content: - self.redis_logger.warning(f'{self.obj.get_global_id()} has a public key message') + print(f'{self.obj.get_global_id()} has a public key message') tag = 'infoleak:automatic-detection="public-key"' self.add_message_to_queue(message=tag, queue='Tags') diff --git a/bin/modules/LibInjection.py b/bin/modules/LibInjection.py index fc7a5956..528473d7 100755 --- a/bin/modules/LibInjection.py +++ b/bin/modules/LibInjection.py @@ -34,10 +34,9 @@ def __init__(self): self.faup = Faup() - self.redis_logger.info(f"Module: {self.module_name} Launched") + self.logger.info(f"Module: {self.module_name} Launched") def compute(self, message): - item = self.get_obj() url = message self.faup.decode(url) @@ -66,12 +65,9 @@ def compute(self, message): # print(f'query is sqli : {result_query}') if result_path['sqli'] is True or result_query['sqli'] is True: - item_id = item.get_id() - print(f"Detected (libinjection) SQL in URL: {item_id}") - print(unquote(url)) - to_print = f'LibInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{self.obj.get_global_id()}' - self.redis_logger.warning(to_print) + self.logger.info(f'Detected SQL in URL;{self.obj.get_global_id()}') + print(unquote(url)) # Add tag tag = 'infoleak:automatic-detection="sql-injection"' diff --git a/bin/modules/MISP_Thehive_Auto_Push.py b/bin/modules/MISP_Thehive_Auto_Push.py index b913c350..0a178719 100755 --- a/bin/modules/MISP_Thehive_Auto_Push.py +++ b/bin/modules/MISP_Thehive_Auto_Push.py @@ -18,7 +18,6 @@ ################################## from modules.abstract_module import AbstractModule from lib.exceptions import MISPConnectionError -from lib.objects.Items import Item from lib import Tag from exporter.MISPExporter import MISPExporterAutoDaily from exporter.TheHiveExporter import TheHiveExporterAlertTag @@ -43,7 +42,7 @@ def compute(self, message): if self.last_refresh < Tag.get_last_auto_push_refreshed() < 0: self.tags = Tag.refresh_auto_push() self.last_refresh = time.time() - self.redis_logger.info('Tags Auto Push refreshed') + self.logger.debug('Tags Auto Push refreshed') tag = message item = self.get_obj() diff --git a/bin/modules/Mail.py b/bin/modules/Mail.py index 29477ed2..2fd431e4 100755 --- a/bin/modules/Mail.py +++ b/bin/modules/Mail.py @@ -95,20 +95,20 @@ def check_mx_record(self, set_mxdomains): # print() except dns.resolver.NoNameservers: - self.redis_logger.debug('NoNameserver, No non-broken nameservers are available to answer the query.') + self.logger.debug('NoNameserver, No non-broken nameservers are available to answer the query.') print('NoNameserver, No non-broken nameservers are available to answer the query.') except dns.resolver.NoAnswer: - self.redis_logger.debug('NoAnswer, The response did not contain an answer to the question.') + self.logger.debug('NoAnswer, The response did not contain an answer to the question.') print('NoAnswer, The response did not contain an answer to the question.') except dns.name.EmptyLabel: - self.redis_logger.debug('SyntaxError: EmptyLabel') + self.logger.debug('SyntaxError: EmptyLabel') print('SyntaxError: EmptyLabel') except dns.resolver.NXDOMAIN: # save_mxdomain_in_cache(mxdomain) - self.redis_logger.debug('The query name does not exist.') + self.logger.debug('The query name does not exist.') print('The query name does not exist.') except dns.name.LabelTooLong: - self.redis_logger.debug('The Label is too long') + self.logger.debug('The Label is too long') print('The Label is too long') except dns.exception.Timeout: print('dns timeout') @@ -134,9 +134,7 @@ def extract(self, obj, content, tag): # # TODO: sanitize mails def compute(self, message): - score = message item = self.get_obj() - item_date = item.get_date() mails = self.regex_findall(self.email_regex, item.id, item.get_content()) mxdomains_email = {} @@ -171,15 +169,15 @@ def compute(self, message): # for tld in mx_tlds: # Statistics.add_module_tld_stats_by_date('mail', item_date, tld, mx_tlds[tld]) - msg = f'Mails;{item.get_source()};{item_date};{item.get_basename()};Checked {num_valid_email} e-mail(s);{self.obj.get_global_id()}' + msg = f'Checked {num_valid_email} e-mail(s);{self.obj.get_global_id()}' if num_valid_email > self.mail_threshold: print(f'{item.id} Checked {num_valid_email} e-mail(s)') - self.redis_logger.warning(msg) + self.logger.info(msg) # Tags tag = 'infoleak:automatic-detection="mail"' self.add_message_to_queue(message=tag, queue='Tags') elif num_valid_email > 0: - self.redis_logger.info(msg) + self.logger.info(msg) if __name__ == '__main__': diff --git a/bin/modules/Onion.py b/bin/modules/Onion.py index 02093a9f..f000bf43 100755 --- a/bin/modules/Onion.py +++ b/bin/modules/Onion.py @@ -4,7 +4,7 @@ The Onion Module ============================ -This module extract url from item and returning only ones which are tor +This module extract url from object and returning only ones which are tor related (.onion). All These urls are send to the crawler discovery queue. Requirements @@ -69,12 +69,11 @@ def compute(self, message): onion_urls = [] domains = [] - score = message - item = self.get_obj() - item_content = item.get_content() + obj = self.get_obj() + content = obj.get_content() # max execution time on regex - res = self.regex_findall(self.onion_regex, item.get_id(), item_content) + res = self.regex_findall(self.onion_regex, obj.get_id(), content) for x in res: # String to tuple x = x[2:-2].replace(" '", "").split("',") @@ -92,16 +91,14 @@ def compute(self, message): if onion_urls: if crawlers.is_crawler_activated(): for domain in domains: - task_uuid = crawlers.create_task(domain, parent=item.get_id(), priority=0, + task_uuid = crawlers.create_task(domain, parent=obj.get_id(), priority=0, har=self.har, screenshot=self.screenshot) if task_uuid: print(f'{domain} added to crawler queue: {task_uuid}') else: - to_print = f'Onion;{item.get_source()};{item.get_date()};{item.get_basename()};' - print(f'{to_print}Detected {len(domains)} .onion(s);{self.obj.get_global_id()}') - self.redis_logger.warning(f'{to_print}Detected {len(domains)} .onion(s);{self.obj.get_global_id()}') + print(f'Detected {len(domains)} .onion(s);{self.obj.get_global_id()}') - # TAG Item + # TAG Object tag = 'infoleak:automatic-detection="onion"' self.add_message_to_queue(message=tag, queue='Tags') diff --git a/bin/modules/Phone.py b/bin/modules/Phone.py index 22686649..2cc22acf 100755 --- a/bin/modules/Phone.py +++ b/bin/modules/Phone.py @@ -62,7 +62,7 @@ def compute(self, message): tag = 'infoleak:automatic-detection="phone-number"' self.add_message_to_queue(message=tag, queue='Tags') - self.redis_logger.warning(f'{self.obj.get_global_id()} contains {len(phone)} Phone numbers') + self.logger.info(f'{self.obj.get_global_id()} contains {len(phone)} Phone numbers') # # List of the regex results in the Item, may be null # results = self.REG_PHONE.findall(content) @@ -70,7 +70,7 @@ def compute(self, message): # # If the list is greater than 4, we consider the Item may contain a list of phone numbers # if len(results) > 4: # self.logger.debug(results) - # self.redis_logger.warning(f'{item.get_id()} contains PID (phone numbers)') + # self.logger.info(f'{item.get_id()} contains PID (phone numbers)') # # msg = f'infoleak:automatic-detection="phone-number";{item.get_id()}' # self.add_message_to_queue(msg, 'Tags') @@ -88,7 +88,7 @@ def compute(self, message): # pass # for country_code in stats: # if stats[country_code] > 4: - # self.redis_logger.warning(f'{item.get_id()} contains Phone numbers with country code {country_code}') + # self.logger.info(f'{item.get_id()} contains Phone numbers with country code {country_code}') if __name__ == '__main__': diff --git a/bin/modules/SQLInjectionDetection.py b/bin/modules/SQLInjectionDetection.py index 953fe82e..ab0acf4a 100755 --- a/bin/modules/SQLInjectionDetection.py +++ b/bin/modules/SQLInjectionDetection.py @@ -16,8 +16,7 @@ import re import urllib.request -from datetime import datetime -from pyfaup.faup import Faup +# from pyfaup.faup import Faup from urllib.parse import unquote sys.path.append(os.environ['AIL_BIN']) @@ -25,8 +24,7 @@ # Import Project packages ################################## from modules.abstract_module import AbstractModule -from lib.ConfigLoader import ConfigLoader -from lib.objects.Items import Item +# from lib.ConfigLoader import ConfigLoader # from lib import Statistics class SQLInjectionDetection(AbstractModule): @@ -39,22 +37,19 @@ class SQLInjectionDetection(AbstractModule): def __init__(self): super(SQLInjectionDetection, self).__init__() - self.faup = Faup() + # self.faup = Faup() self.logger.info(f"Module: {self.module_name} Launched") def compute(self, message): url = message - item = self.get_obj() if self.is_sql_injection(url): - self.faup.decode(url) - url_parsed = self.faup.get() + # self.faup.decode(url) + # url_parsed = self.faup.get() - print(f"Detected SQL in URL: {item.id}") + print(f"Detected SQL in URL: {self.obj.get_global_id()}") print(urllib.request.unquote(url)) - to_print = f'SQLInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{self.obj.get_global_id()}' - self.redis_logger.warning(to_print) # Tag tag = f'infoleak:automatic-detection="sql-injection"' diff --git a/bin/modules/SentimentAnalysis.py b/bin/modules/SentimentAnalysis.py index a5afc3f7..2093cae8 100755 --- a/bin/modules/SentimentAnalysis.py +++ b/bin/modules/SentimentAnalysis.py @@ -167,11 +167,11 @@ def analyse(self, message): provider_timestamp = provider + '_' + str(timestamp) self.db.incr('UniqID') UniqID = self.db.get('UniqID') - self.redis_logger.debug(f'{provider_timestamp}->{UniqID}dropped{num_line_removed}lines') + self.logger.debug(f'{provider_timestamp}->{UniqID}dropped{num_line_removed}lines') self.db.sadd(provider_timestamp, UniqID) self.db.set(UniqID, avg_score) else: - self.redis_logger.debug(f'Dropped:{p_MimeType}') + self.logger.debug(f'Dropped:{p_MimeType}') def isJSON(self, content): diff --git a/bin/modules/SubmitPaste.py b/bin/modules/SubmitPaste.py index 4e264a73..5ac23070 100755 --- a/bin/modules/SubmitPaste.py +++ b/bin/modules/SubmitPaste.py @@ -137,22 +137,21 @@ def _manage_file(self, uuid, file_full_path, ltags, ltagsgalaxies, source): """ Create a paste for given file """ - self.redis_logger.debug('manage') if os.path.exists(file_full_path): - self.redis_logger.debug(f'file exists {file_full_path}') + self.logger.debug(f'file exists {file_full_path}') file_size = os.stat(file_full_path).st_size - self.redis_logger.debug(f'file size {file_size}') + self.logger.debug(f'file size {file_size}') # Verify file length if file_size < SubmitPaste.FILE_MAX_SIZE: # TODO sanitize filename filename = file_full_path.split('/')[-1] - self.redis_logger.debug(f'sanitize filename {filename}') - self.redis_logger.debug('file size allowed') + self.logger.debug(f'sanitize filename {filename}') + self.logger.debug('file size allowed') if not '.' in filename: - self.redis_logger.debug('no extension for filename') + self.logger.debug('no extension for filename') try: # Read file with open(file_full_path,'r') as f: @@ -165,14 +164,14 @@ def _manage_file(self, uuid, file_full_path, ltags, ltagsgalaxies, source): else: file_type = filename.rsplit('.', 1)[1] file_type = file_type.lower() - self.redis_logger.debug(f'file ext {file_type}') + self.logger.debug(f'file ext {file_type}') if file_type in SubmitPaste.ALLOWED_EXTENSIONS: - self.redis_logger.debug('Extension allowed') + self.logger.debug('Extension allowed') # TODO enum of possible file extension ? # TODO verify file hash with virus total ? if not self._is_compressed_type(file_type): - self.redis_logger.debug('Plain text file') + self.logger.debug('Plain text file') # plain txt file with open(file_full_path,'r') as f: content = f.read() @@ -197,7 +196,7 @@ def _manage_file(self, uuid, file_full_path, ltags, ltagsgalaxies, source): # except: # self.abord_file_submission(uuid, "file decompression error") # raise - # self.redis_logger.debug('unpacking {} file'.format(files.unpacker)) + # self.logger.debug('unpacking {} file'.format(files.unpacker)) # if(not files.children): # self.abord_file_submission(uuid, "Empty compressed file") # raise @@ -209,11 +208,11 @@ def _manage_file(self, uuid, file_full_path, ltags, ltagsgalaxies, source): # self.create_paste(uuid, child.contents, ltags, ltagsgalaxies, uuid+'_'+ str(n) , source) # n = n + 1 # else: - # self.redis_logger.error("Error in module %s: bad extention"%(self.module_name)) + # self.logger.error("Error in module %s: bad extention"%(self.module_name)) # self.addError(uuid, 'Bad file extension: {}'.format(child.filename.decode()) ) # except FileNotFoundError: - # self.redis_logger.error("Error in module %s: file not found"%(self.module_name)) + # self.logger.error("Error in module %s: file not found"%(self.module_name)) # self.addError(uuid, 'File not found: {}'.format(file_full_path), uuid ) else: @@ -248,7 +247,7 @@ def remove_submit_uuid(self, uuid): # delete uuid self.r_serv_db.srem('submitted:uuid', uuid) - self.redis_logger.debug(f'{uuid} all file submitted') + self.logger.debug(f'{uuid} all file submitted') print(f'{uuid} all file submitted') def create_paste(self, uuid, paste_content, ltags, ltagsgalaxies, name, source=None): @@ -262,11 +261,11 @@ def create_paste(self, uuid, paste_content, ltags, ltagsgalaxies, name, source=N full_path = os.path.join(ITEMS_FOLDER, save_path) - self.redis_logger.debug(f'file path of the paste {full_path}') + self.logger.debug(f'file path of the paste {full_path}') if not os.path.isfile(full_path): # file not exists in AIL paste directory - self.redis_logger.debug(f"new paste {paste_content}") + self.logger.debug(f"new paste {paste_content}") gzip64encoded = self._compress_encode_content(paste_content, uuid) @@ -274,7 +273,7 @@ def create_paste(self, uuid, paste_content, ltags, ltagsgalaxies, name, source=N # use relative path rel_item_path = save_path.replace(self.PASTES_FOLDER, '', 1) - self.redis_logger.debug(f"relative path {rel_item_path}") + self.logger.debug(f"relative path {rel_item_path}") item = Item(rel_item_path) @@ -295,12 +294,12 @@ def create_paste(self, uuid, paste_content, ltags, ltagsgalaxies, name, source=N if self.r_serv_log_submit.get(f'{uuid}:nb_end') == self.r_serv_log_submit.get(f'{uuid}:nb_total'): self.r_serv_log_submit.set(f'{uuid}:end', 1) - self.redis_logger.debug(f' {rel_item_path} send to Mixer') + self.logger.debug(f' {rel_item_path} send to Mixer') print(f' {rel_item_path} send to Mixer') self.r_serv_log_submit.sadd(f'{uuid}:paste_submit_link', rel_item_path) curr_date = datetime.date.today() - self.redis_logger.debug("paste submitted") + self.logger.debug("paste submitted") else: self.addError(uuid, f'File: {save_path} already exist in submitted pastes') @@ -316,7 +315,7 @@ def _compress_encode_content(self, content, uuid): return gzip64encoded def addError(self, uuid, errorMessage): - self.redis_logger.debug(errorMessage) + self.logger.debug(errorMessage) print(errorMessage) error = self.r_serv_log_submit.get(f'{uuid}:error') if error is not None: @@ -324,7 +323,7 @@ def addError(self, uuid, errorMessage): self.r_serv_log_submit.incr(f'{uuid}:nb_end') def abord_file_submission(self, uuid, errorMessage): - self.redis_logger.debug(f'abord {uuid}, {errorMessage}') + self.logger.debug(f'abord {uuid}, {errorMessage}') self.addError(uuid, errorMessage) self.r_serv_log_submit.set(f'{uuid}:end', 1) diff --git a/bin/modules/Urls.py b/bin/modules/Urls.py index ed510a84..6c701d00 100755 --- a/bin/modules/Urls.py +++ b/bin/modules/Urls.py @@ -85,8 +85,6 @@ def compute(self, message): if len(l_urls) > 0: to_print = f'Urls;{item.get_source()};{item.get_date()};{item.get_basename()};' print(to_print) - # .debug ??? - # self.redis_logger.info(f'{to_print}Detected {len(l_urls)} URL;{item.get_id()}') if __name__ == '__main__': diff --git a/bin/modules/abstract_module.py b/bin/modules/abstract_module.py index 947775df..f984bf4a 100644 --- a/bin/modules/abstract_module.py +++ b/bin/modules/abstract_module.py @@ -18,7 +18,6 @@ ################################## # Import Project packages ################################## -from pubsublogger import publisher from lib import ail_logger from lib.ail_queues import AILQueue from lib import regex_helper @@ -51,15 +50,6 @@ def __init__(self, module_name=None, queue=True): self.obj = None self.sha256_mess = None - # Init Redis Logger - self.redis_logger = publisher - # Port of the redis instance used by pubsublogger - self.redis_logger.port = 6380 - # Channel name to publish logs - # # TODO: refactor logging - # If provided could be a namespaced channel like script: - self.redis_logger.channel = 'Script' - # Cache key self.r_cache_key = regex_helper.generate_redis_cache_key(self.module_name) self.max_execution_time = 30 diff --git a/bin/trackers/Retro_Hunt.py b/bin/trackers/Retro_Hunt.py index 1c9e6dc1..b50fb849 100755 --- a/bin/trackers/Retro_Hunt.py +++ b/bin/trackers/Retro_Hunt.py @@ -42,12 +42,11 @@ def __init__(self): self.obj = None self.tags = [] - self.redis_logger.info(f"Module: {self.module_name} Launched") + self.logger.info(f"Module: {self.module_name} Launched") # # TODO: # start_time # # end_time def compute(self, task_uuid): - self.redis_logger.warning(f'{self.module_name}, starting Retro hunt task {task_uuid}') print(f'starting Retro hunt task {task_uuid}') self.progress = 0 # First launch @@ -58,7 +57,7 @@ def compute(self, task_uuid): timeout = self.retro_hunt.get_timeout() self.tags = self.retro_hunt.get_tags() - self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}') + self.logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}') # Filters filters = self.retro_hunt.get_filters() @@ -111,7 +110,6 @@ def compute(self, task_uuid): # Completed self.retro_hunt.complete() print(f'Retro Hunt {task_uuid} completed') - self.redis_logger.warning(f'{self.module_name}, Retro Hunt {task_uuid} completed') def update_progress(self): if self.nb_objs == 0: @@ -128,7 +126,6 @@ def yara_rules_match(self, data): # print(data) task_uuid = data['namespace'] - self.redis_logger.info(f'{self.module_name}, Retro hunt {task_uuid} match found: {obj_id}') print(f'Retro hunt {task_uuid} match found: {self.obj.get_type()} {obj_id}') self.retro_hunt.add(self.obj.get_type(), self.obj.get_subtype(r_str=True), obj_id) @@ -158,16 +155,15 @@ def run(self): task_uuid = Tracker.get_retro_hunt_task_to_start() if task_uuid: # Module processing with the message from the queue - self.redis_logger.debug(task_uuid) + self.logger.debug(task_uuid) # try: self.compute(task_uuid) # except Exception as err: - # self.redis_logger.error(f'Error in module {self.module_name}: {err}') + # self.logger.error(f'Error in module {self.module_name}: {err}') # # Remove uuid ref # self.remove_submit_uuid(uuid) else: # Wait before next process - self.redis_logger.debug(f'{self.module_name}, waiting for new message, Idling {self.pending_seconds}s') time.sleep(self.pending_seconds) diff --git a/bin/trackers/Tracker_Regex.py b/bin/trackers/Tracker_Regex.py index 9d5f8418..3d7dbf33 100755 --- a/bin/trackers/Tracker_Regex.py +++ b/bin/trackers/Tracker_Regex.py @@ -47,14 +47,13 @@ def __init__(self, queue=True): self.exporters = {'mail': MailExporterTracker(), 'webhook': WebHookExporterTracker()} - self.redis_logger.info(f"Module: {self.module_name} Launched") + self.logger.info(f"Module: {self.module_name} Launched") def compute(self, message): # refresh Tracked regex if self.last_refresh < Tracker.get_tracker_last_updated_by_type('regex'): self.tracked_regexs = Tracker.get_tracked_regexs() self.last_refresh = time.time() - self.redis_logger.debug('Tracked regex refreshed') print('Tracked regex refreshed') obj = self.get_obj() @@ -117,7 +116,6 @@ def new_tracker_found(self, tracker_name, tracker_type, obj, re_matches): continue print(f'new tracked regex found: {tracker_name} in {self.obj.get_global_id()}') - self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {self.obj.get_global_id()}') tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id) diff --git a/bin/trackers/Tracker_Term.py b/bin/trackers/Tracker_Term.py index 2d0a02a5..075e2f3b 100755 --- a/bin/trackers/Tracker_Term.py +++ b/bin/trackers/Tracker_Term.py @@ -62,20 +62,18 @@ def __init__(self, queue=True): self.exporters = {'mail': MailExporterTracker(), 'webhook': WebHookExporterTracker()} - self.redis_logger.info(f"Module: {self.module_name} Launched") + self.logger.info(f"Module: {self.module_name} Launched") def compute(self, message): # refresh Tracked term if self.last_refresh_word < Tracker.get_tracker_last_updated_by_type('word'): self.tracked_words = Tracker.get_tracked_words() self.last_refresh_word = time.time() - self.redis_logger.debug('Tracked word refreshed') print('Tracked word refreshed') if self.last_refresh_set < Tracker.get_tracker_last_updated_by_type('set'): self.tracked_sets = Tracker.get_tracked_sets() self.last_refresh_set = time.time() - self.redis_logger.debug('Tracked set refreshed') print('Tracked set refreshed') obj = self.get_obj() @@ -93,7 +91,7 @@ def compute(self, message): try: dict_words_freq = Tracker.get_text_word_frequency(content) except TimeoutException: - self.redis_logger.warning(f"{self.obj.get_global_id()} processing timeout") + self.logger.warning(f"{self.obj.get_global_id()} processing timeout") else: signal.alarm(0) @@ -125,7 +123,6 @@ def new_tracker_found(self, tracker_name, tracker_type, obj): # TODO FILTER continue print(f'new tracked term {tracker_uuid} found: {tracker_name} in {self.obj.get_global_id()}') - self.redis_logger.warning(f'new tracked term found: {tracker_name} in {self.obj.get_global_id()}') tracker.add(obj.get_type(), obj.get_subtype(), obj_id) diff --git a/bin/trackers/Tracker_Typo_Squatting.py b/bin/trackers/Tracker_Typo_Squatting.py index d81d430f..dd7cb52e 100755 --- a/bin/trackers/Tracker_Typo_Squatting.py +++ b/bin/trackers/Tracker_Typo_Squatting.py @@ -43,14 +43,13 @@ def __init__(self): self.exporters = {'mail': MailExporterTracker(), 'webhook': WebHookExporterTracker()} - self.redis_logger.info(f"Module: {self.module_name} Launched") + self.logger.info(f"Module: {self.module_name} Launched") def compute(self, message): # refresh Tracked typo if self.last_refresh_typosquatting < Tracker.get_tracker_last_updated_by_type('typosquatting'): self.tracked_typosquattings = Tracker.get_tracked_typosquatting() self.last_refresh_typosquatting = time.time() - self.redis_logger.debug('Tracked typosquatting refreshed') print('Tracked typosquatting refreshed') host = message @@ -76,7 +75,6 @@ def new_tracker_found(self, tracked, tracker_type, obj): continue print(f'new tracked typosquatting found: {tracked} in {self.obj.get_global_id()}') - self.redis_logger.warning(f'tracker typosquatting: {tracked} in {self.obj.get_global_id()}') tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id) diff --git a/bin/trackers/Tracker_Yara.py b/bin/trackers/Tracker_Yara.py index 0bf6db4e..5a312f73 100755 --- a/bin/trackers/Tracker_Yara.py +++ b/bin/trackers/Tracker_Yara.py @@ -44,14 +44,13 @@ def __init__(self, queue=True): self.exporters = {'mail': MailExporterTracker(), 'webhook': WebHookExporterTracker()} - self.redis_logger.info(f"Module: {self.module_name} Launched") + self.logger.info(f"Module: {self.module_name} Launched") def compute(self, message): # refresh YARA list if self.last_refresh < Tracker.get_tracker_last_updated_by_type('yara'): self.rules = Tracker.get_tracked_yara_rules() self.last_refresh = time.time() - self.redis_logger.debug('Tracked set refreshed') print('Tracked set refreshed') self.obj = self.get_obj() @@ -69,11 +68,9 @@ def compute(self, message): yara_match = self.rules[obj_type].match(data=content, callback=self.yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=60) if yara_match: - self.redis_logger.warning(f'tracker yara: new match {self.obj.get_global_id()}: {yara_match}') print(f'{self.obj.get_global_id()}: {yara_match}') except yara.TimeoutError: print(f'{self.obj.get_id()}: yara scanning timed out') - self.redis_logger.info(f'{self.obj.get_id()}: yara scanning timed out') def convert_byte_offset_to_string(self, b_content, offset): byte_chunk = b_content[:offset + 1] diff --git a/requirements.txt b/requirements.txt index b9da93dc..43af1fcd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,9 +25,6 @@ xxhash>3.1.0 zmq pyzmq>19.0.0 -# Logging -logbook -pubsublogger # Tokeniser nltk>3.4.5 diff --git a/var/www/Flask_config.py b/var/www/Flask_config.py index 3d54215c..ee0f45e6 100644 --- a/var/www/Flask_config.py +++ b/var/www/Flask_config.py @@ -16,7 +16,6 @@ ################################## sys.path.append(os.environ['AIL_BIN']) from lib import ConfigLoader -from pubsublogger import publisher # FLASK # app = None @@ -34,13 +33,6 @@ r_serv_db = config_loader.get_db_conn("Kvrocks_DB") # TODO remove redis call from blueprint r_serv_tags = config_loader.get_db_conn("Kvrocks_Tags") # TODO remove redis call from blueprint -# Logger (Redis) -redis_logger = publisher -# Port of the redis instance used by pubsublogger -redis_logger.port = 6380 -# Channel name to publish logs -redis_logger.channel = 'Flask' - sys.path.append('../../configs/keys') #### VARIABLES #### diff --git a/var/www/blueprints/ui_submit.py b/var/www/blueprints/ui_submit.py index 3ce0cb88..b3f483d3 100644 --- a/var/www/blueprints/ui_submit.py +++ b/var/www/blueprints/ui_submit.py @@ -39,8 +39,6 @@ r_serv_db = Flask_config.r_serv_db # TODO REMOVE ME r_serv_log_submit = Flask_config.r_serv_log_submit # TODO REMOVE ME -logger = Flask_config.redis_logger - valid_filename_chars = "-_ %s%s" % (string.ascii_letters, string.digits) @@ -57,11 +55,9 @@ def limit_content_length(): def decorator(f): @wraps(f) def wrapper(*args, **kwargs): - logger.debug('decorator') cl = request.content_length if cl is not None: if cl > Flask_config.SUBMIT_PASTE_FILE_MAX_SIZE or ('file' not in request.files and cl > Flask_config.SUBMIT_PASTE_TEXT_MAX_SIZE): - logger.debug('abort') abort(413) return f(*args, **kwargs) return wrapper @@ -75,7 +71,6 @@ def allowed_file(filename): return True else: file_ext = filename.rsplit('.', 1)[1].lower() - logger.debug(file_ext) return file_ext in Flask_config.SUBMIT_PASTE_FILE_ALLOWED_EXTENSIONS def clean_filename(filename, whitelist=valid_filename_chars, replace=' '): @@ -111,7 +106,6 @@ def PasteSubmit_page(): @login_user_no_api @limit_content_length() def submit(): - logger.debug('submit') password = request.form['archive_pass'] ltags = request.form['tags_taxonomies'] @@ -124,12 +118,10 @@ def submit(): paste_source = paste_source.replace('/', '')[:80] if paste_source in ['crawled', 'tests']: content = 'Invalid source' - logger.info(paste_source) return content, 400 if not re.match('^[0-9a-zA-Z-_\+@#&\.;=:!]*$', paste_source): content = f'Invalid source name: Forbidden character(s)' - logger.info(content) return content, 400 is_file = False @@ -139,8 +131,6 @@ def submit(): if file_import.filename: is_file = True - logger.debug(f'is file ? {is_file}') - submitted_tag = 'infoleak:submission="manual"' # active taxonomies @@ -149,13 +139,11 @@ def submit(): active_galaxies = Tag.get_active_galaxies() if ltags or ltagsgalaxies: - logger.debug(f'ltags ? {ltags} {ltagsgalaxies}') ltags = Tag.unpack_str_tags_list(ltags) ltagsgalaxies = Tag.unpack_str_tags_list(ltagsgalaxies) if not Tag.is_valid_tags_taxonomies_galaxy(ltags, ltagsgalaxies): content = 'INVALID TAGS' - logger.info(content) return content, 400 # add submitted tags @@ -164,36 +152,28 @@ def submit(): ltags.append(submitted_tag) if is_file: - logger.debug('file management') if allowed_file(file_import.filename): - logger.debug('file extension allowed') # get UUID UUID = str(uuid.uuid4()) # create submitted dir if not os.path.exists(UPLOAD_FOLDER): - logger.debug('create folder') os.makedirs(UPLOAD_FOLDER) if '.' not in file_import.filename: - logger.debug('add UUID to path') full_path = os.path.join(UPLOAD_FOLDER, UUID) else: if file_import.filename[-6:] == 'tar.gz': - logger.debug('file extension is tar.gz') file_type = 'tar.gz' else: file_type = file_import.filename.rsplit('.', 1)[1] - logger.debug(f'file type {file_type}') name = UUID + '.' + file_type full_path = os.path.join(UPLOAD_FOLDER, name) - logger.debug(f'full path {full_path}') # Flask verify the file size file_import.save(full_path) - logger.debug('file saved') Import_helper.create_import_queue(ltags, ltagsgalaxies, full_path, UUID, password, True) @@ -204,18 +184,13 @@ def submit(): else: content = f'wrong file type, allowed_extensions: {allowed_extensions} or remove the extension' - logger.info(content) return content, 400 elif paste_content != '': - logger.debug(f'entering text paste management') if sys.getsizeof(paste_content) < Flask_config.SUBMIT_PASTE_TEXT_MAX_SIZE: - logger.debug(f'size {sys.getsizeof(paste_content)}') # get id UUID = str(uuid.uuid4()) - logger.debug('create import') Import_helper.create_import_queue(ltags, ltagsgalaxies, paste_content, UUID, password, source=paste_source) - logger.debug('import OK') return render_template("submit_items.html", active_taxonomies = active_taxonomies, active_galaxies = active_galaxies, @@ -223,11 +198,9 @@ def submit(): else: content = f'text paste size is over {Flask_config.SUBMIT_PASTE_TEXT_MAX_SIZE} bytes limit' - logger.info(content) return content, 400 content = 'submit aborded' - logger.error(content) return content, 400 return PasteSubmit_page()