chg: [pubsublogger] remove old redis_logger

bin/LAUNCH.sh

@@ -615,14 +615,13 @@ function launch_all {
     update;
     launch_redis;
     launch_kvrocks;
-    launch_logs;
     launch_scripts;
     launch_flask;
 }
 
 function menu_display {
 
-  options=("Redis" "Kvrocks" "Logs" "Scripts" "Flask" "Killall" "Update" "Update-config" "Update-thirdparty")
+  options=("Redis" "Kvrocks" "Scripts" "Flask" "Killall" "Update" "Update-config" "Update-thirdparty")
 
   menu() {
       echo "What do you want to Launch?:"
@@ -653,9 +652,6 @@ function menu_display {
               Kvrocks)
                   launch_kvrocks;
                   ;;
-              Logs)
-                  launch_logs;
-                  ;;
               Scripts)
                   launch_scripts;
                   ;;

bin/core/Sync_module.py

@@ -88,7 +88,6 @@ def run(self):
             if self.last_refresh_queues < time.time():
                 timeout_processed_objs()
                 self.last_refresh_queues = time.time() + 120
-                self.redis_logger.debug('Timeout queues')
                 # print('Timeout queues')
 
             # Get one message (paste) from the QueueIn (copy of Redis_Global publish)

bin/modules/ApiKey.py

@@ -20,7 +20,6 @@
 # Import Project packages
 ##################################
 from modules.abstract_module import AbstractModule
-from lib.objects.Items import Item
 
 class ApiKey(AbstractModule):
     """ApiKey module for AIL framework"""
@@ -48,31 +47,28 @@ def __init__(self):
 
     def compute(self, message, r_result=False):
         score = message
-        item = self.get_obj()
-        item_content = item.get_content()
+        obj = self.get_obj()
+        content = obj.get_content()
 
-        google_api_key = self.regex_findall(self.re_google_api_key, item.get_id(), item_content, r_set=True)
-        aws_access_key = self.regex_findall(self.re_aws_access_key, item.get_id(), item_content, r_set=True)
+        google_api_key = self.regex_findall(self.re_google_api_key, obj.get_id(), content, r_set=True)
+        aws_access_key = self.regex_findall(self.re_aws_access_key, obj.get_id(), content, r_set=True)
         if aws_access_key:
-            aws_secret_key = self.regex_findall(self.re_aws_secret_key, item.get_id(), item_content, r_set=True)
+            aws_secret_key = self.regex_findall(self.re_aws_secret_key, obj.get_id(), content, r_set=True)
 
         if aws_access_key or google_api_key:
-            to_print = f'ApiKey;{item.get_source()};{item.get_date()};{item.get_basename()};'
+            to_print = obj.get_global_id()
 
             if google_api_key:
                 print(f'found google api key: {to_print}')
-                self.redis_logger.warning(f'{to_print}Checked {len(google_api_key)} found Google API Key;{self.obj.get_global_id()}')
 
                 tag = 'infoleak:automatic-detection="google-api-key"'
                 self.add_message_to_queue(message=tag, queue='Tags')
 
             # # TODO: # FIXME: AWS regex/validate/sanitize KEY + SECRET KEY
             if aws_access_key:
                 print(f'found AWS key: {to_print}')
-                self.redis_logger.warning(f'{to_print}Checked {len(aws_access_key)} found AWS Key;{self.obj.get_global_id()}')
                 if aws_secret_key:
                     print(f'found AWS secret key')
-                    self.redis_logger.warning(f'{to_print}Checked {len(aws_secret_key)} found AWS secret Key;{self.obj.get_global_id()}')
 
                 tag = 'infoleak:automatic-detection="aws-key"'
                 self.add_message_to_queue(message=tag, queue='Tags')

bin/modules/Categ.py

@@ -65,7 +65,7 @@ def __init__(self, categ_files_dir=os.path.join(os.environ['AIL_HOME'], 'files')
         self.matchingThreshold = config_loader.get_config_int("Categ", "matchingThreshold")
 
         self.reload_categ_words()
-        self.redis_logger.info("Script Categ started")
+        self.logger.info("Script Categ started")
 
     # # TODO: trigger reload on change ( save last reload time, ...)
     def reload_categ_words(self):
@@ -103,9 +103,6 @@ def compute(self, message, r_result=False):
                     print(msg, categ)
                     self.add_message_to_queue(message=msg, queue=categ)
 
-                    self.redis_logger.debug(
-                        f'Categ;{obj.get_source()};{obj.get_date()};{obj.get_basename()};Detected {lenfound} as {categ};{obj.get_id()}')
-
         if r_result:
             return categ_found
 

bin/modules/Credential.py

@@ -29,17 +29,14 @@
 import os
 import sys
 import time
-from datetime import datetime
 from pyfaup.faup import Faup
 
 sys.path.append(os.environ['AIL_BIN'])
 ##################################
 # Import Project packages
 ##################################
 from modules.abstract_module import AbstractModule
-from lib.objects.Items import Item
 from lib import ConfigLoader
-# from lib import Statistics
 
 
 class Credential(AbstractModule):
@@ -80,40 +77,36 @@ def __init__(self):
         self.pending_seconds = 10
 
         # Send module state to logs
-        self.redis_logger.info(f"Module {self.module_name} initialized")
+        self.logger.info(f"Module {self.module_name} initialized")
 
     def compute(self, message):
 
-        count = message
-        item = self.get_obj()
+        obj = self.get_obj()
 
-        item_content = item.get_content()
+        content = obj.get_content()
 
         # TODO: USE SETS
         # Extract all credentials
-        all_credentials = self.regex_findall(self.regex_cred, item.get_id(), item_content)
+        all_credentials = self.regex_findall(self.regex_cred, obj.get_id(), content)
         if all_credentials:
             nb_cred = len(all_credentials)
             message = f'Checked {nb_cred} credentials found.'
 
-            all_sites = self.regex_findall(self.regex_web, item.get_id(), item_content, r_set=True)
+            all_sites = self.regex_findall(self.regex_web, obj.get_id(), content, r_set=True)
             if all_sites:
                 discovered_sites = ', '.join(all_sites)
                 message += f' Related websites: {discovered_sites}'
 
             print(message)
 
-            to_print = f'Credential;{item.get_source()};{item.get_date()};{item.get_basename()};{message};{self.obj.get_global_id()}'
-
             # num of creds above threshold, publish an alert
             if nb_cred > self.criticalNumberToAlert:
                 print(f"========> Found more than 10 credentials in this file : {self.obj.get_global_id()}")
-                self.redis_logger.warning(to_print)
 
                 tag = 'infoleak:automatic-detection="credential"'
                 self.add_message_to_queue(message=tag, queue='Tags')
 
-                site_occurrence = self.regex_findall(self.regex_site_for_stats, item.get_id(), item_content)
+                site_occurrence = self.regex_findall(self.regex_site_for_stats, obj.get_id(), content)
 
                 creds_sites = {}
 
@@ -162,8 +155,7 @@ def compute(self, message):
                 # for tld in nb_tlds:
                 #     Statistics.add_module_tld_stats_by_date('credential', date, tld, nb_tlds[tld])
             else:
-                self.redis_logger.info(to_print)
-                print(f'found {nb_cred} credentials')
+                print(f'found {nb_cred} credentials {self.obj.get_global_id()}')
 
             # # TODO: # FIXME: TEMP DESABLE
             # # For searching credential in termFreq

bin/modules/CreditCards.py

@@ -7,7 +7,7 @@
 
 This module is consuming the Redis-list created by the Categ module.
 
-It apply credit card regexes on item content and warn if a valid card number is found.
+It apply credit card regexes on object content and warn if a valid card number is found.
 
 """
 
@@ -23,7 +23,6 @@
 # Import Project packages
 ##################################
 from modules.abstract_module import AbstractModule
-from lib.objects.Items import Item
 from packages import lib_refine
 
 class CreditCards(AbstractModule):
@@ -68,10 +67,9 @@ def extract(self, obj, content, tag):
         return extracted
 
     def compute(self, message, r_result=False):
-        score = message
-        item = self.get_obj()
-        content = item.get_content()
-        all_cards = self.regex_findall(self.regex, item.id, content)
+        obj = self.get_obj()
+        content = obj.get_content()
+        all_cards = self.regex_findall(self.regex, obj.id, content)
 
         if len(all_cards) > 0:
             # self.logger.debug(f'All matching {all_cards}')
@@ -84,19 +82,16 @@ def compute(self, message, r_result=False):
                     creditcard_set.add(valid_card)
 
             # print(creditcard_set)
-            to_print = f'CreditCard;{item.get_source()};{item.get_date()};{item.get_basename()};'
             if creditcard_set:
-                mess = f'{to_print}Checked {len(creditcard_set)} valid number(s);{self.obj.get_global_id()}'
-                print(mess)
-                self.redis_logger.warning(mess)
+                print(f'{len(creditcard_set)} valid number(s);{self.obj.get_global_id()}')
 
                 tag = 'infoleak:automatic-detection="credit-card"'
                 self.add_message_to_queue(message=tag, queue='Tags')
 
                 if r_result:
                     return creditcard_set
             else:
-                self.redis_logger.info(f'{to_print}CreditCard related;{self.obj.get_global_id()}')
+                print(f'CreditCard related;{self.obj.get_global_id()}')
 
 
 if __name__ == '__main__':

bin/modules/Cryptocurrencies.py

@@ -28,7 +28,6 @@
 ##################################
 from modules.abstract_module import AbstractModule
 from lib.objects.CryptoCurrencies import CryptoCurrency
-from lib.objects.Items import Item
 
 ##################################
 ##################################
@@ -151,17 +150,10 @@ def compute(self, message):
 
                             # debug
                             print(private_keys)
-                            to_print = 'Cryptocurrency;{};{};{};'.format(item.get_source(),
-                                                                         item.get_date(),
-                                                                         item.get_basename())
-                            self.redis_logger.warning('{}Detected {} {} private key;{}'.format(
-                                to_print, len(private_keys), currency['name'], self.obj.get_global_id()))
                     else:
                         private_keys = []
 
-                    to_print = f"{currency['name']} found: {len(addresses)} address and {len(private_keys)} private Keys"
-                    print(to_print)
-                    self.redis_logger.warning(to_print)
+                    print(f"{currency['name']} found: {len(addresses)} address and {len(private_keys)} private Keys {self.obj.get_global_id()}")
 
 
 if __name__ == '__main__':

bin/modules/CveModule.py

@@ -44,7 +44,6 @@ def __init__(self):
         self.logger.info(f'Module {self.module_name} initialized')
 
     def compute(self, message):
-        count = message
         item = self.get_obj()
         item_id = item.get_id()
 
@@ -56,9 +55,7 @@ def compute(self, message):
                 cve = Cves.Cve(cve_id)
                 cve.add(date, item)
 
-            warning = f'{self.obj.get_global_id()} contains CVEs {cves}'
-            print(warning)
-            self.redis_logger.warning(warning)
+            print(f'{self.obj.get_global_id()} contains CVEs {cves}')
 
             tag = 'infoleak:automatic-detection="cve"'
             # Send to Tags Queue

bin/modules/DomClassifier.py

@@ -58,10 +58,6 @@ def __init__(self):
     def compute(self, message, r_result=False):
         host = message
 
-        item = self.get_obj()
-        item_basename = item.get_basename()
-        item_date = item.get_date()
-        item_source = item.get_source()
         try:
 
             self.dom_classifier.text(rawtext=host)
@@ -82,19 +78,19 @@ def compute(self, message, r_result=False):
                 localizeddomains = self.dom_classifier.include(expression=self.cc_tld)
                 if localizeddomains:
                     print(localizeddomains)
-                    self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc_tld};{self.obj.get_global_id()}")
+                    self.logger.info(f"{localizeddomains} located in {self.cc_tld};{self.obj.get_global_id()}")
 
             if self.cc:
                 localizeddomains = self.dom_classifier.localizedomain(cc=self.cc)
                 if localizeddomains:
                     print(localizeddomains)
-                    self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc};{self.obj.get_global_id()}")
+                    self.logger.info(f"{localizeddomains} located in {self.cc};{self.obj.get_global_id()}")
 
             if r_result:
                 return self.dom_classifier.vdomain
 
         except IOError as err:
-            self.redis_logger.error(f"Duplicate;{item_source};{item_date};{item_basename};CRC Checksum Failed")
+            self.logger.error(f"{self.obj.get_global_id()};CRC Checksum Failed")
             raise Exception(f"CRC Checksum Failed on: {self.obj.get_global_id()}")
 
 

bin/modules/Duplicates.py

@@ -27,7 +27,6 @@
 from modules.abstract_module import AbstractModule
 from lib.ConfigLoader import ConfigLoader
 from lib import Duplicate
-from lib.objects.Items import Item
 
 
 class Duplicates(AbstractModule):
@@ -92,11 +91,11 @@ def compute(self, message):
             Duplicate.save_object_hash(algo, curr_date_ymonth, self.algos[algo]['hash'], item.get_id())
 
         if nb_duplicates:
-            self.redis_logger.info(f'Duplicate;{item.get_source()};{item.get_date()};{item.get_basename()};Detected {nb_duplicates};{self.obj.get_global_id()}')
+            self.logger.info(f'Duplicates {nb_duplicates};{self.obj.get_global_id()}')
 
         y = time.time()
         print(f'{self.obj.get_global_id()} Processed in {y-x} sec')
-        # self.redis_logger.debug('{}Processed in {} sec'.format(to_print, y-x))
+        # self.logger.debug('{}Processed in {} sec'.format(to_print, y-x))
 
 
 if __name__ == "__main__":

bin/modules/Global.py

@@ -74,7 +74,6 @@ def computeNone(self):
         if int(difftime) > 30:
             to_print = f'Global; ; ; ;glob Processed {self.processed_item} item(s) in {difftime} s'
             print(to_print)
-            self.redis_logger.debug(to_print)
 
             self.time_last_stats = time.time()
             self.processed_item = 0

bin/modules/IPAddress.py

@@ -25,7 +25,6 @@
 ##################################
 from modules.abstract_module import AbstractModule
 from lib.ConfigLoader import ConfigLoader
-from lib.objects.Items import Item
 from lib import regex_helper
 
 # TODO REWRITE ME -> PERF + IPV6 + Tracker ?
@@ -66,11 +65,11 @@ def compute(self, message, r_result=False):
         if not self.ip_networks:
             return None
 
-        item = self.get_obj()
-        content = item.get_content()
+        obj = self.get_obj()
+        content = obj.get_content()
 
         # list of the regex results in the Item
-        results = self.regex_findall(self.re_ipv4, item.get_id(), content)
+        results = self.regex_findall(self.re_ipv4, obj.get_id(), content)
         results = set(results)
         matching_ips = []
         for ip in results:
@@ -83,7 +82,6 @@ def compute(self, message, r_result=False):
 
         if len(matching_ips) > 0:
             self.logger.info(f'{self.obj.get_global_id()} contains {len(matching_ips)} IPs')
-            self.redis_logger.warning(f'{self.obj.get_global_id()} contains IPs')
 
             # Tag message with IP
             tag = 'infoleak:automatic-detection="ip"'
@@ -93,4 +91,3 @@ def compute(self, message, r_result=False):
 if __name__ == "__main__":
     module = IPAddress()
     module.run()
-    # module.compute('submitted/2023/05/15/submitted_8a6136c2-c7f2-4c9e-8f29-e1a62315b482.gz')