failed_request_abort_threshold
+api_failure_abort_threshold
(int)
–
Threshold for setting error state after failed HTTP requests (only takes effect when request_with_fail_count() is used. Default is 5.
Threshold for setting error state after failed HTTP requests (only takes effect when api_request() is used. Default is 5.
class BaseModule:
+1417
+1418
+1419
+1420
+1421
+1422
+1423
+1424
+1425
+1426
+1427
+1428
+1429
+1430
+1431
+1432
+1433
+1434
+1435
+1436
+1437
+1438
+1439
+1440
+1441
+1442
+1443
+1444
+1445
+1446
+1447
+1448
+1449
+1450
+1451
+1452
+1453
+1454
+1455
+1456
+1457
+1458
+1459
+1460
+1461
+1462
+1463
+1464
+1465
+1466
+1467
+1468
+1469
+1470
+1471
+1472
+1473
+1474
+1475
+1476
+1477
+1478
+1479
+1480
+1481
+1482
+1483
+1484
+1485
+1486
+1487
+1488
+1489
+1490
+1491
+1492
+1493
+1494
+1495
+1496
+1497
+1498
+1499
+1500
+1501
+1502
+1503
+1504
+1505
+1506
+1507
+1508
+1509
+1510
+1511
+1512
+1513
+1514
+1515
+1516
+1517
+1518
+1519
+1520
+1521
+1522
+1523
+1524
+1525
+1526
+1527
+1528
+1529
+1530
+1531
+1532
+1533
+1534
+1535
+1536
+1537
+1538
+1539
+1540class BaseModule:
"""The base class for all BBOT modules.
Attributes:
@@ -2571,7 +2708,7 @@
batch_wait (int): Seconds to wait before force-submitting a batch. Default is 10.
- failed_request_abort_threshold (int): Threshold for setting error state after failed HTTP requests (only takes effect when `request_with_fail_count()` is used. Default is 5.
+ api_failure_abort_threshold (int): Threshold for setting error state after failed HTTP requests (only takes effect when `api_request()` is used. Default is 5.
_preserve_graph (bool): When set to True, accept events that may be duplicates but are necessary for construction of complete graph. Typically only enabled for output modules that need to maintain full chains of events, e.g. `neo4j` and `json`. Default is False.
@@ -2611,7 +2748,13 @@
_module_threads = 1
_batch_size = 1
batch_wait = 10
- failed_request_abort_threshold = 5
+
+ # API retries, etc.
+ _api_retries = 2
+ # disable the module after this many failed attempts in a row
+ _api_failure_abort_threshold = 3
+ # sleep for this many seconds after being rate limited
+ _429_sleep_interval = 30
default_discovery_context = "{module} discovered {event.type}: {event.data}"
@@ -2656,8 +2799,10 @@
# string constant
self._custom_filter_criteria_msg = "it did not meet custom filter criteria"
- # track number of failures (for .request_with_fail_count())
- self._request_failures = 0
+ self._api_keys = []
+
+ # track number of failures (for .api_request())
+ self._api_request_failures = 0
self._tasks = []
self._event_received = asyncio.Condition()
@@ -2814,10 +2959,37 @@
self.hugesuccess(f"API is ready")
return True
except Exception as e:
+ self.trace(traceback.format_exc())
return None, f"Error with API ({str(e).strip()})"
else:
return None, "No API key set"
+ @property
+ def api_key(self):
+ if self._api_keys:
+ return self._api_keys[0]
+
+ @api_key.setter
+ def api_key(self, api_keys):
+ if isinstance(api_keys, str):
+ api_keys = [api_keys]
+ self._api_keys = list(api_keys)
+
+ def cycle_api_key(self):
+ if self._api_keys:
+ self.verbose(f"Cycling API key")
+ self._api_keys.insert(0, self._api_keys.pop())
+ else:
+ self.debug(f"No extra API keys to cycle")
+
+ @property
+ def api_retries(self):
+ return max(self._api_retries + 1, len(self._api_keys))
+
+ @property
+ def api_failure_abort_threshold(self):
+ return (self.api_retries * self._api_failure_abort_threshold) + 1
+
async def ping(self):
"""Asynchronously checks the health of the configured API.
@@ -2826,7 +2998,7 @@
Example Usage:
In your implementation, if the API has a "/ping" endpoint:
async def ping(self):
- r = await self.request_with_fail_count(f"{self.base_url}/ping")
+ r = await self.api_request(f"{self.base_url}/ping")
resp_content = getattr(r, "text", "")
assert getattr(r, "status_code", 0) == 200, resp_content
@@ -3573,31 +3745,119 @@
async for line in self.helpers.run_live(*args, **kwargs):
yield line
- async def request_with_fail_count(self, *args, **kwargs):
- """Asynchronously perform an HTTP request while keeping track of consecutive failures.
+ def prepare_api_request(self, url, kwargs):
+ """
+ Prepare an API request by adding the necessary authentication - header, bearer token, etc.
+ """
+ if self.api_key:
+ url = url.format(api_key=self.api_key)
+ if not "headers" in kwargs:
+ kwargs["headers"] = {}
+ kwargs["headers"]["Authorization"] = f"Bearer {self.api_key}"
+ return url, kwargs
+
+ async def api_request(self, *args, **kwargs):
+ """
+ Makes an HTTP request while automatically:
+ - avoiding rate limits (sleep/retry)
+ - cycling API keys
+ - cancelling after too many failed attempts
+ """
+ url = args[0] if args else kwargs.pop("url", "")
+
+ # loop until we have a successful request
+ for _ in range(self.api_retries):
+ if not "headers" in kwargs:
+ kwargs["headers"] = {}
+ new_url, kwargs = self.prepare_api_request(url, kwargs)
+ kwargs["url"] = new_url
+
+ r = await self.helpers.request(**kwargs)
+ success = False if r is None else r.is_success
+
+ if success:
+ self._api_request_failures = 0
+ else:
+ status_code = getattr(r, "status_code", 0)
+ self._api_request_failures += 1
+ if self._api_request_failures >= self.api_failure_abort_threshold:
+ self.set_error_state(
+ f"Setting error state due to {self._api_request_failures:,} failed HTTP requests"
+ )
+ else:
+ # sleep for a bit if we're being rate limited
+ if status_code == 429:
+ self.verbose(
+ f"Sleeping for {self._429_sleep_interval:,} seconds due to rate limit (HTTP status: 429)"
+ )
+ await asyncio.sleep(self._429_sleep_interval)
+ elif self._api_keys:
+ # if request failed, cycle API keys and try again
+ self.cycle_api_key()
+ continue
+ break
+
+ return r
+
+ async def api_page_iter(self, url, page_size=100, json=True, next_key=None, **requests_kwargs):
+ """
+ An asynchronous generator function for iterating through paginated API data.
- This function wraps the `self.helpers.request` method, incrementing a failure counter if
- the request returns None. When the failure counter exceeds `self.failed_request_abort_threshold`,
- the module is set to an error state.
+ This function continuously makes requests to a specified API URL, incrementing the page number
+ or applying a custom pagination function, and yields the received data one page at a time.
+ It is well-suited for APIs that provide paginated results.
Args:
- *args: Positional arguments to pass to `self.helpers.request`.
- **kwargs: Keyword arguments to pass to `self.helpers.request`.
+ url (str): The initial API URL. Can contain placeholders for 'page', 'page_size', and 'offset'.
+ page_size (int, optional): The number of items per page. Defaults to 100.
+ json (bool, optional): If True, attempts to deserialize the response content to a JSON object. Defaults to True.
+ next_key (callable, optional): A function that takes the last page's data and returns the URL for the next page. Defaults to None.
+ **requests_kwargs: Arbitrary keyword arguments that will be forwarded to the HTTP request function.
- Returns:
- Any: The response object or None if the request failed.
+ Yields:
+ dict or httpx.Response: If 'json' is True, yields a dictionary containing the parsed JSON data. Otherwise, yields the raw HTTP response.
- Raises:
- None: Sets the module to an error state when the failure threshold is reached.
+ Note:
+ The loop will continue indefinitely unless manually stopped. Make sure to break out of the loop once the last page has been received.
+
+ Examples:
+ >>> agen = api_page_iter('https://api.example.com/data?page={page}&page_size={page_size}')
+ >>> try:
+ >>> async for page in agen:
+ >>> subdomains = page["subdomains"]
+ >>> self.hugesuccess(subdomains)
+ >>> if not subdomains:
+ >>> break
+ >>> finally:
+ >>> agen.aclose()
"""
- r = await self.helpers.request(*args, **kwargs)
- if r is None:
- self._request_failures += 1
- else:
- self._request_failures = 0
- if self._request_failures >= self.failed_request_abort_threshold:
- self.set_error_state(f"Setting error state due to {self._request_failures:,} failed HTTP requests")
- return r
+ page = 1
+ offset = 0
+ result = None
+ while 1:
+ if result and callable(next_key):
+ try:
+ new_url = next_key(result)
+ except Exception as e:
+ self.debug(f"Failed to extract next page of results from {url}: {e}")
+ self.debug(traceback.format_exc())
+ else:
+ new_url = self.helpers.safe_format(url, page=page, page_size=page_size, offset=offset)
+ result = await self.api_request(new_url, **requests_kwargs)
+ if result is None:
+ self.verbose(f"api_page_iter() got no response for {url}")
+ break
+ try:
+ if json:
+ result = result.json()
+ yield result
+ except Exception:
+ self.warning(f'Error in api_page_iter() for url: "{new_url}"')
+ self.trace(traceback.format_exc())
+ break
+ finally:
+ offset += page_size
+ page += 1
@property
def preset(self):
@@ -4163,13 +4423,7 @@
Source code in bbot/modules/base.py
-119
-120
-121
-122
-123
-124
-125
+125
126
127
128
@@ -4203,7 +4457,15 @@
156
157
158
-159
def __init__(self, scan):
+159
+160
+161
+162
+163
+164
+165
+166
+167
def __init__(self, scan):
"""Initializes a module instance.
Args:
@@ -4235,8 +4497,10 @@
# string constant
self._custom_filter_criteria_msg = "it did not meet custom filter criteria"
- # track number of failures (for .request_with_fail_count())
- self._request_failures = 0
+ self._api_keys = []
+
+ # track number of failures (for .api_request())
+ self._api_request_failures = 0
self._tasks = []
self._event_received = asyncio.Condition()
@@ -4249,165 +4513,481 @@
-
-cleanup
+
+api_page_iter
async
-cleanup()
+api_page_iter(url, page_size=100, json=True, next_key=None, **requests_kwargs)
-Asynchronously performs final cleanup operations after the scan is complete.
-This method can be overridden to implement custom cleanup logic. It is called only once per scan and may not raise events.
-Returns:
+An asynchronous generator function for iterating through paginated API data.
+This function continuously makes requests to a specified API URL, incrementing the page number
+or applying a custom pagination function, and yields the received data one page at a time.
+It is well-suited for APIs that provide paginated results.
+Parameters:
-
+
url
+ (str)
–
-None
+The initial API URL. Can contain placeholders for 'page', 'page_size', and 'offset'.
-
-
-Note
-This method is called only once per scan and may not raise events.
-
-
-Source code in bbot/modules/base.py
-273
-274
-275
-276
-277
-278
-279
-280
-281
-282
-283
-284
async def cleanup(self):
- """Asynchronously performs final cleanup operations after the scan is complete.
-
- This method can be overridden to implement custom cleanup logic. It is called only once per scan and may not raise events.
-
- Returns:
- None
-
- Note:
- This method is called only once per scan and may not raise events.
- """
- return
-
-
-
-
-
-
-critical
-
-critical(*args, trace=True, **kwargs)
-
-
-Logs a whole message in emboldened red text, and optionally the stack trace of the most recent exception.
-Parameters:
-
-
-
*args
+page_size
+ (int, default:
+ 100
+)
–
-Variable-length argument list to pass to the logger.
+The number of items per page. Defaults to 100.
-
-
trace
+json
(bool, default:
True
)
–
-Whether to log the stack trace of the most recently caught exception. Defaults to True.
+If True, attempts to deserialize the response content to a JSON object. Defaults to True.
-
-
**kwargs
+next_key
+ (callable, default:
+ None
+)
–
-Arbitrary keyword arguments to pass to the logger.
+A function that takes the last page's data and returns the URL for the next page. Defaults to None.
+
+
+-
+
**requests_kwargs
+ –
+
+Arbitrary keyword arguments that will be forwarded to the HTTP request function.
+
+
+
+Yields:
+
+-
+ –
+
+
dict or httpx.Response: If 'json' is True, yields a dictionary containing the parsed JSON data. Otherwise, yields the raw HTTP response.
+
+Note
+The loop will continue indefinitely unless manually stopped. Make sure to break out of the loop once the last page has been received.
+
Examples:
->>> self.critical("This is a critical message")
->>> self.critical("This is a critical message with a trace", trace=False)
+>>> agen = api_page_iter('https://api.example.com/data?page={page}&page_size={page_size}')
+>>> try:
+>>> async for page in agen:
+>>> subdomains = page["subdomains"]
+>>> self.hugesuccess(subdomains)
+>>> if not subdomains:
+>>> break
+>>> finally:
+>>> agen.aclose()
Source code in bbot/modules/base.py
-1403
-1404
-1405
-1406
-1407
-1408
-1409
-1410
-1411
-1412
-1413
-1414
-1415
-1416
-1417
def critical(self, *args, trace=True, **kwargs):
- """Logs a whole message in emboldened red text, and optionally the stack trace of the most recent exception.
+1157
+1158
+1159
+1160
+1161
+1162
+1163
+1164
+1165
+1166
+1167
+1168
+1169
+1170
+1171
+1172
+1173
+1174
+1175
+1176
+1177
+1178
+1179
+1180
+1181
+1182
+1183
+1184
+1185
+1186
+1187
+1188
+1189
+1190
+1191
+1192
+1193
+1194
+1195
+1196
+1197
+1198
+1199
+1200
+1201
+1202
+1203
+1204
+1205
+1206
+1207
+1208
+1209
+1210
+1211
+1212
+1213
+1214
+1215
async def api_page_iter(self, url, page_size=100, json=True, next_key=None, **requests_kwargs):
+ """
+ An asynchronous generator function for iterating through paginated API data.
+
+ This function continuously makes requests to a specified API URL, incrementing the page number
+ or applying a custom pagination function, and yields the received data one page at a time.
+ It is well-suited for APIs that provide paginated results.
Args:
- *args: Variable-length argument list to pass to the logger.
- trace (bool, optional): Whether to log the stack trace of the most recently caught exception. Defaults to True.
- **kwargs: Arbitrary keyword arguments to pass to the logger.
+ url (str): The initial API URL. Can contain placeholders for 'page', 'page_size', and 'offset'.
+ page_size (int, optional): The number of items per page. Defaults to 100.
+ json (bool, optional): If True, attempts to deserialize the response content to a JSON object. Defaults to True.
+ next_key (callable, optional): A function that takes the last page's data and returns the URL for the next page. Defaults to None.
+ **requests_kwargs: Arbitrary keyword arguments that will be forwarded to the HTTP request function.
+
+ Yields:
+ dict or httpx.Response: If 'json' is True, yields a dictionary containing the parsed JSON data. Otherwise, yields the raw HTTP response.
+
+ Note:
+ The loop will continue indefinitely unless manually stopped. Make sure to break out of the loop once the last page has been received.
Examples:
- >>> self.critical("This is a critical message")
- >>> self.critical("This is a critical message with a trace", trace=False)
+ >>> agen = api_page_iter('https://api.example.com/data?page={page}&page_size={page_size}')
+ >>> try:
+ >>> async for page in agen:
+ >>> subdomains = page["subdomains"]
+ >>> self.hugesuccess(subdomains)
+ >>> if not subdomains:
+ >>> break
+ >>> finally:
+ >>> agen.aclose()
"""
- self.log.critical(*args, extra={"scan_id": self.scan.id}, **kwargs)
- if trace:
- self.trace()
+ page = 1
+ offset = 0
+ result = None
+ while 1:
+ if result and callable(next_key):
+ try:
+ new_url = next_key(result)
+ except Exception as e:
+ self.debug(f"Failed to extract next page of results from {url}: {e}")
+ self.debug(traceback.format_exc())
+ else:
+ new_url = self.helpers.safe_format(url, page=page, page_size=page_size, offset=offset)
+ result = await self.api_request(new_url, **requests_kwargs)
+ if result is None:
+ self.verbose(f"api_page_iter() got no response for {url}")
+ break
+ try:
+ if json:
+ result = result.json()
+ yield result
+ except Exception:
+ self.warning(f'Error in api_page_iter() for url: "{new_url}"')
+ self.trace(traceback.format_exc())
+ break
+ finally:
+ offset += page_size
+ page += 1
-
-debug
+
+api_request
+
+async
+
-debug(*args, trace=False, **kwargs)
+api_request(*args, **kwargs)
-Logs debug messages and optionally the stack trace of the most recent exception.
-Parameters:
+
+Makes an HTTP request while automatically
--
-
*args
- –
-
-Variable-length argument list to pass to the logger.
-
-
--
-
trace
- (bool, default:
- False
-)
- –
-
-Whether to log the stack trace of the most recently caught exception. Defaults to False.
-
-
--
-
**kwargs
- –
-
-Arbitrary keyword arguments to pass to the logger.
+- avoiding rate limits (sleep/retry)
+- cycling API keys
+- cancelling after too many failed attempts
+
+
+
+Source code in bbot/modules/base.py
+1114
+1115
+1116
+1117
+1118
+1119
+1120
+1121
+1122
+1123
+1124
+1125
+1126
+1127
+1128
+1129
+1130
+1131
+1132
+1133
+1134
+1135
+1136
+1137
+1138
+1139
+1140
+1141
+1142
+1143
+1144
+1145
+1146
+1147
+1148
+1149
+1150
+1151
+1152
+1153
+1154
+1155
async def api_request(self, *args, **kwargs):
+ """
+ Makes an HTTP request while automatically:
+ - avoiding rate limits (sleep/retry)
+ - cycling API keys
+ - cancelling after too many failed attempts
+ """
+ url = args[0] if args else kwargs.pop("url", "")
+
+ # loop until we have a successful request
+ for _ in range(self.api_retries):
+ if not "headers" in kwargs:
+ kwargs["headers"] = {}
+ new_url, kwargs = self.prepare_api_request(url, kwargs)
+ kwargs["url"] = new_url
+
+ r = await self.helpers.request(**kwargs)
+ success = False if r is None else r.is_success
+
+ if success:
+ self._api_request_failures = 0
+ else:
+ status_code = getattr(r, "status_code", 0)
+ self._api_request_failures += 1
+ if self._api_request_failures >= self.api_failure_abort_threshold:
+ self.set_error_state(
+ f"Setting error state due to {self._api_request_failures:,} failed HTTP requests"
+ )
+ else:
+ # sleep for a bit if we're being rate limited
+ if status_code == 429:
+ self.verbose(
+ f"Sleeping for {self._429_sleep_interval:,} seconds due to rate limit (HTTP status: 429)"
+ )
+ await asyncio.sleep(self._429_sleep_interval)
+ elif self._api_keys:
+ # if request failed, cycle API keys and try again
+ self.cycle_api_key()
+ continue
+ break
+
+ return r
+
+
+
+
+
+
+cleanup
+
+async
+
+
+cleanup()
+
+
+Asynchronously performs final cleanup operations after the scan is complete.
+This method can be overridden to implement custom cleanup logic. It is called only once per scan and may not raise events.
+Returns:
+
+-
+ –
+
+
None
+
+
+
+
+Note
+This method is called only once per scan and may not raise events.
+
+
+Source code in bbot/modules/base.py
+281
+282
+283
+284
+285
+286
+287
+288
+289
+290
+291
+292
async def cleanup(self):
+ """Asynchronously performs final cleanup operations after the scan is complete.
+
+ This method can be overridden to implement custom cleanup logic. It is called only once per scan and may not raise events.
+
+ Returns:
+ None
+
+ Note:
+ This method is called only once per scan and may not raise events.
+ """
+ return
+
+
+
+
+
+
+critical
+
+critical(*args, trace=True, **kwargs)
+
+
+Logs a whole message in emboldened red text, and optionally the stack trace of the most recent exception.
+Parameters:
+
+-
+
*args
+ –
+
+Variable-length argument list to pass to the logger.
+
+
+-
+
trace
+ (bool, default:
+ True
+)
+ –
+
+Whether to log the stack trace of the most recently caught exception. Defaults to True.
+
+
+-
+
**kwargs
+ –
+
+Arbitrary keyword arguments to pass to the logger.
+
+
+
+Examples:
+>>> self.critical("This is a critical message")
+>>> self.critical("This is a critical message with a trace", trace=False)
+
+
+Source code in bbot/modules/base.py
+1526
+1527
+1528
+1529
+1530
+1531
+1532
+1533
+1534
+1535
+1536
+1537
+1538
+1539
+1540
def critical(self, *args, trace=True, **kwargs):
+ """Logs a whole message in emboldened red text, and optionally the stack trace of the most recent exception.
+
+ Args:
+ *args: Variable-length argument list to pass to the logger.
+ trace (bool, optional): Whether to log the stack trace of the most recently caught exception. Defaults to True.
+ **kwargs: Arbitrary keyword arguments to pass to the logger.
+
+ Examples:
+ >>> self.critical("This is a critical message")
+ >>> self.critical("This is a critical message with a trace", trace=False)
+ """
+ self.log.critical(*args, extra={"scan_id": self.scan.id}, **kwargs)
+ if trace:
+ self.trace()
+
+
+
+
+
+
+debug
+
+debug(*args, trace=False, **kwargs)
+
+
+Logs debug messages and optionally the stack trace of the most recent exception.
+Parameters:
+
+-
+
*args
+ –
+
+Variable-length argument list to pass to the logger.
+
+
+-
+
trace
+ (bool, default:
+ False
+)
+ –
+
+Whether to log the stack trace of the most recently caught exception. Defaults to False.
+
+
+-
+
**kwargs
+ –
+
+Arbitrary keyword arguments to pass to the logger.
@@ -4417,21 +4997,21 @@
Source code in bbot/modules/base.py
-1223
-1224
-1225
-1226
-1227
-1228
-1229
-1230
-1231
-1232
-1233
-1234
-1235
-1236
-1237
def debug(self, *args, trace=False, **kwargs):
+1346
+1347
+1348
+1349
+1350
+1351
+1352
+1353
+1354
+1355
+1356
+1357
+1358
+1359
+1360
def debug(self, *args, trace=False, **kwargs):
"""Logs debug messages and optionally the stack trace of the most recent exception.
Args:
@@ -4512,45 +5092,45 @@
Source code in bbot/modules/base.py
-443
-444
-445
-446
-447
-448
-449
-450
-451
-452
-453
-454
-455
-456
-457
-458
-459
-460
-461
-462
-463
-464
-465
-466
-467
-468
-469
-470
-471
-472
-473
-474
-475
-476
-477
-478
+478
479
480
-481
async def emit_event(self, *args, **kwargs):
+481
+482
+483
+484
+485
+486
+487
+488
+489
+490
+491
+492
+493
+494
+495
+496
+497
+498
+499
+500
+501
+502
+503
+504
+505
+506
+507
+508
+509
+510
+511
+512
+513
+514
+515
+516
async def emit_event(self, *args, **kwargs):
"""Emit an event to the event queue and distribute it to interested modules.
This is how modules "return" data.
@@ -4634,21 +5214,21 @@
Source code in bbot/modules/base.py
-1367
-1368
-1369
-1370
-1371
-1372
-1373
-1374
-1375
-1376
-1377
-1378
-1379
-1380
-1381
def error(self, *args, trace=True, **kwargs):
+1490
+1491
+1492
+1493
+1494
+1495
+1496
+1497
+1498
+1499
+1500
+1501
+1502
+1503
+1504
def error(self, *args, trace=True, **kwargs):
"""Logs an error message, and optionally the stack trace of the most recent exception.
Args:
@@ -4705,21 +5285,21 @@
Source code in bbot/modules/base.py
-231
-232
-233
-234
-235
-236
-237
-238
-239
+239
240
241
242
243
244
-245
async def filter_event(self, event):
+245
+246
+247
+248
+249
+250
+251
+252
+253
async def filter_event(self, event):
"""Asynchronously filters incoming events based on custom criteria.
Override this method for more granular control over which events are accepted by your module. This method is called automatically before `handle_event()` for each incoming event that matches any in `watched_events`.
@@ -4761,18 +5341,18 @@
Source code in bbot/modules/base.py
-247
-248
-249
-250
-251
-252
-253
-254
-255
+255
256
257
-258
async def finish(self):
+258
+259
+260
+261
+262
+263
+264
+265
+266
async def finish(self):
"""Asynchronously performs final tasks as the scan nears completion.
This method can be overridden to execute any necessary finalization logic. For example, if the module relies on a word cloud, you might wait for the scan to finish to ensure the word cloud is most complete before running an operation.
@@ -4823,24 +5403,24 @@
Source code in bbot/modules/base.py
-980
-981
-982
-983
-984
-985
-986
-987
-988
-989
-990
-991
-992
-993
-994
-995
-996
-997
def get_per_domain_hash(self, event):
+1015
+1016
+1017
+1018
+1019
+1020
+1021
+1022
+1023
+1024
+1025
+1026
+1027
+1028
+1029
+1030
+1031
+1032
def get_per_domain_hash(self, event):
"""
Computes a per-domain hash value for a given event. This method may be optionally overridden in subclasses.
@@ -4897,23 +5477,23 @@
Source code in bbot/modules/base.py
-938
-939
-940
-941
-942
-943
-944
-945
-946
-947
-948
-949
-950
-951
-952
-953
-954
def get_per_host_hash(self, event):
+973
+974
+975
+976
+977
+978
+979
+980
+981
+982
+983
+984
+985
+986
+987
+988
+989
def get_per_host_hash(self, event):
"""
Computes a per-host hash value for a given event. This method may be optionally overridden in subclasses.
@@ -4970,29 +5550,29 @@
Source code in bbot/modules/base.py
-956
-957
-958
-959
-960
-961
-962
-963
-964
-965
-966
-967
-968
-969
-970
-971
-972
-973
-974
-975
-976
-977
-978
def get_per_hostport_hash(self, event):
+ 991
+ 992
+ 993
+ 994
+ 995
+ 996
+ 997
+ 998
+ 999
+1000
+1001
+1002
+1003
+1004
+1005
+1006
+1007
+1008
+1009
+1010
+1011
+1012
+1013
def get_per_hostport_hash(self, event):
"""
Computes a per-host:port hash value for a given event. This method may be optionally overridden in subclasses.
@@ -5039,17 +5619,17 @@
Source code in bbot/modules/base.py
-360
-361
-362
-363
-364
-365
-366
-367
-368
-369
-370
def get_watched_events(self):
+395
+396
+397
+398
+399
+400
+401
+402
+403
+404
+405
def get_watched_events(self):
"""Retrieve the set of events that the module is interested in observing.
Override this method if the set of events the module should watch needs to be determined dynamically, e.g., based on configuration options or other runtime conditions.
@@ -5104,21 +5684,21 @@
Source code in bbot/modules/base.py
-215
-216
-217
-218
-219
-220
-221
-222
-223
+223
224
225
226
227
228
-229
async def handle_batch(self, *events):
+229
+230
+231
+232
+233
+234
+235
+236
+237
async def handle_batch(self, *events):
"""Handles incoming events in batches for optimized processing.
This method is automatically called when multiple events that match any in `watched_events` are encountered and the `batch_size` attribute is set to a value greater than 1. Override this method to implement custom batch event-handling logic for your module.
@@ -5175,21 +5755,21 @@
Source code in bbot/modules/base.py
-199
-200
-201
-202
-203
-204
-205
-206
-207
+207
208
209
210
211
212
-213
async def handle_event(self, event):
+213
+214
+215
+216
+217
+218
+219
+220
+221
async def handle_event(self, event):
"""Asynchronously handles incoming events that the module is configured to watch.
This method is automatically invoked when an event that matches any in `watched_events` is encountered during a scan. Override this method to implement custom event-handling logic for your module.
@@ -5249,21 +5829,21 @@
Source code in bbot/modules/base.py
-1287
-1288
-1289
-1290
-1291
-1292
-1293
-1294
-1295
-1296
-1297
-1298
-1299
-1300
-1301
def hugeinfo(self, *args, trace=False, **kwargs):
+1410
+1411
+1412
+1413
+1414
+1415
+1416
+1417
+1418
+1419
+1420
+1421
+1422
+1423
+1424
def hugeinfo(self, *args, trace=False, **kwargs):
"""Logs a whole message in emboldened blue text, and optionally the stack trace of the most recent exception.
Args:
@@ -5323,21 +5903,21 @@
Source code in bbot/modules/base.py
-1319
-1320
-1321
-1322
-1323
-1324
-1325
-1326
-1327
-1328
-1329
-1330
-1331
-1332
-1333
def hugesuccess(self, *args, trace=False, **kwargs):
+1442
+1443
+1444
+1445
+1446
+1447
+1448
+1449
+1450
+1451
+1452
+1453
+1454
+1455
+1456
def hugesuccess(self, *args, trace=False, **kwargs):
"""Logs a whole message in emboldened green text, and optionally the stack trace of the most recent exception.
Args:
@@ -5397,21 +5977,21 @@
Source code in bbot/modules/base.py
-1255
-1256
-1257
-1258
-1259
-1260
-1261
-1262
-1263
-1264
-1265
-1266
-1267
-1268
-1269
def hugeverbose(self, *args, trace=False, **kwargs):
+1378
+1379
+1380
+1381
+1382
+1383
+1384
+1385
+1386
+1387
+1388
+1389
+1390
+1391
+1392
def hugeverbose(self, *args, trace=False, **kwargs):
"""Logs a whole message in emboldened white text, and optionally the stack trace of the most recent exception.
Args:
@@ -5471,21 +6051,21 @@
Source code in bbot/modules/base.py
-1351
-1352
-1353
-1354
-1355
-1356
-1357
-1358
-1359
-1360
-1361
-1362
-1363
-1364
-1365
def hugewarning(self, *args, trace=True, **kwargs):
+1474
+1475
+1476
+1477
+1478
+1479
+1480
+1481
+1482
+1483
+1484
+1485
+1486
+1487
+1488
def hugewarning(self, *args, trace=True, **kwargs):
"""Logs a whole message in emboldened orange text, and optionally the stack trace of the most recent exception.
Args:
@@ -5545,21 +6125,21 @@
Source code in bbot/modules/base.py
-