Skip to content

Commit

Permalink
[skip actions] [ip] 2023-08-23T14:14:31+03:00
Browse files Browse the repository at this point in the history
  • Loading branch information
@babenek
babenek committed Aug 23, 2023
1 parent 21bef6d commit fa78ef6
Show file tree
Hide file tree
Showing 10 changed files with 528 additions and 5 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/benchmark.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,8 @@ jobs:
- name: Checkout CredData
uses: actions/checkout@v3
with:
repository: Samsung/CredData
repository: babenek/CredData
ref: ipmarkup

- name: Cache data
id: cache-data
Expand Down
1 change: 1 addition & 0 deletions credsweeper/filters/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from credsweeper.filters.value_first_word_check import ValueFirstWordCheck
from credsweeper.filters.value_grafana_check import ValueGrafanaCheck
from credsweeper.filters.value_iban_check import ValueIbanCheck
from credsweeper.filters.value_ip_check import ValueIPCheck
from credsweeper.filters.value_json_web_token_check import ValueJsonWebTokenCheck
from credsweeper.filters.value_last_word_check import ValueLastWordCheck
from credsweeper.filters.value_length_check import ValueLengthCheck
Expand Down
43 changes: 43 additions & 0 deletions credsweeper/filters/value_ip_check.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
import contextlib
import ipaddress

from credsweeper.config import Config
from credsweeper.credentials import LineData
from credsweeper.file_handler.analysis_target import AnalysisTarget
from credsweeper.filters import Filter


class ValueIPCheck(Filter):
"""Filter out some of insensible IP"""

FALSE_POSITIVE_MARKERS = ["version", "oid", "section", "rfc"]

def __init__(self, config: Config = None) -> None:
pass

def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
"""Run filter checks on received credential candidate data 'line_data'.
Args:
line_data: credential candidate data
target: multiline target from which line data was obtained
Return:
True, if need to filter candidate and False if left
"""
if not line_data.value:
return True

with contextlib.suppress(Exception):
ip = ipaddress.ip_address(line_data.value)
if 4 == ip.version:
line_lower = target.line.lower()
for i in ValueIPCheck.FALSE_POSITIVE_MARKERS:
if i in line_lower:
return True
if ip.is_loopback or ip.is_private or ip.is_reserved or ip.is_link_local or ip.is_multicast:
return True
return False

return True
24 changes: 24 additions & 0 deletions credsweeper/rules/config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,30 @@
- api
doc_available: false

- name: IPv4
severity: info
type: pattern
values:
- (^|[^.0-9a-zA-Z])(?P<value>[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2})(?!/([123]?[0-9])([^0-9]|$))([^.0-9a-zA-Z$]|$)
filter_type:
- ValueIPCheck
min_line_len: 10
required_substrings:
- "."
doc_available: false

- name: IPv6
severity: info
type: pattern
values:
- (^|[^:0-9a-zA-Z])(?P<value>[0-9A-Fa-f]{0,4}:(:?[0-9A-Fa-f]{1,4}:?){0,6}:[0-9A-Fa-f]{1,4})([^:0-9a-zA-Z]|$)
filter_type:
- ValueIPCheck
min_line_len: 10
required_substrings:
- ":"
doc_available: false

- name: AWS Client ID
severity: high
type: pattern
Expand Down
8 changes: 4 additions & 4 deletions tests/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
from pathlib import Path

# total number of files in test samples
SAMPLES_FILES_COUNT: int = 109
SAMPLES_FILES_COUNT: int = 111

# credentials count after scan
SAMPLES_CRED_COUNT: int = 112
SAMPLES_CRED_LINE_COUNT: int = 123
SAMPLES_CRED_COUNT: int = 118
SAMPLES_CRED_LINE_COUNT: int = 129

# credentials count after post-processing
SAMPLES_POST_CRED_COUNT: int = 106
SAMPLES_POST_CRED_COUNT: int = 112

# with option --doc
SAMPLES_IN_DOC = 72
Expand Down
132 changes: 132 additions & 0 deletions tests/data/depth_3.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1213,6 +1213,138 @@
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "IPv4",
"severity": "info",
"line_data_list": [
{
"line": "100.64.0.0\u2013100.127.255.255",
"line_num": 14,
"path": "tests/samples/ipv4",
"info": "tests/samples/ipv4|RAW",
"value": "100.64.0.0",
"variable": null,
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 1.5253496664211537,
"valid": false
}
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "IPv4",
"severity": "info",
"line_data_list": [
{
"line": "192.88.99.0\u2013192.88.99.255",
"line_num": 26,
"path": "tests/samples/ipv4",
"info": "tests/samples/ipv4|RAW",
"value": "192.88.99.0",
"variable": null,
"entropy_validation": {
"iterator": "BASE36_CHARS",
"entropy": 1.9018695860849921,
"valid": false
}
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "IPv6",
"severity": "info",
"line_data_list": [
{
"line": "2004:5678::9324",
"line_num": 12,
"path": "tests/samples/ipv6",
"info": "tests/samples/ipv6|RAW",
"value": "2004:5678::9324",
"variable": null,
"entropy_validation": {
"iterator": "BASE36_CHARS",
"entropy": 2.725512476486815,
"valid": false
}
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "IPv6",
"severity": "info",
"line_data_list": [
{
"line": "2004::5678:9",
"line_num": 13,
"path": "tests/samples/ipv6",
"info": "tests/samples/ipv6|RAW",
"value": "2004::5678:9",
"variable": null,
"entropy_validation": {
"iterator": "BASE36_CHARS",
"entropy": 2.5220552088742005,
"valid": false
}
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "IPv6",
"severity": "info",
"line_data_list": [
{
"line": "2041:0000:140F::875B:131B",
"line_num": 14,
"path": "tests/samples/ipv6",
"info": "tests/samples/ipv6|RAW",
"value": "2041:0000:140F::875B:131B",
"variable": null,
"entropy_validation": {
"iterator": "HEX_CHARS",
"entropy": 2.6146939516467023,
"valid": false
}
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "IPv6",
"severity": "info",
"line_data_list": [
{
"line": "2041:0:140F::875B:131B",
"line_num": 15,
"path": "tests/samples/ipv6",
"info": "tests/samples/ipv6|RAW",
"value": "2041:0:140F::875B:131B",
"variable": null,
"entropy_validation": {
"iterator": "HEX_CHARS",
"entropy": 2.684338637030481,
"valid": false
}
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "VALIDATED_KEY",
Expand Down
132 changes: 132 additions & 0 deletions tests/data/ml_threshold_0.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1213,6 +1213,138 @@
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "IPv4",
"severity": "info",
"line_data_list": [
{
"line": "100.64.0.0\u2013100.127.255.255",
"line_num": 14,
"path": "tests/samples/ipv4",
"info": "",
"value": "100.64.0.0",
"variable": null,
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 1.5253496664211537,
"valid": false
}
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "IPv4",
"severity": "info",
"line_data_list": [
{
"line": "192.88.99.0\u2013192.88.99.255",
"line_num": 26,
"path": "tests/samples/ipv4",
"info": "",
"value": "192.88.99.0",
"variable": null,
"entropy_validation": {
"iterator": "BASE36_CHARS",
"entropy": 1.9018695860849921,
"valid": false
}
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "IPv6",
"severity": "info",
"line_data_list": [
{
"line": "2004:5678::9324",
"line_num": 12,
"path": "tests/samples/ipv6",
"info": "",
"value": "2004:5678::9324",
"variable": null,
"entropy_validation": {
"iterator": "BASE36_CHARS",
"entropy": 2.725512476486815,
"valid": false
}
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "IPv6",
"severity": "info",
"line_data_list": [
{
"line": "2004::5678:9",
"line_num": 13,
"path": "tests/samples/ipv6",
"info": "",
"value": "2004::5678:9",
"variable": null,
"entropy_validation": {
"iterator": "BASE36_CHARS",
"entropy": 2.5220552088742005,
"valid": false
}
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "IPv6",
"severity": "info",
"line_data_list": [
{
"line": "2041:0000:140F::875B:131B",
"line_num": 14,
"path": "tests/samples/ipv6",
"info": "",
"value": "2041:0000:140F::875B:131B",
"variable": null,
"entropy_validation": {
"iterator": "HEX_CHARS",
"entropy": 2.6146939516467023,
"valid": false
}
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "IPv6",
"severity": "info",
"line_data_list": [
{
"line": "2041:0:140F::875B:131B",
"line_num": 15,
"path": "tests/samples/ipv6",
"info": "",
"value": "2041:0:140F::875B:131B",
"variable": null,
"entropy_validation": {
"iterator": "HEX_CHARS",
"entropy": 2.684338637030481,
"valid": false
}
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
Expand Down
Loading

0 comments on commit fa78ef6

Please sign in to comment.