Merge branch 'main' into docx

Samsung · Nov 1, 2023 · 34a3476 · 34a3476
2 parents bdde41a + f76d16a
commit 34a3476
Show file tree

Hide file tree

Showing 13 changed files with 338 additions and 348 deletions.
diff --git a/cicd/benchmark.txt b/cicd/benchmark.txt
@@ -10,16 +10,16 @@ Predefined Pattern                  326            2          40
 Private Key                        1001            1           3
 Seed, Salt, Nonce                    40            4           4
 TOTAL:                             5307        63688        5644
-Detected Credentials: 5993
-credsweeper result_cnt : 5337, lost_cnt : 0, true_cnt : 4439, false_cnt : 898
+Detected Credentials: 5997
+credsweeper result_cnt : 5339, lost_cnt : 0, true_cnt : 4441, false_cnt : 898
 Category                      TP    FP        TN    FN        FPR        FNR       ACC       PRC       RCL        F1
 --------------------------  ----  ----  --------  ----  ---------  ---------  --------  --------  --------  --------
 Authentication Key & Token    54     4        28    16  0.125      0.228571   0.803922  0.931034  0.771429  0.84375
 Generic Secret               973     3       215    83  0.0137615  0.0785985  0.932496  0.996926  0.921402  0.957677
-Generic Token                287     7       596    46  0.0116086  0.138138   0.943376  0.97619   0.861862  0.91547
+Generic Token                289     7       596    44  0.0116086  0.132132   0.945513  0.976351  0.867868  0.918919
 Other                        818   750     63395   258  0.0116923  0.239777   0.984545  0.521684  0.760223  0.618759
 Password                     995   130      4150   410  0.0303738  0.291815   0.905013  0.884444  0.708185  0.786561
 Predefined Pattern           309     2        40    17  0.0476191  0.0521472  0.94837   0.993569  0.947853  0.970173
 Private Key                  967     0         4    34             0.033966   0.966169  1         0.966034  0.982724
 Seed, Salt, Nonce             36     2         6     4  0.25       0.1        0.875     0.947368  0.9       0.923077
-                            4439   898  19428253   868  4.622e-05  0.163558   0.999909  0.831741  0.836442  0.834085
+                            4441   898  19428253   866  4.622e-05  0.163181   0.999909  0.831804  0.836819  0.834304
diff --git a/credsweeper/filters/value_allowlist_check.py b/credsweeper/filters/value_allowlist_check.py
@@ -11,7 +11,7 @@ class ValueAllowlistCheck(Filter):
     """Check that patterns from the list is not present in the candidate value."""
 
     ALLOWED = [
-        r"ENC\(.*\)", r"ENC\[.*\]", r"\$\{.*\}", r"#\{.*\}", r"\{\{.+\}\}", r"([.a-z0-9]|->)+\(.*\)", r"\*\*\*\*\*"
+        r"ENC\(.*\)", r"ENC\[.*\]", r"\$\{.*\}", r"#\{.*\}", r"\{\{.+\}\}", r"([.a-z0-9]|->)+\(.*\)", r"\S{0,5}\*{5,}"
     ]
     ALLOWED_PATTERN = re.compile(  #
         Util.get_regex_combine_or(ALLOWED),  #

diff --git a/credsweeper/rules/config.yaml b/credsweeper/rules/config.yaml
@@ -5,6 +5,7 @@
     - (?P<variable>[`'\"]?(?i:token|secret|key|키|암호|암호화|토큰)[`'\"]?)((\s)*[=:](\s)*)(?P<quote>[`'\"(])?(?P<value>\S{4,})(?(quote)[)`'\"])
   filter_type:
     - ValueAllowlistCheck
+    - ValuePatternCheck
   min_line_len: 10
   required_substrings:
     - token
@@ -26,6 +27,7 @@
     - (?P<variable>[`'\"]?(?i:(?<!id[ :/])pa[as]swo?r?ds?|pwd?|p/w|비밀번호|비번|패스워드|암호)[`'\"]?)((\s)*[=:](\s)*)(?P<quote>[`'\"(])?(?P<value>\S{4,})(?(quote)[)`'\"])
   filter_type:
     - ValueAllowlistCheck
+    - ValuePatternCheck
   min_line_len: 10
   required_substrings:
     - pass
@@ -43,9 +45,10 @@
   severity: medium
   type: pattern
   values:
-    - (^|(?P<variable>(?i:\bip[\s/]+id[\s/]+pw[\s/:]*))|(?P<url>://)|\s)(?P<ip>[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2})((?P<lpar>\s*\()?\s*|(?(variable)[\s,/]+|\s*(?(url)[,]|[,/])\s*))[\w.-]{3,}[\s,/]+(?P<value>(?(lpar)[^)\s/]{4,}|(?(url)[^\s/]{4,}|[^\s]{4,})))
+    - (^|(?P<variable>(?i:\bip[\s/]+id[\s/]+pw[\s/:]*))|(?P<url>://)|\s)(?P<ip>[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2})((?P<lpar>\s*(\w+\s+)?\()?\s*|(?(variable)[\s,/]+|\s*(?(url)[,]|[,/]))\s*)[\w.-]{3,}[\s,/]+(?P<value>(?(lpar)[^)\s]{4,}|[^\s/]{4,}))(?:\s|[^/]|$)
   filter_type:
     - ValueAllowlistCheck
+    - ValuePatternCheck
   min_line_len: 10
   required_substrings:
     - "."
@@ -59,6 +62,7 @@
     - (?P<ddash>--)?(?P<variable>(?i:user\s*)?(?i:id|login|account|root|admin|user|name|wifi|role|host|default|계정|아이디))\s*?(?(ddash)[ =]|[ :=])\s*?(?P<value>\S+)
   filter_type:
     - ValueAllowlistCheck
+    - ValuePatternCheck
   min_line_len: 10
   required_substrings:
     - pass
@@ -79,6 +83,7 @@
   filter_type:
     - ValueAllowlistCheck
     - ValueDictionaryKeywordCheck
+    - ValuePatternCheck
   min_line_len: 10
   required_substrings:
     - pw
@@ -194,7 +199,6 @@
   values:
     - (^|[^.0-9A-Za-z_/+-])(?P<value>(ABIA|ACCA|AGPA|AIDA|AIPA|AKIA|ANPA|ANVA|AROA|APKA|ASCA|ASIA)[0-9A-Z]{16,17})([^=0-9A-Za-z_/+-]|$)
   filter_type: GeneralPattern
-  use_ml: true
   required_substrings:
     - A
   min_line_len: 20
@@ -207,7 +211,6 @@
     - (^|[^.0-9A-Za-z_/+-])(?P<value>(AKIA|ASIA)[0-9A-Z]{16,17})([^=0-9A-Za-z_/+-]|$)
     - (?P<value>[0-9a-zA-Z/+]{40})
   filter_type: GeneralPattern
-  use_ml: true
   required_substrings:
     - AKIA
     - ASIA
@@ -219,7 +222,6 @@
   values:
     - (^|[^.0-9A-Za-z_/+-])(?P<value>amzn\.mws\.[0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12})([^=0-9A-Za-z_/+-]|$)
   filter_type: GeneralPattern
-  use_ml: true
   required_substrings:
     - amzn
   min_line_len: 30
@@ -242,7 +244,6 @@
   values:
     - (^|[^.0-9A-Za-z_/+-])(?P<value>dt0[a-zA-Z]{1}[0-9]{2}\.[A-Z0-9]{24}\.[A-Z0-9]{64})([^=0-9A-Za-z_/+-]|$)
   filter_type: GeneralPattern
-  use_ml: true
   required_substrings:
     - dt0
   min_line_len: 90
@@ -253,7 +254,6 @@
   values:
     - (^|[^.0-9A-Za-z_/+-])(?P<value>EAAC[0-9A-Za-z]{27,})
   filter_type: GeneralPattern
-  use_ml: true
   required_substrings:
     - EAAC
   min_line_len: 31
@@ -277,7 +277,6 @@
   values:
     - (^|[^.0-9A-Za-z_/+-])(?P<value>AIza[0-9A-Za-z_-]{35})([^=0-9A-Za-z_/+-]|$)
   filter_type: GeneralPattern
-  use_ml: false
   validations:
     - GoogleApiKeyValidation
   required_substrings:
@@ -291,7 +290,6 @@
     - (?P<value>[0-9]+\-[0-9A-Za-z_]{32}\.apps\.googleusercontent\.com)
     - (?<![0-9a-zA-Z_-])(?P<value>[0-9a-zA-Z_-]{24})([^=0-9A-Za-z_/+-]|$)
   filter_type: GeneralPattern
-  use_ml: false
   validations:
     - GoogleMultiValidation
   required_substrings:
@@ -304,7 +302,6 @@
   values:
     - (^|[^.0-9A-Za-z_/+-])(?P<value>ya29\.[0-9A-Za-z_-]{22,})
   filter_type: GeneralPattern
-  use_ml: true
   required_substrings:
     - ya29.
   min_line_len: 27
@@ -315,7 +312,6 @@
   values:
     - (?i)(?P<value>heroku(.{0,20})?[0-9a-f]{8}(-[0-9a-f]{4})+-[0-9a-f]{12})([^=0-9A-Za-z_/+-]|$)
   filter_type: GeneralPattern
-  use_ml: true
   required_substrings:
     - heroku
   min_line_len: 24
@@ -326,7 +322,6 @@
   values:
     - (^|[^.0-9A-Za-z_/+-])(?P<value>IGQVJ[\w]{100,})
   filter_type: GeneralPattern
-  use_ml: true
   required_substrings:
     - IGQVJ
   min_line_len: 105
@@ -348,7 +343,6 @@
   values:
     - (^|[^.0-9A-Za-z_/+-])(?P<value>[0-9a-zA-Z]{32}-us[0-9]{1,2})([^=0-9A-Za-z_/+-]|$)
   filter_type: GeneralPattern
-  use_ml: false
   validations:
     - MailChimpKeyValidation
   required_substrings:
@@ -361,7 +355,6 @@
   values:
     - (^|[^.0-9A-Za-z_/+-])(?P<value>key-[0-9a-zA-Z]{32})([^=0-9A-Za-z_/+-]|$)
   filter_type: GeneralPattern
-  use_ml: true
   required_substrings:
     - key-
   min_line_len: 36
@@ -385,7 +378,6 @@
   values:
     - (?P<value>access_token\$production\$[0-9a-z]{16}\$[0-9a-z]{32})([^=0-9A-Za-z_/+-]|$)
   filter_type: GeneralPattern
-  use_ml: false
   required_substrings:
     - access_token$production$
   min_line_len: 72
@@ -405,7 +397,6 @@
   values:
     - (?P<value>sk_live_[0-9a-z]{32})([^=0-9A-Za-z_/+-]|$)
   filter_type: GeneralPattern
-  use_ml: false
   required_substrings:
     - sk_live_
   min_line_len: 40
@@ -428,7 +419,6 @@
   values:
     - (?P<value>SG\.[\w_]{16,32}\.[\w_]{16,64})
   filter_type: GeneralPattern
-  use_ml: false
   required_substrings:
     - SG.
   min_line_len: 34
@@ -449,7 +439,6 @@
   values:
     - (^|[^.0-9A-Za-z_/+-])(?P<value>xox[a|b|p|r|o|s]\-[-a-zA-Z0-9]{10,250})
   filter_type: GeneralPattern
-  use_ml: true
   validations:
     - SlackTokenValidation
   required_substrings:
@@ -462,7 +451,6 @@
   values:
     - (?P<value>hooks\.slack\.com/services/T\w{8}/B\w{8}/\w{24})
   filter_type: GeneralPattern
-  use_ml: true
   required_substrings:
     - hooks.slack.com/services/T
   min_line_len: 61
@@ -473,7 +461,6 @@
   values:
     - (?P<value>sk_live_[0-9a-zA-Z]{24})([^=0-9A-Za-z_/+-]|$)
   filter_type: GeneralPattern
-  use_ml: true
   validations:
     - StripeApiKeyValidation
   required_substrings:
@@ -486,7 +473,6 @@
   values:
     - (?P<value>rk_live_[0-9a-zA-Z]{24})([^=0-9A-Za-z_/+-]|$)
   filter_type: GeneralPattern
-  use_ml: true
   required_substrings:
     - rk_live_
   min_line_len: 32
@@ -497,7 +483,6 @@
   values:
     - (^|[^.0-9A-Za-z_/+-])(?P<value>EAAA[0-9A-Za-z_-]{60})([^=0-9A-Za-z_/+-]|$)
   filter_type: GeneralPattern
-  use_ml: true
   validations:
     - SquareAccessTokenValidation
   required_substrings:
@@ -510,7 +495,6 @@
   values:
     - (^|[^.0-9A-Za-z_/+-])(?P<value>sq0[a-z]{3}-[0-9A-Za-z_-]{22})([^=0-9A-Za-z_/+-]|$)
   filter_type: GeneralPattern
-  use_ml: true
   validations:
     - SquareClientIdValidation
   required_substrings:
@@ -523,7 +507,6 @@
   values:
     - (?P<value>sq0csp-[0-9A-Za-z_-]{43})([^=0-9A-Za-z_/+-]|$)
   filter_type: GeneralPattern
-  use_ml: false
   required_substrings:
     - sq0csp
   min_line_len: 50
@@ -546,7 +529,6 @@
   values:
     - (^|[^.0-9A-Za-z_/+-])(?P<value>SK[0-9a-fA-F]{32})([^=0-9A-Za-z_/+-]|$)
   filter_type: GeneralPattern
-  use_ml: true
   required_substrings:
     - SK
   min_line_len: 34

diff --git a/tests/__init__.py b/tests/__init__.py
@@ -8,10 +8,10 @@
 SAMPLES_CRED_LINE_COUNT: int = 402
 
 # credentials count after post-processing
-SAMPLES_POST_CRED_COUNT: int = 293
+SAMPLES_POST_CRED_COUNT: int = 296
 
 # with option --doc
-SAMPLES_IN_DOC = 431
+SAMPLES_IN_DOC = 427
 
 # archived credentials that are not found without --depth
 SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 21