diff --git a/.ci/benchmark.txt b/.ci/benchmark.txt index dd949166c..42a83f0a4 100644 --- a/.ci/benchmark.txt +++ b/.ci/benchmark.txt @@ -1,6 +1,6 @@ -META MD5 f019321883fa9315afcd43fa085b5bf9 -DATA MD5 de85ea0a77bd333be6a0d8422b835df4 -DATA: 16344639 interested lines. MARKUP: 62823 items +META MD5 ef775241a6d575ff10f7220dcfadf5d7 +DATA MD5 51b6d4e4debbd374fc184f2b691e0bb8 +DATA: 16344639 interested lines. MARKUP: 62827 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- 194 28318 71 418 90 @@ -113,7 +113,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .markdown 3 139 3 1 .markerb 3 12 3 .marko 1 21 2 -.md 674 149300 740 2388 621 +.md 674 149300 741 2388 621 .mdx 3 549 7 .mjml 1 18 1 .mjs 22 4424 75 340 @@ -135,7 +135,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .patch 4 109405 4 27 .pbxproj 1 941 2 .pem 48 1169 47 8 -.php 371 75710 128 1622 79 +.php 371 75710 130 1622 79 .pl 16 14727 7 33 .pm 3 744 7 .po 3 2994 15 @@ -153,7 +153,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .pug 2 193 2 .purs 1 69 4 .pxd 1 150 5 2 -.py 890 291553 679 3303 726 +.py 890 291553 681 3303 726 .pyi 4 1361 9 .pyp 1 167 1 .pyx 2 1094 23 @@ -172,7 +172,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .rs 31 9855 2 234 11 .rsc 1 691 1 .rsp 16 7101 20 10 27 -.rst 86 33980 70 323 68 +.rst 86 33980 71 323 68 .rules 1 6 2 .sample 2 25 3 4 4 .sbt 3 570 5 2 @@ -222,8 +222,8 @@ FileType FileNumber ValidLines Positives Negatives Templat .yml 419 36169 559 889 376 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10254 16344639 12221 50501 5104 -credsweeper result_cnt : 11487, lost_cnt : 0, true_cnt : 11308, false_cnt : 179 +TOTAL: 10254 16344639 12227 50501 5104 +credsweeper result_cnt : 11493, lost_cnt : 0, true_cnt : 11314, false_cnt : 179 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ----- ---- ----- ---- -------- -------- -------- -------- -------- -------- API 130 3166 188 125 123 2 3352 7 0.000596 0.053846 0.997417 0.984000 0.946154 0.964706 @@ -266,8 +266,9 @@ Salt 47 76 1 4 Secret 1297 1576 802 1288 1283 5 2373 14 0.002103 0.010794 0.994830 0.996118 0.989206 0.992650 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 Slack Token 4 1 0 4 4 0 1 0 0.000000 0.000000 1.000000 1.000000 1.000000 1.000000 +Tencent WeChat API App ID 6 0 0 6 6 0 0 0 0.000000 1.000000 1.000000 1.000000 1.000000 Token 643 4170 454 616 614 2 4622 29 0.000433 0.045101 0.994114 0.996753 0.954899 0.975377 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 URL Credentials 210 156 216 205 205 0 372 5 0.000000 0.023810 0.991409 1.000000 0.976190 0.987952 UUID 1069 265 0 1068 1067 1 264 2 0.003774 0.001871 0.997751 0.999064 0.998129 0.998596 - 12221 50501 5104 11494 11308 179 50322 913 0.003544 0.074707 0.982590 0.984417 0.925293 0.953940 + 12227 50501 5104 11500 11314 179 50322 913 0.003544 0.074671 0.982592 0.984425 0.925329 0.953963 diff --git a/credsweeper/rules/config.yaml b/credsweeper/rules/config.yaml index 02045e456..1e9f576f9 100644 --- a/credsweeper/rules/config.yaml +++ b/credsweeper/rules/config.yaml @@ -1337,3 +1337,17 @@ target: - code - doc + +- name: Tencent WeChat API App ID + severity: medium + confidence: weak + type: pattern + values: + - (?wx[0-9a-f]{16})(?![0-9A-Za-z_-]) + min_line_len: 18 + filter_type: TokenPattern + required_substrings: + - wx + target: + - code + - doc diff --git a/tests/__init__.py b/tests/__init__.py index 8d4df307e..d2fa7e181 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,20 +1,20 @@ from pathlib import Path # total number of files in test samples -SAMPLES_FILES_COUNT: int = 137 +SAMPLES_FILES_COUNT: int = 139 # the lowest value of ML threshold is used to display possible lowest values NEGLIGIBLE_ML_THRESHOLD = 0.0001 # credentials count after scan -SAMPLES_CRED_COUNT: int = 407 -SAMPLES_CRED_LINE_COUNT: int = 425 +SAMPLES_CRED_COUNT: int = 410 +SAMPLES_CRED_LINE_COUNT: int = 428 # credentials count after post-processing -SAMPLES_POST_CRED_COUNT: int = 364 +SAMPLES_POST_CRED_COUNT: int = 367 # with option --doc -SAMPLES_IN_DOC = 443 +SAMPLES_IN_DOC = 446 # archived credentials that are not found without --depth SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 30 diff --git a/tests/data/depth_3.json b/tests/data/depth_3.json index 215662451..e365faebf 100644 --- a/tests/data/depth_3.json +++ b/tests/data/depth_3.json @@ -13194,6 +13194,87 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Tencent WeChat API App ID", + "severity": "medium", + "confidence": "weak", + "line_data_list": [ + { + "line": "wxe191bc54fc7d529a", + "line_num": 1, + "path": "./tests/samples/wechat_p", + "info": "./tests/samples/wechat_p|RAW", + "value": "wxe191bc54fc7d529a", + "value_start": 0, + "value_end": 18, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.725480556997868, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Tencent WeChat API App ID", + "severity": "medium", + "confidence": "weak", + "line_data_list": [ + { + "line": "wx2274a335e18222d6", + "line_num": 2, + "path": "./tests/samples/wechat_p", + "info": "./tests/samples/wechat_p|RAW", + "value": "wx2274a335e18222d6", + "value_start": 0, + "value_end": 18, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.4138338639736006, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Tencent WeChat API App ID", + "severity": "medium", + "confidence": "weak", + "line_data_list": [ + { + "line": "wx4d82d07e3eb5ba84", + "line_num": 3, + "path": "./tests/samples/wechat_p", + "info": "./tests/samples/wechat_p|RAW", + "value": "wx4d82d07e3eb5ba84", + "value_start": 0, + "value_end": 18, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.614369445886757, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", diff --git a/tests/data/doc.json b/tests/data/doc.json index c9094df05..8dbb9fe81 100644 --- a/tests/data/doc.json +++ b/tests/data/doc.json @@ -13844,6 +13844,87 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Tencent WeChat API App ID", + "severity": "medium", + "confidence": "weak", + "line_data_list": [ + { + "line": "wxe191bc54fc7d529a", + "line_num": 1, + "path": "./tests/samples/wechat_p", + "info": "./tests/samples/wechat_p|RAW", + "value": "wxe191bc54fc7d529a", + "value_start": 0, + "value_end": 18, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.725480556997868, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Tencent WeChat API App ID", + "severity": "medium", + "confidence": "weak", + "line_data_list": [ + { + "line": "wx2274a335e18222d6", + "line_num": 2, + "path": "./tests/samples/wechat_p", + "info": "./tests/samples/wechat_p|RAW", + "value": "wx2274a335e18222d6", + "value_start": 0, + "value_end": 18, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.4138338639736006, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Tencent WeChat API App ID", + "severity": "medium", + "confidence": "weak", + "line_data_list": [ + { + "line": "wx4d82d07e3eb5ba84", + "line_num": 3, + "path": "./tests/samples/wechat_p", + "info": "./tests/samples/wechat_p|RAW", + "value": "wx4d82d07e3eb5ba84", + "value_start": 0, + "value_end": 18, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.614369445886757, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", diff --git a/tests/data/ml_threshold.json b/tests/data/ml_threshold.json index 96bd31aa4..7a6f3e2e1 100644 --- a/tests/data/ml_threshold.json +++ b/tests/data/ml_threshold.json @@ -11159,6 +11159,87 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Tencent WeChat API App ID", + "severity": "medium", + "confidence": "weak", + "line_data_list": [ + { + "line": "30fe6a5c4e4669597b255d6b09201ea9d23dae6e96d27ab8bfeb19f1cd683a5d", + "line_num": 1, + "path": "./tests/samples/wechat_p", + "info": "", + "value": "30fe6a5c4e4669597b255d6b09201ea9d23dae6e96d27ab8bfeb19f1cd683a5d", + "value_start": 0, + "value_end": 18, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.725480556997868, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Tencent WeChat API App ID", + "severity": "medium", + "confidence": "weak", + "line_data_list": [ + { + "line": "459679251cfe8d0f15a79b601a659756a62ceca7252fdf0f01a42fb1ee4f298a", + "line_num": 2, + "path": "./tests/samples/wechat_p", + "info": "", + "value": "459679251cfe8d0f15a79b601a659756a62ceca7252fdf0f01a42fb1ee4f298a", + "value_start": 0, + "value_end": 18, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.4138338639736006, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Tencent WeChat API App ID", + "severity": "medium", + "confidence": "weak", + "line_data_list": [ + { + "line": "0a3524a16f9c2dcaf3310ff08fc999a9579c9a6e0cfe6f44d4da9747db5d1f86", + "line_num": 3, + "path": "./tests/samples/wechat_p", + "info": "", + "value": "0a3524a16f9c2dcaf3310ff08fc999a9579c9a6e0cfe6f44d4da9747db5d1f86", + "value_start": 0, + "value_end": 18, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.614369445886757, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", diff --git a/tests/data/output.json b/tests/data/output.json index f13400ebb..fcb92d2c8 100644 --- a/tests/data/output.json +++ b/tests/data/output.json @@ -10025,6 +10025,87 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Tencent WeChat API App ID", + "severity": "medium", + "confidence": "weak", + "line_data_list": [ + { + "line": "wxe191bc54fc7d529a", + "line_num": 1, + "path": "./tests/samples/wechat_p", + "info": "", + "value": "wxe191bc54fc7d529a", + "value_start": 0, + "value_end": 18, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.725480556997868, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Tencent WeChat API App ID", + "severity": "medium", + "confidence": "weak", + "line_data_list": [ + { + "line": "wx2274a335e18222d6", + "line_num": 2, + "path": "./tests/samples/wechat_p", + "info": "", + "value": "wx2274a335e18222d6", + "value_start": 0, + "value_end": 18, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.4138338639736006, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Tencent WeChat API App ID", + "severity": "medium", + "confidence": "weak", + "line_data_list": [ + { + "line": "wx4d82d07e3eb5ba84", + "line_num": 3, + "path": "./tests/samples/wechat_p", + "info": "", + "value": "wx4d82d07e3eb5ba84", + "value_start": 0, + "value_end": 18, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.614369445886757, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", diff --git a/tests/samples/wechat_f b/tests/samples/wechat_f new file mode 100644 index 000000000..340439a9c --- /dev/null +++ b/tests/samples/wechat_f @@ -0,0 +1,3 @@ +wxb508f3849c445432 +wxabcdf3849c445432 +wxabcdf3849c440000 diff --git a/tests/samples/wechat_p b/tests/samples/wechat_p new file mode 100644 index 000000000..15bc81019 --- /dev/null +++ b/tests/samples/wechat_p @@ -0,0 +1,3 @@ +wxe191bc54fc7d529a +wx2274a335e18222d6 +wx4d82d07e3eb5ba84