From 23f903404a8cda1829fcf6575dfaa2d816fce541 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Tue, 30 Jul 2019 09:02:33 +0200 Subject: [PATCH 01/49] - replaced PyYAML by ruamel.yaml (needed to preserve comments and block style and key ordering) - updated attackcti to v0.2.7 --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index f942e3b6..a99ec44d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -attackcti==0.2.6 +attackcti==0.2.7 simplejson==3.16.0 -PyYAML==5.1.1 plotly==3.10.0 pandas==0.24.2 -xlsxwriter==1.1.8 \ No newline at end of file +xlsxwriter==1.1.8 +ruamel.yaml=0.16.0 \ No newline at end of file From 1ea3547728dfe3286be06a19f3814f6bdc851364 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Tue, 30 Jul 2019 14:41:06 +0200 Subject: [PATCH 02/49] Added an empty data source administration YAML file --- sample-data/data-sources-empty.yaml | 627 ++++++++++++++++++++++++++++ 1 file changed, 627 insertions(+) create mode 100644 sample-data/data-sources-empty.yaml diff --git a/sample-data/data-sources-empty.yaml b/sample-data/data-sources-empty.yaml new file mode 100644 index 00000000..d4e8dc24 --- /dev/null +++ b/sample-data/data-sources-empty.yaml @@ -0,0 +1,627 @@ +%YAML 1.2 +--- +version: 1.0 +file_type: data-source-administration +name: empty-data-source-admin-file +# Fill in the correct MITRE ATT&CK enterprise platform (Windows, Linux or MacOS) +platform: +data_sources: + # A data source is treated as not available when all dimensions of the data quality have a score of 0. + # If desired you are free to add any key-value pairs. + - data_source_name: Process monitoring + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: File monitoring + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Process command-line parameters + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: API monitoring + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Process use of network + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Windows Registry + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Packet capture + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Authentication logs + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Netflow/Enclave netflow + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Windows event logs + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Binary file metadata + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Network protocol analysis + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: DLL monitoring + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Loaded DLLs + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: System calls + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Malware reverse engineering + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: SSL/TLS inspection + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Anti-virus + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Network intrusion detection system + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Data loss prevention + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Application logs + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Email gateway + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Network device logs + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Web proxy + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Windows Error Reporting + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Kernel drivers + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: User interface + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Host network interface + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Third-party application logs + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Services + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Web logs + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Detonation chamber + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Mail server + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Environment variable + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: MBR + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: BIOS + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Web application firewall logs + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Asset management + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: DHCP + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: 'At the time of writing: unknown data source within ATT&CK' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: DNS records + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Browser extensions + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Access tokens + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Digital certificate logs + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Disk forensics + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Component firmware + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: WMI Objects + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: VBR + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Named Pipes + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: Sensor health and status + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: EFI + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 + - data_source_name: PowerShell logs + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False + comment: '' + data_quality: + device_completeness: 0 + data_field_completeness: 0 + timeliness: 0 + consistency: 0 + retention: 0 +exceptions: + # Adding a technique ID below will result in removing that technique in the heat map (meaning not enough data source or quality is available for proper detection). + # Filling in the key-value pair name is optional. + - technique_id: + name: \ No newline at end of file From 6a23aae605e12fb451ab6a1ff6e6da1a3632a76f Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Tue, 30 Jul 2019 20:51:18 +0200 Subject: [PATCH 03/49] Added the data source as available (with a high DQ): Process use of network --- sample-data/data-sources-endpoints.yaml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/sample-data/data-sources-endpoints.yaml b/sample-data/data-sources-endpoints.yaml index b8de3e6f..b20e0371 100644 --- a/sample-data/data-sources-endpoints.yaml +++ b/sample-data/data-sources-endpoints.yaml @@ -56,17 +56,17 @@ data_sources: consistency: 0 retention: 0 - data_source_name: Process use of network - date_registered: - date_connected: - products: [None] - available_for_data_analytics: False + date_registered: 2019-07-25 + date_connected: 2019-07-25 + products: [Sysmon] + available_for_data_analytics: True comment: '' data_quality: - device_completeness: 0 - data_field_completeness: 0 - timeliness: 0 - consistency: 0 - retention: 0 + device_completeness: 5 + data_field_completeness: 5 + timeliness: 5 + consistency: 5 + retention: 5 - data_source_name: Windows Registry date_registered: 2019-03-01 date_connected: 2017-02-01 @@ -620,7 +620,8 @@ data_sources: consistency: 5 retention: 4 exceptions: - # Adding a technique ID below will result in removing that technique in the heat map (meaning not enough data source are available for proper detection). + # Adding a technique ID below will result in removing that technique in the heat map (meaning not enough data source or quality is available for proper detection). # Please note that the below is just an example, many more can exists. + # Filling in the key-value pair name is optional. - technique_id: T1130 name: Install Root Certificate \ No newline at end of file From 539611dac7f74edc87b1f1f28d185b9039ddac83 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Wed, 31 Jul 2019 10:05:44 +0200 Subject: [PATCH 04/49] - Updated to version 1.2 of the tech. administration file. - Improved visibility scores due to the new data source: Process use of network. --- .../techniques-administration-endpoints.yaml | 3044 +++++++++++------ 1 file changed, 2009 insertions(+), 1035 deletions(-) diff --git a/sample-data/techniques-administration-endpoints.yaml b/sample-data/techniques-administration-endpoints.yaml index 4f828d1a..2a43f116 100644 --- a/sample-data/techniques-administration-endpoints.yaml +++ b/sample-data/techniques-administration-endpoints.yaml @@ -1,6 +1,4 @@ -%YAML 1.2 ---- -version: 1.1 +version: 1.2 file_type: technique-administration name: example platform: windows @@ -13,2318 +11,3294 @@ techniques: - technique_id: T1222 technique_name: File Permissions Modification detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1223 technique_name: Compiled HTML File detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1221 technique_name: Template Injection detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' - technique_id: T1220 technique_name: XSL Script Processing detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-07-30 + score: 2 + comment: 'New data source: Process use of network' + auto_generated: true + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1217 technique_name: Browser Bookmark Discovery detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1196 technique_name: Control Panel Items detection: - applicable_to: ['client endpoints'] - date_registered: 2019-01-10 - date_implemented: 2018-12-01 - score: 4 + applicable_to: [client endpoints] location: [EDR] comment: '' + score_logbook: + - date: 2018-12-01 + score: 4 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1214 technique_name: Credentials in Registry detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2018-12-01 - score: 3 + applicable_to: [all] location: [EDR] comment: '' + score_logbook: + - date: 2018-12-01 + score: 3 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1189 technique_name: Drive-by Compromise detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2018-11-01 - score: 1 + applicable_to: [all] location: [SIEM UC 123, Tool Model Y] comment: '' + score_logbook: + - date: 2019-08-05 + score: 3 + comment: 'This detection was improved due to the availability of the new log source Process use of network' + - date: 2018-11-01 + score: 1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-07-30 + score: 2 + comment: 'New data source: Process use of network' + auto_generated: true + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1203 technique_name: Exploitation for Client Execution detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1210 technique_name: Exploitation of Remote Services detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1211 technique_name: Exploitation for Defense Evasion detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1202 technique_name: Indirect Command Execution detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1212 technique_name: Exploitation for Credential Access detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1201 technique_name: Password Policy Discovery detection: - applicable_to: ['domain controllers'] - date_registered: 2019-01-10 - date_implemented: 2017-01-01 - score: 4 + applicable_to: [domain controllers] location: - - 'Third party product A' + - Third party product A comment: '' + score_logbook: + - date: 2017-01-01 + score: 4 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1191 technique_name: CMSTP detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-07-30 + score: 2 + comment: 'New data source: Process use of network' + auto_generated: true + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1219 technique_name: Remote Access Tools detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2017-01-01 - score: 4 + applicable_to: [all] location: - - 'Third party product A' + - Third party product A comment: '' + score_logbook: + - date: 2017-01-01 + score: 4 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-07-30 + score: 3 + comment: 'New data source: Process use of network' + auto_generated: true + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1198 technique_name: SIP and Trust Provider Hijacking detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1218 technique_name: Signed Binary Proxy Execution detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1193 technique_name: Spearphishing Attachment detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1216 technique_name: Signed Script Proxy Execution detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1192 technique_name: Spearphishing Link detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1209 technique_name: Time Providers detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1195 technique_name: Supply Chain Compromise detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2017-01-01 - score: 2 + applicable_to: [all] location: - - 'Third party product A' + - Third party product A comment: '' + score_logbook: + - date: 2017-01-01 + score: 2 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1194 technique_name: Spearphishing via Service detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 4 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 4 + comment: '' + auto_generated: true - technique_id: T1204 technique_name: User Execution detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2018-12-01 - score: 0 + applicable_to: [all] location: [EDR] comment: '' + score_logbook: + - date: 2018-12-01 + score: 0 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1182 technique_name: AppCert DLLs detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2018-12-01 - score: 3 + applicable_to: [all] location: [EDR] comment: '' + score_logbook: + - date: 2018-12-01 + score: 3 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1176 technique_name: Browser Extensions detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1175 technique_name: Distributed Component Object Model detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1185 technique_name: Man in the Browser detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1174 technique_name: Password Filter DLL detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1170 technique_name: Mshta detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1171 technique_name: LLMNR/NBT-NS Poisoning detection: - - applicable_to: ['client endpoints'] - date_registered: 2019-01-10 - date_implemented: 2017-01-01 - score: 2 - location: - - 'Third party product A' - comment: | - This comment will be - multiline in - Excel - - applicable_to: ['servers'] - date_registered: 2019-05-01 - date_implemented: 2019-05-01 + - applicable_to: [client endpoints] + location: + - Third party product A + comment: | + This comment will be + multiline in + Excel + score_logbook: + - date: 2017-01-01 + score: 2 + comment: '' + - applicable_to: [servers] + location: + - Model I + comment: '' + score_logbook: + - date: 2019-05-01 score: 3 - location: - - 'Model I' comment: '' visibility: - - applicable_to: ['client endpoints'] + - applicable_to: [client endpoints] + comment: '' + score_logbook: + - date: 2019-03-01 score: 2 comment: '' - - applicable_to: ['servers'] + - applicable_to: [servers] + comment: | + This comment will be + multiline in + Excel + score_logbook: + - date: 2019-03-01 score: 3 - comment: | - This comment will be - multiline in - Excel + comment: '' - technique_id: T1173 technique_name: Dynamic Data Exchange detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1181 technique_name: Extra Window Memory Injection detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2018-12-01 - score: 4 + applicable_to: [all] location: [EDR] comment: '' + score_logbook: + - date: 2018-12-01 + score: 4 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1179 technique_name: Hooking detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1186 technique_name: Process Doppelgänging detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1172 technique_name: Domain Fronting detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2018-08-01 - score: 5 + applicable_to: [all] location: - - 'Model A' + - Model A comment: '' + score_logbook: + - date: 2018-08-01 + score: 5 + comment: '' visibility: - applicable_to: ['all'] - score: 4 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 4 + comment: '' - technique_id: T1183 technique_name: Image File Execution Options Injection detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2018-11-01 - score: 2 + applicable_to: [all] location: [Tool] comment: '' + score_logbook: + - date: 2018-11-01 + score: 2 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' - technique_id: T1177 technique_name: LSASS Driver detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1180 technique_name: Screensaver detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1134 technique_name: Access Token Manipulation detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2018-12-01 - score: 4 + applicable_to: [all] location: [EDR] comment: '' + score_logbook: + - date: 2018-12-01 + score: 4 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1138 technique_name: Application Shimming detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2018-12-01 - score: 1 + applicable_to: [all] location: [SIEM] comment: '' + score_logbook: + - date: 2018-12-01 + score: 1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1140 technique_name: Deobfuscate/Decode Files or Information detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1136 technique_name: Create Account detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1137 technique_name: Office Application Startup detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1158 technique_name: Hidden Files and Directories detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1135 technique_name: Network Share Discovery detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-07-30 + score: 2 + comment: 'New data source: Process use of network' + auto_generated: true + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1132 technique_name: Data Encoding detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1131 technique_name: Authentication Package detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1129 technique_name: Execution through Module Load detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1128 technique_name: Netsh Helper DLL detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1127 technique_name: Trusted Developer Utilities detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' - technique_id: T1126 technique_name: Network Share Connection Removal detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1125 technique_name: Video Capture detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1124 technique_name: System Time Discovery detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1123 technique_name: Audio Capture detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1122 technique_name: Component Object Model Hijacking detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1121 technique_name: Regsvcs/Regasm detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1118 technique_name: InstallUtil detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1117 technique_name: Regsvr32 detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2018-12-01 - score: 3 + applicable_to: [all] location: [EDR] comment: '' + score_logbook: + - date: 2018-12-01 + score: 3 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1114 technique_name: Email Collection detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-07-30 + score: 2 + comment: 'New data source: Process use of network' + auto_generated: true + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1113 technique_name: Screen Capture detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1112 technique_name: Modify Registry detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1111 technique_name: Two-Factor Authentication Interception detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1109 technique_name: Component Firmware detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1108 technique_name: Redundant Access detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1106 technique_name: Execution through API detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1105 technique_name: Remote File Copy detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1103 technique_name: AppInit DLLs detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1102 technique_name: Web Service detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1101 technique_name: Security Support Provider detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2018-11-01 - score: 4 + applicable_to: [all] location: [SIEM UC 789] comment: '' + score_logbook: + - date: 2018-11-01 + score: 4 + comment: '' visibility: - applicable_to: ['all'] - score: 3 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 3 + comment: '' - technique_id: T1100 technique_name: Web Shell detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1099 technique_name: Timestomp detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2018-11-01 - score: 2 + applicable_to: [all] location: [Tool Model X] comment: '' + score_logbook: + - date: 2018-11-01 + score: 2 + comment: '' visibility: - applicable_to: ['all'] - score: 4 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 4 + comment: '' - technique_id: T1095 technique_name: Standard Non-Application Layer Protocol detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 3 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 3 + comment: '' - technique_id: T1094 technique_name: Custom Command and Control Protocol detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 3 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 3 + comment: '' - technique_id: T1093 technique_name: Process Hollowing detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1090 technique_name: Connection Proxy detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-07-30 + score: 2 + comment: 'New data source: Process use of network' + auto_generated: true + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1089 technique_name: Disabling Security Tools detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1088 technique_name: Bypass User Account Control detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1087 technique_name: Account Discovery detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1086 technique_name: PowerShell detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2018-12-01 - score: 3 + applicable_to: [all] location: [EDR] comment: '' + score_logbook: + - date: 2018-12-01 + score: 3 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' - technique_id: T1085 technique_name: Rundll32 detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2018-12-01 - score: 3 + applicable_to: [all] location: [EDR] comment: '' + score_logbook: + - date: 2018-12-01 + score: 3 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1083 technique_name: File and Directory Discovery detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1082 technique_name: System Information Discovery detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2017-01-01 - score: 3 + applicable_to: [all] location: - - 'Third party product A' + - Third party product A comment: '' + score_logbook: + - date: 2017-01-01 + score: 3 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1080 technique_name: Taint Shared Content detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1079 technique_name: Multilayer Encryption detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-07-30 + score: 2 + comment: 'New data source: Process use of network' + auto_generated: true + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1078 technique_name: Valid Accounts detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - - '' + - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1077 technique_name: Windows Admin Shares detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2018-10-01 - score: 0 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: 2018-10-01 + score: 0 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-07-30 + score: 2 + comment: 'New data source: Process use of network' + auto_generated: true + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1076 technique_name: Remote Desktop Protocol detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1074 technique_name: Data Staged detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1073 technique_name: DLL Side-Loading detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-07-30 + score: 2 + comment: 'New data source: Process use of network' + auto_generated: true + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1072 technique_name: Third-party Software detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-07-30 + score: 2 + comment: 'New data source: Process use of network' + auto_generated: true + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1071 technique_name: Standard Application Layer Protocol detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2018-11-01 - score: -1 + applicable_to: [all] location: [SIEM UC 123] comment: '' + score_logbook: + - date: 2018-11-01 + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' - technique_id: T1070 technique_name: Indicator Removal on Host detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1069 technique_name: Permission Groups Discovery detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1068 technique_name: Exploitation for Privilege Escalation detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1066 technique_name: Indicator Removal from Tools detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-07-30 + score: 2 + comment: 'New data source: Process use of network' + auto_generated: true + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1065 technique_name: Uncommonly Used Port detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2018-10-01 - score: 5 + applicable_to: [all] location: - - 'Model B' + - Model B comment: '' + score_logbook: + - date: 2018-10-01 + score: 5 + comment: '' visibility: - applicable_to: ['all'] - score: 3 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 3 + comment: '' - technique_id: T1064 technique_name: Scripting detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2018-12-01 - score: 3 + applicable_to: [all] location: [EDR, AV Product] comment: '' + score_logbook: + - date: 2018-12-01 + score: 3 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1063 technique_name: Security Software Discovery detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1061 technique_name: Graphical User Interface detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1060 technique_name: Registry Run Keys / Startup Folder detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1059 technique_name: Command-Line Interface detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1058 technique_name: Service Registry Permissions Weakness detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1057 technique_name: Process Discovery detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1056 technique_name: Input Capture detection: - applicable_to: ['client endpoints'] - date_registered: 2019-01-10 - date_implemented: 2018-12-01 - score: 4 + applicable_to: [client endpoints] location: [EDR] comment: '' + score_logbook: + - date: 2018-12-01 + score: 4 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1055 technique_name: Process Injection detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2018-12-01 - score: 4 + applicable_to: [all] location: [EDR] comment: '' + score_logbook: + - date: 2018-12-01 + score: 4 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1054 technique_name: Indicator Blocking detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1053 technique_name: Scheduled Task detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - - '' + - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1051 technique_name: Shared Webroot detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1050 technique_name: New Service detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' - comment: 'Model G' + comment: Model G + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1049 technique_name: System Network Connections Discovery detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1048 technique_name: Exfiltration Over Alternative Protocol detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1047 technique_name: Windows Management Instrumentation detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1043 technique_name: Commonly Used Port detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2018-10-01 - score: 0 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: 2018-10-01 + score: 0 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1042 technique_name: Change Default File Association detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1041 technique_name: Exfiltration Over Command and Control Channel detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2017-01-01 - score: 2 + applicable_to: [all] location: - - 'Third party product A' + - Third party product A comment: '' + score_logbook: + - date: 2017-01-01 + score: 2 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1040 technique_name: Network Sniffing detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1039 technique_name: Data from Network Shared Drive detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1038 technique_name: DLL Search Order Hijacking detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1037 technique_name: Logon Scripts detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2018-05-07 - score: 3 + applicable_to: [all] location: - - 'Model F' + - Model F comment: '' + score_logbook: + - date: 2018-05-07 + score: 3 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1036 technique_name: Masquerading detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2018-02-01 - score: 4 + applicable_to: [all] location: [Model C] comment: '' + score_logbook: + - date: 2018-02-01 + score: 4 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1035 technique_name: Service Execution detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2018-12-01 - score: 4 + applicable_to: [all] location: [EDR] comment: '' + score_logbook: + - date: 2018-12-01 + score: 4 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1034 technique_name: Path Interception detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1033 technique_name: System Owner/User Discovery detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2017-01-01 - score: 3 + applicable_to: [all] location: - - 'Third party product A' + - Third party product A comment: '' + score_logbook: + - date: 2017-01-01 + score: 3 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1032 technique_name: Standard Cryptographic Protocol detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 3 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 3 + comment: '' - technique_id: T1031 technique_name: Modify Existing Service detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1030 technique_name: Data Transfer Size Limits detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - - '' + - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-07-30 + score: 2 + comment: 'New data source: Process use of network' + auto_generated: true + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1029 technique_name: Scheduled Transfer detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-07-30 + score: 2 + comment: 'New data source: Process use of network' + auto_generated: true + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1028 technique_name: Windows Remote Management detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - - '' + - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1027 technique_name: Obfuscated Files or Information detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1026 technique_name: Multiband Communication detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1025 technique_name: Data from Removable Media detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1024 technique_name: Custom Cryptographic Protocol detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2018-12-01 - score: 0 + applicable_to: [all] location: [EDR] comment: '' + score_logbook: + - date: 2018-12-01 + score: 0 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' - technique_id: T1023 technique_name: Shortcut Modification detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1022 technique_name: Data Encrypted detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2017-10-10 - score: 2 + applicable_to: [all] location: - - 'Model D' + - Model D comment: '' + score_logbook: + - date: 2017-10-10 + score: 2 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1020 technique_name: Automated Exfiltration detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-07-30 + score: 2 + comment: 'New data source: Process use of network' + auto_generated: true + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1018 technique_name: Remote System Discovery detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2017-01-01 - score: 3 + applicable_to: [all] location: - - 'Third party product A' + - Third party product A comment: '' + score_logbook: + - date: 2017-01-01 + score: 3 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-07-30 + score: 2 + comment: 'New data source: Process use of network' + auto_generated: true + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1017 technique_name: Application Deployment Software detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-07-30 + score: 2 + comment: 'New data source: Process use of network' + auto_generated: true + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1016 technique_name: System Network Configuration Discovery detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1015 technique_name: Accessibility Features detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1013 technique_name: Port Monitors detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1012 technique_name: Query Registry detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1011 technique_name: Exfiltration Over Other Network Medium detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1010 technique_name: Application Window Discovery detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1008 technique_name: Fallback Channels detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1007 technique_name: System Service Discovery detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1005 technique_name: Data from Local System detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1004 technique_name: Winlogon Helper DLL detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1003 technique_name: Credential Dumping detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2018-12-01 - score: 3 + applicable_to: [all] location: [EDR] comment: '' + score_logbook: + - date: 2018-12-01 + score: 3 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1002 technique_name: Data Compressed detection: - applicable_to: ['all'] - date_registered: 2019-01-10 - date_implemented: 2017-10-10 - score: 2 + applicable_to: [all] location: - - 'Model E' + - Model E comment: '' + score_logbook: + - date: 2017-10-10 + score: 2 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1001 technique_name: Data Obfuscation detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1485 technique_name: Data Destruction detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1486 technique_name: Data Encrypted for Impact detection: - applicable_to: ['all'] - date_registered: 2019-05-01 - date_implemented: 2015-01-01 - score: 4 + applicable_to: [all] location: - - 'Model J' + - Model J comment: '' + score_logbook: + - date: 2015-01-01 + score: 4 + comment: '' visibility: - applicable_to: ['all'] - score: 3 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 3 + comment: '' - technique_id: T1488 technique_name: Disk Content Wipe detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1499 technique_name: Endpoint Denial of Service detection: - applicable_to: ['websites'] - date_registered: 2019-05-01 - date_implemented: 2015-01-01 - score: 5 + applicable_to: [websites] location: - - 'Third party' + - Third party comment: '' + score_logbook: + - date: 2015-01-01 + score: 5 + comment: '' visibility: - applicable_to: ['websites'] - score: 4 + applicable_to: [websites] comment: '' + score_logbook: + - date: 2019-03-01 + score: 4 + comment: '' - technique_id: T1490 technique_name: Inhibit System Recovery detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1498 technique_name: Network Denial of Service detection: - applicable_to: ['websites'] - date_registered: 2019-05-01 - date_implemented: 2015-01-01 - score: 5 + applicable_to: [websites] location: - - 'Third party' + - Third party comment: '' + score_logbook: + - date: 2015-01-01 + score: 5 + comment: '' visibility: - applicable_to: ['websites'] - score: 4 + applicable_to: [websites] comment: '' + score_logbook: + - date: 2019-03-01 + score: 4 + comment: '' - technique_id: T1496 technique_name: Resource Hijacking detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-07-30 + score: 2 + comment: 'New data source: Process use of network' + auto_generated: true + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1494 technique_name: Runtime Data Manipulation detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1489 technique_name: Service Stop detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1500 technique_name: Compile After Delivery detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1483 technique_name: Domain Generation Algorithms detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 1 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 1 + comment: '' + auto_generated: true - technique_id: T1482 technique_name: Domain Trust Discovery detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true - technique_id: T1480 technique_name: Execution Guardrails detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 4 + applicable_to: [all] comment: '' + score_logbook: + - date: 2019-03-01 + score: 4 + comment: '' + auto_generated: true - technique_id: T1497 technique_name: Virtualization/Sandbox Evasion detection: - applicable_to: ['all'] - date_registered: - date_implemented: - score: -1 + applicable_to: [all] location: - '' comment: '' + score_logbook: + - date: null + score: -1 + comment: '' visibility: - applicable_to: ['all'] - score: 2 - comment: '' \ No newline at end of file + applicable_to: [all] + comment: '' + score_logbook: + - date: 2019-03-01 + score: 2 + comment: '' + auto_generated: true +- technique_id: T1187 + technique_name: Forced Authentication + detection: + applicable_to: + - all + location: + - '' + comment: '' + score_logbook: + - date: null + score: -1 + comment: '' + visibility: + applicable_to: + - all + comment: '' + score_logbook: + - date: 2019-07-30 + score: 1 + comment: 'New data source: Process use of network' + auto_generated: true +- technique_id: T1141 + technique_name: Input Prompt + detection: + applicable_to: + - all + location: + - '' + comment: '' + score_logbook: + - date: null + score: -1 + comment: '' + visibility: + applicable_to: + - all + comment: '' + score_logbook: + - date: 2019-07-30 + score: 2 + comment: 'New data source: Process use of network' + auto_generated: true +- technique_id: T1104 + technique_name: Multi-Stage Channels + detection: + applicable_to: + - all + location: + - '' + comment: '' + score_logbook: + - date: null + score: -1 + comment: '' + visibility: + applicable_to: + - all + comment: '' + score_logbook: + - date: 2019-07-30 + score: 1 + comment: 'New data source: Process use of network' + auto_generated: true +- technique_id: T1046 + technique_name: Network Service Scanning + detection: + applicable_to: + - all + location: + - '' + comment: '' + score_logbook: + - date: null + score: -1 + comment: '' + visibility: + applicable_to: + - all + comment: '' + score_logbook: + - date: 2019-07-30 + score: 1 + comment: 'New data source: Process use of network' + auto_generated: true From b0ba153c3225a1aac279bf5b42b58bd885989aa9 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Wed, 31 Jul 2019 10:09:06 +0200 Subject: [PATCH 05/49] - Updated the version to 1.2.0 - Added new constants for the tech v1.1 to v1.2 upgrade - Added new constants for the auto-update visibility scores functionality --- constants.py | 51 +++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/constants.py b/constants.py index 20ebd750..1e706ed2 100644 --- a/constants.py +++ b/constants.py @@ -1,6 +1,8 @@ +import re + APP_NAME = 'DeTT&CT' APP_DESC = 'Detect Tactics, Techniques & Combat Threats' -VERSION = '1.1.2' +VERSION = '1.2.0' EXPIRE_TIME = 60*60*24 @@ -69,12 +71,53 @@ # YAML administration file versions FILE_TYPE_DATA_SOURCE_ADMINISTRATION_VERSION = 1.0 -FILE_TYPE_TECHNIQUE_ADMINISTRATION_VERSION = 1.1 +FILE_TYPE_TECHNIQUE_ADMINISTRATION_VERSION = 1.2 FILE_TYPE_GROUP_ADMINISTRATION_VERSION = 1.0 # YAML file upgrade text -FILE_TYPE_TECHNIQUE_ADMINISTRATION_UPGRADE_TEXT = {1.1: " - Adding new key 'technique_name' containing the ATT&CK technique name.\n" - " - Adding new key 'applicable_to' for both detection and visibility. Default value is ['all']."} +FILE_TYPE_TECHNIQUE_ADMINISTRATION_UPGRADE_TEXT = {1.1: " * Adding new key 'technique_name' containing the ATT&CK technique name.\n" + " * Adding new key 'applicable_to' for both detection and visibility. Default value is ['all'].", + 1.2: " * Detection: removing the key-value pair 'date_registered'.\n" + " You will be asked if you still want to keep this key-value pair even though DeTT&CT no longer makes use of it.\n" + " * Detection: the key-value pair 'date_implemented' will be renamed to 'date'.\n" + " * Visibility: adding a new key-value pair 'date'. You will be asked on what date to fill in for the visibility scores already present.\n" + " * Detection and visibility: the key-value pairs 'score' and 'date' are moved into a 'score_logbook'.\n" + " The primary purpose of doing this is to allow you to keep track of changes in the score."} + +# visibility update questions and answers +V_UPDATE_Q_ALL_MANUAL = 'For all most recent visibility score objects that are eligible for an update. The key-value pair \'auto-generated\' is set to \'false\' or is not present.\n' \ + 'This implies that these scores are manually assigned. How do you want to proceed?:' +V_UPDATE_Q_ALL_AUTO = 'For all most recent visibility score objects that are eligible for an update. The key-value pair \'auto-generated\' is set to \'true\'. \n' \ + 'This implies that these scores are auto-generated. How do you want to proceed?:' +V_UPDATE_Q_MIXED = 'You have visibility scores that are eligible for an update, which are manually assigned and which are calculated based on the nr. of data sources (i.e. auto-generated = true)\n' \ + 'How do you want to proceed?' +V_UPDATE_ANSWER_1 = 'Update all visibility scores that have changed.' +V_UPDATE_ANSWER_2 = 'Decide per visibility score, that has changed if you want to update or not.\n' \ + 'Both the current and new visibility score will be printed.' +V_UPDATE_ANSWER_3 = 'Only auto-update the visibility scores, that have changed, which have \'auto-generated = true\'' +V_UPDATE_ANSWER_4 = '- Auto-update the visibility scores, that have changed, which have \'auto-generated = true\'.\n' \ + '- And decide per manually assigned visibility score, that has changed, if you want to update or not.\n' \ + ' Both the current and new visibility score will be printed.' +V_UPDATE_ANSWER_CANCEL = 'Cancel.' + + +# update actions for visibility scores +V_UPDATE_ACTION_AUTO = 'auto update' +V_UPDATE_ACTION_DIFF = 'the user decides to update or not' + + +# YAML regex +REGEX_YAML_VERSION_10 = re.compile(r'^\s*version:\s+1\.0\s*$', re.IGNORECASE) +REGEX_YAML_TECHNIQUE_ID = re.compile(r'^-\s+technique_id:\s+T[0-9]{4}\s*$', re.IGNORECASE) +REGEX_YAML_TECHNIQUE_ID_FORMAT = re.compile(r'T[0-9]{4}', re.IGNORECASE) +REGEX_YAML_DETECTION = re.compile(r'^\s+detection:\s*$', re.IGNORECASE) +REGEX_YAML_VISIBILITY = re.compile(r'^\s+visibility:\s*$', re.IGNORECASE) +REGEX_YAML_INDENT_CHARS = re.compile(r'(^[\s-]+).*', re.IGNORECASE) +REGEX_YAML_VALID_DATE = re.compile(r'([12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01]))', re.IGNORECASE) +REGEX_YAML_DATE = re.compile(r'^[\s-]+date:.*$', re.IGNORECASE) +REGEX_YAML_TECHNIQUE_ID_GROUP = re.compile(r'^-\s+technique_id:\s+(T[0-9]{4})\s*$', re.IGNORECASE) + + # Interactive menu MENU_NAME_DATA_SOURCE_MAPPING = 'Data source mapping' MENU_NAME_VISIBILITY_MAPPING = 'Visibility coverage mapping' From 94f4913670213d228ec0196e9db2c3d6b8523ea6 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Wed, 31 Jul 2019 10:13:46 +0200 Subject: [PATCH 06/49] - Replaced PyYAML with ruamel.yaml - Added new functionality for the auto-update of visibility scores. - Made compatible with version 1.2 of the technique admin YAML file. --- data_source_mapping.py | 329 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 301 insertions(+), 28 deletions(-) diff --git a/data_source_mapping.py b/data_source_mapping.py index 73337783..35a8edf8 100644 --- a/data_source_mapping.py +++ b/data_source_mapping.py @@ -1,7 +1,8 @@ import simplejson -from generic import * import xlsxwriter import copy +from generic import * + # Imports for pandas and plotly are because of performance reasons in the function that uses these libraries. @@ -23,7 +24,7 @@ def generate_data_sources_layer(filename): output_filename = 'output/data_sources_' + normalize_name_to_filename(name) + '.json' with open(output_filename, 'w') as f: f.write(json_string) - print("File written: " + output_filename) + print('File written: ' + output_filename) def plot_data_sources_graph(filename): @@ -53,7 +54,7 @@ def plot_data_sources_graph(filename): 'layout': go.Layout(title="# of data sources for " + name)}, filename=output_filename, auto_open=False ) - print("File written: " + output_filename) + print("File written: " + output_filename) def export_data_source_list_to_excel(filename): @@ -140,7 +141,7 @@ def export_data_source_list_to_excel(filename): worksheet.freeze_panes(3, 0) try: workbook.close() - print("File written: " + excel_filename) + print("File written: " + excel_filename) except Exception as e: print('[!] Error while writing Excel file: %s' % str(e)) @@ -149,11 +150,12 @@ def _load_data_sources(filename, filter_empty_scores=True): """ Loads the data sources (including all properties) from the given yaml file. :param filename: the filename of the yaml file containing the data sources administration - :return: dictionaty with data sources, name, platform and exceptions list. + :return: dictionary with data sources, name, platform and exceptions list. """ my_data_sources = {} + _yaml = init_yaml() with open(filename, 'r') as yaml_file: - yaml_content = yaml.load(yaml_file, Loader=yaml.FullLoader) + yaml_content = _yaml.load(yaml_file) for d in yaml_content['data_sources']: dq = d['data_quality'] if not filter_empty_scores: @@ -195,9 +197,8 @@ def _map_and_colorize_techniques(my_ds, exceptions): for t, v in my_techniques.items(): if t not in exceptions: for tactic in v['tactics']: - d = {} + d = dict() d['techniqueID'] = t - # d['score'] = 50 d['color'] = technique_colors[t] d['comment'] = '' d['enabled'] = True @@ -211,10 +212,244 @@ def _map_and_colorize_techniques(my_ds, exceptions): return output_techniques -def generate_technique_administration_file(filename): +def _indent_comment(comment, indent): + """ + Indent a multiline general, visibility, detection comment by x spaces + :param comment: The comment to indent + :param indent: The number of spaces to use in the indent + :return: indented comment or the original + """ + if '\n' in comment: + new_comment = comment.replace('\n', '\n' + ' ' * indent) + return new_comment + else: + return comment + + +def _get_technique_yaml_obj(techniques, tech_id): + """ + Get at technique YAML obj from the provided list of techniques YAML objects which as the provided technique ID + :param techniques: list of technique YAML objects + :param tech_id: ATT&CK ID + :return: technique YAML obj + """ + for tech in techniques: + if tech['technique_id'] == tech_id: + return tech + + +def update_technique_administration_file(file_data_sources, file_tech_admin): + """ + Update the visibility scores in the provided technique administration file + :param file_data_sources: file location of the data source admin. file + :param file_tech_admin: file location of the tech. admin. file + :return: + """ + # first we generate the new visibility scores contained within a temporary tech. admin YAML 'file' + new_visibility_scores = generate_technique_administration_file(file_data_sources, write_file=False) + + # we get the date to remove the single quotes at the end of the code + today = new_visibility_scores['techniques'][0]['visibility']['score_logbook'][0]['date'] + + # next we load the current visibility scores from the tech. admin file + cur_visibility_scores, _, platform_tech_admin = load_techniques(file_tech_admin, 'visibility') + + # if the platform does not match between the data source and tech. admin file we return + if new_visibility_scores['platform'] != platform_tech_admin: + print('[!] The MITRE ATT&CK platform key-value pair in the data source administration and technique ' + 'administration file do not match.\n Visibility update canceled.') + return + + # we did not return, so init + _yaml = init_yaml() + with open(file_tech_admin) as fd: + yaml_file_tech_admin = _yaml.load(fd) + + # check if we have tech IDs for which we now have visibility, but which were not yet part of the tech. admin file + cur_tech_ids = cur_visibility_scores.keys() + new_tech_ids = list(map(lambda k: k['technique_id'], new_visibility_scores['techniques'])) + + tech_ids_new = [] + for tid in new_tech_ids: + if tid not in cur_tech_ids: + tech_ids_new.append(tid) + + # Add the new tech. to the ruamel instance: 'yaml_file_tech_admin' + are_scores_updated = False + tech_new_print = [] + if len(tech_ids_new) > 0: + + # do we want fill in a comment for all updated visibility scores? + comment = '' + if ask_yes_no('\nDo you want to fill in the visibility comment for the updated scores?'): + comment = input(' >> Visibility comment for in the new \'score\' object: ') + print('') + + # add new techniques and set the comment + x = 0 + for new_tech in new_visibility_scores['techniques']: + + # set the comment for all new visibility scores + # we will also be needing this later in the code to update the scores of already present techniques + new_visibility_scores['techniques'][x]['visibility']['score_logbook'][0]['comment'] = comment + + if new_tech['technique_id'] in tech_ids_new: + are_scores_updated = True + yaml_file_tech_admin['techniques'].append(new_tech) + tech_new_print.append(' - ' + new_tech['technique_id']+'\n') + x += 1 + + print('The following new technique IDs are added to the technique administration file with a visibility ' + 'score derived from the nr. of data sources:') + print(''.join(tech_new_print)) + else: + print(' - No new techniques, for which we now have visibility, have been added to the techniques administration file.') + + # determine how visibility scores have been assigned in the current YAML file (auto, manually or mixed) + # also determine if we have any scores that can be updated + manually_scored = False + auto_scored = False + mix_scores = False + updated_vis_score_cnt = 0 + for cur_tech, cur_values in cur_visibility_scores.items(): + new_tech = _get_technique_yaml_obj(new_visibility_scores['techniques'], cur_tech) + new_score = new_tech['visibility']['score_logbook'][0]['score'] + + for cur_obj in cur_values['visibility']: + old_score = get_latest_score(cur_obj) + + if get_latest_auto_generated(cur_obj) and old_score != new_score: + auto_scored = True + updated_vis_score_cnt += 1 + elif old_score != new_score: + manually_scored = True + updated_vis_score_cnt += 1 + + if manually_scored and auto_scored: + mix_scores = True + manually_scored = False + auto_scored = False + break + + # stop if none of the present visibility scores are eligible for an update + if not mix_scores and not manually_scored and not auto_scored: + print(' - None of the already present techniques has a visibility score that is eligible for an update.') + else: + print('\nA total of ' + str(updated_vis_score_cnt) + ' visibility scores are eligible for an update.\n') + # ask how the score should be updated + answer = 0 + if manually_scored: + answer = ask_multiple_choice(V_UPDATE_Q_ALL_MANUAL, [V_UPDATE_ANSWER_1, V_UPDATE_ANSWER_2, V_UPDATE_ANSWER_CANCEL]) + elif auto_scored: + answer = ask_multiple_choice(V_UPDATE_Q_ALL_AUTO, [V_UPDATE_ANSWER_1, V_UPDATE_ANSWER_2, V_UPDATE_ANSWER_CANCEL]) + elif mix_scores: + answer = ask_multiple_choice(V_UPDATE_Q_MIXED, [V_UPDATE_ANSWER_3, V_UPDATE_ANSWER_4, V_UPDATE_ANSWER_1, V_UPDATE_ANSWER_2, V_UPDATE_ANSWER_CANCEL]) + if answer == V_UPDATE_ANSWER_CANCEL: + return + + # identify which visibility scores have changed and set the action to perform on the score + # tech_update {tech_id: ..., {obj_idx: { action: 1|2|3, score_obj: {...} } } } + tech_update = dict() + for new_tech in new_visibility_scores['techniques']: + tech_id = new_tech['technique_id'] + new_score_obj = new_tech['visibility']['score_logbook'][0] + new_score = new_score_obj['score'] + + if tech_id in cur_visibility_scores: + old_visibility_objects = cur_visibility_scores[tech_id]['visibility'] + obj_idx = 0 + for old_vis_obj in old_visibility_objects: + old_score = get_latest_score(old_vis_obj) + auto_gen = get_latest_auto_generated(old_vis_obj) + + # continue if score can be updated + if old_score != new_score: + if tech_id not in tech_update: + tech_update[tech_id] = dict() + + if (answer == V_UPDATE_ANSWER_1) or (answer == V_UPDATE_ANSWER_3 and auto_gen): + tech_update[tech_id][obj_idx] = {'action': V_UPDATE_ACTION_AUTO, 'score_obj': new_score_obj} + elif answer == V_UPDATE_ANSWER_2: + tech_update[tech_id][obj_idx] = {'action': V_UPDATE_ACTION_DIFF, 'score_obj': new_score_obj} + elif answer == V_UPDATE_ANSWER_4: + if auto_gen: + tech_update[tech_id][obj_idx] = {'action': V_UPDATE_ACTION_AUTO, 'score_obj': new_score_obj} + else: + tech_update[tech_id][obj_idx] = {'action': V_UPDATE_ACTION_DIFF, 'score_obj': new_score_obj} + obj_idx += 1 + + # perform the above set actions + score_updates_handled = 0 + for old_tech in yaml_file_tech_admin['techniques']: + tech_id = old_tech['technique_id'] + tech_name = old_tech['technique_name'] + obj_idx = 0 + if tech_id in tech_update: + if isinstance(old_tech['visibility'], list): + old_vis_obj = old_tech['visibility'] + else: + old_vis_obj = [old_tech['visibility']] + + while obj_idx <= len(tech_update[tech_id]): + # continue if an action has been set for this visibility object + if obj_idx in tech_update[tech_id]: + update_action = tech_update[tech_id][obj_idx]['action'] + new_score_obj = tech_update[tech_id][obj_idx]['score_obj'] + + if update_action == V_UPDATE_ACTION_AUTO: + are_scores_updated = True + old_vis_obj[obj_idx]['score_logbook'].insert(0, new_score_obj) + print(' - Updated a score in technique ID: ' + tech_id + + ' (applicable to: ' + ', '.join(old_vis_obj[obj_idx]['applicable_to']) + ')') + elif update_action == V_UPDATE_ACTION_DIFF: + print('-' * 80) + tmp_txt = '[updates remaining: ' + str(updated_vis_score_cnt - score_updates_handled) + ']' + print(' ' * (80-len(tmp_txt)) + tmp_txt) + print('') + print('Visibility object:') + print(' - ATT&CK ID/name ' + tech_id + ' / ' + tech_name) + print(' - Applicable to: ' + ', '.join(old_vis_obj[obj_idx]['applicable_to'])) + print(' - General comment: ' + _indent_comment(old_vis_obj[obj_idx]['comment'], 23)) + print('') + print('OLD score object:') + print(' - Date: ' + get_latest_date(old_vis_obj[obj_idx]).strftime('%Y-%m-%d')) + print(' - Score: ' + str(get_latest_score(old_vis_obj[obj_idx]))) + print(' - Visibility comment: ' + _indent_comment(get_latest_comment(old_vis_obj[obj_idx]), 23)) + print('NEW score object:') + print(' - Date: ' + new_score_obj['date']) + print(' - Score: ' + str(new_score_obj['score'])) + print(' - Visibility comment: ' + _indent_comment(new_score_obj['comment'], 23)) + print(' - Auto generated: true') + print('') + if ask_yes_no('Update the score?'): + are_scores_updated = True + old_vis_obj[obj_idx]['score_logbook'].insert(0, new_score_obj) + print(' - Updated a score in technique ID: ' + tech_id + + ' (applicable to: ' + ', '.join(old_vis_obj[obj_idx]['applicable_to']) + ')') + + score_updates_handled += 1 + + obj_idx += 1 + + # create backup of the current tech. admin YAML file + if are_scores_updated: + print('') + backup_file(file_tech_admin) + + yaml_file_tech_admin = fix_date(yaml_file_tech_admin, today, input_reamel=True, return_reamel=True) + + with open(file_tech_admin, 'w') as fd: + _yaml.dump(yaml_file_tech_admin, fd) + print('File written: ' + file_tech_admin) + else: + print('No visibility scores have been updated.') + + +def generate_technique_administration_file(filename, write_file=True): """ Generate a technique administration file based on the data source administration yaml file :param filename: the filename of the yaml file containing the data sources administration + :param write_file: by default the file is written to disk :return: """ my_data_sources, name, platform, exceptions = _load_data_sources(filename) @@ -222,18 +457,37 @@ def generate_technique_administration_file(filename): techniques = load_attack_data(DATA_TYPE_STIX_ALL_TECH_ENTERPRISE) # This is part of the techniques administration YAML file and is used as a template - dict_tech = {'technique_id': '', 'technique_name': '', 'detection': {'applicable_to': ['all'], - 'date_registered': None, - 'date_implemented': None, - 'score': -1, 'location': [''], 'comment': ''}, - 'visibility': {'applicable_to': ['all'], 'score': 0, 'comment': ''}} - - yaml_file = {} + dict_tech = {'technique_id': '', + 'technique_name': '', + 'detection': + {'applicable_to': ['all'], + 'location': [''], + 'comment': '', + 'score_logbook': + [ + {'date': None, + 'score': -1, + 'comment': ''} + ]}, + 'visibility': + {'applicable_to': ['all'], + 'comment': '', + 'score_logbook': + [ + {'date': None, + 'score': 0, + 'comment': '', + 'auto_generated': True} + ] + }} + + yaml_file = dict() yaml_file['version'] = FILE_TYPE_TECHNIQUE_ADMINISTRATION_VERSION yaml_file['file_type'] = FILE_TYPE_TECHNIQUE_ADMINISTRATION yaml_file['name'] = name yaml_file['platform'] = platform yaml_file['techniques'] = [] + today = dt.now().strftime('%Y-%m-%d') # Score visibility based on the number of available data sources and the exceptions for t in techniques: @@ -260,17 +514,36 @@ def generate_technique_administration_file(filename): tech = copy.deepcopy(dict_tech) tech['technique_id'] = tech_id tech['technique_name'] = t['name'] - tech['visibility']['score'] = score + # noinspection PyUnresolvedReferences + tech['visibility']['score_logbook'][0]['score'] = score + # noinspection PyUnresolvedReferences + tech['visibility']['score_logbook'][0]['date'] = today yaml_file['techniques'].append(tech) - yaml_string = '%YAML 1.2\n---\n' + yaml.dump(yaml_file, sort_keys=False).replace('null', '') - output_filename = 'output/techniques-administration-' + normalize_name_to_filename(name+'-'+platform) + '.yaml' - suffix = 1 - while os.path.exists(output_filename): - output_filename = 'output/techniques-administration-' + normalize_name_to_filename(name + '-' + platform) + \ - '_' + str(suffix) + '.yaml' - suffix += 1 - - with open(output_filename, 'w') as f: - f.write(yaml_string) - print("File written: " + output_filename) + if write_file: + # remove the single quotes around the date key-value pair + _yaml = init_yaml() + tmp_file = sys.path[0] + '/.tmp_tech_file' + + # create the file lines by writing it to disk + with open(tmp_file, 'w') as fd_tmp: + _yaml.dump(yaml_file, fd_tmp) + + # remove the single quotes from the date + yaml_file_lines = fix_date(tmp_file, today, input_reamel=False) + + os.remove(tmp_file) + + # create a output filename that prevents overwriting any existing files + output_filename = 'output/techniques-administration-' + normalize_name_to_filename(name+'-'+platform) + '.yaml' + suffix = 1 + while os.path.exists(output_filename): + output_filename = 'output/techniques-administration-' + normalize_name_to_filename(name + '-' + platform) + \ + '_' + str(suffix) + '.yaml' + suffix += 1 + + with open(output_filename, 'w') as f: + f.writelines(yaml_file_lines) + print("File written: " + output_filename) + else: + return yaml_file From 6be77c32608653501ad2d0285dc23762a6179540 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Wed, 31 Jul 2019 10:14:47 +0200 Subject: [PATCH 07/49] - Multiple functions made "private". - Added new menu options. --- dettect.py | 62 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/dettect.py b/dettect.py index aac62281..8f5c4243 100644 --- a/dettect.py +++ b/dettect.py @@ -4,7 +4,7 @@ import signal -def init_menu(): +def _init_menu(): """ Initialise the command line parameter menu. :return: @@ -27,7 +27,10 @@ def init_menu(): description='Create a heat map based on data sources, output data ' 'sources to Excel or generate a data source improvement ' 'graph.') - parser_data_sources.add_argument('-f', '--file', help='path to the data source administration YAML file', + parser_data_sources.add_argument('-ft', '--file-tech', help='path to the technique administration YAML file ' + '(used to score the level of visibility)', + required=False) + parser_data_sources.add_argument('-fd', '--file-ds', help='path to the data source administration YAML file', required=True) parser_data_sources.add_argument('-l', '--layer', help='generate a data source layer for the ATT&CK navigator', action='store_true') @@ -37,8 +40,14 @@ def init_menu(): action='store_true') parser_data_sources.add_argument('-y', '--yaml', help='generate a technique administration YAML file with ' 'visibility scores based on the number of available data ' - 'sources', - action='store_true') + 'sources', action='store_true') + parser_data_sources.add_argument('-u', '--update', help='update the visibility scores within a technique ' + 'administration YAML file based on changes within any of ' + 'the data sources. Past visibility scores are preserved in ' + 'the score_logbook, and manually assigned scores are not ' + 'updated without your approval. The updated visibility ' + 'scores are calculated in the same way as with the option: ' + '-y, --yaml.', action='store_true') # create the visibility parser parser_visibility = subparsers.add_parser('visibility', aliases=['v'], @@ -56,7 +65,7 @@ def init_menu(): parser_visibility.add_argument('-l', '--layer', help='generate a visibility layer for the ATT&CK navigator', action='store_true') parser_visibility.add_argument('-e', '--excel', help='generate an Excel sheet with all administrated techniques', - action='store_true') + action='store_true') parser_visibility.add_argument('-o', '--overlay', help='generate a visibility layer overlaid with detections for ' 'the ATT&CK navigator', action='store_true') parser_visibility.add_argument('--health', help='check the technique YAML file for errors', action='store_true') @@ -69,7 +78,7 @@ def init_menu(): 'improvement graph, output to Excel or check the health of ' 'the technique administration YAML file.') parser_detection.add_argument('-ft', '--file-tech', help='path to the technique administration YAML file (used to ' - 'score the level of visibility)', required=True) + 'score the level of detection)', required=True) parser_detection.add_argument('-fd', '--file-ds', help='path to the data source administration YAML file (used in ' 'the overlay with visibility to add metadata on the ' 'involved data sources)') @@ -79,7 +88,7 @@ def init_menu(): parser_detection.add_argument('-l', '--layer', help='generate detection layer for the ATT&CK navigator', action='store_true') parser_detection.add_argument('-e', '--excel', help='generate an Excel sheet with all administrated techniques', - action='store_true') + action='store_true') parser_detection.add_argument('-o', '--overlay', help='generate a detection layer overlaid with visibility for ' 'the ATT&CK navigator', action='store_true') parser_detection.add_argument('-g', '--graph', help='generate a graph with detections added through time', @@ -135,10 +144,10 @@ def init_menu(): return menu_parser -def menu(menu_parser): +def _menu(menu_parser): """ Parser for the command line parameter menu and calls the appropriate functions. - :param menu_parser: the argparse menu as created with 'init_menu()' + :param menu_parser: the argparse menu as created with '_init_menu()' :return: """ args = menu_parser.parse_args() @@ -147,15 +156,17 @@ def menu(menu_parser): interactive_menu() elif args.subparser in ['datasource', 'ds']: - if check_file(args.file, FILE_TYPE_DATA_SOURCE_ADMINISTRATION): + if check_file(args.file_ds, FILE_TYPE_DATA_SOURCE_ADMINISTRATION): + if args.update and check_file(args.file_tech, FILE_TYPE_TECHNIQUE_ADMINISTRATION): + update_technique_administration_file(args.file_ds, args.file_tech) if args.layer: - generate_data_sources_layer(args.file) + generate_data_sources_layer(args.file_ds) if args.excel: - export_data_source_list_to_excel(args.file) + export_data_source_list_to_excel(args.file_ds) if args.graph: - plot_data_sources_graph(args.file) + plot_data_sources_graph(args.file_ds) if args.yaml: - generate_technique_administration_file(args.file) + generate_technique_administration_file(args.file_ds) elif args.subparser in ['visibility', 'v']: if args.layer or args.overlay: @@ -163,15 +174,14 @@ def menu(menu_parser): print('[!] Generating a visibility layer or doing an overlay requires adding the data source' 'administration YAML file (\'--file-ds\')') quit() - - if check_file(args.file_tech, FILE_TYPE_TECHNIQUE_ADMINISTRATION, args.health) and \ - check_file(args.file_ds, FILE_TYPE_DATA_SOURCE_ADMINISTRATION, args.health): - if args.layer: - generate_visibility_layer(args.file_tech, args.file_ds, False, args.applicable) - if args.overlay: - generate_visibility_layer(args.file_tech, args.file_ds, True, args.applicable) + if not check_file(args.file_ds, FILE_TYPE_DATA_SOURCE_ADMINISTRATION, args.health): + quit() if check_file(args.file_tech, FILE_TYPE_TECHNIQUE_ADMINISTRATION, args.health): + if args.layer: + generate_visibility_layer(args.file_tech, args.file_ds, False, args.applicable) + if args.overlay: + generate_visibility_layer(args.file_tech, args.file_ds, True, args.applicable) if args.excel and args.applicable == 'all': export_techniques_list_to_excel(args.file_tech) if args.excel and args.applicable != 'all': @@ -210,7 +220,7 @@ def menu(menu_parser): menu_parser.print_help() -def prepare_folders(): +def _prepare_folders(): """ Create the folders 'cache' and 'output' if they do not exist. :return: @@ -221,7 +231,7 @@ def prepare_folders(): os.mkdir('output') -def signal_handler(signum, frame): +def _signal_handler(signum, frame): """ Function to handles exiting via Ctrl+C. :param signum: @@ -232,6 +242,6 @@ def signal_handler(signum, frame): if __name__ == '__main__': - signal.signal(signal.SIGINT, signal_handler) - prepare_folders() - menu(init_menu()) + signal.signal(signal.SIGINT, _signal_handler) + _prepare_folders() + _menu(_init_menu()) From d0f2a4946b8579bd7635f0ac00252d6dc051408a Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Wed, 31 Jul 2019 10:18:57 +0200 Subject: [PATCH 08/49] - Made compatible with version 1.2 of the technique admin YAML file. - Added new functionality for the auto-update of visibility scores. - Added multiple new generic functions. - Multiple small improvements to the technique admin YAML file health check. - Replaced PyYAML with ruamel.yaml. - Multiple functions made "private". - Made compatible with v0.2.7 of attackcti. --- generic.py | 456 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 352 insertions(+), 104 deletions(-) diff --git a/generic.py b/generic.py index 07bd7385..4f3325eb 100644 --- a/generic.py +++ b/generic.py @@ -1,10 +1,12 @@ import os +import shutil import pickle +import sys +from ruamel.yaml import YAML +from difflib import SequenceMatcher from datetime import datetime as dt -import yaml from upgrade import upgrade_yaml_file from constants import * -from difflib import SequenceMatcher # Due to performance reasons the import of attackcti is within the function that makes use of this library. @@ -21,23 +23,7 @@ def try_get_key(dictionary, key): return None -def try_except(self, stix_objects, object_type, nested_value=None): - if object_type in stix_objects: - specific_stix_object = stix_objects[object_type] - if isinstance(specific_stix_object, list): - if nested_value is None: - lists = self.handle_list(stix_objects, object_type) - return lists - else: - nested_result = self.handle_nested(stix_objects, object_type, nested_value) - return nested_result - else: - return stix_objects[object_type] - else: - return None - - -def save_attack_data(data, path): +def _save_attack_data(data, path): """ Save ATT&CK data to disk for the purpose of caching. Data can be STIX objects our a custom schema. :param data: the MITRE ATT&CK data to save @@ -70,9 +56,9 @@ def load_attack_data(data_type): attack_data = None if data_type == DATA_TYPE_STIX_ALL_RELATIONSHIPS: - attack_data = mitre.get_all_relationships() + attack_data = mitre.get_relationships() if data_type == DATA_TYPE_STIX_ALL_TECH_ENTERPRISE: - attack_data = mitre.get_all_enterprise_techniques() + attack_data = mitre.get_enterprise_techniques() if data_type == DATA_TYPE_CUSTOM_TECH_BY_GROUP: # First we need to know which technique references (STIX Object type 'attack-pattern') we have for all # groups. This results in a dict: {group_id: Gxxxx, technique_ref/attack-pattern_ref: ...} @@ -112,11 +98,11 @@ def load_attack_data(data_type): attack_data = all_group_use elif data_type == DATA_TYPE_STIX_ALL_TECH: - attack_data = mitre.get_all_techniques() + attack_data = mitre.get_techniques() elif data_type == DATA_TYPE_STIX_ALL_GROUPS: - attack_data = mitre.get_all_groups() + attack_data = mitre.get_groups() elif data_type == DATA_TYPE_STIX_ALL_SOFTWARE: - attack_data = mitre.get_all_software() + attack_data = mitre.get_software() elif data_type == DATA_TYPE_CUSTOM_TECH_BY_SOFTWARE: # First we need to know which technique references (STIX Object type 'attack-pattern') we have for all software # This results in a dict: {software_id: Sxxxx, technique_ref/attack-pattern_ref: ...} @@ -180,11 +166,17 @@ def load_attack_data(data_type): }) attack_data = all_group_use - save_attack_data(attack_data, "cache/" + data_type) + _save_attack_data(attack_data, "cache/" + data_type) return attack_data +def init_yaml(): + _yaml = YAML() + _yaml.Representer.ignore_aliases = lambda *args: True # disable anchors/aliases + return _yaml + + def _get_base_template(name, description, stage, platform, sorting): """ Prepares a base template for the json layer file that can be loaded into the MITRE ATT&CK Navigator. @@ -197,7 +189,7 @@ def _get_base_template(name, description, stage, platform, sorting): :param sorting: sorting :return: layer template dictionary """ - layer = {} + layer = dict() layer['name'] = name layer['version'] = '2.1' layer['domain'] = 'mitre-enterprise' @@ -348,6 +340,22 @@ def get_layer_template_layered(name, description, stage, platform): return layer +def backup_file(filename): + """ + Create a backup of the provided file + :param filename: existing YAML filename + :return: + """ + suffix = 1 + backup_filename = filename.replace('.yaml', '_backup_' + str(suffix) + '.yaml') + while os.path.exists(backup_filename): + backup_filename = backup_filename.replace('_backup_' + str(suffix) + '.yaml', '_backup_' + str(suffix+1) + '.yaml') + suffix += 1 + + shutil.copy2(filename, backup_filename) + print('Written backup file: ' + backup_filename + '\n') + + def get_attack_id(stix_obj): """ Get the Technique, Group or Software ID from the STIX object @@ -386,6 +394,166 @@ def get_technique(techniques, technique_id): return None +def ask_yes_no(question): + """ + Ask the user to a question that needs to be answered with yes or no. + :param question: The question to be asked + :return: boolean value indicating a yes (True) or no (False0 + """ + yes_no = '' + while not re.match('^(y|yes|n|no)$', yes_no, re.IGNORECASE): + yes_no = input(question + '\n >> y(yes) / n(no): ') + print('') + + if re.match('^(y|yes)$', yes_no, re.IGNORECASE): + return True + else: + return False + + +def ask_multiple_choice(question, list_answers): + """ + Ask a multiple choice question. + :param question: the question to ask + :param list_answers: a list of answer + :return: the answer + """ + answer = '' + answers = '' + x = 1 + for a in list_answers: + a = a.replace('\n', '\n ') + answers += ' ' + str(x) + ') ' + a + '\n' + x += 1 + + # noinspection Annotator + while not re.match('(^[1-' + str(len(list_answers)) + ']{1}$)', answer): + print(question) + print(answers) + answer = input(' >> ') + print('') + + return list_answers[int(answer)-1] + + +def fix_date(yaml_file, date, input_reamel=True, return_reamel=False): + """ + Remove the single quotes around the date key-value pair in the provided yaml_file + :param yaml_file: ruamel.yaml instance or location of YAML file + :param date: string date value (e.g. 2019-01-01) + :param input_reamel: input type can be a reamel instance or list + :param return_reamel: return list of YAML file lines or reamel instance + :return: YAML file lines in a list + """ + _yaml = init_yaml() + if input_reamel: + file = sys.path[0] + '/.tmp_tech_file' + + with open(file, 'w') as fd: + _yaml.dump(yaml_file, fd) + else: + file = yaml_file + + with open(file, 'r') as fd: + new_lines = fd.readlines() + x = 0 + for line in new_lines: + if REGEX_YAML_DATE.match(line): + new_lines[x] = line.replace('\'' + date + '\'', date) + x += 1 + + # remove the temporary file + if input_reamel: + os.remove(file) + + if return_reamel: + return _yaml.load(''.join(new_lines)) + else: + return new_lines + + +def _get_latest_score_obj(yaml_object): + """ + Get the the score object in the score_logbook by date + :param yaml_object: a detection or visibility YAML object + :return: the latest score object + """ + if not isinstance(yaml_object['score_logbook'], list): + yaml_object['score_logbook'] = [yaml_object['score_logbook']] + + if len(yaml_object['score_logbook']) > 0 and 'date' in yaml_object['score_logbook'][0]: + # for some weird reason 'sorted()' provides inconsistent results + newest_score_obj = None + newest_date = None + for score_obj in yaml_object['score_logbook']: + if not newest_score_obj or score_obj['date'] > newest_date: + newest_date = score_obj['date'] + newest_score_obj = score_obj + + return newest_score_obj + else: + return None + + +def get_latest_comment(yaml_object, empty=' '): + """ + Return the latest comment present in the score_logbook + :param yaml_object: a detection or visibility YAML object + :param empty: value for an empty comment + :return: comment + """ + score_obj = _get_latest_score_obj(yaml_object) + if score_obj: + if score_obj['comment'] == '' or not score_obj['comment']: + return empty + else: + return score_obj['comment'] + else: + return empty + + +def get_latest_date(yaml_object): + """ + Return the latest date present in the score_logbook + :param yaml_object: a detection or visibility YAML object + :return: date as a datetime object or None + """ + score_obj = _get_latest_score_obj(yaml_object) + if score_obj: + return score_obj['date'] + else: + return None + + +def get_latest_auto_generated(yaml_object): + """ + Return the latest auto_generated value present in the score_logbook + :param yaml_object: a detection or visibility YAML object + :return: True or False + """ + score_obj = _get_latest_score_obj(yaml_object) + if score_obj: + if 'auto_generated' in score_obj: + return score_obj['auto_generated'] + else: + return False + else: + return False + + +def get_latest_score(yaml_object): + """ + Return the latest score present in the score_logbook + :param yaml_object: a detection or visibility YAML object + :return: score as an integer or None + """ + score_obj = _get_latest_score_obj(yaml_object) + if score_obj: + return score_obj['score'] + else: + return None + + def normalize_name_to_filename(name): """ Normalize the input filename to a lowercase filename and replace spaces with dashes. @@ -439,38 +607,39 @@ def get_all_mitre_data_sources(): return sorted(data_sources) -def calculate_score(l, zero_value=0): +def calculate_score(list_detections, zero_value=0): """ - Calculates the average score in the given list which contains dictionaries with 'score' field. - :param l: list + Calculates the average score in the given list which may contain multiple detection dictionaries + :param list_detections: list :param zero_value: the value when no scores are there, default 0 :return: average score """ - s = 0 + avg_score = 0 number = 0 - for v in l: - if v['score'] >= 0: - s += v['score'] + for v in list_detections: + score = get_latest_score(v) + if score >= 0: + avg_score += score number += 1 - s = int(round(s / number, 0) if number > 0 else zero_value) - return s + avg_score = int(round(avg_score / number, 0) if number > 0 else zero_value) + return avg_score -def _add_entry_to_list_in_dictionary(dict, technique_id, key, entry): +def add_entry_to_list_in_dictionary(dictionary, technique_id, key, entry): """ Ensures a list will be created if it doesn't exist in the given dict[technique_id][key] and adds the entry to the list. If the dict[technique_id] doesn't exist yet, it will be created. - :param dict: the dictionary + :param dictionary: the dictionary :param technique_id: the id of the technique in the main dict :param key: the key where the list in the dictionary resides :param entry: the entry to add to the list :return: """ - if technique_id not in dict.keys(): - dict[technique_id] = {} - if not key in dict[technique_id].keys(): - dict[technique_id][key] = [] - dict[technique_id][key].append(entry) + if technique_id not in dictionary.keys(): + dictionary[technique_id] = {} + if key not in dictionary[technique_id].keys(): + dictionary[technique_id][key] = [] + dictionary[technique_id][key].append(entry) def load_techniques(filename, detection_or_visibility='all', filter_applicable_to='all'): @@ -484,26 +653,27 @@ def load_techniques(filename, detection_or_visibility='all', filter_applicable_t """ my_techniques = {} + _yaml = init_yaml() with open(filename, 'r') as yaml_file: - yaml_content = yaml.load(yaml_file, Loader=yaml.FullLoader) + yaml_content = _yaml.load(yaml_file) for d in yaml_content['techniques']: # Add detection items: - if type(d['detection']) == dict: # There is just one detection entry + if isinstance(d['detection'], dict): # There is just one detection entry if detection_or_visibility == 'all' or filter_applicable_to == 'all' or filter_applicable_to in d[detection_or_visibility]['applicable_to'] or 'all' in d[detection_or_visibility]['applicable_to']: - _add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'detection', d['detection']) - elif type(d['detection']) == list: # There are multiple detection entries + add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'detection', d['detection']) + elif isinstance(d['detection'], list): # There are multiple detection entries for de in d['detection']: if detection_or_visibility == 'all' or filter_applicable_to == 'all' or filter_applicable_to in de['applicable_to'] or 'all' in de['applicable_to']: - _add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'detection', de) + add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'detection', de) # Add visibility items - if type(d['visibility']) == dict: # There is just one visibility entry + if isinstance(d['visibility'], dict): # There is just one visibility entry if detection_or_visibility == 'all' or filter_applicable_to == 'all' or filter_applicable_to in d[detection_or_visibility]['applicable_to'] or 'all' in d[detection_or_visibility]['applicable_to']: - _add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'visibility', d['visibility']) - elif type(d['visibility']) == list: # There are multiple visibility entries + add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'visibility', d['visibility']) + elif isinstance(d['visibility'], list): # There are multiple visibility entries for de in d['visibility']: if detection_or_visibility == 'all' or filter_applicable_to == 'all' or filter_applicable_to in de['applicable_to'] or 'all' in de['applicable_to']: - _add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'visibility', de) + add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'visibility', de) name = yaml_content['name'] platform = yaml_content['platform'] @@ -516,6 +686,111 @@ def _print_error_msg(msg, print_error): return True +def _check_health_score_object(yaml_object, object_type, tech_id, health_is_called): + """ + Check the health of a score_logbook inside a visibility or detection YAML object + :param yaml_object: YAML file lines + :param object_type: 'detection' or 'visibility' + :param tech_id: ATT&CK technique ID + :param health_is_called: boolean that specifies if detailed errors in the file will be printed and then quit() + :return: True if the YAML file is unhealthy, otherwise False + """ + has_error = False + min_score = None + max_score = None + + if object_type == 'detection': + min_score = -1 + max_score = 5 + elif object_type == 'visibility': + min_score = 0 + max_score = 4 + + if not isinstance(yaml_object['score_logbook'], list): + yaml_object['score_logbook'] = [yaml_object['score_logbook']] + + try: + for score_obj in yaml_object['score_logbook']: + for key in ['date', 'score', 'comment']: + if key not in score_obj: + has_error = _print_error_msg('[!] Technique ID: ' + tech_id + ' is MISSING a key-value pair in a ' + object_type + ' score object in the \'score_logbook\': ' + key, health_is_called) + + if score_obj['score'] is None: + has_error = _print_error_msg('[!] Technique ID: ' + tech_id + ' has an EMPTY key-value pair in a ' + object_type + ' score object in the \'score_logbook\': score', health_is_called) + + elif not isinstance(score_obj['score'], int): + has_error = _print_error_msg('[!] Technique ID: ' + tech_id + ' has an INVALID score format in a ' + object_type + ' score object in the \'score_logbook\': score should be an integer', health_is_called) + + if 'auto_generated' in score_obj: + if not isinstance(score_obj['auto_generated'], bool): + has_error = _print_error_msg( + '[!] Technique ID: ' + tech_id + ' has an INVALID auto_generated value in a ' + object_type + ' score object in the \'score_logbook\': auto_generated (if present) should be set to \'true\' or \'false\'', health_is_called) + + if isinstance(score_obj['score'], int): + if score_obj['date'] is None and score_obj['score'] > -1: + has_error = _print_error_msg('[!] Technique ID: ' + tech_id + ' has an EMPTY key-value pair in a ' + object_type + ' score object in the \'score_logbook\': date', health_is_called) + + # noinspection PyChainedComparisons + if not (score_obj['score'] >= min_score and score_obj['score'] <= max_score): + has_error = _print_error_msg( + '[!] Technique ID: ' + tech_id + ' has an INVALID ' + object_type + ' score in a score object in the \'score_logbook\': ' + str(score_obj['score']) + ' (should be between ' + str(min_score) + ' and ' + str(max_score) + ')', health_is_called) + + if score_obj['score'] > min_score: + try: + # noinspection PyStatementEffect + score_obj['date'].year + # noinspection PyStatementEffect + score_obj['date'].month + # noinspection PyStatementEffect + score_obj['date'].day + except AttributeError: + has_error = _print_error_msg('[!] Technique ID: ' + tech_id + ' has an INVALID data format in a ' + object_type + ' score object in the \'score_logbook\': date (should be YYYY-MM-DD)', health_is_called) + except KeyError: + pass + + return has_error + + +def _check_health_yaml_object(yaml_object, object_type, tech_id, health_is_called): + """ + Check the health of a visibility or detection YAML object + :param yaml_object: YAML file lines + :param object_type: 'detection' or 'visibility' + :param tech_id: ATT&CK technique ID + :param health_is_called: boolean that specifies if detailed errors in the file will be printed and then quit() + :return: True if the YAML file is unhealthy, otherwise False + """ + has_error = False + + keys = ['applicable_to'] + + if object_type == 'detection': + keys.append('location') + + try: + for key in keys: + if not isinstance(yaml_object[key], list): + has_error = _print_error_msg('[!] Technique ID: ' + tech_id + ' has for the key-value pair \'' + key + '\' in ' + object_type + ' a string value assigned (should be a list)', health_is_called) + else: + try: + if yaml_object[key][0] is None: + has_error = _print_error_msg('[!] Technique ID: ' + tech_id + ' has an EMPTY key-value pair in ' + object_type + ': ' + key, health_is_called) + except TypeError: + has_error = _print_error_msg( + '[!] Technique ID: ' + tech_id + ' has an EMPTY key-value pair in ' + object_type + ': ' + key, health_is_called) + except KeyError: + pass + + return has_error + + +def _update_health_sate(current, update): + if current or update: + return True + else: + return update + + def check_yaml_file_health(filename, file_type, health_is_called): """ Check on error in the provided YAML file. @@ -528,8 +803,9 @@ def check_yaml_file_health(filename, file_type, health_is_called): has_error = False if file_type == FILE_TYPE_TECHNIQUE_ADMINISTRATION: # check for duplicate tech IDs + _yaml = init_yaml() with open(filename, 'r') as yaml_file: - yaml_content = yaml.load(yaml_file, Loader=yaml.FullLoader) + yaml_content = _yaml.load(yaml_file) tech_ids = list(map(lambda x: x['technique_id'], yaml_content['techniques'])) tech_dup = [] @@ -539,17 +815,20 @@ def check_yaml_file_health(filename, file_type, health_is_called): else: has_error = _print_error_msg('[!] Duplicate technique ID: ' + tech, health_is_called) + # check if the technique has a valid format + if not REGEX_YAML_TECHNIQUE_ID_FORMAT.match(tech): + has_error = _print_error_msg('[!] Invalid technique ID: ' + tech, health_is_called) + # checks on: - # - empty key-value pairs: 'date_implemented', 'date_registered', 'location', 'applicable_to', 'score' - # - invalid date format for: 'date_implemented', 'date_registered' + # - empty key-value pairs: 'applicable_to', 'comment', 'location', 'score_logbook' , 'date', 'score' + # - invalid date format for: 'date' # - detection or visibility score out-of-range - # - missing key-value pairs: 'applicable_to', 'date_registered', 'date_implemented', 'score', 'location', 'comment' + # - missing key-value pairs: 'applicable_to', 'comment', 'location', 'score_logbook', 'date', 'score' # - check on 'applicable_to' values which are very similar all_applicable_to = set() techniques = load_techniques(filename) for tech, v in techniques[0].items(): - for key in ['detection', 'visibility']: if key not in v: has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is MISSING ' + key, health_is_called) @@ -558,57 +837,24 @@ def check_yaml_file_health(filename, file_type, health_is_called): all_applicable_to.update([a for v in v[key] for a in v['applicable_to']]) for detection in v['detection']: - for key in ['applicable_to', 'date_registered', 'date_implemented', 'score', 'location', 'comment']: + for key in ['applicable_to', 'location', 'comment', 'score_logbook']: if key not in detection: - has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is MISSING the key-value pair in detection: ' + key, health_is_called) + has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is MISSING a key-value pair in detection: ' + key, health_is_called) - try: - if detection['score'] is None: - has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is has an EMPTY key-value pair in detection: score', health_is_called) - - elif not (detection['score'] >= -1 and detection['score'] <= 5): - has_error = _print_error_msg('[!] Technique ID: ' + tech + ' has an INVALID detection score: ' - + str(detection['score']) + ' (should be between -1 and 5)', health_is_called) - - elif detection['score'] > -1: - for key in ['date_implemented', 'date_registered']: - if not detection[key]: - has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is has an EMPTY key-value pair in detection: ' + key, health_is_called) - break - try: - detection[key].year - detection[key].month - detection[key].day - except AttributeError: - has_error = _print_error_msg('[!] Technique ID: ' + tech + - ' has an INVALID data format for the key-value pair in detection: ' + - key + ' (should be YYYY-MM-DD)', health_is_called) - for key in ['location', 'applicable_to']: - if not isinstance(detection[key], list): - has_error = _print_error_msg('[!] Technique ID: ' + tech + ' has for the key-value pair \'' - + key + '\' a string value assigned (should be a list)', health_is_called) - else: - try: - if detection[key][0] is None: - has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is has an EMPTY key-value pair in detection: ' + key, health_is_called) - except TypeError: - has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is has an EMPTY key-value pair in detection: ' + key, health_is_called) - except KeyError: - pass + health = _check_health_yaml_object(detection, 'detection', tech, health_is_called) + has_error = _update_health_sate(has_error, health) + health = _check_health_score_object(detection, 'detection', tech, health_is_called) + has_error = _update_health_sate(has_error, health) for visibility in v['visibility']: - for key in ['applicable_to', 'score', 'comment']: + for key in ['applicable_to', 'comment', 'score_logbook']: if key not in visibility: - has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is MISSING the key-value pair in visibility: ' + key, health_is_called) + has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is MISSING a key-value pair in visibility: ' + key, health_is_called) - try: - if visibility['score'] is None: - has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is has an EMPTY key-value pair in visibility: score', health_is_called) - elif not (visibility['score'] >= 0 and visibility['score'] <= 4): - has_error = _print_error_msg('[!] Technique ID: ' + tech + ' has an INVALID visibility score: ' - + str(detection['score']) + ' (should be between 0 and 4)', health_is_called) - except KeyError: - pass + health = _check_health_yaml_object(visibility, 'visibility', tech, health_is_called) + has_error = _update_health_sate(has_error, health) + health = _check_health_score_object(visibility, 'visibility', tech, health_is_called) + has_error = _update_health_sate(has_error, health) # get values within the key-value pair 'applicable_to' which are a very close match similar = set() @@ -632,7 +878,7 @@ def check_yaml_file_health(filename, file_type, health_is_called): print('') # print a newline -def check_file_type(filename, file_type=None): +def _check_file_type(filename, file_type=None): """ Check if the provided YAML file has the key 'file_type' and possible if that key matches a specific value. :param filename: path to a YAML file @@ -642,9 +888,11 @@ def check_file_type(filename, file_type=None): if not os.path.exists(filename): print('[!] File: \'' + filename + '\' does not exist') return None + + _yaml = init_yaml() with open(filename, 'r') as yaml_file: try: - yaml_content = yaml.load(yaml_file, Loader=yaml.FullLoader) + yaml_content = _yaml.load(yaml_file) except Exception as e: print('[!] File: \'' + filename + '\' is not a valid YAML file.') print(' ' + str(e)) # print more detailed error information to help the user in fixing the error. @@ -679,7 +927,7 @@ def check_file(filename, file_type=None, health_is_called=False): :return: the file_type if present, else None is returned """ - yaml_content = check_file_type(filename, file_type) + yaml_content = _check_file_type(filename, file_type) # if the file is a valid YAML, continue. Else, return None if yaml_content: From f4121bf4d0962e2da78cbb5a7afe4905c928f2b1 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Wed, 31 Jul 2019 10:19:51 +0200 Subject: [PATCH 09/49] - Replaced PyYAML with ruamel.yaml. - Multiple functions made "private". --- group_mapping.py | 71 +++++++++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/group_mapping.py b/group_mapping.py index 538592e5..cb0bc754 100644 --- a/group_mapping.py +++ b/group_mapping.py @@ -4,7 +4,7 @@ CG_GROUPS = {} -def is_in_group(json_groups, argument_groups): +def _is_in_group(json_groups, argument_groups): """ Check if the two dicts (json_groups and argument_groups) have any groups in common based on their names/aliases. :param json_groups: group aliases from ATT&CK @@ -20,7 +20,7 @@ def is_in_group(json_groups, argument_groups): return False -def is_group_found(groups_found, argument_groups): +def _is_group_found(groups_found, argument_groups): """ Check if a group that has been provided using '-g/--groups'/'-o/--overlay' is present within MITRE ATT&CK. :param groups_found: groups that are found in the ATT&CK data @@ -51,7 +51,7 @@ def is_group_found(groups_found, argument_groups): return True -def get_software_techniques(groups, stage, platform): +def _get_software_techniques(groups, stage, platform): """ Get all techniques (in a dict) from the provided list of groups in relation to the software these groups use, and hence techniques they support. @@ -69,18 +69,20 @@ def get_software_techniques(groups, stage, platform): software_dict = {} for tech in tech_by_software: if tech['software_id'] not in software_dict: + # noinspection PySetFunctionToLiteral software_dict[tech['software_id']] = set([tech['technique_id']]) else: software_dict[tech['software_id']].add(tech['technique_id']) # groups is a YAML file if os.path.isfile(str(groups)): + _yaml = init_yaml() with open(groups, 'r') as yaml_file: - config = yaml.load(yaml_file, Loader=yaml.FullLoader) + config = _yaml.load(yaml_file) for group in config['groups']: if group['enabled']: - group_id = generate_group_id(group['group_name'], group['campaign']) + group_id = _generate_group_id(group['group_name'], group['campaign']) groups_dict[group_id] = dict() groups_dict[group_id]['group_name'] = group['group_name'] @@ -104,7 +106,7 @@ def get_software_techniques(groups, stage, platform): # and the group is a group we are interested in if s['x_mitre_platforms']: # their is some software that do not have a platform, skip those if s['matrix'] == 'mitre-'+stage and (platform in s['x_mitre_platforms'] or platform == 'all') and \ - (groups[0] == 'all' or s['group_id'].lower() in groups or is_in_group(s['aliases'], groups)): + (groups[0] == 'all' or s['group_id'].lower() in groups or _is_in_group(s['aliases'], groups)): if s['group_id'] not in groups_dict: groups_dict[s['group_id']] = {'group_name': s['name']} groups_dict[s['group_id']]['techniques'] = set() @@ -113,7 +115,7 @@ def get_software_techniques(groups, stage, platform): return groups_dict -def generate_group_id(group_name, campaign): +def _generate_group_id(group_name, campaign): # CG_GROUPS = { group_name+campaign: id } } """ Generate a custom group id. @@ -144,7 +146,7 @@ def generate_group_id(group_name, campaign): return CG_GROUPS[group_name + campaign] -def get_group_techniques(groups, stage, platform, file_type): +def _get_group_techniques(groups, stage, platform, file_type): """ Get all techniques (in a dict) from the provided list of groups :param groups: group ID, group name/alias or a YAML file with group(s) data @@ -159,20 +161,21 @@ def get_group_techniques(groups, stage, platform, file_type): # groups is a YAML file if file_type == FILE_TYPE_GROUP_ADMINISTRATION: + _yaml = init_yaml() with open(groups, 'r') as yaml_file: - config = yaml.load(yaml_file, Loader=yaml.FullLoader) + config = _yaml.load(yaml_file) for group in config['groups']: if group['enabled']: campaign = group['campaign'] if group['campaign'] else '' - group_id = generate_group_id(group['group_name'], campaign) + group_id = _generate_group_id(group['group_name'], campaign) groups_dict[group_id] = dict() groups_dict[group_id]['group_name'] = group['group_name'] - if type(group['technique_id']) == list: + if isinstance(group['technique_id'], list): groups_dict[group_id]['techniques'] = set(group['technique_id']) groups_dict[group_id]['weight'] = dict((i, 1) for i in group['technique_id']) - elif type(group['technique_id']) == dict: + elif isinstance(group['technique_id'], dict): groups_dict[group_id]['techniques'] = set(group['technique_id'].keys()) groups_dict[group_id]['weight'] = group['technique_id'] groups_dict[group_id]['campaign'] = group['campaign'] @@ -189,7 +192,7 @@ def get_group_techniques(groups, stage, platform, file_type): # group matches the: matrix/stage, platform and the group(s) we are interested in if gr['matrix'] == 'mitre-'+stage and (platform in platforms or platform == 'all') and \ - (groups[0] == 'all' or gr['group_id'].lower() in groups or is_in_group(gr['aliases'], groups)): + (groups[0] == 'all' or gr['group_id'].lower() in groups or _is_in_group(gr['aliases'], groups)): if gr['group_id'] not in groups_dict: groups_found.add(gr['group_id']) groups_dict[gr['group_id']] = {'group_name': gr['name']} @@ -199,17 +202,17 @@ def get_group_techniques(groups, stage, platform, file_type): groups_dict[gr['group_id']]['techniques'].add(gr['technique_id']) groups_dict[gr['group_id']]['weight'][gr['technique_id']] = 1 - # do not call 'is_group_found' when groups is a YAML file + # do not call '_is_group_found' when groups is a YAML file # (this could contain groups that do not exists within ATT&CK) if not os.path.isfile(str(groups)): - found = is_group_found(groups_found, groups) + found = _is_group_found(groups_found, groups) if not found: return -1 return groups_dict -def get_detection_techniques(filename, filter_applicable_to): +def _get_detection_techniques(filename, filter_applicable_to): """ Get all techniques (in a dict) from the detection administration :param filename: path to the YAML technique administration file @@ -235,9 +238,9 @@ def get_detection_techniques(filename, filter_applicable_to): return groups_dict, detection_techniques -def get_visibility_techniques(filename, filter_applicable_to): +def _get_visibility_techniques(filename, filter_applicable_to): """ - Get all techniques (in a dict) from the detections administration + Get all techniques (in a dict) from the technique administration :param filename: path to the YAML technique administration file :param filter_applicable_to: filter techniques based on applicable_to field in techniques administration YAML file :return: dictionary @@ -261,7 +264,7 @@ def get_visibility_techniques(filename, filter_applicable_to): return groups_dict, visibility_techniques -def get_technique_count(groups, groups_overlay, groups_software, overlay_type, all_techniques): +def _get_technique_count(groups, groups_overlay, groups_software, overlay_type, all_techniques): """ Create a dict with all involved techniques and their relevant count/score :param groups: a dict with data on groups @@ -288,7 +291,7 @@ def get_technique_count(groups, groups_overlay, groups_software, overlay_type, a techniques_dict[tech]['count'] += v['weight'][tech] techniques_dict[tech]['groups'].add(group) - max_count = max(techniques_dict.values(), key=lambda v: v['count'])['count'] + max_count = max(techniques_dict.values(), key=lambda k: k['count'])['count'] # create dict {tech_id: score+max_tech_count} to be used for when doing an overlay of the type visibility or detection if overlay_type != OVERLAY_TYPE_GROUP: @@ -339,8 +342,8 @@ def get_technique_count(groups, groups_overlay, groups_software, overlay_type, a return techniques_dict, max_count -def get_technique_layer(techniques_count, groups, overlay, groups_software, overlay_file_type, overlay_type, - all_techniques): +def _get_technique_layer(techniques_count, groups, overlay, groups_software, overlay_file_type, overlay_type, + all_techniques): """ Create the technique layer that will be part of the ATT&CK navigator json file :param techniques_count: involved techniques with count (to be used within the scores) @@ -430,7 +433,7 @@ def get_technique_layer(techniques_count, groups, overlay, groups_software, over return techniques_layer -def get_group_list(groups, file_type): +def _get_group_list(groups, file_type): """ Make a list of group names for the involved groups. :param groups: a dict with data on groups @@ -495,15 +498,15 @@ def generate_group_heat_map(groups, overlay, overlay_type, stage, platform, soft all_techniques = None if overlay_file_type == FILE_TYPE_TECHNIQUE_ADMINISTRATION: if overlay_type == OVERLAY_TYPE_VISIBILITY: - overlay_dict, all_techniques = get_visibility_techniques(overlay, filter_applicable_to) + overlay_dict, all_techniques = _get_visibility_techniques(overlay, filter_applicable_to) elif overlay_type == OVERLAY_TYPE_DETECTION: - overlay_dict, all_techniques = get_detection_techniques(overlay, filter_applicable_to) + overlay_dict, all_techniques = _get_detection_techniques(overlay, filter_applicable_to) elif len(overlay) > 0: - overlay_dict = get_group_techniques(overlay, stage, platform, overlay_file_type) + overlay_dict = _get_group_techniques(overlay, stage, platform, overlay_file_type) if not overlay_dict: return - groups_dict = get_group_techniques(groups, stage, platform, groups_file_type) + groups_dict = _get_group_techniques(groups, stage, platform, groups_file_type) if groups_dict == -1: return if len(groups_dict) == 0: @@ -514,20 +517,20 @@ def generate_group_heat_map(groups, overlay, overlay_type, stage, platform, soft if software_groups and overlay: # TODO add support for campaign info in layer metadata if overlay_type not in [OVERLAY_TYPE_VISIBILITY, OVERLAY_TYPE_DETECTION]: # if a group overlay is provided, get the software techniques for the overlay - groups_software_dict = get_software_techniques(overlay, stage, platform) + groups_software_dict = _get_software_techniques(overlay, stage, platform) elif software_groups: - groups_software_dict = get_software_techniques(groups, stage, platform) + groups_software_dict = _get_software_techniques(groups, stage, platform) - technique_count, max_count = get_technique_count(groups_dict, overlay_dict, groups_software_dict, overlay_type, all_techniques) - technique_layer = get_technique_layer(technique_count, groups_dict, overlay_dict, groups_software_dict, - overlay_file_type, overlay_type, all_techniques) + technique_count, max_count = _get_technique_count(groups_dict, overlay_dict, groups_software_dict, overlay_type, all_techniques) + technique_layer = _get_technique_layer(technique_count, groups_dict, overlay_dict, groups_software_dict, + overlay_file_type, overlay_type, all_techniques) # make a list group names for the involved groups. if groups == ['all']: groups_list = ['all'] else: - groups_list = get_group_list(groups_dict, groups_file_type) - overlay_list = get_group_list(overlay_dict, overlay_file_type) + groups_list = _get_group_list(groups_dict, groups_file_type) + overlay_list = _get_group_list(overlay_dict, overlay_file_type) desc = 'stage: ' + stage + ' | platform: ' + platform + ' | group(s): ' + ', '.join(groups_list) + \ ' | overlay group(s): ' + ', '.join(overlay_list) From b2fdb25647d95b36d8acef2a007b0da30d3f772b Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Wed, 31 Jul 2019 10:20:21 +0200 Subject: [PATCH 10/49] - Multiple functions made "private". - Added new menu options. --- interactive_menu.py | 182 +++++++++++++++++++++++--------------------- 1 file changed, 95 insertions(+), 87 deletions(-) diff --git a/interactive_menu.py b/interactive_menu.py index e7c85c29..63b39096 100644 --- a/interactive_menu.py +++ b/interactive_menu.py @@ -1,4 +1,3 @@ -import sys import glob from data_source_mapping import * from technique_mapping import * @@ -16,7 +15,7 @@ yaml_path = 'sample-data/' -def clear(): +def _clear(): """ Clears the terminal screen and prints the title and version of the application. :return: @@ -34,7 +33,7 @@ def clear(): print('') -def ask_input(): +def _ask_input(): """ Waits for input from the terminal. :return: @@ -42,7 +41,7 @@ def ask_input(): return input(' >> ') -def wait(): +def _wait(): """ Prints wait statement and wait for pressing ENTER key. :return: @@ -57,7 +56,7 @@ def interactive_menu(): Main menu for interactive mode. :return: """ - clear() + _clear() print('Select a mode:') print('1. %s' % MENU_NAME_DATA_SOURCE_MAPPING) print('2. %s' % MENU_NAME_VISIBILITY_MAPPING) @@ -66,38 +65,38 @@ def interactive_menu(): print('5. Updates') print('6. Statistics') print('9. Quit') - choice = ask_input() + choice = _ask_input() if choice == '1': - menu_data_source(select_file(MENU_NAME_DATA_SOURCE_MAPPING, 'data sources', FILE_TYPE_DATA_SOURCE_ADMINISTRATION)) + _menu_data_source(_select_file(MENU_NAME_DATA_SOURCE_MAPPING, 'data sources', FILE_TYPE_DATA_SOURCE_ADMINISTRATION)) elif choice == '2': - menu_visibility(select_file(MENU_NAME_VISIBILITY_MAPPING, 'techniques (used to score the level of visibility)', FILE_TYPE_TECHNIQUE_ADMINISTRATION), - select_file(MENU_NAME_VISIBILITY_MAPPING, 'data sources (used to add metadata on the involved data sources to the heat map)', FILE_TYPE_DATA_SOURCE_ADMINISTRATION, False)) + _menu_visibility(_select_file(MENU_NAME_VISIBILITY_MAPPING, 'techniques (used to score the level of visibility)', FILE_TYPE_TECHNIQUE_ADMINISTRATION), + _select_file(MENU_NAME_VISIBILITY_MAPPING, 'data sources (used to add metadata on the involved data sources to the heat map)', FILE_TYPE_DATA_SOURCE_ADMINISTRATION, False)) elif choice == '3': - menu_detection(select_file(MENU_NAME_DETECTION_COVERAGE_MAPPING, 'techniques', FILE_TYPE_TECHNIQUE_ADMINISTRATION)) + _menu_detection(_select_file(MENU_NAME_DETECTION_COVERAGE_MAPPING, 'techniques', FILE_TYPE_TECHNIQUE_ADMINISTRATION)) elif choice == '4': - menu_groups() + _menu_groups() elif choice == '5': - menu_updates() + _menu_updates() elif choice == '6': - menu_statistics() + _menu_statistics() elif choice in ['9', 'q']: quit() else: interactive_menu() -def select_file(title, what, expected_file_type, b_clear=True): +def _select_file(title, what, expected_file_type, b_clear=True): """ Prints and handles the file selection in the terminal. It shows just .yaml files. :param title: title to print on top of this menu :param what: print for what purpose the file is selected :param expected_file_type: the expected file type of the YAML file - :param b_clear: clear the terminal before showing this menu + :param b_clear: _clear the terminal before showing this menu :return: filename of the selected file """ global yaml_path if b_clear: - clear() + _clear() print('Menu: %s' % title) print('') print('Select the YAML file with %s:' % what) @@ -116,18 +115,18 @@ def select_file(title, what, expected_file_type, b_clear=True): back_nr = 9 if n < 9 else n + (5 - n % 5) print('%d. Back to main menu.' % back_nr) - choice = ask_input() + choice = _ask_input() if choice == str(change_path_nr): print("Supply full or relative path:") - choice = ask_input() + choice = _ask_input() choice = choice if choice.endswith('/') else choice + '/' if os.path.exists(choice): yaml_path = choice - return select_file(title, what, expected_file_type, b_clear) + return _select_file(title, what, expected_file_type, b_clear) else: print("[!] Path doesn't exist") - wait() - return select_file(title, what, expected_file_type, b_clear) + _wait() + return _select_file(title, what, expected_file_type, b_clear) elif choice == str(back_nr): interactive_menu() elif choice == 'q': @@ -138,21 +137,21 @@ def select_file(title, what, expected_file_type, b_clear=True): file_type = check_file(filename, file_type=expected_file_type) if file_type: print('Selected file: ' + filename) - wait() + _wait() return filename else: print("[!] Invalid choice") - wait() - return select_file(title, what, expected_file_type, b_clear) + _wait() + return _select_file(title, what, expected_file_type, b_clear) -def menu_updates(): +def _menu_updates(): """ Prints and handles the menu for the Updates functionality. :return: """ - clear() + _clear() print('Menu: Updates') print('') @@ -164,55 +163,55 @@ def menu_updates(): print('3. Software (sorted by modified date)') print('3s. Software (sorted by creation date)') print('9. Back to main menu.') - choice = ask_input() + choice = _ask_input() if choice == '1': get_updates('techniques') - wait() + _wait() if choice == '1s': get_updates('techniques', 'created') - wait() + _wait() elif choice == '2': get_updates('groups') - wait() + _wait() elif choice == '2s': get_updates('groups', 'created') - wait() + _wait() elif choice == '3': get_updates('software') - wait() + _wait() elif choice == '3s': get_updates('software', 'created') - wait() + _wait() elif choice == '9': interactive_menu() elif choice == 'q': quit() - menu_updates() + _menu_updates() -def menu_statistics(): +def _menu_statistics(): """ Handles the Statistics functionality. :return: """ - clear() + _clear() print('Menu: Statistics') print('') get_statistics() - wait() + _wait() interactive_menu() -def menu_data_source(filename): +def _menu_data_source(filename_ds): """ Prints and handles the Data source mapping functionality. - :param filename: + :param filename_ds: :return: """ - clear() + _clear() print('Menu: %s' % MENU_NAME_DATA_SOURCE_MAPPING) print('') - print('Selected data source YAML file: %s' % filename) + print('Selected data source YAML file: %s' % filename_ds) print('') print('Select what you want to do:') print('1. Generate a data source layer for the ATT&CK Navigator.') @@ -220,39 +219,48 @@ def menu_data_source(filename): print('3. Generate an Excel sheet with all data sources.') print('4. Generate a technique administration YAML file with visibility scores, based on the number of available ' 'data sources') + print('5. update the visibility scores within a technique administration YAML file based on changes within any of ' + 'the data sources. \nPast visibility scores are preserved in the score_logbook, and manually assigned scores are ' + 'not updated without your approval. \nThe updated visibility are based on the number of available data sources.') print('9. Back to main menu.') - choice = ask_input() + choice = _ask_input() if choice == '1': print('Writing data sources layer...') - generate_data_sources_layer(filename) - wait() + generate_data_sources_layer(filename_ds) + _wait() elif choice == '2': print('Drawing the graph...') - plot_data_sources_graph(filename) - wait() + plot_data_sources_graph(filename_ds) + _wait() elif choice == '3': print('Generating Excel file...') - export_data_source_list_to_excel(filename) - wait() + export_data_source_list_to_excel(filename_ds) + _wait() elif choice == '4': print('Generating YAML file...') - generate_technique_administration_file(filename) - wait() + generate_technique_administration_file(filename_ds) + _wait() + elif choice == '5': + filename_t = _select_file(MENU_NAME_DETECTION_COVERAGE_MAPPING, 'techniques (used to score the level of visibility)', + FILE_TYPE_TECHNIQUE_ADMINISTRATION, False) + print('Updating visibility scores...') + update_technique_administration_file(filename_ds, filename_t) + _wait() elif choice == '9': interactive_menu() elif choice == 'q': quit() - menu_data_source(filename) + _menu_data_source(filename_ds) -def menu_detection(filename_t): +def _menu_detection(filename_t): """ Prints and handles the Detection coverage mapping functionality. :param filename_t: :return: """ global filter_applicable_to - clear() + _clear() print('Menu: %s' % MENU_NAME_DETECTION_COVERAGE_MAPPING) print('') print('Selected techniques YAML file: %s' % filename_t) @@ -268,41 +276,41 @@ def menu_detection(filename_t): print('5. Generate an Excel sheet with all administrated techniques.') print('6. Check the technique YAML file for errors.') print('9. Back to main menu.') - choice = ask_input() + choice = _ask_input() if choice == '1': print('Specify your filter for the applicable_to field:') - filter_applicable_to = ask_input().lower() + filter_applicable_to = _ask_input().lower() elif choice == '2': print('Writing detection coverage layer...') generate_detection_layer(filename_t, None, False, filter_applicable_to) - wait() + _wait() elif choice == '3': - filename_ds = select_file(MENU_NAME_DETECTION_COVERAGE_MAPPING, 'data sources (used to add metadata on the ' - 'involved data sources to the heat map)', - FILE_TYPE_DATA_SOURCE_ADMINISTRATION, False) + filename_ds = _select_file(MENU_NAME_DETECTION_COVERAGE_MAPPING, 'data sources (used to add metadata on the ' + 'involved data sources to the heat map)', + FILE_TYPE_DATA_SOURCE_ADMINISTRATION, False) print('Writing detection coverage layer with visibility as overlay...') generate_detection_layer(filename_t, filename_ds, True, filter_applicable_to) - wait() + _wait() elif choice == '4': print('Drawing the graph...') plot_detection_graph(filename_t, filter_applicable_to) - wait() + _wait() elif choice == '5': print('Generating Excel file...') export_techniques_list_to_excel(filename_t) - wait() + _wait() elif choice == '6': print('Checking the technique YAML file for errors...') check_yaml_file_health(filename_t, FILE_TYPE_TECHNIQUE_ADMINISTRATION, health_is_called=True) - wait() + _wait() elif choice == '9': interactive_menu() elif choice == 'q': quit() - menu_detection(filename_t) + _menu_detection(filename_t) -def menu_visibility(filename_t, filename_ds): +def _menu_visibility(filename_t, filename_ds): """ Prints and handles the Visibility coverage mapping functionality. :param filename_t: @@ -310,7 +318,7 @@ def menu_visibility(filename_t, filename_ds): :return: """ global filter_applicable_to - clear() + _clear() print('Menu: %s' % MENU_NAME_VISIBILITY_MAPPING) print('') print('Selected techniques YAML file: %s' % filename_t) @@ -326,40 +334,40 @@ def menu_visibility(filename_t, filename_ds): print('4. Generate an Excel sheet with all administrated techniques.') print('5. Check the technique YAML file for errors.') print('9. Back to main menu.') - choice = ask_input() + choice = _ask_input() if choice == '1': print('Specify your filter for the applicable_to field:') - filter_applicable_to = ask_input().lower() + filter_applicable_to = _ask_input().lower() elif choice == '2': print('Writing visibility coverage layer...') generate_visibility_layer(filename_t, filename_ds, False, filter_applicable_to) - wait() + _wait() elif choice == '3': print('Writing visibility coverage layer overlaid with detections...') generate_visibility_layer(filename_t, filename_ds, True, filter_applicable_to) - wait() + _wait() elif choice == '4': print('Generating Excel file...') export_techniques_list_to_excel(filename_t) - wait() + _wait() elif choice == '5': print('Checking the technique YAML file for errors...') check_yaml_file_health(filename_t, FILE_TYPE_TECHNIQUE_ADMINISTRATION, health_is_called=True) - wait() + _wait() elif choice == '9': interactive_menu() elif choice == 'q': quit() - menu_visibility(filename_t, filename_ds) + _menu_visibility(filename_t, filename_ds) -def menu_groups(): +def _menu_groups(): """ Prints and handles the Threat actor group mapping functionality. :return: """ global groups, software_group, platform, stage, groups_overlay, overlay_type, filter_applicable_to - clear() + _clear() print('Menu: %s' % MENU_NAME_THREAT_ACTOR_GROUP_MAPPING) print('') print('Options:') @@ -375,22 +383,22 @@ def menu_groups(): print('') print('7. Generate a heat map layer.') print('9. Back to main menu.') - choice = ask_input() + choice = _ask_input() if choice == '1': print('Specify True or False for software group:') - software_group = True if ask_input().lower() == 'true' else False + software_group = True if _ask_input().lower() == 'true' else False elif choice == '2': print('Specify platform (all, Linux, macOS, Windows):') - p = ask_input().lower() + p = _ask_input().lower() platform = 'Windows' if p == 'windows' else 'Linux' if p == 'linux' else 'macOS' if p == 'macos' else 'all' elif choice == '3': print('Specify stage (pre-attack, attack):') - s = ask_input().lower() + s = _ask_input().lower() stage = 'pre-attack' if s == 'pre-attack' else 'attack' elif choice == '4': print('Specify the groups to include separated using commas. Group can be their ID, name or alias ' '(default is all groups). Other option is to provide a YAML file with a custom group(s)') - g = ask_input() + g = _ask_input() groups = g if g is not '' else 'all' elif choice == '5': print('') @@ -398,30 +406,30 @@ def menu_groups(): print('2. Overlay with detections.') print('3. Overlay with visibility.') print('4. No overlay.') - choice = ask_input() + choice = _ask_input() if choice == '1': print('Specify the group(s) to overlay (in a different color) on the one specified in the Groups option. ' 'A group can be their ID, name or alias separated using commas. Other option is to provide a YAML ' 'file with a custom group(s).') overlay_type = OVERLAY_TYPE_GROUP - groups_overlay = ask_input() + groups_overlay = _ask_input() elif choice == '2': overlay_type = OVERLAY_TYPE_DETECTION - groups_overlay = select_file(MENU_NAME_THREAT_ACTOR_GROUP_MAPPING, 'techniques', FILE_TYPE_TECHNIQUE_ADMINISTRATION, False) + groups_overlay = _select_file(MENU_NAME_THREAT_ACTOR_GROUP_MAPPING, 'techniques', FILE_TYPE_TECHNIQUE_ADMINISTRATION, False) elif choice == '3': overlay_type = OVERLAY_TYPE_VISIBILITY - groups_overlay = select_file(MENU_NAME_THREAT_ACTOR_GROUP_MAPPING, 'techniques', FILE_TYPE_TECHNIQUE_ADMINISTRATION, False) + groups_overlay = _select_file(MENU_NAME_THREAT_ACTOR_GROUP_MAPPING, 'techniques', FILE_TYPE_TECHNIQUE_ADMINISTRATION, False) elif choice == '4': overlay_type = '' groups_overlay = '' elif choice == '6': print('Specify your filter for the applicable_to field:') - filter_applicable_to = ask_input().lower() + filter_applicable_to = _ask_input().lower() elif choice == '7': generate_group_heat_map(groups, groups_overlay, overlay_type, stage, platform, software_group, filter_applicable_to) - wait() + _wait() elif choice == '9': interactive_menu() elif choice == 'q': quit() - menu_groups() + _menu_groups() From ef7f494cdad8866b77060a71e4b8a0fcdeffeeb3 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Wed, 31 Jul 2019 10:21:39 +0200 Subject: [PATCH 11/49] - Made compatible with the version 1.2 of the technique admin YAML file. - Replaced PyYAML with ruamel.yaml. --- technique_mapping.py | 121 ++++++++++++++++++++++++------------------- 1 file changed, 67 insertions(+), 54 deletions(-) diff --git a/technique_mapping.py b/technique_mapping.py index f69022bb..ac66dd27 100644 --- a/technique_mapping.py +++ b/technique_mapping.py @@ -61,8 +61,9 @@ def plot_detection_graph(filename, filter_applicable_to): graph_values = [] for t in my_techniques.values(): for detection in t['detection']: - if detection['date_implemented']: - yyyymm = detection['date_implemented'].strftime('%Y-%m') + date = get_latest_date(detection) + if date: + yyyymm = date.strftime('%Y-%m') graph_values.append({'date': yyyymm, 'count': 1}) import pandas as pd @@ -77,7 +78,7 @@ def plot_detection_graph(filename, filter_applicable_to): 'layout': go.Layout(title="# of detections for %s %s" % (name, filter_applicable_to))}, filename=output_filename, auto_open=False ) - print("File written: " + output_filename) + print("File written: " + output_filename) def _load_data_sources(filename): @@ -87,8 +88,9 @@ def _load_data_sources(filename): :return: dictionary with data sources (including properties) """ my_data_sources = {} + _yaml = init_yaml() with open(filename, 'r') as yaml_file: - yaml_content = yaml.load(yaml_file, Loader=yaml.FullLoader) + yaml_content = _yaml.load(yaml_file) for d in yaml_content['data_sources']: dq = d['data_quality'] if dq['device_completeness'] > 0 and dq['data_field_completeness'] > 0 and dq['timeliness'] > 0 and dq['consistency'] > 0: @@ -113,7 +115,7 @@ def _write_layer(layer, mapped_techniques, filename_prefix, filename_suffix, nam output_filename = normalize_name_to_filename('output/%s_%s%s.json' % (filename_prefix, name, filename_suffix)) with open(output_filename, 'w') as f: f.write(json_string) - print("File written: " + output_filename) + print("File written: " + output_filename) def _map_and_colorize_techniques_for_detections(my_techniques): @@ -137,7 +139,7 @@ def _map_and_colorize_techniques_for_detections(my_techniques): technique = get_technique(techniques, technique_id) for tactic in get_tactics(technique): - x = {} + x = dict() x['techniqueID'] = technique_id x['color'] = color x['comment'] = '' @@ -146,17 +148,19 @@ def _map_and_colorize_techniques_for_detections(my_techniques): x['metadata'] = [] x['score'] = s cnt = 1 - tcnt = len([d for d in technique_data['detection'] if d['score'] >= 0]) + tcnt = len([d for d in technique_data['detection'] if get_latest_score(d) >= 0]) for detection in technique_data['detection']: - if detection['score'] >= 0: + d_score = get_latest_score(detection) + if d_score >= 0: location = ', '.join(detection['location']) location = location if location != '' else '-' applicable_to = ', '.join(detection['applicable_to']) - comment = str(detection['comment']) if str(detection['comment']) != '' else '-' + general_comment = str(detection['comment']) if str(detection['comment']) != '' else '-' x['metadata'].append({'name': '-Applicable to', 'value': applicable_to}) - x['metadata'].append({'name': '-Detection score', 'value': str(detection['score'])}) + x['metadata'].append({'name': '-Detection score', 'value': str(d_score)}) x['metadata'].append({'name': '-Detection location', 'value': location}) - x['metadata'].append({'name': '-Comment', 'value': comment}) + x['metadata'].append({'name': '-General comment', 'value': general_comment}) + x['metadata'].append({'name': '-Detection comment', 'value': get_latest_comment(detection)}) if cnt != tcnt: x['metadata'].append({'name': '---', 'value': '---'}) cnt += 1 @@ -192,7 +196,7 @@ def _map_and_colorize_techniques_for_visibility(my_techniques, my_data_sources): color = COLOR_V_1 if s == 1 else COLOR_V_2 if s == 2 else COLOR_V_3 if s == 3 else COLOR_V_4 if s == 4 else '' for tactic in get_tactics(technique): - x = {} + x = dict() x['techniqueID'] = technique_id x['color'] = color x['comment'] = '' @@ -207,11 +211,12 @@ def _map_and_colorize_techniques_for_visibility(my_techniques, my_data_sources): cnt = 1 tcnt = len(technique_data['visibility']) for visibility in technique_data['visibility']: - comment = str(visibility['comment']) if str(visibility['comment']) != '' else '-' applicable_to = ', '.join(visibility['applicable_to']) + general_comment = str(visibility['comment']) if str(visibility['comment']) != '' else '-' x['metadata'].append({'name': '-Applicable to', 'value': applicable_to}) - x['metadata'].append({'name': '-Visibility score', 'value': str(visibility['score'])}) - x['metadata'].append({'name': '-Comment', 'value': comment}) + x['metadata'].append({'name': '-Visibility score', 'value': str(get_latest_score(visibility))}) + x['metadata'].append({'name': '-General comment', 'value': general_comment}) + x['metadata'].append({'name': '-Visibility comment', 'value': get_latest_comment(visibility)}) if cnt != tcnt: x['metadata'].append({'name': '---', 'value': '---'}) cnt += 1 @@ -224,7 +229,7 @@ def _map_and_colorize_techniques_for_visibility(my_techniques, my_data_sources): tactics = get_tactics(t) if tactics: for tactic in tactics: - x = {} + x = dict() x['techniqueID'] = tech_id x['comment'] = '' x['enabled'] = True @@ -279,7 +284,7 @@ def _map_and_colorize_techniques_for_overlaid(my_techniques, my_data_sources, fi technique = get_technique(techniques, technique_id) for tactic in get_tactics(technique): - x = {} + x = dict() x['techniqueID'] = technique_id x['color'] = color x['comment'] = '' @@ -292,17 +297,19 @@ def _map_and_colorize_techniques_for_overlaid(my_techniques, my_data_sources, fi # Metadata for detection: cnt = 1 - tcnt = len([d for d in technique_data['detection'] if d['score'] >= 0 and (filter_applicable_to == 'all' or filter_applicable_to in d['applicable_to'] or 'all' in d['applicable_to'])]) + tcnt = len([d for d in technique_data['detection'] if get_latest_score(d) >= 0 and (filter_applicable_to == 'all' or filter_applicable_to in d['applicable_to'] or 'all' in d['applicable_to'])]) for detection in technique_data['detection']: - if detection['score'] >= 0 and (filter_applicable_to == 'all' or filter_applicable_to in detection['applicable_to'] or 'all' in detection['applicable_to']): + d_score = get_latest_score(detection) + if d_score >= 0 and (filter_applicable_to == 'all' or filter_applicable_to in detection['applicable_to'] or 'all' in detection['applicable_to']): location = ', '.join(detection['location']) location = location if location != '' else '-' applicable_to = ', '.join(detection['applicable_to']) - comment = str(detection['comment']) if str(detection['comment']) != '' else '-' + general_comment = str(detection['comment']) if str(detection['comment']) != '' else '-' x['metadata'].append({'name': '-Applicable to', 'value': applicable_to}) - x['metadata'].append({'name': '-Detection score', 'value': str(detection['score'])}) + x['metadata'].append({'name': '-Detection score', 'value': str(d_score)}) x['metadata'].append({'name': '-Detection location', 'value': location}) - x['metadata'].append({'name': '-Comment', 'value': comment}) + x['metadata'].append({'name': '-General comment', 'value': general_comment}) + x['metadata'].append({'name': '-Detection comment', 'value': get_latest_comment(detection)}) if cnt != tcnt: x['metadata'].append({'name': '---', 'value': '---'}) cnt += 1 @@ -314,11 +321,12 @@ def _map_and_colorize_techniques_for_overlaid(my_techniques, my_data_sources, fi tcnt = len([v for v in technique_data['visibility'] if filter_applicable_to == 'all' or filter_applicable_to in v['applicable_to'] or 'all' in v['applicable_to']]) for visibility in technique_data['visibility']: if filter_applicable_to == 'all' or filter_applicable_to in visibility['applicable_to'] or 'all' in visibility['applicable_to']: - comment = str(visibility['comment']) if str(visibility['comment']) != '' else '-' applicable_to = ', '.join(visibility['applicable_to']) + general_comment = str(visibility['comment']) if str(visibility['comment']) != '' else '-' x['metadata'].append({'name': '-Applicable to', 'value': applicable_to}) - x['metadata'].append({'name': '-Visibility score', 'value': str(visibility['score'])}) - x['metadata'].append({'name': '-Comment', 'value': comment}) + x['metadata'].append({'name': '-Visibility score', 'value': str(get_latest_score(visibility))}) + x['metadata'].append({'name': '-General comment', 'value': general_comment}) + x['metadata'].append({'name': '-Visibility comment', 'value': get_latest_comment(visibility)}) if cnt != tcnt: x['metadata'].append({'name': '---', 'value': '---'}) cnt += 1 @@ -371,7 +379,7 @@ def export_techniques_list_to_excel(filename): worksheet_detections.merge_range(2, 0, 2, 2, 'Technique', format_bold_center_bggrey) worksheet_visibility.merge_range(2, 0, 2, 2, 'Technique', format_bold_center_bggrey) worksheet_detections.merge_range(2, 3, 2, 8, 'Detection', format_bold_center_bgreen) - worksheet_visibility.merge_range(2, 3, 2, 5, 'Visibility', format_bold_center_bgblue) + worksheet_visibility.merge_range(2, 3, 2, 7, 'Visibility', format_bold_center_bgblue) # Writing the detections: y = 3 @@ -379,20 +387,19 @@ def export_techniques_list_to_excel(filename): worksheet_detections.write(y, 1, 'Description', format_bold_left) worksheet_detections.write(y, 2, 'Tactic', format_bold_left) worksheet_detections.write(y, 3, 'Applicable to', format_bold_left) - worksheet_detections.write(y, 4, 'Date registered', format_bold_left) - worksheet_detections.write(y, 5, 'Date implemented', format_bold_left) - worksheet_detections.write(y, 6, 'Score', format_bold_left) - worksheet_detections.write(y, 7, 'Location', format_bold_left) - worksheet_detections.write(y, 8, 'Comment', format_bold_left) + worksheet_detections.write(y, 4, 'Date', format_bold_left) + worksheet_detections.write(y, 5, 'Score', format_bold_left) + worksheet_detections.write(y, 6, 'Location', format_bold_left) + worksheet_detections.write(y, 7, 'General comment', format_bold_left) + worksheet_detections.write(y, 8, 'Detection comment', format_bold_left) worksheet_detections.set_column(0, 0, 14) worksheet_detections.set_column(1, 1, 40) worksheet_detections.set_column(2, 2, 50) worksheet_detections.set_column(3, 3, 18) - worksheet_detections.set_column(4, 4, 15) - worksheet_detections.set_column(5, 5, 18) - worksheet_detections.set_column(6, 6, 8) - worksheet_detections.set_column(7, 7, 25) - worksheet_detections.set_column(8, 8, 40) + worksheet_detections.set_column(4, 4, 11) + worksheet_detections.set_column(5, 5, 8) + worksheet_detections.set_column(6, 6, 25) + worksheet_detections.set_column(7, 8, 40) y = 4 for technique_id, technique_data in my_techniques.items(): # Add row for every detection that is defined: @@ -403,12 +410,13 @@ def export_techniques_list_to_excel(filename): get_tactics(get_technique(mitre_techniques, technique_id))), valign_top) worksheet_detections.write(y, 3, ', '.join(detection['applicable_to']), wrap_text) - worksheet_detections.write(y, 4, str(detection['date_registered']).replace('None', ''), valign_top) - worksheet_detections.write(y, 5, str(detection['date_implemented']).replace('None', ''), valign_top) - ds = detection['score'] - worksheet_detections.write(y, 6, ds, detection_score_0 if ds == 0 else detection_score_1 if ds == 1 else detection_score_2 if ds == 2 else detection_score_3 if ds == 3 else detection_score_4 if ds == 4 else detection_score_5 if ds == 5 else no_score) - worksheet_detections.write(y, 7, '\n'.join(detection['location']), wrap_text) - worksheet_detections.write(y, 8, detection['comment'][:-1] if detection['comment'].endswith('\n') else detection['comment'], wrap_text) + worksheet_detections.write(y, 4, str(get_latest_date(detection)).replace('None', ''), valign_top) + ds = get_latest_score(detection) + worksheet_detections.write(y, 5, ds, detection_score_0 if ds == 0 else detection_score_1 if ds == 1 else detection_score_2 if ds == 2 else detection_score_3 if ds == 3 else detection_score_4 if ds == 4 else detection_score_5 if ds == 5 else no_score) + worksheet_detections.write(y, 6, '\n'.join(detection['location']), wrap_text) + worksheet_detections.write(y, 7, detection['comment'][:-1] if detection['comment'].endswith('\n') else detection['comment'], wrap_text) + d_comment = get_latest_comment(detection, empty='') + worksheet_detections.write(y, 8, d_comment[:-1] if d_comment.endswith('\n') else d_comment, wrap_text) y += 1 worksheet_detections.autofilter(3, 0, 3, 8) worksheet_detections.freeze_panes(4, 0) @@ -419,14 +427,17 @@ def export_techniques_list_to_excel(filename): worksheet_visibility.write(y, 1, 'Description', format_bold_left) worksheet_visibility.write(y, 2, 'Tactic', format_bold_left) worksheet_visibility.write(y, 3, 'Applicable to', format_bold_left) - worksheet_visibility.write(y, 4, 'Score', format_bold_left) - worksheet_visibility.write(y, 5, 'Comment', format_bold_left) + worksheet_visibility.write(y, 4, 'Date', format_bold_left) + worksheet_visibility.write(y, 5, 'Score', format_bold_left) + worksheet_visibility.write(y, 6, 'General Comment', format_bold_left) + worksheet_visibility.write(y, 7, 'Visibility comment', format_bold_left) worksheet_visibility.set_column(0, 0, 14) worksheet_visibility.set_column(1, 1, 40) worksheet_visibility.set_column(2, 2, 50) - worksheet_visibility.set_column(3, 9, 18) - worksheet_visibility.set_column(4, 10, 8) - worksheet_visibility.set_column(5, 11, 40) + worksheet_visibility.set_column(3, 3, 18) + worksheet_visibility.set_column(4, 4, 11) + worksheet_visibility.set_column(5, 5, 8) + worksheet_visibility.set_column(6, 7, 40) y = 4 for technique_id, technique_data in my_techniques.items(): # Add row for every visibility that is defined: @@ -434,18 +445,20 @@ def export_techniques_list_to_excel(filename): worksheet_visibility.write(y, 0, technique_id, valign_top) worksheet_visibility.write(y, 1, get_technique(mitre_techniques, technique_id)['name'], valign_top) worksheet_visibility.write(y, 2, ', '.join(t.capitalize() for t in - get_tactics(get_technique(mitre_techniques, technique_id))), - valign_top) + get_tactics(get_technique(mitre_techniques, technique_id))), valign_top) worksheet_visibility.write(y, 3, ', '.join(visibility['applicable_to']), wrap_text) - vs = visibility['score'] - worksheet_visibility.write(y, 4, vs, visibility_score_1 if vs == 1 else visibility_score_2 if vs == 2 else visibility_score_3 if vs == 3 else visibility_score_4 if vs == 4 else no_score) - worksheet_visibility.write(y, 5, visibility['comment'][:-1] if visibility['comment'].endswith('\n') else visibility['comment'], wrap_text) + worksheet_visibility.write(y, 4, str(get_latest_date(visibility)).replace('None', ''), valign_top) + vs = get_latest_score(visibility) + worksheet_visibility.write(y, 5, vs, visibility_score_1 if vs == 1 else visibility_score_2 if vs == 2 else visibility_score_3 if vs == 3 else visibility_score_4 if vs == 4 else no_score) + v_comment = get_latest_comment(visibility, empty='') + worksheet_visibility.write(y, 6, visibility['comment'][:-1] if visibility['comment'].endswith('\n') else visibility['comment'], wrap_text) + worksheet_visibility.write(y, 7, v_comment[:-1] if v_comment.endswith('\n') else v_comment, wrap_text) y += 1 - worksheet_visibility.autofilter(3, 0, 3, 5) + worksheet_visibility.autofilter(3, 0, 3, 7) worksheet_visibility.freeze_panes(4, 0) try: workbook.close() - print("File written: " + excel_filename) + print("File written: " + excel_filename) except Exception as e: print('[!] Error while writing Excel file: %s' % str(e)) From b3468a1060b70615ae6735738ea1d8b074aa9678 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Wed, 31 Jul 2019 10:23:32 +0200 Subject: [PATCH 12/49] - Added new functionality for the auto-upgrade from tech. admin. YAML file v1.1 to v1.2. - Replaced PyYAML with ruamel.yaml. --- upgrade.py | 323 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 251 insertions(+), 72 deletions(-) diff --git a/upgrade.py b/upgrade.py index 17750a82..869dccda 100644 --- a/upgrade.py +++ b/upgrade.py @@ -1,18 +1,33 @@ -import re -import os -import shutil from constants import * -def _get_attack_id(stix_obj): +def _load_techniques(yaml_file_lines): """ - Get the Technique, Group or Software ID from the STIX object - :param stix_obj: STIX object (Technique, Software or Group) - :return: ATT&CK ID + Loads the techniques (including detection and visibility properties) from the given yaml file. + :param yaml_file_lines: list with the yaml file lines containing the techniques administration + :return: dictionary with techniques (incl. properties) """ - for ext_ref in stix_obj['external_references']: - if ext_ref['source_name'] in ['mitre-attack', 'mitre-mobile-attack', 'mitre-pre-attack']: - return ext_ref['external_id'] + from generic import add_entry_to_list_in_dictionary, init_yaml + + my_techniques = {} + _yaml = init_yaml() + yaml_content = _yaml.load(''.join(yaml_file_lines)) + for d in yaml_content['techniques']: + # Add detection items + if isinstance(d['detection'], dict): # There is just one detection entry + add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'detection', d['detection']) + elif isinstance(d['detection'], list): # There are multiple detection entries + for de in d['detection']: + add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'detection', de) + + # Add visibility items + if isinstance(d['visibility'], dict): # There is just one visibility entry + add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'visibility', d['visibility']) + elif isinstance(d['visibility'], list): # There are multiple visibility entries + for de in d['visibility']: + add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'visibility', de) + + return my_techniques def _create_upgrade_text(file_type, file_version): @@ -27,29 +42,30 @@ def _create_upgrade_text(file_type, file_version): 'The following upgrades will be performed on the techniques administration file:\n' for version in FILE_TYPE_TECHNIQUE_ADMINISTRATION_UPGRADE_TEXT: if file_version < version: + text += '- Version: ' + str(version) + '\n' text += FILE_TYPE_TECHNIQUE_ADMINISTRATION_UPGRADE_TEXT[version] + '\n' return text -def _ask_to_upgrade(filename): +def _get_indent_chars(file_lines): """ - Ask the user to upgrade the YAML file or not. - :param filename: YAML administration file - :return: boolean value indicating if the upgrade can be performed + Identify and return the characters that are used to indent the YAML file + :param file_lines: List of lines in the YAML file + :return: indent characters """ - yes_no = '' - while not re.match('^(y|yes|n|no)$', yes_no, re.IGNORECASE): - yes_no = input('Do you want to upgrade the below file. A backup will be created of the current file.\n' - '[!] Not upgrading the file will brake some functionality within DeTT&CT.\n' - ' - ' + filename + '\n >> y(yes)/n(no): ') + indent_chars = ' ' - if re.match('^(y|yes)$', yes_no, re.IGNORECASE): - return True - else: - return False + for l in file_lines: + if REGEX_YAML_TECHNIQUE_ID.match(l): + indent_chars = REGEX_YAML_INDENT_CHARS.search(l).groups()[0] + indent_chars = len(indent_chars) * ' ' + break + + return indent_chars +# noinspection PyDictCreation def upgrade_yaml_file(filename, file_type, file_version, attack_tech_data): """ Main function to upgrade the YAML file to a new version @@ -59,24 +75,27 @@ def upgrade_yaml_file(filename, file_type, file_version, attack_tech_data): :param attack_tech_data: ATT&CK data on techniques :return: """ + from generic import ask_yes_no, backup_file is_upgraded = False tech_upgrade_func = {} tech_upgrade_func[1.1] = _upgrade_technique_yaml_10_to_11 + tech_upgrade_func[1.2] = _upgrade_technique_yaml_11_to_12 with open(filename, 'r') as file: file_new_lines = file.readlines() if file_type == FILE_TYPE_TECHNIQUE_ADMINISTRATION: - if file_version != FILE_TYPE_TECHNIQUE_ADMINISTRATION_VERSION: + if file_version < FILE_TYPE_TECHNIQUE_ADMINISTRATION_VERSION: upgrade_text = _create_upgrade_text(file_type, file_version) print(upgrade_text) - if _ask_to_upgrade(filename): + upgrade_question = 'Do you want to upgrade the below file. A backup will be created of the current file.\n' + \ + '[!] Not upgrading the file will brake functionality within DeTT&CT.\n' + \ + ' - ' + filename + if ask_yes_no(upgrade_question): is_upgraded = True # create backup of the non-upgraded file - backup_filename = _get_backup_filename(filename) - shutil.copy2(filename, backup_filename) - print('Written backup file: ' + backup_filename) + backup_file(filename) for tech_f in tech_upgrade_func.keys(): if file_version < tech_f: @@ -96,66 +115,226 @@ def upgrade_yaml_file(filename, file_type, file_version, attack_tech_data): print('-'*80) -def _get_technique(techniques, technique_id): - """ - Generic function to lookup a specific technique_id in a list of dictionaries with techniques. - :param techniques: list with all techniques - :param technique_id: technique_id to look for - :return: the technique you're searching for. None if not found. - """ - for t in techniques: - if technique_id == _get_attack_id(t): - return t - return None - - -def _get_backup_filename(filename): - """ - Create a filename to be used for backup of the YAML file - :param filename: existing YAML filename - :return: a name for the backup file - """ - suffix = 1 - backup_filename = filename.replace('.yaml', '_backup_' + str(suffix) + '.yaml') - while os.path.exists(backup_filename): - backup_filename = backup_filename.replace('_backup_' + str(suffix) + '.yaml', '_backup_' + str(suffix+1) + '.yaml') - suffix += 1 - - return backup_filename - - def _upgrade_technique_yaml_10_to_11(file_lines, attack_tech_data): """ Upgrade the YAML technique administration file from 1.0 to 1.1. - :param file_lines: array containing the lines within the tech. admin. file + :param file_lines: list containing the lines within the tech. admin. file :param attack_tech_data: ATT&CK data on techniques :return: array with new lines to be written to disk """ - regex_version = re.compile(r'^\s*version:\s+1\.0\s*$', re.IGNORECASE) - regex_tech = re.compile(r'^-\s+technique_id:\s+T[0-9]{4}\s*$', re.IGNORECASE) - regex_tech_id = re.compile(r'^-\s+technique_id:\s+(T[0-9]{4})\s*$', re.IGNORECASE) - regex_detection = re.compile(r'^\s+detection:\s*$', re.IGNORECASE) - regex_visibility = re.compile(r'^\s+visibility:\s*$', re.IGNORECASE) + from generic import get_technique + + # identify the indent characters used + indent_chars = _get_indent_chars(file_lines) file_new_lines = [] x = 0 for l in file_lines: - if regex_version.match(l): + if REGEX_YAML_VERSION_10.match(l): file_new_lines.append(l.replace('1.0', '1.1')) - elif regex_tech.match(l): + elif REGEX_YAML_TECHNIQUE_ID.match(l): file_new_lines.append(l) - - tech_id = regex_tech_id.search(l).group(1) - tech_name = _get_technique(attack_tech_data, tech_id)['name'] - file_new_lines.append(' technique_name: ' + tech_name+'\n') - elif regex_detection.match(l): + tech_id = REGEX_YAML_TECHNIQUE_ID_GROUP.search(l).group(1) + tech_name = get_technique(attack_tech_data, tech_id)['name'] + file_new_lines.append(indent_chars + 'technique_name: ' + tech_name+'\n') + elif REGEX_YAML_DETECTION.match(l): file_new_lines.append(l) - file_new_lines.append(" applicable_to: ['all']\n") - elif regex_visibility.match(l): + file_new_lines.append((indent_chars * 2) + "applicable_to: ['all']\n") + elif REGEX_YAML_VISIBILITY.match(l): file_new_lines.append(l) - file_new_lines.append(" applicable_to: ['all']\n") + file_new_lines.append((indent_chars * 2) + "applicable_to: ['all']\n") else: file_new_lines.append(l) x += 1 return file_new_lines + + +def _print_error_msg(msg): + print(msg) + return True + + +def _check_yaml_file_health_v11(file_lines): + """ + Check on error in the provided YAML file version 1.1 + :param file_lines: YAML file lines + :return: True for a healthy file, and False when encountering health issues. + """ + from generic import init_yaml + has_error = False + + # check for duplicate tech IDs + _yaml = init_yaml() + yaml_content = _yaml.load(''.join(file_lines)) + + tech_ids = list(map(lambda x: x['technique_id'], yaml_content['techniques'])) + tech_dup = [] + for tech in tech_ids: + if tech not in tech_dup: + tech_dup.append(tech) + else: + has_error = _print_error_msg('[!] Duplicate technique ID: ' + tech) + + # checks on: + # - empty key-value pairs: 'date_implemented', 'location', 'applicable_to', 'score' + # - invalid date format for: 'date_implemented' + # - detection or visibility score out-of-range + # - missing key-value pairs: 'applicable_to', 'date_implemented', 'score', 'location', 'comment' + # - check on 'applicable_to' values which are very similar + + dict_yaml_techniques = _load_techniques(file_lines) + all_applicable_to = set() + for tech, v in dict_yaml_techniques.items(): + for key in ['detection', 'visibility']: + if key not in v: + has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is MISSING ' + key) + else: + # create at set containing all values for 'applicable_to' + all_applicable_to.update([a for v in v[key] for a in v['applicable_to']]) + + for detection in v['detection']: + for key in ['applicable_to', 'date_implemented', 'score', 'location', 'comment']: + if key not in detection: + has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is MISSING the key-value pair in detection: ' + key) + + try: + # noinspection PyChainedComparisons + if detection['score'] is None: + has_error = _print_error_msg('[!] Technique ID: ' + tech + ' has an EMPTY key-value pair in detection: score') + + elif not (detection['score'] >= -1 and detection['score'] <= 5): + has_error = _print_error_msg('[!] Technique ID: ' + tech + ' has an INVALID detection score: ' + + str(detection['score']) + ' (should be between -1 and 5)') + + elif detection['score'] > -1: + if not detection['date_implemented']: + has_error = _print_error_msg('[!] Technique ID: ' + tech + ' has an EMPTY key-value pair in detection: ' + 'date_implemented') + break + try: + # noinspection PyStatementEffect + detection['date_implemented'].year + # noinspection PyStatementEffect + detection['date_implemented'].month + # noinspection PyStatementEffect + detection['date_implemented'].day + except AttributeError: + has_error = _print_error_msg('[!] Technique ID: ' + tech + + ' has an INVALID data format for the key-value pair in detection: ' + + 'date_implemented (should be YYYY-MM-DD)') + for key in ['location', 'applicable_to']: + if not isinstance(detection[key], list): + has_error = _print_error_msg('[!] Technique ID: ' + tech + ' has for the key-value pair \'' + + key + '\' a string value assigned (should be a list)') + else: + try: + if detection[key][0] is None: + has_error = _print_error_msg('[!] Technique ID: ' + tech + ' has an EMPTY key-value pair in detection: ' + key) + except TypeError: + has_error = _print_error_msg('[!] Technique ID: ' + tech + ' has an EMPTY key-value pair in detection: ' + key) + except KeyError: + pass + + for visibility in v['visibility']: + for key in ['applicable_to', 'score', 'comment']: + if key not in visibility: + has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is MISSING the key-value pair in visibility: ' + key) + + try: + if visibility['score'] is None: + has_error = _print_error_msg('[!] Technique ID: ' + tech + ' has an EMPTY key-value pair in visibility: score') + elif not (0 <= visibility['score'] <= 4): + # noinspection PyUnboundLocalVariable + has_error = _print_error_msg('[!] Technique ID: ' + tech + ' has an INVALID visibility score: ' + + str(detection['score']) + ' (should be between 0 and 4)') + except KeyError: + pass + + if has_error: + print('') + return False + else: + return True + + +def _upgrade_technique_yaml_11_to_12(file_lines, attack_tech_data): + """ + Upgrade the YAML technique administration file from 1.1 to 1.2. + :param file_lines: array containing the lines within the tech. admin. file + :param attack_tech_data: Not used, but necessary to be compatible with other upgrade methods. + :return: array with new lines to be written to disk + """ + from generic import ask_yes_no, fix_date, init_yaml + + # we will first do a health check on the tech. admin file version 1.1. Having health issues in the file could + # result in an upgraded file with errors. + print('Checking the health of the file before we to the upgrade from version 1.1 to 1.2') + healthy_file = _check_yaml_file_health_v11(file_lines) + if not healthy_file: + print('[!] Health issues found. It is advisable first to fix the health issues before continuing the upgrade.') + if not ask_yes_no('Are you sure that you want to continue the upgrade?'): + print('Upgrade cancelled') + quit() + else: + print(' - No health issues found. We continue the upgrade to version 1.2\n') + + keep_date_registered = ask_yes_no("Do you want to keep the key-value pair 'date_registered' in your technique " + "administration file even though DeTT&CT no longer makes use of it?") + + date_for_visibility = '' + print("Which date do you want to fill in for the visibility scores already present in the new key-value pair 'date'?") + while not REGEX_YAML_VALID_DATE.match(date_for_visibility): + date_for_visibility = input(' >> YYYY-MM-DD: ') + if not REGEX_YAML_VALID_DATE.match(date_for_visibility): + print(' Invalid date format') + print('') + + auto_generated = ask_yes_no('Are ALL of the current visibility scores within the technique administration file directly derived from the nr. of data sources?\n' + ' * Generated using the option \'-y, --yaml\' from the \'datasoure\' mode in dettect.py\n' + ' * Which means NONE of them have been scored manually?') + + _yaml = init_yaml() + + yaml_file = _yaml.load(''.join(file_lines)) + yaml_file['version'] = 1.2 + + # upgrade to the new v1.2 tech. admin file + for tech in yaml_file['techniques']: + if isinstance(tech['detection'], list): + detections = tech['detection'] + else: + detections = [tech['detection']] + + for d in detections: + score = d['score'] + date = d['date_implemented'] + try: + if not keep_date_registered: + del d['date_registered'] + del d['date_implemented'] + del d['score'] + except KeyError: + pass + + d['score_logbook'] = [{'date': date, 'score': score, 'comment': ''}] + + if isinstance(tech['visibility'], list): + visibility = tech['visibility'] + else: + visibility = [tech['visibility']] + + for v in visibility: + score = v['score'] + try: + del v['score'] + except KeyError: + pass + + v['score_logbook'] = [{'date': date_for_visibility, 'score': score, 'comment': ''}] + if auto_generated: + v['score_logbook'][0]['auto_generated'] = True + + # remove the single quotes around the date + new_lines = fix_date(yaml_file, date_for_visibility) + + return new_lines From a24f77146ec14db96f39c311e753bce93c27e6ee Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Wed, 31 Jul 2019 11:59:17 +0200 Subject: [PATCH 13/49] A pre-attack Navigator layer's filename no longer contains a platform (which it does not support) --- group_mapping.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/group_mapping.py b/group_mapping.py index cb0bc754..8df4878c 100644 --- a/group_mapping.py +++ b/group_mapping.py @@ -540,7 +540,9 @@ def generate_group_heat_map(groups, overlay, overlay_type, stage, platform, soft json_string = simplejson.dumps(layer).replace('}, ', '},\n') - if overlay: + if stage == 'pre-attack': + filename = "output/" + stage + '_' + '_'.join(groups_list) + elif overlay: filename = "output/" + stage + '_' + platform.lower() + '_' + '_'.join(groups_list) + '-overlay_' + '_'.join(overlay_list) + '_' + filter_applicable_to.replace(' ', '_') else: filename = "output/" + stage + '_' + platform.lower() + '_' + '_'.join(groups_list) From 5814446462c476691dfc34e40e202a958ba33f73 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Thu, 1 Aug 2019 15:00:09 +0200 Subject: [PATCH 14/49] Added new constants for STIX --- constants.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/constants.py b/constants.py index 1e706ed2..f6e7836c 100644 --- a/constants.py +++ b/constants.py @@ -15,6 +15,8 @@ DATA_TYPE_STIX_ALL_GROUPS = 'mitre_all_groups' DATA_TYPE_STIX_ALL_SOFTWARE = 'mitre_all_software' DATA_TYPE_STIX_ALL_RELATIONSHIPS = 'mitre_all_relationships' +DATA_TYPE_STIX_ALL_ENTERPRISE_MITIGATIONS = 'mitre_all_mitigations_enterprise' +DATA_TYPE_STIX_ALL_MOBILE_MITIGATIONS = 'mitre_all_mitigations_mobile' # Group colors COLOR_GROUP_OVERLAY_MATCH = '#f9a825' # orange From cf4a55081cec585b9da87a9f96f5d7339eebc5fd Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Thu, 1 Aug 2019 15:02:06 +0200 Subject: [PATCH 15/49] Added new functionality for Mitigations statistics --- dettect.py | 15 ++++++++++----- interactive_menu.py | 40 ++++++++++++++++++++++++++++++---------- 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/dettect.py b/dettect.py index 8f5c4243..7fe4ff82 100644 --- a/dettect.py +++ b/dettect.py @@ -131,9 +131,12 @@ def _init_menu(): help='includes: statistics on ATT&CK data source and updates on techniques' ', groups and software', aliases=['ge']) - parser_generic.add_argument('-s', '--statistics', help='get a sorted count on how much techniques are covered by a ' - 'particular data source', action='store_true') - + parser_generic.add_argument('-ds', '--datasources', help='get a sorted count on how many ATT&CK Enterprise ' + 'techniques are covered by a particular Data Source', + action='store_true') + parser_generic.add_argument('-m', '--mitigations', help='get a sorted count on how many ATT&CK Enterprise or ' + 'Mobile techniques are covered by a Mitigation', + choices=['enterprise', 'mobile']) parser_generic.add_argument('-u', '--updates', help='get a sorted list for when updates were released for ' 'techniques, groups or software', choices=['techniques', 'groups', 'software']) @@ -211,8 +214,10 @@ def _menu(menu_parser): print("[!] Filtering on 'applicable_to' is not supported for Excel output") elif args.subparser in ['generic', 'ge']: - if args.statistics: - get_statistics() + if args.datasources: + get_statistics_data_sources() + elif args.mitigations: + get_statistics_mitigations(args.mitigations) elif args.updates: get_updates(args.updates, args.sort) diff --git a/interactive_menu.py b/interactive_menu.py index 63b39096..75c3d2b8 100644 --- a/interactive_menu.py +++ b/interactive_menu.py @@ -7,8 +7,9 @@ groups = 'all' software_group = False -platform = 'Windows' -stage = 'attack' +default_platform = 'Windows' +default_stage = 'attack' +default_matrix = 'enterprise' groups_overlay = '' overlay_type = 'group' filter_applicable_to = 'all' @@ -194,12 +195,31 @@ def _menu_statistics(): Handles the Statistics functionality. :return: """ + global default_matrix _clear() print('Menu: Statistics') print('') - get_statistics() + print('Options:') + print('1. Matrix: %s' % default_matrix) + print('2. Get a sorted count on how many ATT&CK Enterprise techniques are covered by a particular Data Source.') + print('3. Get a sorted count on how many ATT&CK Enterprise or Mobile techniques are covered by a Mitigation.') + print('9. Back to main menu.') + choice = _ask_input() + if choice == '1': + print('Specify the matrix (enterprise or mobile):') + m = _ask_input().lower() + default_matrix = 'enterprise' if m == 'enterprise' else 'mobile' + elif choice == '2': + get_statistics_data_sources() + elif choice == '3': + get_statistics_mitigations(default_matrix) + elif choice == '9': + interactive_menu() + elif choice == 'q': + quit() + _wait() - interactive_menu() + _menu_statistics() def _menu_data_source(filename_ds): @@ -366,14 +386,14 @@ def _menu_groups(): Prints and handles the Threat actor group mapping functionality. :return: """ - global groups, software_group, platform, stage, groups_overlay, overlay_type, filter_applicable_to + global groups, software_group, default_platform, default_stage, groups_overlay, overlay_type, filter_applicable_to _clear() print('Menu: %s' % MENU_NAME_THREAT_ACTOR_GROUP_MAPPING) print('') print('Options:') print('1. Software group: %s' % str(software_group)) - print('2. Platform: %s' % platform) - print('3. Stage: %s' % stage) + print('2. Platform: %s' % default_platform) + print('3. Stage: %s' % default_stage) print('4. Groups: %s' % groups) print('5. Overlay: ') print(' - %s: %s' % ('File' if os.path.exists(groups_overlay) else 'Groups', groups_overlay)) @@ -390,11 +410,11 @@ def _menu_groups(): elif choice == '2': print('Specify platform (all, Linux, macOS, Windows):') p = _ask_input().lower() - platform = 'Windows' if p == 'windows' else 'Linux' if p == 'linux' else 'macOS' if p == 'macos' else 'all' + default_platform = 'Windows' if p == 'windows' else 'Linux' if p == 'linux' else 'macOS' if p == 'macos' else 'all' elif choice == '3': print('Specify stage (pre-attack, attack):') s = _ask_input().lower() - stage = 'pre-attack' if s == 'pre-attack' else 'attack' + default_stage = 'pre-attack' if s == 'pre-attack' else 'attack' elif choice == '4': print('Specify the groups to include separated using commas. Group can be their ID, name or alias ' '(default is all groups). Other option is to provide a YAML file with a custom group(s)') @@ -426,7 +446,7 @@ def _menu_groups(): print('Specify your filter for the applicable_to field:') filter_applicable_to = _ask_input().lower() elif choice == '7': - generate_group_heat_map(groups, groups_overlay, overlay_type, stage, platform, software_group, filter_applicable_to) + generate_group_heat_map(groups, groups_overlay, overlay_type, default_stage, default_platform, software_group, filter_applicable_to) _wait() elif choice == '9': interactive_menu() From 3d11aa58358683339494c97d7174fa0c34323cd1 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Thu, 1 Aug 2019 15:02:55 +0200 Subject: [PATCH 16/49] - Added new functionality for Mitigations statistics. - Moved multiple functions. --- generic.py | 133 ++++++++++++++++++++++++++++++++++++++++++++++- group_mapping.py | 79 ---------------------------- 2 files changed, 131 insertions(+), 81 deletions(-) diff --git a/generic.py b/generic.py index 4f3325eb..1b8cf1b0 100644 --- a/generic.py +++ b/generic.py @@ -57,9 +57,9 @@ def load_attack_data(data_type): attack_data = None if data_type == DATA_TYPE_STIX_ALL_RELATIONSHIPS: attack_data = mitre.get_relationships() - if data_type == DATA_TYPE_STIX_ALL_TECH_ENTERPRISE: + elif data_type == DATA_TYPE_STIX_ALL_TECH_ENTERPRISE: attack_data = mitre.get_enterprise_techniques() - if data_type == DATA_TYPE_CUSTOM_TECH_BY_GROUP: + elif data_type == DATA_TYPE_CUSTOM_TECH_BY_GROUP: # First we need to know which technique references (STIX Object type 'attack-pattern') we have for all # groups. This results in a dict: {group_id: Gxxxx, technique_ref/attack-pattern_ref: ...} groups = load_attack_data(DATA_TYPE_STIX_ALL_GROUPS) @@ -166,6 +166,12 @@ def load_attack_data(data_type): }) attack_data = all_group_use + elif data_type == DATA_TYPE_STIX_ALL_ENTERPRISE_MITIGATIONS: + attack_data = mitre.get_enterprise_mitigations() + + elif data_type == DATA_TYPE_STIX_ALL_MOBILE_MITIGATIONS: + attack_data = mitre.get_mobile_mitigations() + _save_attack_data(attack_data, "cache/" + data_type) return attack_data @@ -937,3 +943,126 @@ def check_file(filename, file_type=None, health_is_called=False): return yaml_content['file_type'] return yaml_content # value is None + +def get_updates(update_type, sort='modified'): + """ + Print a list of updates for a techniques, groups or software. Sort by modified or creation date. + :param update_type: the type of update: techniques, groups or software + :param sort: sort the list by modified or creation date + :return: + """ + if update_type[:-1] == 'technique': + techniques = load_attack_data(DATA_TYPE_STIX_ALL_TECH) + sorted_techniques = sorted(techniques, key=lambda k: k[sort]) + + for t in sorted_techniques: + print(get_attack_id(t) + ' ' + t['name']) + print(' ' * 6 + 'created: ' + t['created'].strftime('%Y-%m-%d')) + print(' ' * 6 + 'modified: ' + t['modified'].strftime('%Y-%m-%d')) + print(' ' * 6 + 'matrix: ' + t['external_references'][0]['source_name'][6:]) + tactics = get_tactics(t) + if tactics: + print(' ' * 6 + 'tactic: ' + ', '.join(tactics)) + else: + print(' ' * 6 + 'tactic: None') + print('') + + elif update_type[:-1] == 'group': + groups = load_attack_data(DATA_TYPE_STIX_ALL_GROUPS) + sorted_groups = sorted(groups, key=lambda k: k[sort]) + + for g in sorted_groups: + print(get_attack_id(g) + ' ' + g['name']) + print(' ' * 6 + 'created: ' + g['created'].strftime('%Y-%m-%d')) + print(' ' * 6 + 'modified: ' + g['modified'].strftime('%Y-%m-%d')) + print('') + + elif update_type == 'software': + software = load_attack_data(DATA_TYPE_STIX_ALL_SOFTWARE) + sorted_software = sorted(software, key=lambda k: k[sort]) + + for s in sorted_software: + print(get_attack_id(s) + ' ' + s['name']) + print(' ' * 6 + 'created: ' + s['created'].strftime('%Y-%m-%d')) + print(' ' * 6 + 'modified: ' + s['modified'].strftime('%Y-%m-%d')) + print(' ' * 6 + 'matrix: ' + s['external_references'][0]['source_name'][6:]) + print(' ' * 6 + 'type: ' + s['type']) + if 'x_mitre_platforms' in s: + print(' ' * 6 + 'platform: ' + ', '.join(s['x_mitre_platforms'])) + else: + print(' ' * 6 + 'platform: None') + print('') + + +def get_statistics_mitigations(matrix): + """ + Print out statistics related to mitigations and how many techniques they cover + :return: + """ + + if matrix == 'enterprise': + mitigations = load_attack_data(DATA_TYPE_STIX_ALL_ENTERPRISE_MITIGATIONS) + elif matrix == 'mobile': + mitigations = load_attack_data(DATA_TYPE_STIX_ALL_MOBILE_MITIGATIONS) + + mitigations_dict = dict() + for m in mitigations: + if m['external_references'][0]['external_id'].startswith('M'): + mitigations_dict[m['id']] = {'mID': m['external_references'][0]['external_id'], 'name': m['name']} + + relationships = load_attack_data(DATA_TYPE_STIX_ALL_RELATIONSHIPS) + relationships_mitigates = [r for r in relationships if r['relationship_type'] == 'mitigates'] + + # {id: {name: ..., count: ..., name: ...} } + count_dict = dict() + for r in relationships_mitigates: + src_ref = r['source_ref'] + if src_ref.startswith('course-of-action') \ + and r['target_ref'].startswith('attack-pattern') \ + and src_ref in mitigations_dict: + + m = mitigations_dict[src_ref] + if m['mID'] not in count_dict: + count_dict[m['mID']] = dict() + count_dict[m['mID']]['count'] = 1 + count_dict[m['mID']]['name'] = m['name'] + else: + count_dict[m['mID']]['count'] += 1 + + count_dict_sorted = dict(sorted(count_dict.items(), key=lambda kv: kv[1]['count'], reverse=True)) + + str_format = '{:<6s} {:<14s} {:s}' + print(str_format.format('Count', 'Mitigation ID', 'Name')) + print('-' * 60) + for k, v in count_dict_sorted.items(): + print(str_format.format(str(v['count']), k, v['name'])) + + +def get_statistics_data_sources(): + """ + Print out statistics related to data sources and how many techniques they cover. + :return: + """ + techniques = load_attack_data(DATA_TYPE_STIX_ALL_TECH) + + # {data_source: {techniques: [T0001, ...}, count: ...} + data_sources_dict = {} + for tech in techniques: + tech_id = get_attack_id(tech) + # Not every technique has a data source listed + data_sources = try_get_key(tech, 'x_mitre_data_sources') + if data_sources: + for ds in data_sources: + if ds not in data_sources_dict: + data_sources_dict[ds] = {'techniques': [tech_id], 'count': 1} + else: + data_sources_dict[ds]['techniques'].append(tech_id) + data_sources_dict[ds]['count'] += 1 + + # sort the dict on the value of 'count' + data_sources_dict_sorted = dict(sorted(data_sources_dict.items(), key=lambda kv: kv[1]['count'], reverse=True)) + str_format = '{:<6s} {:s}' + print(str_format.format('Count', 'Data Source')) + print('-'*50) + for k, v in data_sources_dict_sorted.items(): + print(str_format.format(str(v['count']), k)) \ No newline at end of file diff --git a/group_mapping.py b/group_mapping.py index 8df4878c..96bf3987 100644 --- a/group_mapping.py +++ b/group_mapping.py @@ -551,82 +551,3 @@ def generate_group_heat_map(groups, overlay, overlay_type, stage, platform, soft f.write(json_string) print('Written layer: ' + filename) - -def get_updates(update_type, sort='modified'): - """ - Print a list of updates for a techniques, groups or software. Sort by modified or creation date. - :param update_type: the type of update: techniques, groups or software - :param sort: sort the list by modified or creation date - :return: - """ - if update_type[:-1] == 'technique': - techniques = load_attack_data(DATA_TYPE_STIX_ALL_TECH) - sorted_techniques = sorted(techniques, key=lambda k: k[sort]) - - for t in sorted_techniques: - print(get_attack_id(t) + ' ' + t['name']) - print(' ' * 6 + 'created: ' + t['created'].strftime('%Y-%m-%d')) - print(' ' * 6 + 'modified: ' + t['modified'].strftime('%Y-%m-%d')) - print(' ' * 6 + 'matrix: ' + t['external_references'][0]['source_name'][6:]) - tactics = get_tactics(t) - if tactics: - print(' ' * 6 + 'tactic: ' + ', '.join(tactics)) - else: - print(' ' * 6 + 'tactic: None') - print('') - - elif update_type[:-1] == 'group': - groups = load_attack_data(DATA_TYPE_STIX_ALL_GROUPS) - sorted_groups = sorted(groups, key=lambda k: k[sort]) - - for g in sorted_groups: - print(get_attack_id(g) + ' ' + g['name']) - print(' ' * 6 + 'created: ' + g['created'].strftime('%Y-%m-%d')) - print(' ' * 6 + 'modified: ' + g['modified'].strftime('%Y-%m-%d')) - print('') - - elif update_type == 'software': - software = load_attack_data(DATA_TYPE_STIX_ALL_SOFTWARE) - sorted_software = sorted(software, key=lambda k: k[sort]) - - for s in sorted_software: - print(get_attack_id(s) + ' ' + s['name']) - print(' ' * 6 + 'created: ' + s['created'].strftime('%Y-%m-%d')) - print(' ' * 6 + 'modified: ' + s['modified'].strftime('%Y-%m-%d')) - print(' ' * 6 + 'matrix: ' + s['external_references'][0]['source_name'][6:]) - print(' ' * 6 + 'type: ' + s['type']) - if 'x_mitre_platforms' in s: - print(' ' * 6 + 'platform: ' + ', '.join(s['x_mitre_platforms'])) - else: - print(' ' * 6 + 'platform: None') - print('') - - -def get_statistics(): - """ - Print out statistics related to data sources and how many techniques they cover. - :return: - """ - techniques = load_attack_data(DATA_TYPE_STIX_ALL_TECH) - - # {data_source: {techniques: [T0001, ...}, count: ...} - data_sources_dict = {} - for tech in techniques: - tech_id = get_attack_id(tech) - # Not every technique has a data source listed - data_sources = try_get_key(tech, 'x_mitre_data_sources') - if data_sources: - for ds in data_sources: - if ds not in data_sources_dict: - data_sources_dict[ds] = {'techniques': [tech_id], 'count': 1} - else: - data_sources_dict[ds]['techniques'].append(tech_id) - data_sources_dict[ds]['count'] += 1 - - # sort the dict on the value of 'count' - data_sources_dict_sorted = dict(sorted(data_sources_dict.items(), key=lambda kv: kv[1]['count'], reverse=True)) - str_format = '{:<6s} {:s}' - print(str_format.format('Count', 'Data Source')) - print('-'*50) - for k, v in data_sources_dict_sorted.items(): - print(str_format.format(str(v['count']), k)) From 9f160a262ccae85194bd4a69b14d93504adbf58f Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Fri, 2 Aug 2019 11:29:50 +0200 Subject: [PATCH 17/49] Removed null from the YAML file lines --- .../techniques-administration-endpoints.yaml | 260 +++++++++--------- 1 file changed, 130 insertions(+), 130 deletions(-) diff --git a/sample-data/techniques-administration-endpoints.yaml b/sample-data/techniques-administration-endpoints.yaml index 2a43f116..5a2de989 100644 --- a/sample-data/techniques-administration-endpoints.yaml +++ b/sample-data/techniques-administration-endpoints.yaml @@ -16,7 +16,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -35,7 +35,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -54,7 +54,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -72,7 +72,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -95,7 +95,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -175,7 +175,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -194,7 +194,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -213,7 +213,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -232,7 +232,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -251,7 +251,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -289,7 +289,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -335,7 +335,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -354,7 +354,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -373,7 +373,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -392,7 +392,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -411,7 +411,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -430,7 +430,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -468,7 +468,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -523,7 +523,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -542,7 +542,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -561,7 +561,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -580,7 +580,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -599,7 +599,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -656,7 +656,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -693,7 +693,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -712,7 +712,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -766,7 +766,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -785,7 +785,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -840,7 +840,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -859,7 +859,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -878,7 +878,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -897,7 +897,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -916,7 +916,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -939,7 +939,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -958,7 +958,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -977,7 +977,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -996,7 +996,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1015,7 +1015,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1033,7 +1033,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1052,7 +1052,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1071,7 +1071,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1090,7 +1090,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1109,7 +1109,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1128,7 +1128,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1147,7 +1147,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1184,7 +1184,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1207,7 +1207,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1226,7 +1226,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1245,7 +1245,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1264,7 +1264,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1283,7 +1283,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1302,7 +1302,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1321,7 +1321,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1340,7 +1340,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1359,7 +1359,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1395,7 +1395,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1431,7 +1431,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1449,7 +1449,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1467,7 +1467,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1486,7 +1486,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1509,7 +1509,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1528,7 +1528,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1547,7 +1547,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1601,7 +1601,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1639,7 +1639,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1658,7 +1658,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1681,7 +1681,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1723,7 +1723,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1742,7 +1742,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1761,7 +1761,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1784,7 +1784,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1824,7 +1824,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1843,7 +1843,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1862,7 +1862,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1881,7 +1881,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1940,7 +1940,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1959,7 +1959,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1978,7 +1978,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -1997,7 +1997,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2016,7 +2016,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2035,7 +2035,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2090,7 +2090,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2109,7 +2109,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2128,7 +2128,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2147,7 +2147,7 @@ techniques: - '' comment: Model G score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2166,7 +2166,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2185,7 +2185,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2204,7 +2204,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2242,7 +2242,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2280,7 +2280,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2299,7 +2299,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2318,7 +2318,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2392,7 +2392,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2430,7 +2430,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2448,7 +2448,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2467,7 +2467,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2490,7 +2490,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2513,7 +2513,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2532,7 +2532,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2551,7 +2551,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2570,7 +2570,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2606,7 +2606,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2644,7 +2644,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2690,7 +2690,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2713,7 +2713,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2732,7 +2732,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2751,7 +2751,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2770,7 +2770,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2789,7 +2789,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2808,7 +2808,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2827,7 +2827,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2846,7 +2846,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2865,7 +2865,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2884,7 +2884,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2940,7 +2940,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2959,7 +2959,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -2996,7 +2996,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -3033,7 +3033,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -3070,7 +3070,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -3093,7 +3093,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -3112,7 +3112,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -3131,7 +3131,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -3150,7 +3150,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -3169,7 +3169,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -3188,7 +3188,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -3207,7 +3207,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -3227,7 +3227,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -3248,7 +3248,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -3269,7 +3269,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: @@ -3290,7 +3290,7 @@ techniques: - '' comment: '' score_logbook: - - date: null + - date: score: -1 comment: '' visibility: From 9100cd217550e5936eba8f3fbb5f691478965a59 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Fri, 2 Aug 2019 11:45:56 +0200 Subject: [PATCH 18/49] Changed the way the file is written. --- data_source_mapping.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data_source_mapping.py b/data_source_mapping.py index 35a8edf8..a54a9dd4 100644 --- a/data_source_mapping.py +++ b/data_source_mapping.py @@ -436,10 +436,10 @@ def update_technique_administration_file(file_data_sources, file_tech_admin): print('') backup_file(file_tech_admin) - yaml_file_tech_admin = fix_date(yaml_file_tech_admin, today, input_reamel=True, return_reamel=True) + yaml_file_tech_admin = fix_date_and_remove_null(yaml_file_tech_admin, today, input_reamel=True) with open(file_tech_admin, 'w') as fd: - _yaml.dump(yaml_file_tech_admin, fd) + fd.writelines(yaml_file_tech_admin) print('File written: ' + file_tech_admin) else: print('No visibility scores have been updated.') @@ -530,7 +530,7 @@ def generate_technique_administration_file(filename, write_file=True): _yaml.dump(yaml_file, fd_tmp) # remove the single quotes from the date - yaml_file_lines = fix_date(tmp_file, today, input_reamel=False) + yaml_file_lines = fix_date_and_remove_null(tmp_file, today, input_reamel=False) os.remove(tmp_file) From c6d25a2f0f5a0ddcf6b90929e2562c462e129857 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Fri, 2 Aug 2019 11:47:58 +0200 Subject: [PATCH 19/49] - Added functionally to remove null values from YAML file lines. - Small improvement in the health check. --- generic.py | 19 +++++++++---------- upgrade.py | 6 +++--- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/generic.py b/generic.py index 1b8cf1b0..c9de49e2 100644 --- a/generic.py +++ b/generic.py @@ -442,9 +442,10 @@ def ask_multiple_choice(question, list_answers): return list_answers[int(answer)-1] -def fix_date(yaml_file, date, input_reamel=True, return_reamel=False): +def fix_date_and_remove_null(yaml_file, date, input_reamel=True, return_reamel=False): """ Remove the single quotes around the date key-value pair in the provided yaml_file + And remove any null values :param yaml_file: ruamel.yaml instance or location of YAML file :param date: string date value (e.g. 2019-01-01) :param input_reamel: input type can be a reamel instance or list @@ -462,20 +463,18 @@ def fix_date(yaml_file, date, input_reamel=True, return_reamel=False): with open(file, 'r') as fd: new_lines = fd.readlines() - x = 0 - for line in new_lines: - if REGEX_YAML_DATE.match(line): - new_lines[x] = line.replace('\'' + date + '\'', date) - x += 1 - # remove the temporary file + fixed_lines = [l.replace('\'' + date + '\'', date).replace('null', '') + if REGEX_YAML_DATE.match(l) else + l.replace('null', '') for l in new_lines] + if input_reamel: os.remove(file) if return_reamel: - return _yaml.load(''.join(new_lines)) + return _yaml.load(''.join(fixed_lines)) else: - return new_lines + return fixed_lines def _get_latest_score_obj(yaml_object): @@ -838,7 +837,7 @@ def check_yaml_file_health(filename, file_type, health_is_called): for key in ['detection', 'visibility']: if key not in v: has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is MISSING ' + key, health_is_called) - else: + elif 'applicable_to' in v: # create at set containing all values for 'applicable_to' all_applicable_to.update([a for v in v[key] for a in v['applicable_to']]) diff --git a/upgrade.py b/upgrade.py index 869dccda..c6a037be 100644 --- a/upgrade.py +++ b/upgrade.py @@ -189,7 +189,7 @@ def _check_yaml_file_health_v11(file_lines): for key in ['detection', 'visibility']: if key not in v: has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is MISSING ' + key) - else: + elif 'applicable_to' in v: # create at set containing all values for 'applicable_to' all_applicable_to.update([a for v in v[key] for a in v['applicable_to']]) @@ -264,7 +264,7 @@ def _upgrade_technique_yaml_11_to_12(file_lines, attack_tech_data): :param attack_tech_data: Not used, but necessary to be compatible with other upgrade methods. :return: array with new lines to be written to disk """ - from generic import ask_yes_no, fix_date, init_yaml + from generic import ask_yes_no, fix_date_and_remove_null, init_yaml # we will first do a health check on the tech. admin file version 1.1. Having health issues in the file could # result in an upgraded file with errors. @@ -335,6 +335,6 @@ def _upgrade_technique_yaml_11_to_12(file_lines, attack_tech_data): v['score_logbook'][0]['auto_generated'] = True # remove the single quotes around the date - new_lines = fix_date(yaml_file, date_for_visibility) + new_lines = fix_date_and_remove_null(yaml_file, date_for_visibility) return new_lines From 8012521ab6cdfd9e6d1f3a1c72943b4470b9b6ac Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Thu, 8 Aug 2019 11:45:03 +0200 Subject: [PATCH 20/49] New functionality for the integration of EQL into DeTT&CT. --- eql_yaml.py | 435 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 435 insertions(+) create mode 100644 eql_yaml.py diff --git a/eql_yaml.py b/eql_yaml.py new file mode 100644 index 00000000..71fcaf7b --- /dev/null +++ b/eql_yaml.py @@ -0,0 +1,435 @@ +from generic import * +import datetime +import sys +from pprint import pprint +import eql +from copy import deepcopy + + +def _traverse_dict(obj, callback=None): + """ + Traverse all items in a dictionary + :param obj: dictionary, list or value + :param callback: a function that will be called to modify a value + :return: value or call callback function + """ + if isinstance(obj, dict): + value = {k: _traverse_dict(v, callback) + for k, v in obj.items()} + elif isinstance(obj, list): + value = [_traverse_dict(elem, callback) + for elem in obj] + else: + value = obj + + if callback is None: # if a callback is provided, call it to get the new value + return value + else: + return callback(value) + + +def _traverse_modify_date(obj): + """ + Modifies a datetime.date object to a string value + :param obj: dictionary + :return: function call + """ + # This will get called for every value in the structure + def _transformer(value): + if isinstance(value, datetime.date): + return str(value) + else: + return value + + return _traverse_dict(obj, callback=_transformer) + + +def _techniques_to_events(techniques, obj_type, include_all_score_objs): + """ + Transform visibility or detection objects into EQL 'events' + :param techniques: visibility or detection YAML objects within a list + :param obj_type: 'visibility' or 'detection' + :param include_all_score_objs: include all score objects within the score_logbook for the EQL query + :return: EQL 'events' + """ + technique_events = [] + + techniques = techniques['techniques'] + + for tech in techniques: + tech_id = tech['technique_id'] + tech_name = tech['technique_name'] + + # first we will make events from detections + if not isinstance(tech[obj_type], list): + tech[obj_type] = [tech[obj_type]] + + # loop over all visibility or detection objects + for d in tech[obj_type]: + app_to = d['applicable_to'] + g_comment = d['comment'] + if obj_type == 'detection': + location = d['location'] + + # latest can be set by the user using the '--latest' argument + if not isinstance(d['score_logbook'], list): + d['score_logbook'] = [d['score_logbook']] + if not include_all_score_objs: + d['score_logbook'] = [get_latest_score_obj(d)] + + # loop over all scores (if we have multiple) create the actual events for EQL + for scr_log in d['score_logbook']: + event_lvl_3 = {'comment': scr_log['comment'], 'date': scr_log['date'], 'score': scr_log['score']} + event_lvl_2 = {'applicable_to': app_to, 'comment': g_comment, 'score_logbook': event_lvl_3} + if obj_type == 'detection': + # noinspection PyUnboundLocalVariable + event_lvl_2['location'] = location + event_lvl_1 = {'event_type': 'techniques', 'technique_id': tech_id, 'technique_name': tech_name, + obj_type: event_lvl_2} + + technique_events.append(event_lvl_1) + + return technique_events + + +def _object_in_technique(obj_event, technique_yaml, obj_type): + """ + Check if the detection/visibility object already exists within the provided technique object ('technique_yaml') + :param obj_event: visibility or detection EQL 'event' + :param technique_yaml: technique object + :param obj_type: 'visibility' or 'detection' + :return: -1 if it does not exists, otherwise the index within the list (this is needed for techniques which have + multiple vicinities or detection objects due to applicable_to) + """ + app_to = obj_event['applicable_to'] + comment = obj_event['comment'] + if obj_type == 'detection': + location = obj_event['location'] + + idx = 0 + for obj in technique_yaml[obj_type]: + if obj_type == 'detection': + # noinspection PyUnboundLocalVariable + if obj['applicable_to'] == app_to and obj['comment'] == comment and obj['location'] == location: + return idx + else: + if obj['applicable_to'] == app_to and obj['comment'] == comment: + return idx + idx += 1 + + # detection not in technique object + return -1 + + +def _value_in_dict_list(dict_list, dict_key, dict_value): + """ + Checks if the provided value is present within a certain dict key against a list of dictionaries + :param dict_list: list of dictionaries + :param dict_key: key name + :param dict_value: key value to match on + :return: true or false + """ + items = set(map(lambda k: k[dict_key], dict_list)) + if dict_value in items: + return True + else: + return False + + +def _get_technique_from_list(techniques, tech_id): + """ + Get a technique object from a list of techniques objects that matches the provided technique ID + :param techniques: list of techniques + :param tech_id: technique_id + :return: technique object or None of no match is found + """ + for tech in techniques: + if tech['technique_id'] == tech_id: + return tech + return None + + +def _events_to_yaml(query_results, obj_type): + """ + Transform the EQL 'events' back to valid YAML objects + :param query_results: list with EQL 'events + :param obj_type: data_sources, detection or visibility EQL 'events' + :return: list containing YAML objects + """ + + if obj_type == 'data_sources': + try: + # Remove the event_type key. We no longer need this. + for r in query_results: + del r['event_type'] + r['date_registered'] = datetime.datetime.strptime(r['date_registered'], '%Y-%m-%d') + r['date_connected'] = datetime.datetime.strptime(r['date_connected'], '%Y-%m-%d') + except KeyError: + # When using an EQL that does not result in a dict having valid YAML objects. Trow an error. + print(EQL_INVALID_RESULT_DS) + pprint(query_results) + quit() + + return query_results + + elif obj_type in ['visibility', 'detection']: + try: + techniques_yaml = [] + # loop over all events and reconstruct the YAML file + for tech_event in query_results: + tech_id = tech_event['technique_id'] + tech_name = tech_event['technique_name'] + obj_event = tech_event[obj_type] + score_logbook_event = tech_event[obj_type]['score_logbook'] + if score_logbook_event['date']: + score_date = datetime.datetime.strptime(score_logbook_event['date'], '%Y-%m-%d') + else: + score_date = None + + # create the technique dict if not already created + if not _value_in_dict_list(techniques_yaml, 'technique_id', tech_id): + tech_yaml = { + 'technique_id': tech_id, 'technique_name': tech_name, 'detection': [], 'visibility': [] + } + techniques_yaml.append(tech_yaml) + else: + # The technique dict was already created. Get a tech. dict from the list with a specific tech. ID + tech_yaml = _get_technique_from_list(techniques_yaml, tech_id) + + # figure out if the detection/visibility dict already exists + obj_idx = _object_in_technique(obj_event, tech_yaml, obj_type) + + score_obj_yaml = {'date': score_date, 'score': score_logbook_event['score'], + 'comment': score_logbook_event['comment']} + + # The detection/visibility dict is missing. Create it. + if obj_idx == -1: + yaml_object = { + 'applicable_to': obj_event['applicable_to'], 'comment': obj_event['comment'], + 'score_logbook': [score_obj_yaml] + } + if obj_type == 'detection': + yaml_object['location'] = obj_event['location'] + + tech_yaml[obj_type].append(yaml_object) + else: + # add the a score object to the score_logbook within the proper detection object using 'obj_idx' + tech_yaml[obj_type][obj_idx]['score_logbook'].append(score_obj_yaml) + + return techniques_yaml + + except KeyError: + print(KeyError) + # When using an EQL that does not in a valid technique administration file. Trow an error. + print(EQL_INVALID_RESULT_TECH + obj_type + ' object(s):') + pprint(query_results) + quit() + + +def _merge_yaml(yaml_content_org, yaml_content_visibility=None, yaml_content_detection=None): + """ + Merge possible filtered detection and visibility objects into a valid technique administration YAML 'file' + :param yaml_content_org: original, untouched, technique administration 'file' + :param yaml_content_visibility: list of visibility YAML objects + :param yaml_content_detection: list of detection YAML objects + :return: technique administration YAML 'file' (i.e. dict) + """ + + # for both a visibility and detection objects an EQL query was provided + if yaml_content_visibility and yaml_content_detection: + techniques_yaml = [] + + # combine visibility objects with detection objects + for tech_vis in yaml_content_visibility: + detection = _get_technique_from_list(yaml_content_detection, tech_vis['technique_id']) + if detection: + detection = detection['detection'] + else: + detection = deepcopy(YAML_OBJ_DETECTION) + + new_tech = tech_vis + new_tech['detection'] = detection + techniques_yaml.append(new_tech) + + # merge detection objects into 'techniques_yaml' which were not already added by the previous step + for tech_d in yaml_content_detection: + if not _value_in_dict_list(techniques_yaml, 'technique_id', tech_d['technique_id']): + visibility = deepcopy(YAML_OBJ_VISIBILITY) + + new_tech = tech_d + new_tech['visibility'] = visibility + techniques_yaml.append(new_tech) + + # only a visibility EQL query was provided + elif yaml_content_visibility: + techniques_yaml = yaml_content_visibility + + for tech_yaml in techniques_yaml: + tech_org = _get_technique_from_list(yaml_content_org['techniques'], tech_yaml['technique_id']) + tech_yaml['detection'] = tech_org['detection'] + # only a detection EQL query was provided + elif yaml_content_detection: + techniques_yaml = yaml_content_detection + + for tech_yaml in techniques_yaml: + tech_org = _get_technique_from_list(yaml_content_org['techniques'], tech_yaml['technique_id']) + tech_yaml['visibility'] = tech_org['visibility'] + + # create the final technique administration YAML 'file'/dict + techniques_yaml_final = yaml_content_org + techniques_yaml_final['techniques'] = techniques_yaml + + return techniques_yaml_final + + +def _prepare_yaml_file(filename, obj_type, include_all_score_objs): + """ + Prepare the YAML file such that it can be used for EQL + :param filename: file location of the YAML file + :param obj_type: technique administration file ('techniques') or data source administration file ('data_sources') + :return: A dict with date fields compatible for JSON and a new key-value pair event-type + for the EQL engine + """ + _yaml = init_yaml() + + with open(filename, 'r') as yaml_file: + yaml_content = _yaml.load(yaml_file) + + yaml_content_eql = _traverse_modify_date(yaml_content) + + # add the event type for EQL + if obj_type == 'data_sources': + for item in yaml_content_eql[obj_type]: + item['event_type'] = obj_type + yaml_content_eql = yaml_content_eql['data_sources'] + + # flatten the technique administration file to events + elif obj_type in ['visibility', 'detection']: + yaml_content_eql = _techniques_to_events(yaml_content_eql, obj_type, include_all_score_objs) + + return yaml_content_eql, yaml_content + + +def _check_query_results(query_results, obj_type): + """ + Check if the EQL query provided results that + :param query_results: EQL events + :param obj_type: 'data_sources', 'visibility' or 'detection' + :return: + """ + # show an error to the user when the query resulted on zero results + result_len = len(query_results) + if result_len == 0: + error = '[!] The search returned 0 ' + obj_type + ' objects. Refine your search to return 1 or more ' \ + + obj_type + ' objects.\nExiting...' + print(error) + quit() + else: + if result_len == 1: + msg = 'The ' + obj_type + ' query executed successfully and provided ' + str(len(query_results)) + ' result.' + else: + msg = 'The ' + obj_type + ' query executed successfully and provided ' + str(len(query_results)) + ' results.' + print(msg) + + +def _execute_eql_query(events, query): + """ + Execute an EQL query against the provided events + :param events: events + :param query: EQL query + :return: the query results (i.e. filtered events) + """ + # learn and load the schema + schema = eql.Schema.learn(events) + schema.default(schema) + + query_results = [] + + def callback(results): + for event in results.events: + query_results.append(event.data) + + # create the engine and parse the query + engine = eql.PythonEngine() + with engine.schema: + try: + eql_query = eql.parse_query(query, implied_any=True, implied_base=True) + engine.add_query(eql_query) + except eql.EqlError as e: + print(e, file=sys.stderr) + print('\nTake into account the following schema:') + pprint(eql.Schema.current().schema) + sys.exit(2) + engine.add_output_hook(callback) + + # execute the query + engine.stream_events(events) + + return query_results + + +def techniques_search(filename, query_visibility=None, query_detection=None, include_all_score_objs=False): + """ + Perform an EQL search on the technique administration file. + :param filename: file location of the YAML file on disk + :param query_visibility: EQL query for the visibility YAML objects + :param query_detection: EQL query for the detection YAML objects + :param include_all_score_objs: include all score objects within the score_logbook for the EQL query + :return: a filtered technique administration YAML 'file' (i.e. dict) + """ + if query_visibility: + visibility_events, yaml_content_org = _prepare_yaml_file(filename, 'visibility', + include_all_score_objs=include_all_score_objs) + + results_visibility = _execute_eql_query(visibility_events, query_visibility) + _check_query_results(results_visibility, 'visibility') + + results_visibility_yaml = _events_to_yaml(results_visibility, 'visibility') + if query_detection: + detection_events, yaml_content_org = _prepare_yaml_file(filename, 'detection', + include_all_score_objs=include_all_score_objs) + + results_detection = _execute_eql_query(detection_events, query_detection) + _check_query_results(results_detection, 'detection') + + results_detection_yaml = _events_to_yaml(results_detection, 'detection') + + if query_visibility and query_detection: + yaml_content = _merge_yaml(yaml_content_org, results_visibility_yaml, results_detection_yaml) + elif query_visibility: + yaml_content = _merge_yaml(yaml_content_org, yaml_content_visibility=results_visibility_yaml) + elif query_detection: + yaml_content = _merge_yaml(yaml_content_org, yaml_content_detection=results_detection_yaml) + else: + return filename + + return yaml_content + + +def search(filename, file_type, query='', include_all_score_objs=False): + """ + Perform an EQL search on the provided YAML file + :param filename: file location of the YAML file on disk + :param file_type: data source administration file, ... + :param query: EQL query + :param include_all_score_objs: include all score objects within the score_logbook for the EQL query + :return: a filtered YAML 'file' (i.e. dict) + """ + + if file_type == FILE_TYPE_DATA_SOURCE_ADMINISTRATION: + obj_type = 'data_sources' + else: + return filename + + yaml_content_eql, yaml_content_org = _prepare_yaml_file(filename, obj_type, + include_all_score_objs=include_all_score_objs) + + query_results = _execute_eql_query(yaml_content_eql, query) + _check_query_results(query_results, obj_type) + + query_results_yaml = _events_to_yaml(query_results, obj_type) + + yaml_content = yaml_content_org + yaml_content[obj_type] = query_results_yaml + + return yaml_content From dbad1f54dafd26325187b4418e8417f2fd64fb1f Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Thu, 8 Aug 2019 11:49:12 +0200 Subject: [PATCH 21/49] Added several new constants. --- constants.py | 54 +++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/constants.py b/constants.py index f6e7836c..596f2904 100644 --- a/constants.py +++ b/constants.py @@ -4,7 +4,7 @@ APP_DESC = 'Detect Tactics, Techniques & Combat Threats' VERSION = '1.2.0' -EXPIRE_TIME = 60*60*24 +EXPIRE_TIME = 60 * 60 * 24 # MITRE ATT&CK data types for custom schema and STIX DATA_TYPE_CUSTOM_TECH_BY_GROUP = 'mitre_techniques_used_by_group' @@ -77,14 +77,15 @@ FILE_TYPE_GROUP_ADMINISTRATION_VERSION = 1.0 # YAML file upgrade text -FILE_TYPE_TECHNIQUE_ADMINISTRATION_UPGRADE_TEXT = {1.1: " * Adding new key 'technique_name' containing the ATT&CK technique name.\n" - " * Adding new key 'applicable_to' for both detection and visibility. Default value is ['all'].", - 1.2: " * Detection: removing the key-value pair 'date_registered'.\n" - " You will be asked if you still want to keep this key-value pair even though DeTT&CT no longer makes use of it.\n" - " * Detection: the key-value pair 'date_implemented' will be renamed to 'date'.\n" - " * Visibility: adding a new key-value pair 'date'. You will be asked on what date to fill in for the visibility scores already present.\n" - " * Detection and visibility: the key-value pairs 'score' and 'date' are moved into a 'score_logbook'.\n" - " The primary purpose of doing this is to allow you to keep track of changes in the score."} +FILE_TYPE_TECHNIQUE_ADMINISTRATION_UPGRADE_TEXT = { + 1.1: " * Adding new key 'technique_name' containing the ATT&CK technique name.\n" + " * Adding new key 'applicable_to' for both detection and visibility. Default value is ['all'].", + 1.2: " * Detection: removing the key-value pair 'date_registered'.\n" + " You will be asked if you still want to keep this key-value pair even though DeTT&CT no longer makes use of it.\n" + " * Detection: the key-value pair 'date_implemented' will be renamed to 'date'.\n" + " * Visibility: adding a new key-value pair 'date'. You will be asked on what date to fill in for the visibility scores already present.\n" + " * Detection and visibility: the key-value pairs 'score' and 'date' are moved into a 'score_logbook'.\n" + " The primary purpose of doing this is to allow you to keep track of changes in the score."} # visibility update questions and answers V_UPDATE_Q_ALL_MANUAL = 'For all most recent visibility score objects that are eligible for an update. The key-value pair \'auto-generated\' is set to \'false\' or is not present.\n' \ @@ -102,12 +103,10 @@ ' Both the current and new visibility score will be printed.' V_UPDATE_ANSWER_CANCEL = 'Cancel.' - # update actions for visibility scores V_UPDATE_ACTION_AUTO = 'auto update' V_UPDATE_ACTION_DIFF = 'the user decides to update or not' - # YAML regex REGEX_YAML_VERSION_10 = re.compile(r'^\s*version:\s+1\.0\s*$', re.IGNORECASE) REGEX_YAML_TECHNIQUE_ID = re.compile(r'^-\s+technique_id:\s+T[0-9]{4}\s*$', re.IGNORECASE) @@ -119,9 +118,42 @@ REGEX_YAML_DATE = re.compile(r'^[\s-]+date:.*$', re.IGNORECASE) REGEX_YAML_TECHNIQUE_ID_GROUP = re.compile(r'^-\s+technique_id:\s+(T[0-9]{4})\s*$', re.IGNORECASE) +# YAML objects +YAML_OBJ_VISIBILITY = {'applicable_to': ['all'], + 'comment': '', + 'score_logbook': + [ + {'date': None, + 'score': 0, + 'comment': '', + 'auto_generated': True} + ] + } +YAML_OBJ_DETECTION = {'applicable_to': ['all'], + 'location': [''], + 'comment': '', + 'score_logbook': + [ + {'date': None, + 'score': -1, + 'comment': ''} + ]} + +YAML_OBJ_TECHNIQUE = {'technique_id': '', + 'technique_name': '', + 'detection': YAML_OBJ_DETECTION, + 'visibility': YAML_OBJ_VISIBILITY} # Interactive menu MENU_NAME_DATA_SOURCE_MAPPING = 'Data source mapping' MENU_NAME_VISIBILITY_MAPPING = 'Visibility coverage mapping' MENU_NAME_DETECTION_COVERAGE_MAPPING = 'Detection coverage mapping' MENU_NAME_THREAT_ACTOR_GROUP_MAPPING = 'Threat actor group mapping' + +# EQL +EQL_INVALID_RESULT_DS = '[!] Invalid data source administration content. Check your EQL query to return data_sources object(s):' +EQL_INVALID_RESULT_TECH = '[!] Invalid technique administration content. Check your EQL query to return ' + +# Health text +HEALTH_HAS_ERROR = '[!] The below YAML file contains possible errors. It\'s recommended to check via the ' \ + '\'--health\' argument or using the option in the interactive menu: \n - ' From 3d66ab004a73c719a0e7edc036c2712dbfcd8929 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Thu, 8 Aug 2019 14:17:31 +0200 Subject: [PATCH 22/49] - Added new CLI arguments for EQL queries. - Removed the CLI argument '-a, --applicable' (replaced by EQL queries). --- dettect.py | 112 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 72 insertions(+), 40 deletions(-) diff --git a/dettect.py b/dettect.py index 7fe4ff82..8789cbe9 100644 --- a/dettect.py +++ b/dettect.py @@ -28,10 +28,13 @@ def _init_menu(): 'sources to Excel or generate a data source improvement ' 'graph.') parser_data_sources.add_argument('-ft', '--file-tech', help='path to the technique administration YAML file ' - '(used to score the level of visibility)', + '(used with the option \'-u, --update\' to update ' + 'the visibility scores)', required=False) parser_data_sources.add_argument('-fd', '--file-ds', help='path to the data source administration YAML file', required=True) + parser_data_sources.add_argument('-s', '--search', help='only include data sources which match the provided EQL ' + 'query') parser_data_sources.add_argument('-l', '--layer', help='generate a data source layer for the ATT&CK navigator', action='store_true') parser_data_sources.add_argument('-e', '--excel', help='generate an Excel sheet with all data source', @@ -44,10 +47,10 @@ def _init_menu(): parser_data_sources.add_argument('-u', '--update', help='update the visibility scores within a technique ' 'administration YAML file based on changes within any of ' 'the data sources. Past visibility scores are preserved in ' - 'the score_logbook, and manually assigned scores are not ' - 'updated without your approval. The updated visibility ' + 'the \'score_logbook\', and manually assigned scores are ' + 'not updated without your approval. The updated visibility ' 'scores are calculated in the same way as with the option: ' - '-y, --yaml.', action='store_true') + '-y, --yaml', action='store_true') # create the visibility parser parser_visibility = subparsers.add_parser('visibility', aliases=['v'], @@ -59,9 +62,14 @@ def _init_menu(): 'score the level of visibility)', required=True) parser_visibility.add_argument('-fd', '--file-ds', help='path to the data source administration YAML file (used to ' 'add metadata on the involved data sources)') - parser_visibility.add_argument('-a', '--applicable', help='filter techniques based on the \'applicable_to\' field ' - 'in the technique administration YAML file. ' - 'Not supported for Excel output', default='all') + parser_visibility.add_argument('-sd', '--search-detection', help='only include detection objects which match the ' + 'provided EQL query') + parser_visibility.add_argument('-sv', '--search-visibility', help='only include visibility objects which match the ' + 'provided EQL query') + parser_visibility.add_argument('--all-scores', help='include all \'score\' objects from the \'score_logbook\' in ' + 'the EQL search. The default behaviour is to only include the ' + 'most recent \'score\' objects', + action='store_true', default=False) parser_visibility.add_argument('-l', '--layer', help='generate a visibility layer for the ATT&CK navigator', action='store_true') parser_visibility.add_argument('-e', '--excel', help='generate an Excel sheet with all administrated techniques', @@ -82,9 +90,14 @@ def _init_menu(): parser_detection.add_argument('-fd', '--file-ds', help='path to the data source administration YAML file (used in ' 'the overlay with visibility to add metadata on the ' 'involved data sources)') - parser_detection.add_argument('-a', '--applicable', help='filter techniques based on the \'applicable_to\' field ' - 'in the technique administration YAML file. ' - 'Not supported for Excel output', default='all') + parser_detection.add_argument('-sd', '--search-detection', help='only include detection objects which match the ' + 'provided EQL query') + parser_detection.add_argument('-sv', '--search-visibility', help='only include visibility objects which match the ' + 'provided EQL query') + parser_detection.add_argument('--all-scores', help='include all \'score\' objects from the \'score_logbook\' in ' + 'the EQL search. The default behaviour is to only include the ' + 'most recent \'score\' objects', + action='store_true', default=False) parser_detection.add_argument('-l', '--layer', help='generate detection layer for the ATT&CK navigator', action='store_true') parser_detection.add_argument('-e', '--excel', help='generate an Excel sheet with all administrated techniques', @@ -100,22 +113,19 @@ def _init_menu(): description='Create threat actor group heat maps, compare group(s) and ' 'compare group(s) with visibility and detection coverage.', help='threat actor group mapping') - parser_group.add_argument('-g', '--groups', help='specify the groups to include separated using commas. ' + parser_group.add_argument('-g', '--groups', help='specify the ATT&CK Groups to include separated using commas. ' 'Group can be their ID, name or alias (default is all groups). ' 'Other option is to provide a YAML file with a custom group(s) ' '(default = all)', default='all') parser_group.add_argument('-o', '--overlay', help='specify what to overlay on the group(s) (provided using the ' 'arguments \'-g/--groups\'): group(s), visibility or detection. ' - 'When overlaying a GROUP: the group can be their ID, name or ' - 'alias separated using commas. Or provide a file path of a YAML ' - 'file with a custom group(s). When overlaying DETECTION or ' - 'VISIBILITY provide a YAML with the technique administration.') + 'When overlaying a GROUP: the group can be their ATT&CK ID, name ' + 'or alias separated using commas. Or provide a file path of a ' + 'YAML file with a custom group(s). When overlaying VISIBILITY or ' + 'DETECTION provide a YAML with the technique administration.') parser_group.add_argument('-t', '--overlay-type', help='specify the type of overlay (default = group)', choices=['group', 'visibility', 'detection'], default='group') - parser_group.add_argument('-a', '--applicable', help='filter techniques in the detection or visibility overlay ' - 'based on the \'applicable_to\' field in the technique ' - 'administration YAML file. ', default='all') parser_group.add_argument('--software-group', help='add techniques to the heat map by checking which software is ' 'used by group(s), and hence which techniques the software ' 'supports (does not influence the scores). If overlay group(s) ' @@ -125,6 +135,15 @@ def _init_menu(): choices=['all', 'Linux', 'macOS', 'Windows'], default='Windows') parser_group.add_argument('-s', '--stage', help='specify the stage (default = attack)', choices=['attack', 'pre-attack'], default='attack') + parser_group.add_argument('-sd', '--search-detection', help='only include detection objects which match the ' + 'provided EQL query') + parser_group.add_argument('-sv', '--search-visibility', help='only include visibility objects which match the ' + 'provided EQL query') + parser_group.add_argument('--all-scores', help='include all \'score\' objects from the \'score_logbook\' in ' + 'the EQL search. The default behaviour is to only include the ' + 'most recent \'score\' objects', + action='store_true', default=False) + parser_group.add_argument('--health', help='check the technique YAML file for errors', action='store_true') # create the generic parser parser_generic = subparsers.add_parser('generic', description='Generic functions which will output to stdout.', @@ -160,58 +179,71 @@ def _menu(menu_parser): elif args.subparser in ['datasource', 'ds']: if check_file(args.file_ds, FILE_TYPE_DATA_SOURCE_ADMINISTRATION): + file_ds = args.file_ds + + if args.search: + file_ds = search(args.file_ds, FILE_TYPE_DATA_SOURCE_ADMINISTRATION, args.search) if args.update and check_file(args.file_tech, FILE_TYPE_TECHNIQUE_ADMINISTRATION): - update_technique_administration_file(args.file_ds, args.file_tech) + update_technique_administration_file(file_ds, args.file_tech) if args.layer: - generate_data_sources_layer(args.file_ds) + generate_data_sources_layer(file_ds) if args.excel: - export_data_source_list_to_excel(args.file_ds) + export_data_source_list_to_excel(file_ds) if args.graph: - plot_data_sources_graph(args.file_ds) + plot_data_sources_graph(file_ds) if args.yaml: - generate_technique_administration_file(args.file_ds) + generate_technique_administration_file(file_ds) elif args.subparser in ['visibility', 'v']: if args.layer or args.overlay: if not args.file_ds: - print('[!] Generating a visibility layer or doing an overlay requires adding the data source' - 'administration YAML file (\'--file-ds\')') + print('[!] Generating a visibility layer or an overlay requires the data source ' + 'administration YAML file (\'-fd, --file-ds\')') quit() if not check_file(args.file_ds, FILE_TYPE_DATA_SOURCE_ADMINISTRATION, args.health): quit() if check_file(args.file_tech, FILE_TYPE_TECHNIQUE_ADMINISTRATION, args.health): + file_tech = args.file_tech + + if args.search_detection or args.search_visibility: + file_tech = techniques_search(args.file_tech, args.search_visibility, args.search_detection, + include_all_score_objs=args.all_scores) if args.layer: - generate_visibility_layer(args.file_tech, args.file_ds, False, args.applicable) + generate_visibility_layer(file_tech, args.file_ds, False) if args.overlay: - generate_visibility_layer(args.file_tech, args.file_ds, True, args.applicable) - if args.excel and args.applicable == 'all': - export_techniques_list_to_excel(args.file_tech) - if args.excel and args.applicable != 'all': - print('[!] Filtering on \'applicable_to\' is not supported for Excel output') + generate_visibility_layer(file_tech, args.file_ds, True) + if args.excel: + export_techniques_list_to_excel(file_tech) + # toto add search capabilities elif args.subparser in ['group', 'g']: - generate_group_heat_map(args.groups, args.overlay, args.overlay_type, args.stage, args.platform, args.software_group, args.applicable) + generate_group_heat_map(args.groups, args.overlay, args.overlay_type, args.stage, args.platform, + args.software_group, args.search_visibility, args.search_detection, args.health, + include_all_score_objs=args.all_scores) elif args.subparser in ['detection', 'd']: if args.overlay: if not args.file_ds: - print('[!] Doing an overlay requires adding the data source administration YAML file (\'--file-ds\')') + print('[!] An overlay requires the data source administration YAML file (\'-fd, --file-ds\')') quit() if not check_file(args.file_ds, FILE_TYPE_DATA_SOURCE_ADMINISTRATION, args.health): quit() if check_file(args.file_tech, FILE_TYPE_TECHNIQUE_ADMINISTRATION, args.health): + file_tech = args.file_tech + + if args.search_detection or args.search_visibility: + file_tech = techniques_search(args.file_tech, args.search_visibility, args.search_detection, + include_all_score_objs=args.all_scores) if args.layer: - generate_detection_layer(args.file_tech, args.file_ds, False, args.applicable) + generate_detection_layer(file_tech, args.file_ds, False) if args.overlay and check_file(args.file_ds, FILE_TYPE_DATA_SOURCE_ADMINISTRATION, args.health): - generate_detection_layer(args.file_tech, args.file_ds, True, args.applicable) + generate_detection_layer(file_tech, args.file_ds, True) if args.graph: - plot_detection_graph(args.file_tech, args.applicable) - if args.excel and args.applicable == 'all': - export_techniques_list_to_excel(args.file_tech) - if args.excel and args.applicable != 'all': - print("[!] Filtering on 'applicable_to' is not supported for Excel output") + plot_detection_graph(file_tech) + if args.excel: + export_techniques_list_to_excel(file_tech) elif args.subparser in ['generic', 'ge']: if args.datasources: From 025c302af54a6554824bfe4e92985a7e2e948af6 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Thu, 8 Aug 2019 14:29:15 +0200 Subject: [PATCH 23/49] - Removed the function 'try_get_key' (replaced by the native dict method 'get'). - Improved the function 'fix_date_and_remove_null' to make use of StringIO instead of writing temporary files to disk. - Made the function 'get_latest_score_obj', 'public'. This function is needed within the module 'eql_yaml.py'. - Removed functionality for the deprecated argument '-a, --applicable'. - Added a try/except block to 'load_techniques', for when an EQL query resulted in invalid technique administration YAML content. - Improved the health check to only to perform the health check, when the content of the YAML file changed. This results in a notable increase in performance. --- generic.py | 372 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 214 insertions(+), 158 deletions(-) diff --git a/generic.py b/generic.py index c9de49e2..babee749 100644 --- a/generic.py +++ b/generic.py @@ -1,7 +1,8 @@ import os import shutil import pickle -import sys +from io import StringIO +from pprint import pprint from ruamel.yaml import YAML from difflib import SequenceMatcher from datetime import datetime as dt @@ -11,18 +12,6 @@ # Due to performance reasons the import of attackcti is within the function that makes use of this library. -def try_get_key(dictionary, key): - """ - Return None if the key does not exists within the provided dict - :param dictionary: dictionary - :param key: key - :return: key value or None - """ - if key in dictionary: - return dictionary[key] - return None - - def _save_attack_data(data, path): """ Save ATT&CK data to disk for the purpose of caching. Data can be STIX objects our a custom schema. @@ -74,7 +63,7 @@ def load_attack_data(data_type): { 'group_id': get_attack_id(g), 'name': g['name'], - 'aliases': try_get_key(g, 'aliases'), + 'aliases': g.get('aliases', None), 'technique_ref': r['target_ref'] }) @@ -91,7 +80,7 @@ def load_attack_data(data_type): 'name': gr['name'], 'aliases': gr['aliases'], 'technique_id': get_attack_id(t), - 'x_mitre_platforms': try_get_key(t, 'x_mitre_platforms'), + 'x_mitre_platforms': t.get('x_mitre_platforms', None), 'matrix': t['external_references'][0]['source_name'] }) @@ -144,7 +133,7 @@ def load_attack_data(data_type): { 'group_id': get_attack_id(g), 'name': g['name'], - 'aliases': try_get_key(g, 'aliases'), + 'aliases': g.get('aliases', None), 'software_ref': r['target_ref'] }) @@ -161,7 +150,7 @@ def load_attack_data(data_type): 'name': gr['name'], 'aliases': gr['aliases'], 'software_id': get_attack_id(s), - 'x_mitre_platforms': try_get_key(s, 'x_mitre_platforms'), + 'x_mitre_platforms': s.get('x_mitre_platforms', None), 'matrix': s['external_references'][0]['source_name'] }) attack_data = all_group_use @@ -442,42 +431,34 @@ def ask_multiple_choice(question, list_answers): return list_answers[int(answer)-1] -def fix_date_and_remove_null(yaml_file, date, input_reamel=True, return_reamel=False): +def fix_date_and_remove_null(yaml_file, date, input_type='ruamel'): """ - Remove the single quotes around the date key-value pair in the provided yaml_file - And remove any null values + Remove the single quotes around the date key-value pair in the provided yaml_file and remove any 'null' values :param yaml_file: ruamel.yaml instance or location of YAML file :param date: string date value (e.g. 2019-01-01) - :param input_reamel: input type can be a reamel instance or list - :param return_reamel: return list of YAML file lines or reamel instance + :param input_type: input type can be a ruamel.yaml instance or list :return: YAML file lines in a list """ _yaml = init_yaml() - if input_reamel: - file = sys.path[0] + '/.tmp_tech_file' - - with open(file, 'w') as fd: - _yaml.dump(yaml_file, fd) - else: - file = yaml_file - - with open(file, 'r') as fd: - new_lines = fd.readlines() + if input_type == 'ruamel': + # ruamel does not support output to a variable. Therefore we make use of StringIO. + file = StringIO() + _yaml.dump(yaml_file, file) + file.seek(0) + new_lines = file.readlines() + elif input_type == 'list': + new_lines = yaml_file + elif input_type == 'file': + new_lines = yaml_file.readlines() fixed_lines = [l.replace('\'' + date + '\'', date).replace('null', '') if REGEX_YAML_DATE.match(l) else l.replace('null', '') for l in new_lines] - if input_reamel: - os.remove(file) - - if return_reamel: - return _yaml.load(''.join(fixed_lines)) - else: - return fixed_lines + return fixed_lines -def _get_latest_score_obj(yaml_object): +def get_latest_score_obj(yaml_object): """ Get the the score object in the score_logbook by date :param yaml_object: a detection or visibility YAML object @@ -507,7 +488,7 @@ def get_latest_comment(yaml_object, empty=' '): :param empty: value for an empty comment :return: comment """ - score_obj = _get_latest_score_obj(yaml_object) + score_obj = get_latest_score_obj(yaml_object) if score_obj: if score_obj['comment'] == '' or not score_obj['comment']: return empty @@ -523,7 +504,7 @@ def get_latest_date(yaml_object): :param yaml_object: a detection or visibility YAML object :return: date as a datetime object or None """ - score_obj = _get_latest_score_obj(yaml_object) + score_obj = get_latest_score_obj(yaml_object) if score_obj: return score_obj['date'] else: @@ -536,7 +517,7 @@ def get_latest_auto_generated(yaml_object): :param yaml_object: a detection or visibility YAML object :return: True or False """ - score_obj = _get_latest_score_obj(yaml_object) + score_obj = get_latest_score_obj(yaml_object) if score_obj: if 'auto_generated' in score_obj: return score_obj['auto_generated'] @@ -552,7 +533,7 @@ def get_latest_score(yaml_object): :param yaml_object: a detection or visibility YAML object :return: score as an integer or None """ - score_obj = _get_latest_score_obj(yaml_object) + score_obj = get_latest_score_obj(yaml_object) if score_obj: return score_obj['score'] else: @@ -588,7 +569,7 @@ def map_techniques_to_data_sources(techniques, my_data_sources): my_techniques[tech_id]['my_data_sources'] = [i_ds, ] my_techniques[tech_id]['data_sources'] = t['x_mitre_data_sources'] # create a list of tactics - my_techniques[tech_id]['tactics'] = list(map(lambda k: k['phase_name'], try_get_key(t, 'kill_chain_phases'))) + my_techniques[tech_id]['tactics'] = list(map(lambda k: k['phase_name'], t.get('kill_chain_phases', None))) my_techniques[tech_id]['products'] = set(my_data_sources[i_ds]['products']) elif t['x_mitre_data_sources'] and i_ds in t['x_mitre_data_sources'] and tech_id in my_techniques.keys(): my_techniques[tech_id]['my_data_sources'].append(i_ds) @@ -626,6 +607,7 @@ def calculate_score(list_detections, zero_value=0): if score >= 0: avg_score += score number += 1 + avg_score = int(round(avg_score / number, 0) if number > 0 else zero_value) return avg_score @@ -647,41 +629,47 @@ def add_entry_to_list_in_dictionary(dictionary, technique_id, key, entry): dictionary[technique_id][key].append(entry) -def load_techniques(filename, detection_or_visibility='all', filter_applicable_to='all'): +def load_techniques(file): """ - Loads the techniques (including detection and visibility properties) from the given yaml file. - :param filename: the filename of the yaml file containing the techniques administration - :param detection_or_visibility: used to indicate to filter applicable_to field for detection or visibility. When - using 'all' no filtering will be applied. - :param filter_applicable_to: filter techniques based on applicable_to field in techniques administration YAML file + Loads the techniques (including detection and visibility properties). + :param file: the file location of the YAML file or a dict containing the techniques administration :return: dictionary with techniques (incl. properties), name and platform """ - my_techniques = {} - _yaml = init_yaml() - with open(filename, 'r') as yaml_file: - yaml_content = _yaml.load(yaml_file) + + if isinstance(file, dict): + # file is a dict and created due to the use of an EQL query by the user + yaml_content = file + else: + # file is a file location on disk + _yaml = init_yaml() + with open(file, 'r') as yaml_file: + yaml_content = _yaml.load(yaml_file) + + try: for d in yaml_content['techniques']: # Add detection items: if isinstance(d['detection'], dict): # There is just one detection entry - if detection_or_visibility == 'all' or filter_applicable_to == 'all' or filter_applicable_to in d[detection_or_visibility]['applicable_to'] or 'all' in d[detection_or_visibility]['applicable_to']: - add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'detection', d['detection']) + add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'detection', d['detection']) elif isinstance(d['detection'], list): # There are multiple detection entries for de in d['detection']: - if detection_or_visibility == 'all' or filter_applicable_to == 'all' or filter_applicable_to in de['applicable_to'] or 'all' in de['applicable_to']: - add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'detection', de) + add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'detection', de) # Add visibility items if isinstance(d['visibility'], dict): # There is just one visibility entry - if detection_or_visibility == 'all' or filter_applicable_to == 'all' or filter_applicable_to in d[detection_or_visibility]['applicable_to'] or 'all' in d[detection_or_visibility]['applicable_to']: - add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'visibility', d['visibility']) + add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'visibility', d['visibility']) elif isinstance(d['visibility'], list): # There are multiple visibility entries for de in d['visibility']: - if detection_or_visibility == 'all' or filter_applicable_to == 'all' or filter_applicable_to in de['applicable_to'] or 'all' in de['applicable_to']: - add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'visibility', de) + add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'visibility', de) + + name = yaml_content['name'] + platform = yaml_content['platform'] + except KeyError: + # When using an EQL that does not in a valid technique administration file. Trow an error. + print(EQL_INVALID_RESULT_TECH + ' detection/visibility object(s):') + pprint(yaml_content) + quit() - name = yaml_content['name'] - platform = yaml_content['platform'] return my_techniques, name, platform @@ -749,7 +737,7 @@ def _check_health_score_object(yaml_object, object_type, tech_id, health_is_call # noinspection PyStatementEffect score_obj['date'].day except AttributeError: - has_error = _print_error_msg('[!] Technique ID: ' + tech_id + ' has an INVALID data format in a ' + object_type + ' score object in the \'score_logbook\': date (should be YYYY-MM-DD)', health_is_called) + has_error = _print_error_msg('[!] Technique ID: ' + tech_id + ' has an INVALID data format in a ' + object_type + ' score object in the \'score_logbook\': date (should be YYYY-MM-DD without quotes)', health_is_called) except KeyError: pass @@ -789,13 +777,76 @@ def _check_health_yaml_object(yaml_object, object_type, tech_id, health_is_calle return has_error -def _update_health_sate(current, update): +def _update_health_state(current, update): if current or update: return True else: return update +def _is_file_modified(filename): + """ + Check if the provided file was modified since the last check + :param filename: file location + :return: true when modified else false + """ + last_modified_file = 'cache/last-modified_' + os.path.basename(filename).rstrip('.yaml') + + def _update_modified_date(date): + with open(last_modified_file, 'wb') as fd: + pickle.dump(date, fd) + + if not os.path.exists(last_modified_file): + last_modified = os.path.getmtime(filename) + _update_modified_date(last_modified) + + return True + else: + with open(last_modified_file, 'rb') as f: + last_modified_cache = pickle.load(f) + last_modified_current = os.path.getmtime(filename) + + if last_modified_cache != last_modified_current: + _update_modified_date(last_modified_current) + return True + else: + return False + + +def _get_health_state_cache(filename): + """ + Get file health state from disk + :param filename: file location + :return: the cached error state + """ + last_error_file = 'cache/last-error-state_' + os.path.basename(filename).rstrip('.yaml') + + if os.path.exists(last_error_file): + with open(last_error_file, 'rb') as f: + last_error_state_cache = pickle.load(f) + + return last_error_state_cache + + +def _update_health_state_cache(filename, has_error): + """ + Write the file health state to disk if changed + :param filename: file location + """ + last_error_file = 'cache/last-error-state_' + os.path.basename(filename).rstrip('.yaml') + + def _update(error): + with open(last_error_file, 'wb') as fd: + pickle.dump(error, fd) + + if not os.path.exists(last_error_file): + _update(has_error) + else: + error_state_cache = _get_health_state_cache(filename) + if error_state_cache != has_error: + _update(has_error) + + def check_yaml_file_health(filename, file_type, health_is_called): """ Check on error in the provided YAML file. @@ -804,83 +855,86 @@ def check_yaml_file_health(filename, file_type, health_is_called): :param health_is_called: boolean that specifies if detailed errors in the file will be printed and then quit() :return: """ - - has_error = False - if file_type == FILE_TYPE_TECHNIQUE_ADMINISTRATION: - # check for duplicate tech IDs - _yaml = init_yaml() - with open(filename, 'r') as yaml_file: - yaml_content = _yaml.load(yaml_file) - - tech_ids = list(map(lambda x: x['technique_id'], yaml_content['techniques'])) - tech_dup = [] - for tech in tech_ids: - if tech not in tech_dup: - tech_dup.append(tech) - else: - has_error = _print_error_msg('[!] Duplicate technique ID: ' + tech, health_is_called) - - # check if the technique has a valid format - if not REGEX_YAML_TECHNIQUE_ID_FORMAT.match(tech): - has_error = _print_error_msg('[!] Invalid technique ID: ' + tech, health_is_called) - - # checks on: - # - empty key-value pairs: 'applicable_to', 'comment', 'location', 'score_logbook' , 'date', 'score' - # - invalid date format for: 'date' - # - detection or visibility score out-of-range - # - missing key-value pairs: 'applicable_to', 'comment', 'location', 'score_logbook', 'date', 'score' - # - check on 'applicable_to' values which are very similar - - all_applicable_to = set() - techniques = load_techniques(filename) - for tech, v in techniques[0].items(): - for key in ['detection', 'visibility']: - if key not in v: - has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is MISSING ' + key, health_is_called) - elif 'applicable_to' in v: - # create at set containing all values for 'applicable_to' - all_applicable_to.update([a for v in v[key] for a in v['applicable_to']]) - - for detection in v['detection']: - for key in ['applicable_to', 'location', 'comment', 'score_logbook']: - if key not in detection: - has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is MISSING a key-value pair in detection: ' + key, health_is_called) - - health = _check_health_yaml_object(detection, 'detection', tech, health_is_called) - has_error = _update_health_sate(has_error, health) - health = _check_health_score_object(detection, 'detection', tech, health_is_called) - has_error = _update_health_sate(has_error, health) - - for visibility in v['visibility']: - for key in ['applicable_to', 'comment', 'score_logbook']: - if key not in visibility: - has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is MISSING a key-value pair in visibility: ' + key, health_is_called) - - health = _check_health_yaml_object(visibility, 'visibility', tech, health_is_called) - has_error = _update_health_sate(has_error, health) - health = _check_health_score_object(visibility, 'visibility', tech, health_is_called) - has_error = _update_health_sate(has_error, health) - - # get values within the key-value pair 'applicable_to' which are a very close match - similar = set() - for i1 in all_applicable_to: - for i2 in all_applicable_to: - match_value = SequenceMatcher(None, i1, i2).ratio() - if match_value > 0.8 and match_value != 1: - similar.add(i1) - similar.add(i2) - - if len(similar) > 0: - has_error = _print_error_msg('[!] There are values in the key-value pair \'applicable_to\' which are very similar. Correct where necessary:', health_is_called) - for s in similar: - _print_error_msg(' - ' + s, health_is_called) - - if has_error and not health_is_called: - print('[!] The below YAML file contains possible errors. It\'s recommended to check via the \'--health\' ' - 'argument or using the option in the interactive menu: \n - ' + filename) - - if has_error: - print('') # print a newline + # first we check if the file was modified. Otherwise, the health check is skipped for performance reasons + if _is_file_modified(filename) or health_is_called: + has_error = False + if file_type == FILE_TYPE_TECHNIQUE_ADMINISTRATION: + # check for duplicate tech IDs + _yaml = init_yaml() + with open(filename, 'r') as yaml_file: + yaml_content = _yaml.load(yaml_file) + + tech_ids = list(map(lambda x: x['technique_id'], yaml_content['techniques'])) + tech_dup = [] + for tech in tech_ids: + if tech not in tech_dup: + tech_dup.append(tech) + else: + has_error = _print_error_msg('[!] Duplicate technique ID: ' + tech, health_is_called) + + # check if the technique has a valid format + if not REGEX_YAML_TECHNIQUE_ID_FORMAT.match(tech): + has_error = _print_error_msg('[!] Invalid technique ID: ' + tech, health_is_called) + + # checks on: + # - empty key-value pairs: 'applicable_to', 'comment', 'location', 'score_logbook' , 'date', 'score' + # - invalid date format for: 'date' + # - detection or visibility score out-of-range + # - missing key-value pairs: 'applicable_to', 'comment', 'location', 'score_logbook', 'date', 'score' + # - check on 'applicable_to' values which are very similar + + all_applicable_to = set() + techniques = load_techniques(filename) + for tech, v in techniques[0].items(): + for key in ['detection', 'visibility']: + if key not in v: + has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is MISSING ' + key, health_is_called) + elif 'applicable_to' in v: + # create at set containing all values for 'applicable_to' + all_applicable_to.update([a for v in v[key] for a in v['applicable_to']]) + + for detection in v['detection']: + for key in ['applicable_to', 'location', 'comment', 'score_logbook']: + if key not in detection: + has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is MISSING a key-value pair in detection: ' + key, health_is_called) + + health = _check_health_yaml_object(detection, 'detection', tech, health_is_called) + has_error = _update_health_state(has_error, health) + health = _check_health_score_object(detection, 'detection', tech, health_is_called) + has_error = _update_health_state(has_error, health) + + for visibility in v['visibility']: + for key in ['applicable_to', 'comment', 'score_logbook']: + if key not in visibility: + has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is MISSING a key-value pair in visibility: ' + key, health_is_called) + + health = _check_health_yaml_object(visibility, 'visibility', tech, health_is_called) + has_error = _update_health_state(has_error, health) + health = _check_health_score_object(visibility, 'visibility', tech, health_is_called) + has_error = _update_health_state(has_error, health) + + # get values within the key-value pair 'applicable_to' which are a very close match + similar = set() + for i1 in all_applicable_to: + for i2 in all_applicable_to: + match_value = SequenceMatcher(None, i1, i2).ratio() + if match_value > 0.8 and match_value != 1: + similar.add(i1) + similar.add(i2) + + if len(similar) > 0: + has_error = _print_error_msg('[!] There are values in the key-value pair \'applicable_to\' which are very similar. Correct where necessary:', health_is_called) + for s in similar: + _print_error_msg(' - ' + s, health_is_called) + + if has_error and not health_is_called: + print(HEALTH_HAS_ERROR + filename) + elif has_error and health_is_called: + print(' - ' + filename) + + _update_health_state_cache(filename, has_error) + elif _get_health_state_cache(filename): + print(HEALTH_HAS_ERROR + filename) def _check_file_type(filename, file_type=None): @@ -943,6 +997,7 @@ def check_file(filename, file_type=None, health_is_called=False): return yaml_content # value is None + def get_updates(update_type, sort='modified'): """ Print a list of updates for a techniques, groups or software. Sort by modified or creation date. @@ -1010,23 +1065,24 @@ def get_statistics_mitigations(matrix): mitigations_dict[m['id']] = {'mID': m['external_references'][0]['external_id'], 'name': m['name']} relationships = load_attack_data(DATA_TYPE_STIX_ALL_RELATIONSHIPS) - relationships_mitigates = [r for r in relationships if r['relationship_type'] == 'mitigates'] + relationships_mitigates = [r for r in relationships + if r['relationship_type'] == 'mitigates' + if r['source_ref'].startswith('course-of-action') + if r['target_ref'].startswith('attack-pattern') + if r['source_ref'] in mitigations_dict] # {id: {name: ..., count: ..., name: ...} } count_dict = dict() for r in relationships_mitigates: src_ref = r['source_ref'] - if src_ref.startswith('course-of-action') \ - and r['target_ref'].startswith('attack-pattern') \ - and src_ref in mitigations_dict: - - m = mitigations_dict[src_ref] - if m['mID'] not in count_dict: - count_dict[m['mID']] = dict() - count_dict[m['mID']]['count'] = 1 - count_dict[m['mID']]['name'] = m['name'] - else: - count_dict[m['mID']]['count'] += 1 + + m = mitigations_dict[src_ref] + if m['mID'] not in count_dict: + count_dict[m['mID']] = dict() + count_dict[m['mID']]['count'] = 1 + count_dict[m['mID']]['name'] = m['name'] + else: + count_dict[m['mID']]['count'] += 1 count_dict_sorted = dict(sorted(count_dict.items(), key=lambda kv: kv[1]['count'], reverse=True)) @@ -1049,7 +1105,7 @@ def get_statistics_data_sources(): for tech in techniques: tech_id = get_attack_id(tech) # Not every technique has a data source listed - data_sources = try_get_key(tech, 'x_mitre_data_sources') + data_sources = tech.get('x_mitre_data_sources', None) if data_sources: for ds in data_sources: if ds not in data_sources_dict: @@ -1064,4 +1120,4 @@ def get_statistics_data_sources(): print(str_format.format('Count', 'Data Source')) print('-'*50) for k, v in data_sources_dict_sorted.items(): - print(str_format.format(str(v['count']), k)) \ No newline at end of file + print(str_format.format(str(v['count']), k)) From 7a0aedb2a3e9406f2189ec82b01b211a18e9004c Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Thu, 8 Aug 2019 14:32:36 +0200 Subject: [PATCH 24/49] Removed unnecessary arguments to the call of the function 'fix_date_and_remove_null'. --- upgrade.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/upgrade.py b/upgrade.py index c6a037be..9f2b6c4a 100644 --- a/upgrade.py +++ b/upgrade.py @@ -3,8 +3,8 @@ def _load_techniques(yaml_file_lines): """ - Loads the techniques (including detection and visibility properties) from the given yaml file. - :param yaml_file_lines: list with the yaml file lines containing the techniques administration + Loads the techniques (including detection and visibility properties) from the given YAML file. + :param yaml_file_lines: list with the YAML file lines containing the techniques administration :return: dictionary with techniques (incl. properties) """ from generic import add_entry_to_list_in_dictionary, init_yaml @@ -112,7 +112,7 @@ def upgrade_yaml_file(filename, file_type, file_version, attack_tech_data): print('Written upgraded file: ' + filename) print('\nUpgrade complete') - print('-'*80) + print('-' * 80) def _upgrade_technique_yaml_10_to_11(file_lines, attack_tech_data): @@ -335,6 +335,6 @@ def _upgrade_technique_yaml_11_to_12(file_lines, attack_tech_data): v['score_logbook'][0]['auto_generated'] = True # remove the single quotes around the date - new_lines = fix_date_and_remove_null(yaml_file, date_for_visibility) + new_lines = fix_date_and_remove_null(yaml_file, date_for_visibility, input_type='ruamel') return new_lines From 1d2fd69a5b953c68db7a89451560a37c5a2d1bdf Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Thu, 8 Aug 2019 14:41:34 +0200 Subject: [PATCH 25/49] - Removed functionality due to the deprecation of the argument '-a, --applicable'. - Renamed the Excel column 'General comment' to 'Technique comment'. - Improved the function '_load_data_sources' to make use of StringIO instead of writing a temporary file to disk. - Before the Excel file is created, it is made sure that the date is written in the following format "%Y-%m%d". This is necessary due to the new EQL query functionality. - Added a try/except block to '_load_data_sources', for when an EQL query resulted in invalid data source administration YAML content. --- technique_mapping.py | 156 +++++++++++++++++++++++-------------------- 1 file changed, 82 insertions(+), 74 deletions(-) diff --git a/technique_mapping.py b/technique_mapping.py index ac66dd27..e2294326 100644 --- a/technique_mapping.py +++ b/technique_mapping.py @@ -1,62 +1,61 @@ import simplejson from generic import * import xlsxwriter +from pprint import pprint +from datetime import datetime # Imports for pandas and plotly are because of performance reasons in the function that uses these libraries. -def generate_detection_layer(filename_techniques, filename_data_sources, overlay, filter_applicable_to): +def generate_detection_layer(filename_techniques, filename_data_sources, overlay): """ Generates layer for detection coverage and optionally an overlaid version with visibility coverage. - :param filename_techniques: the filename of the yaml file containing the techniques administration - :param filename_data_sources: the filename of the yaml file containing the data sources administration + :param filename_techniques: the filename of the YAML file containing the techniques administration + :param filename_data_sources: the filename of the YAML file containing the data sources administration :param overlay: boolean value to specify if an overlay between detection and visibility should be generated - :param filter_applicable_to: filter techniques based on applicable_to field in techniques administration YAML file :return: """ if not overlay: - my_techniques, name, platform = load_techniques(filename_techniques, 'detection', filter_applicable_to) + my_techniques, name, platform = load_techniques(filename_techniques) mapped_techniques_detection = _map_and_colorize_techniques_for_detections(my_techniques) - layer_detection = get_layer_template_detections('Detections ' + name + ' ' + filter_applicable_to, 'description', 'attack', platform) - _write_layer(layer_detection, mapped_techniques_detection, 'detection', filter_applicable_to, name) + layer_detection = get_layer_template_detections('Detections ' + name, 'description', 'attack', platform) + _write_layer(layer_detection, mapped_techniques_detection, 'detection', name) else: - my_techniques, name, platform = load_techniques(filename_techniques, 'all', filter_applicable_to) + my_techniques, name, platform = load_techniques(filename_techniques) my_data_sources = _load_data_sources(filename_data_sources) - mapped_techniques_both = _map_and_colorize_techniques_for_overlaid(my_techniques, my_data_sources, filter_applicable_to) - layer_both = get_layer_template_layered('Visibility and Detection ' + name + ' ' + filter_applicable_to, 'description', 'attack', platform) - _write_layer(layer_both, mapped_techniques_both, 'visibility_and_detection', filter_applicable_to, name) + mapped_techniques_both = _map_and_colorize_techniques_for_overlaid(my_techniques, my_data_sources) + layer_both = get_layer_template_layered('Visibility and Detection ' + name, 'description', 'attack', platform) + _write_layer(layer_both, mapped_techniques_both, 'visibility_and_detection', name) -def generate_visibility_layer(filename_techniques, filename_data_sources, overlay, filter_applicable_to): +def generate_visibility_layer(filename_techniques, filename_data_sources, overlay): """ Generates layer for visibility coverage and optionally an overlaid version with detection coverage. - :param filename_techniques: the filename of the yaml file containing the techniques administration - :param filename_data_sources: the filename of the yaml file containing the data sources administration + :param filename_techniques: the filename of the YAML file containing the techniques administration + :param filename_data_sources: the filename of the YAML file containing the data sources administration :param overlay: boolean value to specify if an overlay between detection and visibility should be generated - :param filter_applicable_to: filter techniques based on applicable_to field in techniques administration YAML file :return: """ my_data_sources = _load_data_sources(filename_data_sources) if not overlay: - my_techniques, name, platform = load_techniques(filename_techniques, 'visibility', filter_applicable_to) + my_techniques, name, platform = load_techniques(filename_techniques) mapped_techniques_visibility = _map_and_colorize_techniques_for_visibility(my_techniques, my_data_sources) - layer_visibility = get_layer_template_visibility('Visibility ' + name + ' ' + filter_applicable_to, 'description', 'attack', platform) - _write_layer(layer_visibility, mapped_techniques_visibility, 'visibility', filter_applicable_to, name) + layer_visibility = get_layer_template_visibility('Visibility ' + name, 'description', 'attack', platform) + _write_layer(layer_visibility, mapped_techniques_visibility, 'visibility', name) else: - my_techniques, name, platform = load_techniques(filename_techniques, 'all', filter_applicable_to) - mapped_techniques_both = _map_and_colorize_techniques_for_overlaid(my_techniques, my_data_sources, filter_applicable_to) - layer_both = get_layer_template_layered('Visibility and Detection ' + name + ' ' + filter_applicable_to, 'description', 'attack', platform) - _write_layer(layer_both, mapped_techniques_both, 'visibility_and_detection', filter_applicable_to, name) + my_techniques, name, platform = load_techniques(filename_techniques) + mapped_techniques_both = _map_and_colorize_techniques_for_overlaid(my_techniques, my_data_sources) + layer_both = get_layer_template_layered('Visibility and Detection ' + name, 'description', 'attack', platform) + _write_layer(layer_both, mapped_techniques_both, 'visibility_and_detection', name) -def plot_detection_graph(filename, filter_applicable_to): +def plot_detection_graph(filename): """ Generates a line graph which shows the improvements on detections through the time. - :param filename: the filename of the yaml file containing the techniques administration - :param filter_applicable_to: filter techniques based on applicable_to field in techniques administration YAML file + :param filename: the filename of the YAML file containing the techniques administration :return: """ - my_techniques, name, platform = load_techniques(filename, 'detection', filter_applicable_to) + my_techniques, name, platform = load_techniques(filename) graph_values = [] for t in my_techniques.values(): @@ -70,49 +69,61 @@ def plot_detection_graph(filename, filter_applicable_to): df = pd.DataFrame(graph_values).groupby('date', as_index=False)[['count']].sum() df['cumcount'] = df.ix[::1, 'count'].cumsum()[::1] - output_filename = 'output/graph_detection_%s.html' % filter_applicable_to + output_filename = 'output/graph_detection.html' import plotly import plotly.graph_objs as go plotly.offline.plot( {'data': [go.Scatter(x=df['date'], y=df['cumcount'])], - 'layout': go.Layout(title="# of detections for %s %s" % (name, filter_applicable_to))}, + 'layout': go.Layout(title="# of detections for %s" % name)}, filename=output_filename, auto_open=False ) print("File written: " + output_filename) -def _load_data_sources(filename): +def _load_data_sources(file): """ - Loads the data sources (including all properties) from the given yaml file. - :param filename: the filename of the yaml file containing the data sources administration - :return: dictionary with data sources (including properties) + Loads the data sources (including all properties) from the given YAML file. + :param file: the file location of the YAML file containing the data sources administration or a dict + :return: dictionary with data sources, name, platform and exceptions list. """ my_data_sources = {} - _yaml = init_yaml() - with open(filename, 'r') as yaml_file: - yaml_content = _yaml.load(yaml_file) + + if isinstance(file, dict): + # file is a dict instance created due to the use of an EQL query by the user + yaml_content = file + else: + # file is a file location on disk + _yaml = init_yaml() + with open(file, 'r') as yaml_file: + yaml_content = _yaml.load(yaml_file) + + try: for d in yaml_content['data_sources']: dq = d['data_quality'] if dq['device_completeness'] > 0 and dq['data_field_completeness'] > 0 and dq['timeliness'] > 0 and dq['consistency'] > 0: my_data_sources[d['data_source_name']] = d + except KeyError: + # When using an EQL that does not result in a dict having 'data_sources' objects. Trow an error. + print(EQL_INVALID_RESULT_DS) + pprint(yaml_content) + quit() + return my_data_sources -def _write_layer(layer, mapped_techniques, filename_prefix, filename_suffix, name): +def _write_layer(layer, mapped_techniques, filename_prefix, name): """ Writes the json layer file to disk. :param layer: the prepped layer dictionary :param mapped_techniques: the techniques section that will be included in the layer :param filename_prefix: the prefix for the output filename - :param filename_suffix: the suffix for the output filename :param name: the name that will be used in the filename together with the prefix :return: """ layer['techniques'] = mapped_techniques json_string = simplejson.dumps(layer).replace('}, ', '},\n') - filename_suffix = '_' + filename_suffix if filename_suffix != '' else '' - output_filename = normalize_name_to_filename('output/%s_%s%s.json' % (filename_prefix, name, filename_suffix)) + output_filename = normalize_name_to_filename('output/%s_%s.json' % (filename_prefix, name)) with open(output_filename, 'w') as f: f.write(json_string) print("File written: " + output_filename) @@ -120,7 +131,7 @@ def _write_layer(layer, mapped_techniques, filename_prefix, filename_suffix, nam def _map_and_colorize_techniques_for_detections(my_techniques): """ - Determine the color of the techniques based on the detection score in the given yaml file. + Determine the color of the techniques based on the detection score in the given YAML file. :param my_techniques: the configured techniques :return: a dictionary with techniques that can be used in the layer's output file """ @@ -159,7 +170,7 @@ def _map_and_colorize_techniques_for_detections(my_techniques): x['metadata'].append({'name': '-Applicable to', 'value': applicable_to}) x['metadata'].append({'name': '-Detection score', 'value': str(d_score)}) x['metadata'].append({'name': '-Detection location', 'value': location}) - x['metadata'].append({'name': '-General comment', 'value': general_comment}) + x['metadata'].append({'name': '-Technique comment', 'value': general_comment}) x['metadata'].append({'name': '-Detection comment', 'value': get_latest_comment(detection)}) if cnt != tcnt: x['metadata'].append({'name': '---', 'value': '---'}) @@ -174,7 +185,7 @@ def _map_and_colorize_techniques_for_detections(my_techniques): def _map_and_colorize_techniques_for_visibility(my_techniques, my_data_sources): """ - Determine the color of the techniques based on the visibility score in the given yaml file. + Determine the color of the techniques based on the visibility score in the given YAML file. :param my_techniques: the configured techniques :param my_data_sources: the configured data sources :return: a dictionary with techniques that can be used in the layer's output file @@ -215,7 +226,7 @@ def _map_and_colorize_techniques_for_visibility(my_techniques, my_data_sources): general_comment = str(visibility['comment']) if str(visibility['comment']) != '' else '-' x['metadata'].append({'name': '-Applicable to', 'value': applicable_to}) x['metadata'].append({'name': '-Visibility score', 'value': str(get_latest_score(visibility))}) - x['metadata'].append({'name': '-General comment', 'value': general_comment}) + x['metadata'].append({'name': '-Technique comment', 'value': general_comment}) x['metadata'].append({'name': '-Visibility comment', 'value': get_latest_comment(visibility)}) if cnt != tcnt: x['metadata'].append({'name': '---', 'value': '---'}) @@ -242,12 +253,11 @@ def _map_and_colorize_techniques_for_visibility(my_techniques, my_data_sources): return mapped_techniques -def _map_and_colorize_techniques_for_overlaid(my_techniques, my_data_sources, filter_applicable_to): +def _map_and_colorize_techniques_for_overlaid(my_techniques, my_data_sources): """ Determine the color of the techniques based on both detection and visibility. :param my_techniques: the configured techniques :param my_data_sources: the configured data sources - :param filter_applicable_to: filter techniques based on applicable_to field in techniques administration YAML file :return: a dictionary with techniques that can be used in the layer's output file """ techniques = load_attack_data(DATA_TYPE_STIX_ALL_TECH) @@ -264,15 +274,6 @@ def _map_and_colorize_techniques_for_overlaid(my_techniques, my_data_sources, fi detection = True if detection_score > 0 else False visibility = True if visibility_score > 0 else False - # Additional filtering based on applicable_to field. Overrules the score. - a2_d = set([a for d in technique_data['detection'] for a in d['applicable_to']]) - a2_v = set([a for v in technique_data['detection'] for a in v['applicable_to']]) - - if filter_applicable_to != 'all' and filter_applicable_to not in a2_d and 'all' not in a2_d: - detection = False - if filter_applicable_to != 'all' and filter_applicable_to not in a2_v and 'all' not in a2_v: - visibility = False - if detection and visibility: color = COLOR_OVERLAY_BOTH elif detection and not visibility: @@ -297,10 +298,10 @@ def _map_and_colorize_techniques_for_overlaid(my_techniques, my_data_sources, fi # Metadata for detection: cnt = 1 - tcnt = len([d for d in technique_data['detection'] if get_latest_score(d) >= 0 and (filter_applicable_to == 'all' or filter_applicable_to in d['applicable_to'] or 'all' in d['applicable_to'])]) + tcnt = len([d for d in technique_data['detection'] if get_latest_score(d) >= 0]) for detection in technique_data['detection']: d_score = get_latest_score(detection) - if d_score >= 0 and (filter_applicable_to == 'all' or filter_applicable_to in detection['applicable_to'] or 'all' in detection['applicable_to']): + if d_score >= 0: location = ', '.join(detection['location']) location = location if location != '' else '-' applicable_to = ', '.join(detection['applicable_to']) @@ -308,7 +309,7 @@ def _map_and_colorize_techniques_for_overlaid(my_techniques, my_data_sources, fi x['metadata'].append({'name': '-Applicable to', 'value': applicable_to}) x['metadata'].append({'name': '-Detection score', 'value': str(d_score)}) x['metadata'].append({'name': '-Detection location', 'value': location}) - x['metadata'].append({'name': '-General comment', 'value': general_comment}) + x['metadata'].append({'name': '-Technique comment', 'value': general_comment}) x['metadata'].append({'name': '-Detection comment', 'value': get_latest_comment(detection)}) if cnt != tcnt: x['metadata'].append({'name': '---', 'value': '---'}) @@ -318,18 +319,17 @@ def _map_and_colorize_techniques_for_overlaid(my_techniques, my_data_sources, fi if tcnt > 0: x['metadata'].append({'name': '---', 'value': '---'}) cnt = 1 - tcnt = len([v for v in technique_data['visibility'] if filter_applicable_to == 'all' or filter_applicable_to in v['applicable_to'] or 'all' in v['applicable_to']]) + tcnt = len([v for v in technique_data['visibility']]) for visibility in technique_data['visibility']: - if filter_applicable_to == 'all' or filter_applicable_to in visibility['applicable_to'] or 'all' in visibility['applicable_to']: - applicable_to = ', '.join(visibility['applicable_to']) - general_comment = str(visibility['comment']) if str(visibility['comment']) != '' else '-' - x['metadata'].append({'name': '-Applicable to', 'value': applicable_to}) - x['metadata'].append({'name': '-Visibility score', 'value': str(get_latest_score(visibility))}) - x['metadata'].append({'name': '-General comment', 'value': general_comment}) - x['metadata'].append({'name': '-Visibility comment', 'value': get_latest_comment(visibility)}) - if cnt != tcnt: - x['metadata'].append({'name': '---', 'value': '---'}) - cnt += 1 + applicable_to = ', '.join(visibility['applicable_to']) + general_comment = str(visibility['comment']) if str(visibility['comment']) != '' else '-' + x['metadata'].append({'name': '-Applicable to', 'value': applicable_to}) + x['metadata'].append({'name': '-Visibility score', 'value': str(get_latest_score(visibility))}) + x['metadata'].append({'name': '-Technique comment', 'value': general_comment}) + x['metadata'].append({'name': '-Visibility comment', 'value': get_latest_comment(visibility)}) + if cnt != tcnt: + x['metadata'].append({'name': '---', 'value': '---'}) + cnt += 1 mapped_techniques.append(x) @@ -339,10 +339,10 @@ def _map_and_colorize_techniques_for_overlaid(my_techniques, my_data_sources, fi def export_techniques_list_to_excel(filename): """ Makes an overview of the MITRE ATT&CK techniques from the YAML administration file. - :param filename: the filename of the yaml file containing the techniques administration + :param filename: the filename of the YAML file containing the techniques administration :return: """ - my_techniques, name, platform = load_techniques(filename, 'all') + my_techniques, name, platform = load_techniques(filename) my_techniques = dict(sorted(my_techniques.items(), key=lambda kv: kv[0], reverse=False)) mitre_techniques = load_attack_data(DATA_TYPE_STIX_ALL_TECH) @@ -390,7 +390,7 @@ def export_techniques_list_to_excel(filename): worksheet_detections.write(y, 4, 'Date', format_bold_left) worksheet_detections.write(y, 5, 'Score', format_bold_left) worksheet_detections.write(y, 6, 'Location', format_bold_left) - worksheet_detections.write(y, 7, 'General comment', format_bold_left) + worksheet_detections.write(y, 7, 'Technique comment', format_bold_left) worksheet_detections.write(y, 8, 'Detection comment', format_bold_left) worksheet_detections.set_column(0, 0, 14) worksheet_detections.set_column(1, 1, 40) @@ -410,7 +410,11 @@ def export_techniques_list_to_excel(filename): get_tactics(get_technique(mitre_techniques, technique_id))), valign_top) worksheet_detections.write(y, 3, ', '.join(detection['applicable_to']), wrap_text) - worksheet_detections.write(y, 4, str(get_latest_date(detection)).replace('None', ''), valign_top) + # make sure the date format is '%Y-%m-%d'. When we've done a EQL query this will become '%Y-%m-%d %H %M $%S' + tmp_date = get_latest_date(detection) + if isinstance(tmp_date, datetime): + tmp_date = tmp_date.strftime('%Y-%m-%d') + worksheet_detections.write(y, 4, str(tmp_date).replace('None', ''), valign_top) ds = get_latest_score(detection) worksheet_detections.write(y, 5, ds, detection_score_0 if ds == 0 else detection_score_1 if ds == 1 else detection_score_2 if ds == 2 else detection_score_3 if ds == 3 else detection_score_4 if ds == 4 else detection_score_5 if ds == 5 else no_score) worksheet_detections.write(y, 6, '\n'.join(detection['location']), wrap_text) @@ -429,7 +433,7 @@ def export_techniques_list_to_excel(filename): worksheet_visibility.write(y, 3, 'Applicable to', format_bold_left) worksheet_visibility.write(y, 4, 'Date', format_bold_left) worksheet_visibility.write(y, 5, 'Score', format_bold_left) - worksheet_visibility.write(y, 6, 'General Comment', format_bold_left) + worksheet_visibility.write(y, 6, 'Technique comment', format_bold_left) worksheet_visibility.write(y, 7, 'Visibility comment', format_bold_left) worksheet_visibility.set_column(0, 0, 14) worksheet_visibility.set_column(1, 1, 40) @@ -447,7 +451,11 @@ def export_techniques_list_to_excel(filename): worksheet_visibility.write(y, 2, ', '.join(t.capitalize() for t in get_tactics(get_technique(mitre_techniques, technique_id))), valign_top) worksheet_visibility.write(y, 3, ', '.join(visibility['applicable_to']), wrap_text) - worksheet_visibility.write(y, 4, str(get_latest_date(visibility)).replace('None', ''), valign_top) + # make sure the date format is '%Y-%m-%d'. When we've done a EQL query this will become '%Y-%m-%d %H %M $%S' + tmp_date = get_latest_date(visibility) + if isinstance(tmp_date, datetime): + tmp_date = tmp_date.strftime('%Y-%m-%d') + worksheet_visibility.write(y, 4, str(tmp_date).replace('None', ''), valign_top) vs = get_latest_score(visibility) worksheet_visibility.write(y, 5, vs, visibility_score_1 if vs == 1 else visibility_score_2 if vs == 2 else visibility_score_3 if vs == 3 else visibility_score_4 if vs == 4 else no_score) v_comment = get_latest_comment(visibility, empty='') From 4a3b77ab007ba74f00d2f3a85d8cfe091a63621c Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Thu, 8 Aug 2019 14:42:31 +0200 Subject: [PATCH 26/49] Added the package eql (v0.7) --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a99ec44d..d4833046 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ simplejson==3.16.0 plotly==3.10.0 pandas==0.24.2 xlsxwriter==1.1.8 -ruamel.yaml=0.16.0 \ No newline at end of file +ruamel.yaml=0.16.0 +eql==0.7 \ No newline at end of file From 823c82a9094827976d83f0b95920260c36dcdc93 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Thu, 8 Aug 2019 14:46:30 +0200 Subject: [PATCH 27/49] - Added new functionality to the function 'generate_group_heat_map' for the new EQL query functionality. And added support for the '--health' argument in the group mode. - Removed functionality due to the deprecation of the argument '-a, --applicable'. --- group_mapping.py | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/group_mapping.py b/group_mapping.py index 96bf3987..0e58e52f 100644 --- a/group_mapping.py +++ b/group_mapping.py @@ -1,5 +1,6 @@ import simplejson from generic import * +from eql_yaml import techniques_search CG_GROUPS = {} @@ -212,17 +213,16 @@ def _get_group_techniques(groups, stage, platform, file_type): return groups_dict -def _get_detection_techniques(filename, filter_applicable_to): +def _get_detection_techniques(filename): """ Get all techniques (in a dict) from the detection administration :param filename: path to the YAML technique administration file - :param filter_applicable_to: filter techniques based on applicable_to field in techniques administration YAML file :return: groups dictionary, loaded techniques from administration YAML file """ # { group_id: {group_name: NAME, techniques: set{id, ...} } } groups_dict = {} - detection_techniques, name, platform = load_techniques(filename, 'detection', filter_applicable_to) + detection_techniques, name, platform = load_techniques(filename) group_id = 'DETECTION' groups_dict[group_id] = {} @@ -238,17 +238,16 @@ def _get_detection_techniques(filename, filter_applicable_to): return groups_dict, detection_techniques -def _get_visibility_techniques(filename, filter_applicable_to): +def _get_visibility_techniques(filename): """ Get all techniques (in a dict) from the technique administration :param filename: path to the YAML technique administration file - :param filter_applicable_to: filter techniques based on applicable_to field in techniques administration YAML file :return: dictionary """ # { group_id: {group_name: NAME, techniques: set{id, ...} } } groups_dict = {} - visibility_techniques, name, platform = load_techniques(filename, 'visibility', filter_applicable_to) + visibility_techniques, name, platform = load_techniques(filename) group_id = 'VISIBILITY' groups_dict[group_id] = {} @@ -454,7 +453,8 @@ def _get_group_list(groups, file_type): return groups -def generate_group_heat_map(groups, overlay, overlay_type, stage, platform, software_groups, filter_applicable_to): +def generate_group_heat_map(groups, overlay, overlay_type, stage, platform, software_groups, + search_visibility, search_detection, health_is_called, include_all_score_objs=False): """ Calls all functions that are necessary for the generation of the heat map and write a json layer to disk. :param groups: threat actor groups @@ -463,8 +463,11 @@ def generate_group_heat_map(groups, overlay, overlay_type, stage, platform, soft :param overlay_type: group, visibility or detection :param stage: attack or pre-attack :param platform: all, Linux, macOS, Windows - :param software_groups: specify if techniques from related software should be included. - :param filter_applicable_to: filter techniques based on applicable_to field in techniques administration YAML file + :param software_groups: specify if techniques from related software should be included + :param search_visibility: visibility EQL search query + :param search_detection: detection EQL search query + :param health_is_called: boolean that specifies if detailed errors in the file will be printed + :param include_all_score_objs: include all score objects within the score_logbook for the EQL query :return: returns nothing when something's wrong """ overlay_dict = {} @@ -472,7 +475,8 @@ def generate_group_heat_map(groups, overlay, overlay_type, stage, platform, soft groups_file_type = None if os.path.isfile(groups): - groups_file_type = check_file(groups, file_type=FILE_TYPE_GROUP_ADMINISTRATION) + groups_file_type = check_file(groups, file_type=FILE_TYPE_GROUP_ADMINISTRATION, + health_is_called=health_is_called) if not groups_file_type: return else: @@ -486,7 +490,7 @@ def generate_group_heat_map(groups, overlay, overlay_type, stage, platform, soft expected_file_type = FILE_TYPE_GROUP_ADMINISTRATION if overlay_type == OVERLAY_TYPE_GROUP \ else FILE_TYPE_TECHNIQUE_ADMINISTRATION \ if overlay_type in [OVERLAY_TYPE_VISIBILITY, OVERLAY_TYPE_DETECTION] else None - overlay_file_type = check_file(overlay, expected_file_type) + overlay_file_type = check_file(overlay, expected_file_type, health_is_called=health_is_called) if not overlay_file_type: return else: @@ -495,12 +499,19 @@ def generate_group_heat_map(groups, overlay, overlay_type, stage, platform, soft else: overlay = [] + # load the techniques (visibility or detection) from the YAML file all_techniques = None if overlay_file_type == FILE_TYPE_TECHNIQUE_ADMINISTRATION: + # filter out visibility and/or detection objects using EQL + if search_detection or search_visibility: + overlay = techniques_search(overlay, search_visibility, search_detection, + include_all_score_objs=include_all_score_objs) + if overlay_type == OVERLAY_TYPE_VISIBILITY: - overlay_dict, all_techniques = _get_visibility_techniques(overlay, filter_applicable_to) + overlay_dict, all_techniques = _get_visibility_techniques(overlay) elif overlay_type == OVERLAY_TYPE_DETECTION: - overlay_dict, all_techniques = _get_detection_techniques(overlay, filter_applicable_to) + overlay_dict, all_techniques = _get_detection_techniques(overlay) + # we are not overlaying visibility or detection, overlay group will therefore contain information another group elif len(overlay) > 0: overlay_dict = _get_group_techniques(overlay, stage, platform, overlay_file_type) if not overlay_dict: @@ -543,7 +554,7 @@ def generate_group_heat_map(groups, overlay, overlay_type, stage, platform, soft if stage == 'pre-attack': filename = "output/" + stage + '_' + '_'.join(groups_list) elif overlay: - filename = "output/" + stage + '_' + platform.lower() + '_' + '_'.join(groups_list) + '-overlay_' + '_'.join(overlay_list) + '_' + filter_applicable_to.replace(' ', '_') + filename = "output/" + stage + '_' + platform.lower() + '_' + '_'.join(groups_list) + '-overlay_' + '_'.join(overlay_list) else: filename = "output/" + stage + '_' + platform.lower() + '_' + '_'.join(groups_list) filename = filename[:255] + '.json' From 0a7be674778fff058c65ca150c6b9853d5e3b8d8 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Thu, 8 Aug 2019 14:48:22 +0200 Subject: [PATCH 28/49] - Multiple functions modified to be compatible with new EQL query functionality. - Making use of StringIO within the function 'generate_technique_administration_file' instead of writing a temporary file to disk. --- data_source_mapping.py | 104 ++++++++++++++++++++--------------------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/data_source_mapping.py b/data_source_mapping.py index a54a9dd4..3e6bd1f9 100644 --- a/data_source_mapping.py +++ b/data_source_mapping.py @@ -1,7 +1,9 @@ import simplejson import xlsxwriter -import copy +from copy import deepcopy from generic import * +from pprint import pprint +from datetime import datetime # Imports for pandas and plotly are because of performance reasons in the function that uses these libraries. @@ -9,7 +11,7 @@ def generate_data_sources_layer(filename): """ Generates a generic layer for data sources. - :param filename: the filename of the yaml file containing the data sources administration + :param filename: the filename of the YAML file containing the data sources administration :return: """ my_data_sources, name, platform, exceptions = _load_data_sources(filename) @@ -30,7 +32,7 @@ def generate_data_sources_layer(filename): def plot_data_sources_graph(filename): """ Generates a line graph which shows the improvements on numbers of data sources through time. - :param filename: the filename of the yaml file containing the data sources administration + :param filename: the filename of the YAML file containing the data sources administration :return: """ my_data_sources, name, platform, exceptions = _load_data_sources(filename) @@ -60,8 +62,8 @@ def plot_data_sources_graph(filename): def export_data_source_list_to_excel(filename): """ Makes an overview of all MITRE ATT&CK data sources (via techniques) and lists which data sources are present - in the yaml administration including all properties and data quality score. - :param filename: the filename of the yaml file containing the data sources administration + in the YAML administration including all properties and data quality score. + :param filename: the filename of the YAML file containing the data sources administration :return: """ my_data_sources, name, platform, exceptions = _load_data_sources(filename, filter_empty_scores=False) @@ -114,8 +116,17 @@ def export_data_source_list_to_excel(filename): worksheet.write(y, 0, d, valign_top) if d in my_data_sources.keys(): ds = my_data_sources[d] - worksheet.write(y, 1, str(ds['date_registered']).replace('None', ''), valign_top) - worksheet.write(y, 2, str(ds['date_connected']).replace('None', ''), valign_top) + + tmp_date_1 = ds['date_registered'] + if isinstance(tmp_date_1, datetime): + tmp_date_1 = tmp_date_1.strftime('%Y-%m-%d') + + tmp_date_2 = ds['date_connected'] + if isinstance(tmp_date_2, datetime): + tmp_date_2 = tmp_date_2.strftime('%Y-%m-%d') + + worksheet.write(y, 1, str(tmp_date_1).replace('None', ''), valign_top) + worksheet.write(y, 2, str(tmp_date_2).replace('None', ''), valign_top) worksheet.write(y, 3, ', '.join(ds['products']).replace('None', ''), valign_top) worksheet.write(y, 4, ds['comment'][:-1] if ds['comment'].endswith('\n') else ds['comment'], wrap_text) worksheet.write(y, 5, str(ds['available_for_data_analytics']), valign_top) @@ -146,25 +157,40 @@ def export_data_source_list_to_excel(filename): print('[!] Error while writing Excel file: %s' % str(e)) -def _load_data_sources(filename, filter_empty_scores=True): +def _load_data_sources(file, filter_empty_scores=True): """ - Loads the data sources (including all properties) from the given yaml file. - :param filename: the filename of the yaml file containing the data sources administration + Loads the data sources (including all properties) from the given YAML file. + :param file: the file location of the YAML file containing the data sources administration or a dict :return: dictionary with data sources, name, platform and exceptions list. """ my_data_sources = {} - _yaml = init_yaml() - with open(filename, 'r') as yaml_file: - yaml_content = _yaml.load(yaml_file) + + if isinstance(file, dict): + # file is a dict created due to the use of an EQL query by the user + yaml_content = file + else: + # file is a file location on disk + _yaml = init_yaml() + with open(file, 'r') as yaml_file: + yaml_content = _yaml.load(yaml_file) + + try: for d in yaml_content['data_sources']: dq = d['data_quality'] if not filter_empty_scores: my_data_sources[d['data_source_name']] = d elif dq['device_completeness'] > 0 and dq['data_field_completeness'] > 0 and dq['timeliness'] > 0 and dq['consistency'] > 0: my_data_sources[d['data_source_name']] = d + name = yaml_content['name'] platform = yaml_content['platform'] exceptions = [t['technique_id'] for t in yaml_content['exceptions']] + except KeyError: + # When using an EQL that does not result in a dict having 'data_sources' objects. Trow an error. + print(EQL_INVALID_RESULT_DS) + pprint(yaml_content) + quit() + return my_data_sources, name, platform, exceptions @@ -252,7 +278,7 @@ def update_technique_administration_file(file_data_sources, file_tech_admin): today = new_visibility_scores['techniques'][0]['visibility']['score_logbook'][0]['date'] # next we load the current visibility scores from the tech. admin file - cur_visibility_scores, _, platform_tech_admin = load_techniques(file_tech_admin, 'visibility') + cur_visibility_scores, _, platform_tech_admin = load_techniques(file_tech_admin) # if the platform does not match between the data source and tech. admin file we return if new_visibility_scores['platform'] != platform_tech_admin: @@ -409,7 +435,7 @@ def update_technique_administration_file(file_data_sources, file_tech_admin): print('Visibility object:') print(' - ATT&CK ID/name ' + tech_id + ' / ' + tech_name) print(' - Applicable to: ' + ', '.join(old_vis_obj[obj_idx]['applicable_to'])) - print(' - General comment: ' + _indent_comment(old_vis_obj[obj_idx]['comment'], 23)) + print(' - Technique comment: ' + _indent_comment(old_vis_obj[obj_idx]['comment'], 23)) print('') print('OLD score object:') print(' - Date: ' + get_latest_date(old_vis_obj[obj_idx]).strftime('%Y-%m-%d')) @@ -436,7 +462,7 @@ def update_technique_administration_file(file_data_sources, file_tech_admin): print('') backup_file(file_tech_admin) - yaml_file_tech_admin = fix_date_and_remove_null(yaml_file_tech_admin, today, input_reamel=True) + yaml_file_tech_admin = fix_date_and_remove_null(yaml_file_tech_admin, today, input_type='ruamel') with open(file_tech_admin, 'w') as fd: fd.writelines(yaml_file_tech_admin) @@ -447,8 +473,8 @@ def update_technique_administration_file(file_data_sources, file_tech_admin): def generate_technique_administration_file(filename, write_file=True): """ - Generate a technique administration file based on the data source administration yaml file - :param filename: the filename of the yaml file containing the data sources administration + Generate a technique administration file based on the data source administration YAML file + :param filename: the filename of the YAML file containing the data sources administration :param write_file: by default the file is written to disk :return: """ @@ -456,31 +482,6 @@ def generate_technique_administration_file(filename, write_file=True): techniques = load_attack_data(DATA_TYPE_STIX_ALL_TECH_ENTERPRISE) - # This is part of the techniques administration YAML file and is used as a template - dict_tech = {'technique_id': '', - 'technique_name': '', - 'detection': - {'applicable_to': ['all'], - 'location': [''], - 'comment': '', - 'score_logbook': - [ - {'date': None, - 'score': -1, - 'comment': ''} - ]}, - 'visibility': - {'applicable_to': ['all'], - 'comment': '', - 'score_logbook': - [ - {'date': None, - 'score': 0, - 'comment': '', - 'auto_generated': True} - ] - }} - yaml_file = dict() yaml_file['version'] = FILE_TYPE_TECHNIQUE_ADMINISTRATION_VERSION yaml_file['file_type'] = FILE_TYPE_TECHNIQUE_ADMINISTRATION @@ -491,7 +492,7 @@ def generate_technique_administration_file(filename, write_file=True): # Score visibility based on the number of available data sources and the exceptions for t in techniques: - platforms_lower = list(map(lambda x: x.lower(), try_get_key(t, 'x_mitre_platforms'))) + platforms_lower = list(map(lambda x: x.lower(), t.get('x_mitre_platforms', None))) if platform in platforms_lower: # not every technique has data source listed if 'x_mitre_data_sources' in t: @@ -511,7 +512,7 @@ def generate_technique_administration_file(filename, write_file=True): techniques_upper = list(map(lambda x: x.upper(), exceptions)) tech_id = get_attack_id(t) if score > 0 and tech_id not in techniques_upper: - tech = copy.deepcopy(dict_tech) + tech = deepcopy(YAML_OBJ_TECHNIQUE) tech['technique_id'] = tech_id tech['technique_name'] = t['name'] # noinspection PyUnresolvedReferences @@ -523,16 +524,15 @@ def generate_technique_administration_file(filename, write_file=True): if write_file: # remove the single quotes around the date key-value pair _yaml = init_yaml() - tmp_file = sys.path[0] + '/.tmp_tech_file' + file = StringIO() - # create the file lines by writing it to disk - with open(tmp_file, 'w') as fd_tmp: - _yaml.dump(yaml_file, fd_tmp) + # create the file lines by writing it to memory + _yaml.dump(yaml_file, file) + file.seek(0) + file_lines = file.readlines() # remove the single quotes from the date - yaml_file_lines = fix_date_and_remove_null(tmp_file, today, input_reamel=False) - - os.remove(tmp_file) + yaml_file_lines = fix_date_and_remove_null(file_lines, today, input_type='list') # create a output filename that prevents overwriting any existing files output_filename = 'output/techniques-administration-' + normalize_name_to_filename(name+'-'+platform) + '.yaml' From b0757b6bbd7e51fadd1a4465ff98fde13753e13d Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Fri, 9 Aug 2019 10:51:57 +0200 Subject: [PATCH 29/49] Fixed a bug that caused a crash when having a 'group_name' or 'campaign' within a Group YAML with only integers. --- group_mapping.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/group_mapping.py b/group_mapping.py index 0e58e52f..cdb6a3cf 100644 --- a/group_mapping.py +++ b/group_mapping.py @@ -83,12 +83,12 @@ def _get_software_techniques(groups, stage, platform): for group in config['groups']: if group['enabled']: - group_id = _generate_group_id(group['group_name'], group['campaign']) + group_id = _generate_group_id(str(group['group_name']), str(group['campaign'])) groups_dict[group_id] = dict() - groups_dict[group_id]['group_name'] = group['group_name'] + groups_dict[group_id]['group_name'] = str(group['group_name']) groups_dict[group_id]['techniques'] = set() - groups_dict[group_id]['campaign'] = group['campaign'] + groups_dict[group_id]['campaign'] = str(group['campaign']) groups_dict[group_id]['software'] = group['software_id'] if group['software_id']: @@ -168,18 +168,18 @@ def _get_group_techniques(groups, stage, platform, file_type): for group in config['groups']: if group['enabled']: - campaign = group['campaign'] if group['campaign'] else '' - group_id = _generate_group_id(group['group_name'], campaign) + campaign = str(group['campaign']) if group['campaign'] else '' + group_id = _generate_group_id(str(group['group_name']), campaign) groups_dict[group_id] = dict() - groups_dict[group_id]['group_name'] = group['group_name'] + groups_dict[group_id]['group_name'] = str(group['group_name']) if isinstance(group['technique_id'], list): groups_dict[group_id]['techniques'] = set(group['technique_id']) groups_dict[group_id]['weight'] = dict((i, 1) for i in group['technique_id']) elif isinstance(group['technique_id'], dict): groups_dict[group_id]['techniques'] = set(group['technique_id'].keys()) groups_dict[group_id]['weight'] = group['technique_id'] - groups_dict[group_id]['campaign'] = group['campaign'] + groups_dict[group_id]['campaign'] = str(group['campaign']) groups_dict[group_id]['software'] = group['software_id'] else: # groups are provided as arguments via the command line From 04591ce76da4ba41efdb0013f0874dc17ba1b36b Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Fri, 9 Aug 2019 20:15:48 +0200 Subject: [PATCH 30/49] Fixed a bug that caused a crash when the key-value pair 'date_registered' or 'date_connected' is not a datetime object. --- eql_yaml.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eql_yaml.py b/eql_yaml.py index 71fcaf7b..ec045512 100644 --- a/eql_yaml.py +++ b/eql_yaml.py @@ -162,8 +162,10 @@ def _events_to_yaml(query_results, obj_type): # Remove the event_type key. We no longer need this. for r in query_results: del r['event_type'] - r['date_registered'] = datetime.datetime.strptime(r['date_registered'], '%Y-%m-%d') - r['date_connected'] = datetime.datetime.strptime(r['date_connected'], '%Y-%m-%d') + if isinstance(r['date_registered'], datetime.datetime): + r['date_registered'] = datetime.datetime.strptime(r['date_registered'], '%Y-%m-%d') + if isinstance(r['date_connected'], datetime.datetime): + r['date_connected'] = datetime.datetime.strptime(r['date_connected'], '%Y-%m-%d') except KeyError: # When using an EQL that does not result in a dict having valid YAML objects. Trow an error. print(EQL_INVALID_RESULT_DS) From a8c925da2758ccd45409b45b881aee52fd8b9d19 Mon Sep 17 00:00:00 2001 From: Ruben Bouman Date: Mon, 12 Aug 2019 15:44:55 +0200 Subject: [PATCH 31/49] fix operator in requirements file --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d4833046..d804bec4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,5 @@ simplejson==3.16.0 plotly==3.10.0 pandas==0.24.2 xlsxwriter==1.1.8 -ruamel.yaml=0.16.0 +ruamel.yaml==0.16.0 eql==0.7 \ No newline at end of file From 94e5470ccdfac09221ce54b72d0c9d3455946575 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Tue, 13 Aug 2019 14:24:03 +0200 Subject: [PATCH 32/49] Modified how the DQ overall scores is calculated. --- data_source_mapping.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/data_source_mapping.py b/data_source_mapping.py index 3e6bd1f9..90cfd240 100644 --- a/data_source_mapping.py +++ b/data_source_mapping.py @@ -138,10 +138,15 @@ def export_data_source_list_to_excel(filename): score = 0 score_count = 0 - for s in ds['data_quality'].values(): - if s != 0: + for k, v in ds['data_quality'].items(): + # the below DQ dimensions are given more weight in the calculation of the DQ score. + print(k) + if k in ['device_completeness', 'data_field_completeness', 'retention']: + score += (v * 2) + score_count += 2 + else: + score += v score_count += 1 - score += s if score > 0: score = score/score_count @@ -185,11 +190,11 @@ def _load_data_sources(file, filter_empty_scores=True): name = yaml_content['name'] platform = yaml_content['platform'] exceptions = [t['technique_id'] for t in yaml_content['exceptions']] - except KeyError: - # When using an EQL that does not result in a dict having 'data_sources' objects. Trow an error. + except KeyError: # todo remove after implemented extra check within the function '_events_to_yaml' + # when using an EQL query that does not result in a dict having 'data_sources' objects. Trow an error. print(EQL_INVALID_RESULT_DS) pprint(yaml_content) - quit() + return None return my_data_sources, name, platform, exceptions From 08b4c9c1ec9ebd8a65b4bb30cf0297b1e2a70987 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Tue, 13 Aug 2019 14:28:43 +0200 Subject: [PATCH 33/49] Made the necessary changes to be compatible with the interactive menu. --- dettect.py | 13 +++++++-- eql_yaml.py | 66 ++++++++++++++++++++++++++++---------------- group_mapping.py | 2 ++ technique_mapping.py | 6 ++-- 4 files changed, 57 insertions(+), 30 deletions(-) diff --git a/dettect.py b/dettect.py index 8789cbe9..04e01387 100644 --- a/dettect.py +++ b/dettect.py @@ -183,6 +183,8 @@ def _menu(menu_parser): if args.search: file_ds = search(args.file_ds, FILE_TYPE_DATA_SOURCE_ADMINISTRATION, args.search) + if not file_ds: + quit() # something went wrong in executing the search or 0 results where returned if args.update and check_file(args.file_tech, FILE_TYPE_TECHNIQUE_ADMINISTRATION): update_technique_administration_file(file_ds, args.file_tech) if args.layer: @@ -209,6 +211,8 @@ def _menu(menu_parser): if args.search_detection or args.search_visibility: file_tech = techniques_search(args.file_tech, args.search_visibility, args.search_detection, include_all_score_objs=args.all_scores) + if not file_tech: + quit() # something went wrong in executing the search or 0 results where returned if args.layer: generate_visibility_layer(file_tech, args.file_ds, False) if args.overlay: @@ -218,9 +222,10 @@ def _menu(menu_parser): # toto add search capabilities elif args.subparser in ['group', 'g']: - generate_group_heat_map(args.groups, args.overlay, args.overlay_type, args.stage, args.platform, - args.software_group, args.search_visibility, args.search_detection, args.health, - include_all_score_objs=args.all_scores) + if not generate_group_heat_map(args.groups, args.overlay, args.overlay_type, args.stage, args.platform, + args.software_group, args.search_visibility, args.search_detection, args.health, + include_all_score_objs=args.all_scores): + quit() # something went wrong in executing the search or 0 results where returned elif args.subparser in ['detection', 'd']: if args.overlay: @@ -236,6 +241,8 @@ def _menu(menu_parser): if args.search_detection or args.search_visibility: file_tech = techniques_search(args.file_tech, args.search_visibility, args.search_detection, include_all_score_objs=args.all_scores) + if not file_tech: + quit() # something went wrong in executing the search or 0 results where returned if args.layer: generate_detection_layer(file_tech, args.file_ds, False) if args.overlay and check_file(args.file_ds, FILE_TYPE_DATA_SOURCE_ADMINISTRATION, args.health): diff --git a/eql_yaml.py b/eql_yaml.py index ec045512..0fd56155 100644 --- a/eql_yaml.py +++ b/eql_yaml.py @@ -154,23 +154,24 @@ def _events_to_yaml(query_results, obj_type): Transform the EQL 'events' back to valid YAML objects :param query_results: list with EQL 'events :param obj_type: data_sources, detection or visibility EQL 'events' - :return: list containing YAML objects + :return: list containing YAML objects or None when the events could not be turned into a valid YAML object """ if obj_type == 'data_sources': try: # Remove the event_type key. We no longer need this. + # todo implement a check to see if the returned data from the EQL query is to the schema of data_sources for r in query_results: del r['event_type'] - if isinstance(r['date_registered'], datetime.datetime): + if r['date_registered'] and isinstance(r['date_registered'], str): r['date_registered'] = datetime.datetime.strptime(r['date_registered'], '%Y-%m-%d') - if isinstance(r['date_connected'], datetime.datetime): + if r['date_connected'] and isinstance(r['date_connected'], str): r['date_connected'] = datetime.datetime.strptime(r['date_connected'], '%Y-%m-%d') except KeyError: - # When using an EQL that does not result in a dict having valid YAML objects. Trow an error. print(EQL_INVALID_RESULT_DS) pprint(query_results) - quit() + # when using an EQL query that does not result in a dict having valid YAML 'data_source' objects. + return None return query_results @@ -183,7 +184,7 @@ def _events_to_yaml(query_results, obj_type): tech_name = tech_event['technique_name'] obj_event = tech_event[obj_type] score_logbook_event = tech_event[obj_type]['score_logbook'] - if score_logbook_event['date']: + if score_logbook_event['date'] and isinstance(score_logbook_event['date'], str): score_date = datetime.datetime.strptime(score_logbook_event['date'], '%Y-%m-%d') else: score_date = None @@ -221,11 +222,10 @@ def _events_to_yaml(query_results, obj_type): return techniques_yaml except KeyError: - print(KeyError) - # When using an EQL that does not in a valid technique administration file. Trow an error. print(EQL_INVALID_RESULT_TECH + obj_type + ' object(s):') pprint(query_results) - quit() + # when using an EQL query that does not in a valid technique administration file. + return None def _merge_yaml(yaml_content_org, yaml_content_visibility=None, yaml_content_detection=None): @@ -319,19 +319,23 @@ def _check_query_results(query_results, obj_type): :param obj_type: 'data_sources', 'visibility' or 'detection' :return: """ + # the EQL query was not compatible with the schema + if query_results is None: + return False # show an error to the user when the query resulted on zero results result_len = len(query_results) if result_len == 0: error = '[!] The search returned 0 ' + obj_type + ' objects. Refine your search to return 1 or more ' \ - + obj_type + ' objects.\nExiting...' + + obj_type + ' objects.' print(error) - quit() + return False else: if result_len == 1: msg = 'The ' + obj_type + ' query executed successfully and provided ' + str(len(query_results)) + ' result.' else: msg = 'The ' + obj_type + ' query executed successfully and provided ' + str(len(query_results)) + ' results.' print(msg) + return True def _execute_eql_query(events, query): @@ -339,7 +343,7 @@ def _execute_eql_query(events, query): Execute an EQL query against the provided events :param events: events :param query: EQL query - :return: the query results (i.e. filtered events) + :return: the query results (i.e. filtered events) or None when the query did not match the schema """ # learn and load the schema schema = eql.Schema.learn(events) @@ -361,7 +365,8 @@ def callback(results): print(e, file=sys.stderr) print('\nTake into account the following schema:') pprint(eql.Schema.current().schema) - sys.exit(2) + # when using an EQL query that does not match the schema, return None. + return None engine.add_output_hook(callback) # execute the query @@ -377,14 +382,17 @@ def techniques_search(filename, query_visibility=None, query_detection=None, inc :param query_visibility: EQL query for the visibility YAML objects :param query_detection: EQL query for the detection YAML objects :param include_all_score_objs: include all score objects within the score_logbook for the EQL query - :return: a filtered technique administration YAML 'file' (i.e. dict) + :return: a filtered technique administration YAML 'file' (i.e. dict) or None when the query was not successful """ + results_visibility_yaml = None + results_detection_yaml = None if query_visibility: visibility_events, yaml_content_org = _prepare_yaml_file(filename, 'visibility', include_all_score_objs=include_all_score_objs) results_visibility = _execute_eql_query(visibility_events, query_visibility) - _check_query_results(results_visibility, 'visibility') + if not _check_query_results(results_visibility, 'visibility'): + return None # the EQL query was not compatible with the schema results_visibility_yaml = _events_to_yaml(results_visibility, 'visibility') if query_detection: @@ -392,15 +400,20 @@ def techniques_search(filename, query_visibility=None, query_detection=None, inc include_all_score_objs=include_all_score_objs) results_detection = _execute_eql_query(detection_events, query_detection) - _check_query_results(results_detection, 'detection') + if not _check_query_results(results_detection, 'detection'): + return None # the EQL query was not compatible with the schema results_detection_yaml = _events_to_yaml(results_detection, 'detection') + if (query_visibility and not results_visibility_yaml) or (query_detection and not results_detection_yaml): + # when using an EQL query that does not result in a dict having a valid technique administration YAML content + return None + if query_visibility and query_detection: yaml_content = _merge_yaml(yaml_content_org, results_visibility_yaml, results_detection_yaml) - elif query_visibility: + elif results_visibility_yaml: yaml_content = _merge_yaml(yaml_content_org, yaml_content_visibility=results_visibility_yaml) - elif query_detection: + elif results_detection_yaml: yaml_content = _merge_yaml(yaml_content_org, yaml_content_detection=results_detection_yaml) else: return filename @@ -415,7 +428,7 @@ def search(filename, file_type, query='', include_all_score_objs=False): :param file_type: data source administration file, ... :param query: EQL query :param include_all_score_objs: include all score objects within the score_logbook for the EQL query - :return: a filtered YAML 'file' (i.e. dict) + :return: a filtered YAML 'file' (i.e. dict) or None when the query was not successful """ if file_type == FILE_TYPE_DATA_SOURCE_ADMINISTRATION: @@ -425,13 +438,18 @@ def search(filename, file_type, query='', include_all_score_objs=False): yaml_content_eql, yaml_content_org = _prepare_yaml_file(filename, obj_type, include_all_score_objs=include_all_score_objs) - query_results = _execute_eql_query(yaml_content_eql, query) - _check_query_results(query_results, obj_type) + + if not _check_query_results(query_results, obj_type): + return # the EQL query was not compatible with the schema query_results_yaml = _events_to_yaml(query_results, obj_type) - yaml_content = yaml_content_org - yaml_content[obj_type] = query_results_yaml + if query_results_yaml: + yaml_content = yaml_content_org + yaml_content[obj_type] = query_results_yaml - return yaml_content + return yaml_content + else: + # when using an EQL query that does not result in a dict having valid YAML objects, return None + return None diff --git a/group_mapping.py b/group_mapping.py index cdb6a3cf..2b7c389a 100644 --- a/group_mapping.py +++ b/group_mapping.py @@ -506,6 +506,8 @@ def generate_group_heat_map(groups, overlay, overlay_type, stage, platform, soft if search_detection or search_visibility: overlay = techniques_search(overlay, search_visibility, search_detection, include_all_score_objs=include_all_score_objs) + if not overlay: + return None # something went wrong in executing the search or 0 results where returned if overlay_type == OVERLAY_TYPE_VISIBILITY: overlay_dict, all_techniques = _get_visibility_techniques(overlay) diff --git a/technique_mapping.py b/technique_mapping.py index e2294326..e6b135a6 100644 --- a/technique_mapping.py +++ b/technique_mapping.py @@ -102,11 +102,11 @@ def _load_data_sources(file): dq = d['data_quality'] if dq['device_completeness'] > 0 and dq['data_field_completeness'] > 0 and dq['timeliness'] > 0 and dq['consistency'] > 0: my_data_sources[d['data_source_name']] = d - except KeyError: - # When using an EQL that does not result in a dict having 'data_sources' objects. Trow an error. + except KeyError: # todo remove after implemented extra check within the function '_events_to_yaml' + # when using an EQL query that does not result in a dict having 'data_sources' objects, return None print(EQL_INVALID_RESULT_DS) pprint(yaml_content) - quit() + return None return my_data_sources From 4fd39d46aa810717cb008c00659489d316ef3263 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Tue, 13 Aug 2019 14:29:54 +0200 Subject: [PATCH 34/49] Removed an unnecessary try/catch block. --- generic.py | 49 +++++++++++++++++++++---------------------------- 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/generic.py b/generic.py index babee749..9e879397 100644 --- a/generic.py +++ b/generic.py @@ -2,7 +2,6 @@ import shutil import pickle from io import StringIO -from pprint import pprint from ruamel.yaml import YAML from difflib import SequenceMatcher from datetime import datetime as dt @@ -646,29 +645,23 @@ def load_techniques(file): with open(file, 'r') as yaml_file: yaml_content = _yaml.load(yaml_file) - try: - for d in yaml_content['techniques']: - # Add detection items: - if isinstance(d['detection'], dict): # There is just one detection entry - add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'detection', d['detection']) - elif isinstance(d['detection'], list): # There are multiple detection entries - for de in d['detection']: - add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'detection', de) - - # Add visibility items - if isinstance(d['visibility'], dict): # There is just one visibility entry - add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'visibility', d['visibility']) - elif isinstance(d['visibility'], list): # There are multiple visibility entries - for de in d['visibility']: - add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'visibility', de) - - name = yaml_content['name'] - platform = yaml_content['platform'] - except KeyError: - # When using an EQL that does not in a valid technique administration file. Trow an error. - print(EQL_INVALID_RESULT_TECH + ' detection/visibility object(s):') - pprint(yaml_content) - quit() + for d in yaml_content['techniques']: + # Add detection items: + if isinstance(d['detection'], dict): # There is just one detection entry + add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'detection', d['detection']) + elif isinstance(d['detection'], list): # There are multiple detection entries + for de in d['detection']: + add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'detection', de) + + # Add visibility items + if isinstance(d['visibility'], dict): # There is just one visibility entry + add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'visibility', d['visibility']) + elif isinstance(d['visibility'], list): # There are multiple visibility entries + for de in d['visibility']: + add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'visibility', de) + + name = yaml_content['name'] + platform = yaml_content['platform'] return my_techniques, name, platform @@ -685,7 +678,7 @@ def _check_health_score_object(yaml_object, object_type, tech_id, health_is_call :param yaml_object: YAML file lines :param object_type: 'detection' or 'visibility' :param tech_id: ATT&CK technique ID - :param health_is_called: boolean that specifies if detailed errors in the file will be printed and then quit() + :param health_is_called: boolean that specifies if detailed errors in the file will be printed :return: True if the YAML file is unhealthy, otherwise False """ has_error = False @@ -750,7 +743,7 @@ def _check_health_yaml_object(yaml_object, object_type, tech_id, health_is_calle :param yaml_object: YAML file lines :param object_type: 'detection' or 'visibility' :param tech_id: ATT&CK technique ID - :param health_is_called: boolean that specifies if detailed errors in the file will be printed and then quit() + :param health_is_called: boolean that specifies if detailed errors in the file will be printed :return: True if the YAML file is unhealthy, otherwise False """ has_error = False @@ -852,7 +845,7 @@ def check_yaml_file_health(filename, file_type, health_is_called): Check on error in the provided YAML file. :param filename: YAML file location :param file_type: currently only 'FILE_TYPE_TECHNIQUE_ADMINISTRATION' is being supported - :param health_is_called: boolean that specifies if detailed errors in the file will be printed and then quit() + :param health_is_called: boolean that specifies if detailed errors in the file will be printed :return: """ # first we check if the file was modified. Otherwise, the health check is skipped for performance reasons @@ -982,7 +975,7 @@ def check_file(filename, file_type=None, health_is_called=False): does the file contain errors. :param filename: path to a YAML file :param file_type: value to check against the 'file_type' key in the YAML file - :param health_is_called: boolean that specifies if detailed errors in the file will be printed by the function 'check_yaml_file_health' and then quit() + :param health_is_called: boolean that specifies if detailed errors in the file will be printed by the function 'check_yaml_file_health' :return: the file_type if present, else None is returned """ From 5c700690c6caa1bbf7a01c75fcc11b7408933298 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Tue, 13 Aug 2019 14:30:43 +0200 Subject: [PATCH 35/49] Added the possibility to use EQL queries. --- interactive_menu.py | 254 +++++++++++++++++++++++++++++--------------- 1 file changed, 171 insertions(+), 83 deletions(-) diff --git a/interactive_menu.py b/interactive_menu.py index 75c3d2b8..5a80fe26 100644 --- a/interactive_menu.py +++ b/interactive_menu.py @@ -2,8 +2,7 @@ from data_source_mapping import * from technique_mapping import * from group_mapping import * -from constants import * - +from eql_yaml import * groups = 'all' software_group = False @@ -12,8 +11,11 @@ default_matrix = 'enterprise' groups_overlay = '' overlay_type = 'group' -filter_applicable_to = 'all' yaml_path = 'sample-data/' +eql_all_scores = False +eql_query_detection = None +eql_query_visibility = None +eql_query_data_sources = None def _clear(): @@ -201,6 +203,8 @@ def _menu_statistics(): print('') print('Options:') print('1. Matrix: %s' % default_matrix) + print('') + print('Select what you want to do:') print('2. Get a sorted count on how many ATT&CK Enterprise techniques are covered by a particular Data Source.') print('3. Get a sorted count on how many ATT&CK Enterprise or Mobile techniques are covered by a Mitigation.') print('9. Back to main menu.') @@ -228,39 +232,56 @@ def _menu_data_source(filename_ds): :param filename_ds: :return: """ + global eql_query_data_sources _clear() print('Menu: %s' % MENU_NAME_DATA_SOURCE_MAPPING) print('') print('Selected data source YAML file: %s' % filename_ds) print('') + print('Options:') + eql_ds_str = '' if not eql_query_data_sources else eql_query_data_sources + print('1. Only include data sources which match the provided EQL query: ' + eql_ds_str) + print('') print('Select what you want to do:') - print('1. Generate a data source layer for the ATT&CK Navigator.') - print('2. Generate a graph with data sources added through time.') - print('3. Generate an Excel sheet with all data sources.') - print('4. Generate a technique administration YAML file with visibility scores, based on the number of available ' + print('2. Generate a data source layer for the ATT&CK Navigator.') + print('3. Generate a graph with data sources added through time.') + print('4. Generate an Excel sheet with all data sources.') + print('5. Generate a technique administration YAML file with visibility scores, based on the number of available ' 'data sources') - print('5. update the visibility scores within a technique administration YAML file based on changes within any of ' + print('6. update the visibility scores within a technique administration YAML file based on changes within any of ' 'the data sources. \nPast visibility scores are preserved in the score_logbook, and manually assigned scores are ' 'not updated without your approval. \nThe updated visibility are based on the number of available data sources.') print('9. Back to main menu.') choice = _ask_input() if choice == '1': - print('Writing data sources layer...') - generate_data_sources_layer(filename_ds) - _wait() - elif choice == '2': - print('Drawing the graph...') - plot_data_sources_graph(filename_ds) - _wait() - elif choice == '3': - print('Generating Excel file...') - export_data_source_list_to_excel(filename_ds) - _wait() - elif choice == '4': - print('Generating YAML file...') - generate_technique_administration_file(filename_ds) - _wait() - elif choice == '5': + print('Specify the EQL query for data source objects:') + eql_query_data_sources = _ask_input().lower() + + elif choice in ['2', '3', '4', '5']: + file_ds = filename_ds + + if eql_query_data_sources: + file_ds = search(filename_ds, FILE_TYPE_DATA_SOURCE_ADMINISTRATION, eql_query_data_sources) + if not file_ds: + _wait() # something went wrong in executing the search or 0 results where returned + _menu_data_source(filename_ds) + if choice == '2': + print('Writing data sources layer...') + generate_data_sources_layer(file_ds) + _wait() + elif choice == '3': + print('Drawing the graph...') + plot_data_sources_graph(file_ds) + _wait() + elif choice == '4': + print('Generating Excel file...') + export_data_source_list_to_excel(file_ds) + _wait() + elif choice == '5': + print('Generating YAML file...') + generate_technique_administration_file(file_ds) + _wait() + elif choice == '6': filename_t = _select_file(MENU_NAME_DETECTION_COVERAGE_MAPPING, 'techniques (used to score the level of visibility)', FILE_TYPE_TECHNIQUE_ADMINISTRATION, False) print('Updating visibility scores...') @@ -279,47 +300,68 @@ def _menu_detection(filename_t): :param filename_t: :return: """ - global filter_applicable_to + global eql_all_scores, eql_query_detection, eql_query_visibility + + filename_str = filename_t _clear() print('Menu: %s' % MENU_NAME_DETECTION_COVERAGE_MAPPING) print('') - print('Selected techniques YAML file: %s' % filename_t) + print('Selected techniques YAML file: %s' % filename_str) print('') print('Options:') - print('1. Filter techniques based on the \'applicable_to\' field in the technique administration YAML file (not ' - 'for Excel output): %s' % filter_applicable_to) + eql_d_str = '' if not eql_query_detection else eql_query_detection + eql_v_str = '' if not eql_query_visibility else eql_query_visibility + print('1. Only include detection objects which match the EQL query: ' + eql_d_str) + print('2. Only include visibility objects which match the EQL query: ' + eql_v_str) + print('3. Include all \'score\' objects from the \'score_logbook\' in the EQL search: ' + str(eql_all_scores)) print('') print('Select what you want to do:') - print('2. Generate a layer for detection coverage for the ATT&CK Navigator.') - print('3. Generate a layer for detection coverage overlaid with visibility for the ATT&CK Navigator.') - print('4. Generate a graph with detections added through time.') - print('5. Generate an Excel sheet with all administrated techniques.') - print('6. Check the technique YAML file for errors.') + print('4. Generate a layer for detection coverage for the ATT&CK Navigator.') + print('5. Generate a layer for detection coverage overlaid with visibility for the ATT&CK Navigator.') + print('6. Generate a graph with detections added through time.') + print('7. Generate an Excel sheet with all administrated techniques.') + print('8. Check the technique YAML file for errors.') print('9. Back to main menu.') choice = _ask_input() if choice == '1': - print('Specify your filter for the applicable_to field:') - filter_applicable_to = _ask_input().lower() + print('Specify the EQL query for detection objects:') + eql_query_detection = _ask_input().lower() elif choice == '2': - print('Writing detection coverage layer...') - generate_detection_layer(filename_t, None, False, filter_applicable_to) - _wait() + print('Specify the EQL query for visibility objects:') + eql_query_visibility = _ask_input().lower() elif choice == '3': - filename_ds = _select_file(MENU_NAME_DETECTION_COVERAGE_MAPPING, 'data sources (used to add metadata on the ' - 'involved data sources to the heat map)', - FILE_TYPE_DATA_SOURCE_ADMINISTRATION, False) - print('Writing detection coverage layer with visibility as overlay...') - generate_detection_layer(filename_t, filename_ds, True, filter_applicable_to) - _wait() - elif choice == '4': - print('Drawing the graph...') - plot_detection_graph(filename_t, filter_applicable_to) - _wait() - elif choice == '5': - print('Generating Excel file...') - export_techniques_list_to_excel(filename_t) - _wait() - elif choice == '6': + print('Specify True or False for to include all scores or not:') + eql_all_scores = True if _ask_input().lower() == 'true' else False + + elif choice in ['4', '5', '6', '7']: + file_tech = filename_t + + if eql_query_detection or eql_query_visibility: + file_tech = techniques_search(filename_t, eql_query_visibility, eql_query_detection, + include_all_score_objs=eql_all_scores) + if not file_tech: + _wait() # something went wrong in executing the search or 0 results where returned + _menu_detection(filename_t) + if choice == '4': + print('Writing detection coverage layer...') + generate_detection_layer(file_tech, None, False) + _wait() + elif choice == '5': + filename_ds = _select_file(MENU_NAME_DETECTION_COVERAGE_MAPPING, 'data sources (used to add metadata on the ' + 'involved data sources to the heat map)', + FILE_TYPE_DATA_SOURCE_ADMINISTRATION, False) + print('Writing detection coverage layer with visibility as overlay...') + generate_detection_layer(file_tech, filename_ds, True) + _wait() + elif choice == '6': + print('Drawing the graph...') + plot_detection_graph(file_tech) + _wait() + elif choice == '7': + print('Generating Excel file...') + export_techniques_list_to_excel(file_tech) + _wait() + elif choice == '8`x': print('Checking the technique YAML file for errors...') check_yaml_file_health(filename_t, FILE_TYPE_TECHNIQUE_ADMINISTRATION, health_is_called=True) _wait() @@ -337,43 +379,64 @@ def _menu_visibility(filename_t, filename_ds): :param filename_ds: :return: """ - global filter_applicable_to + global eql_all_scores, eql_query_detection, eql_query_visibility + + filename_str = filename_t _clear() print('Menu: %s' % MENU_NAME_VISIBILITY_MAPPING) print('') - print('Selected techniques YAML file: %s' % filename_t) + print('Selected techniques YAML file: %s' % filename_str) print('Selected data source YAML file: %s' % filename_ds) print('') print('Options:') - print('1. Filter techniques based on the \'applicable_to\' field in the technique administration YAML file (not for ' - 'Excel output): %s' % filter_applicable_to) + eql_d_str = '' if not eql_query_detection else eql_query_detection + eql_v_str = '' if not eql_query_visibility else eql_query_visibility + print('1. Only include visibility objects which match the EQL query: ' + eql_v_str) + print('2. Only include detection objects which match the EQL query: ' + eql_d_str) + print('3. Include all \'score\' objects from the \'score_logbook\' in the EQL search: ' + str(eql_all_scores)) print('') print('Select what you want to do:') - print('2. Generate a layer for visibility for the ATT&CK Navigator.') - print('3. Generate a layer for visibility overlaid with detection coverage for the ATT&CK Navigator.') - print('4. Generate an Excel sheet with all administrated techniques.') - print('5. Check the technique YAML file for errors.') + print('4. Generate a layer for visibility for the ATT&CK Navigator.') + print('5. Generate a layer for visibility overlaid with detection coverage for the ATT&CK Navigator.') + print('6. Generate an Excel sheet with all administrated techniques.') + print('7. Check the technique YAML file for errors.') print('9. Back to main menu.') choice = _ask_input() if choice == '1': - print('Specify your filter for the applicable_to field:') - filter_applicable_to = _ask_input().lower() + print('Specify the EQL query for visibility objects:') + eql_query_visibility = _ask_input().lower() elif choice == '2': - print('Writing visibility coverage layer...') - generate_visibility_layer(filename_t, filename_ds, False, filter_applicable_to) - _wait() + print('Specify the EQL query for detection objects:') + eql_query_detection = _ask_input().lower() elif choice == '3': - print('Writing visibility coverage layer overlaid with detections...') - generate_visibility_layer(filename_t, filename_ds, True, filter_applicable_to) - _wait() - elif choice == '4': - print('Generating Excel file...') - export_techniques_list_to_excel(filename_t) - _wait() - elif choice == '5': - print('Checking the technique YAML file for errors...') - check_yaml_file_health(filename_t, FILE_TYPE_TECHNIQUE_ADMINISTRATION, health_is_called=True) - _wait() + print('Specify True or False for to include all scores or not:') + eql_all_scores = True if _ask_input().lower() == 'true' else False + + elif choice in ['4', '5', '6', '7']: + file_tech = filename_t + + if eql_query_detection or eql_query_visibility: + file_tech = techniques_search(filename_t, eql_query_visibility, eql_query_detection, + include_all_score_objs=eql_all_scores) + if not file_tech: + _wait() # something went wrong in executing the search or 0 results where returned + _menu_visibility(filename_t, filename_ds) + if choice == '4': + print('Writing visibility coverage layer...') + generate_visibility_layer(file_tech, filename_ds, False) + _wait() + elif choice == '5': + print('Writing visibility coverage layer overlaid with detections...') + generate_visibility_layer(file_tech, filename_ds, True) + _wait() + elif choice == '6': + print('Generating Excel file...') + export_techniques_list_to_excel(file_tech) + _wait() + elif choice == '7': + print('Checking the technique YAML file for errors...') + check_yaml_file_health(file_tech, FILE_TYPE_TECHNIQUE_ADMINISTRATION, health_is_called=True) + _wait() elif choice == '9': interactive_menu() elif choice == 'q': @@ -386,7 +449,8 @@ def _menu_groups(): Prints and handles the Threat actor group mapping functionality. :return: """ - global groups, software_group, default_platform, default_stage, groups_overlay, overlay_type, filter_applicable_to + global groups, software_group, default_platform, default_stage, groups_overlay, overlay_type, eql_all_scores, \ + eql_query_detection, eql_query_visibility _clear() print('Menu: %s' % MENU_NAME_THREAT_ACTOR_GROUP_MAPPING) print('') @@ -398,9 +462,15 @@ def _menu_groups(): print('5. Overlay: ') print(' - %s: %s' % ('File' if os.path.exists(groups_overlay) else 'Groups', groups_overlay)) print(' - Type: %s' % overlay_type) - print('6. Filter techniques in the detection or visibility overlay based on the \'applicable_to\' field in the ' - 'technique administration YAML file: %s' % filter_applicable_to) + print(' - Type: %s' % overlay_type) + print('6. EQL search: ') + eql_d_str = '' if not eql_query_detection else eql_query_detection + eql_v_str = '' if not eql_query_visibility else eql_query_visibility + print(' - Only include detection objects which match the EQL query: ' + eql_d_str) + print(' - Only include visibility objects which match the EQL query: ' + eql_v_str) + print(' - Include all \'score\' objects from the \'score_logbook\' in the EQL search: ' + str(eql_all_scores)) print('') + print('Select what you want to do:') print('7. Generate a heat map layer.') print('9. Back to main menu.') choice = _ask_input() @@ -443,10 +513,28 @@ def _menu_groups(): overlay_type = '' groups_overlay = '' elif choice == '6': - print('Specify your filter for the applicable_to field:') - filter_applicable_to = _ask_input().lower() + print('') + print('1. Only include detection objects which match the EQL query: ' + eql_d_str) + print('2. Only include visibility objects which match the EQL query: ' + eql_v_str) + print('3. Include all \'score\' objects from the \'score_logbook\' in the EQL search: ' + str(eql_all_scores)) + choice = _ask_input() + + if choice == '1': + print('Specify the EQL query for detection objects:') + eql_query_detection = _ask_input().lower() + elif choice == '2': + print('Specify the EQL query for visibility objects:') + eql_query_visibility = _ask_input().lower() + elif choice == '3': + print('Specify True or False for to include all scores or not:') + eql_all_scores = True if _ask_input().lower() == 'true' else False + elif choice == '7': - generate_group_heat_map(groups, groups_overlay, overlay_type, default_stage, default_platform, software_group, filter_applicable_to) + if not generate_group_heat_map(groups, groups_overlay, overlay_type, default_stage, default_platform, + software_group, eql_query_visibility, eql_query_detection, False, + include_all_score_objs=eql_all_scores): + _wait() + _menu_groups() _wait() elif choice == '9': interactive_menu() From 4f045644f1fda81e6134c60589514735fb381537 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Thu, 15 Aug 2019 11:05:11 +0200 Subject: [PATCH 36/49] Fixed a bug that caused a crash when a technique_id was part of the exception list --- data_source_mapping.py | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/data_source_mapping.py b/data_source_mapping.py index 90cfd240..06190163 100644 --- a/data_source_mapping.py +++ b/data_source_mapping.py @@ -344,23 +344,25 @@ def update_technique_administration_file(file_data_sources, file_tech_admin): updated_vis_score_cnt = 0 for cur_tech, cur_values in cur_visibility_scores.items(): new_tech = _get_technique_yaml_obj(new_visibility_scores['techniques'], cur_tech) - new_score = new_tech['visibility']['score_logbook'][0]['score'] - - for cur_obj in cur_values['visibility']: - old_score = get_latest_score(cur_obj) - - if get_latest_auto_generated(cur_obj) and old_score != new_score: - auto_scored = True - updated_vis_score_cnt += 1 - elif old_score != new_score: - manually_scored = True - updated_vis_score_cnt += 1 - - if manually_scored and auto_scored: - mix_scores = True - manually_scored = False - auto_scored = False - break + if new_tech: # new_tech will be None if technique_id is part of the 'exception' list within the + # data source administration file + new_score = new_tech['visibility']['score_logbook'][0]['score'] + + for cur_obj in cur_values['visibility']: + old_score = get_latest_score(cur_obj) + + if get_latest_auto_generated(cur_obj) and old_score != new_score: + auto_scored = True + updated_vis_score_cnt += 1 + elif old_score != new_score: + manually_scored = True + updated_vis_score_cnt += 1 + + if manually_scored and auto_scored: + mix_scores = True + manually_scored = False + auto_scored = False + break # stop if none of the present visibility scores are eligible for an update if not mix_scores and not manually_scored and not auto_scored: From e4eca0116837518bbccd3b518f1337f173dbc2a8 Mon Sep 17 00:00:00 2001 From: Ruben Bouman Date: Thu, 15 Aug 2019 15:34:31 +0200 Subject: [PATCH 37/49] small improvements --- data_source_mapping.py | 14 ++++---------- interactive_menu.py | 2 +- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/data_source_mapping.py b/data_source_mapping.py index 90cfd240..635e5abd 100644 --- a/data_source_mapping.py +++ b/data_source_mapping.py @@ -117,16 +117,11 @@ def export_data_source_list_to_excel(filename): if d in my_data_sources.keys(): ds = my_data_sources[d] - tmp_date_1 = ds['date_registered'] - if isinstance(tmp_date_1, datetime): - tmp_date_1 = tmp_date_1.strftime('%Y-%m-%d') + date_registered = ds['date_registered'].strftime('%Y-%m-%d') if isinstance(ds['date_registered'], datetime) else ds['date_registered'] + date_connected = ds['date_connected'].strftime('%Y-%m-%d') if isinstance(ds['date_connected'], datetime) else ds['date_connected'] - tmp_date_2 = ds['date_connected'] - if isinstance(tmp_date_2, datetime): - tmp_date_2 = tmp_date_2.strftime('%Y-%m-%d') - - worksheet.write(y, 1, str(tmp_date_1).replace('None', ''), valign_top) - worksheet.write(y, 2, str(tmp_date_2).replace('None', ''), valign_top) + worksheet.write(y, 1, str(date_registered).replace('None', ''), valign_top) + worksheet.write(y, 2, str(date_connected).replace('None', ''), valign_top) worksheet.write(y, 3, ', '.join(ds['products']).replace('None', ''), valign_top) worksheet.write(y, 4, ds['comment'][:-1] if ds['comment'].endswith('\n') else ds['comment'], wrap_text) worksheet.write(y, 5, str(ds['available_for_data_analytics']), valign_top) @@ -140,7 +135,6 @@ def export_data_source_list_to_excel(filename): score_count = 0 for k, v in ds['data_quality'].items(): # the below DQ dimensions are given more weight in the calculation of the DQ score. - print(k) if k in ['device_completeness', 'data_field_completeness', 'retention']: score += (v * 2) score_count += 2 diff --git a/interactive_menu.py b/interactive_menu.py index 5a80fe26..db892434 100644 --- a/interactive_menu.py +++ b/interactive_menu.py @@ -94,7 +94,7 @@ def _select_file(title, what, expected_file_type, b_clear=True): :param title: title to print on top of this menu :param what: print for what purpose the file is selected :param expected_file_type: the expected file type of the YAML file - :param b_clear: _clear the terminal before showing this menu + :param b_clear: clear the terminal before showing this menu :return: filename of the selected file """ global yaml_path From 7ad8fe16c7891a93b8e412cf6938635db53b1f2c Mon Sep 17 00:00:00 2001 From: Ruben Bouman Date: Thu, 15 Aug 2019 16:00:06 +0200 Subject: [PATCH 38/49] added same kind of graph for visibility as for detection --- dettect.py | 8 ++++++-- interactive_menu.py | 17 ++++++++++------- technique_mapping.py | 11 ++++++----- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/dettect.py b/dettect.py index 04e01387..6abe6665 100644 --- a/dettect.py +++ b/dettect.py @@ -76,6 +76,8 @@ def _init_menu(): action='store_true') parser_visibility.add_argument('-o', '--overlay', help='generate a visibility layer overlaid with detections for ' 'the ATT&CK navigator', action='store_true') + parser_visibility.add_argument('-g', '--graph', help='generate a graph with visibility items added through time', + action='store_true') parser_visibility.add_argument('--health', help='check the technique YAML file for errors', action='store_true') # create the detection parser @@ -104,7 +106,7 @@ def _init_menu(): action='store_true') parser_detection.add_argument('-o', '--overlay', help='generate a detection layer overlaid with visibility for ' 'the ATT&CK navigator', action='store_true') - parser_detection.add_argument('-g', '--graph', help='generate a graph with detections added through time', + parser_detection.add_argument('-g', '--graph', help='generate a graph with detection items added through time', action='store_true') parser_detection.add_argument('--health', help='check the technique YAML file for errors', action='store_true') @@ -217,6 +219,8 @@ def _menu(menu_parser): generate_visibility_layer(file_tech, args.file_ds, False) if args.overlay: generate_visibility_layer(file_tech, args.file_ds, True) + if args.graph: + plot_graph(file_tech, 'visibility') if args.excel: export_techniques_list_to_excel(file_tech) @@ -248,7 +252,7 @@ def _menu(menu_parser): if args.overlay and check_file(args.file_ds, FILE_TYPE_DATA_SOURCE_ADMINISTRATION, args.health): generate_detection_layer(file_tech, args.file_ds, True) if args.graph: - plot_detection_graph(file_tech) + plot_graph(file_tech, 'detection') if args.excel: export_techniques_list_to_excel(file_tech) diff --git a/interactive_menu.py b/interactive_menu.py index db892434..22fc4dec 100644 --- a/interactive_menu.py +++ b/interactive_menu.py @@ -318,7 +318,7 @@ def _menu_detection(filename_t): print('Select what you want to do:') print('4. Generate a layer for detection coverage for the ATT&CK Navigator.') print('5. Generate a layer for detection coverage overlaid with visibility for the ATT&CK Navigator.') - print('6. Generate a graph with detections added through time.') + print('6. Generate a graph with detection items added through time.') print('7. Generate an Excel sheet with all administrated techniques.') print('8. Check the technique YAML file for errors.') print('9. Back to main menu.') @@ -332,7 +332,6 @@ def _menu_detection(filename_t): elif choice == '3': print('Specify True or False for to include all scores or not:') eql_all_scores = True if _ask_input().lower() == 'true' else False - elif choice in ['4', '5', '6', '7']: file_tech = filename_t @@ -355,7 +354,7 @@ def _menu_detection(filename_t): _wait() elif choice == '6': print('Drawing the graph...') - plot_detection_graph(file_tech) + plot_graph(file_tech, 'detection') _wait() elif choice == '7': print('Generating Excel file...') @@ -398,8 +397,9 @@ def _menu_visibility(filename_t, filename_ds): print('Select what you want to do:') print('4. Generate a layer for visibility for the ATT&CK Navigator.') print('5. Generate a layer for visibility overlaid with detection coverage for the ATT&CK Navigator.') - print('6. Generate an Excel sheet with all administrated techniques.') - print('7. Check the technique YAML file for errors.') + print('6. Generate a graph with visibility items added through time.') + print('7. Generate an Excel sheet with all administrated techniques.') + print('8. Check the technique YAML file for errors.') print('9. Back to main menu.') choice = _ask_input() if choice == '1': @@ -411,7 +411,6 @@ def _menu_visibility(filename_t, filename_ds): elif choice == '3': print('Specify True or False for to include all scores or not:') eql_all_scores = True if _ask_input().lower() == 'true' else False - elif choice in ['4', '5', '6', '7']: file_tech = filename_t @@ -430,10 +429,14 @@ def _menu_visibility(filename_t, filename_ds): generate_visibility_layer(file_tech, filename_ds, True) _wait() elif choice == '6': + print('Drawing the graph...') + plot_graph(file_tech, 'visibility') + _wait() + elif choice == '7': print('Generating Excel file...') export_techniques_list_to_excel(file_tech) _wait() - elif choice == '7': + elif choice == '8': print('Checking the technique YAML file for errors...') check_yaml_file_health(file_tech, FILE_TYPE_TECHNIQUE_ADMINISTRATION, health_is_called=True) _wait() diff --git a/technique_mapping.py b/technique_mapping.py index e6b135a6..5046a4d3 100644 --- a/technique_mapping.py +++ b/technique_mapping.py @@ -49,18 +49,19 @@ def generate_visibility_layer(filename_techniques, filename_data_sources, overla _write_layer(layer_both, mapped_techniques_both, 'visibility_and_detection', name) -def plot_detection_graph(filename): +def plot_graph(filename, type): """ Generates a line graph which shows the improvements on detections through the time. :param filename: the filename of the YAML file containing the techniques administration + :param type: indicates the type of the graph: detection or visibility :return: """ my_techniques, name, platform = load_techniques(filename) graph_values = [] for t in my_techniques.values(): - for detection in t['detection']: - date = get_latest_date(detection) + for item in t[type]: + date = get_latest_date(item) if date: yyyymm = date.strftime('%Y-%m') graph_values.append({'date': yyyymm, 'count': 1}) @@ -69,12 +70,12 @@ def plot_detection_graph(filename): df = pd.DataFrame(graph_values).groupby('date', as_index=False)[['count']].sum() df['cumcount'] = df.ix[::1, 'count'].cumsum()[::1] - output_filename = 'output/graph_detection.html' + output_filename = 'output/graph_%s.html' % type import plotly import plotly.graph_objs as go plotly.offline.plot( {'data': [go.Scatter(x=df['date'], y=df['cumcount'])], - 'layout': go.Layout(title="# of detections for %s" % name)}, + 'layout': go.Layout(title="# of %s items for %s" % (type, name))}, filename=output_filename, auto_open=False ) print("File written: " + output_filename) From 84f9f0440a8d11773f9ff5df698be4324f01b5da Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Thu, 15 Aug 2019 20:31:20 +0200 Subject: [PATCH 39/49] - Non-MITRE ATT&CK data sources are now also exported to Excel. - Any ATT&CK data sources that are missing within the YAML file are added to the Excel with a comment stating it is missing. --- constants.py | 13 ++++++++ data_source_mapping.py | 71 ++++++++++++++++++++++++------------------ generic.py | 4 +-- 3 files changed, 55 insertions(+), 33 deletions(-) diff --git a/constants.py b/constants.py index 596f2904..a4c43f81 100644 --- a/constants.py +++ b/constants.py @@ -144,6 +144,19 @@ 'detection': YAML_OBJ_DETECTION, 'visibility': YAML_OBJ_VISIBILITY} +YAML_OBJ_DATA_SOURCE = {'data_source_name': '', + 'date_registered': None, + 'date_connected': None, + 'products': [''], + 'available_for_data_analytics': False, + 'comment': '', + 'data_quality': { + 'device_completeness': 0, + 'data_field_completeness': 0, + 'timeliness': 0, + 'consistency': 0, + 'retention': 0}} + # Interactive menu MENU_NAME_DATA_SOURCE_MAPPING = 'Data source mapping' MENU_NAME_VISIBILITY_MAPPING = 'Visibility coverage mapping' diff --git a/data_source_mapping.py b/data_source_mapping.py index 0bc039a3..9416957c 100644 --- a/data_source_mapping.py +++ b/data_source_mapping.py @@ -112,39 +112,48 @@ def export_data_source_list_to_excel(filename): # Putting the data sources data: y = 3 - for d in get_all_mitre_data_sources(): + + # check if an ATT&CK data source is missing from the data source YAML administration file + my_ds_list = my_data_sources.keys() + for ds in get_all_mitre_data_sources(): + if ds not in my_ds_list: + ds_obj = deepcopy(YAML_OBJ_DATA_SOURCE) + ds_obj['data_source_name'] = ds + ds_obj['comment'] = 'ATT&CK data source is missing from the YAML file' + my_data_sources[ds] = ds_obj + + for d in sorted(my_data_sources.keys()): + ds = my_data_sources[d] worksheet.write(y, 0, d, valign_top) - if d in my_data_sources.keys(): - ds = my_data_sources[d] - - date_registered = ds['date_registered'].strftime('%Y-%m-%d') if isinstance(ds['date_registered'], datetime) else ds['date_registered'] - date_connected = ds['date_connected'].strftime('%Y-%m-%d') if isinstance(ds['date_connected'], datetime) else ds['date_connected'] - - worksheet.write(y, 1, str(date_registered).replace('None', ''), valign_top) - worksheet.write(y, 2, str(date_connected).replace('None', ''), valign_top) - worksheet.write(y, 3, ', '.join(ds['products']).replace('None', ''), valign_top) - worksheet.write(y, 4, ds['comment'][:-1] if ds['comment'].endswith('\n') else ds['comment'], wrap_text) - worksheet.write(y, 5, str(ds['available_for_data_analytics']), valign_top) - worksheet.write(y, 6, ds['data_quality']['device_completeness'], format_center_valign_top) - worksheet.write(y, 7, ds['data_quality']['data_field_completeness'], format_center_valign_top) - worksheet.write(y, 8, ds['data_quality']['timeliness'], format_center_valign_top) - worksheet.write(y, 9, ds['data_quality']['consistency'], format_center_valign_top) - worksheet.write(y, 10, ds['data_quality']['retention'], format_center_valign_top) - - score = 0 - score_count = 0 - for k, v in ds['data_quality'].items(): - # the below DQ dimensions are given more weight in the calculation of the DQ score. - if k in ['device_completeness', 'data_field_completeness', 'retention']: - score += (v * 2) - score_count += 2 - else: - score += v - score_count += 1 - if score > 0: - score = score/score_count - worksheet.write(y, 11, score, dq_score_1 if score < 2 else dq_score_2 if score < 3 else dq_score_3 if score < 4 else dq_score_4 if score < 5 else dq_score_5 if score < 6 else no_score) + date_registered = ds['date_registered'].strftime('%Y-%m-%d') if isinstance(ds['date_registered'], datetime) else ds['date_registered'] + date_connected = ds['date_connected'].strftime('%Y-%m-%d') if isinstance(ds['date_connected'], datetime) else ds['date_connected'] + + worksheet.write(y, 1, str(date_registered).replace('None', ''), valign_top) + worksheet.write(y, 2, str(date_connected).replace('None', ''), valign_top) + worksheet.write(y, 3, ', '.join(ds['products']).replace('None', ''), valign_top) + worksheet.write(y, 4, ds['comment'][:-1] if ds['comment'].endswith('\n') else ds['comment'], wrap_text) + worksheet.write(y, 5, str(ds['available_for_data_analytics']), valign_top) + worksheet.write(y, 6, ds['data_quality']['device_completeness'], format_center_valign_top) + worksheet.write(y, 7, ds['data_quality']['data_field_completeness'], format_center_valign_top) + worksheet.write(y, 8, ds['data_quality']['timeliness'], format_center_valign_top) + worksheet.write(y, 9, ds['data_quality']['consistency'], format_center_valign_top) + worksheet.write(y, 10, ds['data_quality']['retention'], format_center_valign_top) + + score = 0 + score_count = 0 + for k, v in ds['data_quality'].items(): + # the below DQ dimensions are given more weight in the calculation of the DQ score. + if k in ['device_completeness', 'data_field_completeness', 'retention']: + score += (v * 2) + score_count += 2 + else: + score += v + score_count += 1 + if score > 0: + score = score/score_count + + worksheet.write(y, 11, score, dq_score_1 if score < 2 else dq_score_2 if score < 3 else dq_score_3 if score < 4 else dq_score_4 if score < 5 else dq_score_5 if score < 6 else no_score) y += 1 worksheet.autofilter(2, 0, 2, 11) diff --git a/generic.py b/generic.py index 9e879397..340571c3 100644 --- a/generic.py +++ b/generic.py @@ -579,7 +579,7 @@ def map_techniques_to_data_sources(techniques, my_data_sources): def get_all_mitre_data_sources(): """ - Gets all the data sources from the techniques and make a unique sorted list of it. + Gets all the data sources from the techniques and make a set. :return: a sorted list with all data sources """ techniques = load_attack_data(DATA_TYPE_STIX_ALL_TECH) @@ -589,7 +589,7 @@ def get_all_mitre_data_sources(): if 'x_mitre_data_sources' in t.keys(): for ds in t['x_mitre_data_sources']: data_sources.add(ds) - return sorted(data_sources) + return data_sources def calculate_score(list_detections, zero_value=0): From 90fc9278c92365c44f693c75d09b2ed5b31ee6ac Mon Sep 17 00:00:00 2001 From: Ruben Bouman Date: Tue, 20 Aug 2019 09:15:41 +0200 Subject: [PATCH 40/49] Don't overwrite output files if they already exist, but append a number to the filename as suffix. --- data_source_mapping.py | 18 ++++-------------- generic.py | 35 +++++++++++++++++++++++++++++++++++ technique_mapping.py | 10 ++++------ 3 files changed, 43 insertions(+), 20 deletions(-) diff --git a/data_source_mapping.py b/data_source_mapping.py index 0bc039a3..b1111246 100644 --- a/data_source_mapping.py +++ b/data_source_mapping.py @@ -23,10 +23,7 @@ def generate_data_sources_layer(filename): layer['techniques'] = my_techniques json_string = simplejson.dumps(layer).replace('}, ', '},\n') - output_filename = 'output/data_sources_' + normalize_name_to_filename(name) + '.json' - with open(output_filename, 'w') as f: - f.write(json_string) - print('File written: ' + output_filename) + write_file('data_sources', name, json_string) def plot_data_sources_graph(filename): @@ -47,7 +44,7 @@ def plot_data_sources_graph(filename): df = pd.DataFrame(graph_values).groupby('date', as_index=False)[['count']].sum() df['cumcount'] = df.ix[::1, 'count'].cumsum()[::1] - output_filename = 'output/graph_data_sources.html' + output_filename = get_non_existing_filename('output/graph_data_sources', 'html') import plotly import plotly.graph_objs as go @@ -68,7 +65,7 @@ def export_data_source_list_to_excel(filename): """ my_data_sources, name, platform, exceptions = _load_data_sources(filename, filter_empty_scores=False) - excel_filename = 'output/data_sources.xlsx' + excel_filename = get_non_existing_filename('output/data_sources', 'xlsx') workbook = xlsxwriter.Workbook(excel_filename) worksheet = workbook.add_worksheet('Data sources') @@ -535,14 +532,7 @@ def generate_technique_administration_file(filename, write_file=True): # remove the single quotes from the date yaml_file_lines = fix_date_and_remove_null(file_lines, today, input_type='list') - # create a output filename that prevents overwriting any existing files - output_filename = 'output/techniques-administration-' + normalize_name_to_filename(name+'-'+platform) + '.yaml' - suffix = 1 - while os.path.exists(output_filename): - output_filename = 'output/techniques-administration-' + normalize_name_to_filename(name + '-' + platform) + \ - '_' + str(suffix) + '.yaml' - suffix += 1 - + output_filename = get_non_existing_filename('output/techniques-administration-' + normalize_name_to_filename(name+'-'+platform), 'yaml') with open(output_filename, 'w') as f: f.writelines(yaml_file_lines) print("File written: " + output_filename) diff --git a/generic.py b/generic.py index 9e879397..97c22774 100644 --- a/generic.py +++ b/generic.py @@ -334,6 +334,41 @@ def get_layer_template_layered(name, description, stage, platform): return layer +def write_file(filename_prefix, filename, content): + """ + Writes content to a file and ensures if the file already exists it won't be overwritten by appending a number + as suffix. + :param filename_prefix: prefix part of the filename + :param filename: filename + :param content: the content of the file that needs to be written to the file + :return: + """ + output_filename = 'output/%s_%s' % (filename_prefix, normalize_name_to_filename(filename)) + output_filename = get_non_existing_filename(output_filename, 'json') + + with open(output_filename, 'w') as f: + f.write(content) + + print('File written: ' + output_filename) + + +def get_non_existing_filename(filename, extension): + """ + Generates a filename that doesn't exist based on the given filename by appending a number as suffix. + :param filename: + :param extension: + :return: + """ + if os.path.exists('%s.%s' % (filename, extension)): + suffix = 1 + while os.path.exists('%s_%s.%s' % (filename, suffix, extension)): + suffix += 1 + output_filename = '%s_%s.%s' % (filename, suffix, extension) + else: + output_filename = '%s.%s' % (filename, extension) + return output_filename + + def backup_file(filename): """ Create a backup of the provided file diff --git a/technique_mapping.py b/technique_mapping.py index 5046a4d3..aada4775 100644 --- a/technique_mapping.py +++ b/technique_mapping.py @@ -70,7 +70,8 @@ def plot_graph(filename, type): df = pd.DataFrame(graph_values).groupby('date', as_index=False)[['count']].sum() df['cumcount'] = df.ix[::1, 'count'].cumsum()[::1] - output_filename = 'output/graph_%s.html' % type + output_filename = get_non_existing_filename('output/graph_%s' % type, 'html') + import plotly import plotly.graph_objs as go plotly.offline.plot( @@ -124,10 +125,7 @@ def _write_layer(layer, mapped_techniques, filename_prefix, name): layer['techniques'] = mapped_techniques json_string = simplejson.dumps(layer).replace('}, ', '},\n') - output_filename = normalize_name_to_filename('output/%s_%s.json' % (filename_prefix, name)) - with open(output_filename, 'w') as f: - f.write(json_string) - print("File written: " + output_filename) + write_file(filename_prefix, name, json_string) def _map_and_colorize_techniques_for_detections(my_techniques): @@ -347,7 +345,7 @@ def export_techniques_list_to_excel(filename): my_techniques = dict(sorted(my_techniques.items(), key=lambda kv: kv[0], reverse=False)) mitre_techniques = load_attack_data(DATA_TYPE_STIX_ALL_TECH) - excel_filename = 'output/techniques.xlsx' + excel_filename = get_non_existing_filename('output/techniques', 'xlsx') workbook = xlsxwriter.Workbook(excel_filename) worksheet_detections = workbook.add_worksheet('Detections') worksheet_visibility = workbook.add_worksheet('Visibility') From 47dfc8bb8dfb9ac7fa096b5d857450f79ff3a35e Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Tue, 20 Aug 2019 11:06:09 +0200 Subject: [PATCH 41/49] Bumped the version to 1.2.0 --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4b333224..f264fbc5 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,9 @@ DeTT&CT #### Detect Tactics, Techniques & Combat Threats -Latest version: [1.1.2](https://github.com/rabobank-cdc/DeTTECT/wiki/Changelog#version-112) +Latest version: [1.2.0](https://github.com/rabobank-cdc/DeTTECT/wiki/Changelog#version-120) -To get started with DeTT&CT, check out the -[Wiki](https://github.com/rabobank-cdc/DeTTECT/wiki/Getting-started). +To get started with DeTT&CT, check out this [page](https://github.com/rabobank-cdc/DeTTECT/wiki/Getting-started) and our [blog](https://split.to/FkqwE7U). DeTT&CT aims to assist blue teams using ATT&CK to score and compare data log source quality, visibility coverage, detection coverage and threat actor behaviours. All of which can help, in different ways, to get more resilient against attacks targeting your organisation. The DeTT&CT framework consists of a Python tool, YAML administration files and [scoring tables](https://github.com/rabobank-cdc/DeTTECT/raw/master/scoring_table.xlsx) for the different aspects. From 248c6a07d853a529a9b87a2fdea6187e3812cffd Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Tue, 20 Aug 2019 11:13:25 +0200 Subject: [PATCH 42/49] Added an extra check for a possible empty 'comment' key-value pair. --- upgrade.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/upgrade.py b/upgrade.py index 9f2b6c4a..5f7271a6 100644 --- a/upgrade.py +++ b/upgrade.py @@ -7,7 +7,7 @@ def _load_techniques(yaml_file_lines): :param yaml_file_lines: list with the YAML file lines containing the techniques administration :return: dictionary with techniques (incl. properties) """ - from generic import add_entry_to_list_in_dictionary, init_yaml + from generic import add_entry_to_list_in_dictionary, init_yaml, set_yaml_dv_comments my_techniques = {} _yaml = init_yaml() @@ -15,16 +15,20 @@ def _load_techniques(yaml_file_lines): for d in yaml_content['techniques']: # Add detection items if isinstance(d['detection'], dict): # There is just one detection entry + d['detection'] = set_yaml_dv_comments(d['detection']) add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'detection', d['detection']) elif isinstance(d['detection'], list): # There are multiple detection entries for de in d['detection']: + de = set_yaml_dv_comments(de) add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'detection', de) # Add visibility items if isinstance(d['visibility'], dict): # There is just one visibility entry + d['visibility'] = set_yaml_dv_comments(d['visibility']) add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'visibility', d['visibility']) elif isinstance(d['visibility'], list): # There are multiple visibility entries for de in d['visibility']: + de = set_yaml_dv_comments(de) add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'visibility', de) return my_techniques From 98067447c6fd8351b6e5e5814e82c595e9418719 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Tue, 20 Aug 2019 11:14:07 +0200 Subject: [PATCH 43/49] Implemented a health check for data source administration YAML files. --- dettect.py | 17 +-- health.py | 307 ++++++++++++++++++++++++++++++++++++++++++++ interactive_menu.py | 11 +- 3 files changed, 324 insertions(+), 11 deletions(-) create mode 100644 health.py diff --git a/dettect.py b/dettect.py index 6abe6665..4c67868b 100644 --- a/dettect.py +++ b/dettect.py @@ -51,6 +51,7 @@ def _init_menu(): 'not updated without your approval. The updated visibility ' 'scores are calculated in the same way as with the option: ' '-y, --yaml', action='store_true') + parser_data_sources.add_argument('--health', help='check the YAML file(s) for errors', action='store_true') # create the visibility parser parser_visibility = subparsers.add_parser('visibility', aliases=['v'], @@ -76,9 +77,9 @@ def _init_menu(): action='store_true') parser_visibility.add_argument('-o', '--overlay', help='generate a visibility layer overlaid with detections for ' 'the ATT&CK navigator', action='store_true') - parser_visibility.add_argument('-g', '--graph', help='generate a graph with visibility items added through time', - action='store_true') - parser_visibility.add_argument('--health', help='check the technique YAML file for errors', action='store_true') + parser_visibility.add_argument('-g', '--graph', help='generate a graph with visibility added through time', + action='store_true') + parser_visibility.add_argument('--health', help='check the YAML file for errors', action='store_true') # create the detection parser parser_detection = subparsers.add_parser('detection', aliases=['d'], @@ -106,9 +107,9 @@ def _init_menu(): action='store_true') parser_detection.add_argument('-o', '--overlay', help='generate a detection layer overlaid with visibility for ' 'the ATT&CK navigator', action='store_true') - parser_detection.add_argument('-g', '--graph', help='generate a graph with detection items added through time', + parser_detection.add_argument('-g', '--graph', help='generate a graph with detections added through time', action='store_true') - parser_detection.add_argument('--health', help='check the technique YAML file for errors', action='store_true') + parser_detection.add_argument('--health', help='check the YAML file(s) for errors', action='store_true') # create the group parser parser_group = subparsers.add_parser('group', aliases=['g'], @@ -145,7 +146,7 @@ def _init_menu(): 'the EQL search. The default behaviour is to only include the ' 'most recent \'score\' objects', action='store_true', default=False) - parser_group.add_argument('--health', help='check the technique YAML file for errors', action='store_true') + parser_group.add_argument('--health', help='check the YAML file(s) for errors', action='store_true') # create the generic parser parser_generic = subparsers.add_parser('generic', description='Generic functions which will output to stdout.', @@ -180,14 +181,14 @@ def _menu(menu_parser): interactive_menu() elif args.subparser in ['datasource', 'ds']: - if check_file(args.file_ds, FILE_TYPE_DATA_SOURCE_ADMINISTRATION): + if check_file(args.file_ds, FILE_TYPE_DATA_SOURCE_ADMINISTRATION, args.health): file_ds = args.file_ds if args.search: file_ds = search(args.file_ds, FILE_TYPE_DATA_SOURCE_ADMINISTRATION, args.search) if not file_ds: quit() # something went wrong in executing the search or 0 results where returned - if args.update and check_file(args.file_tech, FILE_TYPE_TECHNIQUE_ADMINISTRATION): + if args.update and check_file(args.file_tech, FILE_TYPE_TECHNIQUE_ADMINISTRATION, args.health): update_technique_administration_file(file_ds, args.file_tech) if args.layer: generate_data_sources_layer(file_ds) diff --git a/health.py b/health.py new file mode 100644 index 00000000..54cefcd1 --- /dev/null +++ b/health.py @@ -0,0 +1,307 @@ +import os +import pickle +from difflib import SequenceMatcher +from constants import * + + +def _print_error_msg(msg, print_error): + if print_error: + print(msg) + return True + + +def _update_health_state(current, update): + if current or update: + return True + else: + return update + + +def _is_file_modified(filename): + """ + Check if the provided file was modified since the last check + :param filename: file location + :return: true when modified else false + """ + last_modified_file = 'cache/last-modified_' + os.path.basename(filename).rstrip('.yaml') + + def _update_modified_date(date): + with open(last_modified_file, 'wb') as fd: + pickle.dump(date, fd) + + if not os.path.exists(last_modified_file): + last_modified = os.path.getmtime(filename) + _update_modified_date(last_modified) + + return True + else: + with open(last_modified_file, 'rb') as f: + last_modified_cache = pickle.load(f) + last_modified_current = os.path.getmtime(filename) + + if last_modified_cache != last_modified_current: + _update_modified_date(last_modified_current) + return True + else: + return False + + +def _get_health_state_cache(filename): + """ + Get file health state from disk + :param filename: file location + :return: the cached error state + """ + last_error_file = 'cache/last-error-state_' + os.path.basename(filename).rstrip('.yaml') + + if os.path.exists(last_error_file): + with open(last_error_file, 'rb') as f: + last_error_state_cache = pickle.load(f) + + return last_error_state_cache + + +def _update_health_state_cache(filename, has_error): + """ + Write the file health state to disk if changed + :param filename: file location + """ + # the function 'check_health_data_sources' will call this function without providing a filename when + # 'check_health_data_sources' is called from '_events_to_yaml' within 'eql_yaml.py' + if filename: + last_error_file = 'cache/last-error-state_' + os.path.basename(filename).rstrip('.yaml') + + def _update(error): + with open(last_error_file, 'wb') as fd: + pickle.dump(error, fd) + + if not os.path.exists(last_error_file): + _update(has_error) + else: + error_state_cache = _get_health_state_cache(filename) + if error_state_cache != has_error: + _update(has_error) + + +def check_health_data_sources(filename, ds_content, health_is_called, no_print=False): + """ + Check on errors in the provided data sources administration YAML file. + :param filename: YAML file location + :param ds_content: content of the YAML file in a list of dicts + :param health_is_called: boolean that specifies if detailed errors in the file will be printed to stdout + :param no_print: specifies if the non-detailed error message is printed to stdout or not + :return: False if no errors have been found, otherwise True + """ + has_error = False + + for ds in ds_content['data_sources']: + # check for missing keys + for key in ['data_source_name', 'date_registered', 'date_connected', 'products', 'available_for_data_analytics', 'comment', 'data_quality']: + if key not in ds: + has_error = _print_error_msg('[!] Data source: \'' + ds['data_source_name'] + '\' is MISSING a key-value pair: ' + key, health_is_called) + + for key in ['date_registered', 'date_connected']: + if key in ds and not ds[key] is None: + try: + # noinspection PyStatementEffect + ds[key].year + # noinspection PyStatementEffect + ds[key].month + # noinspection PyStatementEffect + ds[key].day + except AttributeError: + has_error = _print_error_msg('[!] Data source: \'' + ds['data_source_name'] + '\' has an INVALID data format for the dimension \'' + dimension + + '\': ' + ds[key] + ' (should be YYYY-MM-DD without quotes)', health_is_called) + + if 'available_for_data_analytics' in ds: + if not isinstance(ds['available_for_data_analytics'], bool): + has_error = _print_error_msg('[!] Data source: \'' + ds['data_source_name'] + '\' has an INVALID \'available_for_data_analytics\' value: should be set to \'true\' or \'false\'', health_is_called) + + if 'data_quality' in ds: + if isinstance(ds['data_quality'], dict): + for dimension in ['device_completeness', 'data_field_completeness', 'timeliness', 'consistency', 'retention']: + if dimension not in ds['data_quality']: + has_error = _print_error_msg('[!] Data source: \'' + ds['data_source_name'] + '\' is MISSING a key-value pair in \'data_quality\': ' + dimension, health_is_called) + else: + if isinstance(ds['data_quality'][dimension], int): + if not 0 <= ds['data_quality'][dimension] <= 5: + has_error = _print_error_msg('[!] Data source: \'' + ds['data_source_name'] + '\' has an INVALID data quality score for the dimension \'' + + dimension + '\': ' + str(ds['data_quality'][dimension]) + ' (should be between 0 and 5)', health_is_called) + else: + has_error = _print_error_msg('[!] Data source: \'' + ds['data_source_name'] + '\' has an INVALID data quality score for the dimension \'' + + dimension + '\': ' + str(ds['data_quality'][dimension]) + ' (should be an an integer)', health_is_called) + else: + has_error = _print_error_msg('[!] Data source: \'' + ds['data_source_name'] + '\' the key-value pair \'data_quality\' is NOT a dictionary with data quality dimension scores', health_is_called) + + if has_error and not health_is_called and not no_print: + print(HEALTH_ERROR_TXT + filename) + + _update_health_state_cache(filename, has_error) + + return has_error + + +def _check_health_score_object(yaml_object, object_type, tech_id, health_is_called): + """ + Check the health of a score_logbook inside a visibility or detection YAML object + :param yaml_object: YAML file lines + :param object_type: 'detection' or 'visibility' + :param tech_id: ATT&CK technique ID + :param health_is_called: boolean that specifies if detailed errors in the file will be printed + :return: True if the YAML file is unhealthy, otherwise False + """ + has_error = False + min_score = None + max_score = None + + if object_type == 'detection': + min_score = -1 + max_score = 5 + elif object_type == 'visibility': + min_score = 0 + max_score = 4 + + if not isinstance(yaml_object['score_logbook'], list): + yaml_object['score_logbook'] = [yaml_object['score_logbook']] + + try: + for score_obj in yaml_object['score_logbook']: + for key in ['date', 'score', 'comment']: + if key not in score_obj: + has_error = _print_error_msg('[!] Technique ID: ' + tech_id + ' is MISSING a key-value pair in a ' + object_type + ' score object within the \'score_logbook\': ' + key, health_is_called) + + if score_obj['score'] is None: + has_error = _print_error_msg('[!] Technique ID: ' + tech_id + ' has an EMPTY key-value pair in a ' + object_type + ' score object within the \'score_logbook\': score', health_is_called) + + elif not isinstance(score_obj['score'], int): + has_error = _print_error_msg('[!] Technique ID: ' + tech_id + ' has an INVALID score format in a ' + object_type + ' score object within the \'score_logbook\': ' + score_obj['score'] + ' (should be an integer)', health_is_called) + + if 'auto_generated' in score_obj: + if not isinstance(score_obj['auto_generated'], bool): + has_error = _print_error_msg( + '[!] Technique ID: ' + tech_id + ' has an INVALID \'auto_generated\' value in a ' + object_type + ' score object within the \'score_logbook\': should be set to \'true\' or \'false\'', health_is_called) + + if isinstance(score_obj['score'], int): + if score_obj['date'] is None and score_obj['score'] > -1: + has_error = _print_error_msg('[!] Technique ID: ' + tech_id + ' has an EMPTY key-value pair in a ' + object_type + ' score object within the \'score_logbook\': date', health_is_called) + + # noinspection PyChainedComparisons + if not (score_obj['score'] >= min_score and score_obj['score'] <= max_score): + has_error = _print_error_msg( + '[!] Technique ID: ' + tech_id + ' has an INVALID ' + object_type + ' score in a score object within the \'score_logbook\': ' + str(score_obj['score']) + ' (should be between ' + str(min_score) + ' and ' + str(max_score) + ')', health_is_called) + + if not score_obj['date'] is None: + try: + # noinspection PyStatementEffect + score_obj['date'].year + # noinspection PyStatementEffect + score_obj['date'].month + # noinspection PyStatementEffect + score_obj['date'].day + except AttributeError: + has_error = _print_error_msg('[!] Technique ID: ' + tech_id + ' has an INVALID data format in a ' + object_type + ' score object within the \'score_logbook\': ' + score_obj['date'] + ' (should be YYYY-MM-DD without quotes)', health_is_called) + except KeyError: + pass + + return has_error + + +def _check_health_techniques(filename, technique_content, health_is_called): + """ + Check on errors in the provided technique administration YAML file. + :param filename: YAML file location + :param technique_content: content of the YAML file in a list of dicts + :param health_is_called: boolean that specifies if detailed errors in the file will be printed to stdout + :return: + """ + from generic import load_techniques + + has_error = False + + # create a list of ATT&CK technique IDs and check for duplicates + tech_ids = list(map(lambda x: x['technique_id'], technique_content['techniques'])) + tech_dup = set() + for tech in tech_ids: + if tech not in tech_dup: + tech_dup.add(tech) + else: + has_error = _print_error_msg('[!] Duplicate technique ID: ' + tech, health_is_called) + + # check if the technique has a valid format + if not REGEX_YAML_TECHNIQUE_ID_FORMAT.match(tech): + has_error = _print_error_msg('[!] Invalid technique ID: ' + tech, health_is_called) + + all_applicable_to = set() + + techniques = load_techniques(filename) + for tech, v in techniques[0].items(): + for obj_type in ['detection', 'visibility']: + if obj_type not in v: + has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is MISSING a key-value pair: ' + obj_type, health_is_called) + else: + for obj in v[obj_type]: + obj_keys = ['applicable_to', 'comment', 'score_logbook'] + obj_keys_list = ['applicable_to'] + if obj_type == 'detection': + obj_keys.append('location') + obj_keys_list.append('location') + + for okey in obj_keys: + if okey not in obj: + has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is MISSING a key-value pair in \'' + obj_type + '\': ' + okey, health_is_called) + + for okey in obj_keys_list: + if okey in obj: + if not isinstance(obj[okey], list): + has_error = _print_error_msg('[!] Technique ID: ' + tech + ' the key-value pair \'' + okey + '\' in \'' + obj_type + '\' is NOT a list', health_is_called) + + health = _check_health_score_object(obj, obj_type, tech, health_is_called) + has_error = _update_health_state(has_error, health) + + if 'applicable_to' in obj and isinstance(obj['applicable_to'], list): + all_applicable_to.update(obj['applicable_to']) + + # get values within the key-value pair 'applicable_to' and 'location' which are a very close match + similar = set() + for i1 in all_applicable_to: + for i2 in all_applicable_to: + match_value = SequenceMatcher(None, i1, i2).ratio() + if match_value > 0.8 and match_value != 1: + similar.add(i1) + similar.add(i2) + + if len(similar) > 0: + has_error = _print_error_msg('[!] There are values in the key-value pairs for \'applicable_to\' which are very similar. Correct where necessary:', health_is_called) + for s in similar: + _print_error_msg(' - ' + s, health_is_called) + + if has_error and not health_is_called: + print(HEALTH_ERROR_TXT + filename) + + _update_health_state_cache(filename, has_error) + + +def check_yaml_file_health(filename, file_type, health_is_called): + """ + Check on errors in the provided YAML file. + :param filename: YAML file location + :param file_type: currently FILE_TYPE_TECHNIQUE_ADMINISTRATION and FILE_TYPE_DATA_SOURCE_ADMINISTRATION is supported + :param health_is_called: boolean that specifies if detailed errors in the file will be printed to stdout + :return: + """ + from generic import init_yaml + + # first we check if the file was modified. Otherwise, the health check is skipped for performance reasons + if _is_file_modified(filename) or health_is_called: + + _yaml = init_yaml() + with open(filename, 'r') as yaml_file: + yaml_content = _yaml.load(yaml_file) + + if file_type == FILE_TYPE_DATA_SOURCE_ADMINISTRATION: + check_health_data_sources(filename, yaml_content, health_is_called) + elif file_type == FILE_TYPE_TECHNIQUE_ADMINISTRATION: + _check_health_techniques(filename, yaml_content, health_is_called) + + elif _get_health_state_cache(filename): + print(HEALTH_ERROR_TXT + filename) diff --git a/interactive_menu.py b/interactive_menu.py index 22fc4dec..5345186d 100644 --- a/interactive_menu.py +++ b/interactive_menu.py @@ -251,6 +251,7 @@ def _menu_data_source(filename_ds): print('6. update the visibility scores within a technique administration YAML file based on changes within any of ' 'the data sources. \nPast visibility scores are preserved in the score_logbook, and manually assigned scores are ' 'not updated without your approval. \nThe updated visibility are based on the number of available data sources.') + print('7. Check the data sources YAML file for errors.') print('9. Back to main menu.') choice = _ask_input() if choice == '1': @@ -287,6 +288,10 @@ def _menu_data_source(filename_ds): print('Updating visibility scores...') update_technique_administration_file(filename_ds, filename_t) _wait() + elif choice == '7': + print('Checking the data source YAML for errors...') + check_yaml_file_health(filename_ds, FILE_TYPE_DATA_SOURCE_ADMINISTRATION, health_is_called=True) + _wait() elif choice == '9': interactive_menu() elif choice == 'q': @@ -318,7 +323,7 @@ def _menu_detection(filename_t): print('Select what you want to do:') print('4. Generate a layer for detection coverage for the ATT&CK Navigator.') print('5. Generate a layer for detection coverage overlaid with visibility for the ATT&CK Navigator.') - print('6. Generate a graph with detection items added through time.') + print('6. Generate a graph with detections added through time.') print('7. Generate an Excel sheet with all administrated techniques.') print('8. Check the technique YAML file for errors.') print('9. Back to main menu.') @@ -360,7 +365,7 @@ def _menu_detection(filename_t): print('Generating Excel file...') export_techniques_list_to_excel(file_tech) _wait() - elif choice == '8`x': + elif choice == '8': print('Checking the technique YAML file for errors...') check_yaml_file_health(filename_t, FILE_TYPE_TECHNIQUE_ADMINISTRATION, health_is_called=True) _wait() @@ -397,7 +402,7 @@ def _menu_visibility(filename_t, filename_ds): print('Select what you want to do:') print('4. Generate a layer for visibility for the ATT&CK Navigator.') print('5. Generate a layer for visibility overlaid with detection coverage for the ATT&CK Navigator.') - print('6. Generate a graph with visibility items added through time.') + print('6. Generate a graph with visibility added through time.') print('7. Generate an Excel sheet with all administrated techniques.') print('8. Check the technique YAML file for errors.') print('9. Back to main menu.') From 163d21488aa6f62fb36646f8775c987b2cfefa56 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Tue, 20 Aug 2019 11:14:31 +0200 Subject: [PATCH 44/49] Added improved error checking on invalid YAML content returned from an EQL query. --- eql_yaml.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/eql_yaml.py b/eql_yaml.py index 0fd56155..5629c06c 100644 --- a/eql_yaml.py +++ b/eql_yaml.py @@ -1,4 +1,5 @@ from generic import * +from health import * import datetime import sys from pprint import pprint @@ -66,6 +67,7 @@ def _techniques_to_events(techniques, obj_type, include_all_score_objs): # loop over all visibility or detection objects for d in tech[obj_type]: + d = set_yaml_dv_comments(d) app_to = d['applicable_to'] g_comment = d['comment'] if obj_type == 'detection': @@ -152,7 +154,7 @@ def _get_technique_from_list(techniques, tech_id): def _events_to_yaml(query_results, obj_type): """ Transform the EQL 'events' back to valid YAML objects - :param query_results: list with EQL 'events + :param query_results: list with EQL 'events' :param obj_type: data_sources, detection or visibility EQL 'events' :return: list containing YAML objects or None when the events could not be turned into a valid YAML object """ @@ -160,7 +162,6 @@ def _events_to_yaml(query_results, obj_type): if obj_type == 'data_sources': try: # Remove the event_type key. We no longer need this. - # todo implement a check to see if the returned data from the EQL query is to the schema of data_sources for r in query_results: del r['event_type'] if r['date_registered'] and isinstance(r['date_registered'], str): @@ -173,6 +174,11 @@ def _events_to_yaml(query_results, obj_type): # when using an EQL query that does not result in a dict having valid YAML 'data_source' objects. return None + if check_health_data_sources(None, {'data_sources': query_results}, health_is_called=False, no_print=True): + print(EQL_INVALID_RESULT_DS) + pprint(query_results) + return None + return query_results elif obj_type in ['visibility', 'detection']: From 5bb9b6329d50af025af395a6f8538db73c9abce5 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Tue, 20 Aug 2019 11:14:51 +0200 Subject: [PATCH 45/49] Changed variable name. --- constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/constants.py b/constants.py index a4c43f81..749d15e9 100644 --- a/constants.py +++ b/constants.py @@ -168,5 +168,5 @@ EQL_INVALID_RESULT_TECH = '[!] Invalid technique administration content. Check your EQL query to return ' # Health text -HEALTH_HAS_ERROR = '[!] The below YAML file contains possible errors. It\'s recommended to check via the ' \ +HEALTH_ERROR_TXT = '[!] The below YAML file contains possible errors. It\'s recommended to check via the ' \ '\'--health\' argument or using the option in the interactive menu: \n - ' From da23777631073389db27b143f53b5d1437f9feff Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Tue, 20 Aug 2019 11:15:26 +0200 Subject: [PATCH 46/49] - Excel columns made wider. - Removed depreciated functionality. --- data_source_mapping.py | 31 +++++++++++++------------------ technique_mapping.py | 29 +++++++++++------------------ 2 files changed, 24 insertions(+), 36 deletions(-) diff --git a/data_source_mapping.py b/data_source_mapping.py index 9416957c..82e65ab1 100644 --- a/data_source_mapping.py +++ b/data_source_mapping.py @@ -2,7 +2,6 @@ import xlsxwriter from copy import deepcopy from generic import * -from pprint import pprint from datetime import datetime # Imports for pandas and plotly are because of performance reasons in the function that uses these libraries. @@ -104,7 +103,8 @@ def export_data_source_list_to_excel(filename): worksheet.set_column(0, 0, 35) worksheet.set_column(1, 2, 15) - worksheet.set_column(3, 4, 35) + worksheet.set_column(3, 3, 35) + worksheet.set_column(4, 4, 50) worksheet.set_column(5, 5, 24) worksheet.set_column(6, 7, 25) worksheet.set_column(8, 10, 15) @@ -182,22 +182,17 @@ def _load_data_sources(file, filter_empty_scores=True): with open(file, 'r') as yaml_file: yaml_content = _yaml.load(yaml_file) - try: - for d in yaml_content['data_sources']: - dq = d['data_quality'] - if not filter_empty_scores: - my_data_sources[d['data_source_name']] = d - elif dq['device_completeness'] > 0 and dq['data_field_completeness'] > 0 and dq['timeliness'] > 0 and dq['consistency'] > 0: - my_data_sources[d['data_source_name']] = d - - name = yaml_content['name'] - platform = yaml_content['platform'] - exceptions = [t['technique_id'] for t in yaml_content['exceptions']] - except KeyError: # todo remove after implemented extra check within the function '_events_to_yaml' - # when using an EQL query that does not result in a dict having 'data_sources' objects. Trow an error. - print(EQL_INVALID_RESULT_DS) - pprint(yaml_content) - return None + for d in yaml_content['data_sources']: + d['comment'] = d.get('comment', '') + dq = d['data_quality'] + if not filter_empty_scores: + my_data_sources[d['data_source_name']] = d + elif dq['device_completeness'] > 0 and dq['data_field_completeness'] > 0 and dq['timeliness'] > 0 and dq['consistency'] > 0: + my_data_sources[d['data_source_name']] = d + + name = yaml_content['name'] + platform = yaml_content['platform'] + exceptions = [t['technique_id'] for t in yaml_content['exceptions']] return my_data_sources, name, platform, exceptions diff --git a/technique_mapping.py b/technique_mapping.py index 5046a4d3..48f63b3f 100644 --- a/technique_mapping.py +++ b/technique_mapping.py @@ -1,7 +1,6 @@ import simplejson from generic import * import xlsxwriter -from pprint import pprint from datetime import datetime # Imports for pandas and plotly are because of performance reasons in the function that uses these libraries. @@ -98,16 +97,11 @@ def _load_data_sources(file): with open(file, 'r') as yaml_file: yaml_content = _yaml.load(yaml_file) - try: - for d in yaml_content['data_sources']: - dq = d['data_quality'] - if dq['device_completeness'] > 0 and dq['data_field_completeness'] > 0 and dq['timeliness'] > 0 and dq['consistency'] > 0: - my_data_sources[d['data_source_name']] = d - except KeyError: # todo remove after implemented extra check within the function '_events_to_yaml' - # when using an EQL query that does not result in a dict having 'data_sources' objects, return None - print(EQL_INVALID_RESULT_DS) - pprint(yaml_content) - return None + for d in yaml_content['data_sources']: + d['comment'] = d.get('comment', '') + dq = d['data_quality'] + if dq['device_completeness'] > 0 and dq['data_field_completeness'] > 0 and dq['timeliness'] > 0 and dq['consistency'] > 0: + my_data_sources[d['data_source_name']] = d return my_data_sources @@ -393,14 +387,13 @@ def export_techniques_list_to_excel(filename): worksheet_detections.write(y, 6, 'Location', format_bold_left) worksheet_detections.write(y, 7, 'Technique comment', format_bold_left) worksheet_detections.write(y, 8, 'Detection comment', format_bold_left) - worksheet_detections.set_column(0, 0, 14) + worksheet_detections.set_column(0, 0, 8) worksheet_detections.set_column(1, 1, 40) - worksheet_detections.set_column(2, 2, 50) + worksheet_detections.set_column(2, 2, 40) worksheet_detections.set_column(3, 3, 18) worksheet_detections.set_column(4, 4, 11) worksheet_detections.set_column(5, 5, 8) - worksheet_detections.set_column(6, 6, 25) - worksheet_detections.set_column(7, 8, 40) + worksheet_detections.set_column(6, 8, 50) y = 4 for technique_id, technique_data in my_techniques.items(): # Add row for every detection that is defined: @@ -436,13 +429,13 @@ def export_techniques_list_to_excel(filename): worksheet_visibility.write(y, 5, 'Score', format_bold_left) worksheet_visibility.write(y, 6, 'Technique comment', format_bold_left) worksheet_visibility.write(y, 7, 'Visibility comment', format_bold_left) - worksheet_visibility.set_column(0, 0, 14) + worksheet_visibility.set_column(0, 0, 8) worksheet_visibility.set_column(1, 1, 40) - worksheet_visibility.set_column(2, 2, 50) + worksheet_visibility.set_column(2, 2, 40) worksheet_visibility.set_column(3, 3, 18) worksheet_visibility.set_column(4, 4, 11) worksheet_visibility.set_column(5, 5, 8) - worksheet_visibility.set_column(6, 7, 40) + worksheet_visibility.set_column(6, 7, 50) y = 4 for technique_id, technique_data in my_techniques.items(): # Add row for every visibility that is defined: From 81a8d18eff47016c03f1c0f5309c111b0d23969e Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Tue, 20 Aug 2019 11:15:54 +0200 Subject: [PATCH 47/49] - Removed depreciated functionality. - Moved health check functions to health.py --- generic.py | 312 ++++++----------------------------------------------- 1 file changed, 34 insertions(+), 278 deletions(-) diff --git a/generic.py b/generic.py index 340571c3..294c5ae2 100644 --- a/generic.py +++ b/generic.py @@ -3,10 +3,10 @@ import pickle from io import StringIO from ruamel.yaml import YAML -from difflib import SequenceMatcher from datetime import datetime as dt from upgrade import upgrade_yaml_file from constants import * +from health import check_yaml_file_health # Due to performance reasons the import of attackcti is within the function that makes use of this library. @@ -628,6 +628,20 @@ def add_entry_to_list_in_dictionary(dictionary, technique_id, key, entry): dictionary[technique_id][key].append(entry) +def set_yaml_dv_comments(yaml_object): + """ + Set all comments for the detection or visibility YAML object when missing + :param yaml_object: detection or visibility object + :return: detection or visibility object for which empty comments are no filled with an empty string + """ + yaml_object['comment'] = yaml_object.get('comment', '') + if 'score_logbook' in yaml_object: + for score_obj in yaml_object['score_logbook']: + score_obj['comment'] = score_obj.get('comment', '') + + return yaml_object + + def load_techniques(file): """ Loads the techniques (including detection and visibility properties). @@ -646,19 +660,25 @@ def load_techniques(file): yaml_content = _yaml.load(yaml_file) for d in yaml_content['techniques']: - # Add detection items: - if isinstance(d['detection'], dict): # There is just one detection entry - add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'detection', d['detection']) - elif isinstance(d['detection'], list): # There are multiple detection entries - for de in d['detection']: - add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'detection', de) - - # Add visibility items - if isinstance(d['visibility'], dict): # There is just one visibility entry - add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'visibility', d['visibility']) - elif isinstance(d['visibility'], list): # There are multiple visibility entries - for de in d['visibility']: - add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'visibility', de) + if 'detection' in d: + # Add detection items: + if isinstance(d['detection'], dict): # There is just one detection entry + d['detection'] = set_yaml_dv_comments(d['detection']) + add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'detection', d['detection']) + elif isinstance(d['detection'], list): # There are multiple detection entries + for de in d['detection']: + de = set_yaml_dv_comments(de) + add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'detection', de) + + if 'visibility' in d: + # Add visibility items + if isinstance(d['visibility'], dict): # There is just one visibility entry + d['visibility'] = set_yaml_dv_comments(d['visibility']) + add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'visibility', d['visibility']) + elif isinstance(d['visibility'], list): # There are multiple visibility entries + for de in d['visibility']: + de = set_yaml_dv_comments(de) + add_entry_to_list_in_dictionary(my_techniques, d['technique_id'], 'visibility', de) name = yaml_content['name'] platform = yaml_content['platform'] @@ -666,270 +686,6 @@ def load_techniques(file): return my_techniques, name, platform -def _print_error_msg(msg, print_error): - if print_error: - print(msg) - return True - - -def _check_health_score_object(yaml_object, object_type, tech_id, health_is_called): - """ - Check the health of a score_logbook inside a visibility or detection YAML object - :param yaml_object: YAML file lines - :param object_type: 'detection' or 'visibility' - :param tech_id: ATT&CK technique ID - :param health_is_called: boolean that specifies if detailed errors in the file will be printed - :return: True if the YAML file is unhealthy, otherwise False - """ - has_error = False - min_score = None - max_score = None - - if object_type == 'detection': - min_score = -1 - max_score = 5 - elif object_type == 'visibility': - min_score = 0 - max_score = 4 - - if not isinstance(yaml_object['score_logbook'], list): - yaml_object['score_logbook'] = [yaml_object['score_logbook']] - - try: - for score_obj in yaml_object['score_logbook']: - for key in ['date', 'score', 'comment']: - if key not in score_obj: - has_error = _print_error_msg('[!] Technique ID: ' + tech_id + ' is MISSING a key-value pair in a ' + object_type + ' score object in the \'score_logbook\': ' + key, health_is_called) - - if score_obj['score'] is None: - has_error = _print_error_msg('[!] Technique ID: ' + tech_id + ' has an EMPTY key-value pair in a ' + object_type + ' score object in the \'score_logbook\': score', health_is_called) - - elif not isinstance(score_obj['score'], int): - has_error = _print_error_msg('[!] Technique ID: ' + tech_id + ' has an INVALID score format in a ' + object_type + ' score object in the \'score_logbook\': score should be an integer', health_is_called) - - if 'auto_generated' in score_obj: - if not isinstance(score_obj['auto_generated'], bool): - has_error = _print_error_msg( - '[!] Technique ID: ' + tech_id + ' has an INVALID auto_generated value in a ' + object_type + ' score object in the \'score_logbook\': auto_generated (if present) should be set to \'true\' or \'false\'', health_is_called) - - if isinstance(score_obj['score'], int): - if score_obj['date'] is None and score_obj['score'] > -1: - has_error = _print_error_msg('[!] Technique ID: ' + tech_id + ' has an EMPTY key-value pair in a ' + object_type + ' score object in the \'score_logbook\': date', health_is_called) - - # noinspection PyChainedComparisons - if not (score_obj['score'] >= min_score and score_obj['score'] <= max_score): - has_error = _print_error_msg( - '[!] Technique ID: ' + tech_id + ' has an INVALID ' + object_type + ' score in a score object in the \'score_logbook\': ' + str(score_obj['score']) + ' (should be between ' + str(min_score) + ' and ' + str(max_score) + ')', health_is_called) - - if score_obj['score'] > min_score: - try: - # noinspection PyStatementEffect - score_obj['date'].year - # noinspection PyStatementEffect - score_obj['date'].month - # noinspection PyStatementEffect - score_obj['date'].day - except AttributeError: - has_error = _print_error_msg('[!] Technique ID: ' + tech_id + ' has an INVALID data format in a ' + object_type + ' score object in the \'score_logbook\': date (should be YYYY-MM-DD without quotes)', health_is_called) - except KeyError: - pass - - return has_error - - -def _check_health_yaml_object(yaml_object, object_type, tech_id, health_is_called): - """ - Check the health of a visibility or detection YAML object - :param yaml_object: YAML file lines - :param object_type: 'detection' or 'visibility' - :param tech_id: ATT&CK technique ID - :param health_is_called: boolean that specifies if detailed errors in the file will be printed - :return: True if the YAML file is unhealthy, otherwise False - """ - has_error = False - - keys = ['applicable_to'] - - if object_type == 'detection': - keys.append('location') - - try: - for key in keys: - if not isinstance(yaml_object[key], list): - has_error = _print_error_msg('[!] Technique ID: ' + tech_id + ' has for the key-value pair \'' + key + '\' in ' + object_type + ' a string value assigned (should be a list)', health_is_called) - else: - try: - if yaml_object[key][0] is None: - has_error = _print_error_msg('[!] Technique ID: ' + tech_id + ' has an EMPTY key-value pair in ' + object_type + ': ' + key, health_is_called) - except TypeError: - has_error = _print_error_msg( - '[!] Technique ID: ' + tech_id + ' has an EMPTY key-value pair in ' + object_type + ': ' + key, health_is_called) - except KeyError: - pass - - return has_error - - -def _update_health_state(current, update): - if current or update: - return True - else: - return update - - -def _is_file_modified(filename): - """ - Check if the provided file was modified since the last check - :param filename: file location - :return: true when modified else false - """ - last_modified_file = 'cache/last-modified_' + os.path.basename(filename).rstrip('.yaml') - - def _update_modified_date(date): - with open(last_modified_file, 'wb') as fd: - pickle.dump(date, fd) - - if not os.path.exists(last_modified_file): - last_modified = os.path.getmtime(filename) - _update_modified_date(last_modified) - - return True - else: - with open(last_modified_file, 'rb') as f: - last_modified_cache = pickle.load(f) - last_modified_current = os.path.getmtime(filename) - - if last_modified_cache != last_modified_current: - _update_modified_date(last_modified_current) - return True - else: - return False - - -def _get_health_state_cache(filename): - """ - Get file health state from disk - :param filename: file location - :return: the cached error state - """ - last_error_file = 'cache/last-error-state_' + os.path.basename(filename).rstrip('.yaml') - - if os.path.exists(last_error_file): - with open(last_error_file, 'rb') as f: - last_error_state_cache = pickle.load(f) - - return last_error_state_cache - - -def _update_health_state_cache(filename, has_error): - """ - Write the file health state to disk if changed - :param filename: file location - """ - last_error_file = 'cache/last-error-state_' + os.path.basename(filename).rstrip('.yaml') - - def _update(error): - with open(last_error_file, 'wb') as fd: - pickle.dump(error, fd) - - if not os.path.exists(last_error_file): - _update(has_error) - else: - error_state_cache = _get_health_state_cache(filename) - if error_state_cache != has_error: - _update(has_error) - - -def check_yaml_file_health(filename, file_type, health_is_called): - """ - Check on error in the provided YAML file. - :param filename: YAML file location - :param file_type: currently only 'FILE_TYPE_TECHNIQUE_ADMINISTRATION' is being supported - :param health_is_called: boolean that specifies if detailed errors in the file will be printed - :return: - """ - # first we check if the file was modified. Otherwise, the health check is skipped for performance reasons - if _is_file_modified(filename) or health_is_called: - has_error = False - if file_type == FILE_TYPE_TECHNIQUE_ADMINISTRATION: - # check for duplicate tech IDs - _yaml = init_yaml() - with open(filename, 'r') as yaml_file: - yaml_content = _yaml.load(yaml_file) - - tech_ids = list(map(lambda x: x['technique_id'], yaml_content['techniques'])) - tech_dup = [] - for tech in tech_ids: - if tech not in tech_dup: - tech_dup.append(tech) - else: - has_error = _print_error_msg('[!] Duplicate technique ID: ' + tech, health_is_called) - - # check if the technique has a valid format - if not REGEX_YAML_TECHNIQUE_ID_FORMAT.match(tech): - has_error = _print_error_msg('[!] Invalid technique ID: ' + tech, health_is_called) - - # checks on: - # - empty key-value pairs: 'applicable_to', 'comment', 'location', 'score_logbook' , 'date', 'score' - # - invalid date format for: 'date' - # - detection or visibility score out-of-range - # - missing key-value pairs: 'applicable_to', 'comment', 'location', 'score_logbook', 'date', 'score' - # - check on 'applicable_to' values which are very similar - - all_applicable_to = set() - techniques = load_techniques(filename) - for tech, v in techniques[0].items(): - for key in ['detection', 'visibility']: - if key not in v: - has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is MISSING ' + key, health_is_called) - elif 'applicable_to' in v: - # create at set containing all values for 'applicable_to' - all_applicable_to.update([a for v in v[key] for a in v['applicable_to']]) - - for detection in v['detection']: - for key in ['applicable_to', 'location', 'comment', 'score_logbook']: - if key not in detection: - has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is MISSING a key-value pair in detection: ' + key, health_is_called) - - health = _check_health_yaml_object(detection, 'detection', tech, health_is_called) - has_error = _update_health_state(has_error, health) - health = _check_health_score_object(detection, 'detection', tech, health_is_called) - has_error = _update_health_state(has_error, health) - - for visibility in v['visibility']: - for key in ['applicable_to', 'comment', 'score_logbook']: - if key not in visibility: - has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is MISSING a key-value pair in visibility: ' + key, health_is_called) - - health = _check_health_yaml_object(visibility, 'visibility', tech, health_is_called) - has_error = _update_health_state(has_error, health) - health = _check_health_score_object(visibility, 'visibility', tech, health_is_called) - has_error = _update_health_state(has_error, health) - - # get values within the key-value pair 'applicable_to' which are a very close match - similar = set() - for i1 in all_applicable_to: - for i2 in all_applicable_to: - match_value = SequenceMatcher(None, i1, i2).ratio() - if match_value > 0.8 and match_value != 1: - similar.add(i1) - similar.add(i2) - - if len(similar) > 0: - has_error = _print_error_msg('[!] There are values in the key-value pair \'applicable_to\' which are very similar. Correct where necessary:', health_is_called) - for s in similar: - _print_error_msg(' - ' + s, health_is_called) - - if has_error and not health_is_called: - print(HEALTH_HAS_ERROR + filename) - elif has_error and health_is_called: - print(' - ' + filename) - - _update_health_state_cache(filename, has_error) - elif _get_health_state_cache(filename): - print(HEALTH_HAS_ERROR + filename) - - def _check_file_type(filename, file_type=None): """ Check if the provided YAML file has the key 'file_type' and possible if that key matches a specific value. From 6efd04cb3b62707d667feb0992af89a9e10b7585 Mon Sep 17 00:00:00 2001 From: Ruben Bouman Date: Wed, 21 Aug 2019 10:53:04 +0200 Subject: [PATCH 48/49] Don't overwrite output files if they already exist, but append a number to the filename as suffix. --- group_mapping.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/group_mapping.py b/group_mapping.py index 2b7c389a..799cb534 100644 --- a/group_mapping.py +++ b/group_mapping.py @@ -554,13 +554,10 @@ def generate_group_heat_map(groups, overlay, overlay_type, stage, platform, soft json_string = simplejson.dumps(layer).replace('}, ', '},\n') if stage == 'pre-attack': - filename = "output/" + stage + '_' + '_'.join(groups_list) + filename = '_'.join(groups_list) elif overlay: - filename = "output/" + stage + '_' + platform.lower() + '_' + '_'.join(groups_list) + '-overlay_' + '_'.join(overlay_list) + filename = platform.lower() + '_' + '_'.join(groups_list) + '-overlay_' + '_'.join(overlay_list) else: - filename = "output/" + stage + '_' + platform.lower() + '_' + '_'.join(groups_list) - filename = filename[:255] + '.json' - with open(filename, 'w') as f: # write layer file to disk - f.write(json_string) - print('Written layer: ' + filename) + filename = platform.lower() + '_' + '_'.join(groups_list) + write_file(stage, filename[:255], json_string) From 24c19fcca6adef186e68d203742fb1f872e70209 Mon Sep 17 00:00:00 2001 From: Ruben Bouman Date: Wed, 21 Aug 2019 15:12:12 +0200 Subject: [PATCH 49/49] small changes to interactive menu: makes choices like booleans must faster --- interactive_menu.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/interactive_menu.py b/interactive_menu.py index 5345186d..82e18ef1 100644 --- a/interactive_menu.py +++ b/interactive_menu.py @@ -210,19 +210,19 @@ def _menu_statistics(): print('9. Back to main menu.') choice = _ask_input() if choice == '1': - print('Specify the matrix (enterprise or mobile):') - m = _ask_input().lower() - default_matrix = 'enterprise' if m == 'enterprise' else 'mobile' + default_matrix = 'mobile' if default_matrix == 'enterprise' else 'enterprise' elif choice == '2': get_statistics_data_sources() + _wait() elif choice == '3': get_statistics_mitigations(default_matrix) + _wait() elif choice == '9': interactive_menu() + _wait() elif choice == 'q': quit() - _wait() _menu_statistics() @@ -335,8 +335,7 @@ def _menu_detection(filename_t): print('Specify the EQL query for visibility objects:') eql_query_visibility = _ask_input().lower() elif choice == '3': - print('Specify True or False for to include all scores or not:') - eql_all_scores = True if _ask_input().lower() == 'true' else False + eql_all_scores = not eql_all_scores elif choice in ['4', '5', '6', '7']: file_tech = filename_t @@ -414,8 +413,7 @@ def _menu_visibility(filename_t, filename_ds): print('Specify the EQL query for detection objects:') eql_query_detection = _ask_input().lower() elif choice == '3': - print('Specify True or False for to include all scores or not:') - eql_all_scores = True if _ask_input().lower() == 'true' else False + eql_all_scores = not eql_all_scores elif choice in ['4', '5', '6', '7']: file_tech = filename_t @@ -470,7 +468,6 @@ def _menu_groups(): print('5. Overlay: ') print(' - %s: %s' % ('File' if os.path.exists(groups_overlay) else 'Groups', groups_overlay)) print(' - Type: %s' % overlay_type) - print(' - Type: %s' % overlay_type) print('6. EQL search: ') eql_d_str = '' if not eql_query_detection else eql_query_detection eql_v_str = '' if not eql_query_visibility else eql_query_visibility @@ -483,8 +480,7 @@ def _menu_groups(): print('9. Back to main menu.') choice = _ask_input() if choice == '1': - print('Specify True or False for software group:') - software_group = True if _ask_input().lower() == 'true' else False + software_group = not software_group elif choice == '2': print('Specify platform (all, Linux, macOS, Windows):') p = _ask_input().lower() @@ -534,8 +530,7 @@ def _menu_groups(): print('Specify the EQL query for visibility objects:') eql_query_visibility = _ask_input().lower() elif choice == '3': - print('Specify True or False for to include all scores or not:') - eql_all_scores = True if _ask_input().lower() == 'true' else False + eql_all_scores = not eql_all_scores elif choice == '7': if not generate_group_heat_map(groups, groups_overlay, overlay_type, default_stage, default_platform,