diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a2e7712 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.DS_Store +*.log +.state/ diff --git a/Dovehawk/LICENSE b/Dovehawk/LICENSE new file mode 100644 index 0000000..45f22ff --- /dev/null +++ b/Dovehawk/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright 2018 Cancyber Inc., Michael Kortekaas @mrkortek, Tyler McLellan @tylabs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Dovehawk/README.md b/Dovehawk/README.md new file mode 100644 index 0000000..d05440d --- /dev/null +++ b/Dovehawk/README.md @@ -0,0 +1,94 @@ +# Dovehawk Bro Module + +Threat Hunting with Bro and MISP + + +This modules uses the the built-in Bro Intelligence Framework to load and monitor signatures from MISP automatically. Indicators are downloaded from MISP every 6 hours and hits, called sightings are reported back to MISP immediately. The module also includes a customized version of Jan Grashoefer's expiration code to remove indicators after 7 hours after they are deleted from MISP. + + +Indicators are downloaded automatically every 6 hours. Indicators should expire after 7 hours if removed from MISP. + + +Indicators are downloaded and read into memory. Content signatures in signatures.sig which is not yet automatically downloaded. MISP does not yet support bro content signatures, this module will be updated for downloading those when available. + + +## Official Source + +https://dovehawk.io/ (coming soon) + +https://github.com/tylabs/dovehawk/ + + +## Requirements + +Bro IDS: tested with version version 2.5.4. + +Curl: command line tool for accessing web content, tested with curl 7.54.0. + + +## Quick Start + +Rename misp_config.bro.default to misp_config.bro. Edit misp_config.bro and add your MISP API key and URLs for the Bro Export and Sightings. + + + +## Related Projects + +http://www.misp-project.org/ MISP + +https://www.bro.org Bro IDS + + +## Monitoring and context + +The bro module outputs hits to the console, logs to file, and could send metadata to another web hook. + + +## Usage + +If running bro directly, reference the Dovehawk folder: + +sudo bro -i en1 [FULL PATH]/Dovehawk + +If running using the broctl interface, edit the local.bro configuration file in /usr/local/bro/share/bro/site and, at the bottom, add the line: + +@load [FULL PATH]/Dovehawk + +then run the broctl deploy sequence to have the scripts installed. + + +## BRO Tips + +When running locally (ie running Bro on the same system you are generating traffic from), you may need to use the -C option to ignore checksum validation. + + +## Optional Disable local logging + +Add "Log::default_writer=Log::WRITER_NONE" to the command. + +bro -i en0 Dovehawk Log::default_writer=Log::WRITER_NONE + + +## Maintenance + +For long term monitoring, if not disabling logs as above, use broctl to launch, rotate logs, and restart after crashes. + + + +# Special Thanks + +CanCyber.org for their support in releasing a generic MISP version of their Bro Module as open source. + +Developers: Michael Kortekaas (original module), Tyler McLellan @tylabs (MISP combined import and sightings) + + +# License + +Copyright 2018 Cancyber Inc., Michael Kortekaas @mrkortek, Tyler McLellan @tylabs + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/Dovehawk/__load__.bro b/Dovehawk/__load__.bro new file mode 100644 index 0000000..06b19d1 --- /dev/null +++ b/Dovehawk/__load__.bro @@ -0,0 +1,2 @@ +@load ./dovehawk.bro + diff --git a/Dovehawk/dovehawk.bro b/Dovehawk/dovehawk.bro new file mode 100644 index 0000000..1207aa2 --- /dev/null +++ b/Dovehawk/dovehawk.bro @@ -0,0 +1,370 @@ +# Dovehawk Bro Module V 1.00.000 2018 08 23 + +module Dovehawk; + +@load ./misp_config.bro +@load ./dovehawk_expire.bro + + +@load frameworks/intel/seen +@load frameworks/intel/do_notice + +redef Intel::item_expiration = 7hr; + +export { + global DH_VERSION = "1.00.000"; + + global dh_meta : Intel::MetaData = [ + $source = "MISP", + $do_notice = T, + $dh_expire = 7hr, + $desc = "", + $url = "" + ]; + + global signature_refresh_period = 6hr &redef; + global load_signatures: function(); +} + + + +# Modelled on the original function from ActiveHTTP but they stripped out the newlines and joined +# everything together. Need to keep the original string vector to process individual lines. +# Original Source: https://github.com/bro/bro/blob/master/scripts/base/utils/active-http.bro + +function request2curl(r: ActiveHTTP::Request, bodyfile: string, headersfile: string): string +{ + local cmd = fmt("curl --header \"Authorization: %s\" -s -g -o \"%s\" -D \"%s\" -X \"%s\"", + str_shell_escape(Dovehawk::APIKEY), + str_shell_escape(bodyfile), + str_shell_escape(headersfile), + str_shell_escape(r$method)); + + + cmd = fmt("%s -m %.0f", cmd, r$max_time); + + if ( r?$client_data ) + cmd = fmt("%s -d @-", cmd); + + if ( r?$addl_curl_args ) + cmd = fmt("%s %s", cmd, r$addl_curl_args); + + cmd = fmt("%s \"%s\"", cmd, str_shell_escape(r$url)); + # Make sure file will exist even if curl did not write one. + cmd = fmt("%s && touch %s", cmd, str_shell_escape(bodyfile)); + + return cmd; +} + +function strings_request(req: ActiveHTTP::Request): string_vec +{ + local tmpfile = "/tmp/bro-activehttp-" + unique_id(""); + local bodyfile = fmt("%s_body", tmpfile); + local headersfile = fmt("%s_headers", tmpfile); + + local cmd = request2curl(req, bodyfile, headersfile); + local stdin_data = req?$client_data ? req$client_data : ""; + + return when ( local result = Exec::run([$cmd=cmd, $stdin=stdin_data, $read_files=set(bodyfile, headersfile)]) ) + { + # If there is no response line then nothing else will work either. + if ( ! (result?$files && headersfile in result$files) ) + { + print "download error 1"; + Reporter::error(fmt("There was a failure when requesting \"%s\" with ActiveHTTP.", req$url)); + return vector(); # Empty string vector indicates failure + } + + return result$files[bodyfile]; + } +} + + +# INDICATOR DOWNLOAD FUNCTIONS - Note that +# line counts for signature downloads includes possible comments - output stored in stdout.log to +# be used for verification checking if necessary. + +# Special option to load all the hash strings combined as a single file +function load_all_misp() { + local request: ActiveHTTP::Request = [ + $url = all_export_url + ]; + + print "Downloading Indicators..."; + + when ( local lines = strings_request(request) ) { + if (|lines| > 0 ) { + print "Processing Indicators..."; + print fmt("Number of Indicators %d", |lines|); + + local domaincnt = 0; + local ipcnt = 0; + local subnetcnt = 0; + local urlcnt = 0; + local softwarecnt = 0; + local emailcnt = 0; + local usercnt = 0; + local hashcnt = 0; + local filenamecnt = 0; + + + for (line in lines) { + local sig = strip(lines[line]); + + #local parts: string_array; + local parts = split_string(sig, /\t/); + + if (|parts| > 3) { + local item : Intel::Item = [ + $indicator = parts[0], + $indicator_type = Intel::DOMAIN, + $meta = dh_meta + ]; + + item$meta$desc = parts[4]; + item$meta$url = parts[5]; + + + #For debugging print the items + ##print fmt(" %s=%s", parts[1], parts[0]); + + + + + if (parts[1] == "Intel::ADDR") { + item$indicator_type = Intel::ADDR; + ipcnt += 1; + } else if (parts[1] == "Intel::SUBNET") { + item$indicator_type = Intel::SUBNET; + subnetcnt += 1; + } else if (parts[1] == "Intel::URL") { + item$indicator_type = Intel::URL; + urlcnt += 1; + } else if (parts[1] == "Intel::SOFTWARE") { + item$indicator_type = Intel::SOFTWARE; + softwarecnt += 1; + } else if (parts[1] == "Intel::EMAIL") { + item$indicator_type = Intel::EMAIL; + emailcnt += 1; + } else if (parts[1] == "Intel::USER_NAME") { + item$indicator_type = Intel::USER_NAME; + usercnt += 1; + } else if (parts[1] == "Intel::CERT_HASH") { + item$indicator_type = Intel::FILE_HASH; #cert hash isn't really implemented + hashcnt += 1; + } else if (parts[1] == "Intel::PUBKEY_HASH") { + item$indicator_type = Intel::FILE_HASH; + hashcnt += 1; + } else if (parts[1] == "Intel::FILE_HASH") { + item$indicator_type = Intel::FILE_HASH; + hashcnt += 1; + } else if (parts[1] == "Intel::FILE_NAME") { + item$indicator_type = Intel::FILE_NAME; + filenamecnt += 1; + } else + domaincnt += 1; + + Intel::insert(item); + } + } + + print " Intel Indicator Counts:"; + print fmt(" Intel::DOMAIN: %d", domaincnt); + print fmt(" Intel::IP: %d", ipcnt); + print fmt(" Intel::SOFTWARE: %d", softwarecnt); + print fmt(" Intel::EMAIL: %d", emailcnt); + print fmt(" Intel::USER_NAME: %d", usercnt); + print fmt(" Intel::FILE_HASH: %d", hashcnt); + print fmt(" Intel::FILE_NAME: %d",filenamecnt); + print "Finished Processing Indicators"; + + + } else { + print "indicator download error"; + } + } +} + + + + +function register_hit(hitvalue: string, desc: string) { + local url_string = Dovehawk::sightings_url; + local post_data = fmt("{\"value\": \"%s\"}", hitvalue); + + local request: ActiveHTTP::Request = [ + $url=url_string, + $method="POST", + $client_data=post_data, + $addl_curl_args = fmt("--header \"Authorization: %s\" --header \"Content-Type: application/json\" --header \"Accept: application/json\"", str_shell_escape(Dovehawk::APIKEY)) + ]; + + when ( local resp = ActiveHTTP::request(request) ) { + + if (resp$code == 200) { + print fmt(" Sighting Result ===> %s", resp$body); + } else { + print fmt(" Sighting FAILED ===> %s", resp); + } + } + +} + +# Need to check info variables for null since they are all optional +# Use a | separator since this is not as likely to be part of the data +# Hit data should be kept below 1000 bytes or it will be rejected + +hook Notice::policy(n: Notice::Info) &priority=3 { + if (n$note == Intel::Notice) { + # Each notice contains an email section with the meta source included + # Check on this value to ensure only hits are registered + local isDT = F; + for (ebs in n$email_body_sections) { + if (strstr(n$email_body_sections[ebs],"MISP") > 1) { + isDT = T; + break; + } + } + + if (! isDT) { + return; + } + + local hit = "BRO"; + if (n?$uid) { + hit += fmt("|uid:%s",n$uid); + } + if (n?$ts) { + hit += fmt("|ts:%f",n$ts); + } + + if (n?$id) { + hit += fmt("|orig_h:%s|orig_p:%s|resp_h:%s|resp_p:%s",n$id$orig_h,n$id$orig_p,n$id$resp_h,n$id$resp_p); + } else { + if (n?$src) { + hit += fmt("|src:%s",n$src); + } + if (n?$dst) { + hit += fmt("|dst:%s",n$dst); + } + if (n?$p) { + hit += fmt("|p:%s",n$p); + } + } + + if (n?$fuid) { + hit += fmt("|fuid:%s",n$fuid); + } + if (n?$msg) { + hit += "|msg:" + n$msg; + } + register_hit(n$sub, hit); + print "Intel Signature Hit ===> " + n$sub; + print " Metadata ===> " + hit; + + } +} + + + +event do_reload_signatures() { + if (bro_is_terminating()) { + print "Bro Terminating - Cancelling Scheduled Signature Downloads"; + } else { + load_signatures(); + + schedule signature_refresh_period { do_reload_signatures() }; + } +} + + +function load_signatures() { + print fmt("Downloading Signatures %s", strftime("%Y/%m/%d %H:%M:%S", network_time())); + + print fmt("Source Directory: %s", @DIR); + + # Need to force update this each time to ensure it's not a static constant or zero + dh_meta$dh_last_update = network_time(); + + # Load all contains all MISP bro output combined + load_all_misp(); + + #placeholder for downloading/reingesting full content signatures when misp gains support for them + @load-sigs ./signatures.sig + + + # Force output into stdout.log when using broctl + flush_all(); +} + + +event signature_match(state: signature_state, msg: string, data: string) +{ + + local sig_id = state$sig_id; + + # Ensure this is a MISP signature + if (strstr(msg,"MISP:") == 0) { + return; + } + + local src_addr: addr; + local src_port: port; + local dst_addr: addr; + local dst_port: port; + + if ( state$is_orig ) + { + src_addr = state$conn$id$orig_h; + src_port = state$conn$id$orig_p; + dst_addr = state$conn$id$resp_h; + dst_port = state$conn$id$resp_p; + } + else + { + src_addr = state$conn$id$resp_h; + src_port = state$conn$id$resp_p; + dst_addr = state$conn$id$orig_h; + dst_port = state$conn$id$orig_p; + } + + local hit = "BRO"; + if (state$conn?$uid) { + hit += fmt("|uid:%s",state$conn$uid); + } + if (state$conn?$http && state$conn$http?$ts) { + hit += fmt("|ts:%f",state$conn$http$ts); + } + + hit += fmt("|orig_h:%s|orig_p:%s|resp_h:%s|resp_p:%s",src_addr,src_port,dst_addr,dst_port); + + hit += "|sigid:" + sig_id + "|msg:" + msg; + + # This should always be true but check just in case + if (|hit| < 800) { + # Trim the matched data down to fit the sql hit structure limit + if ( (|data| + |hit|) > 900 ) + data = fmt("%s...", sub_bytes(data, 0, 900-|hit|)); + + hit += "|data:" + data; + } + + register_hit(state$sig_id,hit); + + print "Content Signature Hit ===> " + state$sig_id; +} + + +event bro_init() +{ + schedule signature_refresh_period { do_reload_signatures() }; +} + + +event file_new(f: fa_file) +{ + Files::add_analyzer(f, Files::ANALYZER_MD5); + Files::add_analyzer(f, Files::ANALYZER_SHA1); + Files::add_analyzer(f, Files::ANALYZER_SHA256); +} + diff --git a/Dovehawk/dovehawk_expire.bro b/Dovehawk/dovehawk_expire.bro new file mode 100644 index 0000000..af67236 --- /dev/null +++ b/Dovehawk/dovehawk_expire.bro @@ -0,0 +1,70 @@ +##! This script adds per item expiration for MISP intel items. This +# version does not reset the base time when hits are detected. It is based +# on the intel-extensions package that is Copyright (c) 2016 by Jan Grashoefer + +@load base/frameworks/intel + +module Intel; + +export { + ## Default expiration interval for single intelligence items + ## A negative value disables expiration for these items. + const dh_per_item_expiration = -1min &redef; + + redef record MetaData += { + dh_expire: interval &default = dh_per_item_expiration; + + ## Internal value tracks time of item creation, last update. + dh_last_update: time &default = network_time(); + }; +} + +hook extend_match(info: Info, s: Seen, items: set[Item]) +{ + local matches = |items|; + for ( item in items ) + { + local meta = item$meta; + + if (meta$source != "MISP") { + next; + } + + if ( meta$dh_expire > 0 sec && meta$dh_last_update + meta$dh_expire < network_time() ) + { + # Item already expired + --matches; + print fmt("Removing Expired Intel Item: %s",item$indicator); + remove(item, F); + } + } + + if ( matches < 1 ) { + break; + } +} + + +hook Intel::item_expired(indicator: string, indicator_type: Type, metas: set[MetaData]) +{ + for ( meta in metas ) + { + if (meta$source != "MISP") { + next; + } + + # Check for expired items + if ( meta$dh_expire > 0 sec && meta$dh_last_update + meta$dh_expire < network_time() ) + { + # Recreate the item from the indicator and meta + local item: Intel::Item = [ + $indicator = indicator, + $indicator_type = indicator_type, + $meta = meta + ]; + + print fmt("Removing Expired Intel Item: %s",indicator); + remove(item, F); + } + } +} diff --git a/Dovehawk/misp_config.bro.default b/Dovehawk/misp_config.bro.default new file mode 100644 index 0000000..d23b43d --- /dev/null +++ b/Dovehawk/misp_config.bro.default @@ -0,0 +1,7 @@ +module Dovehawk; + +export { +global APIKEY = "===your misp key Event Actions->Automation==="; +global all_export_url = "https://yourmispsite.com/attributes/bro/download/all"; +global sightings_url = "https://yourmispsite.com/sightings/add/"; +} diff --git a/Dovehawk/signatures.sig b/Dovehawk/signatures.sig new file mode 100644 index 0000000..f794ed5 --- /dev/null +++ b/Dovehawk/signatures.sig @@ -0,0 +1,14 @@ +# Dovehawk Content Signatures - Sig events should have "MISP:" prefix + +signature eicar_test_content { + ip-proto == tcp + payload /.*X5O\!P%@AP\[4\\PZX54\(P\^\)7CC\)7\}\$EICAR\-STANDARD\-ANTIVIRUS\-TEST\-FILE\!\$H\+H\*/ + event "MISP: eicar test file in TCP" +} + + +signature cancyber_test_content { + ip-proto == tcp + payload /.*991CANCYBER_TEST_BAD_SIGNATURE991/ + event "MISP: test content in TCP" +} diff --git a/LICENSE b/LICENSE index 4797ac8..45f22ff 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2018 tylabs +Copyright 2018 Cancyber Inc., Michael Kortekaas @mrkortek, Tyler McLellan @tylabs Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 935a0a6..d05440d 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,94 @@ -# dovehawk -Dovehawk is a Bro module that automatically imports MISP indicators and reports Sightings +# Dovehawk Bro Module + +Threat Hunting with Bro and MISP + + +This modules uses the the built-in Bro Intelligence Framework to load and monitor signatures from MISP automatically. Indicators are downloaded from MISP every 6 hours and hits, called sightings are reported back to MISP immediately. The module also includes a customized version of Jan Grashoefer's expiration code to remove indicators after 7 hours after they are deleted from MISP. + + +Indicators are downloaded automatically every 6 hours. Indicators should expire after 7 hours if removed from MISP. + + +Indicators are downloaded and read into memory. Content signatures in signatures.sig which is not yet automatically downloaded. MISP does not yet support bro content signatures, this module will be updated for downloading those when available. + + +## Official Source + +https://dovehawk.io/ (coming soon) + +https://github.com/tylabs/dovehawk/ + + +## Requirements + +Bro IDS: tested with version version 2.5.4. + +Curl: command line tool for accessing web content, tested with curl 7.54.0. + + +## Quick Start + +Rename misp_config.bro.default to misp_config.bro. Edit misp_config.bro and add your MISP API key and URLs for the Bro Export and Sightings. + + + +## Related Projects + +http://www.misp-project.org/ MISP + +https://www.bro.org Bro IDS + + +## Monitoring and context + +The bro module outputs hits to the console, logs to file, and could send metadata to another web hook. + + +## Usage + +If running bro directly, reference the Dovehawk folder: + +sudo bro -i en1 [FULL PATH]/Dovehawk + +If running using the broctl interface, edit the local.bro configuration file in /usr/local/bro/share/bro/site and, at the bottom, add the line: + +@load [FULL PATH]/Dovehawk + +then run the broctl deploy sequence to have the scripts installed. + + +## BRO Tips + +When running locally (ie running Bro on the same system you are generating traffic from), you may need to use the -C option to ignore checksum validation. + + +## Optional Disable local logging + +Add "Log::default_writer=Log::WRITER_NONE" to the command. + +bro -i en0 Dovehawk Log::default_writer=Log::WRITER_NONE + + +## Maintenance + +For long term monitoring, if not disabling logs as above, use broctl to launch, rotate logs, and restart after crashes. + + + +# Special Thanks + +CanCyber.org for their support in releasing a generic MISP version of their Bro Module as open source. + +Developers: Michael Kortekaas (original module), Tyler McLellan @tylabs (MISP combined import and sightings) + + +# License + +Copyright 2018 Cancyber Inc., Michael Kortekaas @mrkortek, Tyler McLellan @tylabs + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +