From 386e7f1eceef502151b0363a95ec3f3266f464d2 Mon Sep 17 00:00:00 2001 From: Mostafa Abdo Date: Tue, 5 Sep 2023 10:45:53 -0700 Subject: [PATCH 1/5] refactor code --- src/navv/bll.py | 21 --------------------- src/navv/commands.py | 8 ++++---- src/navv/zeek.py | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 25 deletions(-) diff --git a/src/navv/bll.py b/src/navv/bll.py index 7245ce8..981d4e0 100644 --- a/src/navv/bll.py +++ b/src/navv/bll.py @@ -2,7 +2,6 @@ import os import pandas as pd -from navv.zeek import perform_zeekcut from navv.utilities import get_mac_vendor, timeit from navv.validators import is_ipv4_address, is_ipv6_address @@ -10,26 +9,6 @@ MAC_VENDORS_JSON_FILE = os.path.abspath(__file__ + "/../" + "data/mac-vendors.json") -def get_zeek_data(zeek_logs): - """Return a list of Zeek conn.log data.""" - return ( - perform_zeekcut( - fields=[ - "id.orig_h", - "id.resp_h", - "id.resp_p", - "proto", - "conn_state", - "orig_l2_addr", - "resp_l2_addr", - ], - log_file=os.path.join(zeek_logs, "conn.log"), - ) - .decode("utf-8") - .split("\n")[:-1] - ) - - def get_zeek_df(zeek_data: list, dns_data: dict): """Return a pandas dataframe of the conn.log data with its dns data.""" zeek_data = [row.split("\t") for row in zeek_data] diff --git a/src/navv/commands.py b/src/navv/commands.py index 754d5f8..b5ccd25 100644 --- a/src/navv/commands.py +++ b/src/navv/commands.py @@ -5,7 +5,7 @@ # Third-Party Libraries import click -from navv.bll import get_inventory_report_df, get_zeek_data, get_zeek_df +from navv.bll import get_inventory_report_df, get_zeek_df # cisagov Libraries from navv.gui import app @@ -24,7 +24,7 @@ write_stats_sheet, write_unknown_internals_sheet, ) -from navv.zeek import get_dns_data, run_zeek, perform_zeekcut +from navv.zeek import get_conn_data, get_dns_data, run_zeek, perform_zeekcut from navv.utilities import pushd @@ -69,8 +69,8 @@ def generate(customer_name, output_dir, pcap, zeek_logs): else: timer_data["run_zeek"] = "NOT RAN" - # Get zeek data - zeek_data = get_zeek_data(zeek_logs) + # Get zeek conn.log data + zeek_data = get_conn_data(zeek_logs) # Get dns data for resolution json_path = os.path.join(output_dir, f"{customer_name}_dns_data.json") diff --git a/src/navv/zeek.py b/src/navv/zeek.py index 877dc55..0b794f8 100644 --- a/src/navv/zeek.py +++ b/src/navv/zeek.py @@ -16,6 +16,27 @@ def run_zeek(pcap_path, zeek_logs_path, **kwargs): error_msg(e) +@timeit +def get_conn_data(zeek_logs): + """Return a list of Zeek conn.log data.""" + return ( + perform_zeekcut( + fields=[ + "id.orig_h", + "id.resp_h", + "id.resp_p", + "proto", + "conn_state", + "orig_l2_addr", + "resp_l2_addr", + ], + log_file=os.path.join(zeek_logs, "conn.log"), + ) + .decode("utf-8") + .split("\n")[:-1] + ) + + @timeit def get_dns_data(customer_name, output_dir, zeek_logs): """Get DNS data from zeek logs or from a json file if it exists""" @@ -31,6 +52,21 @@ def get_dns_data(customer_name, output_dir, zeek_logs): return trim_dns_data(dns_data) +@timeit +def get_snmp_data(customer_name, output_dir, zeek_logs): + """Get SNMP data from zeek logs or from a json file if it exists""" + json_path = os.path.join(output_dir, f"{customer_name}_dns_data.json") + if os.path.exists(json_path): + with open(json_path, "rb") as json_file: + return json.load(json_file) + + dns_data = perform_zeekcut( + fields=["query", "answers", "qtype", "rcode_name"], + log_file=os.path.join(zeek_logs, "dns.log"), + ) + return trim_dns_data(dns_data) + + def perform_zeekcut(fields, log_file): """Perform the call to zeek-cut with the identified fields on the specified log file""" try: From 674c07e93af5fa5eb133d364afff956c2d258fbe Mon Sep 17 00:00:00 2001 From: Mostafa Abdo Date: Tue, 5 Sep 2023 11:55:30 -0700 Subject: [PATCH 2/5] add snmp dataframe and sheet --- src/navv/bll.py | 17 +++++++++++++ src/navv/commands.py | 23 ++++++++++++------ src/navv/spreadsheet_tools.py | 35 ++++++++++++++++++++++++++- src/navv/zeek.py | 45 +++++++++++++++++++---------------- 4 files changed, 92 insertions(+), 28 deletions(-) diff --git a/src/navv/bll.py b/src/navv/bll.py index 981d4e0..df0a5cc 100644 --- a/src/navv/bll.py +++ b/src/navv/bll.py @@ -142,3 +142,20 @@ def get_inventory_report_df(zeek_df: pd.DataFrame): ) return grouped_df + + +@timeit +def get_snmp_df(zeek_data: list): + """Return a pandas dataframe of the snmp.log data.""" + zeek_data = [row.split("\t") for row in zeek_data] + return pd.DataFrame( + zeek_data, + columns=[ + "src_ip", + "src_port", + "dst_ip", + "dst_port", + "version", + "community", + ], + ) diff --git a/src/navv/commands.py b/src/navv/commands.py index b5ccd25..5589ba4 100644 --- a/src/navv/commands.py +++ b/src/navv/commands.py @@ -5,7 +5,7 @@ # Third-Party Libraries import click -from navv.bll import get_inventory_report_df, get_zeek_df +from navv.bll import get_inventory_report_df, get_snmp_df, get_zeek_df # cisagov Libraries from navv.gui import app @@ -21,10 +21,17 @@ write_conn_states_sheet, write_externals_sheet, write_inventory_report_sheet, + write_snmp_sheet, write_stats_sheet, write_unknown_internals_sheet, ) -from navv.zeek import get_conn_data, get_dns_data, run_zeek, perform_zeekcut +from navv.zeek import ( + get_conn_data, + get_dns_data, + get_snmp_data, + run_zeek, + perform_zeekcut, +) from navv.utilities import pushd @@ -69,17 +76,17 @@ def generate(customer_name, output_dir, pcap, zeek_logs): else: timer_data["run_zeek"] = "NOT RAN" - # Get zeek conn.log data + # Get zeek data from conn.log, dns.log and snmp.log zeek_data = get_conn_data(zeek_logs) + snmp_data = get_snmp_data(zeek_logs) + dns_filtered = get_dns_data(customer_name, output_dir, zeek_logs) # Get dns data for resolution json_path = os.path.join(output_dir, f"{customer_name}_dns_data.json") - # Get dns data from zeek logs - dns_filtered = get_dns_data(customer_name, output_dir, zeek_logs) - - # Get zeek dataframe + # Get zeek dataframes zeek_df = get_zeek_df(zeek_data, dns_filtered) + snmp_df = get_snmp_df(snmp_data) # Get inventory report dataframe inventory_df = get_inventory_report_df(zeek_df) @@ -109,6 +116,8 @@ def generate(customer_name, output_dir, pcap, zeek_logs): write_unknown_internals_sheet(unk_int_IPs, wb) + write_snmp_sheet(snmp_df, wb) + auto_adjust_width(wb["Analysis"]) times = ( diff --git a/src/navv/spreadsheet_tools.py b/src/navv/spreadsheet_tools.py index 5914441..5e3555f 100644 --- a/src/navv/spreadsheet_tools.py +++ b/src/navv/spreadsheet_tools.py @@ -400,7 +400,40 @@ def write_inventory_report_sheet(inventory_df, wb): for cell in ir_sheet[f"{index}:{index}"]: cell.fill = openpyxl.styles.PatternFill("solid", fgColor="AAAAAA") auto_adjust_width(ir_sheet, 40) - # ir_sheet.column_dimensions.width = 39 * 1.2 + + +def write_snmp_sheet(snmp_df, wb): + """Write SNMP log data to excel sheet.""" + sheet = make_sheet(wb, "SNMP", idx=4) + sheet.append( + ["Src IPv4", "Src Port", "Dest IPv4", "Dest Port", "Version", "Community"] + ) + + for index, row in enumerate(snmp_df.to_dict(orient="records"), start=2): + # Source IPv4 column + sheet[f"A{index}"].value = row["src_ip"] + + # Source Port column + sheet[f"B{index}"].value = row["src_port"] + + # Destination IPv4 column + sheet[f"C{index}"].value = row["dst_ip"] + + # Destination Port column + sheet[f"D{index}"].value = row["dst_port"] + + # Version column + sheet[f"E{index}"].value = row["version"] + + # Community column + sheet[f"F{index}"].value = row["community"] + + # Add styling to every other row + if index % 2 == 0: + for cell in sheet[f"{index}:{index}"]: + cell.fill = openpyxl.styles.PatternFill("solid", fgColor="AAAAAA") + + auto_adjust_width(sheet, 40) def write_externals_sheet(IPs, wb): diff --git a/src/navv/zeek.py b/src/navv/zeek.py index 0b794f8..de62745 100644 --- a/src/navv/zeek.py +++ b/src/navv/zeek.py @@ -6,16 +6,6 @@ from navv.utilities import pushd, timeit, trim_dns_data -@timeit -def run_zeek(pcap_path, zeek_logs_path, **kwargs): - with pushd(zeek_logs_path): - # can we add Site::local_nets to the zeek call here? - try: - check_call(["zeek", "-C", "-r", pcap_path, "local.zeek"]) - except Exception as e: - error_msg(e) - - @timeit def get_conn_data(zeek_logs): """Return a list of Zeek conn.log data.""" @@ -53,18 +43,23 @@ def get_dns_data(customer_name, output_dir, zeek_logs): @timeit -def get_snmp_data(customer_name, output_dir, zeek_logs): +def get_snmp_data(zeek_logs): """Get SNMP data from zeek logs or from a json file if it exists""" - json_path = os.path.join(output_dir, f"{customer_name}_dns_data.json") - if os.path.exists(json_path): - with open(json_path, "rb") as json_file: - return json.load(json_file) - - dns_data = perform_zeekcut( - fields=["query", "answers", "qtype", "rcode_name"], - log_file=os.path.join(zeek_logs, "dns.log"), + return ( + perform_zeekcut( + fields=[ + "id.orig_h", + "id.orig_p", + "id.resp_h", + "id.resp_p", + "version", + "community", + ], + log_file=os.path.join(zeek_logs, "snmp.log"), + ) + .decode("utf-8") + .split("\n")[:-1] ) - return trim_dns_data(dns_data) def perform_zeekcut(fields, log_file): @@ -78,3 +73,13 @@ def perform_zeekcut(fields, log_file): except OSError as e: # probably "file does not exist" return b"" + + +@timeit +def run_zeek(pcap_path, zeek_logs_path, **kwargs): + with pushd(zeek_logs_path): + # can we add Site::local_nets to the zeek call here? + try: + check_call(["zeek", "-C", "-r", pcap_path, "local.zeek"]) + except Exception as e: + error_msg(e) From 08cb49b23a75fcaf1266e256df6dd71808c67da7 Mon Sep 17 00:00:00 2001 From: Mostafa Abdo Date: Wed, 6 Sep 2023 08:52:37 -0700 Subject: [PATCH 3/5] Bump version from 3.0.5 to 3.1.0 --- src/navv/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/navv/_version.py b/src/navv/_version.py index 3bc269f..df68702 100644 --- a/src/navv/_version.py +++ b/src/navv/_version.py @@ -1,2 +1,2 @@ """This file defines the version of this module.""" -__version__ = "3.0.5" +__version__ = "3.1.0" From 794815ed297e0acafde469970181ada3dbf5f110 Mon Sep 17 00:00:00 2001 From: Mostafa Abdo Date: Wed, 6 Sep 2023 09:07:42 -0700 Subject: [PATCH 4/5] update makefile --- Makefile | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index a394393..8ef7084 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ -.PHONY: run +.PHONY: generate launch + include .env export @@ -10,17 +11,15 @@ help: install-develop: pip install -e . -# target: install - Install application +# target: install - Install production application install: pip install navv -# target: generate - Generate analysis from pcap +# target: generate - Generate analysis excel sheet +# Optionally set PCAP_PATH to a pcap file +# example: make generate PCAP_PATH=test-path/to/file.pcap generate: - navv generate -o analysis -p test-data/test_data.pcap -z test-data/logs test-customer - -# target: load-metadata - Load metadata -load-metadata: - navv generate -o analysis -z test-data/logs test-customer + navv generate -o analysis -z test-data/logs test-customer $(if $(PCAP_PATH), -p $(PCAP_PATH)) # target: launch - Launch GUI application launch: From 6ee4e4b44ea4a213f0472bd93c150093288350a4 Mon Sep 17 00:00:00 2001 From: Mostafa Abdo Date: Wed, 6 Sep 2023 09:08:18 -0700 Subject: [PATCH 5/5] update make generate comment --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8ef7084..c387b76 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ install: pip install navv # target: generate - Generate analysis excel sheet -# Optionally set PCAP_PATH to a pcap file +# optionally set PCAP_PATH to a relative pcap file path # example: make generate PCAP_PATH=test-path/to/file.pcap generate: navv generate -o analysis -z test-data/logs test-customer $(if $(PCAP_PATH), -p $(PCAP_PATH))