Add IPv6 support. Add parsing of VLAN and QinQ tagged ethernet packets

CHANGELOG.md

@@ -40,4 +40,6 @@
 
 **Version 0.2.20** - Fixing bug output of large communities when using the LineBased formatter
 
-**Version 0.2.20** - Add-Path capability added (RFC7911)
+**Version 0.2.20** - Add-Path capability added (RFC7911)
+
+**Version 0.3.1** - Add basic IPv6 support. Unpacking and filtering of VLAN and QinQ tagged ethernetframes is now possible.

README.md

@@ -72,7 +72,9 @@ Therefore we implemented the following feature. A user is now able to toggle a F
 If the NLRI field contains two 0-Bytes (translated to two 0.0.0.0/0 prefixes which should not occur at all) the programm assumes that the first 4 bytes are a Path Identifier and treates this field as an Add-Path message.  
 
 ## Limitations
-Currently, the parser doesn't perform a reassembly on fragmented TCP packets. This may leads into parsing errors and application warnings when you are trying to parse large BGP packets with several messages.
+Currently, the parser doesn't perform a reassembly on fragmented TCP packets. This may leads into parsing errors and application warnings when you are trying to parse large BGP packets with several messages. 
+
+In general, IPv6 BGP messages are understood, even filtering for sender/receiver addresses is possible (except for shorthand ipv6 notation i.e. 1337::42 has to be extended to 1337:0:0:0:0:0:0:42). Filtering REACH_NLRI and UNREACH_NLRI is not implemented. 
 
 Currently, we are looking into some problems with running pbgpp with Python 2.7 and streaming the output to Kafka. However, Python 3.x works just fine.
 

pbgpp/Application/CLI.py

@@ -56,7 +56,8 @@ def main():
     group_4.add_argument("--filter-message-size", help="only print messages with given message size in bytes (e.g., 128)", nargs="+", action="append", dest="filter_message_size")
     group_4.add_argument("--filter-message-type", help="only print messages with given BGP message type (KEEPALIVE, NOTIFICATION, OPEN, ROUTE-REFRESH, UPDATE, WITHDRAWAL)", nargs="+", action="append", dest="filter_message_type")
     group_4.add_argument("--filter-message-subtype", help="only print UPDATE messages with given message sub type (WITHDRAWAL, ANNOUNCE, BOTH, NONE)", nargs="+", action="append", dest="filter_message_subtype")
-    group_4.add_argument("--filter-nlri", help="only print messages containing the given nlri prefix (e.g., '80.81.82.0/24'", nargs="+", action="append", dest="filter_nlri")
+    group_4.add_argument("--filter-pathid", help="only print messages with the given path identifier (e.g., '7')", nargs="+", action="append", dest="filter_pathid")
+    group_4.add_argument("--filter-nlri", help="only print messages containing the given nlri prefix (e.g., '80.81.82.0/24')", nargs="+", action="append", dest="filter_nlri")
     group_4.add_argument("--filter-withdrawn", help="only print messages containing the given withdrawn routes (e.g., '80.81.82.0/24'", nargs="+", action="append", dest="filter_withdrawn")
     group_4.add_argument("--filter-next-hop", help="only print messages containing the given next hop (e.g., '80.81.82.83')", nargs="+", action="append", dest="filter_next_hop")
     group_4.add_argument("--filter-as", help="only print messages containing the given ASN in path AS_PATH attribute (e.g., '12345')", nargs="+", action="append", dest="filter_asn")
@@ -65,6 +66,8 @@ def main():
     group_4.add_argument("--filter-community-value", help="only print messages containing the given community value (e.g., '12345')", nargs="+", action="append", dest="filter_community_value")
     group_4.add_argument("--filter-source-ip", help="only print messages containing the given source IP address (e.g., '80.81.82.83')", nargs="+", action="append", dest="filter_source_ip")
     group_4.add_argument("--filter-source-mac", help="only print messages containing the given source MAC address (e.g., 'aabbccddeeff')", nargs="+", action="append", dest="filter_source_mac")
+    group_4.add_argument("--filter-customer-vlan", help="only print messages containing the given (customer) VLAN ID (e.g., '42')", nargs="+", action="append", dest="filter_customer_vlan")
+    group_4.add_argument("--filter-service-vlan", help="only print messages containing the given service VLAN ID (e.g., '1337')", nargs="+", action="append", dest="filter_service_vlan")
     group_4.add_argument("--filter-destination-ip", help="only print messages containing the given destination IP address (e.g., '80.81.82.83')", nargs="+", action="append", dest="filter_destination_ip")
     group_4.add_argument("--filter-destination-mac", help="only print messages containing the given destination MAC address (e.g., 'aabbccddeeff')", nargs="+", action="append", dest="filter_destination_mac")
     group_4.add_argument("--filter-large-community", help="only print messages containing one or more matching large communities (e.g., '11:22:33', '11:*:*', '*:22:33')", nargs="+", action="append", dest="filter_large_community")

pbgpp/Application/Handler.py

@@ -46,7 +46,10 @@
 from pbgpp.Output.Filters.MessageTypeFilter import MessageTypeFilter
 from pbgpp.Output.Filters.NLRIFilter import NLRIFilter
 from pbgpp.Output.Filters.NextHopFilter import NextHopFilter
+from pbgpp.Output.Filters.PathIdentifierFilter import PathIdentifierFilter
 from pbgpp.Output.Filters.TimestampFilter import TimestampFilter
+from pbgpp.Output.Filters.VLANCustomerFilter import VLANCustomerFilter
+from pbgpp.Output.Filters.VLANServiceFilter import VLANServiceFilter
 from pbgpp.Output.Filters.WithdrawnFilter import WithdrawnFilter
 from pbgpp.Output.Formatters.HumanReadable import HumanReadableFormatter
 from pbgpp.Output.Formatters.JSON import JSONFormatter
@@ -163,6 +166,12 @@ def __parse_filters(self):
             self.filters.append(MessageSubTypeFilter(filters))
             logger.debug("Added " + str(len(filters)) + " filter(s) of MessageSubTypeFilter")
 
+        if self.args.filter_pathid:
+            values = self.args.filter_pathid
+            filters = list(chain(*values))
+            self.filters.append(PathIdentifierFilter(filters))
+            logger.debug("Added " + str(len(filters)) + " filter(s) of PathIdentfierFilter")
+
         if self.args.filter_nlri:
             values = self.args.filter_nlri
             filters = list(chain(*values))
@@ -244,9 +253,21 @@ def __parse_filters(self):
         if self.args.filter_destination_mac:
             values = self.args.filter_destination_mac
             filters = list(chain(*values))
-            self.prefilters.append(MACDestinationFilter(MACSourceFilter.clear_input(filters)))
+            self.prefilters.append(MACDestinationFilter(MACDestinationFilter.clear_input(filters)))
             logger.debug("Added " + str(len(filters)) + " pre-filter(s) of MACDestinationFilter")
 
+        if self.args.filter_customer_vlan:
+            values = self.args.filter_customer_vlan
+            filters = list(chain(*values))
+            self.prefilters.append(VLANCustomerFilter(filters))
+            logger.debug("Added " + str(len(filters)) + " pre-filter(s) of VLANCustomerFilter")
+
+        if self.args.filter_service_vlan:
+            values = self.args.filter_service_vlan
+            filters = list(chain(*values))
+            self.prefilters.append(VLANServiceFilter(filters))
+            logger.debug("Added " + str(len(filters)) + " pre-filter(s) of VLANServiceFilter")
+
         if self.args.filter_timestamp:
             values = self.args.filter_timestamp
             filters = list(chain(*values))
@@ -318,13 +339,13 @@ def __packet_handler(self, header, payload):
         eth = PCAPEthernet(payload)
 
         # Check for raw ethernet packet
-        if not eth.get_type() == PCAPEthernet.ETH_TYPE_IPV4:
+        if not eth.get_type() == PCAPEthernet.ETH_TYPE_IPV4 and not eth.get_type() == PCAPEthernet.ETH_TYPE_IPV6 and not eth_type == PCAPEthernet.ETH_TYPE_VLAN and not eth_type == PCAPEthernet.ETH_TYPE_QINQ:
 
             # Check for SLL-packet
             eth = PCAPCookedCapture(payload)
 
-            if not eth.get_type() == PCAPCookedCapture.ETH_TYPE_IPV4:
-                logger.debug("Discarding PCAP packet " + str(self.__packet_counter) + " due to non-IPv4 ethernet type.")
+            if not eth.get_type() == PCAPCookedCapture.ETH_TYPE_IPV4 and not eth.get_type() == PCAPCookedCapture.ETH_TYPE_IPV6:
+                logger.debug("Discarding PCAP packet " + str(self.__packet_counter) + " due to non-IPv4 or non-IPv6 ethernet type.")
                 return False
 
         ip = PCAPIP(eth.get_eth_payload())
@@ -335,7 +356,7 @@ def __packet_handler(self, header, payload):
 
         tcp = PCAPTCP(ip.get_ip_payload())
 
-        pcap_information = PCAPInformation(header.getts(), eth.mac, ip.addresses, tcp.ports)
+        pcap_information = PCAPInformation(header.getts(), eth.get_mac(), ip.get_addresses(), tcp.get_ports())
 
         for filter in self.prefilters:
             if not filter.apply(pcap_information):

pbgpp/BGP/Statics.py

@@ -443,3 +443,7 @@ class BGPStatics:
 
     # ** BGPUpdate Path Attribute EXTENDED_COMMUNITIES trans. Generic Experimental Part 3 **
     EXT_COMMUNITY_T_GENERIC_EXPERIMENTAL_PART3_FLOW_SPEC_AS_4BYTE_FORMAT = 8
+
+    # IPV4 and IPV6 needs to be differentiated sometimes:
+    IP4_CODE = 4
+    IP6_CODE = 6 

pbgpp/BGP/Update/Message.py

@@ -64,33 +64,15 @@ def __parse(self):
 
                 # Loop through withdrawals
                 while continue_loop:
-                    # AddPath assumption? look for description in the method for NLRI parsing
-                    if self.flags["addpath"].get_value() == 0: # No AddPath messages
-                        pass
+                    # AddPath assumption? 
+                    current_byte_position = self.__addpath_routine(current_byte_position)
 
-                    else:
-                        pathId_length_bytes = self.payload[current_byte_position:current_byte_position + 4]
-                        pathId = struct.unpack("!I", pathId_length_bytes)[0]
-
-                        if  self.flags["addpath"].get_value() == 1: # Only AddPath
-                            self.add_path = True
-                            self.path_id = pathId
-                            current_byte_position += 4
-
-                        else:                           # Try to find out (using metric)
-                            if pathId < 65536:
-                                self.add_path = True
-                                self.path_id = pathId
-                                current_byte_position += 4
-                            #else: drop the Path Id, its likely that this is not an AddPath msg
-
                     # First of all we need to parse the length of the withdrawn prefix. Depending on the prefix length
                     # we can determine the length following prefix itself
                     prefix_length_bytes = self.payload[current_byte_position:current_byte_position + 1]
                     prefix_length = struct.unpack("!B", prefix_length_bytes)[0]
                     current_byte_position += 1
 
-
                     if prefix_length == 0:
                         prefix_bytes = prefix_length_bytes
                     elif 0 < prefix_length <= 8:
@@ -179,34 +161,7 @@ def __parse(self):
                 current_byte_position = self.path_attributes_length + 4 + self.withdrawn_routes_length
 
                 while continue_loop:
-                    """
-                    The Following is a Fix for missing Add_Path feature.
-                    Due to the lack of a definition for this case, we need depend on the users decision.
-                    See RFC 7911 Chapter 6 p.5 (22.07.2020).
-
-                    In most cases, the pathId is lower than 2**16. Also it is uncommon,
-                    that one BGP UPDATE message contains the 0.0.0.0/0 prefix 2 times.
-                    This leads to the following metric if the user sets the add_path_flag to 2.
-                    """
-                    # AddPath assumption?
-                    if self.flags["addpath"].get_value() == 0: # No AddPath messages
-                        pass
-
-                    else:
-                        pathId_length_bytes = self.payload[current_byte_position:current_byte_position + 4]
-                        pathId = struct.unpack("!I", pathId_length_bytes)[0]
-
-                        if  self.flags["addpath"].get_value() == 1: # Only AddPath
-                            self.add_path = True
-                            self.path_id = pathId
-                            current_byte_position += 4
-
-                        else:                           # Try to find out (using metric)
-                            if pathId < 65536:
-                                self.add_path = True
-                                self.path_id = pathId
-                                current_byte_position += 4
-                            #else: drop the Path Id, its likely that this is not an AddPath msg
+                    current_byte_position = self.__addpath_routine(current_byte_position)
 
                     # First of all we have to check the prefix length as byte-length of the following
                     # prefix depends on its prefix length (This is a 1-byte-field)
@@ -259,3 +214,34 @@ def __parse(self):
             self.error = True
 
         self.error = False
+
+    def __addpath_routine(self, current_byte_position):
+        """
+        The Following is a Fix for missing Add_Path feature.
+        Due to the lack of a definition for this case, we need depend on the users decision.
+        See RFC 7911 Chapter 6 p.5 (22.07.2020).
+
+        In most cases, the pathId is lower than 2**16. Also it is uncommon,
+        that one BGP UPDATE message contains the 0.0.0.0/0 prefix 2 times.
+        This leads to the following metric if the user sets the add_path_flag to 2.
+        """
+        # AddPath assumption?
+        if self.flags["addpath"].get_value() == 0: # No AddPath messages
+            pass
+
+        else:
+            pathId_length_bytes = self.payload[current_byte_position:current_byte_position + 4]
+            pathId = struct.unpack("!I", pathId_length_bytes)[0]
+
+            if  self.flags["addpath"].get_value() == 1: # Only AddPath
+                self.add_path = True
+                self.path_id = pathId
+                current_byte_position += 4
+
+            else:                           # Try to find out (using metric)
+                if pathId < 65536:
+                    self.add_path = True
+                    self.path_id = pathId
+                    current_byte_position += 4
+                #else: drop the Path Id, its likely that this is not an AddPath msg
+        return current_byte_position

pbgpp/BGP/Update/PathAttributes/MPNextHop.py

@@ -0,0 +1,60 @@
+#
+# This file is part of PCAP BGP Parser (pbgpp)
+#
+# Copyright 2016-2020 DE-CIX Management GmbH
+# Author: Christopher Moeller <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from pbgpp.BGP.Statics import BGPStatics
+from pbgpp.BGP.Update.Route import BGPRoute
+
+import struct
+import socket
+
+class MPNextHop:
+    "This class is an extension of the MP_REACH field"
+    def __init__(self, payload, proto):
+        self.payload = payload
+        self.proto = proto
+
+        self.next_hop = None #string representation of address
+
+        self.__parse()
+
+    def __parse(self):
+        try:
+            self.parsed = True
+            self.error = False
+
+            if self.proto == socket.AF_INET:
+                fields = struct.unpack("!4B", self.payload)
+                self.next_hop = str(fields[0]) + "." + str(fields[1]) + "." + str(fields[2]) + "." + str(fields[3])
+
+            else:
+                fields = struct.unpack("!8H", self.payload)
+                next_hop = ""
+                for i in fields:
+                    next_hop += str( hex(i)[2:] ) + ":" 
+                self.next_hop = next_hop[:-1]
+
+        except Exception as e:
+            self.error = True
+
+    def __str__(self):
+        if self.parsed and not self.error:
+            return self.next_hop
+        else:
+            return None
+