From 70800b902decf52f32b1fa448ba42797414a3181 Mon Sep 17 00:00:00 2001 From: Milan Lenco Date: Wed, 10 Jul 2024 18:11:29 +0200 Subject: [PATCH] Add Eden-SDN example for Local NI with multiple ports This example demonstrates Local network instance with multiple ports and multipath static IP routes. EVE API changes were introduced and described here: https://github.com/lf-edge/eve-api/pull/53 Signed-off-by: Milan Lenco --- sdn/examples/ni-with-multiple-ports/README.md | 189 +++++++++++ .../DevicePortConfig/override.json | 13 + .../config-overrides/GlobalConfig/global.json | 9 + .../ni-with-multiple-ports/device-config.json | 306 ++++++++++++++++++ .../ni-with-multiple-ports/network-model.json | 165 ++++++++++ sdn/vm/cmd/sdnagent/config.go | 19 ++ sdn/vm/cmd/sdnagent/ipam.go | 2 + sdn/vm/scripts/get-eve-ip.sh | 2 +- 8 files changed, 704 insertions(+), 1 deletion(-) create mode 100644 sdn/examples/ni-with-multiple-ports/README.md create mode 100644 sdn/examples/ni-with-multiple-ports/config-overrides/DevicePortConfig/override.json create mode 100644 sdn/examples/ni-with-multiple-ports/config-overrides/GlobalConfig/global.json create mode 100644 sdn/examples/ni-with-multiple-ports/device-config.json create mode 100644 sdn/examples/ni-with-multiple-ports/network-model.json diff --git a/sdn/examples/ni-with-multiple-ports/README.md b/sdn/examples/ni-with-multiple-ports/README.md new file mode 100644 index 000000000..bb6a21a64 --- /dev/null +++ b/sdn/examples/ni-with-multiple-ports/README.md @@ -0,0 +1,189 @@ +# SDN Example with Network Instance using multiple ports + +When application is connected to multiple IP networks over multiple network ports, +IP routing must be utilized to properly route network flows so that they reach their +intended destination. + +EVE offers two options for the application traffic routing: + +1. Create a separate network instance for every port needed by app and use DHCP-based propagation + of IP routes into applications. This option is shown in the [app-routing example](../app-routing) + It is more difficult to configure but gives the application full control over the IP routing +2. Create a single local network instance with all those ports assigned and use the IP routing + capabilities of the network instance. From the application perspective this is much simpler + because it will be connected to only a single NI and thus have only one interface. On the app + side a single default route for the single network interface is all that is needed. + The complexity of routing it taken care of by EVE. This option is shown in this example. + +Local Network Instance with multiple ports will have link-local and connected routes +from all the ports present in its routing table. Additionally, user may configure static +IP routes which will be added into the routing table. A static route may reference +a particular gateway IP as the next hop, or a logical label of a port to use as the output +device, or use a shared label to match a subset of NI ports if there are multiple +possible paths to reach the routed destination network. + +For every multi-path route with shared port label, EVE will perform periodic probing +of all matched network ports to determine the connectivity status and select the best +port to use for the route. Note that every multi-path route will have at most one output +port selected and configured at any time - load-balancing is currently not supported. + +The probing method can be customized by the user as part of the route configuration. +If enabled, EVE will check the reachability of the port's next hop (the gateway IP) +every 15 seconds using ICMP ping. The upside of using this probe is fairly quick fail-over +when the currently used port for a given multi-path route looses connectivity. +The downside is that it may generate quite a lot of traffic over time. User may limit +the use of this probe to only ports with low cost or disable this probe altogether. +Additionally, every 2.5 minutes, EVE will run user-defined probe if configured. +This can be either an ICMP ping towards a given IP or hostname, or a TCP handshake against +the given IP/hostname and port. + +A connectivity probe must consecutively success/fail few times in a row to determine +the connectivity as being up/down. EVE will then consider changing the currently used port +for a given route. The requirement for multiple consecutive test passes/failures prevents +from port flapping, i.e. re-routing too often. + +Additionally to connectivity status, there are some other metrics that can affect the port +selection decision. For example, user may enable lower-cost-preference for a given multi-path +route. In that case, with multiple connected ports, EVE will select the lowest-cost port. +Similarly, route that uses multiple wwan ports, can be configured to give preferential +selection to cellular modem with better network signal strength. + +## Example scenario + +In this example, we connect application into a single network instance with 4 ports +attached (using shared label `all`). +Ports `eth0` and `eth2` get their IP configs from DHCP, while `eth1` and `eth3` are assigned +IPs statically. `eth0` and `eth3` have Internet access and are used for management. +Additionally, `eth0`, `eth2` and `eth3` have access to an HTTP server deployed inside +the Eden SDN VM. Shared port label `httpserver` is created to mark these ports and use +them for a multi-path route with the subnet of the HTTP server as destination (`10.88.88.0/24`). +This route is configured to use a TCP handshake with the HTTP server to probe connectivity +of those ports and select the lowest-cost port with a working connectivity. + +Moreover, a default multi-path route is created with a shared label `internet` assigned +to `eth0` and `eth3`. This route is configured to only perform ICMP ping of the gateway +to determine port connectivity status. + +Lastly, port forwarding is configured to enable accessing the app from outside. +However, this is limited to `eth0`, `eth1` and `eth2` using a shared label `portfwd` +(purely for example purposes). + +Here is a diagram depicting the network topology: + +```text + +-------------------+ + ---------->| eth0 (mgmt) |--------------- + | | (DHCP, portfwd) | | + | +-------------------+ | + | | ++-----+ +----------+ +-------------------+ | +| app |-->| Local NI |-->| eth1 (app-shared) | | ++-----+ +----------+ | (static, portfwd) | | + | +-------------------+ | + | | + | +-------------------+ +--------+ +------------+ + ---------->| eth2 (app-shared) |----------| router |--| httpserver | + | | (DHCP, portfwd) | +--------+ +------------+ + | +-------------------+ | + | | + | +-------------------+ | + ---------->| eth3 (mgmt) |---------------- + | (static IP conf) | + +-------------------+ +``` + +Run the example with: + +```shell +make clean && make build-tests +./eden config add default +./eden config set default --key sdn.disable --value false +./eden setup --eve-bootstrap-file $(pwd)/sdn/examples/ni-with-multiple-ports/device-config.json +./eden start --sdn-network-model $(pwd)/sdn/examples/ni-with-multiple-ports/network-model.json +./eden eve onboard +./eden controller edge-node set-config --file $(pwd)/sdn/examples/ni-with-multiple-ports/device-config.json +``` + +Once deployed, check the routing table of the network instance: + +```shell +./eden eve ssh +eve$ ip route show table 801 +default via 172.22.10.1 dev eth0 proto static +unreachable default proto static metric 4294967295 +10.40.40.0/24 dev eth3 proto static scope link src 10.40.40.30 metric 1009 +10.50.0.0/24 dev bn1 proto static scope link src 10.50.0.1 +10.88.88.0/24 via 172.22.10.1 dev eth0 proto static +172.22.10.0/24 dev eth0 proto static scope link src 172.22.10.13 metric 1011 +172.28.20.0/24 dev eth1 proto static scope link src 172.28.20.10 metric 1008 +192.168.30.0/24 dev eth2 proto static scope link src 192.168.30.20 metric 1010 +``` + +Notice that `eth0` is selected for both the default route and the HTTP server route. + +Login to the application and try to access something in the Internet (e.g. 8.8.8.8) +and the HTTP server: + +```shell +eve$ eve attach-app-console 3599588a-17d3-4d02-aae1-bcefe3706cfd.1.1/cons +app$ ping -c 3 8.8.8.8 +PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data. +64 bytes from 8.8.8.8: icmp_seq=1 ttl=252 time=17.6 ms +64 bytes from 8.8.8.8: icmp_seq=2 ttl=252 time=19.2 ms +64 bytes from 8.8.8.8: icmp_seq=3 ttl=252 time=18.2 ms + +--- 8.8.8.8 ping statistics --- +3 packets transmitted, 3 received, 0% packet loss, time 2004ms +rtt min/avg/max/mdev = 17.563/18.318/19.150/0.650 ms + +app$ curl httpserver0.sdn/helloworld +Hello world from HTTP server +``` + +Check the external IP addresses that application will use (depending on the destination): + +```shell +app$ curl http://169.254.169.254/eve/v1/network.json 2>/dev/null | jq +{ + "app-instance-uuid": "3599588a-17d3-4d02-aae1-bcefe3706cfd", + "caller-ip": "10.50.0.2:48588", + "device-name": "6845629e-500d-4b00-be66-801e75ba65b5", + "device-uuid": "6845629e-500d-4b00-be66-801e75ba65b5", + "enterprise-id": "", + "enterprise-name": "", + "external-ipv4": "172.22.10.13,172.28.20.10,192.168.30.20,10.40.40.30", + "hostname": "3599588a-17d3-4d02-aae1-bcefe3706cfd", + "project-name": "", + "project-uuid": "00000000-0000-0000-0000-000000000000" +} +app$ curl http://169.254.169.254/eve/v1/external_ipv4 2>/dev/null +172.22.10.13 +172.28.20.10 +192.168.30.20 +10.40.40.30 +``` + +Next, simulate `eth0` losing the connectivity by changing the network model: + +```shell +./eden sdn net-model get > net-model +jq '(.ports[] | select(.logicalLabel == "eveport0").adminUP) = false' net-model > net-model-eth0-down +./eden sdn net-model apply net-model-eth0-down +``` + +Eventually, default route is re-routed to use `eth3` while the HTTP server route +will use `eth2` (has lower cost than `eth3`): + +```shell +# ssh over eth0 is not going to work, use console access: +./eden eve console +eve$ ip route show table 801 +default via 10.40.40.1 dev eth3 proto static +unreachable default proto static metric 4294967295 +10.40.40.0/24 dev eth3 proto static scope link src 10.40.40.30 metric 1009 +10.50.0.0/24 dev bn1 proto static scope link src 10.50.0.1 +10.88.88.0/24 via 192.168.30.1 dev eth2 proto static +172.22.10.0/24 dev eth0 proto static scope link src 172.22.10.13 metric 1011 +172.28.20.0/24 dev eth1 proto static scope link src 172.28.20.10 metric 1008 +192.168.30.0/24 dev eth2 proto static scope link src 192.168.30.20 metric 1010 +``` diff --git a/sdn/examples/ni-with-multiple-ports/config-overrides/DevicePortConfig/override.json b/sdn/examples/ni-with-multiple-ports/config-overrides/DevicePortConfig/override.json new file mode 100644 index 000000000..2319c6dcd --- /dev/null +++ b/sdn/examples/ni-with-multiple-ports/config-overrides/DevicePortConfig/override.json @@ -0,0 +1,13 @@ +{ + "TimePriority": "2000-01-01T00:00:00Z", + "Version": 1, + "Ports": [ + { + "IfName": "eth0", + "IsMgmt": true, + "Free": true, + "Dhcp": 4, + "Type": 4 + } + ] +} diff --git a/sdn/examples/ni-with-multiple-ports/config-overrides/GlobalConfig/global.json b/sdn/examples/ni-with-multiple-ports/config-overrides/GlobalConfig/global.json new file mode 100644 index 000000000..60b746d60 --- /dev/null +++ b/sdn/examples/ni-with-multiple-ports/config-overrides/GlobalConfig/global.json @@ -0,0 +1,9 @@ +{ + "GlobalSettings": { + "network.fallback.any.eth": { + "Key": "network.fallback.any.eth", + "ItemType": 4, + "TriStateValue": 1 + } + } +} \ No newline at end of file diff --git a/sdn/examples/ni-with-multiple-ports/device-config.json b/sdn/examples/ni-with-multiple-ports/device-config.json new file mode 100644 index 000000000..f42493f51 --- /dev/null +++ b/sdn/examples/ni-with-multiple-ports/device-config.json @@ -0,0 +1,306 @@ +{ + "deviceIoList": [ + { + "ptype": 1, + "phylabel": "eth0", + "phyaddrs": { + "Ifname": "eth0" + }, + "logicallabel": "eth0", + "assigngrp": "eth0", + "usage": 1 + }, + { + "ptype": 1, + "phylabel": "eth1", + "phyaddrs": { + "Ifname": "eth1" + }, + "logicallabel": "eth1", + "assigngrp": "eth1", + "usage": 2 + }, + { + "ptype": 1, + "phylabel": "eth2", + "phyaddrs": { + "Ifname": "eth2" + }, + "logicallabel": "eth2", + "assigngrp": "eth2", + "usage": 2 + }, + { + "ptype": 1, + "phylabel": "eth3", + "phyaddrs": { + "Ifname": "eth3" + }, + "logicallabel": "eth3", + "assigngrp": "eth3", + "usage": 1 + } + ], + "networks": [ + { + "id": "20537543-5dce-4b55-b3d1-733fb416986d", + "type": 4, + "ip": { + "dhcp": 4 + } + }, + { + "id": "6605d17b-3273-4108-8e6e-4965441ebe01", + "type": 4, + "ip": { + "dhcp": 1, + "subnet": "172.28.20.0/24", + "gateway": "172.28.20.1", + "dns": ["10.16.16.25"], + "ntp": "114.30.89.30" + } + }, + { + "id": "baeae1d0-504a-4d1e-ae83-6215e87d41d0", + "type": 4, + "ip": { + "dhcp": 4 + } + }, + { + "id": "d799d9c0-6afc-4d39-b78a-5ddf345f0c8e", + "type": 4, + "ip": { + "dhcp": 1, + "subnet": "10.40.40.0/24", + "gateway": "10.40.40.1", + "dns": ["10.17.17.25"], + "ntp": "114.30.89.30" + } + } + ], + "systemAdapterList": [ + { + "name": "eth0", + "sharedLabels": ["internet", "httpserver", "portfwd"], + "uplink": true, + "networkUUID": "20537543-5dce-4b55-b3d1-733fb416986d" + }, + { + "name": "eth1", + "sharedLabels": ["portfwd"], + "uplink": false, + "networkUUID": "6605d17b-3273-4108-8e6e-4965441ebe01", + "addr": "172.28.20.10" + }, + { + "name": "eth2", + "sharedLabels": ["httpserver", "portfwd"], + "uplink": false, + "cost": 3, + "networkUUID": "baeae1d0-504a-4d1e-ae83-6215e87d41d0" + }, + { + "name": "eth3", + "sharedLabels": ["internet", "httpserver"], + "uplink": true, + "cost": 5, + "networkUUID": "d799d9c0-6afc-4d39-b78a-5ddf345f0c8e", + "addr": "10.40.40.30" + } + ], + "networkInstances": [ + { + "uuidandversion": { + "uuid": "9ca83da9-94e8-48b4-9ae8-3f188c5c694a", + "version": "1" + }, + "displayname": "ni0", + "instType": 2, + "activate": true, + "port": { + "type": 1, + "name": "all" + }, + "cfg": {}, + "ipType": 1, + "ip": { + "subnet": "10.50.0.0/24", + "gateway": "10.50.0.1", + "dns": [ + "10.50.0.1" + ], + "ntp": "129.6.15.28", + "dhcpRange": { + "start": "10.50.0.2", + "end": "10.50.0.254" + } + }, + "propagateConnectedRoutes": true, + "staticRoutes": [ + { + "destinationNetwork": "0.0.0.0/0", + "port": "internet", + "probe": { + "enableGwPing": true, + "gwPingMaxCost": 5 + }, + "preferLowerCost": true, + "preferStrongerWwanSignal": true + }, + { + "destinationNetwork": "10.88.88.0/24", + "port": "httpserver", + "probe": { + "enableGwPing": false, + "customProbe": { + "probeMethod": 2, + "probeEndpoint": { + "host": "httpserver0.sdn", + "port": 80 + } + } + }, + "preferLowerCost": true, + "preferStrongerWwanSignal": true + } + ] + } + ], + "apps": [ + { + "uuidandversion": { + "uuid": "3599588a-17d3-4d02-aae1-bcefe3706cfd", + "version": "1" + }, + "displayname": "eclient", + "fixedresources": { + "memory": 512000, + "maxmem": 512000, + "vcpus": 1, + "virtualizationMode": 1 + }, + "drives": [ + { + "image": { + "uuidandversion": { + "uuid": "d65c67dc-f251-4b93-9d53-f1f84a92ea8b", + "version": "1" + }, + "name": "lfedge/eden-eclient:b96434e", + "iformat": 8, + "dsId": "8e6d22d4-4890-4e0d-a574-0d2e757f951b" + } + } + ], + "activate": true, + "interfaces": [ + { + "name": "ni0", + "networkId": "9ca83da9-94e8-48b4-9ae8-3f188c5c694a", + "acls": [ + { + "matches": [ + { + "type": "ip", + "value": "0.0.0.0/0" + } + ], + "id": 1 + }, + { + "matches": [ + { + "type": "protocol", + "value": "tcp" + }, + { + "type": "lport", + "value": "2223" + }, + { + "type": "adapter", + "value": "portfwd" + } + ], + "actions": [ + { + "portmap": true, + "appPort": 22 + } + ], + "id": 2 + } + ] + } + ], + "volumeRefList": [ + { + "uuid": "d507b2a1-2226-47cc-ac88-dee46077dad1", + "mount_dir": "/" + } + ] + } + ], + "volumes": [ + { + "uuid": "d507b2a1-2226-47cc-ac88-dee46077dad1", + "origin": { + "type": 2, + "downloadContentTreeID": "ce88f026-e73d-4b49-9163-d607e8ec7789" + }, + "displayName": "eclient_0_m_0" + } + ], + "contentInfo": [ + { + "uuid": "ce88f026-e73d-4b49-9163-d607e8ec7789", + "dsId": "8e6d22d4-4890-4e0d-a574-0d2e757f951b", + "URL": "lfedge/eden-eclient:b96434e", + "iformat": 8, + "displayName": "lfedge/eden-eclient:b96434e" + } + ], + "datastores": [ + { + "id": "8e6d22d4-4890-4e0d-a574-0d2e757f951b", + "dType": 5, + "fqdn": "docker://index.docker.io" + } + ], + + "configItems": [ + { + "key": "network.fallback.any.eth", + "value": "disabled" + }, + { + "key": "newlog.allow.fastupload", + "value": "true" + }, + { + "key": "timer.config.interval", + "value": "10" + }, + { + "key": "timer.location.app.interval", + "value": "10" + }, + { + "key": "timer.location.cloud.interval", + "value": "300" + }, + { + "key": "app.allow.vnc", + "value": "true" + }, + { + "key": "timer.download.retry", + "value": "60" + }, + { + "key": "debug.default.loglevel", + "value": "debug" + } + ] +} diff --git a/sdn/examples/ni-with-multiple-ports/network-model.json b/sdn/examples/ni-with-multiple-ports/network-model.json new file mode 100644 index 000000000..f92e54519 --- /dev/null +++ b/sdn/examples/ni-with-multiple-ports/network-model.json @@ -0,0 +1,165 @@ +{ + "ports": [ + { + "logicalLabel": "eveport0", + "adminUP": true + }, + { + "logicalLabel": "eveport1", + "adminUP": true + }, + { + "logicalLabel": "eveport2", + "adminUP": true + }, + { + "logicalLabel": "eveport3", + "adminUP": true + } + ], + "bridges": [ + { + "logicalLabel": "bridge0", + "ports": ["eveport0"] + }, + { + "logicalLabel": "bridge1", + "ports": ["eveport1"] + }, + { + "logicalLabel": "bridge2", + "ports": ["eveport2"] + }, + { + "logicalLabel": "bridge3", + "ports": ["eveport3"] + } + ], + "networks": [ + { + "logicalLabel": "network0", + "bridge": "bridge0", + "subnet": "172.22.10.0/24", + "gwIP": "172.22.10.1", + "dhcp": { + "enable": true, + "ipRange": { + "fromIP": "172.22.10.10", + "toIP": "172.22.10.20" + }, + "domainName": "sdn", + "privateDNS": ["dns-server0"], + "publicNTP": "132.163.96.2" + }, + "router": { + "outsideReachability": true, + "reachableEndpoints": ["dns-server0", "httpserver0"] + } + }, + { + "logicalLabel": "network1", + "bridge": "bridge1", + "subnet": "172.28.20.0/24", + "gwIP": "172.28.20.1", + "dhcp": { + "enable": false + }, + "router": { + "outsideReachability": false, + "reachableEndpoints": ["dns-server0"] + } + }, + { + "logicalLabel": "network2", + "bridge": "bridge2", + "subnet": "192.168.30.0/24", + "gwIP": "192.168.30.1", + "dhcp": { + "enable": true, + "ipRange": { + "fromIP": "192.168.30.10", + "toIP": "192.168.30.20" + }, + "domainName": "sdn", + "privateDNS": ["dns-server1"], + "withoutDefaultRoute": false, + "publicNTP": "128.138.140.44" + }, + "router": { + "outsideReachability": false, + "reachableEndpoints": ["dns-server1", "httpserver0"] + } + }, + { + "logicalLabel": "network3", + "bridge": "bridge3", + "subnet": "10.40.40.0/24", + "gwIP": "10.40.40.1", + "dhcp": { + "enable": false + }, + "router": { + "outsideReachability": true, + "reachableEndpoints": ["dns-server1", "httpserver0"] + } + } + ], + "endpoints": { + "dnsServers": [ + { + "logicalLabel": "dns-server0", + "fqdn": "dns-server0.sdn", + "subnet": "10.16.16.0/24", + "ip": "10.16.16.25", + "staticEntries": [ + { + "fqdn": "mydomain.adam", + "ip": "adam-ip" + }, + { + "fqdn": "endpoint-fqdn.httpserver0", + "ip": "endpoint-ip.httpserver0" + } + ], + "upstreamServers": [ + "1.1.1.1", + "8.8.8.8" + ] + }, + { + "logicalLabel": "dns-server1", + "fqdn": "dns-server1.sdn", + "subnet": "10.17.17.0/24", + "ip": "10.17.17.25", + "staticEntries": [ + { + "fqdn": "mydomain.adam", + "ip": "adam-ip" + }, + { + "fqdn": "endpoint-fqdn.httpserver0", + "ip": "endpoint-ip.httpserver0" + } + ], + "upstreamServers": [ + "8.8.8.8" + ] + } + ], + "httpServers": [ + { + "logicalLabel": "httpserver0", + "fqdn": "httpserver0.sdn", + "subnet": "10.88.88.0/24", + "ip": "10.88.88.70", + "httpPort": 80, + "paths": { + "/helloworld": { + "contentType": "text/plain", + "content": "Hello world from HTTP server\n" + } + } + } + ] + } +} \ No newline at end of file diff --git a/sdn/vm/cmd/sdnagent/config.go b/sdn/vm/cmd/sdnagent/config.go index d60a16fff..0f0c3b7a5 100644 --- a/sdn/vm/cmd/sdnagent/config.go +++ b/sdn/vm/cmd/sdnagent/config.go @@ -583,6 +583,25 @@ func (a *agent) getIntendedNetwork(network api.Network) dg.Graph { Rules: dnatRules, }, nil) } + + // When user is accessing EVE using "sdn fwd" command, the source IP + // is from the internal IP subnet. + // Make sure that the IP address is S-NATed before sending packets to EVE. + // Otherwise, the responses could be routed out via wrong EVE network ports. + intendedCfg.PutItem(configitems.IptablesChain{ + NetNamespace: nsName, + ChainName: "POSTROUTING", + Table: "nat", + ForIPv6: false, + RefersVeths: []string{rtVethName}, + Rules: []configitems.IptablesRule{ + { + Args: []string{"-o", brInIfName, "-s", internalIPv4Subnet.String(), + "-j", "MASQUERADE"}, + Description: "S-NAT traffic leaving SDN VM towards EVE with internal source IP", + }, + }, + }, nil) return intendedCfg } diff --git a/sdn/vm/cmd/sdnagent/ipam.go b/sdn/vm/cmd/sdnagent/ipam.go index 372be20de..089a9c48c 100644 --- a/sdn/vm/cmd/sdnagent/ipam.go +++ b/sdn/vm/cmd/sdnagent/ipam.go @@ -10,10 +10,12 @@ import ( var intOne = big.NewInt(1) var internalIPv4Base *ipAsInt +var internalIPv4Subnet *net.IPNet func init() { // 240.0.0.0/4 is reserved internalIPv4Base = ipToInt(net.ParseIP("240.0.0.0")) + _, internalIPv4Subnet, _ = net.ParseCIDR("240.0.0.0/4") } type ipAsInt struct { diff --git a/sdn/vm/scripts/get-eve-ip.sh b/sdn/vm/scripts/get-eve-ip.sh index 05230d096..b26cbf313 100755 --- a/sdn/vm/scripts/get-eve-ip.sh +++ b/sdn/vm/scripts/get-eve-ip.sh @@ -8,7 +8,7 @@ find_ip() { if [ -n "$NETNS" ]; then PREFIX="ip netns exec $NETNS " fi - $PREFIX arp -a | while read ARP_ENTRY; do + $PREFIX arp -an | while read ARP_ENTRY; do local ENTRY_IP="$(echo "$ARP_ENTRY" | cut -d ' ' -f 2 | tr -d '()')" local ENTRY_MAC="$(echo "$ARP_ENTRY" | cut -d ' ' -f 4)" if [ "$MAC" = "$ENTRY_MAC" ]; then