diff --git a/.github/workflows/buildtest.yml b/.github/workflows/buildtest.yml index c6947cde..1fa235a8 100644 --- a/.github/workflows/buildtest.yml +++ b/.github/workflows/buildtest.yml @@ -36,7 +36,7 @@ jobs: ${{ runner.os }}-bazel- - name: Install pcap run: | - sudo apt-get install libpcap-dev + sudo apt-get install libpcap-dev libnl-genl-3-dev libnl-3-dev - name: Build Lemming run: bazel build //... - name: Save Bazel Cache @@ -63,7 +63,7 @@ jobs: ${{ runner.os }}-bazel- - name: Install pcap run: | - sudo apt-get install libpcap-dev + sudo apt-get install libpcap-dev libnl-genl-3-dev libnl-3-dev - name: Test run: make coverage - name: Coveralls diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index e5f24c6f..140a5a9b 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -36,7 +36,7 @@ jobs: - uses: actions/checkout@v3 - name: Install pcap run: | - sudo apt-get install libpcap-dev + sudo apt-get install libpcap-dev libnl-genl-3-dev libnl-3-dev - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: diff --git a/cloudbuild/buildtest.yaml b/cloudbuild/buildtest.yaml index 28802101..bea38783 100644 --- a/cloudbuild/buildtest.yaml +++ b/cloudbuild/buildtest.yaml @@ -4,12 +4,12 @@ steps: script: | curl -Lo bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.16.0/bazelisk-linux-amd64 && \ install bazel /usr/local/bin/ - apt-get update && apt-get -y install libpcap-dev + apt-get update && apt-get -y install libpcap-dev libnl-genl-3-dev libnl-3-dev bazel build --remote_cache=https://storage.googleapis.com/lemming-bazel-cache --google_default_credentials //... - id: test name: gcr.io/cloud-builders/bazel script: | - apt-get update && apt-get -y install libpcap-dev + apt-get update && apt-get -y install libpcap-dev libnl-genl-3-dev libnl-3-dev bazel test --test_output=errors --combined_report=lcov --remote_cache=https://storage.googleapis.com/lemming-bazel-cache --google_default_credentials \ $(bazel query 'attr(size, small, tests("//...")) + attr(size, medium, tests("//..."))') timeout: 3600s diff --git a/cloudbuild/lemming-test.sh b/cloudbuild/lemming-test.sh index 162b49f6..411b545e 100755 --- a/cloudbuild/lemming-test.sh +++ b/cloudbuild/lemming-test.sh @@ -26,7 +26,7 @@ gopath=$(go env GOPATH) export PATH=${PATH}:$gopath/bin curl -Lo bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.16.0/bazelisk-linux-amd64 && \ sudo install bazel /usr/local/bin/ -sudo apt-get -y install libpcap-dev +sudo apt-get -y install libpcap-dev libnl-genl-3-dev libnl-3-dev cd /tmp/workspace kne deploy ~/kne-internal/deploy/kne/kind-bridge.yaml diff --git a/cloudbuild/operator-test.sh b/cloudbuild/operator-test.sh index a485c433..71d14af7 100755 --- a/cloudbuild/operator-test.sh +++ b/cloudbuild/operator-test.sh @@ -27,7 +27,7 @@ curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/latest/skaffo sudo install skaffold /usr/local/bin/ curl -Lo bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.16.0/bazelisk-linux-amd64 && \ sudo install bazel /usr/local/bin/ -sudo apt-get -y install libpcap-dev +sudo apt-get -y install libpcap-dev libnl-genl-3-dev libnl-3-dev cd /tmp/workspace kne deploy ~/kne-internal/deploy/kne/kind-bridge.yaml diff --git a/cloudbuild/presubmit.sh b/cloudbuild/presubmit.sh index adeea4ac..cfb41e1a 100755 --- a/cloudbuild/presubmit.sh +++ b/cloudbuild/presubmit.sh @@ -35,7 +35,7 @@ curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/latest/skaffo sudo install skaffold /usr/local/bin/ curl -Lo bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.16.0/bazelisk-linux-amd64 && \ sudo install bazel /usr/local/bin/ -sudo apt-get -y install libpcap-dev +sudo apt-get -y install libpcap-dev libnl-genl-3-dev libnl-3-dev cd /tmp/workspace kne deploy ~/kne-internal/deploy/kne/kind-bridge.yaml diff --git a/dataplane/BUILD b/dataplane/BUILD index 1b4d84af..6c61f7e6 100644 --- a/dataplane/BUILD +++ b/dataplane/BUILD @@ -11,6 +11,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//dataplane/dplaneopts", + "//dataplane/kernel/tap", "//dataplane/proto/packetio", "//dataplane/proto/sai", "//dataplane/protocol", diff --git a/dataplane/forwarding/fwd.go b/dataplane/forwarding/fwd.go index 3755861b..f73014e6 100644 --- a/dataplane/forwarding/fwd.go +++ b/dataplane/forwarding/fwd.go @@ -97,26 +97,6 @@ func (e *Server) UpdateNotification(contextID *fwdpb.ContextId, notification fwd return nil } -// UpdatePacketSink updates the packet sink service for a context. If the -// service is set to nil, no packets are delivered externally for the context. -// The address is the address of the packet service (used in queries) -// in the host:port format. -func (e *Server) UpdatePacketSink(contextID *fwdpb.ContextId, packet fwdcontext.PacketCallback) error { - if contextID == nil { - return errors.New("fwd: UpdatePacketSink failed, No context ID") - } - - ctx, err := e.FindContext(contextID) - if err != nil { - return fmt.Errorf("fwd: UpdatePacketSink failed, err %v", err) - } - - ctx.Lock() - defer ctx.Unlock() - ctx.SetPacketSink(packet) - return nil -} - // ContextCreate creates a new context. Note that if the packet sink and/or // notification services are specified but not reachable, the context creation // fails. diff --git a/dataplane/forwarding/fwdport/ports/BUILD b/dataplane/forwarding/fwdport/ports/BUILD index cb5de7ab..a0024421 100644 --- a/dataplane/forwarding/fwdport/ports/BUILD +++ b/dataplane/forwarding/fwdport/ports/BUILD @@ -68,6 +68,7 @@ go_test( "//dataplane/forwarding/protocol/ethernet", "//dataplane/forwarding/protocol/metadata", "//dataplane/forwarding/protocol/opaque", + "//dataplane/proto/packetio", "//proto/forwarding", "@com_github_go_logr_logr//testr", "@com_github_google_go_cmp//cmp", @@ -75,7 +76,6 @@ go_test( "@com_github_google_gopacket//layers", "@com_github_google_gopacket//pcapgo", "@com_github_openconfig_gnmi//errdiff", - "@org_golang_google_protobuf//proto", "@org_uber_go_mock//gomock", ], ) diff --git a/dataplane/forwarding/fwdport/ports/cpu.go b/dataplane/forwarding/fwdport/ports/cpu.go index 3f41c4c1..37583f84 100644 --- a/dataplane/forwarding/fwdport/ports/cpu.go +++ b/dataplane/forwarding/fwdport/ports/cpu.go @@ -98,31 +98,9 @@ func (p *CPUPort) Update(upd *fwdpb.PortUpdateDesc) error { // Write applies output actions and writes a packet to the cable. func (p *CPUPort) Write(packet fwdpacket.Packet) (fwdaction.State, error) { - if p.remote { - if err := p.queue.Write(packet); err != nil { - return fwdaction.DROP, err - } - return fwdaction.CONSUME, nil - } - - // After the CPU packet is processed, the output port may change. Rerun the output actions. - outPort, err := fwdport.OutputPort(packet, p.ctx) - if err != nil { + if err := p.queue.Write(packet); err != nil { return fwdaction.DROP, err } - - // TODO: The types of ports are sent over gRPC should be probably be configurable. - if outPort.Type() == fwdpb.PortType_PORT_TYPE_GENETLINK || outPort.Type() == fwdpb.PortType_PORT_TYPE_CPU_PORT { - if err := p.queue.Write(packet); err != nil { - return fwdaction.DROP, err - } - return fwdaction.CONSUME, nil - } - - if err := fwdport.Output(outPort, packet, fwdpb.PortAction_PORT_ACTION_OUTPUT, p.ctx); err != nil { - return fwdaction.DROP, err - } - return fwdaction.CONSUME, nil } @@ -131,97 +109,30 @@ func (p *CPUPort) Write(packet fwdpacket.Packet) (fwdaction.State, error) { // relock the context. We also do not want to hold the lock when performing // the gRPC request. func (p *CPUPort) punt(v any) { - if p.remote { - p.puntRemotePort(v) - return - } packet, ok := v.(fwdpacket.Packet) if !ok { fwdport.Increment(p, 1, fwdpb.CounterId_COUNTER_ID_TX_ERROR_PACKETS, fwdpb.CounterId_COUNTER_ID_TX_ERROR_OCTETS) return } - p.ctx.RLock() var ingressPID *fwdpb.PortId if port, err := fwdport.InputPort(packet, p.ctx); err == nil { ingressPID = fwdport.GetID(port) } - egressPID := fwdport.GetID(p) - if port, err := fwdport.OutputPort(packet, p.ctx); err == nil { - egressPID = fwdport.GetID(port) - } - var parsed []*fwdpb.PacketFieldBytes - for _, f := range p.export { - value, err := packet.Field(fwdpacket.NewFieldID(f)) - if err != nil { - continue - } - parsed = append(parsed, &fwdpb.PacketFieldBytes{ - FieldId: f, - Bytes: value, - }) - } - response := &fwdpb.PacketSinkResponse{ - Resp: &fwdpb.PacketSinkResponse_Packet{ - Packet: &fwdpb.PacketSinkPacketInfo{ - PortId: fwdport.GetID(p), - Egress: egressPID, - Ingress: ingressPID, - Bytes: packet.Frame(), - ParsedFields: parsed, - }, - }, - } - - ps := p.ctx.PacketSink() - p.ctx.RUnlock() - if ps != nil { - timer := deadlock.NewTimer(deadlock.Timeout, fmt.Sprintf("Punting packet from port %v", p)) - err := ps(response) - timer.Stop() - if err == nil { - return - } - log.Errorf("ports: Unable to punt packet, request %+v, err %v.", response, err) - } - fwdport.Increment(p, packet.Length(), fwdpb.CounterId_COUNTER_ID_TX_ERROR_PACKETS, fwdpb.CounterId_COUNTER_ID_TX_ERROR_OCTETS) -} - -func (p *CPUPort) puntRemotePort(v any) { - packet, ok := v.(fwdpacket.Packet) - if !ok { - fwdport.Increment(p, 1, fwdpb.CounterId_COUNTER_ID_TX_ERROR_PACKETS, fwdpb.CounterId_COUNTER_ID_TX_ERROR_OCTETS) - return - } - - p.ctx.RLock() - var ingressPID *fwdpb.PortId - if port, err := fwdport.InputPort(packet, p.ctx); err == nil { - ingressPID = fwdport.GetID(port) - } - egressPID := fwdport.GetID(p) - if port, err := fwdport.OutputPort(packet, p.ctx); err == nil { - egressPID = fwdport.GetID(port) - } hostPort, err := packet.Field(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_HOST_PORT_ID, 0)) if err != nil { fwdport.Increment(p, packet.Length(), fwdpb.CounterId_COUNTER_ID_TX_ERROR_PACKETS, fwdpb.CounterId_COUNTER_ID_TX_ERROR_OCTETS) return } - ingressID, err := strconv.Atoi(ingressPID.GetObjectId().GetId()) if err != nil { return } - egressID, err := strconv.Atoi(egressPID.GetObjectId().GetId()) - if err != nil { - return - } response := &pktiopb.PacketOut{ Packet: &pktiopb.Packet{ InputPort: uint64(ingressID), - OutputPort: uint64(egressID), + OutputPort: uint64(0), // TODO: If the packet was punted after the FIB, this not be output port. HostPort: binary.BigEndian.Uint64(hostPort), Frame: packet.Frame(), }, diff --git a/dataplane/forwarding/fwdport/ports/cpu_test.go b/dataplane/forwarding/fwdport/ports/cpu_test.go index 55bfe902..78c43bf9 100755 --- a/dataplane/forwarding/fwdport/ports/cpu_test.go +++ b/dataplane/forwarding/fwdport/ports/cpu_test.go @@ -15,14 +15,16 @@ package ports import ( + "encoding/binary" "testing" - "google.golang.org/protobuf/proto" + "github.com/google/go-cmp/cmp" "github.com/openconfig/lemming/dataplane/forwarding/fwdport" "github.com/openconfig/lemming/dataplane/forwarding/infra/fwdcontext" "github.com/openconfig/lemming/dataplane/forwarding/infra/fwdobject" "github.com/openconfig/lemming/dataplane/forwarding/infra/fwdpacket" + pktiopb "github.com/openconfig/lemming/dataplane/proto/packetio" fwdpb "github.com/openconfig/lemming/proto/forwarding" _ "github.com/openconfig/lemming/dataplane/forwarding/protocol/arp" @@ -34,46 +36,33 @@ import ( // A RecordPacketSink records the last injected packet and generates a // notification on its channel. type RecordPacketSink struct { - notify chan bool - lastResponse *fwdpb.PacketSinkResponse + notify chan bool + resp *pktiopb.PacketOut } // PacketSink records the inject packet request and generates a notification. -func (p *RecordPacketSink) PacketSink(resp *fwdpb.PacketSinkResponse) error { - p.lastResponse = resp +func (p *RecordPacketSink) Send(resp *pktiopb.PacketOut) error { + p.resp = resp p.notify <- true return nil } // Tests the writes to the CPU port. func TestCpuWrite(t *testing.T) { - name := "TestPort" + name := "1" ctx := fwdcontext.New("test", "fwd") ps := &RecordPacketSink{ notify: make(chan bool), } - ctx.SetPacketSink(ps.PacketSink) + ctx.SetCPUPortSink(ps.Send, func() {}) - // Create a CPU port that exports the ETHER_TYPE and IP_ADDR_DST desc := &fwdpb.PortDesc{ PortType: fwdpb.PortType_PORT_TYPE_CPU_PORT, PortId: fwdport.MakeID(fwdobject.NewID(name)), } - ethertype := &fwdpb.PacketFieldId{ - Field: &fwdpb.PacketField{ - FieldNum: fwdpb.PacketFieldNum_PACKET_FIELD_NUM_ETHER_TYPE, - Instance: 0, - }, - } - ipaddress := &fwdpb.PacketFieldId{ - Field: &fwdpb.PacketField{ - FieldNum: fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_ADDR_DST, - Instance: 0, - }, - } + cpu := &fwdpb.CPUPortDesc{ - QueueId: name, - ExportFieldIds: []*fwdpb.PacketFieldId{ethertype, ipaddress}, + QueueId: name, } desc.Port = &fwdpb.PortDesc_Cpu{ Cpu: cpu, @@ -95,6 +84,8 @@ func TestCpuWrite(t *testing.T) { if err != nil { t.Fatalf("Unable to create ARP packet, err %v.", err) } + packet.Update(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_HOST_PORT_ID, 1), fwdpacket.OpSet, binary.BigEndian.AppendUint64(nil, 1)) + fwdport.SetInputPort(packet, port) fwdport.SetOutputPort(packet, port) // Write the packet out of the cpu port and wait for it to be received @@ -103,19 +94,12 @@ func TestCpuWrite(t *testing.T) { t.Fatalf("Write failed, err %v.", err) } <-ps.notify - t.Logf("Got request %+v", ps.lastResponse) + t.Logf("Got request %+v", ps.resp) // Verify that the packet was received and the parsed fields only have // the ETHER_TYPE set to ARP. - list := ps.lastResponse.GetPacket().GetParsedFields() - if len(list) != 1 { - t.Fatalf("Write failed to get parsed fields, got %v, want 1.", len(list)) - } - want := &fwdpb.PacketFieldBytes{ - FieldId: ethertype, - Bytes: []byte{0x08, 0x06}, - } - if !proto.Equal(list[0], want) { - t.Fatalf("Write failed to get parsed fields, Got %v, want %v", list[0], want) + got := ps.resp.GetPacket() + if d := cmp.Diff(got.GetFrame(), arp); d != "" { + t.Fatalf("Write failed to get parsed fields, diff: %s", d) } } diff --git a/dataplane/forwarding/fwdport/ports/group_test.go b/dataplane/forwarding/fwdport/ports/group_test.go index b6d3174e..6a56eca8 100755 --- a/dataplane/forwarding/fwdport/ports/group_test.go +++ b/dataplane/forwarding/fwdport/ports/group_test.go @@ -127,7 +127,6 @@ func TestPortGroupHash(t *testing.T) { defer ctrl.Finish() names := []string{"p1", "p2", "p3", "p4"} - ctx := fwdcontext.New("test", "fwd") hashFn := []fwdpb.AggregateHashAlgorithm{ fwdpb.AggregateHashAlgorithm_AGGREGATE_HASH_ALGORITHM_CRC16, @@ -135,6 +134,7 @@ func TestPortGroupHash(t *testing.T) { } for id, fn := range hashFn { + ctx := fwdcontext.New("test", "fwd") var ports []fwdport.Port for _, name := range names { ports = append(ports, porttestutil.CreateTestPort(t, ctx, fmt.Sprintf("%v-%v", name, id))) diff --git a/dataplane/forwarding/infra/fwdcontext/context.go b/dataplane/forwarding/infra/fwdcontext/context.go index c3b1a0ad..2a676856 100644 --- a/dataplane/forwarding/infra/fwdcontext/context.go +++ b/dataplane/forwarding/infra/fwdcontext/context.go @@ -31,10 +31,6 @@ import ( fwdpb "github.com/openconfig/lemming/proto/forwarding" ) -// A PacketCallback transmits packets to a packet sink as specified by the -// injection request. -type PacketCallback func(*fwdpb.PacketSinkResponse) error - // An NotificationCallback generates events to a notification service. type NotificationCallback func(*fwdpb.EventDesc) @@ -65,7 +61,6 @@ type FakePortManager interface { type Context struct { sync.RWMutex // Synchronization between provisioning and forwarding Objects *fwdobject.Table // Set of all visible forwarding objects - packets PacketCallback // Packet service ID string // ID of the context Instance string // Name of the forwarding engine instance Attributes fwdattribute.Set @@ -174,19 +169,6 @@ func (ctx *Context) Notify(event *fwdpb.EventDesc) error { type CPUPortSink func(*pktiopb.PacketOut) error -// SetPacketSink sets the packet sink service for the context. If the packet -// sink service is not set to nil, packets are dropped. -// TODO: Deprecated remove -func (ctx *Context) SetPacketSink(call PacketCallback) error { - ctx.packets = call - return nil -} - -// PacketSink returns a handler to the packet sink service. -func (ctx *Context) PacketSink() PacketCallback { - return ctx.packets -} - // SetCPUPortSink sets the port control service for the context func (ctx *Context) SetCPUPortSink(fn CPUPortSink, doneFn func()) error { ctx.cpuPortSink = fn @@ -204,7 +186,6 @@ func (ctx *Context) CPUPortSink() CPUPortSink { // Then it unblocks the caller by sending a message on the channel. // Then it cleans up the rest of the objects. func (ctx *Context) Cleanup(ch chan bool, isPort func(*fwdpb.ObjectId) bool) { - ctx.SetPacketSink(nil) ctx.SetNotification(nil) if ctx.cpuPortSinkDone != nil { diff --git a/dataplane/kernel/BUILD b/dataplane/kernel/BUILD index e0764711..591291da 100644 --- a/dataplane/kernel/BUILD +++ b/dataplane/kernel/BUILD @@ -2,24 +2,16 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "kernel", - srcs = [ - "genetlink.go", - "kernel.go", - "tap.go", - ], + srcs = ["kernel.go"], importpath = "github.com/openconfig/lemming/dataplane/kernel", visibility = ["//dataplane:__subpackages__"], - deps = [ - "@com_github_vishvananda_netlink//:netlink", - ] + select({ + deps = select({ "@io_bazel_rules_go//go/platform:android": [ - "@com_github_mdlayher_genetlink//:genetlink", - "@com_github_mdlayher_netlink//:netlink", + "@com_github_vishvananda_netlink//:netlink", "@org_golang_x_sys//unix", ], "@io_bazel_rules_go//go/platform:linux": [ - "@com_github_mdlayher_genetlink//:genetlink", - "@com_github_mdlayher_netlink//:netlink", + "@com_github_vishvananda_netlink//:netlink", "@org_golang_x_sys//unix", ], "//conditions:default": [], diff --git a/dataplane/kernel/genetlink.go b/dataplane/kernel/genetlink.go deleted file mode 100644 index 0c01a1cd..00000000 --- a/dataplane/kernel/genetlink.go +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build linux - -package kernel - -import ( - "fmt" - "io" - - "github.com/mdlayher/genetlink" - "github.com/mdlayher/netlink" -) - -// GenetlinkPort is connect to a netlink socket that be written to. -type GenetlinkPort struct { - conn *genetlink.Conn - familyID uint16 -} - -// NewGenetlinkPort creates netlink socket for the given family and multicast group. -func NewGenetlinkPort(family, group string) (*GenetlinkPort, error) { - conn, err := genetlink.Dial(nil) - if err != nil { - return nil, err - } - fam, err := conn.GetFamily(family) - if err != nil { - return nil, fmt.Errorf("could not find %v family", family) - } - grpID := -1 - for _, grp := range fam.Groups { - if grp.Name == group { - grpID = int(grp.ID) - break - } - } - if grpID == -1 { - return nil, fmt.Errorf("could not find multicast group in the %v family", family) - } - if err := conn.JoinGroup(uint32(grpID)); err != nil { - return nil, err - } - return &GenetlinkPort{ - conn: conn, - }, nil -} - -type PacketMetadata struct { - SrcIfIndex int - DstIfIndex int - Context int // Context is extra value that can be set by the forwarding pipeline. -} - -// Writes writes a layer2 frame to the port. -func (p GenetlinkPort) Write(frame []byte, md *PacketMetadata) (int, error) { - data, err := (&NLPacket{ - payload: frame, - srcIfIndex: int16(md.SrcIfIndex), - dstIfIndex: int16(md.DstIfIndex), - contextValue: uint32(md.Context), - }).Encode() - if err != nil { - return 0, err - } - - _, err = p.conn.Send(genetlink.Message{ - Header: genetlink.Header{ - Command: 1, - Version: 1, - }, - Data: data, - }, p.familyID, 0) - return len(data), err -} - -// Read is not implemented. -func (p GenetlinkPort) Read([]byte) (int, error) { - return 0, io.EOF -} - -// Delete closes the netlink connection. -func (p GenetlinkPort) Delete() error { - return p.conn.Close() -} - -// NLPacket contains a packet data. -type NLPacket struct { - srcIfIndex int16 - dstIfIndex int16 - contextValue uint32 - payload []byte -} - -// Constants sourced from https://github.com/sonic-net/sonic-pins/blob/main/p4rt_app/sonic/receive_genetlink.cc#L32 -const ( - AttrDstIfIndex uint16 = iota - AttrSrcIfIndex - AttrContextValue - AttrPayload -) - -// Encode encodes the packet into a netlink-compatible byte slice. -func (nl *NLPacket) Encode() ([]byte, error) { - enc := netlink.NewAttributeEncoder() - enc.Int16(AttrSrcIfIndex, nl.srcIfIndex) - enc.Int16(AttrSrcIfIndex, nl.dstIfIndex) - enc.Uint32(AttrContextValue, nl.contextValue) - enc.Bytes(AttrPayload, nl.payload) - return enc.Encode() -} diff --git a/dataplane/kernel/genetlink/BUILD b/dataplane/kernel/genetlink/BUILD new file mode 100644 index 00000000..a3befaff --- /dev/null +++ b/dataplane/kernel/genetlink/BUILD @@ -0,0 +1,48 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "genetlink", + srcs = [ + "genetlink.c", + "genetlink.go", + "genetlink.h", + ], + cgo = True, + clinkopts = select({ + "@io_bazel_rules_go//go/platform:android": [ + "-lnl-3 -lnl-genl-3", + ], + "@io_bazel_rules_go//go/platform:linux": [ + "-lnl-3 -lnl-genl-3", + ], + "//conditions:default": [], + }), + copts = select({ + "@io_bazel_rules_go//go/platform:android": [ + "-I/usr/include/libnl3", + ], + "@io_bazel_rules_go//go/platform:linux": [ + "-I/usr/include/libnl3", + ], + "//conditions:default": [], + }), + importpath = "github.com/openconfig/lemming/dataplane/kernel/genetlink", + visibility = ["//visibility:public"], + deps = select({ + "@io_bazel_rules_go//go/platform:android": [ + "//dataplane/kernel", + "//dataplane/proto/packetio", + "//dataplane/standalone/pkthandler/pktiohandler", + "@com_github_golang_glog//:glog", + "@com_github_mdlayher_genetlink//:genetlink", + ], + "@io_bazel_rules_go//go/platform:linux": [ + "//dataplane/kernel", + "//dataplane/proto/packetio", + "//dataplane/standalone/pkthandler/pktiohandler", + "@com_github_golang_glog//:glog", + "@com_github_mdlayher_genetlink//:genetlink", + ], + "//conditions:default": [], + }), +) diff --git a/dataplane/kernel/genetlink/genetlink.c b/dataplane/kernel/genetlink/genetlink.c new file mode 100644 index 00000000..7d983ade --- /dev/null +++ b/dataplane/kernel/genetlink/genetlink.c @@ -0,0 +1,82 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "genetlink.h" + +#include +#include +#include +#include +#include +#include +#include + +enum { + /* packet metadata */ + GENL_PACKET_ATTR_IIFINDEX, + GENL_PACKET_ATTR_OIFINDEX, + GENL_PACKET_ATTR_CONTEXT, + GENL_PACKET_ATTR_DATA, +}; + +struct nl_sock* create_port(const char* family, const char* group) { + fprintf(stderr, "creating port\n"); + + struct nl_sock* sock = nl_socket_alloc(); + if (sock == NULL) { + fprintf(stderr, "error: failed to alloc nl socket"); + return NULL; + } + nl_socket_disable_auto_ack(sock); + int error = genl_connect(sock); + if (error < 0) { + fprintf(stderr, "error: failed to disable auto ack: err %d", error); + nl_socket_free(sock); + return NULL; + } + int group_id = genl_ctrl_resolve_grp(sock, family, group); + if (group_id < 0) { + fprintf(stderr, "error: failed to resolve group: err %d", group_id); + nl_socket_free(sock); + return NULL; + } + nl_socket_set_peer_groups(sock, (1 << (group_id - 1))); + return sock; +} + +void delete_port(void* sock) { nl_socket_free(sock); } + +int send_packet(void* sock, int family, const void* pkt, uint32_t size, + int in_ifindex, int out_ifindex, unsigned int context) { + struct nl_msg* msg = nlmsg_alloc(); + if (msg == NULL) { + fprintf(stderr, "failed to allocate packet\n"); + return -1; + } + genlmsg_put(msg, NL_AUTO_PORT, NL_AUTO_SEQ, family, 0, 0, 0, 1); + NLA_PUT_S16(msg, GENL_PACKET_ATTR_IIFINDEX, in_ifindex); + NLA_PUT_S16(msg, GENL_PACKET_ATTR_OIFINDEX, out_ifindex); + NLA_PUT_U32(msg, GENL_PACKET_ATTR_CONTEXT, context); + NLA_PUT(msg, GENL_PACKET_ATTR_DATA, size, pkt); + fprintf(stderr, "sending packet size: %d\n", size); + if (nl_send(sock, msg) < 0) { + fprintf(stderr, "failed to send packet\n"); + return -1; + } + nlmsg_free(msg); + return 0; +nla_put_failure: + nlmsg_free(msg); + return -1; +} diff --git a/dataplane/kernel/genetlink/genetlink.go b/dataplane/kernel/genetlink/genetlink.go new file mode 100644 index 00000000..c52afe51 --- /dev/null +++ b/dataplane/kernel/genetlink/genetlink.go @@ -0,0 +1,112 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux + +package genetlink + +// #cgo LDFLAGS: -lnl-3 -lnl-genl-3 +// #cgo CFLAGS: -I/usr/include/libnl3 +// #include "genetlink.h" +// #include +import "C" + +import ( + "fmt" + "io" + "unsafe" + + log "github.com/golang/glog" + + "github.com/mdlayher/genetlink" + + "github.com/openconfig/lemming/dataplane/kernel" + pktiopb "github.com/openconfig/lemming/dataplane/proto/packetio" + "github.com/openconfig/lemming/dataplane/standalone/pkthandler/pktiohandler" +) + +// GenetlinkPort is connect to a netlink socket that be written to. +type GenetlinkPort struct { + sock unsafe.Pointer + familyID int +} + +// NewGenetlinkPort creates netlink socket for the given family and multicast group. +func New(msg *pktiopb.HostPortControlMessage) (pktiohandler.PortIO, error) { + log.Errorf("creating genl port: %s %s", msg.GetGenetlink().GetFamily(), msg.GetGenetlink().GetGroup()) + + cFamily := C.CString(msg.GetGenetlink().GetFamily()) + defer C.free(unsafe.Pointer(cFamily)) + cGroup := C.CString(msg.GetGenetlink().GetGroup()) + defer C.free(unsafe.Pointer(cGroup)) + + conn, err := genetlink.Dial(nil) + if err != nil { + return nil, err + } + fam, err := conn.GetFamily(msg.GetGenetlink().GetFamily()) + if err != nil { + return nil, err + } + familyID := -1 + for _, grp := range fam.Groups { + if grp.Name == msg.GetGenetlink().GetGroup() { + familyID = int(grp.ID) + } + } + if familyID == -1 { + return nil, fmt.Errorf("failed to find multicast group") + } + + sockAddr := C.create_port(cFamily, cGroup) + if sockAddr == nil { + return nil, fmt.Errorf("failed to create port") + } + + log.Errorf("creating genl port") + return &GenetlinkPort{ + sock: unsafe.Pointer(sockAddr), + familyID: familyID, + }, nil +} + +// Writes writes a layer2 frame to the port. +func (p GenetlinkPort) Write(frame []byte, md *kernel.PacketMetadata) (int, error) { + log.Errorf("writing genl packet: %x", frame) + + packet := C.CBytes(frame) + defer C.free(packet) + + res := C.send_packet(p.sock, C.int(p.familyID), packet, C.uint(uint32(len(frame))), C.int(md.SrcIfIndex), C.int(md.DstIfIndex), C.uint(md.Context)) + if res < 0 { + return 0, fmt.Errorf("failed to write packet") + } + + return len(frame), nil +} + +// Read is not implemented. +func (p GenetlinkPort) Read([]byte) (int, error) { + return 0, io.EOF +} + +// Delete closes the netlink connection. +func (p GenetlinkPort) Delete() error { + C.delete_port(p.sock) + return nil +} + +func init() { + pktiohandler.Register(pktiopb.PortType_PORT_TYPE_GENETLINK, New) +} diff --git a/dataplane/kernel/genetlink/genetlink.h b/dataplane/kernel/genetlink/genetlink.h new file mode 100644 index 00000000..07848bf4 --- /dev/null +++ b/dataplane/kernel/genetlink/genetlink.h @@ -0,0 +1,29 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DATAPLANE_KERNEL_GENETLINK_GENETLINK_H_ +#define DATAPLANE_KERNEL_GENETLINK_GENETLINK_H_ + +#include + +// create_port create genetlink socket. +struct nl_sock* create_port(const char* family, const char* group); + +void delete_port(void* sock); + +// send_packet sends a packet with given metadata to specified port. +int send_packet(void* sock, int family, const void* pkt, uint32_t size, + int in_ifindex, int out_ifindex, unsigned int context); + +#endif // DATAPLANE_KERNEL_GENETLINK_GENETLINK_H_ diff --git a/dataplane/kernel/kernel.go b/dataplane/kernel/kernel.go index 074b4f30..e8c2ef1f 100644 --- a/dataplane/kernel/kernel.go +++ b/dataplane/kernel/kernel.go @@ -183,3 +183,9 @@ func (k *Interfaces) LinkSetNoMaster(link netlink.Link) error { func (k *Interfaces) LinkModify(link netlink.Link) error { return netlink.LinkModify(link) } + +type PacketMetadata struct { + SrcIfIndex int16 + DstIfIndex int16 + Context uint32 // Context is extra value that can be set by the forwarding pipeline. +} diff --git a/dataplane/kernel/tap/BUILD b/dataplane/kernel/tap/BUILD new file mode 100644 index 00000000..ea6ccd66 --- /dev/null +++ b/dataplane/kernel/tap/BUILD @@ -0,0 +1,14 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "tap", + srcs = ["tap.go"], + importpath = "github.com/openconfig/lemming/dataplane/kernel/tap", + visibility = ["//visibility:public"], + deps = [ + "//dataplane/kernel", + "//dataplane/proto/packetio", + "//dataplane/standalone/pkthandler/pktiohandler", + "@com_github_vishvananda_netlink//:netlink", + ], +) diff --git a/dataplane/kernel/tap.go b/dataplane/kernel/tap/tap.go similarity index 70% rename from dataplane/kernel/tap.go rename to dataplane/kernel/tap/tap.go index 716912d2..68e1bb6b 100644 --- a/dataplane/kernel/tap.go +++ b/dataplane/kernel/tap/tap.go @@ -12,19 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. -package kernel +package tap import ( "os" "github.com/vishvananda/netlink" + + "github.com/openconfig/lemming/dataplane/kernel" + pktiopb "github.com/openconfig/lemming/dataplane/proto/packetio" + "github.com/openconfig/lemming/dataplane/standalone/pkthandler/pktiohandler" ) -// NewTap creates a new tap interface. -func NewTap(name string) (*TapInterface, error) { +// New creates a new tap interface. +func New(msg *pktiopb.HostPortControlMessage) (pktiohandler.PortIO, error) { tap := &netlink.Tuntap{ LinkAttrs: netlink.LinkAttrs{ - Name: name, + Name: msg.GetNetdev().GetName(), }, Mode: netlink.TUNTAP_MODE_TAP, Flags: netlink.TUNTAP_MULTI_QUEUE_DEFAULTS, @@ -35,7 +39,7 @@ func NewTap(name string) (*TapInterface, error) { return nil, err } return &TapInterface{ - name: name, + name: msg.GetNetdev().GetName(), File: tap.Fds[0], ifIndex: tap.Index, }, nil @@ -57,10 +61,14 @@ func (t *TapInterface) Delete() error { return netlink.LinkDel(l) } -func (t *TapInterface) Write(frame []byte, _ *PacketMetadata) (int, error) { +func (t *TapInterface) Write(frame []byte, _ *kernel.PacketMetadata) (int, error) { return t.File.Write(frame) } func (t *TapInterface) IfIndex() int { return t.ifIndex } + +func init() { + pktiohandler.Register(pktiopb.PortType_PORT_TYPE_NETDEV, New) +} diff --git a/dataplane/proto/packetio/packetio.pb.go b/dataplane/proto/packetio/packetio.pb.go index a7c6303e..491616ca 100755 --- a/dataplane/proto/packetio/packetio.pb.go +++ b/dataplane/proto/packetio/packetio.pb.go @@ -25,6 +25,55 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type PortType int32 + +const ( + PortType_PORT_TYPE_UNSPECIFIED PortType = 0 + PortType_PORT_TYPE_NETDEV PortType = 1 + PortType_PORT_TYPE_GENETLINK PortType = 2 +) + +// Enum value maps for PortType. +var ( + PortType_name = map[int32]string{ + 0: "PORT_TYPE_UNSPECIFIED", + 1: "PORT_TYPE_NETDEV", + 2: "PORT_TYPE_GENETLINK", + } + PortType_value = map[string]int32{ + "PORT_TYPE_UNSPECIFIED": 0, + "PORT_TYPE_NETDEV": 1, + "PORT_TYPE_GENETLINK": 2, + } +) + +func (x PortType) Enum() *PortType { + p := new(PortType) + *p = x + return p +} + +func (x PortType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (PortType) Descriptor() protoreflect.EnumDescriptor { + return file_dataplane_proto_packetio_packetio_proto_enumTypes[0].Descriptor() +} + +func (PortType) Type() protoreflect.EnumType { + return &file_dataplane_proto_packetio_packetio_proto_enumTypes[0] +} + +func (x PortType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use PortType.Descriptor instead. +func (PortType) EnumDescriptor() ([]byte, []int) { + return file_dataplane_proto_packetio_packetio_proto_rawDescGZIP(), []int{0} +} + type HostPortControlInit struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -652,26 +701,32 @@ var file_dataplane_proto_packetio_packetio_proto_rawDesc = []byte{ 0x4f, 0x75, 0x74, 0x12, 0x39, 0x0a, 0x06, 0x70, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6c, 0x75, 0x63, 0x69, 0x75, 0x73, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x70, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x69, 0x6f, 0x2e, - 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x06, 0x70, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x32, 0xed, - 0x01, 0x0a, 0x08, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x49, 0x4f, 0x12, 0x7d, 0x0a, 0x0f, 0x48, - 0x6f, 0x73, 0x74, 0x50, 0x6f, 0x72, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x12, 0x31, - 0x2e, 0x6c, 0x75, 0x63, 0x69, 0x75, 0x73, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, - 0x65, 0x2e, 0x70, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x69, 0x6f, 0x2e, 0x48, 0x6f, 0x73, 0x74, 0x50, - 0x6f, 0x72, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x31, 0x2e, 0x6c, 0x75, 0x63, 0x69, 0x75, 0x73, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x70, - 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x70, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x69, 0x6f, 0x2e, 0x48, 0x6f, - 0x73, 0x74, 0x50, 0x6f, 0x72, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x4d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x62, 0x0a, 0x0f, 0x43, 0x50, - 0x55, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x23, 0x2e, - 0x6c, 0x75, 0x63, 0x69, 0x75, 0x73, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, - 0x2e, 0x70, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x69, 0x6f, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, - 0x49, 0x6e, 0x1a, 0x24, 0x2e, 0x6c, 0x75, 0x63, 0x69, 0x75, 0x73, 0x2e, 0x64, 0x61, 0x74, 0x61, - 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x70, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x69, 0x6f, 0x2e, 0x50, - 0x61, 0x63, 0x6b, 0x65, 0x74, 0x4f, 0x75, 0x74, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x42, 0x38, - 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, - 0x6e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x6c, 0x65, 0x6d, 0x6d, 0x69, 0x6e, 0x67, 0x2f, - 0x64, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, - 0x70, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x69, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x06, 0x70, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x2a, 0x54, + 0x0a, 0x08, 0x50, 0x6f, 0x72, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x19, 0x0a, 0x15, 0x50, 0x4f, + 0x52, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, + 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x14, 0x0a, 0x10, 0x50, 0x4f, 0x52, 0x54, 0x5f, 0x54, 0x59, + 0x50, 0x45, 0x5f, 0x4e, 0x45, 0x54, 0x44, 0x45, 0x56, 0x10, 0x01, 0x12, 0x17, 0x0a, 0x13, 0x50, + 0x4f, 0x52, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x54, 0x4c, 0x49, + 0x4e, 0x4b, 0x10, 0x02, 0x32, 0xed, 0x01, 0x0a, 0x08, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x49, + 0x4f, 0x12, 0x7d, 0x0a, 0x0f, 0x48, 0x6f, 0x73, 0x74, 0x50, 0x6f, 0x72, 0x74, 0x43, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x12, 0x31, 0x2e, 0x6c, 0x75, 0x63, 0x69, 0x75, 0x73, 0x2e, 0x64, 0x61, + 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x70, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x69, 0x6f, + 0x2e, 0x48, 0x6f, 0x73, 0x74, 0x50, 0x6f, 0x72, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6c, 0x75, 0x63, 0x69, 0x75, 0x73, + 0x2e, 0x64, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x70, 0x61, 0x63, 0x6b, 0x65, + 0x74, 0x69, 0x6f, 0x2e, 0x48, 0x6f, 0x73, 0x74, 0x50, 0x6f, 0x72, 0x74, 0x43, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, + 0x12, 0x62, 0x0a, 0x0f, 0x43, 0x50, 0x55, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x53, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x12, 0x23, 0x2e, 0x6c, 0x75, 0x63, 0x69, 0x75, 0x73, 0x2e, 0x64, 0x61, 0x74, + 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x70, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x69, 0x6f, 0x2e, + 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x49, 0x6e, 0x1a, 0x24, 0x2e, 0x6c, 0x75, 0x63, 0x69, 0x75, + 0x73, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x70, 0x61, 0x63, 0x6b, + 0x65, 0x74, 0x69, 0x6f, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x4f, 0x75, 0x74, 0x22, 0x00, + 0x28, 0x01, 0x30, 0x01, 0x42, 0x38, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x6c, 0x65, + 0x6d, 0x6d, 0x69, 0x6e, 0x67, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x69, 0x6f, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -686,36 +741,38 @@ func file_dataplane_proto_packetio_packetio_proto_rawDescGZIP() []byte { return file_dataplane_proto_packetio_packetio_proto_rawDescData } +var file_dataplane_proto_packetio_packetio_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_dataplane_proto_packetio_packetio_proto_msgTypes = make([]protoimpl.MessageInfo, 9) var file_dataplane_proto_packetio_packetio_proto_goTypes = []interface{}{ - (*HostPortControlInit)(nil), // 0: lucius.dataplane.packetio.HostPortControlInit - (*HostPortControlRequest)(nil), // 1: lucius.dataplane.packetio.HostPortControlRequest - (*NetdevPort)(nil), // 2: lucius.dataplane.packetio.NetdevPort - (*GenetlinkPort)(nil), // 3: lucius.dataplane.packetio.GenetlinkPort - (*HostPortControlMessage)(nil), // 4: lucius.dataplane.packetio.HostPortControlMessage - (*Packet)(nil), // 5: lucius.dataplane.packetio.Packet - (*PacketStreamInit)(nil), // 6: lucius.dataplane.packetio.PacketStreamInit - (*PacketIn)(nil), // 7: lucius.dataplane.packetio.PacketIn - (*PacketOut)(nil), // 8: lucius.dataplane.packetio.PacketOut - (*status.Status)(nil), // 9: google.rpc.Status + (PortType)(0), // 0: lucius.dataplane.packetio.PortType + (*HostPortControlInit)(nil), // 1: lucius.dataplane.packetio.HostPortControlInit + (*HostPortControlRequest)(nil), // 2: lucius.dataplane.packetio.HostPortControlRequest + (*NetdevPort)(nil), // 3: lucius.dataplane.packetio.NetdevPort + (*GenetlinkPort)(nil), // 4: lucius.dataplane.packetio.GenetlinkPort + (*HostPortControlMessage)(nil), // 5: lucius.dataplane.packetio.HostPortControlMessage + (*Packet)(nil), // 6: lucius.dataplane.packetio.Packet + (*PacketStreamInit)(nil), // 7: lucius.dataplane.packetio.PacketStreamInit + (*PacketIn)(nil), // 8: lucius.dataplane.packetio.PacketIn + (*PacketOut)(nil), // 9: lucius.dataplane.packetio.PacketOut + (*status.Status)(nil), // 10: google.rpc.Status } var file_dataplane_proto_packetio_packetio_proto_depIdxs = []int32{ - 0, // 0: lucius.dataplane.packetio.HostPortControlRequest.init:type_name -> lucius.dataplane.packetio.HostPortControlInit - 9, // 1: lucius.dataplane.packetio.HostPortControlRequest.status:type_name -> google.rpc.Status - 2, // 2: lucius.dataplane.packetio.HostPortControlMessage.netdev:type_name -> lucius.dataplane.packetio.NetdevPort - 3, // 3: lucius.dataplane.packetio.HostPortControlMessage.genetlink:type_name -> lucius.dataplane.packetio.GenetlinkPort - 6, // 4: lucius.dataplane.packetio.PacketIn.init:type_name -> lucius.dataplane.packetio.PacketStreamInit - 5, // 5: lucius.dataplane.packetio.PacketIn.packet:type_name -> lucius.dataplane.packetio.Packet - 5, // 6: lucius.dataplane.packetio.PacketOut.packet:type_name -> lucius.dataplane.packetio.Packet - 1, // 7: lucius.dataplane.packetio.PacketIO.HostPortControl:input_type -> lucius.dataplane.packetio.HostPortControlRequest - 7, // 8: lucius.dataplane.packetio.PacketIO.CPUPacketStream:input_type -> lucius.dataplane.packetio.PacketIn - 4, // 9: lucius.dataplane.packetio.PacketIO.HostPortControl:output_type -> lucius.dataplane.packetio.HostPortControlMessage - 8, // 10: lucius.dataplane.packetio.PacketIO.CPUPacketStream:output_type -> lucius.dataplane.packetio.PacketOut - 9, // [9:11] is the sub-list for method output_type - 7, // [7:9] is the sub-list for method input_type - 7, // [7:7] is the sub-list for extension type_name - 7, // [7:7] is the sub-list for extension extendee - 0, // [0:7] is the sub-list for field type_name + 1, // 0: lucius.dataplane.packetio.HostPortControlRequest.init:type_name -> lucius.dataplane.packetio.HostPortControlInit + 10, // 1: lucius.dataplane.packetio.HostPortControlRequest.status:type_name -> google.rpc.Status + 3, // 2: lucius.dataplane.packetio.HostPortControlMessage.netdev:type_name -> lucius.dataplane.packetio.NetdevPort + 4, // 3: lucius.dataplane.packetio.HostPortControlMessage.genetlink:type_name -> lucius.dataplane.packetio.GenetlinkPort + 7, // 4: lucius.dataplane.packetio.PacketIn.init:type_name -> lucius.dataplane.packetio.PacketStreamInit + 6, // 5: lucius.dataplane.packetio.PacketIn.packet:type_name -> lucius.dataplane.packetio.Packet + 6, // 6: lucius.dataplane.packetio.PacketOut.packet:type_name -> lucius.dataplane.packetio.Packet + 2, // 7: lucius.dataplane.packetio.PacketIO.HostPortControl:input_type -> lucius.dataplane.packetio.HostPortControlRequest + 8, // 8: lucius.dataplane.packetio.PacketIO.CPUPacketStream:input_type -> lucius.dataplane.packetio.PacketIn + 5, // 9: lucius.dataplane.packetio.PacketIO.HostPortControl:output_type -> lucius.dataplane.packetio.HostPortControlMessage + 9, // 10: lucius.dataplane.packetio.PacketIO.CPUPacketStream:output_type -> lucius.dataplane.packetio.PacketOut + 9, // [9:11] is the sub-list for method output_type + 7, // [7:9] is the sub-list for method input_type + 7, // [7:7] is the sub-list for extension type_name + 7, // [7:7] is the sub-list for extension extendee + 0, // [0:7] is the sub-list for field type_name } func init() { file_dataplane_proto_packetio_packetio_proto_init() } @@ -850,13 +907,14 @@ func file_dataplane_proto_packetio_packetio_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_dataplane_proto_packetio_packetio_proto_rawDesc, - NumEnums: 0, + NumEnums: 1, NumMessages: 9, NumExtensions: 0, NumServices: 1, }, GoTypes: file_dataplane_proto_packetio_packetio_proto_goTypes, DependencyIndexes: file_dataplane_proto_packetio_packetio_proto_depIdxs, + EnumInfos: file_dataplane_proto_packetio_packetio_proto_enumTypes, MessageInfos: file_dataplane_proto_packetio_packetio_proto_msgTypes, }.Build() File_dataplane_proto_packetio_packetio_proto = out.File diff --git a/dataplane/proto/packetio/packetio.proto b/dataplane/proto/packetio/packetio.proto index 3af69fd5..e2c2470b 100644 --- a/dataplane/proto/packetio/packetio.proto +++ b/dataplane/proto/packetio/packetio.proto @@ -69,6 +69,12 @@ message PacketOut { Packet packet = 1; } +enum PortType { + PORT_TYPE_UNSPECIFIED = 0; + PORT_TYPE_NETDEV = 1; + PORT_TYPE_GENETLINK = 2; +} + service PacketIO { // HostPortControl requests creation and deletion of host ports. // Flow: diff --git a/dataplane/saiserver/hostif_test.go b/dataplane/saiserver/hostif_test.go index 866190a5..3e32f85e 100644 --- a/dataplane/saiserver/hostif_test.go +++ b/dataplane/saiserver/hostif_test.go @@ -83,7 +83,7 @@ func TestCreateHostif(t *testing.T) { dplane := &fakeSwitchDataplane{ ctx: fwdcontext.New("foo", "foo"), } - dplane.ctx.SetPacketSink(func(*fwdpb.PacketSinkResponse) error { return nil }) + dplane.ctx.SetCPUPortSink(func(po *pktiopb.PacketOut) error { return nil }, func() {}) c, mgr, stopFn := newTestHostif(t, dplane) // Create switch and ports mgr.StoreAttributes(mgr.NextID(), &saipb.SwitchAttribute{ diff --git a/dataplane/saiserver/ports.go b/dataplane/saiserver/ports.go index 9628df1c..66c18cea 100644 --- a/dataplane/saiserver/ports.go +++ b/dataplane/saiserver/ports.go @@ -61,7 +61,6 @@ var getInterface = net.InterfaceByName func getPreIngressPipeline() []*fwdpb.ActionDesc { return []*fwdpb.ActionDesc{ fwdconfig.Action(fwdconfig.LookupAction(tunTermTable)).Build(), // Decap the packet if we have a tunnel. - fwdconfig.Action(fwdconfig.LookupAction(VlanTable)).Build(), // Tag VLAN. fwdconfig.Action(fwdconfig.LookupAction(inputIfaceTable)).Build(), // Match packet to interface. fwdconfig.Action(fwdconfig.LookupAction(IngressVRFTable)).Build(), // Match interface to VRF. fwdconfig.Action(fwdconfig.LookupAction(PreIngressActionTable)).Build(), // Run pre-ingress actions. @@ -73,9 +72,9 @@ func getL3Pipeline() []*fwdpb.ActionDesc { return []*fwdpb.ActionDesc{ fwdconfig.Action(fwdconfig.LookupAction(IngressActionTable)).Build(), // Run ingress action. fwdconfig.Action(fwdconfig.DecapAction(fwdpb.PacketHeaderId_PACKET_HEADER_ID_ETHERNET)).Build(), // Decap L2 header. - fwdconfig.Action(fwdconfig.LookupAction(FIBSelectorTable)).Build(), // Lookup in FIB. // Do not forward packets with invalid fields. + fwdconfig.Action(fwdconfig.LookupAction(FIBSelectorTable)).Build(), // Lookup in FIB. fwdconfig.Action(fwdconfig.UpdateAction(fwdpb.UpdateType_UPDATE_TYPE_DEC, fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_HOP).WithValue([]byte{0x1})).Build(), // Decrement TTL. - fwdconfig.Action(fwdconfig.EncapAction(fwdpb.PacketHeaderId_PACKET_HEADER_ID_ETHERNET)).Build(), // Encap L2 header. // Drop invalid packets the FIB. + fwdconfig.Action(fwdconfig.EncapAction(fwdpb.PacketHeaderId_PACKET_HEADER_ID_ETHERNET)).Build(), // Encap L2 header. fwdconfig.Action(fwdconfig.LookupAction(outputIfaceTable)).Build(), // Match interface to port fwdconfig.Action(fwdconfig.LookupAction(NeighborTable)).Build(), // Lookup in the neighbor table. } @@ -90,9 +89,8 @@ func getL2Pipeline() []*fwdpb.ActionDesc { func getEgressPipeline() []*fwdpb.ActionDesc { return []*fwdpb.ActionDesc{ - fwdconfig.Action(fwdconfig.LookupAction(EgressActionTable)).Build(), // Run egress actions - fwdconfig.Action(fwdconfig.LookupAction(SRCMACTable)).Build(), // Lookup interface's MAC addr. - fwdconfig.Action(fwdconfig.DecapAction(fwdpb.PacketHeaderId_PACKET_HEADER_ID_ETHERNET_VLAN)).Build(), // TODO: Revise the code if trunk mode needs to be supported. + fwdconfig.Action(fwdconfig.LookupAction(EgressActionTable)).Build(), // Run egress actions + fwdconfig.Action(fwdconfig.LookupAction(SRCMACTable)).Build(), // Lookup interface's MAC addr. { ActionType: fwdpb.ActionType_ACTION_TYPE_OUTPUT, }, @@ -383,7 +381,6 @@ func (port *port) createCPUPort(ctx context.Context) (uint64, error) { }, Outputs: []*fwdpb.ActionDesc{ fwdconfig.Action(fwdconfig.LookupAction(trapIDToHostifTable)).Build(), // Check if the trap ID sets a hostif, otherwise use the default mapping of port -> hostif. - fwdconfig.Action(fwdconfig.LookupAction(portToHostifTable)).Build(), }, }, }, diff --git a/dataplane/saiserver/switch.go b/dataplane/saiserver/switch.go index be0bb3dd..8295e325 100644 --- a/dataplane/saiserver/switch.go +++ b/dataplane/saiserver/switch.go @@ -572,6 +572,7 @@ func (sw *saiSwitch) CreateSwitch(ctx context.Context, _ *saipb.CreateSwitchRequ ContextId: &fwdpb.ContextId{Id: sw.dataplane.ID()}, Desc: &fwdpb.TableDesc{ TableId: &fwdpb.TableId{ObjectId: &fwdpb.ObjectId{Id: trapIDToHostifTable}}, + Actions: []*fwdpb.ActionDesc{fwdconfig.Action(fwdconfig.LookupAction(portToHostifTable)).Build()}, TableType: fwdpb.TableType_TABLE_TYPE_EXACT, Table: &fwdpb.TableDesc_Exact{ Exact: &fwdpb.ExactTableDesc{ diff --git a/dataplane/server.go b/dataplane/server.go index f9c5434d..1f995ecd 100644 --- a/dataplane/server.go +++ b/dataplane/server.go @@ -27,6 +27,7 @@ import ( "google.golang.org/protobuf/proto" "github.com/openconfig/lemming/dataplane/dplaneopts" + _ "github.com/openconfig/lemming/dataplane/kernel/tap" "github.com/openconfig/lemming/dataplane/protocol" "github.com/openconfig/lemming/dataplane/saiserver" "github.com/openconfig/lemming/dataplane/saiserver/attrmgr" diff --git a/dataplane/standalone/pkthandler/BUILD b/dataplane/standalone/pkthandler/BUILD index 35318531..01cf9142 100644 --- a/dataplane/standalone/pkthandler/BUILD +++ b/dataplane/standalone/pkthandler/BUILD @@ -9,6 +9,8 @@ go_library( importpath = "github.com/openconfig/lemming/dataplane/standalone/pkthandler", visibility = ["//visibility:private"], deps = [ + "//dataplane/kernel/genetlink", + "//dataplane/kernel/tap", "//dataplane/proto/packetio", "//dataplane/standalone/pkthandler/pktiohandler", "@com_github_golang_glog//:glog", diff --git a/dataplane/standalone/pkthandler/main.go b/dataplane/standalone/pkthandler/main.go index a3c9b3dd..26a79f9d 100644 --- a/dataplane/standalone/pkthandler/main.go +++ b/dataplane/standalone/pkthandler/main.go @@ -28,6 +28,9 @@ import ( pktiopb "github.com/openconfig/lemming/dataplane/proto/packetio" "github.com/openconfig/lemming/dataplane/standalone/pkthandler/pktiohandler" + _ "github.com/openconfig/lemming/dataplane/kernel/genetlink" + _ "github.com/openconfig/lemming/dataplane/kernel/tap" + log "github.com/golang/glog" ) diff --git a/dataplane/standalone/pkthandler/pktiohandler/BUILD b/dataplane/standalone/pkthandler/pktiohandler/BUILD index 2844c391..e996bb67 100644 --- a/dataplane/standalone/pkthandler/pktiohandler/BUILD +++ b/dataplane/standalone/pkthandler/pktiohandler/BUILD @@ -10,6 +10,7 @@ go_library( "//dataplane/kernel", "//dataplane/proto/packetio", "@com_github_golang_glog//:glog", + "@com_github_vishvananda_netlink//:netlink", "@org_golang_google_genproto_googleapis_rpc//status", "@org_golang_google_grpc//codes", ], @@ -23,6 +24,7 @@ go_test( "//dataplane/kernel", "//dataplane/proto/packetio", "@com_github_google_go_cmp//cmp", + "@com_github_vishvananda_netlink//:netlink", "@org_golang_google_grpc//codes", "@org_golang_google_protobuf//testing/protocmp", ], diff --git a/dataplane/standalone/pkthandler/pktiohandler/pktiohandler.go b/dataplane/standalone/pkthandler/pktiohandler/pktiohandler.go index ff03d634..fad91a91 100644 --- a/dataplane/standalone/pkthandler/pktiohandler/pktiohandler.go +++ b/dataplane/standalone/pkthandler/pktiohandler/pktiohandler.go @@ -25,6 +25,8 @@ import ( "google.golang.org/genproto/googleapis/rpc/status" "google.golang.org/grpc/codes" + "github.com/vishvananda/netlink" + "github.com/openconfig/lemming/dataplane/forwarding/util/queue" "github.com/openconfig/lemming/dataplane/kernel" @@ -57,12 +59,12 @@ type PacketIOMgr struct { } type port struct { - portIO + PortIO cancelFn func() msg *pktiopb.HostPortControlMessage } -type portIO interface { +type PortIO interface { Delete() error Write([]byte, *kernel.PacketMetadata) (int, error) Read([]byte) (int, error) @@ -117,8 +119,8 @@ func (m *PacketIOMgr) StreamPackets(c pktiopb.PacketIO_CPUPacketStreamClient) er func (m *PacketIOMgr) metadataFromPacket(p *pktiopb.Packet) *kernel.PacketMetadata { md := &kernel.PacketMetadata{ - SrcIfIndex: m.dplanePortIfIndex[p.GetInputPort()], - DstIfIndex: m.dplanePortIfIndex[p.GetOutputPort()], + SrcIfIndex: int16(m.dplanePortIfIndex[p.GetInputPort()]), + DstIfIndex: int16(m.dplanePortIfIndex[p.GetOutputPort()]), } return md @@ -221,29 +223,34 @@ func (m *PacketIOMgr) ManagePorts(c pktiopb.PacketIO_HostPortControlClient) erro } } -var createTAPFunc = kernel.NewTap +var builder = map[pktiopb.PortType]func(*pktiopb.HostPortControlMessage) (PortIO, error){} + +func Register(t pktiopb.PortType, b func(*pktiopb.HostPortControlMessage) (PortIO, error)) { + builder[t] = b +} + +var linkByName = netlink.LinkByName func (m *PacketIOMgr) createPort(msg *pktiopb.HostPortControlMessage) error { - var p portIO + var p PortIO switch msg.GetPort().(type) { case *pktiopb.HostPortControlMessage_Genetlink: - portDesc := msg.GetGenetlink() var err error - p, err = kernel.NewGenetlinkPort(portDesc.Family, portDesc.Group) + p, err = builder[pktiopb.PortType_PORT_TYPE_GENETLINK](msg) if err != nil { return err } - log.Infof("add to new genetlink port: %v %v", portDesc.Family, portDesc.Group) case *pktiopb.HostPortControlMessage_Netdev: - name := msg.GetNetdev().GetName() var err error - kp, err := createTAPFunc(name) + p, err = builder[pktiopb.PortType_PORT_TYPE_NETDEV](msg) + if err != nil { + return err + } + l, err := linkByName(msg.GetNetdev().GetName()) if err != nil { return err } - p = kp - m.dplanePortIfIndex[msg.GetDataplanePort()] = kp.IfIndex() - log.Infof("add to new netdev port: %v", name) + m.dplanePortIfIndex[msg.GetDataplanePort()] = l.Attrs().Index default: return fmt.Errorf("unsupported port type: %v", msg.GetPort()) } @@ -251,7 +258,7 @@ func (m *PacketIOMgr) createPort(msg *pktiopb.HostPortControlMessage) error { doneCh := make(chan struct{}) m.hostifs[msg.GetPortId()] = &port{ - portIO: p, + PortIO: p, cancelFn: func() { close(doneCh) }, msg: msg, } diff --git a/dataplane/standalone/pkthandler/pktiohandler/pktiohandler_test.go b/dataplane/standalone/pkthandler/pktiohandler/pktiohandler_test.go index ebe2425d..c1edc8bd 100644 --- a/dataplane/standalone/pkthandler/pktiohandler/pktiohandler_test.go +++ b/dataplane/standalone/pkthandler/pktiohandler/pktiohandler_test.go @@ -21,6 +21,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + "github.com/vishvananda/netlink" "google.golang.org/grpc/codes" "google.golang.org/protobuf/testing/protocmp" @@ -75,7 +76,7 @@ func TestStreamPackets(t *testing.T) { } fp := &fakePort{} mgr.hostifs[1] = &port{ - portIO: fp, + PortIO: fp, cancelFn: func() {}, } ctx, cancelFn := context.WithCancel(context.Background()) @@ -101,6 +102,11 @@ func TestStreamPackets(t *testing.T) { } } +type fakeLink struct{} + +func (l *fakeLink) Attrs() *netlink.LinkAttrs { return &netlink.LinkAttrs{} } +func (l *fakeLink) Type() string { return "" } + func TestManagePorts(t *testing.T) { tests := []struct { desc string @@ -127,10 +133,12 @@ func TestManagePorts(t *testing.T) { if err != nil { t.Fatalf("unexpected error on New(): %v", err) } - createTAPFunc = func(string) (*kernel.TapInterface, error) { - return &kernel.TapInterface{}, nil + builder[pktiopb.PortType_PORT_TYPE_NETDEV] = func(hpcm *pktiopb.HostPortControlMessage) (PortIO, error) { + return nil, nil + } + linkByName = func(name string) (netlink.Link, error) { + return &fakeLink{}, nil } - hpc := &fakeHostPortControl{ msg: tt.msgs, } @@ -138,7 +146,7 @@ func TestManagePorts(t *testing.T) { t.Fatalf("ManagePorts() unexpected error: %v", err) } if got := codes.Code(hpc.gotReqs[1].GetStatus().GetCode()); got != tt.want { - t.Fatalf("ManagePorts() unexpected result: got %v, want %v", got, tt.want) + t.Fatalf("ManagePorts() unexpected result: got %v, want %v", hpc.gotReqs[1].GetStatus(), tt.want) } }) } @@ -150,7 +158,7 @@ type portWriteData struct { } type fakePort struct { - portIO + PortIO writtenData []*portWriteData }