Support custom port naming based on config (#354)

* wip port naming

* fix

* support port map

* lint

* test

* test fix

* docs

* feedback

Dockerfile.saibuilder

@@ -0,0 +1,18 @@
+FROM us-west1-docker.pkg.dev/openconfig-lemming/internal/builder@sha256:6a960d06bfd63c9cd8cff1f2ab3b3cc21e578b570979b5574d232be644dd0bf9
+WORKDIR /build
+COPY patches/sai.patch sai.patch
+RUN wget -q https://github.com/opencomputeproject/SAI/archive/refs/tags/v1.11.0.tar.gz && tar xf v1.11.0.tar.gz && rm v1.11.0.tar.gz 
+RUN mkdir external && mv SAI-1.11.0 external/com_github_opencomputeproject_sai && patch -p1 -d external/com_github_opencomputeproject_sai < sai.patch 
+COPY go.* .
+COPY dataplane/proto/sai/*.proto dataplane/proto/sai/
+COPY dataplane/cpusink/*.go dataplane/cpusink/
+COPY dataplane/standalone/packetio/*.go dataplane/standalone/packetio/
+COPY dataplane/dplaneopts/*.go dataplane/dplaneopts/
+COPY dataplane/forwarding/ dataplane/forwarding/
+COPY dataplane/internal/kernel/*.go dataplane/internal/kernel/
+COPY proto/forwarding/*.go proto/forwarding/
+COPY dataplane/standalone/sai/*.cc dataplane/standalone/sai/
+COPY dataplane/standalone/sai/*.h dataplane/standalone/sai/
+COPY dataplane/standalone/entrypoint.cc dataplane/standalone/entrypoint.cc
+COPY Makefile .
+RUN make lucius-libsai -j 48

Makefile

@@ -48,4 +48,61 @@ test-race:
 	# Failure in local_tests are due to GoBGP itself unable to issue a Stop
 	# command without conflicting with the running server in another
 	# thread.(e.g. TestRoutePropagation)
-	bazel test --@io_bazel_rules_go//go/config:race --test_output=errors $(shell bazel query 'tests("//...") except "//integration_tests/..." except "//dataplane/..." except "//gnmi/..." except "//bgp/tests/local_tests/..."')
+	bazel test --@io_bazel_rules_go//go/config:race --test_output=errors $(shell bazel query 'tests("//...") except "//integration_tests/..." except "//dataplane/..." except "//gnmi/..." except "//bgp/tests/local_tests/..."')
+
+PROTOS = $(wildcard dataplane/proto/sai/*.proto)
+PROTO_SRC = $(patsubst dataplane/proto/sai/%.proto, dataplane/proto/sai/%.pb.cc, $(PROTOS))
+PROTO_GRPC_SRC = $(patsubst dataplane/proto/sai/%.proto, dataplane/proto/sai/%.grpc.pb.cc, $(PROTOS))
+PROTO_OBJ = $(patsubst dataplane/proto/sai/%.proto, dataplane/proto/sai/%.pb.o, $(PROTOS))
+GRPC_OBJ = $(patsubst dataplane/proto/sai/%.proto, dataplane/proto/sai/%.grpc.pb.o, $(PROTOS))
+SAI_SRC = $(wildcard dataplane/standalone/sai/*.cc)
+SAI_OBJ = $(patsubst dataplane/standalone/sai/%.cc, dataplane/standalone/sai/%.o, $(SAI_SRC))
+
+.PHONY: sai-clean
+sai-clean:
+	rm dataplane/proto/sai/*.cc dataplane/proto/sai/*.h dataplane/proto/sai/*.o
+	rm dataplane/standalone/sai/*.o
+	rm -rf dataplane/standalone/packetio/packetio.a dataplane/standalone/packetio/packetio.h libsai.so pkg
+
+$(PROTO_SRC) $(PROTO_GRPC_SRC) &:
+	protoc dataplane/proto/sai/*.proto --cpp_out=. --grpc_out=. --plugin=protoc-gen-grpc=`which grpc_cpp_plugin` --experimental_allow_proto3_optional
+
+dataplane/proto/sai/%.pb.o: dataplane/proto/sai/%.pb.cc
+	g++ -fPIC -c $< -I . -o $@
+
+dataplane/proto/sai/%.grpc.pb.o:  dataplane/proto/sai/%.grpc.pb.cc
+	g++ -fPIC -c $< -I . -o $@
+
+dataplane/standalone/sai/%.o: dataplane/standalone/sai/%.cc $(PROTO_SRC)
+	g++ -fPIC -c $< -o $@ -I . -I external/com_github_opencomputeproject_sai -I external/com_github_opencomputeproject_sai/inc  -I external/com_github_opencomputeproject_sai/experimental
+
+packetio:
+	go build -o dataplane/standalone/packetio/packetio.a -buildmode=c-archive ./dataplane/standalone/packetio 
+
+libsai.so: $(PROTO_OBJ) $(GRPC_OBJ) $(SAI_OBJ) packetio
+	g++ -fPIC -o libsai.so -shared dataplane/standalone/entrypoint.cc dataplane/proto/sai/*.o dataplane/standalone/sai/*.o dataplane/standalone/packetio/packetio.a -lglog -lprotobuf -lgrpc++ -I . -I external/com_github_opencomputeproject_sai -I external/com_github_opencomputeproject_sai/inc -I external/com_github_opencomputeproject_sai/experimental
+
+define DEB_CONTROL =
+Package: lucius-libsai
+Version: 1.0-3
+Maintainer: OpenConfig
+Architecture: amd64
+Description: SAI implementation for lucius
+endef
+export DEB_CONTROL
+
+lucius-libsai: libsai.so 
+	rm -rf pkg/
+	mkdir -p pkg/lucius-libsai/DEBIAN
+	chmod 0755 pkg/lucius-libsai/DEBIAN
+	mkdir -p pkg/lucius-libsai/usr/lib/x86_64-linux-gnu
+	cp libsai.so pkg/lucius-libsai/usr/lib/x86_64-linux-gnu/libsaivs.so.0.0.0
+	cd pkg/lucius-libsai/usr/lib/x86_64-linux-gnu && ln -s libsaivs.so.0.0.0 libsaivs.so.0
+	echo "$$DEB_CONTROL" > pkg/lucius-libsai/DEBIAN/control
+	dpkg-deb --build pkg/lucius-libsai
+
+lucius-libsai-bullseye:
+	DOCKER_BUILDKIT=1 docker build . -f Dockerfile.saibuilder -t lemming-libsai:latest
+	docker create --name libsai-temp lemming-libsai:latest
+	docker cp libsai-temp:/build/pkg/lucius-libsai.deb .
+	docker rm libsai-temp

dataplane/apigen/ccgen/ccgen.go

@@ -153,22 +153,6 @@ func createCCData(meta *saiast.FuncMetadata, apiName string, sai *saiast.SAIAPI,
 				fmt.Println("skipping due to error: ", err)
 				continue
 			}
-			// When the sai client is running in different namespace than the server,
-			// need to set the name in client namespace, not the server.
-			// TODO: Decide if this needs to ebe supported long term.
-			if meta.TypeName == "HOSTIF" && attr.EnumName == "SAI_HOSTIF_ATTR_NAME" {
-				smt.CustomText = `{
-int idx;
-int count = sscanf( attr_list[i].value.chardata, "Ethernet%d", &idx);
-if (count == 1) {
-  std::ostringstream s;
-  s << "ip link set eth" << idx/4+1 << " name "
-	<< attr_list[i].value.chardata;
-  LOG(INFO) << s.str();
-  system(s.str().c_str());
-}
-}`
-			}
 			smt.EnumValue = attr.EnumName
 			convertFn.AttrSwitch.Attrs = append(convertFn.AttrSwitch.Attrs, smt)
 		}

dataplane/cpusink/BUILD

@@ -6,11 +6,13 @@ go_library(
     importpath = "github.com/openconfig/lemming/dataplane/cpusink",
     visibility = ["//visibility:public"],
     deps = [
+        "//dataplane/dplaneopts",
         "//dataplane/forwarding/attributes",
         "//dataplane/forwarding/fwdconfig",
         "//dataplane/internal/kernel",
         "//proto/forwarding",
         "@com_github_golang_glog//:glog",
         "@com_github_vishvananda_netlink//:netlink",
+        "@org_golang_x_exp//maps",
     ],
 )

dataplane/cpusink/cpusink.go

@@ -18,9 +18,16 @@ package cpusink
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"slices"
+	"strings"
 
 	"github.com/vishvananda/netlink"
+	"golang.org/x/exp/maps"
 
+	"github.com/openconfig/lemming/dataplane/dplaneopts"
 	"github.com/openconfig/lemming/dataplane/forwarding/attributes"
 	"github.com/openconfig/lemming/dataplane/forwarding/fwdconfig"
 	"github.com/openconfig/lemming/dataplane/internal/kernel"
@@ -31,23 +38,48 @@ import (
 )
 
 const (
-	contextID  = "lucius"
-	IP2MeTable = "ip2me"
+	contextID      = "lucius"
+	IP2MeTable     = "ip2me"
+	configPathPath = "/etc/sonic/config_db.json"
 )
 
 // Sink is a CPU port client for a forwarding context.
 type Sink struct {
 	client          fwdpb.ForwardingClient
 	ethDevToPort    map[string]string
 	ethDevToPortNID map[string]uint64
+	// nameToEth maps the modeled name (eg Ethernet8) to the Linux device name (eg eth1).
+	nameToEth map[string]string
 }
 
-func New(client fwdpb.ForwardingClient) *Sink {
+// safeDeviceName returns valid name for a network device from the input name.
+func safeDeviceName(name string) string {
+	return strings.ReplaceAll(name, "/", "_")
+}
+
+func New(client fwdpb.ForwardingClient) (*Sink, error) {
+	data, err := os.ReadFile(configPathPath)
+	if err != nil {
+		return nil, err
+	}
+	config := &dplaneopts.PortConfig{}
+	if err := json.Unmarshal(data, config); err != nil {
+		return nil, err
+	}
+	ports := maps.Keys(config.Ports)
+	slices.Sort(ports)
+	nameToEth := make(map[string]string)
+	for i, port := range ports {
+		log.Infof("port map %v to %v", port, fmt.Sprintf("eth%d", i+1))
+		nameToEth[safeDeviceName(port)] = fmt.Sprintf("eth%d", i+1)
+	}
+
 	return &Sink{
 		client:          client,
 		ethDevToPort:    make(map[string]string),
 		ethDevToPortNID: make(map[string]uint64),
-	}
+		nameToEth:       nameToEth,
+	}, nil
 }
 
 // ReceivePackets from packets from the CPU port and sends them to the correct ports.
@@ -80,6 +112,15 @@ func (sink *Sink) ReceivePackets(ctx context.Context) error {
 				if name == "" {
 					name = desc.GetTap().GetDeviceName()
 				}
+				l, err := netlink.LinkByName(sink.nameToEth[name])
+				if err != nil {
+					log.Errorf("failed to get link name %v, eth %v: %v", name, sink.nameToEth[name], err)
+					continue
+				}
+				if err := netlink.LinkSetName(l, safeDeviceName(name)); err != nil {
+					log.Errorf("failed to set link name: %v", err)
+					continue
+				}
 				// Get the port ID for this hostif.
 				attr, err := sink.client.AttributeQuery(ctx, &fwdpb.AttributeQueryRequest{
 					ContextId: &fwdpb.ContextId{Id: contextID},

dataplane/docs/libsaibuild.md

@@ -2,15 +2,19 @@
 
 ## Bazel
 
-Bazel is the preferred way to build Lemming. There are two bazel targets: the first statically links the library, the other dynamically links glog, protobuf, gRPC libraries.
-The second is used to distribute a deb package for use in syncd. In order to ensure shared library version compatibility, bazel executes the build in a seperate docker container.
-The image is not released publicly, but can be built using the [Dockerfile](../standalone/Dockerfile).
+Bazel is the preferred way to build Lemming. However, to ensure library version compatibility, it may be required to build libsai.so using Make.
+There are two bazel targets: the first statically links the library, the other dynamically links glog, protobuf, gRPC libraries.
+In order to ensure shared library version compatibility, bazel executes the build in a seperate docker container. The image is not released publicly,
+but can be built using the [Dockerfile](../standalone/Dockerfile). P
 
-1. `bazel build //dataplane/standalone:sai-deb --config docker-bullseye`
+1. Option 1: `make lucius-libsai-bullseye`
+   1. This version compiles the protobufs using the libprotobuf from debian bullseye release.
+2. Option 2: `bazel build //dataplane/standalone:sai-deb --config docker-bullseye`
    1. The `--config docker-bullseye` expands all the flags labeled `docker-bullseye` in `.bazelrc`
    2. These flags enable bazel docker sandbox and set up the toolchains for building the target in a custom docker container.
    3. The output of the command is a debian package built using a container based on `debian:bullseye`.
-2. Built debian package is located at `bazel-bin/dataplane/standalone`
+
+3. Built debian package is located at `bazel-bin/dataplane/standalone`
 
 ## RBE configs
 

dataplane/dplaneopts/dplaneopts.go

@@ -26,6 +26,10 @@ type Options struct {
 	HostifNetDevType fwdpb.PortType
 	// PortType is the fwdpb type for the port type.
 	PortType fwdpb.PortType
+	// PortConfigFile is the path of the port config.
+	PortConfigFile string
+	// PortMap maps the modeled port name (Ethernet1/1/1) to Linux port name (eth1).
+	PortMap map[string]string
 }
 
 // Option exposes additional configuration for the dataplane.
@@ -63,13 +67,40 @@ func WithPortType(t fwdpb.PortType) Option {
 	}
 }
 
+// WithPortConfigFile sets the path of the port config file.
+// Default: none
+func WithPortConfigFile(file string) Option {
+	return func(o *Options) {
+		o.PortConfigFile = file
+	}
+}
+
+// WithPortMap configure a map from port name to Linux network device to allow flexible port naming. (eg Ethernet8 -> eth1)
+// Default: none
+func WithPortMap(m map[string]string) Option {
+	return func(o *Options) {
+		o.PortMap = m
+	}
+}
+
+// Port contains configuration data for a single port.
+type Port struct {
+	Lanes string `json:"lanes"`
+}
+
+// PortConfig contains configuration data for the dataplane ports.
+type PortConfig struct {
+	Ports map[string]*Port `json:"PORT"`
+}
+
 // ResolveOpts creates an option struct from the opts.
 func ResolveOpts(opts ...Option) *Options {
 	resolved := &Options{
 		AddrPort:         "127.0.0.1:0",
 		Reconcilation:    true,
 		HostifNetDevType: fwdpb.PortType_PORT_TYPE_TAP,
 		PortType:         fwdpb.PortType_PORT_TYPE_KERNEL,
+		PortMap:          map[string]string{},
 	}
 
 	for _, opt := range opts {

dataplane/saiserver/ports.go

@@ -16,8 +16,11 @@ package saiserver
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net"
+	"os"
+	"strings"
 
 	"google.golang.org/grpc"
 	"google.golang.org/protobuf/proto"
@@ -33,16 +36,27 @@ import (
 	fwdpb "github.com/openconfig/lemming/proto/forwarding"
 )
 
-func newPort(mgr *attrmgr.AttrMgr, dataplane switchDataplaneAPI, s *grpc.Server, opts *dplaneopts.Options) *port {
+func newPort(mgr *attrmgr.AttrMgr, dataplane switchDataplaneAPI, s *grpc.Server, opts *dplaneopts.Options) (*port, error) {
 	p := &port{
 		mgr:       mgr,
 		dataplane: dataplane,
 		portToEth: make(map[uint64]string),
 		nextEth:   1, // Start at eth1
 		opts:      opts,
 	}
+	if opts.PortConfigFile != "" {
+		data, err := os.ReadFile(opts.PortConfigFile)
+		if err != nil {
+			return nil, err
+		}
+		p.config = &dplaneopts.PortConfig{}
+		if err := json.Unmarshal(data, p.config); err != nil {
+			return nil, err
+		}
+	}
+
 	saipb.RegisterPortServer(s, p)
-	return p
+	return p, nil
 }
 
 type port struct {
@@ -52,19 +66,51 @@ type port struct {
 	nextEth   int
 	portToEth map[uint64]string
 	opts      *dplaneopts.Options
+	config    *dplaneopts.PortConfig
 }
 
 // stub for testing
 var getInterface = net.InterfaceByName
 
 // CreatePort creates a new port, mapping the port to ethX, where X is assigned sequentially from 1 to n.
 // Note: If more ports are created than eth devices, no error is returned, but the OperStatus is set to NOT_PRESENT.
-func (port *port) CreatePort(ctx context.Context, _ *saipb.CreatePortRequest) (*saipb.CreatePortResponse, error) {
+func (port *port) CreatePort(ctx context.Context, req *saipb.CreatePortRequest) (*saipb.CreatePortResponse, error) {
 	id := port.mgr.NextID()
 
+	// By default, create port sequentially starting at eth1.
 	dev := fmt.Sprintf("eth%v", port.nextEth)
 	port.nextEth++
 
+	// If a port config is set, then use the hardware lanes to find the interface names.
+	if port.config != nil {
+		if len(req.HwLaneList) == 0 {
+			return nil, fmt.Errorf("port lanes are required got %v", req.HwLaneList)
+		}
+		var b strings.Builder
+		b.WriteString(fmt.Sprint(req.HwLaneList[0]))
+		for _, l := range req.HwLaneList[1:] {
+			b.WriteString(",")
+			b.WriteString(fmt.Sprint(l))
+		}
+		dev = ""
+		lanes := b.String()
+		for n, cfg := range port.config.Ports {
+			if cfg.Lanes == lanes {
+				dev = n
+			}
+		}
+		if dev == "" {
+			return nil, fmt.Errorf("could not find lanes %v in config", lanes)
+		}
+		if port.opts.PortMap != nil && len(port.opts.PortMap) != 0 {
+			if portMapPort := port.opts.PortMap[dev]; portMapPort == "" {
+				log.Warningf("port named %q doesn't exist in the port map", dev)
+			} else {
+				dev = portMapPort
+			}
+		}
+	}
+
 	attrs := &saipb.PortAttribute{
 		QosNumberOfQueues:                proto.Uint32(0),
 		QosQueueList:                     []uint64{},

dataplane/saiserver/saiserver.go

@@ -263,6 +263,10 @@ func New(ctx context.Context, mgr *attrmgr.AttrMgr, s *grpc.Server, opts *dplane
 	if err != nil {
 		return nil, err
 	}
+	sw, err := newSwitch(mgr, fwdCtx, s, opts)
+	if err != nil {
+		return nil, err
+	}
 
 	srv := &Server{
 		mgr:               mgr,
@@ -293,7 +297,7 @@ func New(ctx context.Context, mgr *attrmgr.AttrMgr, s *grpc.Server, opts *dplane
 		schedulerGroup:    &schedulerGroup{},
 		scheduler:         &scheduler{},
 		srv6:              &srv6{},
-		saiSwitch:         newSwitch(mgr, fwdCtx, s, opts),
+		saiSwitch:         sw,
 		systemPort:        &systemPort{},
 		tam:               &tam{},
 		tunnel:            &tunnel{},