From bba830c763c3935a71c10dcd2bcdf0d91e652ea4 Mon Sep 17 00:00:00 2001 From: Serge Hallyn Date: Mon, 16 Oct 2023 11:42:01 -0500 Subject: [PATCH] implement mos boot and support simple service network Closes #4 Closes #7 Implement 'mos boot' which activates all services. Implement a 'simple' network type for services, which isolates a service in a netns with a simple veth on the lxcbr0, with specified host ports forwarded into the container. layers/install: add a zot service layer which uses the simple network. update to v0.0.17 bootkit for 4M firmware kvm provider: specify bootindex 'off' for nics add a hidden subcommand to verify an install.yaml Signed-off-by: Serge Hallyn --- .github/workflows/build.yml | 2 +- Makefile | 4 +- cmd/mosb/main.go | 30 ++++ cmd/mosctl/boot.go | 28 ++++ cmd/mosctl/main.go | 1 + go.mod | 2 +- go.sum | 4 +- layers/install/load-mos-modules | 16 ++ layers/install/mos-boot-setup.service | 18 +++ layers/install/mos-modules.service | 12 ++ layers/install/stacker.yaml | 34 +++++ layers/install/start-zot | 12 ++ layers/install/zot-config.json | 14 ++ pkg/mosconfig/files.go | 24 ++- pkg/mosconfig/manifest.go | 2 + pkg/mosconfig/mos.go | 57 ++++++-- pkg/mosconfig/network.go | 203 +++++++++++++++++++++++++- pkg/mosconfig/uidmap.go | 4 + pkg/provider/kvm.go | 35 ++++- pkg/trust/artifacts.go | 4 +- tests/helpers.bash | 3 +- tests/launch.bats | 2 +- tests/livecd1.bats | 2 +- tests/livecd2.bats | 2 +- 24 files changed, 481 insertions(+), 34 deletions(-) create mode 100644 cmd/mosctl/boot.go create mode 100644 layers/install/load-mos-modules create mode 100644 layers/install/mos-boot-setup.service create mode 100644 layers/install/mos-modules.service create mode 100644 layers/install/start-zot create mode 100644 layers/install/zot-config.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b7e431a..bc6e830 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -131,7 +131,7 @@ jobs: file: 'layers/stacker.yaml' build-args: | ZOT_VERSION=2.0.0-rc5 - ROOTFS_VERSION=v0.0.15.230901 + ROOTFS_VERSION=v0.0.17.231018 url: docker://zothub.io/machine/bootstrap tags: ${{ github.event.release.tag_name }} username: ${{ secrets.ZOTHUB_USERNAME }} diff --git a/Makefile b/Makefile index cdc6afb..0a58bde 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ ORAS_VERSION := 1.0.0-rc.1 REGCTL := $(TOOLSDIR)/bin/regctl REGCTL_VERSION := 0.5.0 TOPDIR := $(shell git rev-parse --show-toplevel) -BOOTKIT_VERSION ?= "v0.0.15.230901" +BOOTKIT_VERSION ?= "v0.0.17.231018" ROOTFS_VERSION = $(BOOTKIT_VERSION) archout = $(shell arch) @@ -33,7 +33,7 @@ all: mosctl mosb trust $(ZOT) $(ORAS) $(REGCTL) VERSION_LDFLAGS=-X github.com/project-machine/mos/pkg/mosconfig.Version=$(MAIN_VERSION) \ -X github.com/project-machine/mos/pkg/trust.Version=$(MAIN_VERSION) \ - -X github.com/project-machine/mos/pkg/mosconfig.LayerVersion=0.0.3 \ + -X github.com/project-machine/mos/pkg/mosconfig.LayerVersion=0.0.4 \ -X github.com/project-machine/mos/pkg/trust.BootkitVersion=$(BOOTKIT_VERSION) mosctl: .made-gofmt $(GO_SRC) diff --git a/cmd/mosb/main.go b/cmd/mosb/main.go index a776e1a..04f82c7 100644 --- a/cmd/mosb/main.go +++ b/cmd/mosb/main.go @@ -1,11 +1,13 @@ package main import ( + "fmt" "os" "github.com/apex/log" "github.com/project-machine/mos/pkg/mosconfig" "github.com/urfave/cli" + "gopkg.in/yaml.v2" ) func main() { @@ -16,6 +18,7 @@ func main() { manifestCmd, mkBootCmd, mkProvisionCmd, + readSpec, } app.Flags = []cli.Flag{ cli.BoolFlag{ @@ -35,3 +38,30 @@ func main() { log.Fatalf("%v\n", err) } } + +var readSpec = cli.Command{ + Name: "readspec", + Usage: "read a manifest.yaml and print out resulting struct", + Action: doReadSpec, + Hidden: true, + UsageText: `in-file + in-file: file to read`, +} + +func doReadSpec(ctx *cli.Context) error { + args := ctx.Args() + if len(args) < 1 { + return fmt.Errorf("input file is a required positional argument") + } + + bytes, err := os.ReadFile(args[0]) + if err != nil { + return err + } + var manifest mosconfig.ImportFile + if err = yaml.Unmarshal(bytes, &manifest); err != nil { + return err + } + fmt.Printf("result: %#v", manifest) + return nil +} diff --git a/cmd/mosctl/boot.go b/cmd/mosctl/boot.go new file mode 100644 index 0000000..0190dfd --- /dev/null +++ b/cmd/mosctl/boot.go @@ -0,0 +1,28 @@ +package main + +import ( + "github.com/pkg/errors" + "github.com/project-machine/mos/pkg/mosconfig" + "github.com/urfave/cli" +) + +var bootCmd = cli.Command{ + Name: "boot", + Usage: "start all services listed in mos manifest", + Action: doBootCmd, +} + +func doBootCmd(ctx *cli.Context) error { + opts := mosconfig.DefaultMosOptions() + mos, err := mosconfig.OpenMos(opts) + if err != nil { + return errors.Wrapf(err, "Failed opening mos") + } + + err = mos.Boot() + if err != nil { + return errors.Wrapf(err, "Failed to boot") + } + + return nil +} diff --git a/cmd/mosctl/main.go b/cmd/mosctl/main.go index 206abb0..7bf157f 100644 --- a/cmd/mosctl/main.go +++ b/cmd/mosctl/main.go @@ -17,6 +17,7 @@ func main() { app.Commands = []cli.Command{ createBootFsCmd, activateCmd, + bootCmd, installCmd, mountCmd, updateCmd, diff --git a/go.mod b/go.mod index e1ab56e..c7aa1a0 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/opencontainers/umoci v0.4.8-0.20220412065115-12453f247749 github.com/pkg/errors v0.9.1 github.com/plus3it/gorecurcopy v0.0.1 - github.com/project-machine/bootkit v0.0.15 + github.com/project-machine/bootkit v0.0.0-20230906152517-964838ab8d93 github.com/project-machine/machine v0.1.2 github.com/project-stacker/stacker v0.21.2 github.com/rekby/gpt v0.0.0-20200219180433-a930afbc6edc diff --git a/go.sum b/go.sum index 72f199c..f494e2b 100644 --- a/go.sum +++ b/go.sum @@ -2691,8 +2691,8 @@ github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQ github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/proglottis/gpgme v0.1.3 h1:Crxx0oz4LKB3QXc5Ea0J19K/3ICfy3ftr5exgUK1AU0= github.com/proglottis/gpgme v0.1.3/go.mod h1:fPbW/EZ0LvwQtH8Hy7eixhp1eF3G39dtx7GUN+0Gmy0= -github.com/project-machine/bootkit v0.0.15 h1:EAeilYMiBnsliLc+3BejqIwN8seir2fQ8aL8Lg93mkY= -github.com/project-machine/bootkit v0.0.15/go.mod h1:02BdQ6/ClxWdwPOIVYxpRNJIJD2eFcg8iz2Q1phJchU= +github.com/project-machine/bootkit v0.0.0-20230906152517-964838ab8d93 h1:OfYRKrxfjxyYUOUNCY8bpNaEhE6xk4QcZb+1rRfZmaw= +github.com/project-machine/bootkit v0.0.0-20230906152517-964838ab8d93/go.mod h1:02BdQ6/ClxWdwPOIVYxpRNJIJD2eFcg8iz2Q1phJchU= github.com/project-machine/machine v0.1.2 h1:/detDExvftlN+PvJWP57tUS6NFLPtHWc+m3b4/NhFq4= github.com/project-machine/machine v0.1.2/go.mod h1:pjru0EkLoBhdLQ1szLxJIqiMkUgezdwMjsaq/ijBZOw= github.com/project-machine/qcli v0.2.1 h1:rIRItjdkeBbD4NIxYyTkxCeJIolGHdniJ51Phfg2Ols= diff --git a/layers/install/load-mos-modules b/layers/install/load-mos-modules new file mode 100644 index 0000000..64491b0 --- /dev/null +++ b/layers/install/load-mos-modules @@ -0,0 +1,16 @@ +#!/bin/bash + +mountpoint /lib/modules && { echo "already mounted"; exit 0; } + +mkdir -p /bootkit +mosctl --debug mount --target=bootkit --dest=/bootkit +mkdir -p /lib/modules +mount /bootkit/bootkit/modules.squashfs /lib/modules/ + +udevadm trigger +modprobe virtio-net +modprobe br_netfilter +modprobe iptables_nat +modprobe iptables_mangle +dhclient +systemctl start lxc-net diff --git a/layers/install/mos-boot-setup.service b/layers/install/mos-boot-setup.service new file mode 100644 index 0000000..b04da00 --- /dev/null +++ b/layers/install/mos-boot-setup.service @@ -0,0 +1,18 @@ +[Unit] +Description=mos-boot-setup +After=local-fs.target +After=systemd-journal-flush.service logs.mount +Requires=local-fs.target +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/bin/mosctl boot +StandardOutput=journal+console +StandardError=journal+console + +# Journal namespaces implementation also affects the mount namespaces. +# Assigning a separate journal namespace to Atomix process hides mount points +# like /config and /tmp from the "main" user namespace. +# LogNamespace=atomix +[Install] +WantedBy=multi-user.target diff --git a/layers/install/mos-modules.service b/layers/install/mos-modules.service new file mode 100644 index 0000000..4a4a815 --- /dev/null +++ b/layers/install/mos-modules.service @@ -0,0 +1,12 @@ +[Unit] +Description=Load mos modules +After=local-fs.target + +[Service] +Type=oneshot +RemainAfterExit=true +ExecStart=/usr/bin/load-mos-modules +StandardOutput=journal+console + +[Install] +WantedBy=multi-user.target diff --git a/layers/install/stacker.yaml b/layers/install/stacker.yaml index a9f68e4..0e49333 100644 --- a/layers/install/stacker.yaml +++ b/layers/install/stacker.yaml @@ -15,6 +15,7 @@ install-rootfs-pkg: cryptsetup \ dosfstools \ e2fsprogs \ + efibootmgr \ iproute2 \ isc-dhcp-client \ keyutils \ @@ -37,6 +38,25 @@ install-rootfs-pkg: DHCP=yes END +demo-zot: + from: + type: built + tag: install-rootfs-pkg + import: + - zot-config.json + - start-zot + - https://github.com/project-zot/zot/releases/download/v${{ZOT_VERSION}}/zot-linux-amd64-minimal + entrypoint: /usr/bin/start-zot + run: | + #!/bin/sh -ex + cp /stacker/imports/zot-config.json /etc/ + + cp /stacker/imports/start-zot /usr/bin/start-zot + chmod 755 /usr/bin/start-zot + cp /stacker/imports/zot-linux-amd64-minimal /usr/bin/zot + chmod 755 /usr/bin/zot + + # The rootfs which we want to run on the system # Note this is for demo purposes only. No one should ever # use this as the target layer. @@ -47,6 +67,9 @@ demo-target-rootfs: import: - ../../mosctl - ../provision/console-helper + - load-mos-modules + - mos-modules.service + - mos-boot-setup.service run: | #!/bin/sh -ex writefile() { @@ -68,10 +91,21 @@ demo-target-rootfs: DHCP=yes END + # lxc needed for mosctl to activate a service \\ + # git needed for mosctl to read manifest + pkgtool install git lxc + cd /stacker/imports cp mosctl console-helper /usr/bin ( cd /usr/bin && chmod 755 mosctl console-helper ) + cp /stacker/imports/load-mos-modules /usr/bin/ + chmod 755 /usr/bin/load-mos-modules + cp /stacker/imports/mos-modules.service /etc/systemd/system/ + systemctl enable mos-modules.service + cp /stacker/imports/mos-boot-setup.service /etc/systemd/system + systemctl enable mos-boot-setup.service + echo root:passw0rd | chpasswd systemctl enable serial-getty@ttyS0 annotations: diff --git a/layers/install/start-zot b/layers/install/start-zot new file mode 100644 index 0000000..6cb8062 --- /dev/null +++ b/layers/install/start-zot @@ -0,0 +1,12 @@ +#!/bin/bash + +if [ -n "$IPV4" ]; then + sed -i "s/0.0.0.0/${IPV4%/*}/" /etc/zot-config.json +elif [ -n "$IPV6" ]; then + sed -i "s/0.0.0.0/${IPV6%/*}/" /etc/zot-config.json +fi + +# Should mos or lxc be doing this for us? +ip route add default via 10.0.3.1 + +exec /usr/bin/zot serve /etc/zot-config.json diff --git a/layers/install/zot-config.json b/layers/install/zot-config.json new file mode 100644 index 0000000..136e43a --- /dev/null +++ b/layers/install/zot-config.json @@ -0,0 +1,14 @@ +{ + "distSpecVersion": "1.0.1-dev", + "storage": { + "rootDirectory": "/zot", + "gc": false + }, + "http": { + "address": "0.0.0.0", + "port": "5000" + }, + "log": { + "level": "error" + } +} diff --git a/pkg/mosconfig/files.go b/pkg/mosconfig/files.go index 6dfc1d0..24ce049 100644 --- a/pkg/mosconfig/files.go +++ b/pkg/mosconfig/files.go @@ -48,12 +48,24 @@ const CurrentInstallFileVersion = 1 type TargetNetworkType string const ( - HostNetwork TargetNetworkType = "host" - NoNetwork TargetNetworkType = "none" + HostNetwork TargetNetworkType = "host" + NoNetwork TargetNetworkType = "none" + SimpleNetwork TargetNetworkType = "simple" ) +// Target Network configuration +// Type dictates which type +// Address is an ipv4 or ipv6 address. +// Ports are ipfilter rules to allow inbound masq. +// +// We will likely want to change this to an array of +// nics, like lxc.net.[i].*. But for now let's just +// support one type TargetNetwork struct { - Type TargetNetworkType `json:"type"` + Type TargetNetworkType `json:"type" yaml:"type"` + Address string `json:"ipv4" yaml:"ipv4"` + Address6 string `json:"ipv6" yaml:"ipv6"` + Ports []SimplePort `json:"ports" yaml:"ports"` } // Service type defines how a service is run. @@ -126,8 +138,14 @@ func (s *SysTargets) Contains(needle SysTarget) (SysTarget, bool) { } type SysManifest struct { + // Persistent stored information UidMaps []IdmapSet `json:"uidmaps"` SysTargets []SysTarget `json:"targets"` + + // Runtime information + DefaultNic string + UsedPorts map[uint]string // map of hostport -> running target name + IpAddrs map[string]string // map of ip4 ip6 addr -> running target name } func (sm *SysManifest) GetTarget(target string) (*SysTarget, error) { diff --git a/pkg/mosconfig/manifest.go b/pkg/mosconfig/manifest.go index 3d7c11f..4978e35 100644 --- a/pkg/mosconfig/manifest.go +++ b/pkg/mosconfig/manifest.go @@ -101,6 +101,8 @@ func (mos *Mos) initManifest(manifestPath, manifestCert, manifestCA, configPath sysmanifest := SysManifest{ UidMaps: uidmaps, SysTargets: targets, + UsedPorts: make(map[uint]string), + IpAddrs: make(map[string]string), } bytes, err := json.Marshal(&sysmanifest) diff --git a/pkg/mosconfig/mos.go b/pkg/mosconfig/mos.go index 1c81f70..f5a30ba 100644 --- a/pkg/mosconfig/mos.go +++ b/pkg/mosconfig/mos.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "strings" + "sync" "time" "github.com/apex/log" @@ -67,6 +68,8 @@ type Mos struct { lockfile *os.File Manifest *SysManifest + + NetLock sync.Mutex } func NewMos(configDir, storeDir string) (*Mos, error) { @@ -166,6 +169,41 @@ func (mos *Mos) Current(name string) (*Target, error) { return nil, errors.Errorf("Target %s not found", name) } +// Called at system boot to do basic setup and +// activate all services +func (mos *Mos) Boot() error { + // For containers to start, /var/lib/lxc needs to be world-x + // at each point so that subuids can get to their RFS. + p := "" + for _, next := range []string{"/", "var", "lib", "lxc"} { + p = filepath.Join(p, next) + if err := os.Chmod(p, 0755); err != nil { + return errors.Wrapf(err, "Failed making %q world-accessible") + } + } + + // Now start the services + return mos.ActivateAll() +} + +// Activate all services +func (mos *Mos) ActivateAll() error { + m, err := mos.CurrentManifest() + if err != nil { + return errors.Wrapf(err, "Failed opening manifest") + } + for _, t := range m.SysTargets { + if t.Name == "hostfs" || t.Name == "bootkit" { + continue + } + if err := mos.Activate(t.Name); err != nil { + return errors.Wrapf(err, "Failed starting %s", t.Name) + } + } + + return nil +} + // Activate a target (service): // If it is not yet running then start it. // If it is already running, but is not at the newest version (i.e. after an @@ -284,17 +322,6 @@ func (mos *Mos) setupContainerService(t *Target) error { return nil } -func (mos *Mos) SetupNetwork(t *Target) ([]string, error) { - switch t.Network.Type { - case HostNetwork: - return []string{"lxc.net.0.type = none"}, nil - case NoNetwork: - return []string{"lxc.net.0.type = empty"}, nil - default: - return []string{}, fmt.Errorf("Unhandled network type: %s", t.Network.Type) - } -} - func (mos *Mos) writeLxcConfig(t *Target) error { // We are guaranteed to have stopped the container before reaching // here @@ -376,7 +403,7 @@ func (mos *Mos) writeLxcConfig(t *Target) error { } lxcConf = append(lxcConf, "lxc.rootfs.path = "+rfs) - netconf, err := mos.SetupNetwork(t) + netconf, err := mos.SetupTargetNetwork(t) if err != nil { return err } @@ -392,6 +419,7 @@ func (mos *Mos) writeLxcConfig(t *Target) error { lxcConf = append(lxcConf, "lxc.apparmor.profile = unchanged") lxcConf = append(lxcConf, fmt.Sprintf("lxc.log.file = %s/%s.log", lxclogDir, t.ServiceName)) + lxcConf = append(lxcConf, "lxc.environment = HOME=/root") for _, env := range syst.OCIConfig.Config.Env { lxcConf = append(lxcConf, fmt.Sprintf("lxc.environment = %s", env)) } @@ -436,6 +464,11 @@ func (mos *Mos) StopTarget(t *Target) error { if rc != 0 && !strings.HasSuffix(outs, "not loaded.\n") { return fmt.Errorf("Failed to stop service %s: %s", t.ServiceName, outs) } + + // TODO What about unhooking network? + if err := mos.StopTargetNetwork(t); err != nil { + log.Warnf("Failed tearing down network for %q: %v", t.ServiceName, err) + } case HostfsService: return fmt.Errorf("Stopping hostfs is not yet supported. Please poweroff") case FsService: diff --git a/pkg/mosconfig/network.go b/pkg/mosconfig/network.go index a1f468f..e4247e2 100644 --- a/pkg/mosconfig/network.go +++ b/pkg/mosconfig/network.go @@ -1,6 +1,207 @@ package mosconfig +import ( + "fmt" + "strings" + + "github.com/apex/log" + "github.com/pkg/errors" + "github.com/project-machine/mos/pkg/utils" +) + +// Validate a network config during manifest load. At this point we +// cannot yet verify whether resources like ports and addresses are in +// use, as that may change before service start. func (t Target) ValidateNetwork() bool { n := t.Network - return n.Type == HostNetwork || n.Type == NoNetwork + switch n.Type { + case HostNetwork, NoNetwork: + return true + case SimpleNetwork: + break + default: + return false + } + + for _, p := range n.Ports { + if p.HostPort > 65536 || p.ContainerPort > 65536 { + log.Warnf("too-high port %d or %d", p.HostPort, p.ContainerPort) + return false + } + if p.HostPort == 0 || p.ContainerPort == 0 { + log.Warnf("zero port %d or %d", p.HostPort, p.ContainerPort) + return false + } + } + + return true +} + +type SimplePort struct { + HostPort uint `json:"host" yaml:"host"` + ContainerPort uint `json:"container" yaml:"container"` +} + +// Pick a lxcbr0 address which is not yet in use. +// For now assume lxcbr0 is 10.0.3.1, TODO we should autodetect this. +func (mos *Mos) UnusedAddress() (string, error) { + for c := 3; c < 255; c++ { + txt := fmt.Sprintf("10.0.3.%d", c) + if _, ok := mos.Manifest.IpAddrs[txt]; !ok { + return txt, nil + } + } + return "", fmt.Errorf("No available addresses") +} + +func (mos *Mos) setupSimpleNet(t *Target) ([]string, error) { + config := []string{"lxc.net.0.type = veth", + "lxc.net.0.type = veth", + "lxc.net.0.link = lxcbr0", + "lxc.net.0.flags = up", + "lxc.net.0.hwaddr = 00:16:3e:xx:xx:xx", + } + + mos.NetLock.Lock() + defer mos.NetLock.Unlock() + + ipv4 := t.Network.Address + ipv6 := t.Network.Address6 + + // Make sure any requested address is not in use + if ipv4 != "" { + if _, ok := mos.Manifest.IpAddrs[ipv4]; ok { + return config, fmt.Errorf("Address in use: %q", ipv4) + } + } + if ipv6 != "" { + if _, ok := mos.Manifest.IpAddrs[ipv6]; ok { + return config, fmt.Errorf("Address in use: %q", ipv6) + } + } + + // If no address requested, choose one. No dhcp, because port fwd... + if ipv4 == "" && ipv6 == "" { + var err error + ipv4, err = mos.UnusedAddress() + if err != nil { + return config, err + } + } + + if ipv4 != "" { + config = append(config, "lxc.net.0.ipv4.address = "+ipv4) + config = append(config, "lxc.environment = IPV4="+ipv4) + mos.Manifest.IpAddrs[ipv4] = t.ServiceName + } + + if ipv6 != "" { + config = append(config, "lxc.net.0.ipv6.address = "+ipv6) + config = append(config, "lxc.environment = IPV6="+ipv6) + mos.Manifest.IpAddrs[ipv6] = t.ServiceName + } + + if err := mos.setupPortFwd(t); err != nil { + if ipv4 != "" { + delete(mos.Manifest.IpAddrs, ipv4) + } + if ipv6 != "" { + delete(mos.Manifest.IpAddrs, ipv6) + } + return config, err + } + + return config, nil +} + +// must be called with mos.NetLock held, since we update mos.Manifest +func (mos *Mos) DefaultNic() (string, error) { + if mos.Manifest.DefaultNic != "" { + return mos.Manifest.DefaultNic, nil + } + out, err := utils.Run("ip", "route") + if err != nil { + return "", errors.Wrapf(err, "Failed getting default route") + } + + lines := strings.Split(string(out), "\n") + for _, l := range lines { + if !strings.HasPrefix(l, "default via") { + continue + } + s := strings.Split(l, " ") + if len(s) < 5 { + continue + } + if s[3] != "dev" { + continue + } + nic := s[4] + mos.Manifest.DefaultNic = nic + return nic, nil + } + + return "", fmt.Errorf("No default route found (%q)", out) +} + +// Setup port forward rules for a container. Must be called with the +// mos.NetLock held. mos.setupSimpleNet() takes that lock. +func (mos *Mos) setupPortFwd(t *Target) error { + // TODO - we need the name of the host nic + nic, err := mos.DefaultNic() + if err != nil { + return errors.Wrapf(err, "Failed to find default nic") + } + ipaddr := "" + if t.Network.Address != "" { + ipaddr = t.Network.Address + } else if t.Network.Address6 != "" { + ipaddr = "[" + t.Network.Address6 + "]" + } else { + return fmt.Errorf("No usable address for port forward destination") + } + for _, p := range t.Network.Ports { + destaddr := strings.Split(ipaddr, "/")[0] // 192.168.2.0/24 + destaddr = fmt.Sprintf("%s:%d", destaddr, p.ContainerPort) + cmd := []string{ + "iptables", "-t", "nat", "-A", "PREROUTING", "-p", "tcp", + "-m", "tcp", "-i", nic, "--dport", fmt.Sprintf("%d", p.HostPort), + "-j", "DNAT", "--to-destination", destaddr} + if err := utils.RunCommand(cmd...); err != nil { + return errors.Wrapf(err, "Failed setting up port forward for %#v", p) + } + } + return nil +} + +func (mos *Mos) SetupTargetNetwork(t *Target) ([]string, error) { + switch t.Network.Type { + case HostNetwork: + return []string{"lxc.net.0.type = none"}, nil + case NoNetwork: + return []string{"lxc.net.0.type = empty"}, nil + case SimpleNetwork: + return mos.setupSimpleNet(t) + default: + return []string{}, fmt.Errorf("Unhandled network type: %s", t.Network.Type) + } +} + +func (mos *Mos) StopTargetNetwork(t *Target) error { + mos.NetLock.Lock() + defer mos.NetLock.Unlock() + + for _, p := range t.Network.Ports { + // TODO - remove the iptables rule for this port + delete(mos.Manifest.UsedPorts, p.HostPort) + } + + if t.Network.Address != "" { + delete(mos.Manifest.IpAddrs, t.Network.Address) + } + if t.Network.Address6 != "" { + delete(mos.Manifest.IpAddrs, t.Network.Address6) + } + + return nil } diff --git a/pkg/mosconfig/uidmap.go b/pkg/mosconfig/uidmap.go index 45f631f..9a4d448 100644 --- a/pkg/mosconfig/uidmap.go +++ b/pkg/mosconfig/uidmap.go @@ -97,6 +97,10 @@ func (mos *Mos) GetUIDMapStr(t *Target) (idmap.IdmapSet, []string, error) { } rangedefs := chooseRangeDefaults() + if t.NSGroup == "none" { + return empty, []string{}, nil + } + for _, u := range manifest.UidMaps { if u.Name == t.NSGroup { uidmap := idmap.IdmapEntry{ diff --git a/pkg/provider/kvm.go b/pkg/provider/kvm.go index 2924b18..f2abbd9 100644 --- a/pkg/provider/kvm.go +++ b/pkg/provider/kvm.go @@ -26,7 +26,7 @@ const KVMTemplate = ` config: name: %s uefi: true - uefi-code: /usr/share/OVMF/OVMF_CODE.secboot.fd + uefi-code: %s uefi-vars: %s cdrom: %s boot: cdrom @@ -35,6 +35,26 @@ const KVMTemplate = ` serial: true tpm-version: 2.0 secure-boot: true + nics: + - device: virtio-net + id: nic0 + bootindex: "off" + network: user + ports: + - protocol: tcp + host: + address: "" + port: 22222 + guest: + address: "" + port: 22 + - protocol: tcp + host: + address: "" + port: 28080 + guest: + address: "" + port: 80 disks: - file: %s type: ssd @@ -113,11 +133,12 @@ func (p KVMProvider) New(mname, keyproject, UUID string) (Machine, error) { // that's worth it. provisionISO := filepath.Join(keysetDir, "artifacts", "provision.iso") uefiVars := filepath.Join(keysetDir, "bootkit", "ovmf-vars.fd") - mData := fmt.Sprintf(KVMTemplate, m.Name, m.Name, uefiVars, provisionISO, - qcowPath, sudiPath) - _, _, err = utils.RunWithStdall(mData, "machine", "init", m.Name) + uefiCode := filepath.Join(keysetDir, "bootkit", "ovmf", "ovmf-code.fd") + mData := fmt.Sprintf(KVMTemplate, m.Name, m.Name, uefiCode, uefiVars, + provisionISO, qcowPath, sudiPath) + stdout, stderr, err := utils.RunWithStdall(mData, "machine", "init", m.Name) if err != nil { - return m, errors.Wrapf(err, "Failed initializing machine") + return m, errors.Wrapf(err, "Failed initializing machine %q with data:%q\nOutput: %q\n%q\n", m.Name, mData, stdout, stderr) } // Write out the details of the machine to persistent storage @@ -294,10 +315,10 @@ func waitForUnix(sockPath, good, bad string) error { return nil } if strings.Contains(s, bad) { - return errors.Errorf("Action failed, as %q was found", bad) + return errors.Errorf("Action failed, as %q was found in %q", bad, s) } - return errors.Errorf("Action timed out, did not find %q nor %q, in %q", good, bad) + return errors.Errorf("Action timed out, did not find %q nor %q, in %q", good, bad, s) } func (m KVMMachine) state(desired string) bool { diff --git a/pkg/trust/artifacts.go b/pkg/trust/artifacts.go index f95b4d1..5a95cfe 100644 --- a/pkg/trust/artifacts.go +++ b/pkg/trust/artifacts.go @@ -171,6 +171,8 @@ bootkit: dest: /bootkit/kernel.efi - path: %s/bootkit/shim.efi dest: /bootkit/shim.efi + - path: %s/bootkit/kernel/modules.squashfs + dest: /bootkit/modules.squashfs ` // SetupBootkit: create a custom bootkit for a keyset. @@ -272,7 +274,7 @@ func SetupBootkit(keysetName, bootkitVersion, mosctlPath string) error { if err := utils.EnsureDir(tmpoci); err != nil { return err } - yamlString := fmt.Sprintf(bootkitOciTemplate, keysetPath, keysetPath) + yamlString := fmt.Sprintf(bootkitOciTemplate, keysetPath, keysetPath, keysetPath) yamlFile := filepath.Join(tmpoci, "stacker.yaml") if err := os.WriteFile(yamlFile, []byte(yamlString), 0644); err != nil { return err diff --git a/tests/helpers.bash b/tests/helpers.bash index 26b89df..78e74df 100644 --- a/tests/helpers.bash +++ b/tests/helpers.bash @@ -29,7 +29,7 @@ function trust_setup { function common_setup { trust_setup - export ROOTFS_VERSION="${ROOTFS_VERSION:-v0.0.15.230901}" + export ROOTFS_VERSION="${ROOTFS_VERSION:-v0.0.17.231018}" echo "ROOTFS_VERSION is ${ROOTFS_VERSION}" if [ ! -d "${PWD}/zothub" ]; then @@ -119,6 +119,7 @@ function common_teardown { if [ -n $TMPUD ]; then lxc-usernsexec -s -- rm -rf $TMPUD fi + trust_teardown } diff --git a/tests/launch.bats b/tests/launch.bats index 101437d..887b5b2 100644 --- a/tests/launch.bats +++ b/tests/launch.bats @@ -24,7 +24,7 @@ product: default update_type: complete targets: - service_name: hostfs - source: "docker://zothub.io/machine/bootkit/demo-target-rootfs:0.0.3-squashfs" + source: "docker://zothub.io/machine/bootkit/demo-target-rootfs:0.0.4-squashfs" version: 1.0.0 service_type: hostfs nsgroup: "none" diff --git a/tests/livecd1.bats b/tests/livecd1.bats index f9035b6..7a33f45 100644 --- a/tests/livecd1.bats +++ b/tests/livecd1.bats @@ -36,7 +36,7 @@ function teardown() { name: ${VMNAME} uefi: true uefi-vars: $HOME/.local/share/machine/trust/keys/snakeoil/bootkit/ovmf-vars.fd - uefi-code: /usr/share/OVMF/OVMF_CODE.secboot.fd + uefi-code: $HOME/.local/share/machine/trust/keys/snakeoil/bootkit/ovmf/ovmf-code.fd cdrom: livecd.iso boot: cdrom tpm: true diff --git a/tests/livecd2.bats b/tests/livecd2.bats index 6938d5a..2fe605d 100644 --- a/tests/livecd2.bats +++ b/tests/livecd2.bats @@ -50,7 +50,7 @@ function teardown() { name: ${VMNAME} uefi: true uefi-vars: $HOME/.local/share/machine/trust/keys/snakeoil/bootkit/ovmf-vars.fd - uefi-code: /usr/share/OVMF/OVMF_CODE.secboot.fd + uefi-code: $HOME/.local/share/machine/trust/keys/snakeoil/bootkit/ovmf/ovmf-code.fd cdrom: $HOME/.local/share/machine/trust/keys/snakeoil/artifacts/provision.iso boot: cdrom tpm: true