diff --git a/cli/containerd-shim-kata-v2/main.go b/cli/containerd-shim-kata-v2/main.go index 5b9260fbc1..df0b3d78e1 100644 --- a/cli/containerd-shim-kata-v2/main.go +++ b/cli/containerd-shim-kata-v2/main.go @@ -6,11 +6,11 @@ package main import ( -"fmt" -"os" + "fmt" + "os" -"github.com/kata-containers/runtime/containerd-shim/kata" -"github.com/containerd/containerd/runtime/v2/shim" + "github.com/containerd/containerd/runtime/v2/shim" + "github.com/kata-containers/runtime/containerd-shim/kata" ) func main() { @@ -19,4 +19,3 @@ func main() { os.Exit(1) } } - diff --git a/containerd-shim/kata/config.go b/containerd-shim/kata/config.go index 410645c726..ef4a0b2d2b 100644 --- a/containerd-shim/kata/config.go +++ b/containerd-shim/kata/config.go @@ -19,7 +19,6 @@ import ( ) var defaultHypervisorPath = "/usr/bin/qemu-lite-system-x86_64" -var defaultImagePath = "/usr/share/kata-containers/kata-containers.img" var defaultKernelPath = "/usr/share/kata-containers/vmlinuz.container" var defaultInitrdPath = "/usr/share/kata-containers/kata-containers-initrd.img" var defaultFirmwarePath = "" @@ -27,7 +26,6 @@ var defaultMachineAccelerators = "" const defaultKernelParams = "" const defaultMachineType = "pc" -const defaultRootDirectory = "/var/run/kata-containers" const systemdUnitName = "kata-containers.target" const defaultVCPUCount uint32 = 1 @@ -35,7 +33,6 @@ const defaultMaxVCPUCount uint32 = 0 const defaultMemSize uint32 = 2048 // MiB const defaultBridgesCount uint32 = 1 const defaultInterNetworkingModel = "macvtap" -const defaultDisableBlockDeviceUse bool = false const defaultBlockDeviceDriver = "virtio-scsi" const defaultEnableIOThreads bool = false const defaultEnableMemPrealloc bool = false diff --git a/containerd-shim/kata/container.go b/containerd-shim/kata/container.go index a31e858efc..d1948fcc39 100644 --- a/containerd-shim/kata/container.go +++ b/containerd-shim/kata/container.go @@ -9,36 +9,35 @@ import ( "github.com/containerd/containerd/api/types/task" "github.com/containerd/containerd/errdefs" taskAPI "github.com/containerd/containerd/runtime/v2/task" - vc "github.com/kata-containers/runtime/virtcontainers" "sync" "time" ) -type Container struct { +type container struct { s *service - pid uint32 - id string - stdin string - stdout string - stderr string - ttyio *TtyIO - terminal bool - + ttyio *ttyIO + execs map[string]*exec exitIOch chan struct{} exitch chan uint32 + pid uint32 + exit uint32 + status task.Status + + id string + stdin string + stdout string + stderr string + bundle string + + terminal bool - bundle string - execs map[string]*Exec - container vc.VCContainer - status task.Status - exit uint32 - time time.Time + time time.Time mu sync.Mutex } -func newContainer(s *service, r *taskAPI.CreateTaskRequest, pid uint32, container vc.VCContainer) *Container { - c := &Container{ +func newContainer(s *service, r *taskAPI.CreateTaskRequest, pid uint32) *container { + c := &container{ s: s, pid: pid, id: r.ID, @@ -47,7 +46,7 @@ func newContainer(s *service, r *taskAPI.CreateTaskRequest, pid uint32, containe stdout: r.Stdout, stderr: r.Stderr, terminal: r.Terminal, - execs: make(map[string]*Exec), + execs: make(map[string]*exec), status: task.StatusCreated, exitIOch: make(chan struct{}), exitch: make(chan uint32, 1), @@ -56,7 +55,7 @@ func newContainer(s *service, r *taskAPI.CreateTaskRequest, pid uint32, containe return c } -func (c *Container) getExec(id string) (*Exec, error) { +func (c *container) getExec(id string) (*exec, error) { exec := c.execs[id] if exec == nil { diff --git a/containerd-shim/kata/create.go b/containerd-shim/kata/create.go index 4e81cbca2a..732694cb55 100644 --- a/containerd-shim/kata/create.go +++ b/containerd-shim/kata/create.go @@ -70,29 +70,7 @@ func create(s *service, containerID, bundlePath, netns string, detach bool, } } - if runtimeConfig.FactoryConfig.Template { - factoryConfig := vf.Config{ - Template: true, - VMConfig: vc.VMConfig{ - HypervisorType: runtimeConfig.HypervisorType, - HypervisorConfig: runtimeConfig.HypervisorConfig, - AgentType: runtimeConfig.AgentType, - AgentConfig: runtimeConfig.AgentConfig, - }, - } - logrus.WithField("factory", factoryConfig).Info("load vm factory") - f, err := vf.NewFactory(factoryConfig, true) - if err != nil { - logrus.WithError(err).Warn("load vm factory failed, about to create new one") - f, err = vf.NewFactory(factoryConfig, false) - if err != nil { - logrus.WithError(err).Warn("create vm factory failed") - } - } - if err != nil { - vci.SetFactory(f) - } - } + setFactory(runtimeConfig) disableOutput := noNeedForOutput(detach, ociSpec.Process.Terminal) @@ -123,6 +101,32 @@ func create(s *service, containerID, bundlePath, netns string, detach bool, return c, nil } +func setFactory(runtimeConfig *oci.RuntimeConfig) { + if runtimeConfig.FactoryConfig.Template { + factoryConfig := vf.Config{ + Template: true, + VMConfig: vc.VMConfig{ + HypervisorType: runtimeConfig.HypervisorType, + HypervisorConfig: runtimeConfig.HypervisorConfig, + AgentType: runtimeConfig.AgentType, + AgentConfig: runtimeConfig.AgentConfig, + }, + } + logrus.WithField("factory", factoryConfig).Info("load vm factory") + f, err := vf.NewFactory(factoryConfig, true) + if err != nil { + logrus.WithError(err).Warn("load vm factory failed, about to create new one") + f, err = vf.NewFactory(factoryConfig, false) + if err != nil { + logrus.WithError(err).Warn("create vm factory failed") + } + } + if err != nil { + vci.SetFactory(f) + } + } +} + var systemdKernelParam = []vc.Param{ { Key: "init", diff --git a/containerd-shim/kata/delete.go b/containerd-shim/kata/delete.go index 5beedfe493..04910dd874 100644 --- a/containerd-shim/kata/delete.go +++ b/containerd-shim/kata/delete.go @@ -12,7 +12,7 @@ import ( "path" ) -func deleteContainer(s *service, c *Container) error { +func deleteContainer(s *service, c *container) error { status, err := s.sandbox.StatusContainer(c.id) if err != nil { diff --git a/containerd-shim/kata/exec.go b/containerd-shim/kata/exec.go index be93b6d999..b4d85f67f4 100644 --- a/containerd-shim/kata/exec.go +++ b/containerd-shim/kata/exec.go @@ -16,15 +16,17 @@ import ( "time" ) -type Exec struct { +type exec struct { + container *container + cmds *vc.Cmd + tty *tty + ttyio *ttyIO id string pid uint32 - container *Container - cmds *vc.Cmd - exitCode int32 - tty *Tty - ttyio *TtyIO - status task.Status + + exitCode int32 + + status task.Status exitIOch chan struct{} exitch chan uint32 @@ -32,7 +34,7 @@ type Exec struct { exitTime time.Time } -type Tty struct { +type tty struct { stdin string stdout string stderr string @@ -60,7 +62,7 @@ func getEnvs(envs []string) []vc.EnvVar { return vcEnvs } -func newExec(c *Container, stdin, stdout, stderr string, terminal bool, jspec *googleProtobuf.Any) (*Exec, error) { +func newExec(c *container, stdin, stdout, stderr string, terminal bool, jspec *googleProtobuf.Any) (*exec, error) { var height uint32 var width uint32 @@ -75,7 +77,7 @@ func newExec(c *Container, stdin, stdout, stderr string, terminal bool, jspec *g width = uint32(spec.ConsoleSize.Width) } - tty := &Tty{ + tty := &tty{ stdin: stdin, stdout: stdout, stderr: stderr, @@ -95,7 +97,7 @@ func newExec(c *Container, stdin, stdout, stderr string, terminal bool, jspec *g NoNewPrivileges: spec.NoNewPrivileges, } - exec := &Exec{ + exec := &exec{ container: c, cmds: cmds, tty: tty, diff --git a/containerd-shim/kata/metrics.go b/containerd-shim/kata/metrics.go index 9611c8f7ac..36c0e43ad8 100644 --- a/containerd-shim/kata/metrics.go +++ b/containerd-shim/kata/metrics.go @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 // + package kata import ( diff --git a/containerd-shim/kata/service.go b/containerd-shim/kata/service.go index 2359260e8a..eb1f5d171c 100644 --- a/containerd-shim/kata/service.go +++ b/containerd-shim/kata/service.go @@ -2,13 +2,14 @@ // // SPDX-License-Identifier: Apache-2.0 // + package kata import ( "context" "encoding/json" "os" - "os/exec" + sysexec "os/exec" "sync" "syscall" "time" @@ -68,10 +69,10 @@ func New(ctx context.Context, id string, publisher events.Publisher) (cdshim.Shi id: id, context: ctx, config: runtimeConfig, - containers: make(map[string]*Container), + containers: make(map[string]*container), processes: make(map[uint32]string), events: make(chan interface{}, 128), - ec: make(chan Exit, bufferSize), + ec: make(chan exit, bufferSize), } go s.processExits() @@ -83,7 +84,7 @@ func New(ctx context.Context, id string, publisher events.Publisher) (cdshim.Shi return s, nil } -type Exit struct { +type exit struct { id string execid string pid int @@ -97,12 +98,12 @@ type service struct { context context.Context sandbox vc.VCSandbox - containers map[string]*Container + containers map[string]*container processes map[uint32]string config *oci.RuntimeConfig events chan interface{} - ec chan Exit + ec chan exit id string } @@ -113,7 +114,7 @@ func (s *service) pid() uint32 { if !ok { break } else { - pidCount += 1 + pidCount++ //if it overflows, recount from 5 if pidCount < 5 { pidCount = 5 @@ -123,7 +124,7 @@ func (s *service) pid() uint32 { return pidCount } -func newCommand(ctx context.Context, containerdBinary, containerdAddress string) (*exec.Cmd, error) { +func newCommand(ctx context.Context, containerdBinary, containerdAddress string) (*sysexec.Cmd, error) { ns, err := namespaces.NamespaceRequired(ctx) if err != nil { return nil, err @@ -141,7 +142,7 @@ func newCommand(ctx context.Context, containerdBinary, containerdAddress string) "-address", containerdAddress, "-publish-binary", containerdBinary, } - cmd := exec.Command(self, args...) + cmd := sysexec.Command(self, args...) cmd.Dir = cwd cmd.Env = append(os.Environ(), "GOMAXPROCS=2") cmd.SysProcAttr = &syscall.SysProcAttr{ @@ -157,30 +158,11 @@ func (s *service) StartShim(ctx context.Context, id, containerdBinary, container return "", err } - // Checks the MUST and MUST NOT from OCI runtime specification - if bundlePath, err = validCreateParams(id, bundlePath); err != nil { - return "", err - } - - ociSpec, err := oci.ParseConfigJSON(bundlePath) - if err != nil { - return "", err - } - - containerType, err := ociSpec.ContainerType() + address, err := getAddress(ctx, bundlePath, id) if err != nil { return "", err } - - if containerType == vc.PodContainer { - sandboxID, err := ociSpec.SandboxID() - if err != nil { - return "", err - } - address, err := cdshim.SocketAddress(ctx, sandboxID) - if err != nil { - return "", err - } + if address != "" { return address, nil } @@ -188,10 +170,12 @@ func (s *service) StartShim(ctx context.Context, id, containerdBinary, container if err != nil { return "", err } - address, err := cdshim.SocketAddress(ctx, id) + + address, err = cdshim.SocketAddress(ctx, id) if err != nil { return "", err } + socket, err := cdshim.NewSocket(address) if err != nil { return "", err @@ -341,13 +325,13 @@ func (s *service) Create(ctx context.Context, r *taskAPI.CreateTaskRequest) (_ * } } - c, err := create(s, r.ID, r.Bundle, ns, !r.Terminal, s.config) + _, err = create(s, r.ID, r.Bundle, ns, !r.Terminal, s.config) if err != nil { return nil, err } pid := s.pid() - container := newContainer(s, r, pid, c) + container := newContainer(s, r, pid) container.status = task.StatusCreated s.containers[r.ID] = container @@ -378,16 +362,16 @@ func (s *service) Start(ctx context.Context, r *taskAPI.StartRequest) (*taskAPI. return &taskAPI.StartResponse{ Pid: c.pid, }, nil - } else { //start an exec - execs, err := startExec(ctx, s, r.ID, r.ExecID) - if err != nil { - return nil, errdefs.ToGRPC(err) - } - - return &taskAPI.StartResponse{ - Pid: execs.pid, - }, nil } + //start an exec + execs, err := startExec(ctx, s, r.ID, r.ExecID) + if err != nil { + return nil, errdefs.ToGRPC(err) + } + + return &taskAPI.StartResponse{ + Pid: execs.pid, + }, nil } // Delete the initial process and container @@ -411,21 +395,21 @@ func (s *service) Delete(ctx context.Context, r *taskAPI.DeleteRequest) (*taskAP ExitedAt: c.time, Pid: c.pid, }, nil - } else { - execs, err := c.getExec(r.ExecID) - if err != nil { - return nil, err - } + } + //deal with the exec case + execs, err := c.getExec(r.ExecID) + if err != nil { + return nil, err + } - delete(s.processes, execs.pid) - delete(c.execs, r.ExecID) + delete(s.processes, execs.pid) + delete(c.execs, r.ExecID) - return &taskAPI.DeleteResponse{ - ExitStatus: uint32(execs.exitCode), - ExitedAt: execs.exitTime, - Pid: execs.pid, - }, nil - } + return &taskAPI.DeleteResponse{ + ExitStatus: uint32(execs.exitCode), + ExitedAt: execs.exitTime, + Pid: execs.pid, + }, nil } // Exec an additional process inside the container @@ -504,23 +488,26 @@ func (s *service) State(ctx context.Context, r *taskAPI.StateRequest) (*taskAPI. Terminal: c.terminal, ExitStatus: c.exit, }, nil - } else { - execs, err := c.getExec(r.ExecID) - if err != nil { - return nil, err - } - return &taskAPI.StateResponse{ - ID: execs.id, - Bundle: c.bundle, - Pid: execs.pid, - Status: execs.status, - Stdin: execs.tty.stdin, - Stdout: execs.tty.stdout, - Stderr: execs.tty.stderr, - Terminal: execs.tty.terminal, - ExitStatus: uint32(execs.exitCode), - }, nil } + + //deal with exec case + execs, err := c.getExec(r.ExecID) + if err != nil { + return nil, err + } + + return &taskAPI.StateResponse{ + ID: execs.id, + Bundle: c.bundle, + Pid: execs.pid, + Status: execs.status, + Stdin: execs.tty.stdin, + Stdout: execs.tty.stdout, + Stderr: execs.tty.stderr, + Terminal: execs.tty.terminal, + ExitStatus: uint32(execs.exitCode), + }, nil + } // Pause the container @@ -765,7 +752,7 @@ func (s *service) processExits() { } } -func (s *service) checkProcesses(e Exit) { +func (s *service) checkProcesses(e exit) { s.mu.Lock() defer s.mu.Unlock() @@ -783,7 +770,7 @@ func (s *service) checkProcesses(e Exit) { return } -func (s *service) getContainer(id string) (*Container, error) { +func (s *service) getContainer(id string) (*container, error) { c := s.containers[id] if c == nil { diff --git a/containerd-shim/kata/start.go b/containerd-shim/kata/start.go index 9640360524..df5cfb47be 100644 --- a/containerd-shim/kata/start.go +++ b/containerd-shim/kata/start.go @@ -13,7 +13,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/pkg/oci" ) -func startContainer(ctx context.Context, s *service, c *Container) error { +func startContainer(ctx context.Context, s *service, c *container) error { //start a container // Checks the MUST and MUST NOT from OCI runtime specification status, sandboxID, err := getExistingContainerInfo(c.id) @@ -44,6 +44,9 @@ func startContainer(ctx context.Context, s *service, c *Container) error { return err } tty, err := newTtyIO(ctx, c.stdin, c.stdout, c.stderr, c.terminal) + if err != nil { + return err + } c.ttyio = tty go ioCopy(c.exitIOch, tty, stdin, stdout, stderr) @@ -53,7 +56,7 @@ func startContainer(ctx context.Context, s *service, c *Container) error { return nil } -func startExec(ctx context.Context, s *service, containerID, execID string) (*Exec, error) { +func startExec(ctx context.Context, s *service, containerID, execID string) (*exec, error) { //start an exec c, err := s.getContainer(containerID) if err != nil { @@ -88,6 +91,9 @@ func startExec(ctx context.Context, s *service, containerID, execID string) (*Ex return nil, err } tty, err := newTtyIO(ctx, execs.tty.stdin, execs.tty.stdout, execs.tty.stderr, execs.tty.terminal) + if err != nil { + return nil, err + } execs.ttyio = tty go ioCopy(execs.exitIOch, tty, stdin, stdout, stderr) diff --git a/containerd-shim/kata/stream.go b/containerd-shim/kata/stream.go index ad5f844937..ae485610d4 100644 --- a/containerd-shim/kata/stream.go +++ b/containerd-shim/kata/stream.go @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 // + package kata import ( @@ -12,13 +13,13 @@ import ( "syscall" ) -type TtyIO struct { +type ttyIO struct { Stdin io.ReadCloser Stdout io.Writer Stderr io.Writer } -func (tty *TtyIO) Close() { +func (tty *ttyIO) close() { if tty.Stdin != nil { tty.Stdin.Close() @@ -35,7 +36,7 @@ func (tty *TtyIO) Close() { cf(tty.Stderr) } -func newTtyIO(ctx context.Context, stdin, stdout, stderr string, console bool) (*TtyIO, error) { +func newTtyIO(ctx context.Context, stdin, stdout, stderr string, console bool) (*ttyIO, error) { var in io.ReadCloser var outw io.Writer var errw io.Writer @@ -62,7 +63,7 @@ func newTtyIO(ctx context.Context, stdin, stdout, stderr string, console bool) ( } } - ttyIO := &TtyIO{ + ttyIO := &ttyIO{ Stdin: in, Stdout: outw, Stderr: errw, @@ -71,7 +72,7 @@ func newTtyIO(ctx context.Context, stdin, stdout, stderr string, console bool) ( return ttyIO, nil } -func ioCopy(exitch chan struct{}, tty *TtyIO, stdinPipe io.WriteCloser, stdoutPipe, stderrPipe io.Reader) { +func ioCopy(exitch chan struct{}, tty *ttyIO, stdinPipe io.WriteCloser, stdoutPipe, stderrPipe io.Reader) { var wg sync.WaitGroup var closeOnce sync.Once @@ -93,7 +94,7 @@ func ioCopy(exitch chan struct{}, tty *TtyIO, stdinPipe io.WriteCloser, stdoutPi defer bufPool.Put(p) io.CopyBuffer(tty.Stdout, stdoutPipe, *p) wg.Done() - closeOnce.Do(tty.Close) + closeOnce.Do(tty.close) }() } @@ -108,6 +109,6 @@ func ioCopy(exitch chan struct{}, tty *TtyIO, stdinPipe io.WriteCloser, stdoutPi } wg.Wait() - closeOnce.Do(tty.Close) + closeOnce.Do(tty.close) close(exitch) } diff --git a/containerd-shim/kata/utils.go b/containerd-shim/kata/utils.go index 06e961f6d4..2654a51cb1 100644 --- a/containerd-shim/kata/utils.go +++ b/containerd-shim/kata/utils.go @@ -12,6 +12,11 @@ import ( "path/filepath" "strings" "time" + + "context" + cdshim "github.com/containerd/containerd/runtime/v2/shim" + vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/pkg/oci" ) const ( @@ -64,7 +69,7 @@ func resolvePath(path string) (string, error) { } func cReap(s *service, pid, status int, id, execid string, exitat time.Time) { - s.ec <- Exit{ + s.ec <- exit{ timestamp: exitat, pid: pid, status: status, @@ -72,3 +77,36 @@ func cReap(s *service, pid, status int, id, execid string, exitat time.Time) { execid: execid, } } + +func getAddress(ctx context.Context, bundlePath, id string) (string, error) { + var err error + + // Checks the MUST and MUST NOT from OCI runtime specification + if bundlePath, err = validCreateParams(id, bundlePath); err != nil { + return "", err + } + + ociSpec, err := oci.ParseConfigJSON(bundlePath) + if err != nil { + return "", err + } + + containerType, err := ociSpec.ContainerType() + if err != nil { + return "", err + } + + if containerType == vc.PodContainer { + sandboxID, err := ociSpec.SandboxID() + if err != nil { + return "", err + } + address, err := cdshim.SocketAddress(ctx, sandboxID) + if err != nil { + return "", err + } + return address, nil + } + + return "", nil +} diff --git a/containerd-shim/kata/wait.go b/containerd-shim/kata/wait.go index 4f50078862..ba7a8e0016 100644 --- a/containerd-shim/kata/wait.go +++ b/containerd-shim/kata/wait.go @@ -10,8 +10,8 @@ import ( "time" ) -func wait(s *service, c *Container, execID string) (int32, error) { - var execs *Exec +func wait(s *service, c *container, execID string) (int32, error) { + var execs *exec var err error processID := c.id