Add configuration to OutputLimit & CpuSet

README.md

@@ -62,6 +62,8 @@ The `executorserver` need root privilege to create `cgroup`. Either creates sub-
 - `-auth-token` to add token-based authentication to REST / gRPC
 - `-src-prefix` to restrict `src` copyIn path (need to be absolute path)
 - `-time-limit-checker-interval` specifies time limit checker interval (default 100ms)
+- `-output-limit` specifies size limit of POSIX rlimit of output
+- `-cpuset` specifies `cpuset.cpus` cgroup for each container
 
 #### Environment Variables
 

cmd/executorserver/config/config.go

@@ -26,6 +26,8 @@ type Config struct {
 	// runner limit
 	TimeLimitCheckerInterval time.Duration `flagUsage:"specifies time limit checker interval" default:"100ms"`
 	ExtraMemoryLimit         *envexec.Size `flagUsage:"specifies extra memory buffer for check memory limit" default:"16k"`
+	OutputLimit              *envexec.Size `flagUsage:"specifies POSIX rlimit for output for each command" default:"256m"`
+	Cpuset                   string        `flagUsage:"control the usage of cpuset for all containerd process"`
 
 	// server config
 	HTTPAddr      string `flagUsage:"specifies the http binding address" default:":5050"`

cmd/executorserver/main.go

@@ -75,6 +75,7 @@ func main() {
 		TmpFsParam:        conf.TmpFsParam,
 		NetShare:          conf.NetShare,
 		CgroupPrefix:      conf.CgroupPrefix,
+		Cpuset:            conf.Cpuset,
 		Logger:            logger.Sugar(),
 	})
 	if err != nil {
@@ -102,6 +103,7 @@ func main() {
 		WorkDir:               conf.Dir,
 		TimeLimitTickInterval: conf.TimeLimitCheckerInterval,
 		ExtraMemoryLimit:      *conf.ExtraMemoryLimit,
+		OutputLimit:           *conf.OutputLimit,
 	})
 	work.Start()
 	logger.Sugar().Infof("Starting worker with parallelism=%d, workdir=%s, timeLimitCheckInterval=%v",

env/config.go

@@ -15,5 +15,6 @@ type Config struct {
 	NetShare          bool
 	MountConf         string
 	CgroupPrefix      string
+	Cpuset            string
 	Logger
 }

env/env_linux.go

@@ -75,7 +75,11 @@ func NewBuilder(c Config) (pool.EnvBuilder, error) {
 		DomainName:    domainName,
 		WorkDir:       workDir,
 	}
-	cgb, err := cgroup.NewBuilder(c.CgroupPrefix).WithCPUAcct().WithMemory().WithPids().FilterByEnv()
+	cgb := cgroup.NewBuilder(c.CgroupPrefix).WithCPUAcct().WithMemory().WithPids()
+	if c.Cpuset != "" {
+		cgb = cgb.WithCPUSet()
+	}
+	cgb, err = cgb.FilterByEnv()
 	if err != nil {
 		return nil, err
 	}
@@ -92,7 +96,7 @@ func NewBuilder(c Config) (pool.EnvBuilder, error) {
 	if cgb != nil {
 		cgroupPool = pool.NewFakeCgroupPool(cgb)
 	}
-	return pool.NewEnvBuilder(b, cgroupPool, workDir), nil
+	return pool.NewEnvBuilder(b, cgroupPool, workDir, c.Cpuset), nil
 }
 
 type credGen struct {

go.mod

@@ -5,7 +5,7 @@ go 1.14
 require (
 	cloud.google.com/go v0.73.0 // indirect
 	github.com/creack/pty v1.1.11
-	github.com/criyle/go-sandbox v0.5.2
+	github.com/criyle/go-sandbox v0.5.4
 	github.com/fatih/camelcase v1.0.0 // indirect
 	github.com/fatih/structs v1.1.0 // indirect
 	github.com/gin-contrib/pprof v1.3.0
@@ -30,7 +30,7 @@ require (
 	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
 	golang.org/x/sys v0.0.0-20201204225414-ed752295db88
 	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
-	golang.org/x/tools v0.0.0-20201204222352-654352759326 // indirect
+	golang.org/x/tools v0.0.0-20201206230334-368bee879bfd // indirect
 	google.golang.org/genproto v0.0.0-20201204160425-06b3db808446 // indirect
 	google.golang.org/grpc v1.34.0
 	google.golang.org/grpc/examples v0.0.0-20201204235607-0d6a24f68a5f // indirect

go.sum

@@ -85,8 +85,8 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/criyle/go-sandbox v0.5.2 h1:P6I5vE3dHq6peu3PXBHXpahimKJHnbt/vbV3Eby0qz4=
-github.com/criyle/go-sandbox v0.5.2/go.mod h1:6iDDyseyHVugTM2ixPleIBuRx3pVuMKh2qTm6IRmvd8=
+github.com/criyle/go-sandbox v0.5.4 h1:6CNSbw3zjCyI8PxGhOH0Liz9F05P2UDGTW4uvb3DS+8=
+github.com/criyle/go-sandbox v0.5.4/go.mod h1:6iDDyseyHVugTM2ixPleIBuRx3pVuMKh2qTm6IRmvd8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -710,8 +710,8 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd h1:kJP9fbfkpUoA4y03Nxor8be
 golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201202200335-bef1c476418a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201204222352-654352759326 h1:XKLw9EEEfGJFE2K5Ni4nXgtFBIfI+zKPIi2SlRYmIG4=
-golang.org/x/tools v0.0.0-20201204222352-654352759326/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201206230334-368bee879bfd h1:EqFvKLTxjH6gEy2baWxX2AgJwZkBIDIcZFYcoYlI9RA=
+golang.org/x/tools v0.0.0-20201206230334-368bee879bfd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=

pkg/envexec/cmd.go

@@ -33,6 +33,7 @@ type Cmd struct {
 	MemoryLimit      Size
 	StackLimit       Size
 	ExtraMemoryLimit Size
+	OutputLimit      Size
 	ProcLimit        uint64
 
 	// file contents to copyin before exec

pkg/envexec/interface.go

@@ -33,6 +33,7 @@ type Limit struct {
 	Memory Size          // Memory limit
 	Proc   uint64        // Process count limit
 	Stack  Size          // Stack limit
+	Output Size          // Output limit
 }
 
 // Usage defines the peak process resource usage

pkg/pool/cgroup_wrapper_linux.go

@@ -13,6 +13,10 @@ var (
 
 type wCgroup cgroup.Cgroup
 
+func (c *wCgroup) SetCpuset(s string) error {
+	return (*cgroup.Cgroup)(c).SetCpusetCpus([]byte(s))
+}
+
 func (c *wCgroup) SetMemoryLimit(s envexec.Size) error {
 	return (*cgroup.Cgroup)(c).SetMemoryLimitInBytes(uint64(s))
 }

pkg/pool/cgrouppool_linux.go

@@ -9,6 +9,7 @@ import (
 
 // Cgroup defines interface to limit and monitor resources consumption of a process
 type Cgroup interface {
+	SetCpuset(string) error
 	SetMemoryLimit(envexec.Size) error
 	SetProcLimit(uint64) error
 

pkg/pool/envbuilder_linux.go

@@ -11,14 +11,16 @@ type environmentBuilder struct {
 	builder EnvironmentBuilder
 	cgPool  CgroupPool
 	workDir string
+	cpuset  string
 }
 
 // NewEnvBuilder creates builder for linux container pools
-func NewEnvBuilder(builder EnvironmentBuilder, cgPool CgroupPool, workDir string) EnvBuilder {
+func NewEnvBuilder(builder EnvironmentBuilder, cgPool CgroupPool, workDir, cpuset string) EnvBuilder {
 	return &environmentBuilder{
 		builder: builder,
 		cgPool:  cgPool,
 		workDir: workDir,
+		cpuset:  cpuset,
 	}
 }
 
@@ -40,5 +42,6 @@ func (b *environmentBuilder) Build() (Environment, error) {
 		Environment: m,
 		cgPool:      b.cgPool,
 		wd:          wd[0],
+		cpuset:      b.cpuset,
 	}, nil
 }

pkg/pool/environment_linux.go

@@ -12,15 +12,14 @@ import (
 	"github.com/criyle/go-sandbox/pkg/rlimit"
 )
 
-const outputLimit = 256 << 20 // 256M
-
 var _ envexec.Environment = &environ{}
 
 // environ defines interface to access container resources
 type environ struct {
 	container.Environment
 	cgPool CgroupPool
 	wd     *os.File // container work dir
+	cpuset string
 }
 
 // Destory destories the environment
@@ -45,6 +44,9 @@ func (c *environ) Execve(ctx context.Context, param envexec.ExecveParam) (envexe
 		if err != nil {
 			return nil, fmt.Errorf("execve: failed to get cgroup %v", err)
 		}
+		if c.cpuset != "" {
+			cg.SetCpuset(c.cpuset)
+		}
 		cg.SetMemoryLimit(param.Limit.Memory)
 		cg.SetProcLimit(param.Limit.Proc)
 		syncFunc = cg.AddProc
@@ -53,7 +55,7 @@ func (c *environ) Execve(ctx context.Context, param envexec.ExecveParam) (envexe
 	rLimits := rlimit.RLimits{
 		CPU:      uint64(param.Limit.Time.Truncate(time.Second)/time.Second) + 1,
 		Data:     param.Limit.Memory.Byte(),
-		FileSize: outputLimit,
+		FileSize: param.Limit.Output.Byte(),
 		Stack:    param.Limit.Stack.Byte(),
 	}
 

worker/worker.go

@@ -22,6 +22,7 @@ type Config struct {
 	WorkDir               string
 	TimeLimitTickInterval time.Duration
 	ExtraMemoryLimit      envexec.Size
+	OutputLimit           envexec.Size
 }
 
 // Worker defines interface for executor
@@ -41,6 +42,7 @@ type worker struct {
 
 	timeLimitTickInterval time.Duration
 	extraMemoryLimit      envexec.Size
+	outputLimit           envexec.Size
 
 	startOnce sync.Once
 	stopOnce  sync.Once
@@ -64,6 +66,7 @@ func New(conf Config) Worker {
 		workDir:               conf.WorkDir,
 		timeLimitTickInterval: conf.TimeLimitTickInterval,
 		extraMemoryLimit:      conf.ExtraMemoryLimit,
+		outputLimit:           conf.OutputLimit,
 	}
 }
 
@@ -285,6 +288,7 @@ func (w *worker) prepareCmd(rc Cmd) (*envexec.Cmd, map[string]bool, error) {
 		MemoryLimit:      envexec.Size(rc.MemoryLimit),
 		StackLimit:       envexec.Size(rc.StackLimit),
 		ExtraMemoryLimit: w.extraMemoryLimit,
+		OutputLimit:      w.outputLimit,
 		ProcLimit:        rc.ProcLimit,
 		CopyIn:           copyIn,
 		CopyOut:          copyOut,