From e1f8c954e0ef02c84fc3f9a59a978cd51622e6c8 Mon Sep 17 00:00:00 2001 From: criyle Date: Sun, 6 Dec 2020 18:45:56 -0800 Subject: [PATCH] Add configuration to OutputLimit & CpuSet --- README.md | 2 ++ cmd/executorserver/config/config.go | 2 ++ cmd/executorserver/main.go | 2 ++ env/config.go | 1 + env/env_linux.go | 8 ++++++-- go.mod | 4 ++-- go.sum | 8 ++++---- pkg/envexec/cmd.go | 1 + pkg/envexec/interface.go | 1 + pkg/pool/cgroup_wrapper_linux.go | 4 ++++ pkg/pool/cgrouppool_linux.go | 1 + pkg/pool/envbuilder_linux.go | 5 ++++- pkg/pool/environment_linux.go | 8 +++++--- worker/worker.go | 4 ++++ 14 files changed, 39 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 92d5da2..e303851 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,8 @@ The `executorserver` need root privilege to create `cgroup`. Either creates sub- - `-auth-token` to add token-based authentication to REST / gRPC - `-src-prefix` to restrict `src` copyIn path (need to be absolute path) - `-time-limit-checker-interval` specifies time limit checker interval (default 100ms) +- `-output-limit` specifies size limit of POSIX rlimit of output +- `-cpuset` specifies `cpuset.cpus` cgroup for each container #### Environment Variables diff --git a/cmd/executorserver/config/config.go b/cmd/executorserver/config/config.go index 6c9df3f..24beb4f 100644 --- a/cmd/executorserver/config/config.go +++ b/cmd/executorserver/config/config.go @@ -26,6 +26,8 @@ type Config struct { // runner limit TimeLimitCheckerInterval time.Duration `flagUsage:"specifies time limit checker interval" default:"100ms"` ExtraMemoryLimit *envexec.Size `flagUsage:"specifies extra memory buffer for check memory limit" default:"16k"` + OutputLimit *envexec.Size `flagUsage:"specifies POSIX rlimit for output for each command" default:"256m"` + Cpuset string `flagUsage:"control the usage of cpuset for all containerd process"` // server config HTTPAddr string `flagUsage:"specifies the http binding address" default:":5050"` diff --git a/cmd/executorserver/main.go b/cmd/executorserver/main.go index 23e33b3..256e2de 100644 --- a/cmd/executorserver/main.go +++ b/cmd/executorserver/main.go @@ -75,6 +75,7 @@ func main() { TmpFsParam: conf.TmpFsParam, NetShare: conf.NetShare, CgroupPrefix: conf.CgroupPrefix, + Cpuset: conf.Cpuset, Logger: logger.Sugar(), }) if err != nil { @@ -102,6 +103,7 @@ func main() { WorkDir: conf.Dir, TimeLimitTickInterval: conf.TimeLimitCheckerInterval, ExtraMemoryLimit: *conf.ExtraMemoryLimit, + OutputLimit: *conf.OutputLimit, }) work.Start() logger.Sugar().Infof("Starting worker with parallelism=%d, workdir=%s, timeLimitCheckInterval=%v", diff --git a/env/config.go b/env/config.go index 318a2c1..8ccdbe6 100644 --- a/env/config.go +++ b/env/config.go @@ -15,5 +15,6 @@ type Config struct { NetShare bool MountConf string CgroupPrefix string + Cpuset string Logger } diff --git a/env/env_linux.go b/env/env_linux.go index 5debde1..ae1c765 100644 --- a/env/env_linux.go +++ b/env/env_linux.go @@ -75,7 +75,11 @@ func NewBuilder(c Config) (pool.EnvBuilder, error) { DomainName: domainName, WorkDir: workDir, } - cgb, err := cgroup.NewBuilder(c.CgroupPrefix).WithCPUAcct().WithMemory().WithPids().FilterByEnv() + cgb := cgroup.NewBuilder(c.CgroupPrefix).WithCPUAcct().WithMemory().WithPids() + if c.Cpuset != "" { + cgb = cgb.WithCPUSet() + } + cgb, err = cgb.FilterByEnv() if err != nil { return nil, err } @@ -92,7 +96,7 @@ func NewBuilder(c Config) (pool.EnvBuilder, error) { if cgb != nil { cgroupPool = pool.NewFakeCgroupPool(cgb) } - return pool.NewEnvBuilder(b, cgroupPool, workDir), nil + return pool.NewEnvBuilder(b, cgroupPool, workDir, c.Cpuset), nil } type credGen struct { diff --git a/go.mod b/go.mod index 178fbeb..1ef06f1 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.14 require ( cloud.google.com/go v0.73.0 // indirect github.com/creack/pty v1.1.11 - github.com/criyle/go-sandbox v0.5.2 + github.com/criyle/go-sandbox v0.5.4 github.com/fatih/camelcase v1.0.0 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/gin-contrib/pprof v1.3.0 @@ -30,7 +30,7 @@ require ( golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 golang.org/x/sys v0.0.0-20201204225414-ed752295db88 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect - golang.org/x/tools v0.0.0-20201204222352-654352759326 // indirect + golang.org/x/tools v0.0.0-20201206230334-368bee879bfd // indirect google.golang.org/genproto v0.0.0-20201204160425-06b3db808446 // indirect google.golang.org/grpc v1.34.0 google.golang.org/grpc/examples v0.0.0-20201204235607-0d6a24f68a5f // indirect diff --git a/go.sum b/go.sum index 49833d6..33b92dd 100644 --- a/go.sum +++ b/go.sum @@ -85,8 +85,8 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/criyle/go-sandbox v0.5.2 h1:P6I5vE3dHq6peu3PXBHXpahimKJHnbt/vbV3Eby0qz4= -github.com/criyle/go-sandbox v0.5.2/go.mod h1:6iDDyseyHVugTM2ixPleIBuRx3pVuMKh2qTm6IRmvd8= +github.com/criyle/go-sandbox v0.5.4 h1:6CNSbw3zjCyI8PxGhOH0Liz9F05P2UDGTW4uvb3DS+8= +github.com/criyle/go-sandbox v0.5.4/go.mod h1:6iDDyseyHVugTM2ixPleIBuRx3pVuMKh2qTm6IRmvd8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -710,8 +710,8 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd h1:kJP9fbfkpUoA4y03Nxor8be golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201202200335-bef1c476418a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201204222352-654352759326 h1:XKLw9EEEfGJFE2K5Ni4nXgtFBIfI+zKPIi2SlRYmIG4= -golang.org/x/tools v0.0.0-20201204222352-654352759326/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201206230334-368bee879bfd h1:EqFvKLTxjH6gEy2baWxX2AgJwZkBIDIcZFYcoYlI9RA= +golang.org/x/tools v0.0.0-20201206230334-368bee879bfd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/pkg/envexec/cmd.go b/pkg/envexec/cmd.go index 3c3445d..723a827 100644 --- a/pkg/envexec/cmd.go +++ b/pkg/envexec/cmd.go @@ -33,6 +33,7 @@ type Cmd struct { MemoryLimit Size StackLimit Size ExtraMemoryLimit Size + OutputLimit Size ProcLimit uint64 // file contents to copyin before exec diff --git a/pkg/envexec/interface.go b/pkg/envexec/interface.go index f4dd2dd..1093a38 100644 --- a/pkg/envexec/interface.go +++ b/pkg/envexec/interface.go @@ -33,6 +33,7 @@ type Limit struct { Memory Size // Memory limit Proc uint64 // Process count limit Stack Size // Stack limit + Output Size // Output limit } // Usage defines the peak process resource usage diff --git a/pkg/pool/cgroup_wrapper_linux.go b/pkg/pool/cgroup_wrapper_linux.go index 07bae97..138c2a3 100644 --- a/pkg/pool/cgroup_wrapper_linux.go +++ b/pkg/pool/cgroup_wrapper_linux.go @@ -13,6 +13,10 @@ var ( type wCgroup cgroup.Cgroup +func (c *wCgroup) SetCpuset(s string) error { + return (*cgroup.Cgroup)(c).SetCpusetCpus([]byte(s)) +} + func (c *wCgroup) SetMemoryLimit(s envexec.Size) error { return (*cgroup.Cgroup)(c).SetMemoryLimitInBytes(uint64(s)) } diff --git a/pkg/pool/cgrouppool_linux.go b/pkg/pool/cgrouppool_linux.go index d6951c3..58205b6 100644 --- a/pkg/pool/cgrouppool_linux.go +++ b/pkg/pool/cgrouppool_linux.go @@ -9,6 +9,7 @@ import ( // Cgroup defines interface to limit and monitor resources consumption of a process type Cgroup interface { + SetCpuset(string) error SetMemoryLimit(envexec.Size) error SetProcLimit(uint64) error diff --git a/pkg/pool/envbuilder_linux.go b/pkg/pool/envbuilder_linux.go index 5f10f1e..5e73149 100644 --- a/pkg/pool/envbuilder_linux.go +++ b/pkg/pool/envbuilder_linux.go @@ -11,14 +11,16 @@ type environmentBuilder struct { builder EnvironmentBuilder cgPool CgroupPool workDir string + cpuset string } // NewEnvBuilder creates builder for linux container pools -func NewEnvBuilder(builder EnvironmentBuilder, cgPool CgroupPool, workDir string) EnvBuilder { +func NewEnvBuilder(builder EnvironmentBuilder, cgPool CgroupPool, workDir, cpuset string) EnvBuilder { return &environmentBuilder{ builder: builder, cgPool: cgPool, workDir: workDir, + cpuset: cpuset, } } @@ -40,5 +42,6 @@ func (b *environmentBuilder) Build() (Environment, error) { Environment: m, cgPool: b.cgPool, wd: wd[0], + cpuset: b.cpuset, }, nil } diff --git a/pkg/pool/environment_linux.go b/pkg/pool/environment_linux.go index a879af4..9ea6e95 100644 --- a/pkg/pool/environment_linux.go +++ b/pkg/pool/environment_linux.go @@ -12,8 +12,6 @@ import ( "github.com/criyle/go-sandbox/pkg/rlimit" ) -const outputLimit = 256 << 20 // 256M - var _ envexec.Environment = &environ{} // environ defines interface to access container resources @@ -21,6 +19,7 @@ type environ struct { container.Environment cgPool CgroupPool wd *os.File // container work dir + cpuset string } // Destory destories the environment @@ -45,6 +44,9 @@ func (c *environ) Execve(ctx context.Context, param envexec.ExecveParam) (envexe if err != nil { return nil, fmt.Errorf("execve: failed to get cgroup %v", err) } + if c.cpuset != "" { + cg.SetCpuset(c.cpuset) + } cg.SetMemoryLimit(param.Limit.Memory) cg.SetProcLimit(param.Limit.Proc) syncFunc = cg.AddProc @@ -53,7 +55,7 @@ func (c *environ) Execve(ctx context.Context, param envexec.ExecveParam) (envexe rLimits := rlimit.RLimits{ CPU: uint64(param.Limit.Time.Truncate(time.Second)/time.Second) + 1, Data: param.Limit.Memory.Byte(), - FileSize: outputLimit, + FileSize: param.Limit.Output.Byte(), Stack: param.Limit.Stack.Byte(), } diff --git a/worker/worker.go b/worker/worker.go index 40c8fcd..60911fa 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -22,6 +22,7 @@ type Config struct { WorkDir string TimeLimitTickInterval time.Duration ExtraMemoryLimit envexec.Size + OutputLimit envexec.Size } // Worker defines interface for executor @@ -41,6 +42,7 @@ type worker struct { timeLimitTickInterval time.Duration extraMemoryLimit envexec.Size + outputLimit envexec.Size startOnce sync.Once stopOnce sync.Once @@ -64,6 +66,7 @@ func New(conf Config) Worker { workDir: conf.WorkDir, timeLimitTickInterval: conf.TimeLimitTickInterval, extraMemoryLimit: conf.ExtraMemoryLimit, + outputLimit: conf.OutputLimit, } } @@ -285,6 +288,7 @@ func (w *worker) prepareCmd(rc Cmd) (*envexec.Cmd, map[string]bool, error) { MemoryLimit: envexec.Size(rc.MemoryLimit), StackLimit: envexec.Size(rc.StackLimit), ExtraMemoryLimit: w.extraMemoryLimit, + OutputLimit: w.outputLimit, ProcLimit: rc.ProcLimit, CopyIn: copyIn, CopyOut: copyOut,