From eba44b478c3f9c67be765aa3c31042d7d7243251 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 15 Nov 2024 16:21:23 +0100 Subject: [PATCH] wip: progress reporting via jsonseq-monitor --- cmd/image-builder/build.go | 100 ++++++++++++++++++++++++++++++++-- cmd/image-builder/manifest.go | 4 ++ go.mod | 4 ++ go.sum | 5 ++ 4 files changed, 109 insertions(+), 4 deletions(-) diff --git a/cmd/image-builder/build.go b/cmd/image-builder/build.go index 038f6c3772..6442235457 100644 --- a/cmd/image-builder/build.go +++ b/cmd/image-builder/build.go @@ -5,12 +5,106 @@ import ( "fmt" "io" "os" + "os/exec" "path/filepath" + "strings" + + "github.com/cheggaaa/pb/v3" "github.com/osbuild/images/pkg/arch" - "github.com/osbuild/images/pkg/osbuild" + "github.com/osbuild/images/pkg/osbuildmonitor" ) +// XXX: merge back into images/pkg/osbuild/osbuild-exec.go or +// into osbuildmonitor +func runOSBuild(manifest []byte, store, outputDirectory string, exports, extraEnv []string) error { + rp, wp, err := os.Pipe() + if err != nil { + return fmt.Errorf("cannot create pipe for osbuild: %w", err) + } + defer rp.Close() + defer wp.Close() + + cmd := exec.Command( + "osbuild", + "--store", store, + "--output-directory", outputDirectory, + "--monitor=JSONSeqMonitor", + "--monitor-fd=3", + "-", + ) + for _, export := range exports { + cmd.Args = append(cmd.Args, "--export", export) + } + + cmd.Env = append(os.Environ(), extraEnv...) + cmd.Stdin = bytes.NewBuffer(manifest) + cmd.Stderr = os.Stderr + // we could use "--json" here and would get the build-result + // exported here + cmd.Stdout = nil + cmd.ExtraFiles = []*os.File{wp} + + if err := cmd.Start(); err != nil { + return fmt.Errorf("error starting osbuild: %v", err) + } + wp.Close() + + // XXX: extract helper + spinnerPb := pb.New(0) + spinnerPb.SetTemplate(`Building [{{ (cycle . "|" "/" "-" "\\") }}]`) + mainPb := pb.New(0) + progressBarTmplFmt := `[{{ counters . }}] %s: {{ bar .}} {{ percent . }}` + mainPb.SetTemplateString(fmt.Sprintf(progressBarTmplFmt, "step")) + subPb := pb.New(0) + subPb.SetTemplateString(fmt.Sprintf(progressBarTmplFmt, "module")) + + tracePb := pb.New(0) + tracePb.SetTemplate(`last msg: {{ string . "msg" }}`) + + pool, err := pb.StartPool(spinnerPb, mainPb, subPb, tracePb) + if err != nil { + return err + } + defer pool.Stop() + + buildLog := bytes.NewBuffer(nil) + scanner := osbuildmonitor.NewStatusScanner(rp) + for { + status, err := scanner.Status() + if err != nil { + return err + } + if status == nil { + break + } + + if status.Progress.Total > 0 { + mainPb.SetTotal(int64(status.Progress.Total + 1)) + mainPb.SetCurrent(int64(status.Progress.Done + 1)) + } + // XXX: support more levels + if status.Progress.SubProgress != nil && status.Progress.SubProgress.Total > 0 { + subPb.SetTotal(int64(status.Progress.SubProgress.Total + 1)) + subPb.SetCurrent(int64(status.Progress.SubProgress.Done + 1)) + } + // XXX: this is ugly, lines that are too long cause scrolling + // and the messages are too low-level anyway, find something + // nicer + cutoff := min(len(status.Trace), 60) + tracePb.Set("msg", strings.TrimSpace(status.Trace)[:cutoff]) + + // log all traces + fmt.Fprintf(buildLog, "[%s] %s\n", status.Timestamp.Format("2006-01-02 15:04:05"), status.Trace) + } + + if err := cmd.Wait(); err != nil { + return fmt.Errorf("error running osbuild: %w\nfull log:\n%s", err, buildLog.String()) + } + + return nil +} + func buildImage(out io.Writer, distroName, imgTypeStr, outputFilename string) error { // cross arch building is not possible, we would have to download // a pre-populated buildroot (tar,container) with rpm for that @@ -33,7 +127,5 @@ func buildImage(out io.Writer, distroName, imgTypeStr, outputFilename string) er outputDir := "." buildName := fmt.Sprintf("%s-%s-%s", distroName, imgTypeStr, archStr) jobOutputDir := filepath.Join(outputDir, buildName) - // XXX: support stremaing via statusWriter - _, err = osbuild.RunOSBuild(mf.Bytes(), osbuildStoreDir, jobOutputDir, imgType.Exports(), nil, nil, false, os.Stderr) - return err + return runOSBuild(mf.Bytes(), osbuildStoreDir, jobOutputDir, imgType.Exports(), nil) } diff --git a/cmd/image-builder/manifest.go b/cmd/image-builder/manifest.go index b597e09512..ffcf7111dc 100644 --- a/cmd/image-builder/manifest.go +++ b/cmd/image-builder/manifest.go @@ -72,7 +72,11 @@ func outputManifest(out io.Writer, distroName, imgTypeStr, archStr string, opts if err != nil { return err } + + // XXX: create progress abstraction for this too + println("depsolve") packageSpecs, _, err := depsolve(cacheDir, preManifest.GetPackageSetChains(), dist, archStr) + println("depsolve done") if err != nil { return err } diff --git a/go.mod b/go.mod index c60fabc771..6eaef97b3c 100644 --- a/go.mod +++ b/go.mod @@ -68,6 +68,7 @@ require ( github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cheggaaa/pb/v3 v3.1.5 // indirect github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect github.com/containerd/cgroups/v3 v3.0.3 // indirect github.com/containerd/errdefs v0.1.0 // indirect @@ -87,6 +88,7 @@ require ( github.com/dougm/pretty v0.0.0-20171025230240-2ee9d7453c02 // indirect github.com/envoyproxy/go-control-plane v0.13.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect + github.com/fatih/color v1.16.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-jose/go-jose/v4 v4.0.2 // indirect github.com/go-logr/logr v1.4.2 // indirect @@ -122,6 +124,8 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/letsencrypt/boulder v0.0.0-20240418210053-89b07f4543e0 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/miekg/pkcs11 v1.1.1 // indirect diff --git a/go.sum b/go.sum index 193ea09fab..e3f7212d54 100644 --- a/go.sum +++ b/go.sum @@ -100,6 +100,8 @@ github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMr github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cheggaaa/pb/v3 v3.1.5 h1:QuuUzeM2WsAqG2gMqtzaWithDJv0i+i6UlnwSCI4QLk= +github.com/cheggaaa/pb/v3 v3.1.5/go.mod h1:CrxkeghYTXi1lQBEI7jSn+3svI3cuc19haAj6jM60XI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 h1:QVw89YDxXxEe+l8gU8ETbOasdwEV+avkR75ZzsVV9WI= @@ -293,6 +295,7 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= @@ -511,7 +514,9 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=