From cab158535bcead180919c5b735d3d08dbf33f017 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Tue, 20 Aug 2024 11:09:33 -0400 Subject: [PATCH] enhance(flags): add compressed caching, log timestamp, and ignore path flags (#197) * enhance(flags): add compressed caching, log timestamp, and ignore path flags * fix reviewdog --- .github/workflows/reviewdog.yml | 4 +- cmd/vela-kaniko/build.go | 4 + cmd/vela-kaniko/main.go | 38 +++- cmd/vela-kaniko/plugin.go | 17 ++ cmd/vela-kaniko/plugin_test.go | 317 +++++++++++++++++--------------- cmd/vela-kaniko/repo.go | 2 + 6 files changed, 228 insertions(+), 154 deletions(-) diff --git a/.github/workflows/reviewdog.yml b/.github/workflows/reviewdog.yml index fcb4440..252b903 100644 --- a/.github/workflows/reviewdog.yml +++ b/.github/workflows/reviewdog.yml @@ -26,7 +26,7 @@ jobs: uses: reviewdog/action-golangci-lint@7708105983c614f7a2725e2172908b7709d1c3e4 # v2.6.2 with: github_token: ${{ secrets.github_token }} - golangci_lint_flags: "--config=.golangci.yml" + golangci_lint_flags: "--config=.golangci.yml --timeout=5m" fail_on_error: true filter_mode: diff_context reporter: github-pr-review @@ -50,6 +50,6 @@ jobs: uses: reviewdog/action-golangci-lint@7708105983c614f7a2725e2172908b7709d1c3e4 # v2.6.2 with: github_token: ${{ secrets.github_token }} - golangci_lint_flags: "--config=.golangci.yml" + golangci_lint_flags: "--config=.golangci.yml --timeout=5m" fail_on_error: false filter_mode: nofilter diff --git a/cmd/vela-kaniko/build.go b/cmd/vela-kaniko/build.go index 21443ee..4f4ac33 100644 --- a/cmd/vela-kaniko/build.go +++ b/cmd/vela-kaniko/build.go @@ -27,6 +27,10 @@ type Build struct { SingleSnapshot bool // https://github.com/GoogleContainerTools/kaniko#flag---ignore-var-run IgnoreVarRun bool + // https://github.com/GoogleContainerTools/kaniko#flag---ignore-path + IgnorePath []string + // https://github.com/GoogleContainerTools/kaniko#flag---log-timestamp + LogTimestamp bool } // SnapshotModeValues represents the available options for setting a snapshot mode. diff --git a/cmd/vela-kaniko/main.go b/cmd/vela-kaniko/main.go index 1f5be91..033f6d7 100644 --- a/cmd/vela-kaniko/main.go +++ b/cmd/vela-kaniko/main.go @@ -116,6 +116,18 @@ func main() { Usage: "By default Kaniko ignores /var/run when taking image snapshot. Include this parameter to preserve /var/run/* in destination image.", Value: true, }, + &cli.StringSliceFlag{ + EnvVars: []string{"PARAMETER_IGNORE_PATH", "KANIKO_IGNORE_PATH"}, + FilePath: "/vela/parameters/kaniko/ignore_path,/vela/secrets/kaniko/ignore_path", + Name: "build.ignore_path", + Usage: "ignore paths when taking image snapshot", + }, + &cli.BoolFlag{ + EnvVars: []string{"PARAMETER_LOG_TIMESTAMPS", "KANIKO_LOG_TIMESTAMPS"}, + FilePath: "/vela/parameters/kaniko/log_timestamps,/vela/secrets/kaniko/log_timestamps", + Name: "build.log_timestamps", + Usage: "enable timestamps in logs", + }, // Image Flags @@ -248,6 +260,13 @@ func main() { Name: "repo.compression_level", Usage: "set the compression level (1-9, inclusive)", }, + &cli.BoolFlag{ + EnvVars: []string{"PARAMETER_COMPRESSED_CACHING", "KANIKO_COMPRESSED_CACHING"}, + FilePath: "/vela/parameters/kaniko/compressed_caching,/vela/secrets/kaniko/compressed_caching", + Name: "repo.compressed_caching", + Usage: "when set to false, will prevent tar compression for cached layers", + Value: true, + }, &cli.StringFlag{ EnvVars: []string{"PARAMETER_REPO", "KANIKO_REPO"}, FilePath: "/vela/parameters/kaniko/repo,/vela/secrets/kaniko/repo", @@ -374,6 +393,8 @@ func run(c *cli.Context) error { TarPath: c.String("build.tar_path"), SingleSnapshot: c.Bool("build.single_snapshot"), IgnoreVarRun: c.Bool("build.ignore_var_run"), + IgnorePath: c.StringSlice("build.ignore_path"), + LogTimestamp: c.Bool("build.log_timestamps"), }, // image configuration Image: &Image{ @@ -398,14 +419,15 @@ func run(c *cli.Context) error { }, // repo configuration Repo: &Repo{ - AutoTag: c.Bool("repo.auto_tag"), - Cache: c.Bool("repo.cache"), - CacheName: c.String("repo.cache_name"), - Compression: c.String("repo.compression"), - CompressionLevel: c.Int("repo.compression_level"), - Name: c.String("repo.name"), - Tags: c.StringSlice("repo.tags"), - TopicsFilter: c.String("repo.topics_filter"), + AutoTag: c.Bool("repo.auto_tag"), + Cache: c.Bool("repo.cache"), + CacheName: c.String("repo.cache_name"), + Compression: c.String("repo.compression"), + CompressionLevel: c.Int("repo.compression_level"), + CompressedCaching: c.Bool("repo.compressed_caching"), + Name: c.String("repo.name"), + Tags: c.StringSlice("repo.tags"), + TopicsFilter: c.String("repo.topics_filter"), Label: &Label{ AuthorEmail: c.String("label.author_email"), Commit: c.String("label.commit"), diff --git a/cmd/vela-kaniko/plugin.go b/cmd/vela-kaniko/plugin.go index 8e67721..b9200ab 100644 --- a/cmd/vela-kaniko/plugin.go +++ b/cmd/vela-kaniko/plugin.go @@ -65,6 +65,18 @@ func (p *Plugin) Command() *exec.Cmd { flags = append(flags, fmt.Sprintf("--ignore-var-run=%s", strconv.FormatBool(p.Build.IgnoreVarRun))) + // add paths to be ignored if provided + if len(p.Build.IgnorePath) > 0 { + for _, path := range p.Build.IgnorePath { + flags = append(flags, fmt.Sprintf("--ignore-path=%s", path)) + } + } + + // add timestamps if enabled + if p.Build.LogTimestamp { + flags = append(flags, "--log-timestamp") + } + // iterate through all image build args for _, arg := range p.Image.Args { // add flag for build args from provided image build arg @@ -96,6 +108,11 @@ func (p *Plugin) Command() *exec.Cmd { flags = append(flags, fmt.Sprintf("--compression-level=%d", p.Repo.CompressionLevel)) } + // check if compressed caching is disabled + if !p.Repo.CompressedCaching { + flags = append(flags, "--compressed-caching=false") + } + // add flag for context from provided image context flags = append(flags, fmt.Sprintf("--context=%s", p.Image.Context)) diff --git a/cmd/vela-kaniko/plugin_test.go b/cmd/vela-kaniko/plugin_test.go index a27d946..42914a5 100644 --- a/cmd/vela-kaniko/plugin_test.go +++ b/cmd/vela-kaniko/plugin_test.go @@ -95,6 +95,8 @@ func TestDocker_Plugin_Command(t *testing.T) { Sha: "7fd1a60b01f91b314f59955a4e4d4e80d8edf11d", Tag: "v0.0.0", IgnoreVarRun: true, + IgnorePath: []string{"/tmp", ".git"}, + LogTimestamp: true, }, Image: &Image{ Args: []string{"foo=bar"}, @@ -113,18 +115,22 @@ func TestDocker_Plugin_Command(t *testing.T) { InsecurePush: true, }, Repo: &Repo{ - Cache: true, - CacheName: "index.docker.io/target/vela-kaniko", - Name: "index.docker.io/target/vela-kaniko", - Tags: []string{"latest"}, - AutoTag: true, - Label: testLabel(), + Cache: true, + CacheName: "index.docker.io/target/vela-kaniko", + Name: "index.docker.io/target/vela-kaniko", + Tags: []string{"latest"}, + AutoTag: true, + Label: testLabel(), + CompressedCaching: true, }, } want := exec.Command( kanikoBin, "--ignore-var-run=true", + "--ignore-path=/tmp", + "--ignore-path=.git", + "--log-timestamp", "--build-arg=foo=bar", "--cache", "--cache-repo=index.docker.io/target/vela-kaniko", @@ -184,12 +190,13 @@ func TestDocker_Plugin_Command_AutoTag_TagBuild(t *testing.T) { InsecurePush: true, }, Repo: &Repo{ - Cache: true, - CacheName: "index.docker.io/target/vela-kaniko", - Name: "index.docker.io/target/vela-kaniko", - Tags: []string{"latest"}, - AutoTag: true, - Label: testLabel(), + Cache: true, + CacheName: "index.docker.io/target/vela-kaniko", + Name: "index.docker.io/target/vela-kaniko", + Tags: []string{"latest"}, + AutoTag: true, + Label: testLabel(), + CompressedCaching: true, }, } @@ -259,12 +266,13 @@ func TestDocker_Plugin_Command_AutoTag_PushBuild(t *testing.T) { InsecurePush: true, }, Repo: &Repo{ - Cache: true, - CacheName: "index.docker.io/target/vela-kaniko", - Name: "index.docker.io/target/vela-kaniko", - Tags: []string{"latest"}, - AutoTag: true, - Label: testLabel(), + Cache: true, + CacheName: "index.docker.io/target/vela-kaniko", + Name: "index.docker.io/target/vela-kaniko", + Tags: []string{"latest"}, + AutoTag: true, + Label: testLabel(), + CompressedCaching: true, }, } @@ -330,13 +338,14 @@ func TestDocker_Plugin_Command_With_Labels(t *testing.T) { PushRetry: 1, }, Repo: &Repo{ - Cache: true, - CacheName: "index.docker.io/target/vela-kaniko", - Name: "index.docker.io/target/vela-kaniko", - Tags: []string{"latest"}, - AutoTag: true, - Label: testLabel(), - Labels: []string{"key1=tag1"}, + Cache: true, + CacheName: "index.docker.io/target/vela-kaniko", + Name: "index.docker.io/target/vela-kaniko", + Tags: []string{"latest"}, + AutoTag: true, + Label: testLabel(), + Labels: []string{"key1=tag1"}, + CompressedCaching: true, }, } @@ -397,12 +406,13 @@ func TestDocker_Plugin_Command_With_MultipleTopics(t *testing.T) { PushRetry: 1, }, Repo: &Repo{ - Cache: true, - CacheName: "index.docker.io/target/vela-kaniko", - Name: "index.docker.io/target/vela-kaniko", - Tags: []string{"latest"}, - AutoTag: true, - Label: label, + Cache: true, + CacheName: "index.docker.io/target/vela-kaniko", + Name: "index.docker.io/target/vela-kaniko", + Tags: []string{"latest"}, + AutoTag: true, + Label: label, + CompressedCaching: true, }, } @@ -462,13 +472,14 @@ func TestDocker_Plugin_Command_With_MultipleTopicsWithFilter(t *testing.T) { PushRetry: 1, }, Repo: &Repo{ - Cache: true, - CacheName: "index.docker.io/target/vela-kaniko", - Name: "index.docker.io/target/vela-kaniko", - Tags: []string{"latest"}, - AutoTag: true, - TopicsFilter: "^id", - Label: label, + Cache: true, + CacheName: "index.docker.io/target/vela-kaniko", + Name: "index.docker.io/target/vela-kaniko", + Tags: []string{"latest"}, + AutoTag: true, + TopicsFilter: "^id", + Label: label, + CompressedCaching: true, }, } @@ -528,12 +539,13 @@ func TestDocker_Plugin_Command_With_MultipleTopicsNoTopics(t *testing.T) { PushRetry: 1, }, Repo: &Repo{ - Cache: true, - CacheName: "index.docker.io/target/vela-kaniko", - Name: "index.docker.io/target/vela-kaniko", - Tags: []string{"latest"}, - AutoTag: true, - Label: label, + Cache: true, + CacheName: "index.docker.io/target/vela-kaniko", + Name: "index.docker.io/target/vela-kaniko", + Tags: []string{"latest"}, + AutoTag: true, + Label: label, + CompressedCaching: true, }, } @@ -591,12 +603,13 @@ func TestDocker_Plugin_Command_With_SnapshotMode(t *testing.T) { PushRetry: 1, }, Repo: &Repo{ - Cache: true, - CacheName: "index.docker.io/target/vela-kaniko", - Name: "index.docker.io/target/vela-kaniko", - Tags: []string{"latest"}, - AutoTag: true, - Label: testLabel(), + Cache: true, + CacheName: "index.docker.io/target/vela-kaniko", + Name: "index.docker.io/target/vela-kaniko", + Tags: []string{"latest"}, + AutoTag: true, + Label: testLabel(), + CompressedCaching: true, }, } @@ -657,12 +670,13 @@ func TestDocker_Plugin_Command_With_UseNewRun(t *testing.T) { PushRetry: 1, }, Repo: &Repo{ - Cache: true, - CacheName: "index.docker.io/target/vela-kaniko", - Name: "index.docker.io/target/vela-kaniko", - Tags: []string{"latest"}, - AutoTag: true, - Label: testLabel(), + Cache: true, + CacheName: "index.docker.io/target/vela-kaniko", + Name: "index.docker.io/target/vela-kaniko", + Tags: []string{"latest"}, + AutoTag: true, + Label: testLabel(), + CompressedCaching: true, }, } @@ -723,12 +737,13 @@ func TestDocker_Plugin_Command_With_TarPath(t *testing.T) { PushRetry: 1, }, Repo: &Repo{ - Cache: true, - CacheName: "index.docker.io/target/vela-kaniko", - Name: "index.docker.io/target/vela-kaniko", - Tags: []string{"latest"}, - AutoTag: true, - Label: testLabel(), + Cache: true, + CacheName: "index.docker.io/target/vela-kaniko", + Name: "index.docker.io/target/vela-kaniko", + Tags: []string{"latest"}, + AutoTag: true, + Label: testLabel(), + CompressedCaching: true, }, } @@ -789,12 +804,13 @@ func TestDocker_Plugin_Command_With_UseSingleSnapshot(t *testing.T) { PushRetry: 1, }, Repo: &Repo{ - Cache: true, - CacheName: "index.docker.io/target/vela-kaniko", - Name: "index.docker.io/target/vela-kaniko", - Tags: []string{"latest"}, - AutoTag: true, - Label: testLabel(), + Cache: true, + CacheName: "index.docker.io/target/vela-kaniko", + Name: "index.docker.io/target/vela-kaniko", + Tags: []string{"latest"}, + AutoTag: true, + Label: testLabel(), + CompressedCaching: true, }, } @@ -854,12 +870,13 @@ func TestDocker_Plugin_Command_With_UseIgnoreVarRunFalse(t *testing.T) { PushRetry: 1, }, Repo: &Repo{ - Cache: true, - CacheName: "index.docker.io/target/vela-kaniko", - Name: "index.docker.io/target/vela-kaniko", - Tags: []string{"latest"}, - AutoTag: true, - Label: testLabel(), + Cache: true, + CacheName: "index.docker.io/target/vela-kaniko", + Name: "index.docker.io/target/vela-kaniko", + Tags: []string{"latest"}, + AutoTag: true, + Label: testLabel(), + CompressedCaching: true, }, } @@ -918,12 +935,13 @@ func TestDocker_Plugin_Command_With_UseIgnoreVarRunTrue(t *testing.T) { PushRetry: 1, }, Repo: &Repo{ - Cache: true, - CacheName: "index.docker.io/target/vela-kaniko", - Name: "index.docker.io/target/vela-kaniko", - Tags: []string{"latest"}, - AutoTag: true, - Label: testLabel(), + Cache: true, + CacheName: "index.docker.io/target/vela-kaniko", + Name: "index.docker.io/target/vela-kaniko", + Tags: []string{"latest"}, + AutoTag: true, + Label: testLabel(), + CompressedCaching: true, }, } @@ -983,12 +1001,13 @@ func TestDocker_Plugin_Command_With_ForceBuildMetaData(t *testing.T) { PushRetry: 1, }, Repo: &Repo{ - Cache: true, - CacheName: "index.docker.io/target/vela-kaniko", - Name: "index.docker.io/target/vela-kaniko", - Tags: []string{"latest"}, - AutoTag: true, - Label: testLabel(), + Cache: true, + CacheName: "index.docker.io/target/vela-kaniko", + Name: "index.docker.io/target/vela-kaniko", + Tags: []string{"latest"}, + AutoTag: true, + Label: testLabel(), + CompressedCaching: true, }, } @@ -1049,12 +1068,13 @@ func TestDocker_Plugin_Command_With_Mirror(t *testing.T) { PushRetry: 1, }, Repo: &Repo{ - Cache: true, - CacheName: "index.docker.io/target/vela-kaniko", - Name: "index.docker.io/target/vela-kaniko", - Tags: []string{"latest"}, - AutoTag: true, - Label: testLabel(), + Cache: true, + CacheName: "index.docker.io/target/vela-kaniko", + Name: "index.docker.io/target/vela-kaniko", + Tags: []string{"latest"}, + AutoTag: true, + Label: testLabel(), + CompressedCaching: true, }, } @@ -1114,14 +1134,15 @@ func TestDocker_Plugin_Command_With_Compression(t *testing.T) { PushRetry: 1, }, Repo: &Repo{ - Cache: true, - CacheName: "index.docker.io/target/vela-kaniko", - Compression: "zstd", - CompressionLevel: 3, - Name: "index.docker.io/target/vela-kaniko", - Tags: []string{"latest"}, - AutoTag: true, - Label: testLabel(), + Cache: true, + CacheName: "index.docker.io/target/vela-kaniko", + Compression: "zstd", + CompressionLevel: 3, + Name: "index.docker.io/target/vela-kaniko", + Tags: []string{"latest"}, + AutoTag: true, + Label: testLabel(), + CompressedCaching: true, }, } @@ -1182,11 +1203,12 @@ func TestDocker_Plugin_Command_NoCacheRepo(t *testing.T) { PushRetry: 1, }, Repo: &Repo{ - Cache: true, - Name: "index.docker.io/target/vela-kaniko", - Tags: []string{"latest"}, - AutoTag: true, - Label: testLabel(), + Cache: true, + Name: "index.docker.io/target/vela-kaniko", + Tags: []string{"latest"}, + AutoTag: true, + Label: testLabel(), + CompressedCaching: true, }, } @@ -1244,12 +1266,13 @@ func TestDocker_Plugin_Command_NoDryRun(t *testing.T) { PushRetry: 1, }, Repo: &Repo{ - Cache: true, - CacheName: "index.docker.io/target/vela-kaniko", - Name: "index.docker.io/target/vela-kaniko", - Tags: []string{"latest"}, - AutoTag: true, - Label: testLabel(), + Cache: true, + CacheName: "index.docker.io/target/vela-kaniko", + Name: "index.docker.io/target/vela-kaniko", + Tags: []string{"latest"}, + AutoTag: true, + Label: testLabel(), + CompressedCaching: true, }, } @@ -1310,12 +1333,13 @@ func TestDocker_Plugin_Command_CustomPlatform(t *testing.T) { InsecurePush: true, }, Repo: &Repo{ - Cache: true, - CacheName: "index.docker.io/target/vela-kaniko", - Name: "index.docker.io/target/vela-kaniko", - Tags: []string{"latest"}, - AutoTag: true, - Label: testLabel(), + Cache: true, + CacheName: "index.docker.io/target/vela-kaniko", + Name: "index.docker.io/target/vela-kaniko", + Tags: []string{"latest"}, + AutoTag: true, + Label: testLabel(), + CompressedCaching: true, }, } @@ -1378,12 +1402,13 @@ func TestDocker_Plugin_Validate(t *testing.T) { DryRun: false, }, Repo: &Repo{ - Cache: true, - CacheName: "index.docker.io/target/vela-kaniko", - Name: "index.docker.io/target/vela-kaniko", - Tags: []string{"latest"}, - AutoTag: true, - Label: &Label{}, + Cache: true, + CacheName: "index.docker.io/target/vela-kaniko", + Name: "index.docker.io/target/vela-kaniko", + Tags: []string{"latest"}, + AutoTag: true, + Label: &Label{}, + CompressedCaching: true, }, } @@ -1410,11 +1435,12 @@ func TestDocker_Plugin_Validate_NoBuild(t *testing.T) { DryRun: false, }, Repo: &Repo{ - Cache: true, - CacheName: "index.docker.io/target/vela-kaniko", - Name: "index.docker.io/target/vela-kaniko", - Tags: []string{"latest"}, - AutoTag: true, + Cache: true, + CacheName: "index.docker.io/target/vela-kaniko", + Name: "index.docker.io/target/vela-kaniko", + Tags: []string{"latest"}, + AutoTag: true, + CompressedCaching: true, }, } @@ -1447,12 +1473,13 @@ func TestDocker_Plugin_Validate_AutoTag_InvalidBuildTag(t *testing.T) { DryRun: false, }, Repo: &Repo{ - Cache: true, - CacheName: "index.docker.io/target/vela-kaniko", - Name: "index.docker.io/target/vela-kaniko", - Tags: []string{"latest"}, - AutoTag: true, - Label: &Label{}, + Cache: true, + CacheName: "index.docker.io/target/vela-kaniko", + Name: "index.docker.io/target/vela-kaniko", + Tags: []string{"latest"}, + AutoTag: true, + Label: &Label{}, + CompressedCaching: true, }, } @@ -1482,11 +1509,12 @@ func TestDocker_Plugin_Validate_NoImage(t *testing.T) { DryRun: false, }, Repo: &Repo{ - Cache: true, - CacheName: "index.docker.io/target/vela-kaniko", - Name: "index.docker.io/target/vela-kaniko", - Tags: []string{"latest"}, - AutoTag: true, + Cache: true, + CacheName: "index.docker.io/target/vela-kaniko", + Name: "index.docker.io/target/vela-kaniko", + Tags: []string{"latest"}, + AutoTag: true, + CompressedCaching: true, }, } @@ -1513,11 +1541,12 @@ func TestDocker_Plugin_Validate_NoRegistry(t *testing.T) { }, Registry: &Registry{}, Repo: &Repo{ - Cache: true, - CacheName: "index.docker.io/target/vela-kaniko", - Name: "index.docker.io/target/vela-kaniko", - Tags: []string{"latest"}, - AutoTag: true, + Cache: true, + CacheName: "index.docker.io/target/vela-kaniko", + Name: "index.docker.io/target/vela-kaniko", + Tags: []string{"latest"}, + AutoTag: true, + CompressedCaching: true, }, } diff --git a/cmd/vela-kaniko/repo.go b/cmd/vela-kaniko/repo.go index 450439f..40070aa 100644 --- a/cmd/vela-kaniko/repo.go +++ b/cmd/vela-kaniko/repo.go @@ -23,6 +23,8 @@ type ( Compression string // level of compression - 1 to 9 (inclusive) CompressionLevel int + // prevent tar compression for cached layers + CompressedCaching bool // used for translating the pre-defined image labels Label *Label // labels of the image for the repository