Allow using anchored diff for large resources

Add --diff-anchored flag

Signed-off-by: Praveen Rewar <[email protected]>

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/cppforlife/go-patch v0.0.0-20240118020416-2147782e467b
 	github.com/google/go-cmp v0.5.9
 	github.com/hashicorp/go-version v1.6.0
-	github.com/k14s/difflib v0.0.0-20201117154628-0c031775bf57
+	github.com/k14s/difflib v0.0.0-20240118055029-596a7a5585c3
 	github.com/k14s/ytt v0.36.0
 	github.com/mitchellh/go-wordwrap v1.0.1
 	github.com/spf13/cobra v1.7.0

go.sum

@@ -169,8 +169,9 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/k14s/difflib v0.0.0-20201117154628-0c031775bf57 h1:CwBRArr+BWBopnUJhDjJw86rPL/jGbEjfHWKzTasSqE=
 github.com/k14s/difflib v0.0.0-20201117154628-0c031775bf57/go.mod h1:B0xN2MiNBGWOWi9CcfAo9LBI8IU4J1utlbOIJCsmKr4=
+github.com/k14s/difflib v0.0.0-20240118055029-596a7a5585c3 h1:q2ikACDbDDbyUcN9JkDcNMGhIx1EBRkctAsPZMr35qM=
+github.com/k14s/difflib v0.0.0-20240118055029-596a7a5585c3/go.mod h1:B0xN2MiNBGWOWi9CcfAo9LBI8IU4J1utlbOIJCsmKr4=
 github.com/k14s/starlark-go v0.0.0-20200720175618-3a5c849cc368 h1:4bcRTTSx+LKSxMWibIwzHnDNmaN1x52oEpvnjCy+8vk=
 github.com/k14s/starlark-go v0.0.0-20200720175618-3a5c849cc368/go.mod h1:lKGj1op99m4GtQISxoD2t+K+WO/q2NzEPKvfXFQfbCA=
 github.com/k14s/ytt v0.36.0 h1:ERr7q+r3ziYJv91fvTx2b76d1MIo3SI/EsAS01WU+Zo=

pkg/kapp/cmd/app/delete.go

@@ -233,7 +233,7 @@ func (o *DeleteOptions) calculateAndPresentChanges(existingResources []ctlres.Re
 	)
 
 	{ // Figure out changes for X existing resources -> 0 new resources
-		changeFactory := ctldiff.NewChangeFactory(nil, nil, nil)
+		changeFactory := ctldiff.NewChangeFactory(nil, nil, nil, ctldiff.ChangeOpts{o.DiffFlags.AnchoredDiff})
 		changeSetFactory := ctldiff.NewChangeSetFactory(o.DiffFlags.ChangeSetOpts, changeFactory)
 
 		changes, err := changeSetFactory.New(existingResources, nil).Calculate()

pkg/kapp/cmd/app/deploy.go

@@ -410,7 +410,7 @@ func (o *DeployOptions) calculateAndPresentChanges(existingResources,
 	var clusterChangeSet ctlcap.ClusterChangeSet
 
 	{ // Figure out changes for X existing resources -> X new resources
-		changeFactory := ctldiff.NewChangeFactory(conf.RebaseMods(), conf.DiffAgainstLastAppliedFieldExclusionMods(), conf.DiffAgainstExistingFieldExclusionMods())
+		changeFactory := ctldiff.NewChangeFactory(conf.RebaseMods(), conf.DiffAgainstLastAppliedFieldExclusionMods(), conf.DiffAgainstExistingFieldExclusionMods(), ctldiff.ChangeOpts{o.DiffFlags.AnchoredDiff})
 		changeSetFactory := ctldiff.NewChangeSetFactory(o.DiffFlags.ChangeSetOpts, changeFactory)
 
 		err := ctldiff.NewRenewableResources(existingResources, newResources).Prepare()

pkg/kapp/cmd/tools/diff.go

@@ -53,7 +53,7 @@ func (o *DiffOptions) Run() error {
 		return err
 	}
 
-	changeFactory := ctldiff.NewChangeFactory(nil, nil, nil)
+	changeFactory := ctldiff.NewChangeFactory(nil, nil, nil, ctldiff.ChangeOpts{o.DiffFlags.AnchoredDiff})
 
 	changes, err := ctldiff.NewChangeSet(existingResources, newResources, o.DiffFlags.ChangeSetOpts, changeFactory).Calculate()
 	if err != nil {

pkg/kapp/cmd/tools/diff_flags.go

@@ -17,6 +17,8 @@ type DiffFlags struct {
 	Run        bool
 	ExitStatus bool
 	UI         bool
+
+	AnchoredDiff bool
 }
 
 func (s *DiffFlags) SetWithPrefix(prefix string, cmd *cobra.Command) {
@@ -39,4 +41,6 @@ func (s *DiffFlags) SetWithPrefix(prefix string, cmd *cobra.Command) {
 
 	cmd.Flags().StringVar(&s.Filter, prefix+"filter", "", `Set changes filter (example: {"and":[{"ops":["update"]},{"existingResource":{"kinds":["Deployment"]}]})`)
 	cmd.Flags().BoolVar(&s.ChangesYAML, prefix+"changes-yaml", false, "Print YAML to be applied")
+
+	cmd.Flags().BoolVar(&s.AnchoredDiff, prefix+"anchored", false, "Allow using anchored diff for large resources")
 }

pkg/kapp/config/default_test.go

@@ -15,7 +15,7 @@ import (
 func TestDefaultTemplateRules(t *testing.T) {
 	_, defaultConfig, err := config.NewConfFromResourcesWithDefaults([]ctlres.Resource{})
 	require.NoError(t, err)
-	changeFactory := ctldiff.NewChangeFactory(defaultConfig.RebaseMods(), defaultConfig.DiffAgainstLastAppliedFieldExclusionMods(), defaultConfig.DiffAgainstExistingFieldExclusionMods())
+	changeFactory := ctldiff.NewChangeFactory(defaultConfig.RebaseMods(), defaultConfig.DiffAgainstLastAppliedFieldExclusionMods(), defaultConfig.DiffAgainstExistingFieldExclusionMods(), ctldiff.ChangeOpts{false})
 
 	testCases := []struct {
 		description  string

pkg/kapp/diff/change.go

@@ -45,11 +45,13 @@ type ChangeImpl struct {
 	configurableTextDiff *ConfigurableTextDiff
 	opsDiff              *OpsDiff
 	changeOpVal          ChangeOp
+
+	opts ChangeOpts
 }
 
 var _ Change = &ChangeImpl{}
 
-func NewChange(existingRes, newRes, appliedRes, clusterOriginalRes ctlres.Resource) *ChangeImpl {
+func NewChange(existingRes, newRes, appliedRes, clusterOriginalRes ctlres.Resource, opts ChangeOpts) *ChangeImpl {
 	if existingRes == nil && newRes == nil {
 		panic("Expected either existingRes or newRes be non-nil")
 	}
@@ -67,7 +69,7 @@ func NewChange(existingRes, newRes, appliedRes, clusterOriginalRes ctlres.Resour
 		clusterOriginalRes = clusterOriginalRes.DeepCopy()
 	}
 
-	return &ChangeImpl{existingRes: existingRes, newRes: newRes, appliedRes: appliedRes, clusterOriginalRes: clusterOriginalRes}
+	return &ChangeImpl{existingRes: existingRes, newRes: newRes, appliedRes: appliedRes, clusterOriginalRes: clusterOriginalRes, opts: opts}
 }
 
 func (d *ChangeImpl) NewOrExistingResource() ctlres.Resource {
@@ -129,7 +131,7 @@ func (d *ChangeImpl) isIgnoredTransient() bool {
 func (d *ChangeImpl) ConfigurableTextDiff() *ConfigurableTextDiff {
 	// diff is called very often, so memoize
 	if d.configurableTextDiff == nil {
-		d.configurableTextDiff = NewConfigurableTextDiff(d.existingRes, d.newRes, d.IsIgnored())
+		d.configurableTextDiff = NewConfigurableTextDiff(d.existingRes, d.newRes, d.IsIgnored(), d.opts)
 	}
 	return d.configurableTextDiff
 }

pkg/kapp/diff/change_factory.go

@@ -11,12 +11,17 @@ type ChangeFactory struct {
 	rebaseMods                               []ctlres.ResourceModWithMultiple
 	diffAgainstLastAppliedFieldExclusionMods []ctlres.FieldRemoveMod
 	diffAgainstExistingFieldExclusionRules   []ctlres.FieldRemoveMod
+	opts                                     ChangeOpts
+}
+
+type ChangeOpts struct {
+	AllowAnchoredDiff bool
 }
 
 func NewChangeFactory(rebaseMods []ctlres.ResourceModWithMultiple,
-	diffAgainstLastAppliedFieldExclusionMods []ctlres.FieldRemoveMod, diffAgainstExistingFieldExclusionRules []ctlres.FieldRemoveMod) ChangeFactory {
+	diffAgainstLastAppliedFieldExclusionMods []ctlres.FieldRemoveMod, diffAgainstExistingFieldExclusionRules []ctlres.FieldRemoveMod, opts ChangeOpts) ChangeFactory {
 
-	return ChangeFactory{rebaseMods, diffAgainstLastAppliedFieldExclusionMods, diffAgainstExistingFieldExclusionRules}
+	return ChangeFactory{rebaseMods, diffAgainstLastAppliedFieldExclusionMods, diffAgainstExistingFieldExclusionRules, opts}
 }
 
 func (f ChangeFactory) NewChangeAgainstLastApplied(existingRes, newRes ctlres.Resource) (Change, error) {
@@ -59,7 +64,7 @@ func (f ChangeFactory) NewChangeAgainstLastApplied(existingRes, newRes ctlres.Re
 		return nil, err
 	}
 
-	return NewChange(existingRes, rebasedNewRes, newRes, existingResForRebasing), nil
+	return NewChange(existingRes, rebasedNewRes, newRes, existingResForRebasing, f.opts), nil
 }
 
 func (f ChangeFactory) NewExactChange(existingRes, newRes ctlres.Resource) (Change, error) {
@@ -86,7 +91,7 @@ func (f ChangeFactory) NewExactChange(existingRes, newRes ctlres.Resource) (Chan
 		return nil, err
 	}
 
-	return NewChange(existingRes, rebasedNewRes, newRes, existingRes), nil
+	return NewChange(existingRes, rebasedNewRes, newRes, existingRes, f.opts), nil
 }
 
 func (f ChangeFactory) NewResourceWithHistory(resource ctlres.Resource) ResourceWithHistory {

pkg/kapp/diff/change_set_test.go

@@ -48,7 +48,7 @@ metadata:
 		},
 	}
 
-	changeFactory := ctldiff.NewChangeFactory(mods, nil, nil)
+	changeFactory := ctldiff.NewChangeFactory(mods, nil, nil, ctldiff.ChangeOpts{false})
 	changeSet := ctldiff.NewChangeSet([]ctlres.Resource{existingRes}, []ctlres.Resource{newRes},
 		ctldiff.ChangeSetOpts{}, changeFactory)
 
@@ -106,7 +106,7 @@ metadata:
 		},
 	}
 
-	changeFactory := ctldiff.NewChangeFactory(mods, nil, nil)
+	changeFactory := ctldiff.NewChangeFactory(mods, nil, nil, ctldiff.ChangeOpts{false})
 	changeSet := ctldiff.NewChangeSet([]ctlres.Resource{existingRes}, []ctlres.Resource{newRes},
 		ctldiff.ChangeSetOpts{}, changeFactory)
 
@@ -174,7 +174,7 @@ metadata:
 		},
 	}
 
-	changeFactory := ctldiff.NewChangeFactory(rebaseMods, ignoreFieldsMods, nil)
+	changeFactory := ctldiff.NewChangeFactory(rebaseMods, ignoreFieldsMods, nil, ctldiff.ChangeOpts{false})
 	changeSet := ctldiff.NewChangeSet([]ctlres.Resource{existingRes}, []ctlres.Resource{newRes},
 		ctldiff.ChangeSetOpts{AgainstLastApplied: true}, changeFactory)
 
@@ -246,7 +246,7 @@ metadata:
 		},
 	}
 
-	changeFactory := ctldiff.NewChangeFactory(rebaseMods, ignoreFieldsMods, nil)
+	changeFactory := ctldiff.NewChangeFactory(rebaseMods, ignoreFieldsMods, nil, ctldiff.ChangeOpts{false})
 	changeSet := ctldiff.NewChangeSet([]ctlres.Resource{existingRes}, []ctlres.Resource{newRes},
 		ctldiff.ChangeSetOpts{AgainstLastApplied: true}, changeFactory)
 
@@ -304,7 +304,7 @@ metadata:
 		},
 	}
 
-	changeFactory := ctldiff.NewChangeFactory(mods, nil, nil)
+	changeFactory := ctldiff.NewChangeFactory(mods, nil, nil, ctldiff.ChangeOpts{false})
 	changeSet := ctldiff.NewChangeSet([]ctlres.Resource{existingRes}, []ctlres.Resource{newRes},
 		ctldiff.ChangeSetOpts{}, changeFactory)
 

pkg/kapp/diff/change_set_with_versioned_rs.go

@@ -198,7 +198,7 @@ func (d ChangeSetWithVersionedRs) noopAndDeleteChanges(
 }
 
 func (d ChangeSetWithVersionedRs) newKeepChange(existingRes ctlres.Resource) Change {
-	return NewChangePrecalculated(existingRes, nil, nil, ChangeOpKeep, NewConfigurableTextDiff(existingRes, nil, true), OpsDiff{})
+	return NewChangePrecalculated(existingRes, nil, nil, ChangeOpKeep, NewConfigurableTextDiff(existingRes, nil, true, ChangeOpts{false}), OpsDiff{})
 }
 
 func (d ChangeSetWithVersionedRs) newNoopChange(existingRes ctlres.Resource) Change {

pkg/kapp/diff/configurable_text_diff.go

@@ -16,10 +16,12 @@ type ConfigurableTextDiff struct {
 	ignored             bool
 
 	memoizedTextDiff *TextDiff
+
+	opts ChangeOpts
 }
 
-func NewConfigurableTextDiff(existingRes, newRes ctlres.Resource, ignored bool) *ConfigurableTextDiff {
-	return &ConfigurableTextDiff{existingRes, newRes, ignored, nil}
+func NewConfigurableTextDiff(existingRes, newRes ctlres.Resource, ignored bool, opts ChangeOpts) *ConfigurableTextDiff {
+	return &ConfigurableTextDiff{existingRes, newRes, ignored, nil, opts}
 }
 
 func (d ConfigurableTextDiff) Full() TextDiff {
@@ -73,5 +75,5 @@ func (d ConfigurableTextDiff) calculate(existingRes, newRes ctlres.Resource) Tex
 		newLines = existingLines // show as no changes
 	}
 
-	return NewTextDiff(existingLines, newLines)
+	return NewTextDiff(existingLines, newLines, d.opts.AllowAnchoredDiff)
 }

pkg/kapp/diff/text_diff.go

@@ -15,7 +15,11 @@ type TextDiff struct {
 	recs []difflib.DiffRecord
 }
 
-func NewTextDiff(existingLines, newLines []string) TextDiff {
+func NewTextDiff(existingLines, newLines []string, allowAnchoredDiff bool) TextDiff {
+	if allowAnchoredDiff && (len(existingLines) > 500 || len(newLines) > 500) {
+		// Diff is memory hungry, use AnchoredDiff for large resources
+		return TextDiff{difflib.AnchoredDiff(existingLines, newLines)}
+	}
 	return TextDiff{difflib.Diff(existingLines, newLines)}
 }
 

test/e2e/diff_test.go

@@ -486,3 +486,125 @@ metadata:
 			RunOpts{IntoNs: true, StdinReader: strings.NewReader(yaml1)})
 	})
 }
+
+func TestAnchoredDiff(t *testing.T) {
+	env := BuildEnv(t)
+	logger := Logger{}
+	kapp := Kapp{t, env.Namespace, env.KappBinaryPath, logger}
+
+	name := "test-anchored-diff"
+	cleanUp := func() {
+		kapp.Run([]string{"delete", "-a", name})
+	}
+
+	cleanUp()
+	defer cleanUp()
+
+	yaml1 := `apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cm-1
+  annotations:
+    kbld.k14s.io/images: |
+      - origins:
+        - resolved:
+            tag: 9.5.5
+             url: docker.io/grafana/grafana:9.5.5
+        url: index.docker.io/grafana/grafana@sha256:6c6fe32401b6b14e1886e61a7bacd5cc4b6fbd0de1e58e985db0e48f99fe1be1
+      - origins:
+        - resolved:
+            tag: 1.24.3
+            url: quay.io/kiwigrid/k8s-sidecar:1.24.3
+        url: quay.io/kiwigrid/k8s-sidecar@sha256:5af76eebbba79edf4f7471bf1c3d5f2b40858114730c92d95eafe5716abe1fe8
+data:
+
+`
+
+	yaml2 := `apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cm-1
+  annotations:
+    kbld.k14s.io/images: |
+      - origins:
+        - resolved:
+            tag: 10.1.4
+            url: docker.io/grafana/grafana:10.1.4
+        url: index.docker.io/grafana/grafana@sha256:29f39e23705d3ef653fa84ca3c01731e0771f1fedbd69ecb99868270cdeb0572
+      - origins:
+        - resolved:
+            tag: 1.25.1
+            url: quay.io/kiwigrid/k8s-sidecar:1.25.1
+        url: quay.io/kiwigrid/k8s-sidecar@sha256:415d07ee1027c3ff7af9e26e05e03ffd0ec0ccf9f619ac00ab24366efe4343bd
+data:
+
+`
+	// Add keys so that number of lines in the yamls are > 500
+	for i := 0; i <= 500; i++ {
+		line := fmt.Sprintf("  key%v: value%v\n", i, i)
+		yaml1 += line
+		yaml2 += line
+	}
+
+	logger.Section("deploy initial", func() {
+		kapp.RunWithOpts([]string{"deploy", "-f", "-", "-a", name}, RunOpts{IntoNs: true, StdinReader: strings.NewReader(yaml1)})
+	})
+
+	logger.Section("deploy without anchored diff", func() {
+		out, _ := kapp.RunWithOpts([]string{"deploy", "-f", "-", "-a", name, "-c", "--diff-run", "--diff-summary=false"}, RunOpts{IntoNs: true, StdinReader: strings.NewReader(yaml2)})
+		expectedDiff := `
+@@ update configmap/cm-1 (v1) namespace: kapp-test @@
+  ...
+508,508           - resolved:
+509     -             tag: 9.5.5
+510     -              url: docker.io/grafana/grafana:9.5.5
+511     -         url: index.docker.io/grafana/grafana@sha256:6c6fe32401b6b14e1886e61a7bacd5cc4b6fbd0de1e58e985db0e48f99fe1be1
+    509 +             tag: 10.1.4
+    510 +             url: docker.io/grafana/grafana:10.1.4
+    511 +         url: index.docker.io/grafana/grafana@sha256:29f39e23705d3ef653fa84ca3c01731e0771f1fedbd69ecb99868270cdeb0572
+512,512         - origins:
+513,513           - resolved:
+514     -             tag: 1.24.3
+515     -             url: quay.io/kiwigrid/k8s-sidecar:1.24.3
+516     -         url: quay.io/kiwigrid/k8s-sidecar@sha256:5af76eebbba79edf4f7471bf1c3d5f2b40858114730c92d95eafe5716abe1fe8
+    514 +             tag: 1.25.1
+    515 +             url: quay.io/kiwigrid/k8s-sidecar:1.25.1
+    516 +         url: quay.io/kiwigrid/k8s-sidecar@sha256:415d07ee1027c3ff7af9e26e05e03ffd0ec0ccf9f619ac00ab24366efe4343bd
+517,517     creationTimestamp: "2006-01-02T15:04:05Z07:00"
+518,518     labels:
+
+Succeeded
+`
+		require.Equal(t, expectedDiff, replaceTimestampWithDfaultValue(replaceTarget(out)))
+	})
+
+	logger.Section("deploy with anchored diff", func() {
+		out, _ := kapp.RunWithOpts([]string{"deploy", "-f", "-", "-a", name, "-c", "--diff-run", "--diff-summary=false", "--diff-anchored"}, RunOpts{IntoNs: true, StdinReader: strings.NewReader(yaml2)})
+		expectedDiff := `
+@@ update configmap/cm-1 (v1) namespace: kapp-test @@
+  ...
+508,508           - resolved:
+509     -             tag: 9.5.5
+510     -              url: docker.io/grafana/grafana:9.5.5
+511     -         url: index.docker.io/grafana/grafana@sha256:6c6fe32401b6b14e1886e61a7bacd5cc4b6fbd0de1e58e985db0e48f99fe1be1
+512     -       - origins:
+513     -         - resolved:
+514     -             tag: 1.24.3
+515     -             url: quay.io/kiwigrid/k8s-sidecar:1.24.3
+516     -         url: quay.io/kiwigrid/k8s-sidecar@sha256:5af76eebbba79edf4f7471bf1c3d5f2b40858114730c92d95eafe5716abe1fe8
+    509 +             tag: 10.1.4
+    510 +             url: docker.io/grafana/grafana:10.1.4
+    511 +         url: index.docker.io/grafana/grafana@sha256:29f39e23705d3ef653fa84ca3c01731e0771f1fedbd69ecb99868270cdeb0572
+    512 +       - origins:
+    513 +         - resolved:
+    514 +             tag: 1.25.1
+    515 +             url: quay.io/kiwigrid/k8s-sidecar:1.25.1
+    516 +         url: quay.io/kiwigrid/k8s-sidecar@sha256:415d07ee1027c3ff7af9e26e05e03ffd0ec0ccf9f619ac00ab24366efe4343bd
+517,517     creationTimestamp: "2006-01-02T15:04:05Z07:00"
+518,518     labels:
+
+Succeeded
+`
+		require.Equal(t, expectedDiff, replaceTimestampWithDfaultValue(replaceTarget(out)))
+	})
+}