Add support for loading UTF16 encoded files

If they have a correct BOM (Byte-Order-Mark https://en.wikipedia.org/wiki/Byte_order_mark) at the beginning of the file, support non utf8 files. Falls back on UTF8 encoding, if no understandable BOM is present. Signed-off-by: Soule BA <[email protected]>
fluxcd · Sep 11, 2023 · 298f254 · 298f254
1 parent b4f3acd
commit 298f254
Show file tree

Hide file tree

Showing 5 changed files with 22 additions and 3 deletions.
diff --git a/kustomize/go.mod b/kustomize/go.mod
@@ -15,12 +15,13 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/onsi/gomega v1.27.10
 	github.com/otiai10/copy v1.12.0
+	golang.org/x/text v0.11.0
 	k8s.io/api v0.27.4
 	k8s.io/apiextensions-apiserver v0.27.4
 	k8s.io/apimachinery v0.27.4
 	k8s.io/client-go v0.27.4
 	sigs.k8s.io/controller-runtime v0.15.1
-	sigs.k8s.io/kustomize/api v0.13.4
+	sigs.k8s.io/kustomize/api v0.14.0
 	sigs.k8s.io/kustomize/kyaml v0.14.2
 	sigs.k8s.io/yaml v1.3.0
 )
@@ -72,7 +73,6 @@ require (
 	golang.org/x/oauth2 v0.5.0 // indirect
 	golang.org/x/sys v0.10.0 // indirect
 	golang.org/x/term v0.10.0 // indirect
-	golang.org/x/text v0.11.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect

diff --git a/kustomize/kustomize_generator.go b/kustomize/kustomize_generator.go
@@ -27,6 +27,8 @@ import (
 	"sync"
 
 	securefs "github.com/fluxcd/pkg/kustomize/filesys"
+	"golang.org/x/text/encoding/unicode"
+	"golang.org/x/text/transform"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -510,7 +512,16 @@ func scanManifests(fs filesys.FileSystem, base string) ([]string, error) {
 			return
 		}
 
-		fContents, err := fs.ReadFile(path)
+		f, err := os.Open(path)
+		if err != nil {
+			walkErr = err
+			return
+		}
+		defer f.Close()
+
+		utf16bom := unicode.BOMOverride(unicode.UTF8.NewDecoder())
+		reader := transform.NewReader(f, utf16bom)
+		fContents, err := io.ReadAll(reader)
 		if err != nil {
 			walkErr = err
 			return

diff --git a/kustomize/kustomize_generator_whitebox_test.go b/kustomize/kustomize_generator_whitebox_test.go
@@ -49,6 +49,14 @@ func TestScanManifests(t *testing.T) {
 			base:    "./testdata/nokustomization/panic",
 			wantErr: true,
 		},
+		{
+			name: "utf-16LE encoding",
+			base: "./testdata/nokustomization/utf16le",
+			wantPaths: []string{
+				"testdata/nokustomization/utf16le/configmap.yaml",
+				"testdata/nokustomization/utf16le/secret.yaml",
+			},
+		},
 	}
 
 	for _, tt := range tests {

diff --git a/kustomize/testdata/nokustomization/utf16le/configmap.yaml b/kustomize/testdata/nokustomization/utf16le/configmap.yaml
diff --git a/kustomize/testdata/nokustomization/utf16le/secret.yaml b/kustomize/testdata/nokustomization/utf16le/secret.yaml