From ae00b0a198bb62f13914f46c0f6275ac12b14a2d Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Fri, 8 Dec 2023 18:34:10 +0100 Subject: [PATCH] installer: add support for data URLs RFC 015 proposes the introduction of data URLs to materialize static content to files on disk. This commit adds support for data URLs to the installer. The corresponding content will be added to versions.go in a subsequent commit. --- bazel/toolchains/go_module_deps.bzl | 8 +++ go.mod | 1 + go.sum | 2 + internal/installer/BUILD.bazel | 1 + internal/installer/installer.go | 85 ++++++++++++++++++++++------ internal/installer/installer_test.go | 18 ++++++ 6 files changed, 97 insertions(+), 18 deletions(-) diff --git a/bazel/toolchains/go_module_deps.bzl b/bazel/toolchains/go_module_deps.bzl index d2271f9b92..add2ec96e6 100644 --- a/bazel/toolchains/go_module_deps.bzl +++ b/bazel/toolchains/go_module_deps.bzl @@ -4876,6 +4876,14 @@ def go_dependencies(): sum = "h1:AalPS4VGiKavpAzIlBjrn7bhqXiXi4jbMYY/2+UC+4o=", version = "v1.1.0", ) + go_repository( + name = "com_github_vincent_petithory_dataurl", + build_file_generation = "on", + build_file_proto_mode = "disable_global", + importpath = "github.com/vincent-petithory/dataurl", + sum = "h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI=", + version = "v1.0.0", + ) go_repository( name = "com_github_vishvananda_netlink", build_file_generation = "on", diff --git a/go.mod b/go.mod index dedf430781..671cb6f6c8 100644 --- a/go.mod +++ b/go.mod @@ -113,6 +113,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/theupdateframework/go-tuf v0.5.2 github.com/tink-crypto/tink-go/v2 v2.0.0 + github.com/vincent-petithory/dataurl v1.0.0 go.uber.org/goleak v1.3.0 go.uber.org/zap v1.26.0 golang.org/x/crypto v0.14.0 diff --git a/go.sum b/go.sum index 5e144cfa07..07d383cee9 100644 --- a/go.sum +++ b/go.sum @@ -916,6 +916,8 @@ github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 h1:e/5i7d4oYZ+C github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399/go.mod h1:LdwHTNJT99C5fTAzDz0ud328OgXz+gierycbcIx2fRs= github.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG81+twTK4= github.com/transparency-dev/merkle v0.0.2/go.mod h1:pqSy+OXefQ1EDUVmAJ8MUhHB9TXGuzVAT58PqBoHz1A= +github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= +github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= diff --git a/internal/installer/BUILD.bazel b/internal/installer/BUILD.bazel index 0ad838ac41..c68ec2e900 100644 --- a/internal/installer/BUILD.bazel +++ b/internal/installer/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "//internal/retry", "//internal/versions/components", "@com_github_spf13_afero//:afero", + "@com_github_vincent_petithory_dataurl//:dataurl", "@io_k8s_utils//clock", ], ) diff --git a/internal/installer/installer.go b/internal/installer/installer.go index 113d753e5d..dd26ea12e7 100644 --- a/internal/installer/installer.go +++ b/internal/installer/installer.go @@ -9,6 +9,7 @@ package installer import ( "archive/tar" + "bytes" "compress/gzip" "context" "crypto/sha256" @@ -17,13 +18,16 @@ import ( "io" "io/fs" "net/http" + "net/url" "os" "path" + "slices" "time" "github.com/edgelesssys/constellation/v2/internal/retry" "github.com/edgelesssys/constellation/v2/internal/versions/components" "github.com/spf13/afero" + "github.com/vincent-petithory/dataurl" "k8s.io/utils/clock" ) @@ -178,36 +182,81 @@ func (i *OsInstaller) retryDownloadToTempDir(ctx context.Context, url string) (f return doer.path, nil } -// downloadToTempDir downloads a file to a temporary location. -func (i *OsInstaller) downloadToTempDir(ctx context.Context, url string) (fileName string, retErr error) { - out, err := afero.TempFile(i.fs, "", "") - if err != nil { - return "", fmt.Errorf("creating destination temp file: %w", err) - } - // Remove the created file if an error occurs. - defer func() { - if retErr != nil { - _ = i.fs.Remove(fileName) - retErr = &retriableError{err: retErr} // mark any error after this point as retriable - } - }() - defer out.Close() +// retriableHTTPStatusCodes are status codes that might flip to 200 if retried. +// This arguably depends on the web server implementation, but below list is +// a reasonable selection, cf. https://stackoverflow.com/a/74627395. +var retriableHTTPStatusCodes = []int{ + http.StatusRequestTimeout, + http.StatusTooEarly, + http.StatusTooManyRequests, + http.StatusBadGateway, + http.StatusServiceUnavailable, + http.StatusGatewayTimeout, +} +// downloadHTTP downloads the given URL with the embedded HTTP client and writes the content to out. +func (i *OsInstaller) downloadHTTP(ctx context.Context, url string, out io.Writer) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { - return "", fmt.Errorf("request to download %q: %w", url, err) + return fmt.Errorf("request to download %q: %w", url, err) } resp, err := i.hClient.Do(req) if err != nil { - return "", fmt.Errorf("request to download %q: %w", url, err) + // A failure at this point might be transient, such as network connectivity. + return fmt.Errorf("request to download %q: %w", url, &retriableError{err: err}) } if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("request to download %q failed with status code: %v", url, resp.Status) + // The HTTP request went through, but the result is not what we + // expected. Wrap the error return in case we think the request could + // be retried. + err = fmt.Errorf("request to download %q failed with status code: %v", url, resp.Status) + if slices.Contains(retriableHTTPStatusCodes, resp.StatusCode) { + err = &retriableError{err: err} + } + return err } defer resp.Body.Close() if _, err = io.Copy(out, resp.Body); err != nil { - return "", fmt.Errorf("downloading %q: %w", url, err) + return fmt.Errorf("downloading %q: %w", url, &retriableError{err: err}) + } + return nil +} + +// unpackData parses the given data URL and writes the content to out. +func (i *OsInstaller) unpackData(url string, out io.Writer) error { + dataURL, err := dataurl.DecodeString(url) + if err != nil { + return fmt.Errorf("parsing data URL: %w", err) + } + buf := bytes.NewBuffer(dataURL.Data) + if _, err = io.Copy(out, buf); err != nil { + return fmt.Errorf("writing content of data URL %q: %w", url, err) + } + return nil +} + +// downloadToTempDir downloads a file from the given URL to a temporary location and returns the path to the downloaded file. +func (i *OsInstaller) downloadToTempDir(ctx context.Context, u string) (string, error) { + url, err := url.Parse(u) + if err != nil { + return "", fmt.Errorf("parsing component URL: %w", err) + } + + out, err := afero.TempFile(i.fs, "", "") + if err != nil { + return "", fmt.Errorf("creating destination temp file: %w", err) + } + + if url.Scheme == "data" { + err = i.unpackData(u, out) + } else { + err = i.downloadHTTP(ctx, u, out) + } + out.Close() + if err != nil { + removeErr := i.fs.Remove(out.Name()) + return "", errors.Join(err, removeErr) } return out.Name(), nil } diff --git a/internal/installer/installer_test.go b/internal/installer/installer_test.go index 1ea1f1900c..dead113c04 100644 --- a/internal/installer/installer_test.go +++ b/internal/installer/installer_test.go @@ -88,6 +88,24 @@ func TestInstall(t *testing.T) { }, wantErr: true, }, + "dataurl works": { + server: newHTTPBufconnServer(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(500) }), + component: &components.Component{ + Url: "data:text/plain,file-contents", + Hash: "", + InstallPath: "/destination", + }, + wantFiles: map[string][]byte{"/destination": []byte("file-contents")}, + }, + "broken dataurl fails": { + server: newHTTPBufconnServer(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(500) }), + component: &components.Component{ + Url: "data:file-contents", + Hash: "", + InstallPath: "/destination", + }, + wantErr: true, + }, } for name, tc := range testCases {