diff --git a/bazel/toolchains/go_module_deps.bzl b/bazel/toolchains/go_module_deps.bzl index d2271f9b92..add2ec96e6 100644 --- a/bazel/toolchains/go_module_deps.bzl +++ b/bazel/toolchains/go_module_deps.bzl @@ -4876,6 +4876,14 @@ def go_dependencies(): sum = "h1:AalPS4VGiKavpAzIlBjrn7bhqXiXi4jbMYY/2+UC+4o=", version = "v1.1.0", ) + go_repository( + name = "com_github_vincent_petithory_dataurl", + build_file_generation = "on", + build_file_proto_mode = "disable_global", + importpath = "github.com/vincent-petithory/dataurl", + sum = "h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI=", + version = "v1.0.0", + ) go_repository( name = "com_github_vishvananda_netlink", build_file_generation = "on", diff --git a/go.mod b/go.mod index dedf430781..671cb6f6c8 100644 --- a/go.mod +++ b/go.mod @@ -113,6 +113,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/theupdateframework/go-tuf v0.5.2 github.com/tink-crypto/tink-go/v2 v2.0.0 + github.com/vincent-petithory/dataurl v1.0.0 go.uber.org/goleak v1.3.0 go.uber.org/zap v1.26.0 golang.org/x/crypto v0.14.0 diff --git a/go.sum b/go.sum index 5e144cfa07..07d383cee9 100644 --- a/go.sum +++ b/go.sum @@ -916,6 +916,8 @@ github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 h1:e/5i7d4oYZ+C github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399/go.mod h1:LdwHTNJT99C5fTAzDz0ud328OgXz+gierycbcIx2fRs= github.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG81+twTK4= github.com/transparency-dev/merkle v0.0.2/go.mod h1:pqSy+OXefQ1EDUVmAJ8MUhHB9TXGuzVAT58PqBoHz1A= +github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= +github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= diff --git a/internal/installer/BUILD.bazel b/internal/installer/BUILD.bazel index 0ad838ac41..c68ec2e900 100644 --- a/internal/installer/BUILD.bazel +++ b/internal/installer/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "//internal/retry", "//internal/versions/components", "@com_github_spf13_afero//:afero", + "@com_github_vincent_petithory_dataurl//:dataurl", "@io_k8s_utils//clock", ], ) diff --git a/internal/installer/installer.go b/internal/installer/installer.go index 113d753e5d..dd26ea12e7 100644 --- a/internal/installer/installer.go +++ b/internal/installer/installer.go @@ -9,6 +9,7 @@ package installer import ( "archive/tar" + "bytes" "compress/gzip" "context" "crypto/sha256" @@ -17,13 +18,16 @@ import ( "io" "io/fs" "net/http" + "net/url" "os" "path" + "slices" "time" "github.com/edgelesssys/constellation/v2/internal/retry" "github.com/edgelesssys/constellation/v2/internal/versions/components" "github.com/spf13/afero" + "github.com/vincent-petithory/dataurl" "k8s.io/utils/clock" ) @@ -178,36 +182,81 @@ func (i *OsInstaller) retryDownloadToTempDir(ctx context.Context, url string) (f return doer.path, nil } -// downloadToTempDir downloads a file to a temporary location. -func (i *OsInstaller) downloadToTempDir(ctx context.Context, url string) (fileName string, retErr error) { - out, err := afero.TempFile(i.fs, "", "") - if err != nil { - return "", fmt.Errorf("creating destination temp file: %w", err) - } - // Remove the created file if an error occurs. - defer func() { - if retErr != nil { - _ = i.fs.Remove(fileName) - retErr = &retriableError{err: retErr} // mark any error after this point as retriable - } - }() - defer out.Close() +// retriableHTTPStatusCodes are status codes that might flip to 200 if retried. +// This arguably depends on the web server implementation, but below list is +// a reasonable selection, cf. https://stackoverflow.com/a/74627395. +var retriableHTTPStatusCodes = []int{ + http.StatusRequestTimeout, + http.StatusTooEarly, + http.StatusTooManyRequests, + http.StatusBadGateway, + http.StatusServiceUnavailable, + http.StatusGatewayTimeout, +} +// downloadHTTP downloads the given URL with the embedded HTTP client and writes the content to out. +func (i *OsInstaller) downloadHTTP(ctx context.Context, url string, out io.Writer) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { - return "", fmt.Errorf("request to download %q: %w", url, err) + return fmt.Errorf("request to download %q: %w", url, err) } resp, err := i.hClient.Do(req) if err != nil { - return "", fmt.Errorf("request to download %q: %w", url, err) + // A failure at this point might be transient, such as network connectivity. + return fmt.Errorf("request to download %q: %w", url, &retriableError{err: err}) } if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("request to download %q failed with status code: %v", url, resp.Status) + // The HTTP request went through, but the result is not what we + // expected. Wrap the error return in case we think the request could + // be retried. + err = fmt.Errorf("request to download %q failed with status code: %v", url, resp.Status) + if slices.Contains(retriableHTTPStatusCodes, resp.StatusCode) { + err = &retriableError{err: err} + } + return err } defer resp.Body.Close() if _, err = io.Copy(out, resp.Body); err != nil { - return "", fmt.Errorf("downloading %q: %w", url, err) + return fmt.Errorf("downloading %q: %w", url, &retriableError{err: err}) + } + return nil +} + +// unpackData parses the given data URL and writes the content to out. +func (i *OsInstaller) unpackData(url string, out io.Writer) error { + dataURL, err := dataurl.DecodeString(url) + if err != nil { + return fmt.Errorf("parsing data URL: %w", err) + } + buf := bytes.NewBuffer(dataURL.Data) + if _, err = io.Copy(out, buf); err != nil { + return fmt.Errorf("writing content of data URL %q: %w", url, err) + } + return nil +} + +// downloadToTempDir downloads a file from the given URL to a temporary location and returns the path to the downloaded file. +func (i *OsInstaller) downloadToTempDir(ctx context.Context, u string) (string, error) { + url, err := url.Parse(u) + if err != nil { + return "", fmt.Errorf("parsing component URL: %w", err) + } + + out, err := afero.TempFile(i.fs, "", "") + if err != nil { + return "", fmt.Errorf("creating destination temp file: %w", err) + } + + if url.Scheme == "data" { + err = i.unpackData(u, out) + } else { + err = i.downloadHTTP(ctx, u, out) + } + out.Close() + if err != nil { + removeErr := i.fs.Remove(out.Name()) + return "", errors.Join(err, removeErr) } return out.Name(), nil } diff --git a/internal/installer/installer_test.go b/internal/installer/installer_test.go index 1ea1f1900c..dead113c04 100644 --- a/internal/installer/installer_test.go +++ b/internal/installer/installer_test.go @@ -88,6 +88,24 @@ func TestInstall(t *testing.T) { }, wantErr: true, }, + "dataurl works": { + server: newHTTPBufconnServer(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(500) }), + component: &components.Component{ + Url: "data:text/plain,file-contents", + Hash: "", + InstallPath: "/destination", + }, + wantFiles: map[string][]byte{"/destination": []byte("file-contents")}, + }, + "broken dataurl fails": { + server: newHTTPBufconnServer(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(500) }), + component: &components.Component{ + Url: "data:file-contents", + Hash: "", + InstallPath: "/destination", + }, + wantErr: true, + }, } for name, tc := range testCases {