Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add support for registry mirrors #8244

Merged
merged 22 commits into from
Jan 22, 2025
Merged
Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
b9d46d7
feat(flag): add registry.mirrors flag
DmitriyLewen Jan 14, 2025
f798d68
feat(mage): handle map[string][]string flags + add example for regist…
DmitriyLewen Jan 14, 2025
04851e1
feat(options): add RegistryMirrors
DmitriyLewen Jan 15, 2025
8c4f6c1
feat(remote): check mirrors for `Get` and `Image` functions
DmitriyLewen Jan 15, 2025
696be1f
fix(remote): use replace to set mirror
DmitriyLewen Jan 15, 2025
392c920
test: add testcases
DmitriyLewen Jan 15, 2025
678ba5c
test: update testcase
DmitriyLewen Jan 15, 2025
c3682e0
test: add testcase for image without docker.io prefix
DmitriyLewen Jan 15, 2025
ccbf7c7
docs: add info about mirrors in docs
DmitriyLewen Jan 16, 2025
f9b6d01
refactor: use `index.docker.io` in example docs
DmitriyLewen Jan 16, 2025
3d6e223
docs: update mirror title
DmitriyLewen Jan 17, 2025
f097a43
refactor: don't use `ctx` variable
DmitriyLewen Jan 17, 2025
8d269d2
refactor: return error for wrong mirrors
DmitriyLewen Jan 17, 2025
ad56617
refactor: add tryImage and tryGet functions
DmitriyLewen Jan 17, 2025
d7a1b7d
test: add `bad credential for multiple mirrors`
DmitriyLewen Jan 17, 2025
d082591
docs: add order and example for mirrors
DmitriyLewen Jan 17, 2025
539e9f4
fix: linter error
DmitriyLewen Jan 17, 2025
f3247f1
docs: fix display of steps
DmitriyLewen Jan 17, 2025
04f1bc5
refactor(mage): remove example for mirrors
DmitriyLewen Jan 17, 2025
e3a9123
chore: add comment about WithMirror
DmitriyLewen Jan 17, 2025
7102e45
fix: logs about used mirrors
DmitriyLewen Jan 20, 2025
2981a6f
refactor: reduce duplicates
knqyf263 Jan 21, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions docs/docs/configuration/others.md
Original file line number Diff line number Diff line change
Expand Up @@ -117,3 +117,23 @@ The following example will fail when a critical vulnerability is found or the OS
```
$ trivy image --exit-code 1 --exit-on-eol 1 --severity CRITICAL alpine:3.16.3
```

## Mirrors support
DmitriyLewen marked this conversation as resolved.
Show resolved Hide resolved

!!! warning "EXPERIMENTAL"
This feature might change without preserving backwards compatibility.

Trivy supports mirrors for [remote container images](../target/container_image.md#container-registry) and [databases](./db.md).

To configure them, add a list of mirrors along with the host to the [trivy config file](../references/configuration/config-file.md#registry-options).
knqyf263 marked this conversation as resolved.
Show resolved Hide resolved

!!! note
Use the `index.docker.io` host for images from `Docker Hub`, even if you don't use that prefix.

Example for `index.docker.io`:
```yaml
registry:
mirrors:
index.docker.io:
- mirror.gcr.io
```
5 changes: 5 additions & 0 deletions docs/docs/references/configuration/config-file.md
Original file line number Diff line number Diff line change
Expand Up @@ -461,6 +461,11 @@ pkg:

```yaml
registry:
mirrors:
index.docker.io:
- harbor.example.com/docker.io
- mirror.gcr.io
knqyf263 marked this conversation as resolved.
Show resolved Hide resolved

# Same as '--password'
password: []

Expand Down
24 changes: 23 additions & 1 deletion magefiles/docs.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ func writeFlags(group flag.FlagGroup, w *os.File) {
}
w.WriteString(ind + parts[i] + ":")
if isLastPart {
writeFlagValue(flg.GetDefaultValue(), ind, w)
writeFlagValue(value(flg), ind, w)
}
w.WriteString("\n")
}
Expand All @@ -147,9 +147,31 @@ func writeFlagValue(val any, ind string, w *os.File) {
} else {
w.WriteString(" []\n")
}
case map[string][]string:
w.WriteString("\n")
for k, vv := range v {
fmt.Fprintf(w, "%s %s:\n", ind, k)
for _, vvv := range vv {
fmt.Fprintf(w, " %s - %s\n", ind, vvv)
}
}
case string:
fmt.Fprintf(w, " %q\n", v)
default:
fmt.Fprintf(w, " %v\n", v)
}
}

var registryMirrorsExample = map[string][]string{
"index.docker.io": {
"harbor.example.com/docker.io",
"mirror.gcr.io",
},
}

func value(flg flag.Flagger) any {
if flg.GetConfigName() == flag.RegistryMirrorsFlag.ConfigName {
return registryMirrorsExample
}
return flg.GetDefaultValue()
}
3 changes: 3 additions & 0 deletions pkg/fanal/types/image.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,9 @@ type RegistryOptions struct {
// RegistryToken is a bearer token to be sent to a registry
RegistryToken string

// RegistryMirrors is a map of hosts with mirrors for them
RegistryMirrors map[string][]string

// SSL/TLS
Insecure bool

Expand Down
15 changes: 9 additions & 6 deletions pkg/flag/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import (
)

type FlagType interface {
int | string | []string | bool | time.Duration | float64
int | string | []string | bool | time.Duration | float64 | map[string][]string
}

type Flag[T FlagType] struct {
Expand Down Expand Up @@ -161,6 +161,8 @@ func (f *Flag[T]) cast(val any) any {
return cast.ToFloat64(val)
case time.Duration:
return cast.ToDuration(val)
case map[string][]string:
return cast.ToStringMapStringSlice(val)
knqyf263 marked this conversation as resolved.
Show resolved Hide resolved
case []string:
if s, ok := val.(string); ok && strings.Contains(s, ",") {
// Split environmental variables by comma as it is not done by viper.
Expand Down Expand Up @@ -467,11 +469,12 @@ func (o *Options) ScanOpts() types.ScanOptions {
// RegistryOpts returns options for OCI registries
func (o *Options) RegistryOpts() ftypes.RegistryOptions {
return ftypes.RegistryOptions{
Credentials: o.Credentials,
RegistryToken: o.RegistryToken,
Insecure: o.Insecure,
Platform: o.Platform,
AWSRegion: o.AWSOptions.Region,
Credentials: o.Credentials,
RegistryToken: o.RegistryToken,
Insecure: o.Insecure,
Platform: o.Platform,
AWSRegion: o.AWSOptions.Region,
RegistryMirrors: o.RegistryMirrors,
}
}

Expand Down
33 changes: 21 additions & 12 deletions pkg/flag/registry_flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,26 +31,33 @@ var (
ConfigName: "registry.token",
Usage: "registry token",
}
RegistryMirrorsFlag = Flag[map[string][]string]{
knqyf263 marked this conversation as resolved.
Show resolved Hide resolved
ConfigName: "registry.mirrors",
Usage: "map of hosts and registries for them.",
}
)

type RegistryFlagGroup struct {
Username *Flag[[]string]
Password *Flag[[]string]
PasswordStdin *Flag[bool]
RegistryToken *Flag[string]
Username *Flag[[]string]
Password *Flag[[]string]
PasswordStdin *Flag[bool]
RegistryToken *Flag[string]
RegistryMirrors *Flag[map[string][]string]
}

type RegistryOptions struct {
Credentials []types.Credential
RegistryToken string
Credentials []types.Credential
RegistryToken string
RegistryMirrors map[string][]string
}

func NewRegistryFlagGroup() *RegistryFlagGroup {
return &RegistryFlagGroup{
Username: UsernameFlag.Clone(),
Password: PasswordFlag.Clone(),
PasswordStdin: PasswordStdinFlag.Clone(),
RegistryToken: RegistryTokenFlag.Clone(),
Username: UsernameFlag.Clone(),
Password: PasswordFlag.Clone(),
PasswordStdin: PasswordStdinFlag.Clone(),
RegistryToken: RegistryTokenFlag.Clone(),
RegistryMirrors: RegistryMirrorsFlag.Clone(),
}
}

Expand All @@ -64,6 +71,7 @@ func (f *RegistryFlagGroup) Flags() []Flagger {
f.Password,
f.PasswordStdin,
f.RegistryToken,
f.RegistryMirrors,
}
}

Expand Down Expand Up @@ -97,7 +105,8 @@ func (f *RegistryFlagGroup) ToOptions() (RegistryOptions, error) {
}

return RegistryOptions{
Credentials: credentials,
RegistryToken: f.RegistryToken.Value(),
Credentials: credentials,
RegistryToken: f.RegistryToken.Value(),
RegistryMirrors: f.RegistryMirrors.Value(),
}, nil
}
112 changes: 75 additions & 37 deletions pkg/remote/remote.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"fmt"
"net"
"net/http"
"strings"
"time"

"github.com/google/go-containerregistry/pkg/authn"
Expand Down Expand Up @@ -36,39 +37,46 @@ func Get(ctx context.Context, ref name.Reference, option types.RegistryOptions)
}

var errs error
// Try each authentication method until it succeeds
for _, authOpt := range authOptions(ctx, ref, option) {
remoteOpts := []remote.Option{
remote.WithTransport(tr),
authOpt,
}

if option.Platform.Platform != nil {
p, err := resolvePlatform(ref, option.Platform, remoteOpts)
if err != nil {
return nil, xerrors.Errorf("platform error: %w", err)
// Try each mirrors/host until it succeeds
for _, r := range append(registryMirrors(ref, option), ref) {
knqyf263 marked this conversation as resolved.
Show resolved Hide resolved
// Try each authentication method until it succeeds
knqyf263 marked this conversation as resolved.
Show resolved Hide resolved
for _, authOpt := range authOptions(ctx, r, option) {
remoteOpts := []remote.Option{
remote.WithTransport(tr),
authOpt,
}
// Don't pass platform when the specified image is single-arch.
if p.Platform != nil {
remoteOpts = append(remoteOpts, remote.WithPlatform(*p.Platform))

if option.Platform.Platform != nil {
p, err := resolvePlatform(r, option.Platform, remoteOpts)
if err != nil {
return nil, xerrors.Errorf("platform error: %w", err)
}
// Don't pass platform when the specified image is single-arch.
if p.Platform != nil {
remoteOpts = append(remoteOpts, remote.WithPlatform(*p.Platform))
}
}
}

desc, err := remote.Get(ref, remoteOpts...)
if err != nil {
errs = multierror.Append(errs, err)
continue
}
var desc *remote.Descriptor
desc, err = remote.Get(r, remoteOpts...)
if err != nil {
errs = multierror.Append(errs, err)
continue
}

if option.Platform.Force {
if err = satisfyPlatform(desc, lo.FromPtr(option.Platform.Platform)); err != nil {
return nil, err
if option.Platform.Force {
if err = satisfyPlatform(desc, lo.FromPtr(option.Platform.Platform)); err != nil {
return nil, err
}
}
if ref.Context().RegistryStr() != r.Context().RegistryStr() {
log.WithPrefix("remote").Info("Mirror was used to get remote image", log.String("image", ref.String()), log.String("mirror", r.Context().RegistryStr()))
}
return desc, nil
}
return desc, nil
}

// No authentication succeeded
// No authentication for mirrors/host succeeded
return nil, errs
}

Expand All @@ -81,21 +89,28 @@ func Image(ctx context.Context, ref name.Reference, option types.RegistryOptions
}

var errs error
// Try each authentication method until it succeeds
for _, authOpt := range authOptions(ctx, ref, option) {
remoteOpts := []remote.Option{
remote.WithTransport(tr),
authOpt,
}
index, err := remote.Image(ref, remoteOpts...)
if err != nil {
errs = multierror.Append(errs, err)
continue
// Try each mirrors/host until it succeeds
for _, r := range append(registryMirrors(ref, option), ref) {
// Try each authentication method until it succeeds
for _, authOpt := range authOptions(ctx, r, option) {
remoteOpts := []remote.Option{
remote.WithTransport(tr),
authOpt,
}
index, err := remote.Image(r, remoteOpts...)
if err != nil {
errs = multierror.Append(errs, err)
continue
}

if ref.Context().RegistryStr() != r.Context().RegistryStr() {
log.WithPrefix("remote").Info("Mirror was used to get remote image", log.String("image", ref.String()), log.String("mirror", r.Context().RegistryStr()))
}
return index, nil
}
return index, nil
}

// No authentication succeeded
// No authentication for mirrors/host succeeded
return nil, errs
}

Expand Down Expand Up @@ -126,6 +141,29 @@ func Referrers(ctx context.Context, d name.Digest, option types.RegistryOptions)
return nil, errs
}

func registryMirrors(hostRef name.Reference, option types.RegistryOptions) []name.Reference {
var mirrors []name.Reference

ctx := hostRef.Context()
reg := ctx.RegistryStr()
DmitriyLewen marked this conversation as resolved.
Show resolved Hide resolved
if ms, ok := option.RegistryMirrors[reg]; ok {
for _, m := range ms {
var nameOpts []name.Option
if option.Insecure {
nameOpts = append(nameOpts, name.Insecure)
}
mirrorImageName := strings.Replace(hostRef.Name(), reg, m, 1)
ref, err := name.ParseReference(mirrorImageName, nameOpts...)
if err != nil {
log.WithPrefix("remote").Warn("Unable to parse mirror of image", log.String("mirror", mirrorImageName))
knqyf263 marked this conversation as resolved.
Show resolved Hide resolved
continue
}
mirrors = append(mirrors, ref)
}
}
return mirrors
}

func httpTransport(option types.RegistryOptions) (http.RoundTripper, error) {
d := &net.Dialer{
Timeout: 10 * time.Minute,
Expand Down
Loading
Loading