diff --git a/cmd/csaf_downloader/config.go b/cmd/csaf_downloader/config.go index 929ed608..10eef12e 100644 --- a/cmd/csaf_downloader/config.go +++ b/cmd/csaf_downloader/config.go @@ -126,7 +126,7 @@ func parseArgsConfig() ([]string, *config, error) { } // prepareDirectory ensures that the working directory -// exists and is setup properly. +// exists and is set up properly. func (cfg *config) prepareDirectory() error { // If not given use current working directory. if cfg.Directory == "" { @@ -213,7 +213,7 @@ func (cfg *config) prepareCertificates() error { return nil } -// Prepare prepares internal state of a loaded configuration. +// GetDownloadConfig Prepare prepares internal state of a loaded configuration. func (cfg *config) GetDownloadConfig() (*downloader.Config, error) { for _, prepare := range []func(*config) error{ (*config).prepareDirectory, @@ -233,7 +233,7 @@ func (cfg *config) GetDownloadConfig() (*downloader.Config, error) { ClientPassphrase: cfg.ClientPassphrase, Rate: cfg.Rate, Worker: cfg.Worker, - Range: cfg.Range, + Range: (*[2]time.Time)(cfg.Range), IgnorePattern: cfg.ignorePattern, ExtraHeader: cfg.ExtraHeader, diff --git a/lib/downloader/config.go b/lib/downloader/config.go index 1bd3200d..60e07ad4 100644 --- a/lib/downloader/config.go +++ b/lib/downloader/config.go @@ -13,9 +13,8 @@ import ( "fmt" "log/slog" "net/http" - - "github.com/csaf-poc/csaf_distribution/v3/internal/filter" - "github.com/csaf-poc/csaf_distribution/v3/internal/models" + "regexp" + "time" ) // ValidationMode specifies the strict the validation is. @@ -37,8 +36,8 @@ type Config struct { ClientPassphrase *string Rate *float64 Worker int - Range *models.TimeRange - IgnorePattern filter.PatternMatcher + Range *[2]time.Time + IgnorePattern []*regexp.Regexp ExtraHeader http.Header RemoteValidator string @@ -82,7 +81,13 @@ func (vm *ValidationMode) UnmarshalFlag(value string) error { // ignoreFile returns true if the given URL should not be downloaded. func (cfg *Config) ignoreURL(u string) bool { - return cfg.IgnorePattern.Matches(u) + for _, expr := range cfg.IgnorePattern { + if expr.MatchString(u) { + return true + } + } + return false + } // verbose is considered a log level equal or less debug. diff --git a/lib/downloader/downloader.go b/lib/downloader/downloader.go index c74d2f44..708845da 100644 --- a/lib/downloader/downloader.go +++ b/lib/downloader/downloader.go @@ -17,6 +17,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/csaf-poc/csaf_distribution/v3/internal/models" "hash" "io" "log/slog" @@ -25,7 +26,6 @@ import ( "path/filepath" "strings" "sync" - "time" "github.com/ProtonMail/gopenpgp/v2/crypto" "golang.org/x/time/rate" @@ -56,11 +56,6 @@ type DownloadedDocument struct { ValStatus ValidationStatus } -// failedValidationDir is the name of the sub folder -// where advisories are stored that fail validation in -// unsafe mode. -const failedValidationDir = "failed_validation" - // NewDownloader constructs a new downloader given the configuration. func NewDownloader(cfg *Config) (*Downloader, error) { var validator csaf.RemoteValidator @@ -179,7 +174,7 @@ func (d *Downloader) enumerate(domain string) error { loader := csaf.NewProviderMetadataLoader(client) lpmd := loader.Enumerate(domain) - docs := []any{} + var docs []any for _, pmd := range lpmd { if d.cfg.verbose() { @@ -246,9 +241,10 @@ func (d *Downloader) download(ctx context.Context, domain string) error { // Do we need time range based filtering? if d.cfg.Range != nil { + timeRange := models.NewTimeInterval(d.cfg.Range[0], d.cfg.Range[1]) d.cfg.Logger.Debug("Setting up filter to accept advisories within", - "timerange", d.cfg.Range) - afp.AgeAccept = d.cfg.Range.Contains + "timerange", timeRange) + afp.AgeAccept = timeRange.Contains } return afp.Process(func(label csaf.TLPLabel, files []csaf.AdvisoryFile) error { @@ -425,7 +421,7 @@ func (d *Downloader) downloadWorker( client = d.httpClient() data bytes.Buffer stats = stats{} - expr = util.NewPathEval() + expr = util.NewPathEval() ) // Add collected stats back to total.