From 3c2c1e7f38f80d8657c2d3060e9aec771617f8af Mon Sep 17 00:00:00 2001 From: Manith Date: Wed, 24 Jul 2024 11:11:35 +0000 Subject: [PATCH 1/3] Priority for repo-bundles and repos Some rpms can be distributed in multiple repos, hence we need to specify priority to the repo for eext to decide where to pick dependencies from. Priority 1 (highest) is given to the local deps available for eext to check before checking upstream repos. The priority should be defined at a repo-bundle level in dnfconfig.yaml, with a priorty > 1. Or within the package specific eext.yaml file at a repo specific level. This is an optional flag, the default priority is assigned by the repo-bundle. --- cmd/testData/mrtparse-2/eext.yaml | 1 + configfiles/dnfconfig.yaml | 9 +++++- dnfconfig/dnfconfig.go | 41 +++++++++++++++++++----- dnfconfig/testData/sample-dnfconfig.yaml | 2 ++ impl/testData/dnfconfig.yaml | 3 ++ impl/testData/expected-mock.cfg | 2 +- impl/testData/manifest-with-deps.yaml | 1 + impl/testData/manifest.yaml | 1 + manifest/testData/sampleManifest1.yaml | 1 + 9 files changed, 51 insertions(+), 10 deletions(-) diff --git a/cmd/testData/mrtparse-2/eext.yaml b/cmd/testData/mrtparse-2/eext.yaml index cc452a2..8779ba2 100644 --- a/cmd/testData/mrtparse-2/eext.yaml +++ b/cmd/testData/mrtparse-2/eext.yaml @@ -14,3 +14,4 @@ package: repo-bundle: - name: el9 version: 9.1 + priority: 2 diff --git a/configfiles/dnfconfig.yaml b/configfiles/dnfconfig.yaml index 63ecb32..f4d3399 100644 --- a/configfiles/dnfconfig.yaml +++ b/configfiles/dnfconfig.yaml @@ -23,6 +23,7 @@ repo-bundle: enabled: false version-labels: default: 9.3 + priority: 2 # -------------------------------------------------------------------------------------------- # -------------------------------------------------------------------------------------------- @@ -43,6 +44,7 @@ repo-bundle: # The eext team is responsible for creating these snapshots. # default points to the latest such snapshot. default: v20240522-1 + priority: 2 # -------------------------------------------------------------------------------------------- # ******************************************************************************************** @@ -57,7 +59,7 @@ repo-bundle: gpgcheck: true gpgkey: file:///usr/share/distribution-gpg-keys/alma/RPM-GPG-KEY-AlmaLinux-9 - baseurl: '{{.Host}}/artifactory/eext-snapshots-local/el9/{{.Version}}/9/{{.RepoName}}/{{.Arch}}/os' + baseurl: "{{.Host}}/artifactory/eext-snapshots-local/el9/{{.Version}}/9/{{.RepoName}}/{{.Arch}}/os" repo: AppStream: enabled: true @@ -69,6 +71,7 @@ repo-bundle: enabled: false extras: enabled: false + priority: 2 #--------------------------------------------------------------------------------------------- # -------------------------------------------------------------------------------------------- @@ -98,6 +101,7 @@ repo-bundle: # default=9 always points to upstream latest dot release 9.x, # which upstream updates regularly. default: 9 + priority: 2 # -------------------------------------------------------------------------------------------- # -------------------------------------------------------------------------------------------- @@ -127,6 +131,7 @@ repo-bundle: # default always points to upstream latest dot release 9.x's beta version, # which upstream updates regularly. default: 9.4-beta + priority: 2 # -------------------------------------------------------------------------------------------- # -------------------------------------------------------------------------------------------- @@ -151,6 +156,7 @@ repo-bundle: version-labels: # default always points to upstream stable repo which receives updates. default: 9 + priority: 2 # -------------------------------------------------------------------------------------------- # -------------------------------------------------------------------------------------------- @@ -174,4 +180,5 @@ repo-bundle: version-labels: # default always points to upstream next/beta repo which receives updates. default: 9 + priority: 2 # -------------------------------------------------------------------------------------------- diff --git a/dnfconfig/dnfconfig.go b/dnfconfig/dnfconfig.go index 34f6efa..8ba7d00 100644 --- a/dnfconfig/dnfconfig.go +++ b/dnfconfig/dnfconfig.go @@ -6,7 +6,6 @@ package dnfconfig import ( "bytes" "fmt" - "io/ioutil" "os" "text/template" @@ -17,8 +16,7 @@ import ( ) const ( - RepoHighPriority int = 1 - RepoStandardPriority = 2 + RepoHighPriority int = 1 ) // DnfRepoConfig holds baseURL format template(/string) @@ -35,6 +33,7 @@ type DnfRepoBundleConfig struct { UseBaseArch bool `yaml:"use-base-arch"` DnfRepoConfig map[string]*DnfRepoConfig `yaml:"repo"` VersionLabels map[string]string `yaml:"version-labels"` + Priority int `yaml:"priority"` baseURLFormatTemplate *template.Template } @@ -54,8 +53,9 @@ type DnfRepoURLData struct { // RepoParamsOverride spec // this is used to override default parameters for repos in the bundle. type DnfRepoParamsOverride struct { - Enabled bool `yaml:"enabled"` - Exclude string `yaml:"exclude"` + Enabled bool `yaml:"enabled"` + Exclude string `yaml:"exclude"` + Priority int `yaml:"priority"` } // RepoData holds dnf repo name and baseurl for mock.cfg generation @@ -149,14 +149,28 @@ func (b *DnfRepoBundleConfig) GetDnfRepoParams( return nil, err } + repoPriority := b.Priority + var enabled bool var exclude string + var priority int repoOverride, isOverride := repoOverrides[repoName] if isOverride { enabled = repoOverride.Enabled exclude = repoOverride.Exclude + priorityOverride := repoOverride.Priority + if priorityOverride != 0 { + if priorityOverride == 1 { + return nil, fmt.Errorf("%sRepo %s priority cannot be 1. Provide a priority > 1", errPrefix, repoName) + } else { + priority = priorityOverride + } + } else { + priority = repoPriority + } } else { enabled = repoConfig.Enabled + priority = repoPriority } return &DnfRepoParams{ @@ -166,7 +180,7 @@ func (b *DnfRepoBundleConfig) GetDnfRepoParams( Exclude: exclude, GpgCheck: b.GpgCheck, GpgKey: b.GpgKey, - Priority: RepoStandardPriority, + Priority: priority, }, nil } @@ -185,9 +199,9 @@ func LoadDnfConfig() (*DnfConfig, error) { cfgPath, statErr) } - yamlContents, readErr := ioutil.ReadFile(cfgPath) + yamlContents, readErr := os.ReadFile(cfgPath) if readErr != nil { - return nil, fmt.Errorf("dnfconfig.LoadDnfConfig: ioutil.ReadFile on %s returned %s", + return nil, fmt.Errorf("dnfconfig.LoadDnfConfig: os.ReadFile on %s returned %s", cfgPath, readErr) } @@ -206,6 +220,17 @@ func LoadDnfConfig() (*DnfConfig, error) { repoBundleConfig.BaseURLFormat, bundleName) } repoBundleConfig.baseURLFormatTemplate = t + + priority := repoBundleConfig.Priority + if priority > 0 { + if priority == 1 { + return nil, fmt.Errorf( + "dnfconfig.LoadDnfConfig: Priority 1 is reserved for local deps, please provide a priority > 1") + } + } else { + return nil, fmt.Errorf("dnfconfig.LoadDnfConfig: Wrong priority %d provided / Priority not set."+ + " Please provide a valid priority > 1", priority) + } } return &config, nil } diff --git a/dnfconfig/testData/sample-dnfconfig.yaml b/dnfconfig/testData/sample-dnfconfig.yaml index edba969..7726598 100644 --- a/dnfconfig/testData/sample-dnfconfig.yaml +++ b/dnfconfig/testData/sample-dnfconfig.yaml @@ -12,6 +12,7 @@ repo-bundle: version-labels: latest: 999 default: 1 + priority: 2 bundle2: baseurl: "{{.Host}}/bundle2-{{.Version}}/{{.RepoName}}/{{.Arch}}/" use-base-arch: true @@ -23,3 +24,4 @@ repo-bundle: version-labels: latest: 999 default: 1 + priority: 3 diff --git a/impl/testData/dnfconfig.yaml b/impl/testData/dnfconfig.yaml index 39d172f..7122b25 100644 --- a/impl/testData/dnfconfig.yaml +++ b/impl/testData/dnfconfig.yaml @@ -15,6 +15,7 @@ repo-bundle: enabled: true version-labels: default: v1 + priority: 2 bundle-boo2: baseurl: "{{.Host}}/boo2-{{.Version}}/{{.RepoName}}/{{.Arch}}/" use-base-arch: true @@ -24,6 +25,7 @@ repo-bundle: version-labels: default: v1 latest: v3 + priority: 2 bundle-boo3: baseurl: "{{.Host}}/boo3-{{.Version}}/{{.RepoName}}/{{.Arch}}/" repo: @@ -32,3 +34,4 @@ repo-bundle: version-labels: default: v1 latest: v3 + priority: 3 diff --git a/impl/testData/expected-mock.cfg b/impl/testData/expected-mock.cfg index b432f01..43f132b 100644 --- a/impl/testData/expected-mock.cfg +++ b/impl/testData/expected-mock.cfg @@ -79,7 +79,7 @@ baseurl = https://foo.org/boo1-v1/repo-roo12/x86_64/ enabled = 0 gpgcheck = 1 gpgkey = file:///keyfile -priority = 2 +priority = 3 [repo-roo13] name = repo-roo13 diff --git a/impl/testData/manifest-with-deps.yaml b/impl/testData/manifest-with-deps.yaml index e321371..3649e30 100644 --- a/impl/testData/manifest-with-deps.yaml +++ b/impl/testData/manifest-with-deps.yaml @@ -13,6 +13,7 @@ package: exclude: "roo1-rpm.rpm" repo-roo12: enabled: false + priority: 3 - name: bundle-boo2 version: v2 - name: bundle-boo2 diff --git a/impl/testData/manifest.yaml b/impl/testData/manifest.yaml index 37c4273..ad233d1 100644 --- a/impl/testData/manifest.yaml +++ b/impl/testData/manifest.yaml @@ -13,6 +13,7 @@ package: exclude: "roo1-rpm.rpm" repo-roo12: enabled: false + priority: 3 - name: bundle-boo2 version: v2 - name: bundle-boo2 diff --git a/manifest/testData/sampleManifest1.yaml b/manifest/testData/sampleManifest1.yaml index 6d1be68..7f957a0 100644 --- a/manifest/testData/sampleManifest1.yaml +++ b/manifest/testData/sampleManifest1.yaml @@ -15,6 +15,7 @@ package: rfoo: enabled: true exclude: "rfoo.rpm" + priority: 4 - name: bar - name: tcpdump upstream-sources: From 9457d9621fe730df1b9b9797c4a7bd0164f78ff9 Mon Sep 17 00:00:00 2001 From: Manith Date: Wed, 24 Jul 2024 12:45:33 +0000 Subject: [PATCH 2/3] Add support for fc40-unsafe and fc40-snapshot repo eext packages have a dependency on Fedora distro rpms. Adding these rpms to eext was cumbersome due to the number of new rpms introduced. Instead we will satisfy Fedora dependencies through a new repo bundle, fc40-unsafe and create a snapshot fc40-snapshot to ensure build reproducibility. fc40-unsafe is a local cache of the required dependencies pulled in from upstream by eext. eext developers snapshot this repo when requested by repo owners to pull in fedora dependencies. --- configfiles/dnfconfig.yaml | 41 +++++++++++++++++++++++++++++++++ dnfconfig/defaultconfig_test.go | 32 +++++++++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/configfiles/dnfconfig.yaml b/configfiles/dnfconfig.yaml index f4d3399..999e3c8 100644 --- a/configfiles/dnfconfig.yaml +++ b/configfiles/dnfconfig.yaml @@ -74,6 +74,25 @@ repo-bundle: priority: 2 #--------------------------------------------------------------------------------------------- +#--------------------------------------------------------------------------------------------- + fc40-snapshot: + # DO NOT use fc40-snapshot as a repo-bundle in your eext.yaml, + # unless recommended to you by the eext team. + # The eext team is responsible for creating these snapshots. + + gpgcheck: true + gpgkey: file:///usr/share/distribution-gpg-keys/fedora/RPM-GPG-KEY-fedora-40-primary + + # fc40 publishes i686 RPMS in x86_64 repo. This ensures that we point to the x86_64 repo + # when target is i686. + use-base-arch: true + baseurl: "{{.Host}}/artifactory/eext-snapshots-local/fc40/{{.Version}}/{{.RepoName}}/40/Everything/{{.Arch}}/os" + repo: + releases: + enabled: true + priority: 3 +#--------------------------------------------------------------------------------------------- + # -------------------------------------------------------------------------------------------- el9-unsafe: # DO NOT use el9-unsafe as a repo-bundle in your eext.yaml unless you know what you're doing. @@ -182,3 +201,25 @@ repo-bundle: default: 9 priority: 2 # -------------------------------------------------------------------------------------------- + +#--------------------------------------------------------------------------------------------- + fc40-unsafe: + # DO NOT use fc40-unsafe as a repo-bundle in your eext.yaml + # unless you know what you're doing. Use fc40-snapshot instead because this + # will ensure build reproducibility. + # fc40-unsafe is used by the eext team for experiments. + + gpgcheck: true + gpgkey: file:///usr/share/distribution-gpg-keys/fedora/RPM-GPG-KEY-fedora-40-primary + + # fc40 publishes i686 RPMS in x86_64 repo. This ensures that we point to the x86_64 repo + # when target is i686. + use-base-arch: true + baseurl: "{{.Host}}/artifactory/eext-fedora-linux/{{.RepoName}}/{{.Version}}/Everything/{{.Arch}}/os" + repo: + releases: + enabled: true + version-labels: + default: 40 + priority: 3 +#--------------------------------------------------------------------------------------------- diff --git a/dnfconfig/defaultconfig_test.go b/dnfconfig/defaultconfig_test.go index 9d15ab4..d5a9471 100644 --- a/dnfconfig/defaultconfig_test.go +++ b/dnfconfig/defaultconfig_test.go @@ -153,6 +153,38 @@ func TestDefaultDnfRepoConfig(t *testing.T) { }, defaultVersion: "9", }, + "fc40-unsafe": ExpectedDefaultRepoBundle{ + repoToURLFormatString: map[string]string{ + "releases": "%s/artifactory/%s/releases/%s/Everything/%s/os", + }, + archToArtifactoryRepo: map[string]string{ + "i686": "eext-fedora-linux", + "x86_64": "eext-fedora-linux", + "aarch64": "eext-fedora-linux", + }, + archToURLFormatArch: map[string]string{ + "i686": "x86_64", // baseArch + "x86_64": "x86_64", + "aarch64": "aarch64", + }, + defaultVersion: "40", + }, + "fc40-snapshot": ExpectedDefaultRepoBundle{ + repoToURLFormatString: map[string]string{ + "releases": "%s/artifactory/%s/fc40/default/releases/%s/Everything/%s/os", + }, + archToArtifactoryRepo: map[string]string{ + "i686": "eext-snapshots-local", + "x86_64": "eext-snapshots-local", + "aarch64": "eext-snapshots-local", + }, + archToURLFormatArch: map[string]string{ + "i686": "x86_64", // baseArch + "x86_64": "x86_64", + "aarch64": "aarch64", + }, + defaultVersion: "40", + }, } t.Log("Testing expected defaults") From 7374f585ee4bf37265abd7c62b64f50c0b8de3c4 Mon Sep 17 00:00:00 2001 From: Manith Date: Thu, 25 Jul 2024 15:07:36 +0000 Subject: [PATCH 3/3] Add check to prevent negative priority Update code to add check for negative priority that could be provided by the user. --- dnfconfig/dnfconfig.go | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/dnfconfig/dnfconfig.go b/dnfconfig/dnfconfig.go index 8ba7d00..a50ed2f 100644 --- a/dnfconfig/dnfconfig.go +++ b/dnfconfig/dnfconfig.go @@ -149,7 +149,7 @@ func (b *DnfRepoBundleConfig) GetDnfRepoParams( return nil, err } - repoPriority := b.Priority + repoBundlePriority := b.Priority var enabled bool var exclude string @@ -159,18 +159,21 @@ func (b *DnfRepoBundleConfig) GetDnfRepoParams( enabled = repoOverride.Enabled exclude = repoOverride.Exclude priorityOverride := repoOverride.Priority - if priorityOverride != 0 { - if priorityOverride == 1 { + if priorityOverride == 0 { + // If repo priority not set, use repo-bundle priority + priority = repoBundlePriority + } else { + if priorityOverride > 1 { + priority = priorityOverride + } else if priorityOverride == 1 { return nil, fmt.Errorf("%sRepo %s priority cannot be 1. Provide a priority > 1", errPrefix, repoName) } else { - priority = priorityOverride + return nil, fmt.Errorf("%sInvaid repo priority %d. Provide a priority > 1", errPrefix, priorityOverride) } - } else { - priority = repoPriority } } else { enabled = repoConfig.Enabled - priority = repoPriority + priority = repoBundlePriority } return &DnfRepoParams{ @@ -222,14 +225,14 @@ func LoadDnfConfig() (*DnfConfig, error) { repoBundleConfig.baseURLFormatTemplate = t priority := repoBundleConfig.Priority - if priority > 0 { - if priority == 1 { - return nil, fmt.Errorf( - "dnfconfig.LoadDnfConfig: Priority 1 is reserved for local deps, please provide a priority > 1") - } - } else { - return nil, fmt.Errorf("dnfconfig.LoadDnfConfig: Wrong priority %d provided / Priority not set."+ - " Please provide a valid priority > 1", priority) + if priority == 0 { + return nil, fmt.Errorf("dnfconfig.LoadDnfConfig: Priority not set. Please provide a valid priority > 1") + } else if priority == 1 { + return nil, fmt.Errorf( + "dnfconfig.LoadDnfConfig: Priority 1 is reserved for local deps, please provide a priority > 1") + } else if priority < 0 { + return nil, fmt.Errorf( + "dnfconfig.LoadDnfConfig: Wrong priority %d provided Please provide a valid priority > 1", priority) } } return &config, nil