From 45099dffedec49f7b7cbc6b868a1e0fb8daf102d Mon Sep 17 00:00:00 2001 From: Patrik Segedy Date: Mon, 6 Nov 2023 16:18:00 +0100 Subject: [PATCH] RHINENG-2473: parse repository_paths and use in vmaas request --- base/inventory/inventory.go | 8 +-- base/vmaas/vmaas.go | 15 +++--- docs/v3/openapi.json | 6 +++ listener/upload.go | 97 ++++++++++++++++++++++++++++--------- listener/upload_test.go | 45 +++++++++++++++++ 5 files changed, 139 insertions(+), 32 deletions(-) diff --git a/base/inventory/inventory.go b/base/inventory/inventory.go index 24e0dfa6e..c45cf4782 100644 --- a/base/inventory/inventory.go +++ b/base/inventory/inventory.go @@ -40,9 +40,11 @@ type OperatingSystem struct { } type YumRepo struct { - ID string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - Enabled bool `json:"enabled,omitempty"` + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Enabled bool `json:"enabled,omitempty"` + Mirrorlist string `json:"mirrorlist,omitempty"` + BaseURL string `json:"base_url,omitempty"` } type DnfModule struct { diff --git a/base/vmaas/vmaas.go b/base/vmaas/vmaas.go index f16f5a17f..b86f51d83 100644 --- a/base/vmaas/vmaas.go +++ b/base/vmaas/vmaas.go @@ -6,13 +6,14 @@ import ( ) type UpdatesV3Request struct { - PackageList []string `json:"package_list"` - RepositoryList []string `json:"repository_list"` - ModulesList *[]UpdatesV3RequestModulesList `json:"modules_list,omitempty"` - Releasever *string `json:"releasever,omitempty"` - Basearch *string `json:"basearch,omitempty"` - SecurityOnly *bool `json:"security_only,omitempty"` - LatestOnly *bool `json:"latest_only,omitempty"` + PackageList []string `json:"package_list"` + RepositoryList []string `json:"repository_list"` + RepositoryPaths []string `json:"repository_paths"` + ModulesList *[]UpdatesV3RequestModulesList `json:"modules_list,omitempty"` + Releasever *string `json:"releasever,omitempty"` + Basearch *string `json:"basearch,omitempty"` + SecurityOnly *bool `json:"security_only,omitempty"` + LatestOnly *bool `json:"latest_only,omitempty"` // Include content from \"third party\" repositories into the response, disabled by default. ThirdParty *bool `json:"third_party,omitempty"` // Search for updates of unknown package EVRAs. diff --git a/docs/v3/openapi.json b/docs/v3/openapi.json index cb92f9834..f00bbb444 100644 --- a/docs/v3/openapi.json +++ b/docs/v3/openapi.json @@ -7322,6 +7322,12 @@ "type": "string" } }, + "repository_paths": { + "type": "array", + "items": { + "type": "string" + } + }, "security_only": { "type": "boolean" }, diff --git a/listener/upload.go b/listener/upload.go index 2131dbe00..1a8f3b8b3 100644 --- a/listener/upload.go +++ b/listener/upload.go @@ -16,6 +16,8 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" + "regexp" "strings" "sync" "time" @@ -26,24 +28,29 @@ import ( ) const ( - WarnSkippingNoPackages = "skipping profile with no packages" - WarnSkippingReporter = "skipping excluded reporter" - WarnSkippingHostType = "skipping excluded host type" - WarnPayloadTracker = "unable to send message to payload tracker" - ErrorNoAccountProvided = "no account provided in host message" - ErrorKafkaSend = "unable to send evaluation message" - ErrorProcessUpload = "unable to process upload" - UploadSuccessNoEval = "upload event handled successfully, no eval required" - UploadSuccess = "upload event handled successfully" - FlushedFullBuffer = "flushing full eval event buffer" - FlushedTimeoutBuffer = "flushing eval event buffer after timeout" - ErrorUnmarshalMetadata = "unable to unmarshall platform metadata value" - ErrorStatus = "error" - SuccessStatus = "success" + WarnSkippingNoPackages = "skipping profile with no packages" + WarnSkippingReporter = "skipping excluded reporter" + WarnSkippingHostType = "skipping excluded host type" + WarnPayloadTracker = "unable to send message to payload tracker" + ErrorNoAccountProvided = "no account provided in host message" + ErrorKafkaSend = "unable to send evaluation message" + ErrorProcessUpload = "unable to process upload" + UploadSuccessNoEval = "upload event handled successfully, no eval required" + UploadSuccess = "upload event handled successfully" + FlushedFullBuffer = "flushing full eval event buffer" + FlushedTimeoutBuffer = "flushing eval event buffer after timeout" + ErrorUnmarshalMetadata = "unable to unmarshall platform metadata value" + ErrorStatus = "error" + SuccessStatus = "success" + RhuiPathPart = "/rhui/" + RepoPathPattern = "(/content/.*)" + RepoBasearchPlaceholder = "$basearch" + RepoReleaseverPlaceholder = "$releasever" ) var ( DeletionThreshold = time.Hour * time.Duration(utils.GetIntEnvOrDefault("SYSTEM_DELETE_HRS", 4)) + repoPathRegex = regexp.MustCompile(RepoPathPattern) httpClient *api.Client ) @@ -522,11 +529,47 @@ func deleteOtherSystemRepos(tx *gorm.DB, rhAccountID int, systemID int64, repoID return res.DeletedCount, nil } -func processRepos(systemProfile *inventory.SystemProfile) []string { +// get rhui repository path form `system_profile.yum_repos.mirrorlist` or `system_profile.yum_repos.base_url` +func getRepoPath(systemProfile *inventory.SystemProfile, repo *inventory.YumRepo) (string, error) { + var repoPath string + if systemProfile == nil || repo == nil { + return repoPath, nil + } + if len(repo.Mirrorlist) == 0 && len(repo.BaseURL) == 0 { + return repoPath, nil + } + + repoURL := repo.Mirrorlist + if len(repoURL) == 0 { + repoURL = repo.BaseURL + } + + url, err := url.Parse(repoURL) + if err != nil { + return repoPath, errors.Wrap(err, "couldn't parse repo mirrorlist or base_url") + } + + foundRepoPath := repoPathRegex.FindString(url.Path) + if strings.Contains(foundRepoPath, RhuiPathPart) { + repoPath = foundRepoPath + if systemProfile.Arch != nil { + repoPath = strings.ReplaceAll(repoPath, RepoBasearchPlaceholder, *systemProfile.Arch) + } + if systemProfile.Releasever != nil { + repoPath = strings.ReplaceAll(repoPath, RepoReleaseverPlaceholder, *systemProfile.Releasever) + } + return repoPath, nil + } + return repoPath, nil +} + +func processRepos(systemProfile *inventory.SystemProfile) ([]string, []string) { yumRepos := systemProfile.GetYumRepos() seen := make(map[string]bool, len(yumRepos)) repos := make([]string, 0, len(yumRepos)) + repoPaths := make([]string, 0, len(yumRepos)) for _, r := range yumRepos { + r := r rID := r.ID if seen[rID] { // remove duplicate repos @@ -541,9 +584,17 @@ func processRepos(systemProfile *inventory.SystemProfile) []string { if r.Enabled { repos = append(repos, rID) } + + repoPath, err := getRepoPath(systemProfile, &r) + if err != nil { + utils.LogWarn("repo", rID, "mirrorlist", r.Mirrorlist, "base_url", r.BaseURL, "invalid repository_path") + } + if len(repoPath) > 0 { + repoPaths = append(repoPaths, repoPath) + } } fixEpelRepos(systemProfile, repos) - return repos + return repos, repoPaths } func processModules(systemProfile *inventory.SystemProfile) *[]vmaas.UpdatesV3RequestModulesList { @@ -571,14 +622,16 @@ func processUpload(host *Host, yumUpdates *YumUpdates) (*models.SystemPlatform, } systemProfile := host.SystemProfile + repos, repoPaths := processRepos(&systemProfile) // Prepare VMaaS request updatesReq := vmaas.UpdatesV3Request{ - PackageList: systemProfile.GetInstalledPackages(), - RepositoryList: processRepos(&systemProfile), - ModulesList: processModules(&systemProfile), - Basearch: systemProfile.Arch, - SecurityOnly: utils.PtrBool(false), - LatestOnly: utils.PtrBool(true), + PackageList: systemProfile.GetInstalledPackages(), + RepositoryList: repos, + RepositoryPaths: repoPaths, + ModulesList: processModules(&systemProfile), + Basearch: systemProfile.Arch, + SecurityOnly: utils.PtrBool(false), + LatestOnly: utils.PtrBool(true), } // use rhsm version if set diff --git a/listener/upload_test.go b/listener/upload_test.go index 0d100c648..3c88737df 100644 --- a/listener/upload_test.go +++ b/listener/upload_test.go @@ -370,3 +370,48 @@ func TestStoreOrUpdateSysPlatform(t *testing.T) { maxInc := currval - nextval assert.Equal(t, countInc, maxInc) } + +func TestGetRepoPath(t *testing.T) { + repoPath, err := getRepoPath(nil, nil) + assert.Nil(t, err) + assert.Empty(t, repoPath) + + repo := inventory.YumRepo{} + sp := inventory.SystemProfile{} + + repoPath, err = getRepoPath(&sp, &repo) + assert.Nil(t, err) + assert.Empty(t, repoPath) + + repo = inventory.YumRepo{Mirrorlist: "://asdf"} + repoPath, err = getRepoPath(&sp, &repo) + assert.NotNil(t, err) + assert.Empty(t, repoPath) + + repo = inventory.YumRepo{Mirrorlist: "https://rhui.redhat.com["} + repoPath, err = getRepoPath(&sp, &repo) + assert.Nil(t, err) + assert.Empty(t, repoPath) + + repo = inventory.YumRepo{BaseURL: "https://rhui.redhat.com/"} + repoPath, err = getRepoPath(&sp, &repo) + assert.Nil(t, err) + assert.Empty(t, repoPath) + + repo = inventory.YumRepo{ + BaseURL: "https://rhui.redhat.com/pulp/mirror/content/dist/rhel8/rhui/8.4/x86_64/baseos/os", + } + repoPath, err = getRepoPath(&sp, &repo) + assert.Nil(t, err) + assert.Equal(t, "/content/dist/rhel8/rhui/8.4/x86_64/baseos/os", repoPath) + + repo = inventory.YumRepo{ + BaseURL: "https://rhui.redhat.com/pulp/mirror/content/dist/rhel8/rhui/$releasever/$basearch/baseos/os", + } + arch := "x86_64" + release := "8.4" + sp = inventory.SystemProfile{Arch: &arch, Releasever: &release} + repoPath, err = getRepoPath(&sp, &repo) + assert.Nil(t, err) + assert.Equal(t, "/content/dist/rhel8/rhui/8.4/x86_64/baseos/os", repoPath) +}