RHINENG-2473: parse repository_paths and use in vmaas request

base/inventory/inventory.go

@@ -40,9 +40,11 @@ type OperatingSystem struct {
 }
 
 type YumRepo struct {
-	ID      string `json:"id,omitempty"`
-	Name    string `json:"name,omitempty"`
-	Enabled bool   `json:"enabled,omitempty"`
+	ID         string `json:"id,omitempty"`
+	Name       string `json:"name,omitempty"`
+	Enabled    bool   `json:"enabled,omitempty"`
+	Mirrorlist string `json:"mirrorlist,omitempty"`
+	BaseURL    string `json:"base_url,omitempty"`
 }
 
 type DnfModule struct {

base/vmaas/vmaas.go

@@ -6,13 +6,14 @@ import (
 )
 
 type UpdatesV3Request struct {
-	PackageList    []string                       `json:"package_list"`
-	RepositoryList []string                       `json:"repository_list"`
-	ModulesList    *[]UpdatesV3RequestModulesList `json:"modules_list,omitempty"`
-	Releasever     *string                        `json:"releasever,omitempty"`
-	Basearch       *string                        `json:"basearch,omitempty"`
-	SecurityOnly   *bool                          `json:"security_only,omitempty"`
-	LatestOnly     *bool                          `json:"latest_only,omitempty"`
+	PackageList     []string                       `json:"package_list"`
+	RepositoryList  []string                       `json:"repository_list"`
+	RepositoryPaths []string                       `json:"repository_paths"`
+	ModulesList     *[]UpdatesV3RequestModulesList `json:"modules_list,omitempty"`
+	Releasever      *string                        `json:"releasever,omitempty"`
+	Basearch        *string                        `json:"basearch,omitempty"`
+	SecurityOnly    *bool                          `json:"security_only,omitempty"`
+	LatestOnly      *bool                          `json:"latest_only,omitempty"`
 	// Include content from \"third party\" repositories into the response, disabled by default.
 	ThirdParty *bool `json:"third_party,omitempty"`
 	// Search for updates of unknown package EVRAs.

docs/v3/openapi.json

@@ -7322,6 +7322,12 @@
                             "type": "string"
                         }
                     },
+                    "repository_paths": {
+                        "type": "array",
+                        "items": {
+                            "type": "string"
+                        }
+                    },
                     "security_only": {
                         "type": "boolean"
                     },

listener/upload.go

@@ -16,6 +16,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"net/url"
+	"regexp"
 	"strings"
 	"sync"
 	"time"
@@ -26,24 +28,29 @@ import (
 )
 
 const (
-	WarnSkippingNoPackages = "skipping profile with no packages"
-	WarnSkippingReporter   = "skipping excluded reporter"
-	WarnSkippingHostType   = "skipping excluded host type"
-	WarnPayloadTracker     = "unable to send message to payload tracker"
-	ErrorNoAccountProvided = "no account provided in host message"
-	ErrorKafkaSend         = "unable to send evaluation message"
-	ErrorProcessUpload     = "unable to process upload"
-	UploadSuccessNoEval    = "upload event handled successfully, no eval required"
-	UploadSuccess          = "upload event handled successfully"
-	FlushedFullBuffer      = "flushing full eval event buffer"
-	FlushedTimeoutBuffer   = "flushing eval event buffer after timeout"
-	ErrorUnmarshalMetadata = "unable to unmarshall platform metadata value"
-	ErrorStatus            = "error"
-	SuccessStatus          = "success"
+	WarnSkippingNoPackages    = "skipping profile with no packages"
+	WarnSkippingReporter      = "skipping excluded reporter"
+	WarnSkippingHostType      = "skipping excluded host type"
+	WarnPayloadTracker        = "unable to send message to payload tracker"
+	ErrorNoAccountProvided    = "no account provided in host message"
+	ErrorKafkaSend            = "unable to send evaluation message"
+	ErrorProcessUpload        = "unable to process upload"
+	UploadSuccessNoEval       = "upload event handled successfully, no eval required"
+	UploadSuccess             = "upload event handled successfully"
+	FlushedFullBuffer         = "flushing full eval event buffer"
+	FlushedTimeoutBuffer      = "flushing eval event buffer after timeout"
+	ErrorUnmarshalMetadata    = "unable to unmarshall platform metadata value"
+	ErrorStatus               = "error"
+	SuccessStatus             = "success"
+	RhuiPathPart              = "/rhui/"
+	RepoPathPattern           = "(/content/.*)"
+	RepoBasearchPlaceholder   = "$basearch"
+	RepoReleaseverPlaceholder = "$releasever"
 )
 
 var (
 	DeletionThreshold = time.Hour * time.Duration(utils.GetIntEnvOrDefault("SYSTEM_DELETE_HRS", 4))
+	repoPathRegex     = regexp.MustCompile(RepoPathPattern)
 	httpClient        *api.Client
 )
 
@@ -522,11 +529,47 @@ func deleteOtherSystemRepos(tx *gorm.DB, rhAccountID int, systemID int64, repoID
 	return res.DeletedCount, nil
 }
 
-func processRepos(systemProfile *inventory.SystemProfile) []string {
+// get rhui repository path form `system_profile.yum_repos.mirrorlist` or `system_profile.yum_repos.base_url`
+func getRepoPath(systemProfile *inventory.SystemProfile, repo *inventory.YumRepo) (string, error) {
+	var repoPath string
+	if systemProfile == nil || repo == nil {
+		return repoPath, nil
+	}
+	if len(repo.Mirrorlist) == 0 && len(repo.BaseURL) == 0 {
+		return repoPath, nil
+	}
+
+	repoURL := repo.Mirrorlist
+	if len(repoURL) == 0 {
+		repoURL = repo.BaseURL
+	}
+
+	url, err := url.Parse(repoURL)
+	if err != nil {
+		return repoPath, errors.Wrap(err, "couldn't parse repo mirrorlist or base_url")
+	}
+
+	foundRepoPath := repoPathRegex.FindString(url.Path)
+	if strings.Contains(foundRepoPath, RhuiPathPart) {
+		repoPath = foundRepoPath
+		if systemProfile.Arch != nil {
+			repoPath = strings.ReplaceAll(repoPath, RepoBasearchPlaceholder, *systemProfile.Arch)
+		}
+		if systemProfile.Releasever != nil {
+			repoPath = strings.ReplaceAll(repoPath, RepoReleaseverPlaceholder, *systemProfile.Releasever)
+		}
+		return repoPath, nil
+	}
+	return repoPath, nil
+}
+
+func processRepos(systemProfile *inventory.SystemProfile) ([]string, []string) {
 	yumRepos := systemProfile.GetYumRepos()
 	seen := make(map[string]bool, len(yumRepos))
 	repos := make([]string, 0, len(yumRepos))
+	repoPaths := make([]string, 0, len(yumRepos))
 	for _, r := range yumRepos {
+		r := r
 		rID := r.ID
 		if seen[rID] {
 			// remove duplicate repos
@@ -541,9 +584,17 @@ func processRepos(systemProfile *inventory.SystemProfile) []string {
 		if r.Enabled {
 			repos = append(repos, rID)
 		}
+
+		repoPath, err := getRepoPath(systemProfile, &r)
+		if err != nil {
+			utils.LogWarn("repo", rID, "mirrorlist", r.Mirrorlist, "base_url", r.BaseURL, "invalid repository_path")
+		}
+		if len(repoPath) > 0 {
+			repoPaths = append(repoPaths, repoPath)
+		}
 	}
 	fixEpelRepos(systemProfile, repos)
-	return repos
+	return repos, repoPaths
 }
 
 func processModules(systemProfile *inventory.SystemProfile) *[]vmaas.UpdatesV3RequestModulesList {
@@ -571,14 +622,16 @@ func processUpload(host *Host, yumUpdates *YumUpdates) (*models.SystemPlatform,
 	}
 
 	systemProfile := host.SystemProfile
+	repos, repoPaths := processRepos(&systemProfile)
 	// Prepare VMaaS request
 	updatesReq := vmaas.UpdatesV3Request{
-		PackageList:    systemProfile.GetInstalledPackages(),
-		RepositoryList: processRepos(&systemProfile),
-		ModulesList:    processModules(&systemProfile),
-		Basearch:       systemProfile.Arch,
-		SecurityOnly:   utils.PtrBool(false),
-		LatestOnly:     utils.PtrBool(true),
+		PackageList:     systemProfile.GetInstalledPackages(),
+		RepositoryList:  repos,
+		RepositoryPaths: repoPaths,
+		ModulesList:     processModules(&systemProfile),
+		Basearch:        systemProfile.Arch,
+		SecurityOnly:    utils.PtrBool(false),
+		LatestOnly:      utils.PtrBool(true),
 	}
 
 	// use rhsm version if set

listener/upload_test.go

@@ -370,3 +370,48 @@ func TestStoreOrUpdateSysPlatform(t *testing.T) {
 	maxInc := currval - nextval
 	assert.Equal(t, countInc, maxInc)
 }
+
+func TestGetRepoPath(t *testing.T) {
+	repoPath, err := getRepoPath(nil, nil)
+	assert.Nil(t, err)
+	assert.Empty(t, repoPath)
+
+	repo := inventory.YumRepo{}
+	sp := inventory.SystemProfile{}
+
+	repoPath, err = getRepoPath(&sp, &repo)
+	assert.Nil(t, err)
+	assert.Empty(t, repoPath)
+
+	repo = inventory.YumRepo{Mirrorlist: "://asdf"}
+	repoPath, err = getRepoPath(&sp, &repo)
+	assert.NotNil(t, err)
+	assert.Empty(t, repoPath)
+
+	repo = inventory.YumRepo{Mirrorlist: "https://rhui.redhat.com["}
+	repoPath, err = getRepoPath(&sp, &repo)
+	assert.Nil(t, err)
+	assert.Empty(t, repoPath)
+
+	repo = inventory.YumRepo{BaseURL: "https://rhui.redhat.com/"}
+	repoPath, err = getRepoPath(&sp, &repo)
+	assert.Nil(t, err)
+	assert.Empty(t, repoPath)
+
+	repo = inventory.YumRepo{
+		BaseURL: "https://rhui.redhat.com/pulp/mirror/content/dist/rhel8/rhui/8.4/x86_64/baseos/os",
+	}
+	repoPath, err = getRepoPath(&sp, &repo)
+	assert.Nil(t, err)
+	assert.Equal(t, "/content/dist/rhel8/rhui/8.4/x86_64/baseos/os", repoPath)
+
+	repo = inventory.YumRepo{
+		BaseURL: "https://rhui.redhat.com/pulp/mirror/content/dist/rhel8/rhui/$releasever/$basearch/baseos/os",
+	}
+	arch := "x86_64"
+	release := "8.4"
+	sp = inventory.SystemProfile{Arch: &arch, Releasever: &release}
+	repoPath, err = getRepoPath(&sp, &repo)
+	assert.Nil(t, err)
+	assert.Equal(t, "/content/dist/rhel8/rhui/8.4/x86_64/baseos/os", repoPath)
+}