Skip to content

Commit

Permalink
RHINENG-2473: parse repository_paths and use in vmaas request
Browse files Browse the repository at this point in the history
  • Loading branch information
psegedy committed Nov 6, 2023
1 parent 8ec7385 commit 45099df
Show file tree
Hide file tree
Showing 5 changed files with 139 additions and 32 deletions.
8 changes: 5 additions & 3 deletions base/inventory/inventory.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,11 @@ type OperatingSystem struct {
}

type YumRepo struct {
ID string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
Enabled bool `json:"enabled,omitempty"`
ID string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
Enabled bool `json:"enabled,omitempty"`
Mirrorlist string `json:"mirrorlist,omitempty"`
BaseURL string `json:"base_url,omitempty"`
}

type DnfModule struct {
Expand Down
15 changes: 8 additions & 7 deletions base/vmaas/vmaas.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,14 @@ import (
)

type UpdatesV3Request struct {
PackageList []string `json:"package_list"`
RepositoryList []string `json:"repository_list"`
ModulesList *[]UpdatesV3RequestModulesList `json:"modules_list,omitempty"`
Releasever *string `json:"releasever,omitempty"`
Basearch *string `json:"basearch,omitempty"`
SecurityOnly *bool `json:"security_only,omitempty"`
LatestOnly *bool `json:"latest_only,omitempty"`
PackageList []string `json:"package_list"`
RepositoryList []string `json:"repository_list"`
RepositoryPaths []string `json:"repository_paths"`
ModulesList *[]UpdatesV3RequestModulesList `json:"modules_list,omitempty"`
Releasever *string `json:"releasever,omitempty"`
Basearch *string `json:"basearch,omitempty"`
SecurityOnly *bool `json:"security_only,omitempty"`
LatestOnly *bool `json:"latest_only,omitempty"`
// Include content from \"third party\" repositories into the response, disabled by default.
ThirdParty *bool `json:"third_party,omitempty"`
// Search for updates of unknown package EVRAs.
Expand Down
6 changes: 6 additions & 0 deletions docs/v3/openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -7322,6 +7322,12 @@
"type": "string"
}
},
"repository_paths": {
"type": "array",
"items": {
"type": "string"
}
},
"security_only": {
"type": "boolean"
},
Expand Down
97 changes: 75 additions & 22 deletions listener/upload.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ import (
"encoding/json"
"fmt"
"net/http"
"net/url"
"regexp"
"strings"
"sync"
"time"
Expand All @@ -26,24 +28,29 @@ import (
)

const (
WarnSkippingNoPackages = "skipping profile with no packages"
WarnSkippingReporter = "skipping excluded reporter"
WarnSkippingHostType = "skipping excluded host type"
WarnPayloadTracker = "unable to send message to payload tracker"
ErrorNoAccountProvided = "no account provided in host message"
ErrorKafkaSend = "unable to send evaluation message"
ErrorProcessUpload = "unable to process upload"
UploadSuccessNoEval = "upload event handled successfully, no eval required"
UploadSuccess = "upload event handled successfully"
FlushedFullBuffer = "flushing full eval event buffer"
FlushedTimeoutBuffer = "flushing eval event buffer after timeout"
ErrorUnmarshalMetadata = "unable to unmarshall platform metadata value"
ErrorStatus = "error"
SuccessStatus = "success"
WarnSkippingNoPackages = "skipping profile with no packages"
WarnSkippingReporter = "skipping excluded reporter"
WarnSkippingHostType = "skipping excluded host type"
WarnPayloadTracker = "unable to send message to payload tracker"
ErrorNoAccountProvided = "no account provided in host message"
ErrorKafkaSend = "unable to send evaluation message"
ErrorProcessUpload = "unable to process upload"
UploadSuccessNoEval = "upload event handled successfully, no eval required"
UploadSuccess = "upload event handled successfully"
FlushedFullBuffer = "flushing full eval event buffer"
FlushedTimeoutBuffer = "flushing eval event buffer after timeout"
ErrorUnmarshalMetadata = "unable to unmarshall platform metadata value"
ErrorStatus = "error"
SuccessStatus = "success"
RhuiPathPart = "/rhui/"
RepoPathPattern = "(/content/.*)"
RepoBasearchPlaceholder = "$basearch"
RepoReleaseverPlaceholder = "$releasever"
)

var (
DeletionThreshold = time.Hour * time.Duration(utils.GetIntEnvOrDefault("SYSTEM_DELETE_HRS", 4))
repoPathRegex = regexp.MustCompile(RepoPathPattern)
httpClient *api.Client
)

Expand Down Expand Up @@ -522,11 +529,47 @@ func deleteOtherSystemRepos(tx *gorm.DB, rhAccountID int, systemID int64, repoID
return res.DeletedCount, nil
}

func processRepos(systemProfile *inventory.SystemProfile) []string {
// get rhui repository path form `system_profile.yum_repos.mirrorlist` or `system_profile.yum_repos.base_url`
func getRepoPath(systemProfile *inventory.SystemProfile, repo *inventory.YumRepo) (string, error) {
var repoPath string
if systemProfile == nil || repo == nil {
return repoPath, nil
}
if len(repo.Mirrorlist) == 0 && len(repo.BaseURL) == 0 {
return repoPath, nil
}

repoURL := repo.Mirrorlist
if len(repoURL) == 0 {
repoURL = repo.BaseURL
}

url, err := url.Parse(repoURL)
if err != nil {
return repoPath, errors.Wrap(err, "couldn't parse repo mirrorlist or base_url")
}

foundRepoPath := repoPathRegex.FindString(url.Path)
if strings.Contains(foundRepoPath, RhuiPathPart) {
repoPath = foundRepoPath
if systemProfile.Arch != nil {
repoPath = strings.ReplaceAll(repoPath, RepoBasearchPlaceholder, *systemProfile.Arch)
}
if systemProfile.Releasever != nil {
repoPath = strings.ReplaceAll(repoPath, RepoReleaseverPlaceholder, *systemProfile.Releasever)
}
return repoPath, nil
}
return repoPath, nil
}

func processRepos(systemProfile *inventory.SystemProfile) ([]string, []string) {
yumRepos := systemProfile.GetYumRepos()
seen := make(map[string]bool, len(yumRepos))
repos := make([]string, 0, len(yumRepos))
repoPaths := make([]string, 0, len(yumRepos))
for _, r := range yumRepos {
r := r
rID := r.ID
if seen[rID] {
// remove duplicate repos
Expand All @@ -541,9 +584,17 @@ func processRepos(systemProfile *inventory.SystemProfile) []string {
if r.Enabled {
repos = append(repos, rID)
}

repoPath, err := getRepoPath(systemProfile, &r)
if err != nil {
utils.LogWarn("repo", rID, "mirrorlist", r.Mirrorlist, "base_url", r.BaseURL, "invalid repository_path")
}
if len(repoPath) > 0 {
repoPaths = append(repoPaths, repoPath)
}
}
fixEpelRepos(systemProfile, repos)
return repos
return repos, repoPaths
}

func processModules(systemProfile *inventory.SystemProfile) *[]vmaas.UpdatesV3RequestModulesList {
Expand Down Expand Up @@ -571,14 +622,16 @@ func processUpload(host *Host, yumUpdates *YumUpdates) (*models.SystemPlatform,
}

systemProfile := host.SystemProfile
repos, repoPaths := processRepos(&systemProfile)
// Prepare VMaaS request
updatesReq := vmaas.UpdatesV3Request{
PackageList: systemProfile.GetInstalledPackages(),
RepositoryList: processRepos(&systemProfile),
ModulesList: processModules(&systemProfile),
Basearch: systemProfile.Arch,
SecurityOnly: utils.PtrBool(false),
LatestOnly: utils.PtrBool(true),
PackageList: systemProfile.GetInstalledPackages(),
RepositoryList: repos,
RepositoryPaths: repoPaths,
ModulesList: processModules(&systemProfile),
Basearch: systemProfile.Arch,
SecurityOnly: utils.PtrBool(false),
LatestOnly: utils.PtrBool(true),
}

// use rhsm version if set
Expand Down
45 changes: 45 additions & 0 deletions listener/upload_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -370,3 +370,48 @@ func TestStoreOrUpdateSysPlatform(t *testing.T) {
maxInc := currval - nextval
assert.Equal(t, countInc, maxInc)
}

func TestGetRepoPath(t *testing.T) {
repoPath, err := getRepoPath(nil, nil)
assert.Nil(t, err)
assert.Empty(t, repoPath)

repo := inventory.YumRepo{}
sp := inventory.SystemProfile{}

repoPath, err = getRepoPath(&sp, &repo)
assert.Nil(t, err)
assert.Empty(t, repoPath)

repo = inventory.YumRepo{Mirrorlist: "://asdf"}
repoPath, err = getRepoPath(&sp, &repo)
assert.NotNil(t, err)
assert.Empty(t, repoPath)

repo = inventory.YumRepo{Mirrorlist: "https://rhui.redhat.com["}
repoPath, err = getRepoPath(&sp, &repo)
assert.Nil(t, err)
assert.Empty(t, repoPath)

repo = inventory.YumRepo{BaseURL: "https://rhui.redhat.com/"}
repoPath, err = getRepoPath(&sp, &repo)
assert.Nil(t, err)
assert.Empty(t, repoPath)

repo = inventory.YumRepo{
BaseURL: "https://rhui.redhat.com/pulp/mirror/content/dist/rhel8/rhui/8.4/x86_64/baseos/os",
}
repoPath, err = getRepoPath(&sp, &repo)
assert.Nil(t, err)
assert.Equal(t, "/content/dist/rhel8/rhui/8.4/x86_64/baseos/os", repoPath)

repo = inventory.YumRepo{
BaseURL: "https://rhui.redhat.com/pulp/mirror/content/dist/rhel8/rhui/$releasever/$basearch/baseos/os",
}
arch := "x86_64"
release := "8.4"
sp = inventory.SystemProfile{Arch: &arch, Releasever: &release}
repoPath, err = getRepoPath(&sp, &repo)
assert.Nil(t, err)
assert.Equal(t, "/content/dist/rhel8/rhui/8.4/x86_64/baseos/os", repoPath)
}

0 comments on commit 45099df

Please sign in to comment.