diff --git a/base/database/database.go b/base/database/database.go index ffecf5bbb..68628284e 100644 --- a/base/database/database.go +++ b/base/database/database.go @@ -24,7 +24,7 @@ func OnConflictUpdateMulti(db *gorm.DB, keys []string, updateCols ...string) *go type UpExpr struct { Name string - Expr string + Expr interface{} } func OnConflictDoUpdateExpr(db *gorm.DB, keys []string, updateExprs ...UpExpr) *gorm.DB { diff --git a/base/models/models.go b/base/models/models.go index dadee693e..8aaa56958 100644 --- a/base/models/models.go +++ b/base/models/models.go @@ -122,6 +122,26 @@ type SystemPackage struct { NameID int64 } +type SystemPackageData struct { + RhAccountID int `gorm:"primaryKey"` + SystemID int64 `gorm:"primaryKey"` + UpdateData []byte +} + +func (SystemPackageData) TableName() string { + return "system_package_data" +} + +type PackageSystemData struct { + RhAccountID int `gorm:"primaryKey"` + PackageNameID int64 `gorm:"primaryKey"` + UpdateData []byte +} + +func (PackageSystemData) TableName() string { + return "package_system_data" +} + type PackageUpdate struct { EVRA string `json:"evra"` Advisory string `json:"-"` // don't show it in API, we can probably remove it completely later @@ -131,9 +151,15 @@ type PackageUpdate struct { type PackageUpdateData struct { Installed string `json:"installed,omitempty"` Installable string `json:"installable,omitempty"` - Applicable string `'json:"applicable,omitempty"` + Applicable string `json:"applicable,omitempty"` } +// for a given system holds map[package_id]->update_data_json +type SystemPackageUpdateData map[int64]PackageUpdateData + +// for given package_name holds map[system_id]->update_data_json +type PackageSystemUpdateData map[int64]PackageUpdateData + type DeletedSystem struct { InventoryID string WhenDeleted time.Time diff --git a/evaluator/evaluate_packages.go b/evaluator/evaluate_packages.go index 8d3019bde..1f46ffc3d 100644 --- a/evaluator/evaluate_packages.go +++ b/evaluator/evaluate_packages.go @@ -5,8 +5,8 @@ import ( "app/base/models" "app/base/utils" "app/base/vmaas" - "bytes" "encoding/json" + "strconv" "strings" "time" @@ -27,13 +27,13 @@ func analyzePackages(tx *gorm.DB, system *models.SystemPlatform, vmaasData *vmaa return 0, 0, errors.Wrap(err, "lazy package save failed") } - pkgByName, err := loadPackages(tx, system, vmaasData) + pkgByName, installed, updatable, err := loadPackages(tx, system, vmaasData) if err != nil { evaluationCnt.WithLabelValues("error-pkg-data").Inc() return 0, 0, errors.Wrap(err, "Unable to load package data") } - installed, updatable, err = updateSystemPackages(tx, system, pkgByName, vmaasData) + err = updateSystemPackages(tx, system, pkgByName) if err != nil { evaluationCnt.WithLabelValues("error-system-pkgs").Inc() return 0, 0, errors.Wrap(err, "Unable to update system packages") @@ -123,235 +123,214 @@ func updatePackageNameDB(missing *models.PackageName) error { // Find relevant package data based on vmaas results func loadPackages(tx *gorm.DB, system *models.SystemPlatform, - vmaasData *vmaas.UpdatesV3Response) (*map[string]namedPackage, error) { + vmaasData *vmaas.UpdatesV3Response) (map[string]namedPackage, int, int, error) { defer utils.ObserveSecondsSince(time.Now(), evaluationPartDuration.WithLabelValues("packages-load")) - packages, err := loadSystemNEVRAsFromDB(tx, system, vmaasData) + packages, installed, updatable, err := loadSystemNEVRAsFromDB(tx, system, vmaasData) if err != nil { - return nil, errors.Wrap(err, "loading packages") + return nil, 0, 0, errors.Wrap(err, "loading packages") } - - pkgByNevra := packages2NevraMap(packages) - return &pkgByNevra, nil -} - -func packages2NevraMap(packages []namedPackage) map[string]namedPackage { - pkgByNevra := make(map[string]namedPackage, len(packages)) - for _, p := range packages { - // make sure nevra contains epoch even if epoch==0 - nevraString := utils.NEVRAStringE(p.Name, p.EVRA, true) - pkgByNevra[nevraString] = p - } - return pkgByNevra + return packages, installed, updatable, nil } +// nolint: funlen func loadSystemNEVRAsFromDB(tx *gorm.DB, system *models.SystemPlatform, - vmaasData *vmaas.UpdatesV3Response) ([]namedPackage, error) { + vmaasData *vmaas.UpdatesV3Response) (map[string]namedPackage, int, int, error) { + installed := 0 + updatable := 0 updates := vmaasData.GetUpdateList() numUpdates := len(updates) - packageIDs := make([]int64, 0, numUpdates) - packages := make([]namedPackage, 0, numUpdates) - id2index := make(map[int64]int, numUpdates) - i := 0 - for nevra := range updates { + packages := make(map[string]namedPackage, numUpdates*2) + for nevra, pkgUpdate := range updates { + if !isValidNevra(nevra) { + continue + } + installed++ + availableUpdates := pkgUpdate.GetAvailableUpdates() + if len(availableUpdates) > 0 { + updatable++ + } pkgMeta, ok := memoryPackageCache.GetByNevra(nevra) + pkgUpdateData := packageUpdateData(pkgMeta.Evra, availableUpdates) + // before we used nevra.EVRAString() function which shows only non zero epoch, keep it consistent + // maybe we need here something like: evra := strings.TrimPrefix(upData.GetEVRA(), "0:") if ok { - packageIDs = append(packageIDs, pkgMeta.ID) - p := namedPackage{ + packages[nevra] = namedPackage{ NameID: pkgMeta.NameID, Name: pkgMeta.Name, PackageID: pkgMeta.ID, EVRA: pkgMeta.Evra, - WasStored: false, - UpdateData: nil, + Change: Add, + UpdateData: pkgUpdateData, } - packages = append(packages, p) - id2index[pkgMeta.ID] = i - i++ } } - rows, err := tx.Table("system_package"). - Select("package_id, update_data"). - Where("rh_account_id = ? AND system_id = ?", system.RhAccountID, system.ID). - Where("package_id in (?)", packageIDs). + + rows, err := tx.Table("(?) as t", database.SystemPackageDataShort(tx, system.RhAccountID)). + Joins("JOIN package p ON p.id = t.package_id"). + Joins("JOIN package_name pn on pn.id = p.name_id"). + Select("t.package_id, pn.name, p.name_id, p.evra, t.update_data"). + Where("system_id = ?", system.ID). Rows() if err != nil { - return nil, err + return nil, 0, 0, err } - var columns namedPackage for rows.Next() { - err = tx.ScanRows(rows, &columns) + var packageID int64 + var nameID int64 + var name string + var evra string + var jsonb []byte + var updateData models.PackageUpdateData + err = rows.Scan(&packageID, &name, &nameID, &evra, &jsonb) if err != nil { - return nil, err + return nil, 0, 0, err + } + nevra := utils.NEVRAStringE(name, evra, true) + err = json.Unmarshal(jsonb, &updateData) + if err != nil { + return nil, 0, 0, err + } + if p, ok := packages[nevra]; ok { + if isEqual(p.UpdateData, updateData) { + p.Change = Keep + } else { + p.Change = Update + } + } else { + packages[nevra] = namedPackage{ + NameID: nameID, + PackageID: packageID, + EVRA: evra, + Change: Remove, + UpdateData: updateData, + } } - index := id2index[columns.PackageID] - packages[index].WasStored = true - packages[index].UpdateData = columns.UpdateData - } - utils.LogInfo("inventoryID", system.InventoryID, "packages", numUpdates, "already stored", len(packages)) - return packages, err -} - -func isValidNevra(nevra string, packagesByNEVRA *map[string]namedPackage) bool { - // skip "phantom" package - if strings.HasPrefix(nevra, "gpg-pubkey") { - return false } - - // Check whether we have that NEVRA in DB - currentNamedPackage := (*packagesByNEVRA)[nevra] - if currentNamedPackage.PackageID == 0 { - utils.LogTrace("nevra", nevra, "Unknown package") - return false + if err := rows.Close(); err != nil { + return nil, 0, 0, err } - return true + utils.LogInfo("inventoryID", system.InventoryID, "packages", numUpdates, "already stored", len(packages)) + return packages, installed, updatable, err } -func updateDataChanged(currentNamedPackage *namedPackage, updateDataJSON []byte) bool { - if bytes.Equal(updateDataJSON, currentNamedPackage.UpdateData) { - // If the update_data we want to store is null, we skip only if there was a row for this specific - // system_package already stored. - if updateDataJSON == nil && currentNamedPackage.WasStored { - return false - } - - // If its not null, then the previous check ensured that the old update data matches new one - if updateDataJSON != nil { - return false +func packageUpdateData(installedEvra string, + availableUpdates []vmaas.UpdatesV3ResponseAvailableUpdates) models.PackageUpdateData { + data := models.PackageUpdateData{Installed: installedEvra} + for _, p := range availableUpdates { + if p.Package != nil { + // before we used nevra.EVRAString() function which shows only non zero epoch, keep it consistent + evra := strings.TrimPrefix(*p.EVRA, "0:") + switch p.StatusID { + case APPLICABLE: + data.Applicable = evra + case INSTALLABLE: + data.Installable = evra + } } } - return true + return data } -func createSystemPackage(nevra string, - updateData *vmaas.UpdatesV3ResponseUpdateList, - system *models.SystemPlatform, - packagesByNEVRA *map[string]namedPackage) (systemPackagePtr *models.SystemPackage, updatesChanged bool) { - updateDataJSON, err := vmaasResponse2UpdateDataJSON(updateData) - if err != nil { - utils.LogError("nevra", nevra, "VMaaS updates response parsing failed") - return nil, false - } - - // Skip overwriting entries which have the same data as before - currentNamedPackage := (*packagesByNEVRA)[nevra] - if !updateDataChanged(¤tNamedPackage, updateDataJSON) { - return nil, false - } +func isEqual(a, b models.PackageUpdateData) bool { + return a.Applicable == b.Applicable && a.Installable == b.Installable && a.Installed == b.Installed +} - systemPackage := models.SystemPackage{ - RhAccountID: system.RhAccountID, - SystemID: system.ID, - PackageID: currentNamedPackage.PackageID, - UpdateData: updateDataJSON, - NameID: currentNamedPackage.NameID, - } - return &systemPackage, true +func isValidNevra(nevra string) bool { + // skip "phantom" package + return !strings.HasPrefix(nevra, "gpg-pubkey") } func updateSystemPackages(tx *gorm.DB, system *models.SystemPlatform, - packagesByNEVRA *map[string]namedPackage, - vmaasData *vmaas.UpdatesV3Response) (installed, updatable int, err error) { + packagesByNEVRA map[string]namedPackage) error { defer utils.ObserveSecondsSince(time.Now(), evaluationPartDuration.WithLabelValues("packages-store")) - updates := vmaasData.GetUpdateList() - if err := deleteOldSystemPackages(tx, system, packagesByNEVRA); err != nil { - return 0, 0, err + // update system_package_data + if err := updateSystemPackageData(tx, system, packagesByNEVRA); err != nil { + return err } - toStore := make([]models.SystemPackage, 0, len(updates)) - for nevra, updateData := range updates { - isValid := isValidNevra(nevra, packagesByNEVRA) - if !isValid { - continue - } - installed++ - if len(updateData.GetAvailableUpdates()) > 0 { - updatable++ - } - - systemPackagePtr, updatesChanged := createSystemPackage(nevra, updateData, system, packagesByNEVRA) - if updatesChanged { - toStore = append(toStore, *systemPackagePtr) - } - } - return installed, updatable, errors.Wrap( - database.UnnestInsert(tx, - "INSERT INTO system_package (rh_account_id, system_id, package_id, update_data, name_id)"+ - " (select * from unnest($1::int[], $2::bigint[], $3::bigint[], $4::jsonb[], $5::bigint[]))"+ - " ON CONFLICT (rh_account_id, system_id, package_id) DO UPDATE SET update_data = EXCLUDED.update_data", toStore), - "Storing system packages") + // update package_system_data + err := updatePackageSystemData(tx, system, packagesByNEVRA) + return err } -func vmaasResponse2UpdateDataJSON(updateData *vmaas.UpdatesV3ResponseUpdateList) ([]byte, error) { - var latestInstallable models.PackageUpdate - var latestApplicable models.PackageUpdate - uniqUpdates := make(map[models.PackageUpdate]bool) - pkgUpdates := make([]models.PackageUpdate, 0, len(updateData.GetAvailableUpdates())) - for _, upData := range updateData.GetAvailableUpdates() { - if len(upData.GetPackage()) == 0 { - // no update +func systemPackageUpdateData(pkgDataMap map[string]namedPackage) models.SystemPackageUpdateData { + updateData := make(models.SystemPackageUpdateData, len(pkgDataMap)) + for _, pkg := range pkgDataMap { + if pkg.Change == Remove { continue } - // before we used nevra.EVRAString() function which shows only non zero epoch, keep it consistent - evra := strings.TrimPrefix(upData.GetEVRA(), "0:") - // Keep only unique entries for each update in the list - pkgUpdate := models.PackageUpdate{ - EVRA: evra, Advisory: upData.GetErratum(), Status: STATUS[upData.StatusID], - } - if !uniqUpdates[pkgUpdate] { - pkgUpdates = append(pkgUpdates, pkgUpdate) - uniqUpdates[pkgUpdate] = true - switch upData.StatusID { - case INSTALLABLE: - latestInstallable = pkgUpdate - case APPLICABLE: - latestApplicable = pkgUpdate - } - } + updateData[pkg.PackageID] = pkg.UpdateData } + return updateData +} - if prunePackageLatestOnly && len(pkgUpdates) > 1 { - pkgUpdates = make([]models.PackageUpdate, 0, 2) - if latestInstallable.EVRA != "" { - pkgUpdates = append(pkgUpdates, latestInstallable) - } - if latestApplicable.EVRA != "" { - pkgUpdates = append(pkgUpdates, latestApplicable) - } +func updateSystemPackageData(tx *gorm.DB, system *models.SystemPlatform, + pkgDataMap map[string]namedPackage) error { + jsonb, err := json.Marshal(systemPackageUpdateData(pkgDataMap)) + if err != nil { + return err + } + row := models.SystemPackageData{RhAccountID: system.RhAccountID, SystemID: system.ID, UpdateData: jsonb} + if len(pkgDataMap) > 0 { + return database.OnConflictUpdateMulti(tx, []string{"rh_account_id", "system_id"}, "update_data").Create(row).Error } + return tx.Delete(&models.SystemPackageData{}, system.RhAccountID, system.ID).Error +} - var updateDataJSON []byte - var err error - if len(pkgUpdates) > 0 { - updateDataJSON, err = json.Marshal(pkgUpdates) - if err != nil { - return nil, errors.Wrap(err, "Serializing pkg json") +func updatePackageSystemData(tx *gorm.DB, system *models.SystemPlatform, pkgDataMap map[string]namedPackage) error { + removeNameIDs := make([]int64, 0, len(pkgDataMap)) + tx = tx.Session(&gorm.Session{PrepareStmt: true}) + for _, pkg := range pkgDataMap { + switch pkg.Change { + case Remove: + removeNameIDs = append(removeNameIDs, pkg.NameID) + case Add: + fallthrough + case Update: + // handle updated packages + jsonb, err := json.Marshal(models.PackageSystemUpdateData{system.ID: pkg.UpdateData}) + if err != nil { + return err + } + row := models.PackageSystemData{RhAccountID: system.RhAccountID, PackageNameID: pkg.NameID, UpdateData: jsonb} + err = database.OnConflictDoUpdateExpr(tx, []string{"rh_account_id", "package_name_id"}, + database.UpExpr{Name: "update_data", + Expr: gorm.Expr("package_system_data.update_data || ?", jsonb)}). + Create(row).Error + if err != nil { + return err + } } } - return updateDataJSON, nil -} -func deleteOldSystemPackages(tx *gorm.DB, system *models.SystemPlatform, - packagesByNEVRA *map[string]namedPackage) error { - pkgIds := make([]int64, 0, len(*packagesByNEVRA)) - for _, pkg := range *packagesByNEVRA { - pkgIds = append(pkgIds, pkg.PackageID) + // handle removed packages + if len(removeNameIDs) > 0 { + // TODO: remove package names with no systems + tx.Model(&models.PackageSystemData{}).Where("update_data IS NULL OR update_date == '{}'::jsonb") + return tx.Model(&models.PackageSystemData{}). + Where("rh_account_id = ? and package_name_id in (?)", system.RhAccountID, removeNameIDs). + Update("update_data", gorm.Expr("update_data - ?", strconv.FormatInt(system.ID, 10))).Error } + return nil +} - err := tx.Where("rh_account_id = ? ", system.RhAccountID). - Where("system_id = ?", system.ID). - Where("package_id not in (?)", pkgIds). - Delete(&models.SystemPackage{}).Error +type ChangeType int8 - return errors.Wrap(err, "Deleting outdated system packages") -} +const ( + None ChangeType = iota + Add + Keep + Update + Remove +) type namedPackage struct { NameID int64 Name string PackageID int64 EVRA string - WasStored bool - UpdateData []byte + Change ChangeType + UpdateData models.PackageUpdateData }