diff --git a/deploy/clowdapp.yaml b/deploy/clowdapp.yaml index f61967492..615c965f5 100644 --- a/deploy/clowdapp.yaml +++ b/deploy/clowdapp.yaml @@ -508,6 +508,11 @@ objects: - {name: DB_USER, value: vmaas_sync} - {name: DB_PASSWD, valueFrom: {secretKeyRef: {name: patchman-engine-database-passwords, key: vmaas-sync-database-password}}} + - {name: PACKAGE_CACHE_SIZE, value: '${PACKAGE_CACHE_SIZE}'} + - {name: PACKAGE_NAME_CACHE_SIZE, value: '${PACKAGE_NAME_CACHE_SIZE}'} + resources: + limits: {cpu: '${RES_LIMIT_CPU_MIGRATE_JOB}', memory: '${RES_LIMIT_MEM_MIGRATE_JOB}'} + requests: {cpu: '${RES_REQUEST_CPU_MIGRATE_JOB}', memory: '${RES_REQUEST_MEM_MIGRATE_JOB}'} database: name: patchman @@ -729,6 +734,10 @@ parameters: - {name: SKIP_N_ACCOUNTS_REFRESH, value: '0'} # Skip advisory cache refresh for N first accounts in case the previous refresh didn't finish # Migrate system package data - {name: MIGRATE_SYSTEM_PACKAGE_DISABLED, value: 'true'} +- {name: RES_REQUEST_CPU_MIGRATE_JOB, value: '1'} +- {name: RES_LIMIT_CPU_MIGRATE_JOB, value: '2'} +- {name: RES_REQUEST_MEM_MIGRATE_JOB, value: '512Mi'} +- {name: RES_LIMIT_MEM_MIGRATE_JOB, value: '1024Mi'} # Database admin - {name: IMAGE_TAG_DATABASE_ADMIN, value: v3.5.31} diff --git a/tasks/migration/migrate_system_package_data.go b/tasks/migration/migrate_system_package_data.go index b029a3543..a6cdc52f1 100644 --- a/tasks/migration/migrate_system_package_data.go +++ b/tasks/migration/migrate_system_package_data.go @@ -4,16 +4,24 @@ import ( "app/base/core" "app/base/models" "app/base/utils" + "app/evaluator" "app/tasks" "encoding/json" + "fmt" "sync" "gorm.io/gorm" ) +var memoryPackageCache *evaluator.PackageCache + func RunSystemPackageDataMigration() { tasks.HandleContextCancel(tasks.WaitAndExit) core.ConfigureApp() + packageCacheSize := utils.GetIntEnvOrDefault("PACKAGE_CACHE_SIZE", 1000000) + packageNameCacheSize := utils.GetIntEnvOrDefault("PACKAGE_NAME_CACHE_SIZE", 60000) + memoryPackageCache = evaluator.NewPackageCache(true, true, packageCacheSize, packageNameCacheSize) + memoryPackageCache.Load() utils.LogInfo("Migrating installable/applicable advisories from system_package to system_package2") MigrateSystemPackageData() } @@ -178,6 +186,37 @@ func getPackageIDs(u SystemPackageRecord, i int, latestApplicable, latestInstall return 0, 0 } + var applicableID, installableID int64 + + name, ok := memoryPackageCache.GetNameByID(u.NameID) + if ok { + var applicable, installable *evaluator.PackageCacheMetadata + // assume both evras will be found in cache + applicableInCache := true + installableInCache := true + + if len(latestApplicable) > 0 { + nevraApplicable := fmt.Sprintf("%s-%s", name, latestApplicable) + applicable, applicableInCache = memoryPackageCache.GetByNevra(nevraApplicable) + if applicableInCache { + applicableID = applicable.ID + } + } + + if len(latestInstallable) > 0 { + nevraInstallable := fmt.Sprintf("%s-%s", name, latestInstallable) + installable, installableInCache = memoryPackageCache.GetByNevra(nevraInstallable) + if installableInCache { + installableID = installable.ID + } + } + + if applicableInCache && installableInCache { + // return ids only if both evras are found in cache + return applicableID, installableID + } + } + var packages []Package err := tasks.WithReadReplicaTx(func(db *gorm.DB) error { return db.Table("package"). @@ -190,7 +229,6 @@ func getPackageIDs(u SystemPackageRecord, i int, latestApplicable, latestInstall utils.LogWarn("#", i, "Failed to load packages") } - var applicableID, installableID int64 for _, p := range packages { if p.Evra == latestApplicable { applicableID = p.ID