POC: use package cache in migrate job to get package id

deploy/clowdapp.yaml

@@ -508,6 +508,11 @@ objects:
         - {name: DB_USER, value: vmaas_sync}
         - {name: DB_PASSWD, valueFrom: {secretKeyRef: {name: patchman-engine-database-passwords,
                                                       key: vmaas-sync-database-password}}}
+        - {name: PACKAGE_CACHE_SIZE, value: '${PACKAGE_CACHE_SIZE}'}
+        - {name: PACKAGE_NAME_CACHE_SIZE, value: '${PACKAGE_NAME_CACHE_SIZE}'}
+        resources:
+          limits: {cpu: '${RES_LIMIT_CPU_MIGRATE_JOB}', memory: '${RES_LIMIT_MEM_MIGRATE_JOB}'}
+          requests: {cpu: '${RES_REQUEST_CPU_MIGRATE_JOB}', memory: '${RES_REQUEST_MEM_MIGRATE_JOB}'}
 
     database:
       name: patchman
@@ -729,6 +734,10 @@ parameters:
 - {name: SKIP_N_ACCOUNTS_REFRESH, value: '0'} # Skip advisory cache refresh for N first accounts in case the previous refresh didn't finish
 # Migrate system package data
 - {name: MIGRATE_SYSTEM_PACKAGE_DISABLED, value: 'true'}
+- {name: RES_REQUEST_CPU_MIGRATE_JOB, value: '1'}
+- {name: RES_LIMIT_CPU_MIGRATE_JOB, value: '2'}
+- {name: RES_REQUEST_MEM_MIGRATE_JOB, value: '512Mi'}
+- {name: RES_LIMIT_MEM_MIGRATE_JOB, value: '1024Mi'}
 
 # Database admin
 - {name: IMAGE_TAG_DATABASE_ADMIN, value: v3.5.31}

tasks/migration/migrate_system_package_data.go

@@ -4,16 +4,24 @@ import (
 	"app/base/core"
 	"app/base/models"
 	"app/base/utils"
+	"app/evaluator"
 	"app/tasks"
 	"encoding/json"
+	"fmt"
 	"sync"
 
 	"gorm.io/gorm"
 )
 
+var memoryPackageCache *evaluator.PackageCache
+
 func RunSystemPackageDataMigration() {
 	tasks.HandleContextCancel(tasks.WaitAndExit)
 	core.ConfigureApp()
+	packageCacheSize := utils.GetIntEnvOrDefault("PACKAGE_CACHE_SIZE", 1000000)
+	packageNameCacheSize := utils.GetIntEnvOrDefault("PACKAGE_NAME_CACHE_SIZE", 60000)
+	memoryPackageCache = evaluator.NewPackageCache(true, true, packageCacheSize, packageNameCacheSize)
+	memoryPackageCache.Load()
 	utils.LogInfo("Migrating installable/applicable advisories from system_package to system_package2")
 	MigrateSystemPackageData()
 }
@@ -178,6 +186,37 @@ func getPackageIDs(u SystemPackageRecord, i int, latestApplicable, latestInstall
 		return 0, 0
 	}
 
+	var applicableID, installableID int64
+
+	name, ok := memoryPackageCache.GetNameByID(u.NameID)
+	if ok {
+		var applicable, installable *evaluator.PackageCacheMetadata
+		// assume both evras will be found in cache
+		applicableInCache := true
+		installableInCache := true
+
+		if len(latestApplicable) > 0 {
+			nevraApplicable := fmt.Sprintf("%s-%s", name, latestApplicable)
+			applicable, applicableInCache = memoryPackageCache.GetByNevra(nevraApplicable)
+			if applicableInCache {
+				applicableID = applicable.ID
+			}
+		}
+
+		if len(latestInstallable) > 0 {
+			nevraInstallable := fmt.Sprintf("%s-%s", name, latestInstallable)
+			installable, installableInCache = memoryPackageCache.GetByNevra(nevraInstallable)
+			if installableInCache {
+				installableID = installable.ID
+			}
+		}
+
+		if applicableInCache && installableInCache {
+			// return ids only if both evras are found in cache
+			return applicableID, installableID
+		}
+	}
+
 	var packages []Package
 	err := tasks.WithReadReplicaTx(func(db *gorm.DB) error {
 		return db.Table("package").
@@ -190,7 +229,6 @@ func getPackageIDs(u SystemPackageRecord, i int, latestApplicable, latestInstall
 		utils.LogWarn("#", i, "Failed to load packages")
 	}
 
-	var applicableID, installableID int64
 	for _, p := range packages {
 		if p.Evra == latestApplicable {
 			applicableID = p.ID