RHINENG-9778: test repo-package_name based recalc

base/database/testing.go

@@ -322,6 +322,10 @@ func DeleteNewlyAddedPackages(t *testing.T) {
 
 func DeleteNewlyAddedAdvisories(t *testing.T) {
 	query := DB.Model(models.AdvisoryMetadata{}).Where("id >= 100")
+	querySa := DB.Model(models.SystemAdvisories{}).Where("advisory_id >= 100")
+	queryAad := DB.Model(models.AdvisoryAccountData{}).Where("advisory_id >= 100")
+	assert.Nil(t, querySa.Delete(models.SystemAdvisories{}).Error)
+	assert.Nil(t, queryAad.Delete(models.AdvisoryAccountData{}).Error)
 	assert.Nil(t, query.Delete(models.AdvisoryMetadata{}).Error)
 	var cnt int64
 	assert.Nil(t, query.Count(&cnt).Error)

base/database/utils_test.go

@@ -11,8 +11,8 @@ var (
 	// counts of systems from system_platform JOIN inventory.hosts
 	nGroup1    int64 = 6
 	nGroup2    int64 = 2
-	nUngrouped int64 = 6
-	nAll       int64 = 16
+	nUngrouped int64 = 7
+	nAll       int64 = 17
 )
 
 var testCases = []map[int64]map[string]string{

dev/test_data.sql

@@ -56,7 +56,8 @@ INSERT INTO system_platform (id, inventory_id, display_name, rh_account_id,  vma
  '{"update_list": {"suricata-0:6.0.3-2.fc35.i686": {"available_updates": [{"erratum": "RHSA-2021:3801", "basearch": "i686", "releasever": "ser1", "repository": "group_oisf:suricata-6.0", "package": "suricata-0:6.0.4-2.fc35.i686"}]}}, "basearch": "i686", "releasever": "ser1"}');
 
 INSERT INTO system_platform (id, inventory_id, display_name, rh_account_id,  vmaas_json, json_checksum, last_evaluation, last_upload, packages_installed, packages_installable, packages_applicable, yum_updates, baseline_id, template_id) VALUES
-(16, '00000000-0000-0000-0000-000000000016','00000000-0000-0000-0000-000000000016', 3, '{ "package_list": [ "kernel-2.6.32-696.20.1.el6.x86_64" ], "repository_list": [ "rhel-6-server-rpms" ] }', '1', '2018-09-22 12:00:00-04', '2018-01-22 12:00:00-04', 1,1,1, NULL, 4, 4);
+(16, '00000000-0000-0000-0000-000000000016','00000000-0000-0000-0000-000000000016', 3, '{ "package_list": [ "kernel-2.6.32-696.20.1.el6.x86_64" ], "repository_list": [ "rhel-6-server-rpms" ] }', '1', '2018-09-22 12:00:00-04', '2018-01-22 12:00:00-04', 1,1,1, NULL, 4, 4),
+(17, '00000000-0000-0000-0000-000000000017','00000000-0000-0000-0000-000000000017', 1, '{ "package_list": [ "kernel-2.6.32-696.20.1.el6.x86_64" ], "repository_list": [ "rhel-6-server-rpms" ] }', '1', '2018-09-22 12:00:00-04', '2018-01-22 12:00:00-04',2,2,2, NULL, NULL, NULL);
 
 INSERT INTO advisory_metadata (id, name, description, synopsis, summary, solution, advisory_type_id,
                                public_date, modified_date, url, severity_id, cve_list, release_versions) VALUES
@@ -111,7 +112,8 @@ INSERT INTO repo (id, name, third_party) VALUES
 INSERT INTO system_repo (rh_account_id, system_id, repo_id) VALUES
 (1, 2, 1),
 (1, 3, 1),
-(1, 2, 2);
+(1, 2, 2),
+(1, 17, 1);
 
 
 INSERT INTO package_name(id, name, summary) VALUES
@@ -164,13 +166,19 @@ INSERT INTO package(id, name_id, evra, description_hash, summary_hash, advisory_
 (13, 102, '77.0.1-1.fc31.x86_64', '22', '2', null, true); -- firefox
 
 INSERT INTO system_package2 (rh_account_id, system_id, name_id, package_id, installable_id, applicable_id) VALUES
+(1, 2, 101, 1, 11, null),
+(1, 2, 102, 2, 12, null),
+(1, 3, 101, 1, 11, null),
+(1, 3, 102, 2, 12, null),
 (3, 12, 101, 1, 11, null),
 (3, 12, 102, 2, 12, null),
 (3, 13, 101, 1, null, null),
 (3, 13, 102, 2, 12, 13),
 (3, 13, 103, 3, null, null),
 (3, 13, 104, 4, null, null),
-(3, 16, 101, 1, 11, 11);
+(3, 16, 101, 1, 11, 11),
+(1, 17, 101, 1, 11, null),
+(1, 17, 102, 2, 12, null);
 
 INSERT INTO timestamp_kv (name, value) VALUES
 ('last_eval_repo_based', '2018-04-05T01:23:45+02:00');

listener/common_test.go

@@ -30,7 +30,7 @@ func deleteData(t *testing.T) {
 		"USING rh_account ra WHERE ra.id = aad.rh_account_id AND ra.name = ?", id).Error)
 	assert.Nil(t, database.DB.Unscoped().Where("first_reported > timestamp '2020-01-01'").
 		Delete(&models.SystemAdvisories{}).Error)
-	assert.Nil(t, database.DB.Unscoped().Where("repo_id NOT IN (1) OR system_id NOT IN (2, 3)").
+	assert.Nil(t, database.DB.Unscoped().Where("repo_id NOT IN (1, 2) OR system_id NOT IN (2, 3, 17)").
 		Delete(&models.SystemRepo{}).Error)
 	assert.Nil(t, database.DB.Unscoped().Where("name NOT IN ('repo1', 'repo2', 'repo3', 'repo4')").
 		Delete(&models.Repo{}).Error)

manager/controllers/systems_auth_test.go

@@ -19,7 +19,7 @@ func testAccountSystemCounts(t *testing.T, acc int, count int) {
 
 func TestMissingAccount(t *testing.T) {
 	testAccountSystemCounts(t, 0, 0)
-	testAccountSystemCounts(t, 1, 8)
+	testAccountSystemCounts(t, 1, 9)
 	testAccountSystemCounts(t, 2, 3)
 	testAccountSystemCounts(t, 3, 5)
 	testAccountSystemCounts(t, 4, 0)

manager/controllers/systems_export_test.go

@@ -30,7 +30,7 @@ func TestSystemsExportJSON(t *testing.T) {
 
 	var output []SystemDBLookup
 	CheckResponse(t, w, http.StatusOK, &output)
-	assert.Equal(t, 8, len(output))
+	assert.Equal(t, 9, len(output))
 	assert.Equal(t, "00000000-0000-0000-0000-000000000001", output[0].ID)
 	assert.Equal(t, 2, output[0].SystemItemAttributes.RhsaCount)
 	assert.Equal(t, 2, output[0].SystemItemAttributes.RhbaCount)
@@ -53,7 +53,7 @@ func TestSystemsExportCSV(t *testing.T) {
 	body := w.Body.String()
 	lines := strings.Split(body, "\r\n")
 
-	assert.Equal(t, 10, len(lines))
+	assert.Equal(t, 11, len(lines))
 	assert.Equal(t, SystemCsvHeader, lines[0])
 
 	assert.Equal(t, "00000000-0000-0000-0000-000000000001,00000000-0000-0000-0000-000000000001,RHEL 8.10,8.10,"+

manager/controllers/systems_ids_test.go

@@ -14,7 +14,7 @@ func TestSystemsIDsDefault(t *testing.T) {
 	output := testSystemsIDs(t, ``, 1)
 
 	// data
-	assert.Equal(t, 8, len(output.IDs))
+	assert.Equal(t, 9, len(output.IDs))
 	assert.Equal(t, "00000000-0000-0000-0000-000000000001", output.IDs[0])
 }
 
@@ -40,7 +40,7 @@ func TestSystemsIDsWrongSort(t *testing.T) {
 
 func TestSystemsIDsSearch(t *testing.T) {
 	output := testSystemsIDs(t, "?search=001", 1)
-	assert.Equal(t, 1, len(output.IDs))
+	assert.Equal(t, 2, len(output.IDs))
 	assert.Equal(t, "00000000-0000-0000-0000-000000000001", output.IDs[0])
 }
 
@@ -140,15 +140,15 @@ func TestSystemsIDsFilterNotExisting(t *testing.T) {
 func TestSystemsIDsFilterPartialOS(t *testing.T) {
 	outputIDs := testSystemsIDs(t, "?filter[osname]=RHEL&filter[osmajor]=8&filter[osminor]=1", 1)
 	output := testSystems(t, "?filter[osname]=RHEL&filter[osmajor]=8&filter[osminor]=1", 1)
-	assert.Equal(t, 2, len(outputIDs.IDs))
+	assert.Equal(t, 3, len(outputIDs.IDs))
 	assert.Equal(t, output.Data[0].ID, outputIDs.IDs[0])
 	assert.Equal(t, output.Data[1].ID, outputIDs.IDs[1])
 }
 
 func TestSystemsIDsFilterOS(t *testing.T) {
 	outputIDs := testSystemsIDs(t, `?filter[os]=in:RHEL 8.1,RHEL 7.3&sort=os`, 1)
 	output := testSystems(t, `?filter[os]=in:RHEL 8.1,RHEL 7.3&sort=os`, 1)
-	assert.Equal(t, 3, len(outputIDs.IDs))
+	assert.Equal(t, 4, len(outputIDs.IDs))
 	assert.Equal(t, output.Data[0].ID, outputIDs.IDs[0])
 	assert.Equal(t, output.Data[1].ID, outputIDs.IDs[1])
 	assert.Equal(t, output.Data[2].ID, outputIDs.IDs[2])

manager/controllers/systems_test.go

@@ -16,7 +16,7 @@ func TestSystemsDefault(t *testing.T) {
 	output := testSystems(t, ``, 1)
 
 	// data
-	assert.Equal(t, 8, len(output.Data))
+	assert.Equal(t, 9, len(output.Data))
 	assert.Equal(t, "00000000-0000-0000-0000-000000000001", output.Data[0].ID)
 	assert.Equal(t, "system", output.Data[0].Type)
 	assert.Equal(t, "2020-09-22 16:00:00 +0000 UTC", output.Data[0].Attributes.LastUpload.String())
@@ -45,9 +45,9 @@ func TestSystemsDefault(t *testing.T) {
 	// test meta
 	assert.Equal(t, 0, output.Meta.Offset)
 	assert.Equal(t, core.DefaultLimit, output.Meta.Limit)
-	assert.Equal(t, 8, output.Meta.TotalItems)
+	assert.Equal(t, 9, output.Meta.TotalItems)
 	assert.Equal(t, 8, output.Meta.SubTotals["patched"])
-	assert.Equal(t, 0, output.Meta.SubTotals["unpatched"])
+	assert.Equal(t, 1, output.Meta.SubTotals["unpatched"])
 	assert.Equal(t, 0, output.Meta.SubTotals["stale"])
 }
 
@@ -56,15 +56,15 @@ func TestSystemsOffset(t *testing.T) {
 	assert.Equal(t, 4, len(output.Data))
 	assert.Equal(t, 0, output.Meta.Offset)
 	assert.Equal(t, 4, output.Meta.Limit)
-	assert.Equal(t, 8, output.Meta.TotalItems)
+	assert.Equal(t, 9, output.Meta.TotalItems)
 }
 
 func TestSystemsOffsetLimit(t *testing.T) {
 	output := testSystems(t, `?offset=4&limit=4`, 1)
 	assert.Equal(t, 4, len(output.Data))
 	assert.Equal(t, 4, output.Meta.Offset)
 	assert.Equal(t, 4, output.Meta.Limit)
-	assert.Equal(t, 8, output.Meta.TotalItems)
+	assert.Equal(t, 9, output.Meta.TotalItems)
 }
 
 func TestSystemsWrongOffset(t *testing.T) {
@@ -79,7 +79,7 @@ func TestSystemsWrongSort(t *testing.T) {
 
 func TestSystemsSearch(t *testing.T) {
 	output := testSystems(t, "?search=001", 1)
-	assert.Equal(t, 1, len(output.Data))
+	assert.Equal(t, 2, len(output.Data))
 	assert.Equal(t, "00000000-0000-0000-0000-000000000001", output.Data[0].ID)
 	assert.Equal(t, "system", output.Data[0].Type)
 	assert.Equal(t, "00000000-0000-0000-0000-000000000001", output.Data[0].Attributes.DisplayName)
@@ -208,7 +208,7 @@ func TestSystemsFilterNotExisting(t *testing.T) {
 
 func TestSystemsFilterOS(t *testing.T) {
 	output := testSystems(t, `?filter[os]=in:RHEL 8.1,RHEL 7.3&sort=os`, 1)
-	assert.Equal(t, 3, len(output.Data))
+	assert.Equal(t, 4, len(output.Data))
 	assert.Equal(t, "RHEL 7.3", output.Data[0].Attributes.OS)
 	assert.Equal(t, "RHEL 8.1", output.Data[1].Attributes.OS)
 	assert.Equal(t, "RHEL 8.1", output.Data[2].Attributes.OS)
@@ -219,11 +219,12 @@ func TestSystemsOrderOS(t *testing.T) {
 	assert.Equal(t, "RHEL 7.3", output.Data[0].Attributes.OS)
 	assert.Equal(t, "RHEL 8.1", output.Data[1].Attributes.OS)
 	assert.Equal(t, "RHEL 8.1", output.Data[2].Attributes.OS)
-	assert.Equal(t, "RHEL 8.2", output.Data[3].Attributes.OS)
-	assert.Equal(t, "RHEL 8.3", output.Data[4].Attributes.OS)
+	assert.Equal(t, "RHEL 8.1", output.Data[3].Attributes.OS)
+	assert.Equal(t, "RHEL 8.2", output.Data[4].Attributes.OS)
 	assert.Equal(t, "RHEL 8.3", output.Data[5].Attributes.OS)
-	assert.Equal(t, "RHEL 8.10", output.Data[6].Attributes.OS)
-	assert.Equal(t, "RHEL 8.x", output.Data[7].Attributes.OS) // yes, we should be robust against this
+	assert.Equal(t, "RHEL 8.3", output.Data[6].Attributes.OS)
+	assert.Equal(t, "RHEL 8.10", output.Data[7].Attributes.OS)
+	assert.Equal(t, "RHEL 8.x", output.Data[8].Attributes.OS) // yes, we should be robust against this
 }
 
 func testSystems(t *testing.T, queryString string, account int) SystemsResponse {
@@ -318,7 +319,7 @@ func TestAAPSystemMeta2(t *testing.T) {
 func TestAAPSystemMeta3(t *testing.T) {
 	const (
 		ID         = "00000000-0000-0000-0000-000000000007"
-		totalItems = 1
+		totalItems = 2
 	)
 	output := testSystems(t, `?filter[system_profile][ansible][controller_version]=1.0`, 1)
 	assert.Equal(t, ID, output.Data[0].ID)
@@ -346,7 +347,7 @@ func TestMSSQLSystemMeta2(t *testing.T) {
 func TestMSSQLSystemMeta3(t *testing.T) {
 	const (
 		ID         = "00000000-0000-0000-0000-000000000006"
-		totalItems = 1
+		totalItems = 2
 	)
 	output := testSystems(t, `?filter[system_profile][mssql][version]=15.3.0`, 1)
 	assert.Equal(t, ID, output.Data[0].ID)

platform/vmaas.go

@@ -205,7 +205,7 @@ func reposHandler(c *gin.Context) {
     "repository_list": {
         "repo1": [],
         "repo2": [],
-        "repo3": []
+        "repo3": [{"updated_package_names": ["kernel", "firefox"]}]
     }}`
 	c.Data(http.StatusOK, gin.MIMEJSON, []byte(data))
 }

tasks/system_culling/system_culling_test.go

@@ -91,7 +91,7 @@ func TestMarkSystemsStale(t *testing.T) {
 	}
 	nMarked, err := markSystemsStale(database.DB, 500)
 	assert.Nil(t, err)
-	assert.Equal(t, 16, nMarked)
+	assert.Equal(t, 17, nMarked)
 
 	assert.NoError(t, database.DB.Find(&systems).Error)
 	for i, s := range systems {

tasks/vmaas_sync/metrics_test.go

@@ -29,7 +29,7 @@ func TestSystemsCounts(t *testing.T) {
 	assert.Equal(t, 2, counts[systemsCntLabels{staleOff, lastUploadLast1D}])
 	assert.Equal(t, 5, counts[systemsCntLabels{staleOff, lastUploadLast7D}])
 	assert.Equal(t, 8, counts[systemsCntLabels{staleOff, lastUploadLast30D}])
-	assert.Equal(t, 16, counts[systemsCntLabels{staleOff, lastUploadAll}])
+	assert.Equal(t, 17, counts[systemsCntLabels{staleOff, lastUploadAll}])
 	assert.Equal(t, 0, counts[systemsCntLabels{staleOn, lastUploadLast1D}])
 	assert.Equal(t, 0, counts[systemsCntLabels{staleOn, lastUploadLast7D}])
 	assert.Equal(t, 0, counts[systemsCntLabels{staleOn, lastUploadLast30D}])

tasks/vmaas_sync/repo_based_test.go

@@ -12,17 +12,19 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+var orgID1 = "org_1"
+
 func TestGetCurrentRepoBasedInventoryIDs(t *testing.T) {
 	utils.SkipWithoutDB(t)
 	core.SetupTestEnvironment()
 	Configure()
 
-	orgID := "org_1"
 	inventoryAIDs, err := getCurrentRepoBasedInventoryIDs()
 	assert.Nil(t, err)
 	assert.Equal(t, []mqueue.EvalData{
-		{InventoryID: "00000000-0000-0000-0000-000000000002", RhAccountID: 1, OrgID: &orgID},
-		{InventoryID: "00000000-0000-0000-0000-000000000003", RhAccountID: 1, OrgID: &orgID}},
+		{InventoryID: "00000000-0000-0000-0000-000000000002", RhAccountID: 1, OrgID: &orgID1},
+		{InventoryID: "00000000-0000-0000-0000-000000000003", RhAccountID: 1, OrgID: &orgID1},
+		{InventoryID: "00000000-0000-0000-0000-000000000017", RhAccountID: 1, OrgID: &orgID1}},
 		inventoryAIDs)
 	resetLastEvalTimestamp(t)
 }
@@ -66,17 +68,53 @@ func TestUpdateRepoBaseEvalTimestamp(t *testing.T) {
 	resetLastEvalTimestamp(t)
 }
 
-func TestGetRepoBasedInventoryIDs(t *testing.T) {
+func TestGetRepoOnlyBasedInventoryIDs(t *testing.T) {
 	utils.SkipWithoutDB(t)
 	core.SetupTestEnvironment()
 
-	orgID := "org_1"
 	repos := []string{"repo1", "repo2"}
-	inventoryAIDs, err := getRepoBasedInventoryIDs(repos)
+	inventoryAIDs, err := getRepoBasedInventoryIDs(nil, repos)
+	assert.Nil(t, err)
+	assert.Equal(t, []mqueue.EvalData{
+		{InventoryID: "00000000-0000-0000-0000-000000000002", RhAccountID: 1, OrgID: &orgID1},
+		{InventoryID: "00000000-0000-0000-0000-000000000003", RhAccountID: 1, OrgID: &orgID1},
+		{InventoryID: "00000000-0000-0000-0000-000000000017", RhAccountID: 1, OrgID: &orgID1}},
+		inventoryAIDs)
+}
+
+func TestGetRepoPackageBasedInventoryIDs(t *testing.T) {
+	utils.SkipWithoutDB(t)
+	core.SetupTestEnvironment()
+
+	repos := [][]string{{"repo1", "not_installed_pkg"}, {"repo2", "not_installed_pkg"}, {"repo2", "kernel"}}
+	inventoryAIDs, err := getRepoBasedInventoryIDs(repos, nil)
+	assert.Nil(t, err)
+	assert.Equal(t, []mqueue.EvalData{
+		// "kernel" in "repo2"
+		{InventoryID: "00000000-0000-0000-0000-000000000002", RhAccountID: 1, OrgID: &orgID1}},
+		// 00000000-0000-0000-0000-000000000017 does not have "not_installed_pkg" in "repo1"
+		inventoryAIDs)
+
+	repos = [][]string{{"repo1", "not_installed_pkg"}, {"repo2", "not_installed_pkg"}}
+	inventoryAIDs, err = getRepoBasedInventoryIDs(repos, nil)
+	assert.Nil(t, err)
+	assert.Len(t, inventoryAIDs, 0)
+}
+
+func TestGetRepoBasedInventoryIDs(t *testing.T) {
+	utils.SkipWithoutDB(t)
+	core.SetupTestEnvironment()
+
+	repos := []string{"repo1"}
+	repoPackages := [][]string{{"repo1", "not_installed_pkg"}, {"repo2", "not_installed_pkg"}, {"repo2", "kernel"}}
+	inventoryAIDs, err := getRepoBasedInventoryIDs(repoPackages, repos)
 	assert.Nil(t, err)
 	assert.Equal(t, []mqueue.EvalData{
-		{InventoryID: "00000000-0000-0000-0000-000000000002", RhAccountID: 1, OrgID: &orgID},
-		{InventoryID: "00000000-0000-0000-0000-000000000003", RhAccountID: 1, OrgID: &orgID}},
+		// from repoPackages
+		{InventoryID: "00000000-0000-0000-0000-000000000002", RhAccountID: 1, OrgID: &orgID1},
+		// systems added from repos
+		{InventoryID: "00000000-0000-0000-0000-000000000003", RhAccountID: 1, OrgID: &orgID1},
+		{InventoryID: "00000000-0000-0000-0000-000000000017", RhAccountID: 1, OrgID: &orgID1}},
 		inventoryAIDs)
 }
 
@@ -85,7 +123,8 @@ func TestGetRepoBasedInventoryIDsEmpty(t *testing.T) {
 	core.SetupTestEnvironment()
 
 	repos := []string{}
-	inventoryIDs, err := getRepoBasedInventoryIDs(repos)
+	repoPackages := [][]string{}
+	inventoryIDs, err := getRepoBasedInventoryIDs(repoPackages, repos)
 	assert.Nil(t, err)
 	assert.Equal(t, 0, len(inventoryIDs))
 }
@@ -95,10 +134,11 @@ func TestGetUpdatedRepos(t *testing.T) {
 	Configure()
 
 	modifiedSince := time.Now().Format(types.Rfc3339NoTz)
-	redhat, thirdparty, _, err := getUpdatedRepos(time.Now(), &modifiedSince)
+	thirdParty := true
+	repoPackages, repoNoPackages, _, err := getUpdatedRepos(time.Now(), &modifiedSince, &thirdParty)
 	assert.Nil(t, err)
-	assert.Equal(t, 3, len(redhat))
-	assert.Equal(t, 0, len(thirdparty))
+	assert.Equal(t, 2, len(repoPackages[0]))
+	assert.Equal(t, 2, len(repoNoPackages))
 }
 
 func resetLastEvalTimestamp(t *testing.T) {