From 35e8a09542656178f732b8bf377881bcd325256e Mon Sep 17 00:00:00 2001 From: Michael Mraka Date: Tue, 12 Dec 2023 11:13:11 +0100 Subject: [PATCH 1/2] RHINENG-6031: drop old system_package table --- .../122_remove_system_package.down.sql | 153 ++++++++++++++++++ .../122_remove_system_package.up.sql | 102 ++++++++++++ database_admin/schema/create_schema.sql | 58 +------ dev/test_data.sql | 1 - tasks/vmaas_sync/metrics_db_test.go | 4 +- 5 files changed, 258 insertions(+), 60 deletions(-) create mode 100644 database_admin/migrations/122_remove_system_package.down.sql create mode 100644 database_admin/migrations/122_remove_system_package.up.sql diff --git a/database_admin/migrations/122_remove_system_package.down.sql b/database_admin/migrations/122_remove_system_package.down.sql new file mode 100644 index 000000000..13d1a0df4 --- /dev/null +++ b/database_admin/migrations/122_remove_system_package.down.sql @@ -0,0 +1,153 @@ +CREATE OR REPLACE FUNCTION delete_system(inventory_id_in uuid) + RETURNS TABLE + ( + deleted_inventory_id uuid + ) +AS +$delete_system$ +DECLARE + v_system_id INT; + v_account_id INT; +BEGIN + -- opt out to refresh cache and then delete + SELECT id, rh_account_id + FROM system_platform + WHERE inventory_id = inventory_id_in + LIMIT 1 + FOR UPDATE OF system_platform + INTO v_system_id, v_account_id; + + IF v_system_id IS NULL OR v_account_id IS NULL THEN + RAISE NOTICE 'Not found'; + RETURN; + END IF; + + UPDATE system_platform + SET stale = true + WHERE rh_account_id = v_account_id + AND id = v_system_id; + + DELETE + FROM system_advisories + WHERE rh_account_id = v_account_id + AND system_id = v_system_id; + + DELETE + FROM system_repo + WHERE rh_account_id = v_account_id + AND system_id = v_system_id; + + DELETE + FROM system_package + WHERE rh_account_id = v_account_id + AND system_id = v_system_id; + + DELETE + FROM system_package2 + WHERE rh_account_id = v_account_id + AND system_id = v_system_id; + + RETURN QUERY DELETE FROM system_platform + WHERE rh_account_id = v_account_id AND + id = v_system_id + RETURNING inventory_id; +END; +$delete_system$ LANGUAGE 'plpgsql'; + +CREATE OR REPLACE FUNCTION delete_systems(inventory_ids UUID[]) + RETURNS INTEGER +AS +$$ +DECLARE + tmp_cnt INTEGER; +BEGIN + + WITH systems as ( + SELECT rh_account_id, id + FROM system_platform + WHERE inventory_id = ANY (inventory_ids) + ORDER BY rh_account_id, id FOR UPDATE OF system_platform), + marked as ( + UPDATE system_platform sp + SET stale = true + WHERE (rh_account_id, id) in (select rh_account_id, id from systems) + ), + advisories as ( + DELETE + FROM system_advisories + WHERE (rh_account_id, system_id) in (select rh_account_id, id from systems) + ), + repos as ( + DELETE + FROM system_repo + WHERE (rh_account_id, system_id) in (select rh_account_id, id from systems) + ), + packages as ( + DELETE + FROM system_package + WHERE (rh_account_id, system_id) in (select rh_account_id, id from systems) + ), + packages2 as ( + DELETE + FROM system_package2 + WHERE (rh_account_id, system_id) in (select rh_account_id, id from systems) + ), + deleted as ( + DELETE + FROM system_platform + WHERE (rh_account_id, id) in (select rh_account_id, id from systems) + RETURNING id + ) + SELECT count(*) + FROM deleted + INTO tmp_cnt; + + RETURN tmp_cnt; +END +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION update_status(update_data jsonb) + RETURNS TEXT as +$$ +DECLARE + len int; +BEGIN + len = jsonb_array_length(update_data); + IF len IS NULL or len = 0 THEN + RETURN 'None'; + END IF; + len = jsonb_array_length(jsonb_path_query_array(update_data, '$ ? (@.status == "Installable")')); + IF len > 0 THEN + RETURN 'Installable'; + END IF; + RETURN 'Applicable'; +END; +$$ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; + +CREATE TABLE IF NOT EXISTS system_package +( + rh_account_id INT NOT NULL REFERENCES rh_account, + system_id BIGINT NOT NULL, + package_id BIGINT NOT NULL REFERENCES package, + -- Use null to represent up-to-date packages + update_data JSONB DEFAULT NULL, + latest_evra TEXT GENERATED ALWAYS AS ( ((update_data ->> -1)::jsonb ->> 'evra')::text) STORED + CHECK(NOT empty(latest_evra)), + name_id BIGINT REFERENCES package_name (id) NOT NULL, + + PRIMARY KEY (rh_account_id, system_id, package_id) INCLUDE (latest_evra) +) PARTITION BY HASH (rh_account_id); + +CREATE INDEX IF NOT EXISTS system_package_name_pkg_system_idx + ON system_package (rh_account_id, name_id, package_id, system_id) INCLUDE (latest_evra); + +CREATE INDEX IF NOT EXISTS system_package_package_id_idx on system_package (package_id); + +GRANT SELECT, INSERT, UPDATE, DELETE ON system_package TO evaluator; +GRANT SELECT, UPDATE, DELETE ON system_package TO listener; +GRANT SELECT, UPDATE, DELETE ON system_package TO manager; +GRANT SELECT, UPDATE, DELETE ON system_package TO vmaas_sync; + +SELECT create_table_partitions('system_package', 128, + $$WITH (fillfactor = '70', autovacuum_vacuum_scale_factor = '0.05')$$); diff --git a/database_admin/migrations/122_remove_system_package.up.sql b/database_admin/migrations/122_remove_system_package.up.sql new file mode 100644 index 000000000..0a855569d --- /dev/null +++ b/database_admin/migrations/122_remove_system_package.up.sql @@ -0,0 +1,102 @@ +CREATE OR REPLACE FUNCTION delete_system(inventory_id_in uuid) + RETURNS TABLE + ( + deleted_inventory_id uuid + ) +AS +$delete_system$ +DECLARE + v_system_id INT; + v_account_id INT; +BEGIN + -- opt out to refresh cache and then delete + SELECT id, rh_account_id + FROM system_platform + WHERE inventory_id = inventory_id_in + LIMIT 1 + FOR UPDATE OF system_platform + INTO v_system_id, v_account_id; + + IF v_system_id IS NULL OR v_account_id IS NULL THEN + RAISE NOTICE 'Not found'; + RETURN; + END IF; + + UPDATE system_platform + SET stale = true + WHERE rh_account_id = v_account_id + AND id = v_system_id; + + DELETE + FROM system_advisories + WHERE rh_account_id = v_account_id + AND system_id = v_system_id; + + DELETE + FROM system_repo + WHERE rh_account_id = v_account_id + AND system_id = v_system_id; + + DELETE + FROM system_package2 + WHERE rh_account_id = v_account_id + AND system_id = v_system_id; + + RETURN QUERY DELETE FROM system_platform + WHERE rh_account_id = v_account_id AND + id = v_system_id + RETURNING inventory_id; +END; +$delete_system$ LANGUAGE 'plpgsql'; + +CREATE OR REPLACE FUNCTION delete_systems(inventory_ids UUID[]) + RETURNS INTEGER +AS +$$ +DECLARE + tmp_cnt INTEGER; +BEGIN + + WITH systems as ( + SELECT rh_account_id, id + FROM system_platform + WHERE inventory_id = ANY (inventory_ids) + ORDER BY rh_account_id, id FOR UPDATE OF system_platform), + marked as ( + UPDATE system_platform sp + SET stale = true + WHERE (rh_account_id, id) in (select rh_account_id, id from systems) + ), + advisories as ( + DELETE + FROM system_advisories + WHERE (rh_account_id, system_id) in (select rh_account_id, id from systems) + ), + repos as ( + DELETE + FROM system_repo + WHERE (rh_account_id, system_id) in (select rh_account_id, id from systems) + ), + packages2 as ( + DELETE + FROM system_package2 + WHERE (rh_account_id, system_id) in (select rh_account_id, id from systems) + ), + deleted as ( + DELETE + FROM system_platform + WHERE (rh_account_id, id) in (select rh_account_id, id from systems) + RETURNING id + ) + SELECT count(*) + FROM deleted + INTO tmp_cnt; + + RETURN tmp_cnt; +END +$$ LANGUAGE plpgsql; + + +DROP FUNCTION IF EXISTS update_status(update_data jsonb); + +DROP TABLE IF EXISTS system_package; diff --git a/database_admin/schema/create_schema.sql b/database_admin/schema/create_schema.sql index ee424d595..fc5275288 100644 --- a/database_admin/schema/create_schema.sql +++ b/database_admin/schema/create_schema.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS schema_migrations INSERT INTO schema_migrations -VALUES (121, false); +VALUES (122, false); -- --------------------------------------------------------------------------- -- Functions @@ -382,11 +382,6 @@ BEGIN WHERE rh_account_id = v_account_id AND system_id = v_system_id; - DELETE - FROM system_package - WHERE rh_account_id = v_account_id - AND system_id = v_system_id; - DELETE FROM system_package2 WHERE rh_account_id = v_account_id @@ -427,11 +422,6 @@ BEGIN FROM system_repo WHERE (rh_account_id, system_id) in (select rh_account_id, id from systems) ), - packages as ( - DELETE - FROM system_package - WHERE (rh_account_id, system_id) in (select rh_account_id, id from systems) - ), packages2 as ( DELETE FROM system_package2 @@ -610,25 +600,6 @@ END; $$ LANGUAGE plpgsql; -CREATE OR REPLACE FUNCTION update_status(update_data jsonb) - RETURNS TEXT as -$$ -DECLARE - len int; -BEGIN - len = jsonb_array_length(update_data); - IF len IS NULL or len = 0 THEN - RETURN 'None'; - END IF; - len = jsonb_array_length(jsonb_path_query_array(update_data, '$ ? (@.status == "Installable")')); - IF len > 0 THEN - RETURN 'Installable'; - END IF; - RETURN 'Applicable'; -END; -$$ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; - - -- --------------------------------------------------------------------------- -- Tables -- --------------------------------------------------------------------------- @@ -1015,33 +986,6 @@ CREATE UNIQUE INDEX IF NOT EXISTS package_evra_idx on package (evra, name_id); GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE package TO vmaas_sync; GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE package TO evaluator; -CREATE TABLE IF NOT EXISTS system_package -( - rh_account_id INT NOT NULL REFERENCES rh_account, - system_id BIGINT NOT NULL, - package_id BIGINT NOT NULL REFERENCES package, - -- Use null to represent up-to-date packages - update_data JSONB DEFAULT NULL, - latest_evra TEXT GENERATED ALWAYS AS ( ((update_data ->> -1)::jsonb ->> 'evra')::text) STORED - CHECK(NOT empty(latest_evra)), - name_id BIGINT REFERENCES package_name (id) NOT NULL, - - PRIMARY KEY (rh_account_id, system_id, package_id) INCLUDE (latest_evra) -) PARTITION BY HASH (rh_account_id); - -CREATE INDEX IF NOT EXISTS system_package_name_pkg_system_idx - ON system_package (rh_account_id, name_id, package_id, system_id) INCLUDE (latest_evra); - -CREATE INDEX IF NOT EXISTS system_package_package_id_idx on system_package (package_id); - -GRANT SELECT, INSERT, UPDATE, DELETE ON system_package TO evaluator; -GRANT SELECT, UPDATE, DELETE ON system_package TO listener; -GRANT SELECT, UPDATE, DELETE ON system_package TO manager; -GRANT SELECT, UPDATE, DELETE ON system_package TO vmaas_sync; - -SELECT create_table_partitions('system_package', 128, - $$WITH (fillfactor = '70', autovacuum_vacuum_scale_factor = '0.05')$$); - CREATE TABLE IF NOT EXISTS system_package2 ( rh_account_id INT NOT NULL, diff --git a/dev/test_data.sql b/dev/test_data.sql index 64bbd6514..5eeb880bc 100644 --- a/dev/test_data.sql +++ b/dev/test_data.sql @@ -1,6 +1,5 @@ DELETE FROM system_advisories; DELETE FROM system_repo; -DELETE FROM system_package; DELETE FROM system_package2; DELETE FROM system_platform; DELETE FROM deleted_system; diff --git a/tasks/vmaas_sync/metrics_db_test.go b/tasks/vmaas_sync/metrics_db_test.go index 633adc14d..51a8efbc4 100644 --- a/tasks/vmaas_sync/metrics_db_test.go +++ b/tasks/vmaas_sync/metrics_db_test.go @@ -17,8 +17,8 @@ func TestTableSizes(t *testing.T) { for _, item := range tableSizes { uniqueTables[item.Key] = true } - assert.Equal(t, 342, len(tableSizes)) - assert.Equal(t, 342, len(uniqueTables)) + assert.Equal(t, 213, len(tableSizes)) + assert.Equal(t, 213, len(uniqueTables)) assert.True(t, uniqueTables["system_platform"]) // check whether table names were loaded assert.True(t, uniqueTables["package"]) assert.True(t, uniqueTables["repo"]) From db53b8dbc67d24b91aa724c6f2853626c95c88a4 Mon Sep 17 00:00:00 2001 From: Michael Mraka Date: Tue, 12 Dec 2023 11:19:05 +0100 Subject: [PATCH 2/2] RHINENG-6031: remove migration task --- manager/routes/routes.go | 1 - turnpike/controllers/admin.go | 60 ----------------------------------- 2 files changed, 61 deletions(-) diff --git a/manager/routes/routes.go b/manager/routes/routes.go index fc186f82d..47c07200b 100644 --- a/manager/routes/routes.go +++ b/manager/routes/routes.go @@ -97,7 +97,6 @@ func InitAdmin(app *gin.Engine) { api.GET("/check-caches", admin.CheckCaches) api.PUT("/refresh-packages", admin.RefreshPackagesHandler) api.PUT("/refresh-packages/:account", admin.RefreshPackagesAccountHandler) - api.PUT("/migrate_system_package", admin.MigrateSystemPackage) api.GET("/sessions", admin.GetActiveSessionsHandler) api.GET("/sessions/:search", admin.GetActiveSessionsHandler) api.DELETE("/sessions/:pid", admin.TerminateSessionHandler) diff --git a/turnpike/controllers/admin.go b/turnpike/controllers/admin.go index a41aa4fb9..599bc1461 100644 --- a/turnpike/controllers/admin.go +++ b/turnpike/controllers/admin.go @@ -12,7 +12,6 @@ import ( "time" "github.com/gin-gonic/gin" - "gorm.io/gorm" ) type Session struct { @@ -287,62 +286,3 @@ func getPprof(address, param, query string) ([]byte, error) { } return resBody, nil } - -// @Summary Migrate system_package data to system_package2 -// @Description Migrate system_package data to system_package2 -// @ID migrateSystemPackage -// @Security RhIdentity -// @Accept json -// @Produce json -// @Success 200 {object} string -// @Failure 500 {object} map[string]interface{} -// @Router /migrate_system_package [put] -func MigrateSystemPackage(c *gin.Context) { - utils.LogInfo("starting system_package data migration") - var cnt int64 - db := database.Db - - db.Table("system_package2").Count(&cnt) - if cnt > 0 { - utils.LogInfo("System_package2 table is not empty") - c.JSON(http.StatusNoContent, "System_package2 table is not empty, nothing to do.") - return - } - - // nolint:lll - go func() { - execQuery(db, "VACUUM ANALYZE system_package;") - execQuery(db, "ALTER TABLE system_package2 DROP CONSTRAINT system_package2_applicable_id_fkey;") - execQuery(db, "ALTER TABLE system_package2 DROP CONSTRAINT system_package2_installable_id_fkey;") - execQuery(db, "ALTER TABLE system_package2 DROP CONSTRAINT system_package2_name_id_fkey;") - execQuery(db, "ALTER TABLE system_package2 DROP CONSTRAINT system_package2_package_id_fkey;") - execQuery(db, "ALTER TABLE system_package2 DROP CONSTRAINT system_package2_rh_account_id_system_id_fkey;") - execQuery(db, "DROP INDEX system_package2_account_pkg_name_idx;") - execQuery(db, "DROP INDEX system_package2_package_id_idx;") - - if err := db.Exec("CALL copy_system_packages();").Error; err != nil { - // truncate system_package2 table on failed migration - execQuery(db, "TRUNCATE TABLE system_package2;") - utils.LogError("err", err.Error(), "Migration failed") - return - } - - execQuery(db, "ALTER TABLE system_package2 ADD CONSTRAINT system_package2_applicable_id_fkey FOREIGN KEY (applicable_id) REFERENCES package(id);") - execQuery(db, "ALTER TABLE system_package2 ADD CONSTRAINT system_package2_installable_id_fkey FOREIGN KEY (installable_id) REFERENCES package(id);") - execQuery(db, "ALTER TABLE system_package2 ADD CONSTRAINT system_package2_name_id_fkey FOREIGN KEY (name_id) REFERENCES package_name(id);") - execQuery(db, "ALTER TABLE system_package2 ADD CONSTRAINT system_package2_package_id_fkey FOREIGN KEY (package_id) REFERENCES package(id);") - execQuery(db, "ALTER TABLE system_package2 ADD CONSTRAINT system_package2_rh_account_id_system_id_fkey FOREIGN KEY (rh_account_id, system_id) REFERENCES system_platform (rh_account_id, id);") - execQuery(db, `CREATE INDEX IF NOT EXISTS system_package2_account_pkg_name_idx - ON system_package2 (rh_account_id, name_id) INCLUDE (system_id, package_id, installable_id, applicable_id);`) - execQuery(db, "CREATE INDEX IF NOT EXISTS system_package2_package_id_idx on system_package2 (package_id);") - utils.LogInfo("System_package migration completed") - }() - c.JSON(http.StatusOK, "Migration started") -} - -func execQuery(db *gorm.DB, query string) { - err := db.Exec(query).Error - if err != nil { - utils.LogWarn("err", err.Error(), "query", query, "Exec of query failed.") - } -}