From e24d99c88c83bb6202d165f79d99e751ac0d4661 Mon Sep 17 00:00:00 2001 From: Patrik Segedy Date: Mon, 11 Dec 2023 10:20:34 +0100 Subject: [PATCH 1/2] RHINENG-5394: execute all add/drop queries and log error --- turnpike/controllers/admin.go | 54 +++++++++++++++++------------------ 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/turnpike/controllers/admin.go b/turnpike/controllers/admin.go index 7d081575f..a1418102a 100644 --- a/turnpike/controllers/admin.go +++ b/turnpike/controllers/admin.go @@ -12,6 +12,7 @@ import ( "time" "github.com/gin-gonic/gin" + "gorm.io/gorm" ) type Session struct { @@ -301,6 +302,7 @@ func MigrateSystemPackage(c *gin.Context) { var cnt int64 db := database.Db + execQuery(db, "VACUUM ANALYZE system_package;") db.Table("system_package2").Count(&cnt) if cnt > 0 { utils.LogInfo("System_package2 table is not empty") @@ -308,43 +310,39 @@ func MigrateSystemPackage(c *gin.Context) { return } + // nolint:lll go func() { - if err := db.Exec(` - ALTER TABLE system_package2 DROP CONSTRAINT system_package2_applicable_id_fkey; - ALTER TABLE system_package2 DROP CONSTRAINT system_package2_installable_id_fkey; - ALTER TABLE system_package2 DROP CONSTRAINT system_package2_name_id_fkey; - ALTER TABLE system_package2 DROP CONSTRAINT system_package2_package_id_fkey; - ALTER TABLE system_package2 DROP CONSTRAINT system_package2_rh_account_id_system_id_fkey; - DROP INDEX system_package2_account_pkg_name_idx; - DROP INDEX system_package2_package_id_idx; - `).Error; err != nil { - // don't fail when we couldn't remove constraints and indexes - // continue with migration - utils.LogWarn("err", err.Error(), "Couldn't remove constraints and indexes") - } + execQuery(db, "ALTER TABLE system_package2 DROP CONSTRAINT system_package2_applicable_id_fkey;") + execQuery(db, "ALTER TABLE system_package2 DROP CONSTRAINT system_package2_installable_id_fkey;") + execQuery(db, "ALTER TABLE system_package2 DROP CONSTRAINT system_package2_name_id_fkey;") + execQuery(db, "ALTER TABLE system_package2 DROP CONSTRAINT system_package2_package_id_fkey;") + execQuery(db, "ALTER TABLE system_package2 DROP CONSTRAINT system_package2_rh_account_id_system_id_fkey;") + execQuery(db, "DROP INDEX system_package2_account_pkg_name_idx;") + execQuery(db, "DROP INDEX system_package2_package_id_idx;") if err := db.Exec("CALL copy_system_packages();").Error; err != nil { // truncate system_package2 table on failed migration - db.Exec("TRUNCATE TABLE system_package2;") + execQuery(db, "TRUNCATE TABLE system_package2;") utils.LogError("err", err.Error(), "Migration failed") return } - // nolint:lll - if err := db.Exec(` - ALTER TABLE system_package2 ADD CONSTRAINT system_package2_applicable_id_fkey FOREIGN KEY (applicable_id) REFERENCES package(id); - ALTER TABLE system_package2 ADD CONSTRAINT system_package2_installable_id_fkey FOREIGN KEY (installable_id) REFERENCES package(id); - ALTER TABLE system_package2 ADD CONSTRAINT system_package2_name_id_fkey FOREIGN KEY (name_id) REFERENCES package_name(id); - ALTER TABLE system_package2 ADD CONSTRAINT system_package2_package_id_fkey FOREIGN KEY (package_id) REFERENCES package(id); - ALTER TABLE system_package2 ADD CONSTRAINT system_package2_rh_account_id_system_id_fkey FOREIGN KEY (rh_account_id, system_id) REFERENCES system_platform (rh_account_id, id); - CREATE INDEX IF NOT EXISTS system_package2_account_pkg_name_idx - ON system_package2 (rh_account_id, name_id) INCLUDE (system_id, package_id, installable_id, applicable_id); - CREATE INDEX IF NOT EXISTS system_package2_package_id_idx on system_package2 (package_id); - `).Error; err != nil { - utils.LogError("err", err.Error(), "Couldn't add constraints and indexes") - return - } + execQuery(db, "ALTER TABLE system_package2 ADD CONSTRAINT system_package2_applicable_id_fkey FOREIGN KEY (applicable_id) REFERENCES package(id);") + execQuery(db, "ALTER TABLE system_package2 ADD CONSTRAINT system_package2_installable_id_fkey FOREIGN KEY (installable_id) REFERENCES package(id);") + execQuery(db, "ALTER TABLE system_package2 ADD CONSTRAINT system_package2_name_id_fkey FOREIGN KEY (name_id) REFERENCES package_name(id);") + execQuery(db, "ALTER TABLE system_package2 ADD CONSTRAINT system_package2_package_id_fkey FOREIGN KEY (package_id) REFERENCES package(id);") + execQuery(db, "ALTER TABLE system_package2 ADD CONSTRAINT system_package2_rh_account_id_system_id_fkey FOREIGN KEY (rh_account_id, system_id) REFERENCES system_platform (rh_account_id, id);") + execQuery(db, `CREATE INDEX IF NOT EXISTS system_package2_account_pkg_name_idx + ON system_package2 (rh_account_id, name_id) INCLUDE (system_id, package_id, installable_id, applicable_id);`) + execQuery(db, "CREATE INDEX IF NOT EXISTS system_package2_package_id_idx on system_package2 (package_id);") utils.LogInfo("System_package migration completed") }() c.JSON(http.StatusOK, "Migration started") } + +func execQuery(db *gorm.DB, query string) { + err := db.Exec(query).Error + if err != nil { + utils.LogWarn("err", err.Error(), "query", query, "Exec of query failed.") + } +} From a6cca3d24c2e0228401f0a35a4d0787b1e334522 Mon Sep 17 00:00:00 2001 From: Patrik Segedy Date: Mon, 11 Dec 2023 10:21:05 +0100 Subject: [PATCH 2/2] RHINENG-5394: migrate system_package per account Co-authored-by: dkuc --- .../119_migrate_system_package2.up.sql | 41 +++++++++++++++++++ database_admin/schema/create_schema.sql | 2 +- 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 database_admin/migrations/119_migrate_system_package2.up.sql diff --git a/database_admin/migrations/119_migrate_system_package2.up.sql b/database_admin/migrations/119_migrate_system_package2.up.sql new file mode 100644 index 000000000..68e29b69e --- /dev/null +++ b/database_admin/migrations/119_migrate_system_package2.up.sql @@ -0,0 +1,41 @@ +CREATE OR REPLACE PROCEDURE copy_system_packages() + LANGUAGE plpgsql +AS +$$ +DECLARE + cnt bigint := 0; + prev_cnt bigint := 0; + rows_inserted bigint := 0; + account int := 0; +BEGIN + FOR account IN (SELECT id from rh_account ORDER BY hash_partition_id(id, 128), id) + LOOP + INSERT INTO system_package2 + SELECT system_package.rh_account_id, + system_id, + name_id, + package_id, + (SELECT id + FROM package + WHERE package.name_id = system_package.name_id + AND evra = + JSONB_PATH_QUERY_ARRAY(update_data, '$[*] ? (@.status== "Installable").evra') ->> 0), + (SELECT id + FROM package + WHERE package.name_id = system_package.name_id + AND evra = JSONB_PATH_QUERY_ARRAY(update_data, '$[*] ? (@.status== "Applicable").evra') ->> 0) + FROM system_package + JOIN system_platform ON system_platform.id = system_package.system_id AND system_platform.rh_account_id = system_package.rh_account_id + WHERE system_package.rh_account_id = account; + COMMIT; + + GET DIAGNOSTICS rows_inserted = ROW_COUNT; + + cnt := cnt + rows_inserted; + IF (cnt/1000000)::int > (prev_cnt/1000000)::int THEN + RAISE NOTICE 'inserted % rows, account: %, partition: %', cnt, account, hash_partition_id(account, 128); + prev_cnt := cnt; + END IF; + END LOOP; +END +$$; diff --git a/database_admin/schema/create_schema.sql b/database_admin/schema/create_schema.sql index ea22af09c..57ebc5ea8 100644 --- a/database_admin/schema/create_schema.sql +++ b/database_admin/schema/create_schema.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS schema_migrations INSERT INTO schema_migrations -VALUES (118, false); +VALUES (119, false); -- --------------------------------------------------------------------------- -- Functions