From 6e2b6a7a415719b00c25c0323b6e963f1933a854 Mon Sep 17 00:00:00 2001 From: Patrik Segedy Date: Tue, 5 Dec 2023 14:35:37 +0100 Subject: [PATCH] POC: migrate system_package with admin api Co-authored-by: dkuc --- .../117_migrate_system_package2.down.sql | 1 + .../117_migrate_system_package2.up.sql | 99 ++++++++++++++++--- docs/admin/openapi.json | 82 ++++++++------- manager/routes/routes.go | 1 + turnpike/controllers/admin.go | 59 +++++++++++ 5 files changed, 194 insertions(+), 48 deletions(-) diff --git a/database_admin/migrations/117_migrate_system_package2.down.sql b/database_admin/migrations/117_migrate_system_package2.down.sql index e69de29bb..b15d03341 100644 --- a/database_admin/migrations/117_migrate_system_package2.down.sql +++ b/database_admin/migrations/117_migrate_system_package2.down.sql @@ -0,0 +1 @@ +DROP PROCEDURE copy_system_packages; diff --git a/database_admin/migrations/117_migrate_system_package2.up.sql b/database_admin/migrations/117_migrate_system_package2.up.sql index db65f9f19..1bd4324bc 100644 --- a/database_admin/migrations/117_migrate_system_package2.up.sql +++ b/database_admin/migrations/117_migrate_system_package2.up.sql @@ -1,16 +1,87 @@ +/* +Loops through the PK(account_id, system_id, package_id) in chunks of 1000 and inserts it into the new table. +Assumptions: +1. the system_packages2 table is empty. +2. No new system_packages comes in when this is running. +*/ --- migrate syste_package partitions one by one -DO +CREATE OR REPLACE PROCEDURE copy_system_packages() + LANGUAGE plpgsql +AS $$ - DECLARE - part text; - BEGIN - FOR part IN (SELECT tablename from pg_tables t where t.tablename ~ '^system_package_[0-9]+$') - LOOP - RAISE NOTICE 'Copying the % partition', part; - EXECUTE 'INSERT INTO system_package2 (rh_account_id, system_id, name_id, package_id, installable_id, applicable_id) - ( SELECT rh_account_id, system_id, name_id, package_id, NULL, NULL FROM ' || - part || ') ON CONFLICT DO NOTHING'; - END LOOP; - END -$$ LANGUAGE plpgsql; +DECLARE + rows_inserted INTEGER := 0; + account_idx integer := 0; + system_idx bigint := 0; + package_idx bigint := 0; +BEGIN + LOOP + INSERT INTO system_package2 + SELECT rh_account_id, + system_id, + name_id, + package_id, + (SELECT id + FROM package + WHERE package.name_id = system_package.name_id + AND evra = + JSONB_PATH_QUERY_ARRAY(update_data, '$[*] ? (@.status== "Installable").evra') ->> 0), + (SELECT id + FROM package + WHERE package.name_id = system_package.name_id + AND evra = JSONB_PATH_QUERY_ARRAY(update_data, '$[*] ? (@.status== "Applicable").evra') ->> 0) + FROM system_package + WHERE (rh_account_id = account_idx AND system_id = system_idx AND package_id > package_idx) + OR (rh_account_id = account_idx AND system_id > system_idx) + ORDER BY rh_account_id, system_id, package_id + LIMIT 1000; + + GET DIAGNOSTICS rows_inserted = ROW_COUNT; + + COMMIT; + + /* I Originally did not include this if statement but I couldn't get the select to use the PK index when I had all 3 where clauses + + 1. rh_account_id = account_idx AND system_id = system_idx AND package_id > package_idx + 2. rh_account_id = account_idx AND system_id > system_idx + 3. rh_account_id > account_idx + + So I just copied and pasted the same select / insert and used the final clause when the first two return 0 rows + */ + IF rows_inserted = 0 THEN + INSERT INTO system_package2 + SELECT rh_account_id, + system_id, + name_id, + package_id, + (SELECT id + FROM package + WHERE package.name_id = system_package.name_id + AND evra = JSONB_PATH_QUERY_ARRAY(update_data, '$[*] ? (@.status== "Installable").evra') ->> 0), + (SELECT id + FROM package + WHERE package.name_id = system_package.name_id + AND evra = JSONB_PATH_QUERY_ARRAY(update_data, '$[*] ? (@.status== "Applicable").evra') ->> 0) + FROM system_package + WHERE rh_account_id > account_idx + ORDER BY rh_account_id, system_id, package_id + LIMIT 1000; + + GET DIAGNOSTICS rows_inserted = ROW_COUNT; + + COMMIT; + END IF; + + EXIT WHEN rows_inserted = 0; + + /* Store the highest values of our account/system/package ids are up to. + Should be O(1) because it can just look at the tail of the PK index*/ + SELECT rh_account_id, system_id, package_id + INTO account_idx, system_idx, package_idx + FROM system_package2 + ORDER BY rh_account_id DESC, system_id DESC, package_id DESC + LIMIT 1; + + END LOOP; +END +$$; diff --git a/docs/admin/openapi.json b/docs/admin/openapi.json index 3d340a7c5..510051d49 100644 --- a/docs/admin/openapi.json +++ b/docs/admin/openapi.json @@ -48,9 +48,42 @@ "application/json": { "schema": { "type": "object", - "additionalProperties": { - "type": "object" - } + "additionalProperties": true + } + } + } + } + }, + "security": [ + { + "RhIdentity": [] + } + ] + } + }, + "/migrate_system_package": { + "put": { + "summary": "Migrate system_package data to system_package2", + "description": "Migrate system_package data to system_package2", + "operationId": "migrateSystemPackage", + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "string" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true } } } @@ -90,9 +123,7 @@ "application/octet-stream": { "schema": { "type": "object", - "additionalProperties": { - "type": "object" - } + "additionalProperties": true } } } @@ -132,9 +163,7 @@ "application/octet-stream": { "schema": { "type": "object", - "additionalProperties": { - "type": "object" - } + "additionalProperties": true } } } @@ -174,9 +203,7 @@ "application/octet-stream": { "schema": { "type": "object", - "additionalProperties": { - "type": "object" - } + "additionalProperties": true } } } @@ -216,9 +243,7 @@ "application/octet-stream": { "schema": { "type": "object", - "additionalProperties": { - "type": "object" - } + "additionalProperties": true } } } @@ -253,9 +278,7 @@ "application/json": { "schema": { "type": "object", - "additionalProperties": { - "type": "object" - } + "additionalProperties": true } } } @@ -300,9 +323,7 @@ "application/json": { "schema": { "type": "object", - "additionalProperties": { - "type": "object" - } + "additionalProperties": true } } } @@ -358,9 +379,7 @@ "application/json": { "schema": { "type": "object", - "additionalProperties": { - "type": "object" - } + "additionalProperties": true } } } @@ -416,9 +435,7 @@ "application/json": { "schema": { "type": "object", - "additionalProperties": { - "type": "object" - } + "additionalProperties": true } } } @@ -477,9 +494,7 @@ "application/json": { "schema": { "type": "object", - "additionalProperties": { - "type": "object" - } + "additionalProperties": true } } } @@ -514,9 +529,7 @@ "application/json": { "schema": { "type": "object", - "additionalProperties": { - "type": "object" - } + "additionalProperties": true } } } @@ -551,5 +564,6 @@ "in": "header" } } - } + }, + "x-original-swagger-version": "2.0" } diff --git a/manager/routes/routes.go b/manager/routes/routes.go index c5e9cad09..840a5dad8 100644 --- a/manager/routes/routes.go +++ b/manager/routes/routes.go @@ -96,6 +96,7 @@ func InitAdmin(app *gin.Engine) { api.GET("/check-caches", admin.CheckCaches) api.PUT("/refresh-packages", admin.RefreshPackagesHandler) api.PUT("/refresh-packages/:account", admin.RefreshPackagesAccountHandler) + api.PUT("/migrate_system_package", admin.MigrateSystemPackage) api.GET("/sessions", admin.GetActiveSessionsHandler) api.GET("/sessions/:search", admin.GetActiveSessionsHandler) api.DELETE("/sessions/:pid", admin.TerminateSessionHandler) diff --git a/turnpike/controllers/admin.go b/turnpike/controllers/admin.go index 599bc1461..f702f7359 100644 --- a/turnpike/controllers/admin.go +++ b/turnpike/controllers/admin.go @@ -286,3 +286,62 @@ func getPprof(address, param, query string) ([]byte, error) { } return resBody, nil } + +// @Summary Migrate system_package data to system_package2 +// @Description Migrate system_package data to system_package2 +// @ID migrateSystemPackage +// @Security RhIdentity +// @Accept json +// @Produce json +// @Success 200 {object} string +// @Failure 500 {object} map[string]interface{} +// @Router /migrate_system_package [put] +func MigrateSystemPackage(c *gin.Context) { + utils.LogInfo("starting system_package data migration") + var cnt int64 + db := database.Db + + db.Table("system_package2").Count(&cnt) + if cnt > 0 { + utils.LogInfo("System_package2 table is not empty") + c.JSON(http.StatusNoContent, "System_package2 table is not empty, nothing to do.") + return + } + + go func() { + if err := db.Exec(` + ALTER TABLE system_package2 DROP CONSTRAINT system_package2_applicable_id_fkey; + ALTER TABLE system_package2 DROP CONSTRAINT system_package2_installable_id_fkey; + ALTER TABLE system_package2 DROP CONSTRAINT system_package2_name_id_fkey; + ALTER TABLE system_package2 DROP CONSTRAINT system_package2_package_id_fkey; + ALTER TABLE system_package2 DROP CONSTRAINT system_package2_rh_account_id_system_id_fkey; + DROP INDEX system_package2_account_pkg_name_idx; + DROP INDEX system_package2_package_id_idx; + `).Error; err != nil { + utils.LogError("err", err.Error(), "Couldn't remove constraints and indexes") + return + } + + if err := db.Exec("CALL copy_system_packages();").Error; err != nil { + utils.LogError("err", err.Error(), "Migration failed") + return + } + + // nolint:lll + if err := db.Exec(` + ALTER TABLE system_package2 ADD CONSTRAINT system_package2_applicable_id_fkey FOREIGN KEY (applicable_id) REFERENCES package(id); + ALTER TABLE system_package2 ADD CONSTRAINT system_package2_installable_id_fkey FOREIGN KEY (installable_id) REFERENCES package(id); + ALTER TABLE system_package2 ADD CONSTRAINT system_package2_name_id_fkey FOREIGN KEY (name_id) REFERENCES package_name(id); + ALTER TABLE system_package2 ADD CONSTRAINT system_package2_package_id_fkey FOREIGN KEY (package_id) REFERENCES package(id); + ALTER TABLE system_package2 ADD CONSTRAINT system_package2_rh_account_id_system_id_fkey FOREIGN KEY (rh_account_id, system_id) REFERENCES system_platform (rh_account_id, id); + CREATE INDEX IF NOT EXISTS system_package2_account_pkg_name_idx + ON system_package2 (rh_account_id, name_id) INCLUDE (system_id, package_id, installable_id, applicable_id); + CREATE INDEX IF NOT EXISTS system_package2_package_id_idx on system_package2 (package_id); + `).Error; err != nil { + utils.LogError("err", err.Error(), "Couldn't add constraints and indexes") + return + } + utils.LogInfo("System_package migration completed") + }() + c.JSON(http.StatusOK, "Migration started") +}