|
| 1 | +import kx, { Knex } from "knex"; |
| 2 | + |
| 3 | +import { TableName } from "../schemas"; |
| 4 | + |
| 5 | +const INTERMEDIATE_AUDIT_LOG_TABLE = "intermediate_audit_logs"; |
| 6 | + |
| 7 | +const formatPartitionDate = (date: Date) => { |
| 8 | + const year = date.getFullYear(); |
| 9 | + const month = String(date.getMonth() + 1).padStart(2, "0"); |
| 10 | + const day = String(date.getDate()).padStart(2, "0"); |
| 11 | + |
| 12 | + return `${year}-${month}-${day}`; |
| 13 | +}; |
| 14 | + |
| 15 | +const createAuditLogPartition = async (knex: Knex, startDate: Date, endDate: Date) => { |
| 16 | + const startDateStr = formatPartitionDate(startDate); |
| 17 | + const endDateStr = formatPartitionDate(endDate); |
| 18 | + |
| 19 | + const partitionName = `${TableName.AuditLog}_${startDateStr.replace(/-/g, "")}_${endDateStr.replace(/-/g, "")}`; |
| 20 | + |
| 21 | + await knex.schema.raw( |
| 22 | + `CREATE TABLE ${partitionName} PARTITION OF ${TableName.AuditLog} FOR VALUES FROM ('${startDateStr}') TO ('${endDateStr}')` |
| 23 | + ); |
| 24 | +}; |
| 25 | + |
| 26 | +const up = async (knex: Knex): Promise<void> => { |
| 27 | + console.info("Dropping primary key of audit log table..."); |
| 28 | + await knex.schema.alterTable(TableName.AuditLog, (t) => { |
| 29 | + // remove existing keys |
| 30 | + t.dropPrimary(); |
| 31 | + }); |
| 32 | + |
| 33 | + // Get all indices of the audit log table and drop them |
| 34 | + const indexNames: { rows: { indexname: string }[] } = await knex.raw( |
| 35 | + ` |
| 36 | + SELECT indexname |
| 37 | + FROM pg_indexes |
| 38 | + WHERE tablename = '${TableName.AuditLog}' |
| 39 | + ` |
| 40 | + ); |
| 41 | + |
| 42 | + console.log( |
| 43 | + "Deleting existing audit log indices:", |
| 44 | + indexNames.rows.map((e) => e.indexname) |
| 45 | + ); |
| 46 | + |
| 47 | + for await (const row of indexNames.rows) { |
| 48 | + await knex.raw(`DROP INDEX IF EXISTS ${row.indexname}`); |
| 49 | + } |
| 50 | + |
| 51 | + // renaming audit log to intermediate table |
| 52 | + console.log("Renaming audit log table to the intermediate name"); |
| 53 | + await knex.schema.renameTable(TableName.AuditLog, INTERMEDIATE_AUDIT_LOG_TABLE); |
| 54 | + |
| 55 | + if (!(await knex.schema.hasTable(TableName.AuditLog))) { |
| 56 | + const createTableSql = knex.schema |
| 57 | + .createTable(TableName.AuditLog, (t) => { |
| 58 | + t.uuid("id").defaultTo(knex.fn.uuid()); |
| 59 | + t.string("actor").notNullable(); |
| 60 | + t.jsonb("actorMetadata").notNullable(); |
| 61 | + t.string("ipAddress"); |
| 62 | + t.string("eventType").notNullable(); |
| 63 | + t.jsonb("eventMetadata"); |
| 64 | + t.string("userAgent"); |
| 65 | + t.string("userAgentType"); |
| 66 | + t.datetime("expiresAt"); |
| 67 | + t.timestamps(true, true, true); |
| 68 | + t.uuid("orgId"); |
| 69 | + t.string("projectId"); |
| 70 | + t.string("projectName"); |
| 71 | + t.primary(["id", "createdAt"]); |
| 72 | + }) |
| 73 | + .toString(); |
| 74 | + |
| 75 | + console.info("Creating partition table..."); |
| 76 | + await knex.schema.raw(` |
| 77 | + ${createTableSql} PARTITION BY RANGE ("createdAt"); |
| 78 | + `); |
| 79 | + |
| 80 | + console.log("Adding indices..."); |
| 81 | + await knex.schema.alterTable(TableName.AuditLog, (t) => { |
| 82 | + t.index(["projectId", "createdAt"]); |
| 83 | + t.index(["orgId", "createdAt"]); |
| 84 | + t.index("expiresAt"); |
| 85 | + t.index("orgId"); |
| 86 | + t.index("projectId"); |
| 87 | + }); |
| 88 | + |
| 89 | + console.log("Adding GIN indices..."); |
| 90 | + |
| 91 | + await knex.raw( |
| 92 | + `CREATE INDEX IF NOT EXISTS "audit_logs_actorMetadata_idx" ON ${TableName.AuditLog} USING gin("actorMetadata" jsonb_path_ops)` |
| 93 | + ); |
| 94 | + console.log("GIN index for actorMetadata done"); |
| 95 | + |
| 96 | + await knex.raw( |
| 97 | + `CREATE INDEX IF NOT EXISTS "audit_logs_eventMetadata_idx" ON ${TableName.AuditLog} USING gin("eventMetadata" jsonb_path_ops)` |
| 98 | + ); |
| 99 | + console.log("GIN index for eventMetadata done"); |
| 100 | + |
| 101 | + // create default partition |
| 102 | + console.log("Creating default partition..."); |
| 103 | + await knex.schema.raw(`CREATE TABLE ${TableName.AuditLog}_default PARTITION OF ${TableName.AuditLog} DEFAULT`); |
| 104 | + |
| 105 | + const nextDate = new Date(); |
| 106 | + nextDate.setDate(nextDate.getDate() + 1); |
| 107 | + const nextDateStr = formatPartitionDate(nextDate); |
| 108 | + |
| 109 | + console.log("Attaching existing audit log table as a partition..."); |
| 110 | + await knex.schema.raw(` |
| 111 | + ALTER TABLE ${INTERMEDIATE_AUDIT_LOG_TABLE} ADD CONSTRAINT audit_log_old |
| 112 | + CHECK ( "createdAt" < DATE '${nextDateStr}' ); |
| 113 | +
|
| 114 | + ALTER TABLE ${TableName.AuditLog} ATTACH PARTITION ${INTERMEDIATE_AUDIT_LOG_TABLE} |
| 115 | + FOR VALUES FROM (MINVALUE) TO ('${nextDateStr}' ); |
| 116 | + `); |
| 117 | + |
| 118 | + // create partition from now until end of month |
| 119 | + console.log("Creating audit log partitions ahead of time... next date:", nextDateStr); |
| 120 | + await createAuditLogPartition(knex, nextDate, new Date(nextDate.getFullYear(), nextDate.getMonth() + 1)); |
| 121 | + |
| 122 | + // create partitions 4 years ahead |
| 123 | + const partitionMonths = 4 * 12; |
| 124 | + const partitionPromises: Promise<void>[] = []; |
| 125 | + for (let x = 1; x <= partitionMonths; x += 1) { |
| 126 | + partitionPromises.push( |
| 127 | + createAuditLogPartition( |
| 128 | + knex, |
| 129 | + new Date(nextDate.getFullYear(), nextDate.getMonth() + x, 1), |
| 130 | + new Date(nextDate.getFullYear(), nextDate.getMonth() + (x + 1), 1) |
| 131 | + ) |
| 132 | + ); |
| 133 | + } |
| 134 | + |
| 135 | + await Promise.all(partitionPromises); |
| 136 | + console.log("Partition migration complete"); |
| 137 | + } |
| 138 | +}; |
| 139 | + |
| 140 | +export const executeMigration = async (url: string) => { |
| 141 | + console.log("Executing migration..."); |
| 142 | + const knex = kx({ |
| 143 | + client: "pg", |
| 144 | + connection: url |
| 145 | + }); |
| 146 | + |
| 147 | + await knex.transaction(async (tx) => { |
| 148 | + await up(tx); |
| 149 | + }); |
| 150 | +}; |
| 151 | + |
| 152 | +const dbUrl = process.env.AUDIT_LOGS_DB_CONNECTION_URI; |
| 153 | +if (!dbUrl) { |
| 154 | + console.error("Please provide a DB connection URL to the AUDIT_LOGS_DB_CONNECTION_URI env"); |
| 155 | + process.exit(1); |
| 156 | +} |
| 157 | + |
| 158 | +void executeMigration(dbUrl).then(() => { |
| 159 | + console.log("Migration: partition-audit-logs DONE"); |
| 160 | + process.exit(0); |
| 161 | +}); |
0 commit comments