From 5309d9a8dc9fcbc56ed3e853adddb5ef6cd36380 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 15 Aug 2023 16:55:36 -0600 Subject: [PATCH] Expire entries from the Media ID hold table Fixes https://github.com/turt2live/matrix-media-repo/issues/426 --- database/table_media_hold.go | 15 ++++++++++++-- ...24_add_timestamp_to_media_id_hold_down.sql | 1 + .../24_add_timestamp_to_media_id_hold_up.sql | 1 + tasks/all.go | 1 + tasks/schedule.go | 7 ++++--- tasks/task_runner/purge_held_media_ids.go | 20 +++++++++++++++++++ 6 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 migrations/24_add_timestamp_to_media_id_hold_down.sql create mode 100644 migrations/24_add_timestamp_to_media_id_hold_up.sql create mode 100644 tasks/task_runner/purge_held_media_ids.go diff --git a/database/table_media_hold.go b/database/table_media_hold.go index 0911f957..da05b42e 100644 --- a/database/table_media_hold.go +++ b/database/table_media_hold.go @@ -5,6 +5,7 @@ import ( "errors" "github.com/turt2live/matrix-media-repo/common/rcontext" + "github.com/turt2live/matrix-media-repo/util" ) type DbHeldMedia struct { @@ -19,10 +20,12 @@ const ( ForCreateHeldReason HeldReason = "media_create" ) -const insertHeldMedia = "INSERT INTO media_id_hold (origin, media_id, reason) VALUES ($1, $2, $3);" +const insertHeldMedia = "INSERT INTO media_id_hold (origin, media_id, reason, held_ts) VALUES ($1, $2, $3, $4);" +const deleteHeldMedia = "DELETE FROM media_id_hold WHERE reason = $1 AND held_ts <= $2;" type heldMediaTableStatements struct { insertHeldMedia *sql.Stmt + deleteHeldMedia *sql.Stmt } type heldMediaTableWithContext struct { @@ -37,6 +40,9 @@ func prepareHeldMediaTables(db *sql.DB) (*heldMediaTableStatements, error) { if stmts.insertHeldMedia, err = db.Prepare(insertHeldMedia); err != nil { return nil, errors.New("error preparing insertHeldMedia: " + err.Error()) } + if stmts.deleteHeldMedia, err = db.Prepare(deleteHeldMedia); err != nil { + return nil, errors.New("error preparing deleteHeldMedia: " + err.Error()) + } return stmts, nil } @@ -49,6 +55,11 @@ func (s *heldMediaTableStatements) Prepare(ctx rcontext.RequestContext) *heldMed } func (s *heldMediaTableWithContext) TryInsert(origin string, mediaId string, reason HeldReason) error { - _, err := s.statements.insertHeldMedia.ExecContext(s.ctx, origin, mediaId, reason) + _, err := s.statements.insertHeldMedia.ExecContext(s.ctx, origin, mediaId, reason, util.NowMillis()) + return err +} + +func (s *heldMediaTableWithContext) DeleteOlderThan(reason HeldReason, olderThanTs int64) error { + _, err := s.statements.deleteHeldMedia.ExecContext(s.ctx, reason, olderThanTs) return err } diff --git a/migrations/24_add_timestamp_to_media_id_hold_down.sql b/migrations/24_add_timestamp_to_media_id_hold_down.sql new file mode 100644 index 00000000..53a0631f --- /dev/null +++ b/migrations/24_add_timestamp_to_media_id_hold_down.sql @@ -0,0 +1 @@ +ALTER TABLE media_id_hold DROP COLUMN held_ts; diff --git a/migrations/24_add_timestamp_to_media_id_hold_up.sql b/migrations/24_add_timestamp_to_media_id_hold_up.sql new file mode 100644 index 00000000..b4b3ac2f --- /dev/null +++ b/migrations/24_add_timestamp_to_media_id_hold_up.sql @@ -0,0 +1 @@ +ALTER TABLE media_id_hold ADD COLUMN held_ts BIGINT NOT NULL DEFAULT 0; diff --git a/tasks/all.go b/tasks/all.go index a4eb7e01..edbeb6d7 100644 --- a/tasks/all.go +++ b/tasks/all.go @@ -10,6 +10,7 @@ func StartAll() { scheduleHourly(RecurringTaskPurgeRemoteMedia, task_runner.PurgeRemoteMedia) scheduleHourly(RecurringTaskPurgeThumbnails, task_runner.PurgeThumbnails) scheduleHourly(RecurringTaskPurgePreviews, task_runner.PurgePreviews) + scheduleHourly(RecurringTaskPurgeHeldMediaIds, task_runner.PurgeHeldMediaIds) scheduleUnfinished() } diff --git a/tasks/schedule.go b/tasks/schedule.go index 576c33b0..aeed88b7 100644 --- a/tasks/schedule.go +++ b/tasks/schedule.go @@ -24,9 +24,10 @@ const ( TaskImportData TaskName = "import_data" ) const ( - RecurringTaskPurgeThumbnails RecurringTaskName = "recurring_purge_thumbnails" - RecurringTaskPurgePreviews RecurringTaskName = "recurring_purge_previews" - RecurringTaskPurgeRemoteMedia RecurringTaskName = "recurring_purge_remote_media" + RecurringTaskPurgeThumbnails RecurringTaskName = "recurring_purge_thumbnails" + RecurringTaskPurgePreviews RecurringTaskName = "recurring_purge_previews" + RecurringTaskPurgeRemoteMedia RecurringTaskName = "recurring_purge_remote_media" + RecurringTaskPurgeHeldMediaIds RecurringTaskName = "recurring_purge_held_media_ids" ) const ExecutingMachineId = int64(0) diff --git a/tasks/task_runner/purge_held_media_ids.go b/tasks/task_runner/purge_held_media_ids.go new file mode 100644 index 00000000..b0793571 --- /dev/null +++ b/tasks/task_runner/purge_held_media_ids.go @@ -0,0 +1,20 @@ +package task_runner + +import ( + "github.com/getsentry/sentry-go" + "github.com/turt2live/matrix-media-repo/common/rcontext" + "github.com/turt2live/matrix-media-repo/database" + "github.com/turt2live/matrix-media-repo/util" +) + +func PurgeHeldMediaIds(ctx rcontext.RequestContext) { + // dev note: don't use ctx for config lookup to avoid misreading it + + beforeTs := util.NowMillis() - int64(7*24*60*60*1000) // 7 days + db := database.GetInstance().HeldMedia.Prepare(ctx) + + if err := db.DeleteOlderThan(database.ForCreateHeldReason, beforeTs); err != nil { + ctx.Log.Error("Error deleting held media IDs: ", err) + sentry.CaptureException(err) + } +}