Skip to content

Commit

Permalink
Schedule compression policy more often
Browse files Browse the repository at this point in the history 
By default, the compression policy is scheduled for every
chunk_time_interval / 2 in the current implementation, equal to three
days and twelve hours with our default settings. This schedule interval
was sufficient for previous versions of TimescaleDB. However, with the
introduction of features like mutable compression and ON CONFLICT .. DO
UPDATE queries, regular DML operations decompress data. To ensure that
modified data is compressed earlier, this patch reduces the schedule
interval of the compression policy to run at least every 12 hours.
  • Loading branch information
@jnidzwetzki
jnidzwetzki committed Sep 21, 2023
1 parent 8c41757 commit 95b135e
Show file tree
Hide file tree
Showing 10 changed files with 238 additions and 36 deletions.
23 changes: 21 additions & 2 deletions tsl/src/bgw_policy/compression_api.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,11 @@
#define DEFAULT_RETRY_PERIOD \
DatumGetIntervalP(DirectFunctionCall3(interval_in, CStringGetDatum("1 hour"), InvalidOid, -1))

/* Default max schedule period for the compression policy is 12 hours. The actual schedule period
* will be chunk_interval/2 if the chunk_interval is < 12 hours. */
#define DEFAULT_MAX_SCHEDULE_PERIOD \
DatumGetIntervalP(DirectFunctionCall3(interval_in, CStringGetDatum("12 hours"), InvalidOid, -1))

static Hypertable *validate_compress_chunks_hypertable(Cache *hcache, Oid user_htoid,
bool *is_cagg);

Expand Down Expand Up @@ -248,8 +253,22 @@ policy_compression_add_internal(Oid user_rel_oid, Datum compress_after_datum,
if (dim && IS_TIMESTAMP_TYPE(ts_dimension_get_partition_type(dim)) &&
!user_defined_schedule_interval)
{
default_schedule_interval = DatumGetIntervalP(
ts_internal_to_interval_value(dim->fd.interval_length / 2, INTERVALOID));
int64 hypertable_schedule_interval = dim->fd.interval_length / 2;
int64 max_schedule_interval =
ts_interval_value_to_internal(IntervalPGetDatum(DEFAULT_MAX_SCHEDULE_PERIOD),
INTERVALOID);

/* On hypertables with a small chunk_time_interval, schedule the compression job more often
* than DEFAULT_MAX_SCHEDULE_PERIOD */
if (max_schedule_interval > hypertable_schedule_interval)
{
default_schedule_interval = DatumGetIntervalP(
ts_internal_to_interval_value(hypertable_schedule_interval, INTERVALOID));
}
else
{
default_schedule_interval = DEFAULT_MAX_SCHEDULE_PERIOD;
}
}

/* insert a new job into jobs table */
Expand Down
8 changes: 4 additions & 4 deletions tsl/test/expected/bgw_custom.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -476,7 +476,7 @@ order by id;
select t.schedule_interval FROM alter_job(:job_id_4, next_start=> now() ) t;
schedule_interval
-------------------
@ 7 days 12 hours
@ 12 hours
(1 row)

SELECT wait_for_job_to_run(:job_id_4, 2);
Expand Down Expand Up @@ -917,9 +917,9 @@ ALTER TABLE sensor_data SET (timescaledb.compress, timescaledb.compress_orderby
SELECT add_compression_policy('sensor_data', INTERVAL '1' minute) AS compressjob_id \gset
-- set recompress to true
SELECT alter_job(id,config:=jsonb_set(config,'{recompress}', 'true')) FROM _timescaledb_config.bgw_job WHERE id = :compressjob_id;
alter_job
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
(1014,"@ 3 days 12 hours","@ 0",-1,"@ 1 hour",t,"{""recompress"": true, ""hypertable_id"": 4, ""compress_after"": ""@ 1 min""}",-infinity,_timescaledb_functions.policy_compression_check,f,,)
alter_job
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
(1014,"@ 12 hours","@ 0",-1,"@ 1 hour",t,"{""recompress"": true, ""hypertable_id"": 4, ""compress_after"": ""@ 1 min""}",-infinity,_timescaledb_functions.policy_compression_check,f,,)
(1 row)

-- create new chunks
Expand Down
8 changes: 4 additions & 4 deletions tsl/test/expected/bgw_db_scheduler_fixed.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1696,7 +1696,7 @@ select * from _timescaledb_config.bgw_job;
id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone
------+--------------------------------------------+-------------------+-------------+-------------+--------------+------------------------+-------------------------------------+------------+-----------+----------------+----------------------------------+---------------+--------------------------------------------------------------------------------+------------------------+-------------------------------------------+---------------
1026 | Retention Policy [1026] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_functions | policy_retention | super_user | t | t | Fri Dec 31 16:00:00 1999 PST | 1 | {"drop_after": "@ 2 years", "hypertable_id": 1} | _timescaledb_functions | policy_retention_check | Europe/Berlin
1027 | Compression Policy [1027] | @ 15 days | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | super_user | t | t | Fri Dec 31 16:00:00 1999 PST | 1 | {"hypertable_id": 1, "compress_after": "@ 1 year"} | _timescaledb_functions | policy_compression_check | Europe/Berlin
1027 | Compression Policy [1027] | @ 12 hours | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | super_user | t | t | Fri Dec 31 16:00:00 1999 PST | 1 | {"hypertable_id": 1, "compress_after": "@ 1 year"} | _timescaledb_functions | policy_compression_check | Europe/Berlin
1028 | Refresh Continuous Aggregate Policy [1028] | @ 21 days | @ 0 | -1 | @ 21 days | _timescaledb_functions | policy_refresh_continuous_aggregate | super_user | t | t | Fri Dec 31 16:00:00.005 1999 PST | 2 | {"end_offset": "@ 2 mons", "start_offset": "@ 1 year", "mat_hypertable_id": 2} | _timescaledb_functions | policy_refresh_continuous_aggregate_check | Europe/Athens
(3 rows)

Expand All @@ -1711,7 +1711,7 @@ SELECT * from _timescaledb_internal.bgw_job_stat;
job_id | last_start | last_finish | next_start | last_successful_finish | last_run_success | total_runs | total_duration | total_duration_failures | total_successes | total_failures | total_crashes | consecutive_failures | consecutive_crashes | flags
--------+----------------------------------+----------------------------------+----------------------------------+----------------------------------+------------------+------------+----------------+-------------------------+-----------------+----------------+---------------+----------------------+---------------------+-------
1026 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 16:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST | t | 1 | @ 0 | @ 0 | 1 | 0 | 0 | 0 | 0 | 0
1027 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Sat Jan 15 16:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST | t | 1 | @ 0 | @ 0 | 1 | 0 | 0 | 0 | 0 | 0
1027 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 04:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST | t | 1 | @ 0 | @ 0 | 1 | 0 | 0 | 0 | 0 | 0
1028 | Fri Dec 31 16:00:00.005 1999 PST | Fri Dec 31 16:00:00.005 1999 PST | Fri Jan 21 16:00:00.005 2000 PST | Fri Dec 31 16:00:00.005 1999 PST | t | 1 | @ 0 | @ 0 | 1 | 0 | 0 | 0 | 0 | 0
(3 rows)

Expand Down Expand Up @@ -1780,7 +1780,7 @@ select * from _timescaledb_config.bgw_job order by id;
id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone
------+--------------------------------------------+-------------------+-------------+-------------+--------------+------------------------+-------------------------------------+------------+-----------+----------------+----------------------------------+---------------+--------------------------------------------------------------------------------+------------------------+-------------------------------------------+---------------
1026 | Retention Policy [1026] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_functions | policy_retention | super_user | t | t | Fri Dec 31 16:00:00 1999 PST | 1 | {"drop_after": "@ 2 years", "hypertable_id": 1} | _timescaledb_functions | policy_retention_check | Europe/Berlin
1027 | Compression Policy [1027] | @ 15 days | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | super_user | t | t | Fri Dec 31 16:00:00 1999 PST | 1 | {"hypertable_id": 1, "compress_after": "@ 1 year"} | _timescaledb_functions | policy_compression_check | Europe/Berlin
1027 | Compression Policy [1027] | @ 12 hours | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | super_user | t | t | Fri Dec 31 16:00:00 1999 PST | 1 | {"hypertable_id": 1, "compress_after": "@ 1 year"} | _timescaledb_functions | policy_compression_check | Europe/Berlin
1028 | Refresh Continuous Aggregate Policy [1028] | @ 21 days | @ 0 | -1 | @ 21 days | _timescaledb_functions | policy_refresh_continuous_aggregate | super_user | t | t | Fri Dec 31 16:00:00.005 1999 PST | 2 | {"end_offset": "@ 2 mons", "start_offset": "@ 1 year", "mat_hypertable_id": 2} | _timescaledb_functions | policy_refresh_continuous_aggregate_check | Europe/Athens
1029 | User-Defined Action [1029] | @ 7 mons | @ 0 | -1 | @ 5 mins | public | job_test_fixed | super_user | t | t | Fri Dec 31 16:00:00.01 1999 PST | | | | |
1030 | User-Defined Action [1030] | @ 7 mons | @ 0 | -1 | @ 5 mins | public | job_test_fixed | super_user | t | t | Fri Dec 31 16:00:00.01 1999 PST | | | | | Europe/Athens
Expand All @@ -1798,7 +1798,7 @@ ORDER BY job_id;
job_id | last_start | last_finish | next_start | last_successful_finish
--------+------------------------------+------------------------------+------------------------------+------------------------------
1026 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 16:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST
1027 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Sat Jan 15 16:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST
1027 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 04:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST
1028 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Fri Jan 21 16:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST
1029 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Mon Jul 31 16:00:00 2000 PDT | Fri Dec 31 16:00:00 1999 PST
1030 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Mon Jul 31 16:00:00 2000 PDT | Fri Dec 31 16:00:00 1999 PST
Expand Down
10 changes: 5 additions & 5 deletions tsl/test/expected/bgw_policy.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -714,11 +714,11 @@ select create_hypertable('test_missing_schedint_integer', 'time', chunk_time_int
alter table test_missing_schedint_integer set (timescaledb.compress);
select add_compression_policy('test_missing_schedint_integer', BIGINT '600000') as compression_id_integer \gset
select * from _timescaledb_config.bgw_job where id in (:retenion_id_missing_schedint, :compression_id_missing_schedint, :compression_id_integer);
id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone
------+---------------------------+--------------------+-------------+-------------+--------------+------------------------+--------------------+---------------------+-----------+----------------+---------------+---------------+-----------------------------------------------------+------------------------+--------------------------+----------
1008 | Retention Policy [1008] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_functions | policy_retention | default_perm_user_2 | t | f | | 8 | {"drop_after": "@ 14 days", "hypertable_id": 8} | _timescaledb_functions | policy_retention_check |
1009 | Compression Policy [1009] | @ 15 days 12 hours | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | default_perm_user_2 | t | f | | 8 | {"hypertable_id": 8, "compress_after": "@ 60 days"} | _timescaledb_functions | policy_compression_check |
1010 | Compression Policy [1010] | @ 1 day | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | default_perm_user_2 | t | f | | 10 | {"hypertable_id": 10, "compress_after": 600000} | _timescaledb_functions | policy_compression_check |
id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone
------+---------------------------+-------------------+-------------+-------------+--------------+------------------------+--------------------+---------------------+-----------+----------------+---------------+---------------+-----------------------------------------------------+------------------------+--------------------------+----------
1008 | Retention Policy [1008] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_functions | policy_retention | default_perm_user_2 | t | f | | 8 | {"drop_after": "@ 14 days", "hypertable_id": 8} | _timescaledb_functions | policy_retention_check |
1009 | Compression Policy [1009] | @ 12 hours | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | default_perm_user_2 | t | f | | 8 | {"hypertable_id": 8, "compress_after": "@ 60 days"} | _timescaledb_functions | policy_compression_check |
1010 | Compression Policy [1010] | @ 1 day | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | default_perm_user_2 | t | f | | 10 | {"hypertable_id": 10, "compress_after": 600000} | _timescaledb_functions | policy_compression_check |
(3 rows)

-- test policy check functions with NULL args
Expand Down
Loading

0 comments on commit 95b135e

Please sign in to comment.