Skip to content

Commit

Permalink
Schedule compression policy more often
Browse files Browse the repository at this point in the history
By default, the compression policy is scheduled for every
chunk_time_interval / 2 in the current implementation, equal to three
days and twelve hours with our default settings. This schedule interval
was sufficient for previous versions of TimescaleDB. However, with the
introduction of features like mutable compression and ON CONFLICT .. DO
UPDATE queries, regular DML operations decompress data. To ensure that
modified data is compressed earlier, this patch reduces the schedule
interval of the compression policy to run at least every 12 hours.
  • Loading branch information
jnidzwetzki committed Sep 21, 2023
1 parent 8c41757 commit 0d7f95b
Show file tree
Hide file tree
Showing 4 changed files with 204 additions and 2 deletions.
23 changes: 21 additions & 2 deletions tsl/src/bgw_policy/compression_api.c
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,11 @@
#define DEFAULT_RETRY_PERIOD \
DatumGetIntervalP(DirectFunctionCall3(interval_in, CStringGetDatum("1 hour"), InvalidOid, -1))

/* Default max schedule period for the compression policy is 12 hours. The actual schedule period
* will be chunk_interval/2 if the chunk_interval is < 12 hours. */
#define DEFAULT_MAX_SCHEDULE_PERIOD \
DatumGetIntervalP(DirectFunctionCall3(interval_in, CStringGetDatum("12 hours"), InvalidOid, -1))

static Hypertable *validate_compress_chunks_hypertable(Cache *hcache, Oid user_htoid,
bool *is_cagg);

Expand Down Expand Up @@ -248,8 +253,22 @@ policy_compression_add_internal(Oid user_rel_oid, Datum compress_after_datum,
if (dim && IS_TIMESTAMP_TYPE(ts_dimension_get_partition_type(dim)) &&
!user_defined_schedule_interval)
{
default_schedule_interval = DatumGetIntervalP(
ts_internal_to_interval_value(dim->fd.interval_length / 2, INTERVALOID));
int64 hypertable_schedule_interval = dim->fd.interval_length / 2;
int64 max_schedule_interval =
ts_interval_value_to_internal(IntervalPGetDatum(DEFAULT_MAX_SCHEDULE_PERIOD),
INTERVALOID);

/* On hypertables with a small chunk_time_interval, schedule the compression job more often
* than DEFAULT_MAX_SCHEDULE_PERIOD */
if (max_schedule_interval > hypertable_schedule_interval)
{
default_schedule_interval = DatumGetIntervalP(
ts_internal_to_interval_value(hypertable_schedule_interval, INTERVALOID));
}
else
{
default_schedule_interval = DEFAULT_MAX_SCHEDULE_PERIOD;
}
}

/* insert a new job into jobs table */
Expand Down
103 changes: 103 additions & 0 deletions tsl/test/expected/compression_policy.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
-- This file and its contents are licensed under the Timescale License.
-- Please see the included NOTICE for copyright information and
-- LICENSE-TIMESCALE for a copy of the license.
----
-- Chunk interval 1 month - compression schedule interval should be 12 hours
----
CREATE TABLE sensor_data_1month(
time timestamptz not null,
sensor_id integer not null
);
SELECT FROM create_hypertable('sensor_data_1month', 'time', chunk_time_interval => INTERVAL '1 month');
--
(1 row)

ALTER TABLE sensor_data_1month SET (timescaledb.compress);
-- Add a compression policy and check the schedule interval
SELECT add_compression_policy('sensor_data_1month','7 days'::INTERVAL) as compression_job \gset
SELECT schedule_interval from timescaledb_information.jobs where job_id = :compression_job;
schedule_interval
-------------------
@ 12 hours
(1 row)

----
-- Chunk interval 1 week - compression schedule interval should be 12 hours
----
CREATE TABLE sensor_data_1week(
time timestamptz not null,
sensor_id integer not null
);
SELECT FROM create_hypertable('sensor_data_1week', 'time', chunk_time_interval => INTERVAL '1 week');
--
(1 row)

ALTER TABLE sensor_data_1week SET (timescaledb.compress);
-- Add a compression policy and check the schedule interval
SELECT add_compression_policy('sensor_data_1week','7 days'::INTERVAL) as compression_job \gset
SELECT schedule_interval from timescaledb_information.jobs where job_id = :compression_job;
schedule_interval
-------------------
@ 12 hours
(1 row)

----
-- Chunk interval 1 day - compression schedule interval should be 12 hours
----
CREATE TABLE sensor_data_1day(
time timestamptz not null,
sensor_id integer not null
);
SELECT FROM create_hypertable('sensor_data_1day', 'time', chunk_time_interval => INTERVAL '1 day');
--
(1 row)

ALTER TABLE sensor_data_1day SET (timescaledb.compress);
-- Add a compression policy and check the schedule interval
SELECT add_compression_policy('sensor_data_1day','7 days'::INTERVAL) as compression_job \gset
SELECT schedule_interval from timescaledb_information.jobs where job_id = :compression_job;
schedule_interval
-------------------
@ 12 hours
(1 row)

----
-- Chunk interval 1 hour - compression schedule interval should be 30 minutes
----
CREATE TABLE sensor_data_1hour(
time timestamptz not null,
sensor_id integer not null
);
SELECT FROM create_hypertable('sensor_data_1hour', 'time', chunk_time_interval => INTERVAL '1 hour');
--
(1 row)

ALTER TABLE sensor_data_1hour SET (timescaledb.compress);
-- Add a compression policy and check the schedule interval
SELECT add_compression_policy('sensor_data_1hour','7 days'::INTERVAL) as compression_job \gset
SELECT schedule_interval from timescaledb_information.jobs where job_id = :compression_job;
schedule_interval
-------------------
@ 30 mins
(1 row)

----
-- Chunk interval 1 hour - compression schedule is set to a custom value
----
CREATE TABLE sensor_data_1hour_custom(
time timestamptz not null,
sensor_id integer not null
);
SELECT FROM create_hypertable('sensor_data_1hour_custom', 'time', chunk_time_interval => INTERVAL '1 hour');
--
(1 row)

ALTER TABLE sensor_data_1hour_custom SET (timescaledb.compress);
-- Add a compression policy and check the schedule interval
SELECT add_compression_policy('sensor_data_1hour_custom','7 days'::INTERVAL, schedule_interval => '7 days') as compression_job \gset
SELECT schedule_interval from timescaledb_information.jobs where job_id = :compression_job;
schedule_interval
-------------------
@ 7 days
(1 row)

1 change: 1 addition & 0 deletions tsl/test/sql/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ set(TEST_FILES
compression_bgw.sql
compression_conflicts.sql
compression_insert.sql
compression_policy.sql
compression_qualpushdown.sql
exp_cagg_monthly.sql
exp_cagg_next_gen.sql
Expand Down
79 changes: 79 additions & 0 deletions tsl/test/sql/compression_policy.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
-- This file and its contents are licensed under the Timescale License.
-- Please see the included NOTICE for copyright information and
-- LICENSE-TIMESCALE for a copy of the license.

----
-- Chunk interval 1 month - compression schedule interval should be 12 hours
----
CREATE TABLE sensor_data_1month(
time timestamptz not null,
sensor_id integer not null
);

SELECT FROM create_hypertable('sensor_data_1month', 'time', chunk_time_interval => INTERVAL '1 month');
ALTER TABLE sensor_data_1month SET (timescaledb.compress);

-- Add a compression policy and check the schedule interval
SELECT add_compression_policy('sensor_data_1month','7 days'::INTERVAL) as compression_job \gset
SELECT schedule_interval from timescaledb_information.jobs where job_id = :compression_job;

----
-- Chunk interval 1 week - compression schedule interval should be 12 hours
----
CREATE TABLE sensor_data_1week(
time timestamptz not null,
sensor_id integer not null
);

SELECT FROM create_hypertable('sensor_data_1week', 'time', chunk_time_interval => INTERVAL '1 week');
ALTER TABLE sensor_data_1week SET (timescaledb.compress);

-- Add a compression policy and check the schedule interval
SELECT add_compression_policy('sensor_data_1week','7 days'::INTERVAL) as compression_job \gset
SELECT schedule_interval from timescaledb_information.jobs where job_id = :compression_job;

----
-- Chunk interval 1 day - compression schedule interval should be 12 hours
----
CREATE TABLE sensor_data_1day(
time timestamptz not null,
sensor_id integer not null
);

SELECT FROM create_hypertable('sensor_data_1day', 'time', chunk_time_interval => INTERVAL '1 day');
ALTER TABLE sensor_data_1day SET (timescaledb.compress);

-- Add a compression policy and check the schedule interval
SELECT add_compression_policy('sensor_data_1day','7 days'::INTERVAL) as compression_job \gset
SELECT schedule_interval from timescaledb_information.jobs where job_id = :compression_job;


----
-- Chunk interval 1 hour - compression schedule interval should be 30 minutes
----
CREATE TABLE sensor_data_1hour(
time timestamptz not null,
sensor_id integer not null
);

SELECT FROM create_hypertable('sensor_data_1hour', 'time', chunk_time_interval => INTERVAL '1 hour');
ALTER TABLE sensor_data_1hour SET (timescaledb.compress);

-- Add a compression policy and check the schedule interval
SELECT add_compression_policy('sensor_data_1hour','7 days'::INTERVAL) as compression_job \gset
SELECT schedule_interval from timescaledb_information.jobs where job_id = :compression_job;

----
-- Chunk interval 1 hour - compression schedule is set to a custom value
----
CREATE TABLE sensor_data_1hour_custom(
time timestamptz not null,
sensor_id integer not null
);

SELECT FROM create_hypertable('sensor_data_1hour_custom', 'time', chunk_time_interval => INTERVAL '1 hour');
ALTER TABLE sensor_data_1hour_custom SET (timescaledb.compress);

-- Add a compression policy and check the schedule interval
SELECT add_compression_policy('sensor_data_1hour_custom','7 days'::INTERVAL, schedule_interval => '7 days') as compression_job \gset
SELECT schedule_interval from timescaledb_information.jobs where job_id = :compression_job;

0 comments on commit 0d7f95b

Please sign in to comment.