Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Schedule compression policy more often #6102

Merged
merged 1 commit into from
Sep 22, 2023
Merged

Schedule compression policy more often #6102

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion scripts/docker-build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ build_timescaledb()
cd /build/debug \
&& git config --global --add safe.directory /src \
&& cmake -DGENERATE_DOWNGRADE_SCRIPT=${GENERATE_DOWNGRADE_SCRIPT} -DENABLE_DEBUG_UTILS=off -DUSE_OPENSSL=${USE_OPENSSL} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} /src \
&& make && make install \
&& make -j $(nproc) && make install \
&& echo \"shared_preload_libraries = 'timescaledb'\" >> /usr/local/share/postgresql/postgresql.conf.sample \
&& echo \"timescaledb.telemetry_level=off\" >> /usr/local/share/postgresql/postgresql.conf.sample \
&& cd / && rm -rf /build"
Expand Down
22 changes: 17 additions & 5 deletions test/sql/updates/setup.policies.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,21 +20,33 @@ SELECT

DO LANGUAGE PLPGSQL $$
DECLARE
ts_version TEXT;
ts_major INT;
ts_minor INT;
BEGIN

SELECT extversion INTO ts_version FROM pg_extension WHERE extname = 'timescaledb';
WITH timescale_version AS (
SELECT string_to_array(extversion,'.') AS v
FROM pg_extension
WHERE extname = 'timescaledb'
)
SELECT v[1], v[2]
INTO ts_major, ts_minor
FROM timescale_version;

PERFORM add_reorder_policy('policy_test_timestamptz','policy_test_timestamptz_time_idx');

-- some policy API functions got renamed for 2.0 so we need to make
-- sure to use the right name for the version
IF ts_version < '2.0.0' THEN
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Changed to ts_major and ts_minor because semantic version compares with a text variable does not really work.

-- sure to use the right name for the version. The schedule_interval
-- parameter of add_compression_policy was introduced in 2.8.0
IF ts_major < 2 THEN
PERFORM add_drop_chunks_policy('policy_test_timestamptz','60d'::interval);
PERFORM add_compress_chunks_policy('policy_test_timestamptz','10d'::interval);
ELSE
ELSIF ts_major <= 2 AND ts_minor < 8 THEN
PERFORM add_retention_policy('policy_test_timestamptz','60d'::interval);
PERFORM add_compression_policy('policy_test_timestamptz','10d'::interval);
ELSE
PERFORM add_retention_policy('policy_test_timestamptz','60d'::interval);
PERFORM add_compression_policy('policy_test_timestamptz','10d'::interval, schedule_interval => '3 days 12:00:00'::interval);
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added to use the same schedule_interval across all timescaledb versions and make the upgrade tests pass.

END IF;
END
$$;
Expand Down
20 changes: 18 additions & 2 deletions tsl/src/bgw_policy/compression_api.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@
#define DEFAULT_RETRY_PERIOD \
DatumGetIntervalP(DirectFunctionCall3(interval_in, CStringGetDatum("1 hour"), InvalidOid, -1))

/* Default max schedule period for the compression policy is 12 hours. The actual schedule period
* will be chunk_interval/2 if the chunk_interval is < 12 hours. */
#define DEFAULT_MAX_SCHEDULE_PERIOD (int64)(12 * 3600 * 1000 * (int64) 1000)

static Hypertable *validate_compress_chunks_hypertable(Cache *hcache, Oid user_htoid,
bool *is_cagg);

Expand Down Expand Up @@ -248,8 +252,20 @@ policy_compression_add_internal(Oid user_rel_oid, Datum compress_after_datum,
if (dim && IS_TIMESTAMP_TYPE(ts_dimension_get_partition_type(dim)) &&
!user_defined_schedule_interval)
{
default_schedule_interval = DatumGetIntervalP(
ts_internal_to_interval_value(dim->fd.interval_length / 2, INTERVALOID));
int64 hypertable_schedule_interval = dim->fd.interval_length / 2;

/* On hypertables with a small chunk_time_interval, schedule the compression job more often
* than DEFAULT_MAX_SCHEDULE_PERIOD */
if (DEFAULT_MAX_SCHEDULE_PERIOD > hypertable_schedule_interval)
{
default_schedule_interval = DatumGetIntervalP(
ts_internal_to_interval_value(hypertable_schedule_interval, INTERVALOID));
}
else
{
default_schedule_interval = DatumGetIntervalP(
ts_internal_to_interval_value(DEFAULT_MAX_SCHEDULE_PERIOD, INTERVALOID));
}
}

/* insert a new job into jobs table */
Expand Down
8 changes: 4 additions & 4 deletions tsl/test/expected/bgw_custom.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -476,7 +476,7 @@ order by id;
select t.schedule_interval FROM alter_job(:job_id_4, next_start=> now() ) t;
schedule_interval
-------------------
@ 7 days 12 hours
@ 12 hours
(1 row)

SELECT wait_for_job_to_run(:job_id_4, 2);
Expand Down Expand Up @@ -917,9 +917,9 @@ ALTER TABLE sensor_data SET (timescaledb.compress, timescaledb.compress_orderby
SELECT add_compression_policy('sensor_data', INTERVAL '1' minute) AS compressjob_id \gset
-- set recompress to true
SELECT alter_job(id,config:=jsonb_set(config,'{recompress}', 'true')) FROM _timescaledb_config.bgw_job WHERE id = :compressjob_id;
alter_job
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
(1014,"@ 3 days 12 hours","@ 0",-1,"@ 1 hour",t,"{""recompress"": true, ""hypertable_id"": 4, ""compress_after"": ""@ 1 min""}",-infinity,_timescaledb_functions.policy_compression_check,f,,)
alter_job
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
(1014,"@ 12 hours","@ 0",-1,"@ 1 hour",t,"{""recompress"": true, ""hypertable_id"": 4, ""compress_after"": ""@ 1 min""}",-infinity,_timescaledb_functions.policy_compression_check,f,,)
(1 row)

-- create new chunks
Expand Down
8 changes: 4 additions & 4 deletions tsl/test/expected/bgw_db_scheduler_fixed.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1696,7 +1696,7 @@ select * from _timescaledb_config.bgw_job;
id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone
------+--------------------------------------------+-------------------+-------------+-------------+--------------+------------------------+-------------------------------------+------------+-----------+----------------+----------------------------------+---------------+--------------------------------------------------------------------------------+------------------------+-------------------------------------------+---------------
1026 | Retention Policy [1026] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_functions | policy_retention | super_user | t | t | Fri Dec 31 16:00:00 1999 PST | 1 | {"drop_after": "@ 2 years", "hypertable_id": 1} | _timescaledb_functions | policy_retention_check | Europe/Berlin
1027 | Compression Policy [1027] | @ 15 days | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | super_user | t | t | Fri Dec 31 16:00:00 1999 PST | 1 | {"hypertable_id": 1, "compress_after": "@ 1 year"} | _timescaledb_functions | policy_compression_check | Europe/Berlin
1027 | Compression Policy [1027] | @ 12 hours | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | super_user | t | t | Fri Dec 31 16:00:00 1999 PST | 1 | {"hypertable_id": 1, "compress_after": "@ 1 year"} | _timescaledb_functions | policy_compression_check | Europe/Berlin
1028 | Refresh Continuous Aggregate Policy [1028] | @ 21 days | @ 0 | -1 | @ 21 days | _timescaledb_functions | policy_refresh_continuous_aggregate | super_user | t | t | Fri Dec 31 16:00:00.005 1999 PST | 2 | {"end_offset": "@ 2 mons", "start_offset": "@ 1 year", "mat_hypertable_id": 2} | _timescaledb_functions | policy_refresh_continuous_aggregate_check | Europe/Athens
(3 rows)

Expand All @@ -1711,7 +1711,7 @@ SELECT * from _timescaledb_internal.bgw_job_stat;
job_id | last_start | last_finish | next_start | last_successful_finish | last_run_success | total_runs | total_duration | total_duration_failures | total_successes | total_failures | total_crashes | consecutive_failures | consecutive_crashes | flags
--------+----------------------------------+----------------------------------+----------------------------------+----------------------------------+------------------+------------+----------------+-------------------------+-----------------+----------------+---------------+----------------------+---------------------+-------
1026 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 16:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST | t | 1 | @ 0 | @ 0 | 1 | 0 | 0 | 0 | 0 | 0
1027 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Sat Jan 15 16:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST | t | 1 | @ 0 | @ 0 | 1 | 0 | 0 | 0 | 0 | 0
1027 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 04:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST | t | 1 | @ 0 | @ 0 | 1 | 0 | 0 | 0 | 0 | 0
1028 | Fri Dec 31 16:00:00.005 1999 PST | Fri Dec 31 16:00:00.005 1999 PST | Fri Jan 21 16:00:00.005 2000 PST | Fri Dec 31 16:00:00.005 1999 PST | t | 1 | @ 0 | @ 0 | 1 | 0 | 0 | 0 | 0 | 0
(3 rows)

Expand Down Expand Up @@ -1780,7 +1780,7 @@ select * from _timescaledb_config.bgw_job order by id;
id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone
------+--------------------------------------------+-------------------+-------------+-------------+--------------+------------------------+-------------------------------------+------------+-----------+----------------+----------------------------------+---------------+--------------------------------------------------------------------------------+------------------------+-------------------------------------------+---------------
1026 | Retention Policy [1026] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_functions | policy_retention | super_user | t | t | Fri Dec 31 16:00:00 1999 PST | 1 | {"drop_after": "@ 2 years", "hypertable_id": 1} | _timescaledb_functions | policy_retention_check | Europe/Berlin
1027 | Compression Policy [1027] | @ 15 days | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | super_user | t | t | Fri Dec 31 16:00:00 1999 PST | 1 | {"hypertable_id": 1, "compress_after": "@ 1 year"} | _timescaledb_functions | policy_compression_check | Europe/Berlin
1027 | Compression Policy [1027] | @ 12 hours | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | super_user | t | t | Fri Dec 31 16:00:00 1999 PST | 1 | {"hypertable_id": 1, "compress_after": "@ 1 year"} | _timescaledb_functions | policy_compression_check | Europe/Berlin
1028 | Refresh Continuous Aggregate Policy [1028] | @ 21 days | @ 0 | -1 | @ 21 days | _timescaledb_functions | policy_refresh_continuous_aggregate | super_user | t | t | Fri Dec 31 16:00:00.005 1999 PST | 2 | {"end_offset": "@ 2 mons", "start_offset": "@ 1 year", "mat_hypertable_id": 2} | _timescaledb_functions | policy_refresh_continuous_aggregate_check | Europe/Athens
1029 | User-Defined Action [1029] | @ 7 mons | @ 0 | -1 | @ 5 mins | public | job_test_fixed | super_user | t | t | Fri Dec 31 16:00:00.01 1999 PST | | | | |
1030 | User-Defined Action [1030] | @ 7 mons | @ 0 | -1 | @ 5 mins | public | job_test_fixed | super_user | t | t | Fri Dec 31 16:00:00.01 1999 PST | | | | | Europe/Athens
Expand All @@ -1798,7 +1798,7 @@ ORDER BY job_id;
job_id | last_start | last_finish | next_start | last_successful_finish
--------+------------------------------+------------------------------+------------------------------+------------------------------
1026 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 16:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST
1027 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Sat Jan 15 16:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST
1027 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 04:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST
1028 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Fri Jan 21 16:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST
1029 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Mon Jul 31 16:00:00 2000 PDT | Fri Dec 31 16:00:00 1999 PST
1030 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Mon Jul 31 16:00:00 2000 PDT | Fri Dec 31 16:00:00 1999 PST
Expand Down
10 changes: 5 additions & 5 deletions tsl/test/expected/bgw_policy.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -714,11 +714,11 @@ select create_hypertable('test_missing_schedint_integer', 'time', chunk_time_int
alter table test_missing_schedint_integer set (timescaledb.compress);
select add_compression_policy('test_missing_schedint_integer', BIGINT '600000') as compression_id_integer \gset
select * from _timescaledb_config.bgw_job where id in (:retenion_id_missing_schedint, :compression_id_missing_schedint, :compression_id_integer);
id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone
------+---------------------------+--------------------+-------------+-------------+--------------+------------------------+--------------------+---------------------+-----------+----------------+---------------+---------------+-----------------------------------------------------+------------------------+--------------------------+----------
1008 | Retention Policy [1008] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_functions | policy_retention | default_perm_user_2 | t | f | | 8 | {"drop_after": "@ 14 days", "hypertable_id": 8} | _timescaledb_functions | policy_retention_check |
1009 | Compression Policy [1009] | @ 15 days 12 hours | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | default_perm_user_2 | t | f | | 8 | {"hypertable_id": 8, "compress_after": "@ 60 days"} | _timescaledb_functions | policy_compression_check |
1010 | Compression Policy [1010] | @ 1 day | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | default_perm_user_2 | t | f | | 10 | {"hypertable_id": 10, "compress_after": 600000} | _timescaledb_functions | policy_compression_check |
id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone
------+---------------------------+-------------------+-------------+-------------+--------------+------------------------+--------------------+---------------------+-----------+----------------+---------------+---------------+-----------------------------------------------------+------------------------+--------------------------+----------
1008 | Retention Policy [1008] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_functions | policy_retention | default_perm_user_2 | t | f | | 8 | {"drop_after": "@ 14 days", "hypertable_id": 8} | _timescaledb_functions | policy_retention_check |
1009 | Compression Policy [1009] | @ 12 hours | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | default_perm_user_2 | t | f | | 8 | {"hypertable_id": 8, "compress_after": "@ 60 days"} | _timescaledb_functions | policy_compression_check |
1010 | Compression Policy [1010] | @ 1 day | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | default_perm_user_2 | t | f | | 10 | {"hypertable_id": 10, "compress_after": 600000} | _timescaledb_functions | policy_compression_check |
(3 rows)

-- test policy check functions with NULL args
Expand Down
Loading
Loading