diff --git a/.unreleased/enhancement_6343 b/.unreleased/enhancement_6343 new file mode 100644 index 00000000000..361d0e6be3b --- /dev/null +++ b/.unreleased/enhancement_6343 @@ -0,0 +1,2 @@ +Implements: #6343 Enable segmentwise recompression in compression policy +Thanks @fetchezar for reporting the issue diff --git a/sql/policy_internal.sql b/sql/policy_internal.sql index 2ea47567b39..2190c1e11fc 100644 --- a/sql/policy_internal.sql +++ b/sql/policy_internal.sql @@ -56,6 +56,7 @@ DECLARE bit_frozen int := 4; bit_compressed_partial int := 8; creation_lag INTERVAL := NULL; + compressed_chunk_index REGCLASS; BEGIN -- procedures with SET clause cannot execute transaction @@ -123,26 +124,19 @@ BEGIN ) ) AND recompress_enabled IS TRUE THEN BEGIN - PERFORM @extschema@.decompress_chunk(chunk_rec.oid, if_compressed => true); + -- first check if there's an index. Might have to use a heuristic to determine if index usage would be efficient, + -- or if we'd better fall back to decompressing & recompressing entire chunk + IF _timescaledb_functions.get_compressed_chunk_index_for_recompression(chunk_rec.oid) IS NOT NULL THEN + PERFORM _timescaledb_functions.recompress_chunk_segmentwise(chunk_rec.oid); + ELSE + PERFORM @extschema@.decompress_chunk(chunk_rec.oid, if_compressed => true); + PERFORM @extschema@.compress_chunk(chunk_rec.oid); + END IF; EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS _message = MESSAGE_TEXT, _detail = PG_EXCEPTION_DETAIL; - RAISE WARNING 'decompressing chunk "%" failed when compression policy is executed', chunk_rec.oid::regclass::text - USING DETAIL = format('Message: (%s), Detail: (%s).', _message, _detail), - ERRCODE = sqlstate; - END; - -- SET LOCAL is only active until end of transaction. - -- While we could use SET at the start of the function we do not - -- want to bleed out search_path to caller, so we do SET LOCAL - -- again after COMMIT - BEGIN - PERFORM @extschema@.compress_chunk(chunk_rec.oid); - EXCEPTION WHEN OTHERS THEN - GET STACKED DIAGNOSTICS - _message = MESSAGE_TEXT, - _detail = PG_EXCEPTION_DETAIL; - RAISE WARNING 'compressing chunk "%" failed when compression policy is executed', chunk_rec.oid::regclass::text + RAISE WARNING 'recompressing chunk "%" failed when compression policy is executed', chunk_rec.oid::regclass::text USING DETAIL = format('Message: (%s), Detail: (%s).', _message, _detail), ERRCODE = sqlstate; END; diff --git a/tsl/test/expected/bgw_custom-15.out b/tsl/test/expected/bgw_custom-15.out index 6f308e7d361..8795a704a6f 100644 --- a/tsl/test/expected/bgw_custom-15.out +++ b/tsl/test/expected/bgw_custom-15.out @@ -572,7 +572,7 @@ FROM _timescaledb_config.bgw_job WHERE id = :job_id_5; --verify that job is dropped when cagg is dropped DROP MATERIALIZED VIEW conditions_summary_daily; -NOTICE: drop cascades to table _timescaledb_internal._hyper_3_10_chunk +NOTICE: drop cascades to table _timescaledb_internal._hyper_3_7_chunk SELECT id, proc_name, hypertable_id FROM _timescaledb_config.bgw_job WHERE id = :job_id_5; id | proc_name | hypertable_id diff --git a/tsl/test/expected/compression_bgw-13.out b/tsl/test/expected/compression_bgw-13.out index 74818f05ac4..004e0eb1cd3 100644 --- a/tsl/test/expected/compression_bgw-13.out +++ b/tsl/test/expected/compression_bgw-13.out @@ -474,15 +474,16 @@ ORDER BY 1, 2; Thu Apr 02 17:00:00 2020 PDT | 13 | 1 (4 rows) ---chunk status should be unordered for the previously compressed chunk +--chunk status should be partially compressed for the previously compressed chunk SELECT chunk_status, - chunk_name as "CHUNK_NAME" + chunk_name as "CHUNK_NAME", + compressed_chunk_name as "COMPRESSED_CHUNK_NAME" FROM compressed_chunk_info_view WHERE hypertable_name = 'test2' ORDER BY chunk_name; - chunk_status | CHUNK_NAME ---------------+-------------------- - 9 | _hyper_14_62_chunk - 0 | _hyper_14_64_chunk + chunk_status | CHUNK_NAME | COMPRESSED_CHUNK_NAME +--------------+--------------------+---------------------------- + 9 | _hyper_14_62_chunk | compress_hyper_15_63_chunk + 0 | _hyper_14_64_chunk | (2 rows) SELECT add_compression_policy AS job_id @@ -490,23 +491,27 @@ SELECT add_compression_policy AS job_id CALL run_job(:job_id); CALL run_job(:job_id); -- status should be compressed --- +-- compressed chunk name should not change for +-- the partially compressed chunk indicating +-- it was done segmentwise SELECT chunk_status, - chunk_name as "CHUNK_NAME" + chunk_name as "CHUNK_NAME", + compressed_chunk_name as "COMPRESSED_CHUNK_NAME" FROM compressed_chunk_info_view WHERE hypertable_name = 'test2' ORDER BY chunk_name; - chunk_status | CHUNK_NAME ---------------+-------------------- - 1 | _hyper_14_62_chunk - 1 | _hyper_14_64_chunk + chunk_status | CHUNK_NAME | COMPRESSED_CHUNK_NAME +--------------+--------------------+---------------------------- + 1 | _hyper_14_62_chunk | compress_hyper_15_63_chunk + 1 | _hyper_14_64_chunk | compress_hyper_15_65_chunk (2 rows) \set ON_ERROR_STOP 0 -- call recompress_chunk when status is not unordered CALL recompress_chunk(:'CHUNK_NAME'::regclass, true); -psql:include/recompress_basic.sql:110: NOTICE: nothing to recompress in chunk "_hyper_14_62_chunk" +psql:include/recompress_basic.sql:115: NOTICE: nothing to recompress in chunk "_hyper_14_62_chunk" -- This will succeed and compress the chunk for the test below. CALL recompress_chunk(:'CHUNK_NAME'::regclass, false); -psql:include/recompress_basic.sql:113: ERROR: nothing to recompress in chunk "_hyper_14_62_chunk" +psql:include/recompress_basic.sql:118: ERROR: nothing to recompress in chunk "_hyper_14_62_chunk" --now decompress it , then try and recompress SELECT decompress_chunk(:'CHUNK_NAME'::regclass); decompress_chunk @@ -515,7 +520,7 @@ SELECT decompress_chunk(:'CHUNK_NAME'::regclass); (1 row) CALL recompress_chunk(:'CHUNK_NAME'::regclass); -psql:include/recompress_basic.sql:117: ERROR: call compress_chunk instead of recompress_chunk +psql:include/recompress_basic.sql:122: ERROR: call compress_chunk instead of recompress_chunk \set ON_ERROR_STOP 1 -- test recompress policy CREATE TABLE metrics(time timestamptz NOT NULL); @@ -620,7 +625,7 @@ SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'met ---- nothing to do yet CALL run_job(:JOB_RECOMPRESS); -psql:include/recompress_basic.sql:189: NOTICE: no chunks for hypertable "public.metrics" that satisfy recompress chunk policy +psql:include/recompress_basic.sql:194: NOTICE: no chunks for hypertable "public.metrics" that satisfy recompress chunk policy ---- status should be 1 SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'metrics'; chunk_status diff --git a/tsl/test/expected/compression_bgw-14.out b/tsl/test/expected/compression_bgw-14.out index ceea22edd46..1b0947c2b5f 100644 --- a/tsl/test/expected/compression_bgw-14.out +++ b/tsl/test/expected/compression_bgw-14.out @@ -474,15 +474,16 @@ ORDER BY 1, 2; Thu Apr 02 17:00:00 2020 PDT | 13 | 1 (4 rows) ---chunk status should be unordered for the previously compressed chunk +--chunk status should be partially compressed for the previously compressed chunk SELECT chunk_status, - chunk_name as "CHUNK_NAME" + chunk_name as "CHUNK_NAME", + compressed_chunk_name as "COMPRESSED_CHUNK_NAME" FROM compressed_chunk_info_view WHERE hypertable_name = 'test2' ORDER BY chunk_name; - chunk_status | CHUNK_NAME ---------------+-------------------- - 9 | _hyper_14_62_chunk - 0 | _hyper_14_64_chunk + chunk_status | CHUNK_NAME | COMPRESSED_CHUNK_NAME +--------------+--------------------+---------------------------- + 9 | _hyper_14_62_chunk | compress_hyper_15_63_chunk + 0 | _hyper_14_64_chunk | (2 rows) SELECT add_compression_policy AS job_id @@ -490,23 +491,27 @@ SELECT add_compression_policy AS job_id CALL run_job(:job_id); CALL run_job(:job_id); -- status should be compressed --- +-- compressed chunk name should not change for +-- the partially compressed chunk indicating +-- it was done segmentwise SELECT chunk_status, - chunk_name as "CHUNK_NAME" + chunk_name as "CHUNK_NAME", + compressed_chunk_name as "COMPRESSED_CHUNK_NAME" FROM compressed_chunk_info_view WHERE hypertable_name = 'test2' ORDER BY chunk_name; - chunk_status | CHUNK_NAME ---------------+-------------------- - 1 | _hyper_14_62_chunk - 1 | _hyper_14_64_chunk + chunk_status | CHUNK_NAME | COMPRESSED_CHUNK_NAME +--------------+--------------------+---------------------------- + 1 | _hyper_14_62_chunk | compress_hyper_15_63_chunk + 1 | _hyper_14_64_chunk | compress_hyper_15_65_chunk (2 rows) \set ON_ERROR_STOP 0 -- call recompress_chunk when status is not unordered CALL recompress_chunk(:'CHUNK_NAME'::regclass, true); -psql:include/recompress_basic.sql:110: NOTICE: nothing to recompress in chunk "_hyper_14_62_chunk" +psql:include/recompress_basic.sql:115: NOTICE: nothing to recompress in chunk "_hyper_14_62_chunk" -- This will succeed and compress the chunk for the test below. CALL recompress_chunk(:'CHUNK_NAME'::regclass, false); -psql:include/recompress_basic.sql:113: ERROR: nothing to recompress in chunk "_hyper_14_62_chunk" +psql:include/recompress_basic.sql:118: ERROR: nothing to recompress in chunk "_hyper_14_62_chunk" --now decompress it , then try and recompress SELECT decompress_chunk(:'CHUNK_NAME'::regclass); decompress_chunk @@ -515,7 +520,7 @@ SELECT decompress_chunk(:'CHUNK_NAME'::regclass); (1 row) CALL recompress_chunk(:'CHUNK_NAME'::regclass); -psql:include/recompress_basic.sql:117: ERROR: call compress_chunk instead of recompress_chunk +psql:include/recompress_basic.sql:122: ERROR: call compress_chunk instead of recompress_chunk \set ON_ERROR_STOP 1 -- test recompress policy CREATE TABLE metrics(time timestamptz NOT NULL); @@ -620,7 +625,7 @@ SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'met ---- nothing to do yet CALL run_job(:JOB_RECOMPRESS); -psql:include/recompress_basic.sql:189: NOTICE: no chunks for hypertable "public.metrics" that satisfy recompress chunk policy +psql:include/recompress_basic.sql:194: NOTICE: no chunks for hypertable "public.metrics" that satisfy recompress chunk policy ---- status should be 1 SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'metrics'; chunk_status diff --git a/tsl/test/expected/compression_bgw-15.out b/tsl/test/expected/compression_bgw-15.out index ceea22edd46..1b0947c2b5f 100644 --- a/tsl/test/expected/compression_bgw-15.out +++ b/tsl/test/expected/compression_bgw-15.out @@ -474,15 +474,16 @@ ORDER BY 1, 2; Thu Apr 02 17:00:00 2020 PDT | 13 | 1 (4 rows) ---chunk status should be unordered for the previously compressed chunk +--chunk status should be partially compressed for the previously compressed chunk SELECT chunk_status, - chunk_name as "CHUNK_NAME" + chunk_name as "CHUNK_NAME", + compressed_chunk_name as "COMPRESSED_CHUNK_NAME" FROM compressed_chunk_info_view WHERE hypertable_name = 'test2' ORDER BY chunk_name; - chunk_status | CHUNK_NAME ---------------+-------------------- - 9 | _hyper_14_62_chunk - 0 | _hyper_14_64_chunk + chunk_status | CHUNK_NAME | COMPRESSED_CHUNK_NAME +--------------+--------------------+---------------------------- + 9 | _hyper_14_62_chunk | compress_hyper_15_63_chunk + 0 | _hyper_14_64_chunk | (2 rows) SELECT add_compression_policy AS job_id @@ -490,23 +491,27 @@ SELECT add_compression_policy AS job_id CALL run_job(:job_id); CALL run_job(:job_id); -- status should be compressed --- +-- compressed chunk name should not change for +-- the partially compressed chunk indicating +-- it was done segmentwise SELECT chunk_status, - chunk_name as "CHUNK_NAME" + chunk_name as "CHUNK_NAME", + compressed_chunk_name as "COMPRESSED_CHUNK_NAME" FROM compressed_chunk_info_view WHERE hypertable_name = 'test2' ORDER BY chunk_name; - chunk_status | CHUNK_NAME ---------------+-------------------- - 1 | _hyper_14_62_chunk - 1 | _hyper_14_64_chunk + chunk_status | CHUNK_NAME | COMPRESSED_CHUNK_NAME +--------------+--------------------+---------------------------- + 1 | _hyper_14_62_chunk | compress_hyper_15_63_chunk + 1 | _hyper_14_64_chunk | compress_hyper_15_65_chunk (2 rows) \set ON_ERROR_STOP 0 -- call recompress_chunk when status is not unordered CALL recompress_chunk(:'CHUNK_NAME'::regclass, true); -psql:include/recompress_basic.sql:110: NOTICE: nothing to recompress in chunk "_hyper_14_62_chunk" +psql:include/recompress_basic.sql:115: NOTICE: nothing to recompress in chunk "_hyper_14_62_chunk" -- This will succeed and compress the chunk for the test below. CALL recompress_chunk(:'CHUNK_NAME'::regclass, false); -psql:include/recompress_basic.sql:113: ERROR: nothing to recompress in chunk "_hyper_14_62_chunk" +psql:include/recompress_basic.sql:118: ERROR: nothing to recompress in chunk "_hyper_14_62_chunk" --now decompress it , then try and recompress SELECT decompress_chunk(:'CHUNK_NAME'::regclass); decompress_chunk @@ -515,7 +520,7 @@ SELECT decompress_chunk(:'CHUNK_NAME'::regclass); (1 row) CALL recompress_chunk(:'CHUNK_NAME'::regclass); -psql:include/recompress_basic.sql:117: ERROR: call compress_chunk instead of recompress_chunk +psql:include/recompress_basic.sql:122: ERROR: call compress_chunk instead of recompress_chunk \set ON_ERROR_STOP 1 -- test recompress policy CREATE TABLE metrics(time timestamptz NOT NULL); @@ -620,7 +625,7 @@ SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'met ---- nothing to do yet CALL run_job(:JOB_RECOMPRESS); -psql:include/recompress_basic.sql:189: NOTICE: no chunks for hypertable "public.metrics" that satisfy recompress chunk policy +psql:include/recompress_basic.sql:194: NOTICE: no chunks for hypertable "public.metrics" that satisfy recompress chunk policy ---- status should be 1 SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'metrics'; chunk_status diff --git a/tsl/test/expected/compression_bgw-16.out b/tsl/test/expected/compression_bgw-16.out index ceea22edd46..1b0947c2b5f 100644 --- a/tsl/test/expected/compression_bgw-16.out +++ b/tsl/test/expected/compression_bgw-16.out @@ -474,15 +474,16 @@ ORDER BY 1, 2; Thu Apr 02 17:00:00 2020 PDT | 13 | 1 (4 rows) ---chunk status should be unordered for the previously compressed chunk +--chunk status should be partially compressed for the previously compressed chunk SELECT chunk_status, - chunk_name as "CHUNK_NAME" + chunk_name as "CHUNK_NAME", + compressed_chunk_name as "COMPRESSED_CHUNK_NAME" FROM compressed_chunk_info_view WHERE hypertable_name = 'test2' ORDER BY chunk_name; - chunk_status | CHUNK_NAME ---------------+-------------------- - 9 | _hyper_14_62_chunk - 0 | _hyper_14_64_chunk + chunk_status | CHUNK_NAME | COMPRESSED_CHUNK_NAME +--------------+--------------------+---------------------------- + 9 | _hyper_14_62_chunk | compress_hyper_15_63_chunk + 0 | _hyper_14_64_chunk | (2 rows) SELECT add_compression_policy AS job_id @@ -490,23 +491,27 @@ SELECT add_compression_policy AS job_id CALL run_job(:job_id); CALL run_job(:job_id); -- status should be compressed --- +-- compressed chunk name should not change for +-- the partially compressed chunk indicating +-- it was done segmentwise SELECT chunk_status, - chunk_name as "CHUNK_NAME" + chunk_name as "CHUNK_NAME", + compressed_chunk_name as "COMPRESSED_CHUNK_NAME" FROM compressed_chunk_info_view WHERE hypertable_name = 'test2' ORDER BY chunk_name; - chunk_status | CHUNK_NAME ---------------+-------------------- - 1 | _hyper_14_62_chunk - 1 | _hyper_14_64_chunk + chunk_status | CHUNK_NAME | COMPRESSED_CHUNK_NAME +--------------+--------------------+---------------------------- + 1 | _hyper_14_62_chunk | compress_hyper_15_63_chunk + 1 | _hyper_14_64_chunk | compress_hyper_15_65_chunk (2 rows) \set ON_ERROR_STOP 0 -- call recompress_chunk when status is not unordered CALL recompress_chunk(:'CHUNK_NAME'::regclass, true); -psql:include/recompress_basic.sql:110: NOTICE: nothing to recompress in chunk "_hyper_14_62_chunk" +psql:include/recompress_basic.sql:115: NOTICE: nothing to recompress in chunk "_hyper_14_62_chunk" -- This will succeed and compress the chunk for the test below. CALL recompress_chunk(:'CHUNK_NAME'::regclass, false); -psql:include/recompress_basic.sql:113: ERROR: nothing to recompress in chunk "_hyper_14_62_chunk" +psql:include/recompress_basic.sql:118: ERROR: nothing to recompress in chunk "_hyper_14_62_chunk" --now decompress it , then try and recompress SELECT decompress_chunk(:'CHUNK_NAME'::regclass); decompress_chunk @@ -515,7 +520,7 @@ SELECT decompress_chunk(:'CHUNK_NAME'::regclass); (1 row) CALL recompress_chunk(:'CHUNK_NAME'::regclass); -psql:include/recompress_basic.sql:117: ERROR: call compress_chunk instead of recompress_chunk +psql:include/recompress_basic.sql:122: ERROR: call compress_chunk instead of recompress_chunk \set ON_ERROR_STOP 1 -- test recompress policy CREATE TABLE metrics(time timestamptz NOT NULL); @@ -620,7 +625,7 @@ SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'met ---- nothing to do yet CALL run_job(:JOB_RECOMPRESS); -psql:include/recompress_basic.sql:189: NOTICE: no chunks for hypertable "public.metrics" that satisfy recompress chunk policy +psql:include/recompress_basic.sql:194: NOTICE: no chunks for hypertable "public.metrics" that satisfy recompress chunk policy ---- status should be 1 SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'metrics'; chunk_status diff --git a/tsl/test/sql/include/recompress_basic.sql b/tsl/test/sql/include/recompress_basic.sql index eeba8f233fa..95033656c0c 100644 --- a/tsl/test/sql/include/recompress_basic.sql +++ b/tsl/test/sql/include/recompress_basic.sql @@ -88,9 +88,10 @@ FROM test2 GROUP BY time_bucket(INTERVAL '2 hour', timec), b ORDER BY 1, 2; ---chunk status should be unordered for the previously compressed chunk +--chunk status should be partially compressed for the previously compressed chunk SELECT chunk_status, - chunk_name as "CHUNK_NAME" + chunk_name as "CHUNK_NAME", + compressed_chunk_name as "COMPRESSED_CHUNK_NAME" FROM compressed_chunk_info_view WHERE hypertable_name = 'test2' ORDER BY chunk_name; @@ -100,8 +101,12 @@ CALL run_job(:job_id); CALL run_job(:job_id); -- status should be compressed --- +-- compressed chunk name should not change for +-- the partially compressed chunk indicating +-- it was done segmentwise SELECT chunk_status, - chunk_name as "CHUNK_NAME" + chunk_name as "CHUNK_NAME", + compressed_chunk_name as "COMPRESSED_CHUNK_NAME" FROM compressed_chunk_info_view WHERE hypertable_name = 'test2' ORDER BY chunk_name;