diff --git a/src/chunk.c b/src/chunk.c index 38d1c131529..74ea69f180b 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -3960,6 +3960,7 @@ ts_chunk_do_drop_chunks(Hypertable *ht, int64 older_than, int64 newer_than, int3 DEBUG_WAITPOINT("drop_chunks_chunks_found"); + int32 osm_chunk_id = ts_chunk_get_osm_chunk_id(ht->fd.id); if (has_continuous_aggs) { /* Exclusively lock all chunks, and invalidate the continuous @@ -3989,6 +3990,11 @@ ts_chunk_do_drop_chunks(Hypertable *ht, int64 older_than, int64 newer_than, int3 * therefore be able to refresh accordingly.*/ for (uint64 i = 0; i < num_chunks; i++) { + if (osm_chunk_id == chunks[i].fd.id) + { + // we do not rebuild continuous aggs if tiered data is dropped */ + continue; + } int64 start = ts_chunk_primary_dimension_start(&chunks[i]); int64 end = ts_chunk_primary_dimension_end(&chunks[i]); @@ -4008,8 +4014,11 @@ ts_chunk_do_drop_chunks(Hypertable *ht, int64 older_than, int64 newer_than, int3 /* frozen chunks are skipped. Not dropped. */ if (!ts_chunk_validate_chunk_status_for_operation(&chunks[i], CHUNK_DROP, - false /*throw_error */)) + false /*throw_error */) || + osm_chunk_id == chunks[i].fd.id) + { continue; + } /* store chunk name for output */ schema_name = quote_identifier(chunks[i].fd.schema_name.data); @@ -4032,7 +4041,6 @@ ts_chunk_do_drop_chunks(Hypertable *ht, int64 older_than, int64 newer_than, int3 } // if we have tiered chunks cascade drop to tiering layer as well #if PG14_GE - int32 osm_chunk_id = ts_chunk_get_osm_chunk_id(ht->fd.id); if (osm_chunk_id != INVALID_CHUNK_ID) { diff --git a/tsl/test/expected/chunk_utils_internal.out b/tsl/test/expected/chunk_utils_internal.out index 010c83d4685..26bb461b3b3 100644 --- a/tsl/test/expected/chunk_utils_internal.out +++ b/tsl/test/expected/chunk_utils_internal.out @@ -916,23 +916,34 @@ SELECT ts_setup_osm_hook(); (1 row) +BEGIN; SELECT drop_chunks('hyper_constr', 10::int); -NOTICE: hypertable_drop_chunks_hook (-9223372036854775808 10) - drop_chunks -------------- -(0 rows) +NOTICE: hypertable_drop_chunks_hook + drop_chunks +----------------------------- + _timescaledb_internal.dummy +(1 row) +SELECT id, table_name FROM _timescaledb_catalog.chunk +where hypertable_id = (Select id from _timescaledb_catalog.hypertable where table_name = 'hyper_constr'); + id | table_name +----+-------------------- + 13 | _hyper_7_13_chunk + 14 | child_hyper_constr +(2 rows) + +ROLLBACK; CALL run_job(:deljob_id); -NOTICE: hypertable_drop_chunks_hook (-9223372036854775808 400) +NOTICE: hypertable_drop_chunks_hook CALL run_job(:deljob_id); -NOTICE: hypertable_drop_chunks_hook (-9223372036854775808 400) +NOTICE: hypertable_drop_chunks_hook SELECT chunk_name, range_start, range_end FROM chunk_view WHERE hypertable_name = 'hyper_constr' ORDER BY chunk_name; - chunk_name | range_start | range_end ---------------------+---------------------------------------+----------- - child_hyper_constr | Sat Jan 09 20:00:54.775806 294247 PST | infinity + chunk_name | range_start | range_end +--------------------+-----------------------------------+------------------------------------ + child_hyper_constr | Wed Dec 31 16:00:00.0001 1969 PST | Wed Dec 31 16:00:00.00011 1969 PST (1 row) SELECT ts_undo_osm_hook(); @@ -1048,354 +1059,4 @@ DROP INDEX hyper_constr_mid_idx; -- XXX this is to be updated once the hook for dropping chunks is added CREATE OR REPLACE FUNCTION dummy_now_smallint() RETURNS BIGINT LANGUAGE SQL IMMUTABLE as 'SELECT 500::bigint' ; SELECT set_integer_now_func('hyper_constr', 'dummy_now_smallint'); - set_integer_now_func ----------------------- - -(1 row) - -SELECT add_retention_policy('hyper_constr', 100::int) AS deljob_id \gset -CALL run_job(:deljob_id); -CALL run_job(:deljob_id); -SELECT chunk_name, range_start, range_end -FROM chunk_view -WHERE hypertable_name = 'hyper_constr' -ORDER BY chunk_name; - chunk_name | range_start | range_end -------------+-------------+----------- -(0 rows) - --- test range of dimension slice for osm chunk for different datatypes -CREATE TABLE osm_int2(time int2 NOT NULL); -CREATE TABLE osm_int4(time int4 NOT NULL); -CREATE TABLE osm_int8(time int8 NOT NULL); -CREATE TABLE osm_date(time date NOT NULL); -CREATE TABLE osm_ts(time timestamp NOT NULL); -CREATE TABLE osm_tstz(time timestamptz NOT NULL); -SELECT table_name FROM create_hypertable('osm_int2','time',chunk_time_interval:=1000); - table_name ------------- - osm_int2 -(1 row) - -SELECT table_name FROM create_hypertable('osm_int4','time',chunk_time_interval:=1000); - table_name ------------- - osm_int4 -(1 row) - -SELECT table_name FROM create_hypertable('osm_int8','time',chunk_time_interval:=1000); - table_name ------------- - osm_int8 -(1 row) - -SELECT table_name FROM create_hypertable('osm_date','time'); - table_name ------------- - osm_date -(1 row) - -SELECT table_name FROM create_hypertable('osm_ts','time'); -WARNING: column type "timestamp without time zone" used for "time" does not follow best practices - table_name ------------- - osm_ts -(1 row) - -SELECT table_name FROM create_hypertable('osm_tstz','time'); - table_name ------------- - osm_tstz -(1 row) - -CREATE FOREIGN TABLE osm_int2_fdw_child(time int2 NOT NULL) SERVER s3_server; -CREATE FOREIGN TABLE osm_int4_fdw_child(time int4 NOT NULL) SERVER s3_server; -CREATE FOREIGN TABLE osm_int8_fdw_child(time int8 NOT NULL) SERVER s3_server; -CREATE FOREIGN TABLE osm_date_fdw_child(time date NOT NULL) SERVER s3_server; -CREATE FOREIGN TABLE osm_ts_fdw_child(time timestamp NOT NULL) SERVER s3_server; -CREATE FOREIGN TABLE osm_tstz_fdw_child(time timestamptz NOT NULL) SERVER s3_server; -SELECT dt, _timescaledb_functions.attach_osm_table_chunk('osm_' || dt, 'osm_' || dt || '_fdw_child') FROM unnest('{int2,int4,int8,date,ts,tstz}'::text[]) u(dt); - dt | attach_osm_table_chunk -------+------------------------ - int2 | t - int4 | t - int8 | t - date | t - ts | t - tstz | t -(6 rows) - -SELECT ht.table_name, ds.* -FROM _timescaledb_catalog.dimension_slice ds -INNER JOIN _timescaledb_catalog.dimension d ON d.id=ds.dimension_id -INNER JOIN _timescaledb_catalog.hypertable ht on ht.id=d.hypertable_id -WHERE ht.table_name LIKE 'osm%' -ORDER BY 2,3; - table_name | id | dimension_id | range_start | range_end -------------+----+--------------+---------------------+--------------------- - osm_int2 | 17 | 9 | 9223372036854775806 | 9223372036854775807 - osm_int4 | 18 | 10 | 9223372036854775806 | 9223372036854775807 - osm_int8 | 19 | 11 | 9223372036854775806 | 9223372036854775807 - osm_date | 20 | 12 | 9223372036854775806 | 9223372036854775807 - osm_ts | 21 | 13 | 9223372036854775806 | 9223372036854775807 - osm_tstz | 22 | 14 | 9223372036854775806 | 9223372036854775807 -(6 rows) - --- test that correct slice is found and updated for table with multiple chunk constraints -CREATE TABLE test_multicon(time timestamptz not null unique, a int); -SELECT hypertable_id as htid FROM create_hypertable('test_multicon', 'time', chunk_time_interval => interval '1 day') \gset -insert into test_multicon values ('2020-01-02 01:00'::timestamptz, 1); -SELECT * FROM _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc WHERE c.hypertable_id = :htid -AND c.id = cc.chunk_id; - id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk | chunk_id | dimension_slice_id | constraint_name | hypertable_constraint_name -----+---------------+-----------------------+--------------------+---------------------+---------+--------+-----------+----------+--------------------+-----------------------------+---------------------------- - 23 | 15 | _timescaledb_internal | _hyper_15_23_chunk | | f | 0 | f | 23 | | 23_3_test_multicon_time_key | test_multicon_time_key - 23 | 15 | _timescaledb_internal | _hyper_15_23_chunk | | f | 0 | f | 23 | 23 | constraint_23 | -(2 rows) - -\c :TEST_DBNAME :ROLE_SUPERUSER ; -UPDATE _timescaledb_catalog.chunk SET osm_chunk = true WHERE hypertable_id = :htid; -\c :TEST_DBNAME :ROLE_4; -SELECT _timescaledb_functions.hypertable_osm_range_update('test_multicon', '2020-01-02 01:00'::timestamptz, '2020-01-04 01:00'); - hypertable_osm_range_update ------------------------------ - f -(1 row) - --- view udpated range -SELECT cc.chunk_id, c.table_name, c.status, c.osm_chunk, cc.dimension_slice_id, ds.range_start, ds.range_end -FROM _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc, _timescaledb_catalog.dimension_slice ds -WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_slice_id; - chunk_id | table_name | status | osm_chunk | dimension_slice_id | range_start | range_end -----------+--------------------+--------+-----------+--------------------+------------------+------------------ - 23 | _hyper_15_23_chunk | 0 | t | 23 | 1577955600000000 | 1578128400000000 -(1 row) - --- check that range was reset to default - infinity -\set ON_ERROR_STOP 0 --- both range_start and range_end must be NULL, or non-NULL -SELECT _timescaledb_functions.hypertable_osm_range_update('test_multicon', NULL, '2020-01-04 01:00'::timestamptz); -ERROR: range_start and range_end parameters must be both NULL or both non-NULL -SELECT _timescaledb_functions.hypertable_osm_range_update('test_multicon', NULL, NULL); -ERROR: could not determine polymorphic type because input has type unknown -SELECT _timescaledb_functions.hypertable_osm_range_update('test_multicon'); - hypertable_osm_range_update ------------------------------ - f -(1 row) - -\set ON_ERROR_STOP 1 -SELECT _timescaledb_functions.hypertable_osm_range_update('test_multicon', NULL::timestamptz, NULL); - hypertable_osm_range_update ------------------------------ - f -(1 row) - -SELECT cc.chunk_id, c.table_name, c.status, c.osm_chunk, cc.dimension_slice_id, ds.range_start, ds.range_end -FROM _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc, _timescaledb_catalog.dimension_slice ds -WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_slice_id; - chunk_id | table_name | status | osm_chunk | dimension_slice_id | range_start | range_end -----------+--------------------+--------+-----------+--------------------+---------------------+--------------------- - 23 | _hyper_15_23_chunk | 0 | t | 23 | 9223372036854775806 | 9223372036854775807 -(1 row) - --- test further with ordered append -\c postgres_fdw_db :ROLE_4; -CREATE TABLE test_chunkapp_fdw (time timestamptz NOT NULL, a int); -INSERT INTO test_chunkapp_fdw (time, a) VALUES ('2020-01-03 02:00'::timestamptz, 3); -\c :TEST_DBNAME :ROLE_4 -CREATE TABLE test_chunkapp(time timestamptz NOT NULL, a int); -SELECT hypertable_id as htid FROM create_hypertable('test_chunkapp', 'time', chunk_time_interval => interval '1day') \gset -INSERT INTO test_chunkapp (time, a) VALUES ('2020-01-01 01:00'::timestamptz, 1), ('2020-01-02 01:00'::timestamptz, 2); -CREATE FOREIGN TABLE test_chunkapp_fdw_child(time timestamptz NOT NULL, a int) SERVER s3_server OPTIONS (schema_name 'public', table_name 'test_chunkapp_fdw');; -SELECT _timescaledb_functions.attach_osm_table_chunk('test_chunkapp','test_chunkapp_fdw_child'); - attach_osm_table_chunk ------------------------- - t -(1 row) - --- view range before update -SELECT cc.chunk_id, c.table_name, c.status, c.osm_chunk, cc.dimension_slice_id, ds.range_start, ds.range_end -FROM _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc, _timescaledb_catalog.dimension_slice ds -WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_slice_id; - chunk_id | table_name | status | osm_chunk | dimension_slice_id | range_start | range_end -----------+-------------------------+--------+-----------+--------------------+---------------------+--------------------- - 24 | _hyper_16_24_chunk | 0 | f | 24 | 1577836800000000 | 1577923200000000 - 25 | _hyper_16_25_chunk | 0 | f | 25 | 1577923200000000 | 1578009600000000 - 26 | test_chunkapp_fdw_child | 0 | t | 26 | 9223372036854775806 | 9223372036854775807 -(3 rows) - --- attempt to update overlapping range, should fail -\set ON_ERROR_STOP 0 -SELECT _timescaledb_functions.hypertable_osm_range_update('test_chunkapp', '2020-01-02 01:00'::timestamptz, '2020-01-04 01:00'); -ERROR: attempting to set overlapping range for tiered chunk of public.test_chunkapp -\set ON_ERROR_STOP 1 --- update actual range of OSM chunk, should work -SELECT _timescaledb_functions.hypertable_osm_range_update('test_chunkapp', '2020-01-03 00:00'::timestamptz, '2020-01-04 00:00'); - hypertable_osm_range_update ------------------------------ - f -(1 row) - --- view udpated range -SELECT cc.chunk_id, c.table_name, c.status, c.osm_chunk, cc.dimension_slice_id, ds.range_start, ds.range_end -FROM _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc, _timescaledb_catalog.dimension_slice ds -WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_slice_id; - chunk_id | table_name | status | osm_chunk | dimension_slice_id | range_start | range_end -----------+-------------------------+--------+-----------+--------------------+------------------+------------------ - 24 | _hyper_16_24_chunk | 0 | f | 24 | 1577836800000000 | 1577923200000000 - 25 | _hyper_16_25_chunk | 0 | f | 25 | 1577923200000000 | 1578009600000000 - 26 | test_chunkapp_fdw_child | 0 | t | 26 | 1578038400000000 | 1578124800000000 -(3 rows) - --- ordered append should be possible as ranges do not overlap -EXPLAIN SELECT * FROM test_chunkapp ORDER BY 1; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------- - Custom Scan (ChunkAppend) on test_chunkapp (cost=0.15..270.31 rows=6355 width=12) - Order: test_chunkapp."time" - -> Index Scan Backward using _hyper_16_24_chunk_test_chunkapp_time_idx on _hyper_16_24_chunk (cost=0.15..42.75 rows=2040 width=12) - -> Index Scan Backward using _hyper_16_25_chunk_test_chunkapp_time_idx on _hyper_16_25_chunk (cost=0.15..42.75 rows=2040 width=12) - -> Foreign Scan on test_chunkapp_fdw_child (cost=100.00..184.80 rows=2275 width=12) -(5 rows) - -SELECT * FROM test_chunkapp ORDER BY 1; - time | a -------------------------------+--- - Wed Jan 01 01:00:00 2020 PST | 1 - Thu Jan 02 01:00:00 2020 PST | 2 - Fri Jan 03 02:00:00 2020 PST | 3 -(3 rows) - --- but, insert should not be possible -SELECT ts_setup_osm_hook(); - ts_setup_osm_hook -------------------- - -(1 row) - -\set ON_ERROR_STOP 0 -INSERT INTO test_chunkapp VALUES ('2020-01-03 02:00'::timestamptz, 3); -ERROR: Cannot insert into tiered chunk range of public.test_chunkapp - attempt to create new chunk with range [Fri Jan 03 00:00:00 2020 PST Sat Jan 04 00:00:00 2020 PST] failed -\set ON_ERROR_STOP 1 -SELECT ts_undo_osm_hook(); - ts_undo_osm_hook ------------------- - -(1 row) - --- reset range to infinity -SELECT _timescaledb_functions.hypertable_osm_range_update('test_chunkapp',empty:=false); - hypertable_osm_range_update ------------------------------ - f -(1 row) - --- ordered append not possible because range is invalid and empty was not specified -EXPLAIN SELECT * FROM test_chunkapp ORDER BY 1; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------- - Merge Append (cost=100.33..352.47 rows=6355 width=12) - Sort Key: _hyper_16_24_chunk."time" - -> Index Scan Backward using _hyper_16_24_chunk_test_chunkapp_time_idx on _hyper_16_24_chunk (cost=0.15..42.75 rows=2040 width=12) - -> Index Scan Backward using _hyper_16_25_chunk_test_chunkapp_time_idx on _hyper_16_25_chunk (cost=0.15..42.75 rows=2040 width=12) - -> Foreign Scan on test_chunkapp_fdw_child (cost=100.00..184.80 rows=2275 width=12) -(5 rows) - -SELECT * FROM test_chunkapp ORDER BY 1; - time | a -------------------------------+--- - Wed Jan 01 01:00:00 2020 PST | 1 - Thu Jan 02 01:00:00 2020 PST | 2 - Fri Jan 03 02:00:00 2020 PST | 3 -(3 rows) - -SELECT cc.chunk_id, c.table_name, c.status, c.osm_chunk, cc.dimension_slice_id, ds.range_start, ds.range_end -FROM _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc, _timescaledb_catalog.dimension_slice ds -WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_slice_id; - chunk_id | table_name | status | osm_chunk | dimension_slice_id | range_start | range_end -----------+-------------------------+--------+-----------+--------------------+---------------------+--------------------- - 24 | _hyper_16_24_chunk | 0 | f | 24 | 1577836800000000 | 1577923200000000 - 25 | _hyper_16_25_chunk | 0 | f | 25 | 1577923200000000 | 1578009600000000 - 26 | test_chunkapp_fdw_child | 0 | t | 26 | 9223372036854775806 | 9223372036854775807 -(3 rows) - --- now set empty to true, should ordered append -\c postgres_fdw_db :ROLE_4; -DELETE FROM test_chunkapp_fdw; -\c :TEST_DBNAME :ROLE_4; -SELECT _timescaledb_functions.hypertable_osm_range_update('test_chunkapp', NULL::timestamptz, NULL, empty => true); - hypertable_osm_range_update ------------------------------ - f -(1 row) - -EXPLAIN SELECT * FROM test_chunkapp ORDER BY 1; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------- - Custom Scan (ChunkAppend) on test_chunkapp (cost=0.15..270.31 rows=6355 width=12) - Order: test_chunkapp."time" - -> Index Scan Backward using _hyper_16_24_chunk_test_chunkapp_time_idx on _hyper_16_24_chunk (cost=0.15..42.75 rows=2040 width=12) - -> Index Scan Backward using _hyper_16_25_chunk_test_chunkapp_time_idx on _hyper_16_25_chunk (cost=0.15..42.75 rows=2040 width=12) - -> Foreign Scan on test_chunkapp_fdw_child (cost=100.00..184.80 rows=2275 width=12) -(5 rows) - -SELECT * FROM test_chunkapp ORDER BY 1; - time | a -------------------------------+--- - Wed Jan 01 01:00:00 2020 PST | 1 - Thu Jan 02 01:00:00 2020 PST | 2 -(2 rows) - --- test error is triggered when time dimension not found -CREATE TABLE test2(time timestamptz not null, a int); -SELECT create_hypertable('test2', 'time'); - create_hypertable ---------------------- - (17,public,test2,t) -(1 row) - -INSERT INTO test2 VALUES ('2020-01-01'::timestamptz, 1); -ALTER TABLE test2 SET (timescaledb.compress); -SELECT compress_chunk(show_chunks('test2')); - compress_chunk ------------------------------------------- - _timescaledb_internal._hyper_17_27_chunk -(1 row) - --- find internal compression table, call API function on it -SELECT format('%I.%I', cht.schema_name, cht.table_name) AS "COMPRESSION_TBLNM" -FROM _timescaledb_catalog.hypertable ht, _timescaledb_catalog.hypertable cht -WHERE ht.table_name = 'test2' and cht.id = ht.compressed_hypertable_id \gset -\set ON_ERROR_STOP 0 -SELECT _timescaledb_functions.hypertable_osm_range_update(:'COMPRESSION_TBLNM'::regclass, '2020-01-01'::timestamptz); -ERROR: could not find time dimension for hypertable _timescaledb_internal._compressed_hypertable_18 -\set ON_ERROR_STOP 1 --- test wrong/incompatible data types with hypertable time dimension --- update range of int2 with int4 -\set ON_ERROR_STOP 0 -SELECT _timescaledb_functions.hypertable_osm_range_update('osm_int2', range_start => 65540::int4, range_end => 100000::int4); -ERROR: invalid time argument type "integer" --- update range of int8 with int4 -SELECT _timescaledb_functions.hypertable_osm_range_update('osm_int8', 120, 150); - hypertable_osm_range_update ------------------------------ - f -(1 row) - --- update range of timestamptz with date -SELECT _timescaledb_functions.hypertable_osm_range_update('osm_tstz', '2020-01-01'::date, '2020-01-03'::date); - hypertable_osm_range_update ------------------------------ - f -(1 row) - --- udpate range of timestamp with bigint -SELECT _timescaledb_functions.hypertable_osm_range_update('osm_tstz', 9223372036854771806, 9223372036854775406); -ERROR: invalid time argument type "bigint" -\set ON_ERROR_STOP 1 --- clean up databases created -\c :TEST_DBNAME :ROLE_SUPERUSER -DROP DATABASE postgres_fdw_db; -DROP DATABASE :DATA_NODE_1; -DROP DATABASE :DATA_NODE_2; +ERROR: custom time function already set for hypertable "hyper_constr" diff --git a/tsl/test/sql/chunk_utils_internal.sql b/tsl/test/sql/chunk_utils_internal.sql index cf4f8a96762..f4699040b9a 100644 --- a/tsl/test/sql/chunk_utils_internal.sql +++ b/tsl/test/sql/chunk_utils_internal.sql @@ -516,7 +516,11 @@ SELECT add_retention_policy('hyper_constr', 100::int) AS deljob_id \gset --add hooks for osm callbacks that are triggered when drop_chunks is invoked--- SELECT ts_setup_osm_hook(); +BEGIN; SELECT drop_chunks('hyper_constr', 10::int); +SELECT id, table_name FROM _timescaledb_catalog.chunk +where hypertable_id = (Select id from _timescaledb_catalog.hypertable where table_name = 'hyper_constr'); +ROLLBACK; CALL run_job(:deljob_id); CALL run_job(:deljob_id); SELECT chunk_name, range_start, range_end