Merge branch 'main' into pg16-rteperminfo

.github/workflows/apt-arm-packages.yaml

@@ -16,9 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # Debian images:  10 (buster), 11 (bullseye)
-        # Ubuntu images:  20.04 LTS (focal), 22.04 (jammy)
-        image: [ "debian:10-slim","debian:11-slim","ubuntu:focal", "ubuntu:jammy"]
+        image: [ "debian:10-slim","debian:11-slim","ubuntu:20.04", "ubuntu:22.04"]
         pg: [ 13, 14, 15 ]
 
     steps:

.github/workflows/apt-packages.yaml

@@ -20,9 +20,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # Debian images:  10 (buster), 11 (bullseye), 12 (bookworm)
-        # Ubuntu images:  18.04 LTS (bionic), 20.04 LTS (focal), 21.10 (impish), 22.04 (jammy), 22.10 (kinetic)
-        image: [ "debian:10-slim", "debian:11-slim", "debian:12-slim", "ubuntu:focal", "ubuntu:jammy", "ubuntu:kinetic"]
+        image: [ "debian:10-slim", "debian:11-slim", "debian:12-slim", "ubuntu:20.04", "ubuntu:22.04" ]
         pg: [ 13, 14, 15 ]
         license: [ "TSL", "Apache"]
         include:

.github/workflows/coverity.yaml

@@ -39,7 +39,7 @@ jobs:
 
     - name: Build TimescaleDB
       run: |
-        PATH="$GITHUB_WORKSPACE/coverity/bin:$PATH"
+        PATH="$GITHUB_WORKSPACE/coverity/bin:/usr/lib/postgresql/${{ matrix.pg }}/bin:$PATH"
         ./bootstrap -DCMAKE_BUILD_TYPE=Release
         cov-build --dir cov-int make -C build
 

.pull-review

@@ -42,16 +42,12 @@ reviewers:
   fabriziomello: {}
   konskov: {}
   erimatnor: {}
-  pmwkaa: {}
   nikkhils: {}
   akuzm: {}
-  RafiaSabih: {}
   gayyappan: {}
   jnidzwetzki: {}
   mahipv: {}
-  lkshminarayanan: {}
-  sb230132: {}
-  shhnwz: {}
+  antekresic: {}
 
 # list of users who will never be notified
 review_blacklist:

.unreleased/PR_6036

@@ -0,0 +1 @@
+Implements: #6036 Add API function for updating OSM chunk ranges

cmake/ScriptFiles.cmake

@@ -59,7 +59,8 @@ set(SOURCE_FILES
     policy_internal.sql
     cagg_utils.sql
     cagg_migrate.sql
-    job_error_log_retention.sql)
+    job_error_log_retention.sql
+    osm_api.sql)
 
 if(ENABLE_DEBUG_UTILS AND CMAKE_BUILD_TYPE MATCHES Debug)
   list(APPEND SOURCE_FILES debug_utils.sql)

sql/osm_api.sql

@@ -0,0 +1,14 @@
+-- This file and its contents are licensed under the Apache License 2.0.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-APACHE for a copy of the license.
+
+-- This function updates the dimension slice range stored in the catalog with the min and max
+-- values that the OSM chunk contains. Since there is only one OSM chunk per hypertable with
+-- only a time dimension, the hypertable is used to determine the corresponding slice
+CREATE OR REPLACE FUNCTION _timescaledb_functions.hypertable_osm_range_update(
+    hypertable REGCLASS,
+    range_start ANYELEMENT = NULL::bigint,
+    range_end ANYELEMENT = NULL,
+    empty BOOL = false
+) RETURNS BOOL AS '@MODULE_PATHNAME@',
+'ts_hypertable_osm_range_update' LANGUAGE C VOLATILE;

sql/pre_install/tables.sql

@@ -52,6 +52,7 @@ CREATE TABLE _timescaledb_catalog.hypertable (
   compression_state smallint NOT NULL DEFAULT 0,
   compressed_hypertable_id integer,
   replication_factor smallint NULL,
+  status int NOT NULL DEFAULT 0,
   -- table constraints
   CONSTRAINT hypertable_pkey PRIMARY KEY (id),
   CONSTRAINT hypertable_associated_schema_name_associated_table_prefix_key UNIQUE (associated_schema_name, associated_table_prefix),

sql/updates/latest-dev.sql

@@ -174,3 +174,154 @@ FROM _timescaledb_catalog.chunk_constraint cc
 INNER JOIN _timescaledb_catalog.chunk c ON c.id = cc.chunk_id AND c.osm_chunk
 WHERE cc.dimension_slice_id = ds.id AND ds.range_start <> 9223372036854775806;
 
+-- OSM support - table must be rebuilt to ensure consistent attribute numbers
+-- we cannot just ALTER TABLE .. ADD COLUMN
+ALTER TABLE _timescaledb_config.bgw_job
+    DROP CONSTRAINT bgw_job_hypertable_id_fkey;
+ALTER TABLE _timescaledb_catalog.chunk
+    DROP CONSTRAINT chunk_hypertable_id_fkey;
+ALTER TABLE _timescaledb_catalog.chunk_index
+    DROP CONSTRAINT chunk_index_hypertable_id_fkey;
+ALTER TABLE _timescaledb_catalog.continuous_agg
+    DROP CONSTRAINT continuous_agg_mat_hypertable_id_fkey,
+    DROP CONSTRAINT continuous_agg_raw_hypertable_id_fkey;
+ALTER TABLE _timescaledb_catalog.continuous_aggs_bucket_function
+    DROP CONSTRAINT continuous_aggs_bucket_function_mat_hypertable_id_fkey;
+ALTER TABLE _timescaledb_catalog.continuous_aggs_invalidation_threshold
+    DROP CONSTRAINT continuous_aggs_invalidation_threshold_hypertable_id_fkey;
+ALTER TABLE _timescaledb_catalog.dimension
+    DROP CONSTRAINT dimension_hypertable_id_fkey;
+ALTER TABLE _timescaledb_catalog.hypertable
+    DROP CONSTRAINT hypertable_compressed_hypertable_id_fkey;
+ALTER TABLE _timescaledb_catalog.hypertable_compression
+    DROP CONSTRAINT hypertable_compression_hypertable_id_fkey;
+ALTER TABLE _timescaledb_catalog.hypertable_data_node
+    DROP CONSTRAINT hypertable_data_node_hypertable_id_fkey;
+ALTER TABLE _timescaledb_catalog.tablespace
+    DROP CONSTRAINT tablespace_hypertable_id_fkey;
+
+DROP VIEW IF EXISTS timescaledb_information.hypertables;
+DROP VIEW IF EXISTS timescaledb_information.job_stats;
+DROP VIEW IF EXISTS timescaledb_information.jobs;
+DROP VIEW IF EXISTS timescaledb_information.continuous_aggregates;
+DROP VIEW IF EXISTS timescaledb_information.chunks;
+DROP VIEW IF EXISTS timescaledb_information.dimensions;
+DROP VIEW IF EXISTS timescaledb_information.compression_settings;
+DROP VIEW IF EXISTS _timescaledb_internal.hypertable_chunk_local_size;
+DROP VIEW IF EXISTS _timescaledb_internal.compressed_chunk_stats;
+DROP VIEW IF EXISTS timescaledb_experimental.chunk_replication_status;
+DROP VIEW IF EXISTS timescaledb_experimental.policies;
+
+-- recreate table
+CREATE TABLE _timescaledb_catalog.hypertable_tmp AS SELECT * FROM _timescaledb_catalog.hypertable;
+CREATE TABLE _timescaledb_catalog.tmp_hypertable_seq_value AS SELECT last_value, is_called FROM _timescaledb_catalog.hypertable_id_seq;
+
+ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.hypertable;
+ALTER EXTENSION timescaledb DROP SEQUENCE _timescaledb_catalog.hypertable_id_seq;
+
+SET timescaledb.restoring = on; -- must disable the hooks otherwise we can't do anything without the table _timescaledb_catalog.hypertable
+
+DROP TABLE _timescaledb_catalog.hypertable;
+
+CREATE SEQUENCE _timescaledb_catalog.hypertable_id_seq MINVALUE 1;
+SELECT setval('_timescaledb_catalog.hypertable_id_seq', last_value, is_called) FROM _timescaledb_catalog.tmp_hypertable_seq_value;
+DROP TABLE _timescaledb_catalog.tmp_hypertable_seq_value;
+
+CREATE TABLE _timescaledb_catalog.hypertable (
+    id INTEGER PRIMARY KEY NOT NULL DEFAULT nextval('_timescaledb_catalog.hypertable_id_seq'),
+    schema_name name NOT NULL,
+    table_name name NOT NULL,
+    associated_schema_name name NOT NULL,
+    associated_table_prefix name NOT NULL,
+    num_dimensions smallint NOT NULL,
+    chunk_sizing_func_schema name NOT NULL,
+    chunk_sizing_func_name name NOT NULL,
+    chunk_target_size bigint NOT NULL, -- size in bytes
+    compression_state smallint NOT NULL DEFAULT 0,
+    compressed_hypertable_id integer,
+    replication_factor smallint NULL,
+    status integer NOT NULL DEFAULT 0
+);
+
+SET timescaledb.restoring = off;
+
+INSERT INTO _timescaledb_catalog.hypertable (
+    id,
+    schema_name,
+    table_name,
+    associated_schema_name,
+    associated_table_prefix,
+    num_dimensions,
+    chunk_sizing_func_schema,
+    chunk_sizing_func_name,
+    chunk_target_size,
+    compression_state,
+    compressed_hypertable_id,
+    replication_factor
+)
+SELECT
+    id,
+    schema_name,
+    table_name,
+    associated_schema_name,
+    associated_table_prefix,
+    num_dimensions,
+    chunk_sizing_func_schema,
+    chunk_sizing_func_name,
+    chunk_target_size,
+    compression_state,
+    compressed_hypertable_id,
+    replication_factor
+FROM
+    _timescaledb_catalog.hypertable_tmp
+ORDER BY id;
+
+UPDATE _timescaledb_catalog.hypertable h
+SET status = 3
+WHERE EXISTS (
+  SELECT FROM _timescaledb_catalog.chunk c WHERE c.osm_chunk AND c.hypertable_id = h.id
+);
+
+ALTER SEQUENCE _timescaledb_catalog.hypertable_id_seq OWNED BY _timescaledb_catalog.hypertable.id;
+SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable', 'WHERE id >= 1');
+SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable_id_seq', '');
+
+GRANT SELECT ON _timescaledb_catalog.hypertable TO PUBLIC;
+GRANT SELECT ON _timescaledb_catalog.hypertable_id_seq TO PUBLIC;
+
+DROP TABLE _timescaledb_catalog.hypertable_tmp;
+-- now add any constraints
+ALTER TABLE _timescaledb_catalog.hypertable
+    ADD CONSTRAINT hypertable_associated_schema_name_associated_table_prefix_key UNIQUE (associated_schema_name, associated_table_prefix),
+    ADD CONSTRAINT hypertable_table_name_schema_name_key UNIQUE (table_name, schema_name),
+    ADD CONSTRAINT hypertable_schema_name_check CHECK (schema_name != '_timescaledb_catalog'),
+    ADD CONSTRAINT hypertable_dim_compress_check CHECK (num_dimensions > 0 OR compression_state = 2),
+    ADD CONSTRAINT hypertable_chunk_target_size_check CHECK (chunk_target_size >= 0),
+    ADD CONSTRAINT hypertable_compress_check CHECK ( (compression_state = 0 OR compression_state = 1 )  OR (compression_state = 2 AND compressed_hypertable_id IS NULL)),
+    ADD CONSTRAINT hypertable_replication_factor_check CHECK (replication_factor > 0 OR replication_factor = -1),
+    ADD CONSTRAINT hypertable_compressed_hypertable_id_fkey FOREIGN KEY (compressed_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id);
+
+GRANT SELECT ON TABLE _timescaledb_catalog.hypertable TO PUBLIC;
+
+-- 3. reestablish constraints on other tables
+ALTER TABLE _timescaledb_config.bgw_job
+    ADD CONSTRAINT bgw_job_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE;
+ALTER TABLE _timescaledb_catalog.chunk
+    ADD CONSTRAINT chunk_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id);
+ALTER TABLE _timescaledb_catalog.chunk_index
+    ADD CONSTRAINT chunk_index_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE;
+ALTER TABLE _timescaledb_catalog.continuous_agg
+    ADD CONSTRAINT continuous_agg_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE,
+    ADD CONSTRAINT continuous_agg_raw_hypertable_id_fkey FOREIGN KEY (raw_hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE;
+ALTER TABLE _timescaledb_catalog.continuous_aggs_bucket_function
+    ADD CONSTRAINT continuous_aggs_bucket_function_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE;
+ALTER TABLE _timescaledb_catalog.continuous_aggs_invalidation_threshold
+    ADD CONSTRAINT continuous_aggs_invalidation_threshold_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE;
+ALTER TABLE _timescaledb_catalog.dimension
+    ADD CONSTRAINT dimension_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE;
+ALTER TABLE _timescaledb_catalog.hypertable_compression
+    ADD CONSTRAINT hypertable_compression_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE;
+ALTER TABLE _timescaledb_catalog.hypertable_data_node
+    ADD CONSTRAINT hypertable_data_node_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id);
+ALTER TABLE _timescaledb_catalog.tablespace
+    ADD CONSTRAINT tablespace_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE;