Refactor compression setting storage

This patch drops the catalog table _timescaledb_catalog.hypertable_compression
and stores those settings in _timescaledb_catalog.compression_settings instead.
The storage format is changed and the new table will have 1 entry per relation
instead of 1 entry per column and has no dependancy on hypertables.
All other aspects of compression will remain the same. This is refactoring is
to enable per chunk compression settings in a follow-up patch.

sql/pre_install/tables.sql

@@ -466,23 +466,19 @@ CREATE TABLE _timescaledb_catalog.compression_algorithm (
   CONSTRAINT compression_algorithm_pkey PRIMARY KEY (id)
 );
 
-CREATE TABLE _timescaledb_catalog.hypertable_compression (
-  hypertable_id integer NOT NULL,
-  attname name NOT NULL,
-  compression_algorithm_id smallint,
-  segmentby_column_index smallint,
-  orderby_column_index smallint,
-  orderby_asc boolean,
-  orderby_nullsfirst boolean,
-  -- table constraints
-  CONSTRAINT hypertable_compression_pkey PRIMARY KEY (hypertable_id, attname),
-  CONSTRAINT hypertable_compression_hypertable_id_orderby_column_index_key UNIQUE (hypertable_id, orderby_column_index),
-  CONSTRAINT hypertable_compression_hypertable_id_segmentby_column_index_key UNIQUE (hypertable_id, segmentby_column_index),
-  CONSTRAINT hypertable_compression_compression_algorithm_id_fkey FOREIGN KEY (compression_algorithm_id) REFERENCES _timescaledb_catalog.compression_algorithm (id),
-  CONSTRAINT hypertable_compression_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE
+CREATE TABLE _timescaledb_catalog.compression_settings (
+	relid regclass NOT NULL,
+  segmentby text[],
+  orderby text[],
+  orderby_desc bool[],
+  orderby_nullsfirst bool[],
+  CONSTRAINT compression_settings_pkey PRIMARY KEY (relid),
+  CONSTRAINT compression_settings_check_segmentby CHECK (array_ndims(segmentby) = 1),
+  CONSTRAINT compression_settings_check_orderby_null CHECK ((orderby IS NULL AND orderby_desc IS NULL AND orderby_nullsfirst IS NULL) OR (orderby IS NOT NULL AND orderby_desc IS NOT NULL AND orderby_nullsfirst IS NOT NULL)),
+  CONSTRAINT compression_settings_check_orderby_cardinality CHECK (array_ndims(orderby) = 1 AND array_ndims(orderby_desc) = 1 AND array_ndims(orderby_nullsfirst) = 1 AND cardinality(orderby) = cardinality(orderby_desc) AND cardinality(orderby) = cardinality(orderby_nullsfirst))
 );
 
-SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable_compression', '');
+SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.compression_settings', '');
 
 CREATE TABLE _timescaledb_catalog.compression_chunk_size (
   chunk_id integer NOT NULL,

sql/updates/latest-dev.sql

@@ -43,3 +43,34 @@ DROP FUNCTION IF EXISTS @[email protected]_data_node;
 DROP PROCEDURE IF EXISTS @[email protected]_exec;
 DROP FUNCTION IF EXISTS @[email protected]_distributed_restore_point;
 DROP FUNCTION IF EXISTS @[email protected]_replication_factor;
+
+CREATE TABLE _timescaledb_catalog.compression_settings (
+	relid regclass NOT NULL,
+  segmentby text[],
+  orderby text[],
+  orderby_desc bool[],
+  orderby_nullsfirst bool[],
+  CONSTRAINT compression_settings_pkey PRIMARY KEY (relid),
+  CONSTRAINT compression_settings_check_segmentby CHECK (array_ndims(segmentby) = 1),
+  CONSTRAINT compression_settings_check_orderby_null CHECK ( (orderby IS NULL AND orderby_desc IS NULL AND orderby_nullsfirst IS NULL) OR (orderby IS NOT NULL AND orderby_desc IS NOT NULL AND orderby_nullsfirst IS NOT NULL) ),
+  CONSTRAINT compression_settings_check_orderby_cardinality CHECK (array_ndims(orderby) = 1 AND array_ndims(orderby_desc) = 1 AND array_ndims(orderby_nullsfirst) = 1 AND cardinality(orderby) = cardinality(orderby_desc) AND cardinality(orderby) = cardinality(orderby_nullsfirst))
+);
+
+INSERT INTO _timescaledb_catalog.compression_settings(relid, segmentby, orderby, orderby_desc, orderby_nullsfirst)
+  SELECT
+    format('%I.%I', ht.schema_name, ht.table_name)::regclass,
+    array_agg(attname ORDER BY segmentby_column_index) FILTER(WHERE segmentby_column_index >= 1) AS compress_segmentby,
+    array_agg(attname ORDER BY orderby_column_index) FILTER(WHERE orderby_column_index >= 1) AS compress_orderby,
+    array_agg(NOT orderby_asc ORDER BY orderby_column_index) FILTER(WHERE orderby_column_index >= 1) AS compress_orderby_desc,
+    array_agg(orderby_nullsfirst ORDER BY orderby_column_index) FILTER(WHERE orderby_column_index >= 1) AS compress_orderby_nullsfirst
+  FROM _timescaledb_catalog.hypertable_compression hc
+		INNER JOIN _timescaledb_catalog.hypertable ht ON ht.id = hc.hypertable_id
+	GROUP BY hypertable_id, ht.schema_name, ht.table_name;
+
+GRANT SELECT ON _timescaledb_catalog.compression_settings TO PUBLIC;
+SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.compression_settings', '');
+
+ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.hypertable_compression;
+DROP VIEW IF EXISTS timescaledb_information.compression_settings;
+DROP TABLE _timescaledb_catalog.hypertable_compression;
+

sql/updates/reverse-dev.sql

@@ -141,3 +141,53 @@ CREATE FUNCTION @[email protected]_replication_factor(
 ) RETURNS VOID
 AS '@MODULE_PATHNAME@', 'ts_hypertable_distributed_set_replication_factor' LANGUAGE C VOLATILE;
 
+CREATE TABLE _timescaledb_catalog.hypertable_compression (
+  hypertable_id integer NOT NULL,
+  attname name NOT NULL,
+  compression_algorithm_id smallint,
+  segmentby_column_index smallint,
+  orderby_column_index smallint,
+  orderby_asc boolean,
+  orderby_nullsfirst boolean,
+  -- table constraints
+  CONSTRAINT hypertable_compression_pkey PRIMARY KEY (hypertable_id, attname),
+  CONSTRAINT hypertable_compression_hypertable_id_orderby_column_index_key UNIQUE (hypertable_id, orderby_column_index),
+  CONSTRAINT hypertable_compression_hypertable_id_segmentby_column_index_key UNIQUE (hypertable_id, segmentby_column_index),
+  CONSTRAINT hypertable_compression_compression_algorithm_id_fkey FOREIGN KEY (compression_algorithm_id) REFERENCES _timescaledb_catalog.compression_algorithm (id),
+  CONSTRAINT hypertable_compression_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE
+);
+
+INSERT INTO _timescaledb_catalog.hypertable_compression(
+	hypertable_id,
+	attname,
+	compression_algorithm_id,
+	segmentby_column_index,
+	orderby_column_index,
+	orderby_asc,
+	orderby_nullsfirst
+) SELECT
+  ht.id,
+  att.attname,
+  CASE
+    WHEN att.attname = ANY(cs.segmentby) THEN 0
+		WHEN att.atttypid IN ('numeric'::regtype) THEN 1
+		WHEN att.atttypid IN ('float4'::regtype,'float8'::regtype) THEN 3
+		WHEN att.atttypid IN ('int2'::regtype,'int4'::regtype,'int8'::regtype,'date'::regtype,'timestamp'::regtype,'timestamptz'::regtype) THEN 4
+		WHEN EXISTS(SELECT FROM pg_operator op WHERE op.oprname = '=' AND op.oprkind = 'b' AND op.oprcanhash = true AND op.oprleft = att.atttypid AND op.oprright = att.atttypid) THEN 2
+    ELSE 1
+  END AS compression_algorithm_id,
+  CASE WHEN att.attname = ANY(cs.segmentby) THEN array_position(cs.segmentby, att.attname::text) ELSE NULL END AS segmentby_column_index,
+  CASE WHEN att.attname = ANY(cs.orderby) THEN array_position(cs.orderby, att.attname::text) ELSE NULL END AS orderby_column_index,
+  CASE WHEN att.attname = ANY(cs.orderby) THEN NOT cs.orderby_desc[array_position(cs.orderby, att.attname::text)] ELSE false END AS orderby_asc,
+  CASE WHEN att.attname = ANY(cs.orderby) THEN cs.orderby_nullsfirst[array_position(cs.orderby, att.attname::text)] ELSE false END AS orderby_nullsfirst
+FROM _timescaledb_catalog.hypertable ht
+INNER JOIN _timescaledb_catalog.compression_settings cs ON cs.relid = format('%I.%I',ht.schema_name,ht.table_name)::regclass
+LEFT JOIN pg_attribute att ON att.attrelid = format('%I.%I',ht.schema_name,ht.table_name)::regclass AND attnum > 0
+WHERE compressed_hypertable_id IS NOT NULL;
+
+SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable_compression', '');
+GRANT SELECT ON _timescaledb_catalog.hypertable_compression TO PUBLIC;
+
+DROP VIEW timescaledb_information.compression_settings;
+ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.compression_settings;
+DROP TABLE _timescaledb_catalog.compression_settings;

sql/views.sql

@@ -287,19 +287,30 @@ WHERE dim.hypertable_id = ht.id;
 
 ---compression parameters information ---
 CREATE OR REPLACE VIEW timescaledb_information.compression_settings AS
-SELECT ht.schema_name AS hypertable_schema,
-  ht.table_name AS hypertable_name,
-  segq.attname,
-  segq.segmentby_column_index,
-  segq.orderby_column_index,
-  segq.orderby_asc,
-  segq.orderby_nullsfirst
-FROM _timescaledb_catalog.hypertable_compression segq,
-  _timescaledb_catalog.hypertable ht
-WHERE segq.hypertable_id = ht.id
-  AND (segq.segmentby_column_index IS NOT NULL
-    OR segq.orderby_column_index IS NOT NULL)
-ORDER BY table_name,
+SELECT
+	schema_name AS hypertable_schema,
+  table_name AS hypertable_name,
+  (unnest(cs.segmentby))::name COLLATE "C" AS attname,
+  generate_series(1,array_length(cs.segmentby,1))::smallint AS segmentby_column_index,
+  NULL::smallint AS orderby_column_index,
+  NULL::bool AS orderby_asc,
+  NULL::bool AS orderby_nullsfirst
+FROM _timescaledb_catalog.hypertable ht
+INNER JOIN _timescaledb_catalog.compression_settings cs ON cs.relid = format('%I.%I',ht.schema_name,ht.table_name)::regclass AND cs.segmentby IS NOT NULL
+WHERE compressed_hypertable_id IS NOT NULL
+UNION ALL
+SELECT
+	schema_name AS hypertable_schema,
+  table_name AS hypertable_name,
+  (unnest(cs.orderby))::name COLLATE "C" AS attname,
+  NULL::smallint AS segmentby_column_index,
+  generate_series(1,array_length(cs.orderby,1))::smallint AS orderby_column_index,
+  unnest(array_replace(array_replace(array_replace(cs.orderby_desc,false,NULL),true,false),NULL,true)) AS orderby_asc,
+  unnest(cs.orderby_nullsfirst) AS orderby_nullsfirst
+FROM _timescaledb_catalog.hypertable ht
+INNER JOIN _timescaledb_catalog.compression_settings cs ON cs.relid = format('%I.%I',ht.schema_name,ht.table_name)::regclass AND cs.orderby IS NOT NULL
+WHERE compressed_hypertable_id IS NOT NULL
+ORDER BY hypertable_name,
   segmentby_column_index,
   orderby_column_index;
 

src/hypertable.c

@@ -39,14 +39,14 @@
 #include <parser/parse_coerce.h>
 
 #include "hypertable.h"
+#include "ts_catalog/compression_settings.h"
 #include "ts_catalog/hypertable_data_node.h"
 #include "ts_catalog/catalog.h"
 #include "ts_catalog/metadata.h"
 #include "hypercube.h"
 #include "dimension.h"
 #include "chunk.h"
 #include "chunk_adaptive.h"
-#include "ts_catalog/hypertable_compression.h"
 #include "subspace_store.h"
 #include "hypertable_cache.h"
 #include "trigger.h"
@@ -628,7 +628,7 @@ hypertable_tuple_delete(TupleInfo *ti, void *data)
 	ts_continuous_agg_drop_hypertable_callback(hypertable_id);
 
 	/* remove any associated compression definitions */
-	ts_hypertable_compression_delete_by_hypertable_id(hypertable_id);
+	ts_compression_settings_delete(ts_hypertable_id_to_relid(hypertable_id, true));
 
 	if (!compressed_hypertable_id_isnull)
 	{

src/process_utility.c

@@ -60,7 +60,9 @@
 #include "hypercube.h"
 #include "hypertable.h"
 #include "hypertable_cache.h"
+#include "ts_catalog/compression_settings.h"
 #include "ts_catalog/hypertable_data_node.h"
+#include "ts_catalog/array_utils.h"
 #include "dimension_vector.h"
 #include "indexing.h"
 #include "scan_iterator.h"
@@ -1952,11 +1954,11 @@ process_rename_column(ProcessUtilityArgs *args, Cache *hcache, Oid relid, Rename
 	Hypertable *ht = ts_hypertable_cache_get_entry(hcache, relid, CACHE_FLAG_MISSING_OK);
 	Dimension *dim;
 
-	if (NULL == ht)
+	if (!ht)
 	{
 		Chunk *chunk = ts_chunk_get_by_relid(relid, false);
 
-		if (NULL != chunk)
+		if (chunk)
 			ereport(ERROR,
 					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
 					 errmsg("cannot rename column \"%s\" of hypertable chunk \"%s\"",
@@ -2019,6 +2021,7 @@ process_rename_column(ProcessUtilityArgs *args, Cache *hcache, Oid relid, Rename
 	 * we don't do anything. */
 	if (ht)
 	{
+		ts_compression_settings_rename_column(ht->main_table_relid, stmt->subname, stmt->newname);
 		add_hypertable_to_process_args(args, ht);
 		dim = ts_hyperspace_get_mutable_dimension_by_name(ht->space,
 														  DIMENSION_TYPE_ANY,

src/ts_catalog/CMakeLists.txt

@@ -3,10 +3,10 @@ set(SOURCES
     ${CMAKE_CURRENT_SOURCE_DIR}/catalog.c
     ${CMAKE_CURRENT_SOURCE_DIR}/chunk_data_node.c
     ${CMAKE_CURRENT_SOURCE_DIR}/compression_chunk_size.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/compression_settings.c
     ${CMAKE_CURRENT_SOURCE_DIR}/continuous_agg.c
     ${CMAKE_CURRENT_SOURCE_DIR}/continuous_aggs_watermark.c
     ${CMAKE_CURRENT_SOURCE_DIR}/dimension_partition.c
-    ${CMAKE_CURRENT_SOURCE_DIR}/hypertable_compression.c
     ${CMAKE_CURRENT_SOURCE_DIR}/hypertable_data_node.c
     ${CMAKE_CURRENT_SOURCE_DIR}/metadata.c
     ${CMAKE_CURRENT_SOURCE_DIR}/tablespace.c)

src/ts_catalog/array_utils.c

@@ -101,6 +101,50 @@ ts_array_position(ArrayType *arr, const char *name)
 	return found ? pos : 0;
 }
 
+extern TSDLLEXPORT ArrayType *
+ts_array_replace_text(ArrayType *arr, const char *old, const char *new)
+{
+	if (!arr)
+		return NULL;
+
+	Assert(ARR_NDIM(arr) == 1);
+	Assert(arr->elemtype == TEXTOID);
+
+	Datum datum;
+	bool null;
+	int pos = 1;
+	ArrayIterator it = array_create_iterator(arr, 0, NULL);
+
+	while (array_iterate(it, &datum, &null))
+	{
+		/*
+		 * Our internal catalog arrays should either be NULL or
+		 * have non-NULL members. During normal operation it should
+		 * never have NULL members. If we have NULL members either
+		 * the catalog is corrupted or some catalog tampering has
+		 * happened.
+		 */
+		Ensure(!null, "array element was NULL");
+		if (strncmp(TextDatumGetCString(datum), old, NAMEDATALEN) == 0)
+		{
+			datum = array_set_element(PointerGetDatum(arr),
+									  1,
+									  &pos,
+									  CStringGetTextDatum(new),
+									  false,
+									  -1,
+									  -1,
+									  false,
+									  TYPALIGN_INT);
+			arr = DatumGetArrayTypeP(datum);
+		}
+		pos++;
+	}
+
+	array_free_iterator(it);
+	return arr;
+}
+
 extern TSDLLEXPORT bool
 ts_array_get_element_bool(ArrayType *arr, int position)
 {

src/ts_catalog/array_utils.h

@@ -10,6 +10,9 @@
 
 #include "export.h"
 
+#ifndef TIMESCALEDB_ARRAY_UTILS_H
+#define TIMESCALEDB_ARRAY_UTILS_H
+
 /*
  * Array helper function for internal catalog arrays.
  * These are not suitable for arbitrary dimension
@@ -20,7 +23,14 @@
 extern TSDLLEXPORT int ts_array_length(ArrayType *arr);
 extern TSDLLEXPORT bool ts_array_is_member(ArrayType *arr, const char *name);
 extern TSDLLEXPORT int ts_array_position(ArrayType *arr, const char *name);
+
 extern TSDLLEXPORT bool ts_array_get_element_bool(ArrayType *arr, int position);
 extern TSDLLEXPORT const char *ts_array_get_element_text(ArrayType *arr, int position);
-extern TSDLLEXPORT ArrayType *ts_array_add_element_text(ArrayType *arr, const char *value);
+
 extern TSDLLEXPORT ArrayType *ts_array_add_element_bool(ArrayType *arr, bool value);
+extern TSDLLEXPORT ArrayType *ts_array_add_element_text(ArrayType *arr, const char *value);
+
+extern TSDLLEXPORT ArrayType *ts_array_replace_text(ArrayType *arr, const char *old,
+													const char *new);
+
+#endif

src/ts_catalog/catalog.c

@@ -97,9 +97,9 @@ static const TableInfoDef catalog_table_names[_MAX_CATALOG_TABLES + 1] = {
 		.schema_name = CATALOG_SCHEMA_NAME,
 		.table_name = CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG_TABLE_NAME,
 	},
-	[HYPERTABLE_COMPRESSION] = {
+	[COMPRESSION_SETTINGS] = {
 		.schema_name = CATALOG_SCHEMA_NAME,
-		.table_name = HYPERTABLE_COMPRESSION_TABLE_NAME,
+		.table_name = COMPRESSION_SETTINGS_TABLE_NAME,
 	},
 	[COMPRESSION_CHUNK_SIZE] = {
 		.schema_name = CATALOG_SCHEMA_NAME,
@@ -268,10 +268,10 @@ static const TableIndexDef catalog_table_index_definitions[_MAX_CATALOG_TABLES]
 			[CONTINUOUS_AGGS_WATERMARK_PKEY] = "continuous_aggs_watermark_pkey",
 		},
 	},
-	[HYPERTABLE_COMPRESSION] = {
-		.length =  _MAX_HYPERTABLE_COMPRESSION_INDEX,
+	[COMPRESSION_SETTINGS] = {
+		.length =  _MAX_COMPRESSION_SETTINGS_INDEX,
 		.names = (char *[]) {
-			[HYPERTABLE_COMPRESSION_PKEY] = "hypertable_compression_pkey",
+			[COMPRESSION_SETTINGS_PKEY] = "compression_settings_pkey",
 		},
 	},
 	[COMPRESSION_CHUNK_SIZE] = {
@@ -317,7 +317,7 @@ static const char *catalog_table_serial_id_names[_MAX_CATALOG_TABLES] = {
 	[CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG] = NULL,
 	[CONTINUOUS_AGGS_INVALIDATION_THRESHOLD] = NULL,
 	[CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG] = NULL,
-	[HYPERTABLE_COMPRESSION] = NULL,
+	[COMPRESSION_SETTINGS] = NULL,
 	[COMPRESSION_CHUNK_SIZE] = NULL,
 	[REMOTE_TXN] = NULL,
 	[CHUNK_COPY_OPERATION] = CATALOG_SCHEMA_NAME ".chunk_copy_operation_id_seq",