Skip to content

Commit

Permalink
Restrict creation of partial cagg
Browse files Browse the repository at this point in the history
Block creation of old format cagg. However, partial aggregates can be
created under a custom boolean variable `block_old_format_cagg`. This
variable can be used only for development purposes like validating
migration from old format to new format cagg until we completely remove
the old format cagg.
  • Loading branch information
pdipesh02 committed Sep 10, 2023
1 parent ef783c4 commit 1935d26
Show file tree
Hide file tree
Showing 25 changed files with 247 additions and 127 deletions.
1 change: 1 addition & 0 deletions .unreleased/PR_6056
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Implements: #6056 Restrict creation of partial cagg
12 changes: 12 additions & 0 deletions src/guc.c
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ bool ts_guc_enable_async_append = true;
TSDLLEXPORT bool ts_guc_enable_compression_indexscan = true;
TSDLLEXPORT bool ts_guc_enable_bulk_decompression = true;
TSDLLEXPORT bool ts_guc_enable_skip_scan = true;
TSDLLEXPORT bool ts_guc_block_old_format_cagg = true;
/* default value of ts_guc_max_open_chunks_per_insert and ts_guc_max_cached_chunks_per_hypertable
* will be set as their respective boot-value when the GUC mechanism starts up */
int ts_guc_max_open_chunks_per_insert;
Expand Down Expand Up @@ -476,6 +477,17 @@ _guc_init(void)
NULL,
NULL);

DefineCustomBoolVariable("timescaledb.block_old_format_cagg",
"Block creation of old format continuous aggregate",
"Disable creation of continuous aggregate with partials",
&ts_guc_block_old_format_cagg,
true,
PGC_USERSET,
0,
NULL,
NULL,
NULL);

DefineCustomIntVariable("timescaledb.max_insert_batch_size",
"The max number of tuples to batch before sending to a data node",
"When acting as a access node, TimescaleDB splits batches of "
Expand Down
1 change: 1 addition & 0 deletions src/guc.h
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ extern TSDLLEXPORT bool ts_guc_enable_per_data_node_queries;
extern TSDLLEXPORT bool ts_guc_enable_parameterized_data_node_scan;
extern TSDLLEXPORT bool ts_guc_enable_async_append;
extern TSDLLEXPORT bool ts_guc_enable_skip_scan;
extern TSDLLEXPORT bool ts_guc_block_old_format_cagg;
extern bool ts_guc_restoring;
extern int ts_guc_max_open_chunks_per_insert;
extern int ts_guc_max_cached_chunks_per_hypertable;
Expand Down
15 changes: 5 additions & 10 deletions tsl/src/continuous_aggs/common.c
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
*/

#include "common.h"
#include "guc.h"

static Const *check_time_bucket_argument(Node *arg, char *position);
static void caggtimebucketinfo_init(CAggTimebucketInfo *src, int32 hypertable_id,
Expand Down Expand Up @@ -419,14 +420,10 @@ cagg_agg_validate(Node *node, void *context)
static bool
cagg_query_supported(const Query *query, StringInfo hint, StringInfo detail, const bool finalized)
{
/*
* For now deprecate partial aggregates on release builds only.
* Once migration tests are made compatible with PG15 enable deprecation
* on debug builds as well.
*/
#ifndef DEBUG
#if PG15_GE
if (!finalized)
/*
* Deprecate partial aggregates to support only finalized aggregates.
*/
if (!finalized && ts_guc_block_old_format_cagg)
{
/* continuous aggregates with old format will not be allowed */
appendStringInfoString(detail,
Expand All @@ -436,8 +433,6 @@ cagg_query_supported(const Query *query, StringInfo hint, StringInfo detail, con
"to true.");
return false;
}
#endif
#endif
if (!query->jointree->fromlist)
{
appendStringInfoString(hint, "FROM clause missing in the query");
Expand Down
2 changes: 2 additions & 0 deletions tsl/test/expected/cagg_errors_deprecated.out
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ select table_name from create_hypertable( 'conditions', 'timec');
conditions
(1 row)

-- Continuous aggregates with partials are supported under a custom boolean flag.
SET timescaledb.block_old_format_cagg TO OFF;
CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.finalized = false, timescaledb.myfill = 1)
as
select location , min(temperature)
Expand Down
2 changes: 2 additions & 0 deletions tsl/test/expected/cagg_joins.out
Original file line number Diff line number Diff line change
Expand Up @@ -938,6 +938,8 @@ ERROR: invalid continuous aggregate view
DETAIL: Unsupported expression in join clause.
HINT: Only equality conditions are supported in continuous aggregates.
--With old format cagg definition
-- Continuous aggregates with partials are supported under a custom boolean flag.
SET timescaledb.block_old_format_cagg TO OFF;
CREATE MATERIALIZED VIEW cagg_cagg_old
WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE, timescaledb.finalized = FALSE) AS
SELECT time_bucket(INTERVAL '1 day', day) AS bucket,
Expand Down
246 changes: 129 additions & 117 deletions tsl/test/expected/cagg_migrate.out

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions tsl/test/expected/cagg_repair.out
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,8 @@ UNION ALL
GROUP BY (time_bucket('@ 7 days'::interval, conditions."time"));

-- Tests with old cagg format
-- Continuous aggregates with partials are supported under a custom boolean flag.
SET timescaledb.block_old_format_cagg TO OFF;
CREATE MATERIALIZED VIEW conditions_summary_old_format
WITH (timescaledb.continuous, timescaledb.finalized=false) AS
SELECT
Expand Down
12 changes: 12 additions & 0 deletions tsl/test/expected/continuous_aggs-13.out
Original file line number Diff line number Diff line change
Expand Up @@ -2213,6 +2213,18 @@ AS
SELECT time_bucket('1day', timec), min(location), sum(temperature), sum(humidity)
FROM conditions
GROUP BY time_bucket('1day', timec) WITH NO DATA;
-- Aggregates with partials are not supported by default
\set ON_ERROR_STOP 0
CREATE MATERIALIZED VIEW conditions_summary_old(timec, minl, sumt, sumh)
WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.finalized=false)
AS
SELECT time_bucket('1day', timec), min(location), sum(temperature), sum(humidity)
FROM conditions
GROUP BY time_bucket('1day', timec) WITH NO DATA;
ERROR: invalid continuous aggregate query
\set ON_ERROR_STOP 1
-- Aggregates with partials are supported under a custom boolean flag
SET timescaledb.block_old_format_cagg TO OFF;
CREATE MATERIALIZED VIEW conditions_summary_old(timec, minl, sumt, sumh)
WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.finalized=false)
AS
Expand Down
12 changes: 12 additions & 0 deletions tsl/test/expected/continuous_aggs-14.out
Original file line number Diff line number Diff line change
Expand Up @@ -2212,6 +2212,18 @@ AS
SELECT time_bucket('1day', timec), min(location), sum(temperature), sum(humidity)
FROM conditions
GROUP BY time_bucket('1day', timec) WITH NO DATA;
-- Aggregates with partials are not supported by default
\set ON_ERROR_STOP 0
CREATE MATERIALIZED VIEW conditions_summary_old(timec, minl, sumt, sumh)
WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.finalized=false)
AS
SELECT time_bucket('1day', timec), min(location), sum(temperature), sum(humidity)
FROM conditions
GROUP BY time_bucket('1day', timec) WITH NO DATA;
ERROR: invalid continuous aggregate query
\set ON_ERROR_STOP 1
-- Aggregates with partials are supported under a custom boolean flag
SET timescaledb.block_old_format_cagg TO OFF;
CREATE MATERIALIZED VIEW conditions_summary_old(timec, minl, sumt, sumh)
WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.finalized=false)
AS
Expand Down
12 changes: 12 additions & 0 deletions tsl/test/expected/continuous_aggs-15.out
Original file line number Diff line number Diff line change
Expand Up @@ -2214,6 +2214,18 @@ AS
SELECT time_bucket('1day', timec), min(location), sum(temperature), sum(humidity)
FROM conditions
GROUP BY time_bucket('1day', timec) WITH NO DATA;
-- Aggregates with partials are not supported by default
\set ON_ERROR_STOP 0
CREATE MATERIALIZED VIEW conditions_summary_old(timec, minl, sumt, sumh)
WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.finalized=false)
AS
SELECT time_bucket('1day', timec), min(location), sum(temperature), sum(humidity)
FROM conditions
GROUP BY time_bucket('1day', timec) WITH NO DATA;
ERROR: invalid continuous aggregate query
\set ON_ERROR_STOP 1
-- Aggregates with partials are supported under a custom boolean flag
SET timescaledb.block_old_format_cagg TO OFF;
CREATE MATERIALIZED VIEW conditions_summary_old(timec, minl, sumt, sumh)
WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.finalized=false)
AS
Expand Down
4 changes: 4 additions & 0 deletions tsl/test/expected/continuous_aggs_deprecated-13.out
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ SELECT set_integer_now_func('foo', 'integer_now_foo');

(1 row)

-- Continuous aggregates with partials are supported under a custom boolean flag.
SET timescaledb.block_old_format_cagg TO OFF;
CREATE MATERIALIZED VIEW mat_m1(a, countb)
WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.finalized=false)
as
Expand Down Expand Up @@ -741,6 +743,8 @@ select table_name from create_hypertable( 'conditions', 'timec');
(1 row)

--no data in hyper table on purpose so that CASCADE is not required because of chunks
-- Continuous aggregates with partials are supported under a custom boolean flag.
SET timescaledb.block_old_format_cagg TO OFF;
CREATE MATERIALIZED VIEW mat_drop_test(timec, minl, sumt , sumh)
WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.finalized=false)
as
Expand Down
4 changes: 4 additions & 0 deletions tsl/test/expected/continuous_aggs_deprecated-14.out
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ SELECT set_integer_now_func('foo', 'integer_now_foo');

(1 row)

-- Continuous aggregates with partials are supported under a custom boolean flag.
SET timescaledb.block_old_format_cagg TO OFF;
CREATE MATERIALIZED VIEW mat_m1(a, countb)
WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.finalized=false)
as
Expand Down Expand Up @@ -741,6 +743,8 @@ select table_name from create_hypertable( 'conditions', 'timec');
(1 row)

--no data in hyper table on purpose so that CASCADE is not required because of chunks
-- Continuous aggregates with partials are supported under a custom boolean flag.
SET timescaledb.block_old_format_cagg TO OFF;
CREATE MATERIALIZED VIEW mat_drop_test(timec, minl, sumt , sumh)
WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.finalized=false)
as
Expand Down
4 changes: 4 additions & 0 deletions tsl/test/expected/continuous_aggs_deprecated-15.out
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ SELECT set_integer_now_func('foo', 'integer_now_foo');

(1 row)

-- Continuous aggregates with partials are supported under a custom boolean flag.
SET timescaledb.block_old_format_cagg TO OFF;
CREATE MATERIALIZED VIEW mat_m1(a, countb)
WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.finalized=false)
as
Expand Down Expand Up @@ -741,6 +743,8 @@ select table_name from create_hypertable( 'conditions', 'timec');
(1 row)

--no data in hyper table on purpose so that CASCADE is not required because of chunks
-- Continuous aggregates with partials are supported under a custom boolean flag.
SET timescaledb.block_old_format_cagg TO OFF;
CREATE MATERIALIZED VIEW mat_drop_test(timec, minl, sumt , sumh)
WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.finalized=false)
as
Expand Down
2 changes: 2 additions & 0 deletions tsl/test/expected/telemetry_stats-13.out
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ FROM
hyper
GROUP BY hour, device;
NOTICE: continuous aggregate "contagg" is already up-to-date
-- Continuous aggregates with partials are supported under a custom boolean flag.
SET timescaledb.block_old_format_cagg TO OFF;
CREATE MATERIALIZED VIEW contagg_old
WITH (timescaledb.continuous, timescaledb.finalized=false) AS
SELECT
Expand Down
2 changes: 2 additions & 0 deletions tsl/test/expected/telemetry_stats-14.out
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ FROM
hyper
GROUP BY hour, device;
NOTICE: continuous aggregate "contagg" is already up-to-date
-- Continuous aggregates with partials are supported under a custom boolean flag.
SET timescaledb.block_old_format_cagg TO OFF;
CREATE MATERIALIZED VIEW contagg_old
WITH (timescaledb.continuous, timescaledb.finalized=false) AS
SELECT
Expand Down
4 changes: 4 additions & 0 deletions tsl/test/expected/telemetry_stats-15.out
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ FROM
hyper
GROUP BY hour, device;
NOTICE: continuous aggregate "contagg" is already up-to-date
-- Continuous aggregates with partials are supported under a custom boolean flag.
SET timescaledb.block_old_format_cagg TO OFF;
CREATE MATERIALIZED VIEW contagg_old
WITH (timescaledb.continuous, timescaledb.finalized=false) AS
SELECT
Expand Down Expand Up @@ -920,6 +922,8 @@ FROM
disthyper
GROUP BY hour, device;
NOTICE: refreshing continuous aggregate "distcontagg"
-- Continuous aggregates with partials are supported under a custom boolean flag.
SET timescaledb.block_old_format_cagg TO OFF;
CREATE MATERIALIZED VIEW distcontagg_old
WITH (timescaledb.continuous, timescaledb.finalized=false) AS
SELECT
Expand Down
3 changes: 3 additions & 0 deletions tsl/test/sql/cagg_errors_deprecated.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@ CREATE TABLE conditions (
);
select table_name from create_hypertable( 'conditions', 'timec');

-- Continuous aggregates with partials are supported under a custom boolean flag.
SET timescaledb.block_old_format_cagg TO OFF;

CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.finalized = false, timescaledb.myfill = 1)
as
select location , min(temperature)
Expand Down
3 changes: 3 additions & 0 deletions tsl/test/sql/cagg_joins.sql
Original file line number Diff line number Diff line change
Expand Up @@ -381,6 +381,9 @@ AND conditions.city = devices.location AND
GROUP BY name, bucket;

--With old format cagg definition
-- Continuous aggregates with partials are supported under a custom boolean flag.
SET timescaledb.block_old_format_cagg TO OFF;

CREATE MATERIALIZED VIEW cagg_cagg_old
WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE, timescaledb.finalized = FALSE) AS
SELECT time_bucket(INTERVAL '1 day', day) AS bucket,
Expand Down
2 changes: 2 additions & 0 deletions tsl/test/sql/cagg_repair.sql
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,8 @@ CALL _timescaledb_internal.cagg_try_repair('conditions_summary_nojoin', TRUE);
\d+ conditions_summary_nojoin

-- Tests with old cagg format
-- Continuous aggregates with partials are supported under a custom boolean flag.
SET timescaledb.block_old_format_cagg TO OFF;
CREATE MATERIALIZED VIEW conditions_summary_old_format
WITH (timescaledb.continuous, timescaledb.finalized=false) AS
SELECT
Expand Down
13 changes: 13 additions & 0 deletions tsl/test/sql/continuous_aggs.sql.in
Original file line number Diff line number Diff line change
Expand Up @@ -1490,6 +1490,19 @@ SELECT time_bucket('1day', timec), min(location), sum(temperature), sum(humidity
FROM conditions
GROUP BY time_bucket('1day', timec) WITH NO DATA;

-- Aggregates with partials are not supported by default
\set ON_ERROR_STOP 0
CREATE MATERIALIZED VIEW conditions_summary_old(timec, minl, sumt, sumh)
WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.finalized=false)
AS
SELECT time_bucket('1day', timec), min(location), sum(temperature), sum(humidity)
FROM conditions
GROUP BY time_bucket('1day', timec) WITH NO DATA;
\set ON_ERROR_STOP 1

-- Aggregates with partials are supported under a custom boolean flag
SET timescaledb.block_old_format_cagg TO OFF;

CREATE MATERIALIZED VIEW conditions_summary_old(timec, minl, sumt, sumh)
WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.finalized=false)
AS
Expand Down
5 changes: 5 additions & 0 deletions tsl/test/sql/continuous_aggs_deprecated.sql.in
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ CREATE OR REPLACE FUNCTION integer_now_foo() returns int LANGUAGE SQL STABLE as
SELECT set_integer_now_func('foo', 'integer_now_foo');


-- Continuous aggregates with partials are supported under a custom boolean flag.
SET timescaledb.block_old_format_cagg TO OFF;

CREATE MATERIALIZED VIEW mat_m1(a, countb)
WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.finalized=false)
as
Expand Down Expand Up @@ -574,6 +577,8 @@ select table_name from create_hypertable( 'conditions', 'timec');

--no data in hyper table on purpose so that CASCADE is not required because of chunks

-- Continuous aggregates with partials are supported under a custom boolean flag.
SET timescaledb.block_old_format_cagg TO OFF;
CREATE MATERIALIZED VIEW mat_drop_test(timec, minl, sumt , sumh)
WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.finalized=false)
as
Expand Down
4 changes: 4 additions & 0 deletions tsl/test/sql/include/cagg_migrate_common.sql
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,8 @@ CALL cagg_migrate('conditions_summary_daily_new');
\set ON_ERROR_STOP 1

-- older continuous aggregate to be migrated
-- Continuous aggregates with partials are supported under a custom boolean flag.
SET timescaledb.block_old_format_cagg TO OFF;
CREATE MATERIALIZED VIEW conditions_summary_daily
WITH (timescaledb.continuous, timescaledb.finalized=false) AS
SELECT
Expand Down Expand Up @@ -275,6 +277,8 @@ DROP MATERIALIZED VIEW conditions_summary_daily;
GRANT ALL ON TABLE conditions TO :ROLE_DEFAULT_PERM_USER;
SET ROLE :ROLE_DEFAULT_PERM_USER;

-- Continuous aggregates with partials are supported under a custom boolean flag.
SET timescaledb.block_old_format_cagg TO OFF;
CREATE MATERIALIZED VIEW conditions_summary_daily
WITH (timescaledb.continuous, timescaledb.finalized=false) AS
SELECT
Expand Down
3 changes: 3 additions & 0 deletions tsl/test/sql/include/cont_agg_equal_deprecated.sql
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@

DROP MATERIALIZED VIEW IF EXISTS mat_test;

-- Aggregates with partials are supported under a custom boolean flag
SET timescaledb.block_old_format_cagg TO OFF;

CREATE MATERIALIZED VIEW mat_test
WITH (timescaledb.continuous, timescaledb.finalized=false)
as :QUERY
Expand Down
4 changes: 4 additions & 0 deletions tsl/test/sql/telemetry_stats.sql.in
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ FROM
hyper
GROUP BY hour, device;

-- Continuous aggregates with partials are supported under a custom boolean flag.
SET timescaledb.block_old_format_cagg TO OFF;
CREATE MATERIALIZED VIEW contagg_old
WITH (timescaledb.continuous, timescaledb.finalized=false) AS
SELECT
Expand Down Expand Up @@ -227,6 +229,8 @@ FROM
disthyper
GROUP BY hour, device;

-- Continuous aggregates with partials are supported under a custom boolean flag.
SET timescaledb.block_old_format_cagg TO OFF;
CREATE MATERIALIZED VIEW distcontagg_old
WITH (timescaledb.continuous, timescaledb.finalized=false) AS
SELECT
Expand Down

0 comments on commit 1935d26

Please sign in to comment.