From d373115f3e51762775252c18fbab14b383688551 Mon Sep 17 00:00:00 2001 From: Bharathy Date: Sun, 3 Sep 2023 18:43:37 +0530 Subject: [PATCH] Refactor recompress_chunk() API This patch will improve performance of recompress_chunk() API by identifying only affected segments which needs to be recompressed. All unaffected segments are never decompressed. If affected compressed segments exceeds a certain limit we fallback to legacy way of recompressing, where we decompres all compressed segments and recompress them. Fixes #392 --- .unreleased/feature_5867 | 1 + tsl/src/compression/CMakeLists.txt | 1 + tsl/src/compression/api.c | 574 +------ tsl/src/compression/compression.c | 52 +- tsl/src/compression/compression.h | 25 +- tsl/src/compression/recompression.c | 1421 +++++++++++++++++ tsl/src/compression/recompression.h | 21 + .../recompress_chunk_segmentwise-v2.out | 1390 ++++++++++++++++ .../expected/recompress_chunk_segmentwise.out | 44 +- .../expected/compression_ddl_iso.out | 10 +- .../isolation/specs/compression_ddl_iso.spec | 2 +- tsl/test/sql/CMakeLists.txt | 3 +- .../sql/recompress_chunk_segmentwise-v2.sql | 459 ++++++ tsl/test/sql/recompress_chunk_segmentwise.sql | 12 +- 14 files changed, 3387 insertions(+), 628 deletions(-) create mode 100644 .unreleased/feature_5867 create mode 100644 tsl/src/compression/recompression.c create mode 100644 tsl/src/compression/recompression.h create mode 100644 tsl/test/expected/recompress_chunk_segmentwise-v2.out create mode 100644 tsl/test/sql/recompress_chunk_segmentwise-v2.sql diff --git a/.unreleased/feature_5867 b/.unreleased/feature_5867 new file mode 100644 index 00000000000..4280c0b1d5e --- /dev/null +++ b/.unreleased/feature_5867 @@ -0,0 +1 @@ +Implements: #5867 Refactor recompress_chunk() API diff --git a/tsl/src/compression/CMakeLists.txt b/tsl/src/compression/CMakeLists.txt index 9e8e9724872..d90a8c36c3e 100644 --- a/tsl/src/compression/CMakeLists.txt +++ b/tsl/src/compression/CMakeLists.txt @@ -7,5 +7,6 @@ set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/deltadelta.c ${CMAKE_CURRENT_SOURCE_DIR}/dictionary.c ${CMAKE_CURRENT_SOURCE_DIR}/gorilla.c + ${CMAKE_CURRENT_SOURCE_DIR}/recompression.c ${CMAKE_CURRENT_SOURCE_DIR}/segment_meta.c) target_sources(${TSL_LIBRARY_NAME} PRIVATE ${SOURCES}) diff --git a/tsl/src/compression/api.c b/tsl/src/compression/api.c index 3113fa27518..f4f38903dae 100644 --- a/tsl/src/compression/api.c +++ b/tsl/src/compression/api.c @@ -43,6 +43,7 @@ #include "api.h" #include "compression.h" #include "compat/compat.h" +#include "recompression.h" #include "scanner.h" #include "scan_iterator.h" #include "utils.h" @@ -181,83 +182,6 @@ compression_chunk_size_catalog_update_merged(int32 chunk_id, const RelationSize return updated; } -/* - * This function updates catalog chunk compression statistics - * for an existing compressed chunk after it has been recompressed - * segmentwise in-place (as opposed to creating a new compressed chunk). - * Note that because of this it is not possible to accurately report - * the fields - * uncompressed_chunk_size, uncompressed_index_size, uncompressed_toast_size - * anymore, so these are not updated. - */ -static int -compression_chunk_size_catalog_update_recompressed(int32 uncompressed_chunk_id, - int32 compressed_chunk_id, - const RelationSize *recompressed_size, - int64 rowcnt_pre_compression, - int64 rowcnt_post_compression) -{ - ScanIterator iterator = - ts_scan_iterator_create(COMPRESSION_CHUNK_SIZE, RowExclusiveLock, CurrentMemoryContext); - bool updated = false; - - iterator.ctx.index = - catalog_get_index(ts_catalog_get(), COMPRESSION_CHUNK_SIZE, COMPRESSION_CHUNK_SIZE_PKEY); - ts_scan_iterator_scan_key_init(&iterator, - Anum_compression_chunk_size_pkey_chunk_id, - BTEqualStrategyNumber, - F_INT4EQ, - Int32GetDatum(uncompressed_chunk_id)); - ts_scanner_foreach(&iterator) - { - Datum values[Natts_compression_chunk_size]; - bool replIsnull[Natts_compression_chunk_size] = { false }; - bool repl[Natts_compression_chunk_size] = { false }; - bool should_free; - TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); - HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); - HeapTuple new_tuple; - heap_deform_tuple(tuple, ts_scanner_get_tupledesc(ti), values, replIsnull); - - /* Only update the information pertaining to the compressed chunk */ - /* these fields are about the compressed chunk so they can be updated */ - values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_heap_size)] = - Int64GetDatum(recompressed_size->heap_size); - repl[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_heap_size)] = true; - - values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_toast_size)] = - Int64GetDatum(recompressed_size->toast_size); - repl[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_toast_size)] = true; - - values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_index_size)] = - Int64GetDatum(recompressed_size->index_size); - repl[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_index_size)] = true; - - values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_numrows_pre_compression)] = - Int64GetDatum(rowcnt_pre_compression); - repl[AttrNumberGetAttrOffset(Anum_compression_chunk_size_numrows_pre_compression)] = true; - - values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_numrows_post_compression)] = - Int64GetDatum(rowcnt_post_compression); - repl[AttrNumberGetAttrOffset(Anum_compression_chunk_size_numrows_post_compression)] = true; - - new_tuple = - heap_modify_tuple(tuple, ts_scanner_get_tupledesc(ti), values, replIsnull, repl); - ts_catalog_update(ti->scanrel, new_tuple); - heap_freetuple(new_tuple); - - if (should_free) - heap_freetuple(tuple); - - updated = true; - break; - } - - ts_scan_iterator_end(&iterator); - ts_scan_iterator_close(&iterator); - return updated; -} - static void get_hypertable_or_cagg_name(Hypertable *ht, Name objname) { @@ -920,86 +844,6 @@ tsl_recompress_chunk_wrapper(Chunk *uncompressed_chunk) return true; } -/* This is a wrapper around row_compressor_append_sorted_rows. */ -static void -recompress_segment(Tuplesortstate *tuplesortstate, Relation compressed_chunk_rel, - RowCompressor *row_compressor) -{ - row_compressor_append_sorted_rows(row_compressor, - tuplesortstate, - RelationGetDescr(compressed_chunk_rel)); -} - -static bool -compressed_chunk_column_is_segmentby(PerCompressedColumn per_compressed_col) -{ - /* not compressed and not metadata, offset should be -1 for metadata */ - return !per_compressed_col.is_compressed && per_compressed_col.decompressed_column_offset >= 0; -} - -static void -decompress_segment_update_current_segment(CompressedSegmentInfo **current_segment, - TupleTableSlot *slot, PerCompressedColumn *per_col, - int16 *segby_col_offsets_compressed, int nsegmentby_cols) -{ - Datum val; - bool is_null; - int seg_idx = 0; - for (int i = 0; i < nsegmentby_cols; i++) - { - int16 col_offset = segby_col_offsets_compressed[i]; - if (!compressed_chunk_column_is_segmentby(per_col[col_offset])) - continue; - else - { - val = slot_getattr(slot, AttrOffsetGetAttrNumber(col_offset), &is_null); - if (!segment_info_datum_is_in_group(current_segment[seg_idx++]->segment_info, - val, - is_null)) - { - /* new segment, need to do per-segment processing */ - pfree( - current_segment[seg_idx - 1]->segment_info); /* because increased previously */ - SegmentInfo *segment_info = - segment_info_new(TupleDescAttr(slot->tts_tupleDescriptor, col_offset)); - segment_info_update(segment_info, val, is_null); - current_segment[seg_idx - 1]->segment_info = segment_info; - current_segment[seg_idx - 1]->decompressed_chunk_offset = - per_col[col_offset].decompressed_column_offset; - } - } - } -} - -static bool -decompress_segment_changed_group(CompressedSegmentInfo **current_segment, TupleTableSlot *slot, - PerCompressedColumn *per_col, int16 *segby_col_offsets_compressed, - int nsegmentby_cols) -{ - Datum val; - bool is_null; - bool changed_segment = false; - int seg_idx = 0; - for (int i = 0; i < nsegmentby_cols; i++) - { - int16 col_offset = segby_col_offsets_compressed[i]; - if (!compressed_chunk_column_is_segmentby(per_col[col_offset])) - continue; - else - { - val = slot_getattr(slot, AttrOffsetGetAttrNumber(col_offset), &is_null); - if (!segment_info_datum_is_in_group(current_segment[seg_idx++]->segment_info, - val, - is_null)) - { - changed_segment = true; - break; - } - } - } - return changed_segment; -} - static bool recompress_remote_chunk(FunctionCallInfo fcinfo, Chunk *chunk) { @@ -1071,7 +915,8 @@ tsl_get_compressed_chunk_index_for_recompression(PG_FUNCTION_ARGS) in_column_offsets, compressed_rel_tupdesc->natts, true /*need_bistate*/, - true /*reset_sequence*/); + true /*reset_sequence*/, + RECOMPRESS); /* * Keep the ExclusiveLock on the compressed chunk. This lock will be requested @@ -1093,130 +938,6 @@ tsl_get_compressed_chunk_index_for_recompression(PG_FUNCTION_ARGS) PG_RETURN_NULL(); } -/* - * This function fetches the remaining uncompressed chunk rows into - * the tuplesort for recompression. - * `unmatched` here means that no existing segment in the compressed - * chunk matches the segmentby column values for these remaining rows, - * so new segments must be created for them. - */ -static void -fetch_unmatched_uncompressed_chunk_into_tuplesort(Tuplesortstate *segment_tuplesortstate, - Relation uncompressed_chunk_rel, - bool *unmatched_rows_exist) -{ - TableScanDesc heapScan; - HeapTuple uncompressed_tuple; - TupleDesc uncompressed_rel_tupdesc = RelationGetDescr(uncompressed_chunk_rel); - - heapScan = table_beginscan(uncompressed_chunk_rel, GetLatestSnapshot(), 0, NULL); - TupleTableSlot *heap_tuple_slot = - MakeTupleTableSlot(uncompressed_rel_tupdesc, &TTSOpsHeapTuple); - - while ((uncompressed_tuple = heap_getnext(heapScan, ForwardScanDirection)) != NULL) - { - if (!(*unmatched_rows_exist)) - *unmatched_rows_exist = true; - - ExecStoreHeapTuple(uncompressed_tuple, heap_tuple_slot, false); - slot_getallattrs(heap_tuple_slot); - - tuplesort_puttupleslot(segment_tuplesortstate, heap_tuple_slot); - - simple_heap_delete(uncompressed_chunk_rel, &uncompressed_tuple->t_self); - } - ExecDropSingleTupleTableSlot(heap_tuple_slot); - table_endscan(heapScan); -} - -static void -fetch_matching_uncompressed_chunk_into_tuplesort(Tuplesortstate *segment_tuplesortstate, - int nsegmentby_cols, - Relation uncompressed_chunk_rel, - CompressedSegmentInfo **current_segment) -{ - TableScanDesc heapScan; - HeapTuple uncompressed_tuple; - TupleDesc uncompressed_rel_tupdesc = RelationGetDescr(uncompressed_chunk_rel); - int index = 0; - int nsegbycols_nonnull = 0; - Bitmapset *null_segbycols = NULL; - - for (int seg_col = 0; seg_col < nsegmentby_cols; seg_col++) - { - if (!current_segment[seg_col]->segment_info->is_null) - nsegbycols_nonnull++; - /* also build a bms of null columns to check later on */ - else - { - int16 attno = current_segment[seg_col]->decompressed_chunk_offset + 1; - null_segbycols = bms_add_member(null_segbycols, attno); - } - } - - ScanKeyData *scankey = - (nsegbycols_nonnull > 0) ? palloc0(sizeof(*scankey) * nsegbycols_nonnull) : NULL; - - for (int seg_col = 0; seg_col < nsegmentby_cols; seg_col++) - { - Datum val = current_segment[seg_col]->segment_info->val; - /* Get the attno of this segby col in the uncompressed chunk */ - int16 attno = - current_segment[seg_col]->decompressed_chunk_offset + 1; /* offset is attno - 1 */ - - if (current_segment[seg_col]->segment_info->is_null) - continue; - ScanKeyEntryInitializeWithInfo(&scankey[index], - 0 /*flags */, - attno, - BTEqualStrategyNumber, - InvalidOid, - current_segment[seg_col]->segment_info->collation, - ¤t_segment[seg_col]->segment_info->eq_fn, - val); - index++; - } - - heapScan = - table_beginscan(uncompressed_chunk_rel, GetLatestSnapshot(), nsegbycols_nonnull, scankey); - TupleTableSlot *heap_tuple_slot = - MakeTupleTableSlot(uncompressed_rel_tupdesc, &TTSOpsHeapTuple); - - while ((uncompressed_tuple = heap_getnext(heapScan, ForwardScanDirection)) != NULL) - { - bool valid = true; - /* check for NULL values in this segment manually */ - for (int attno = bms_next_member(null_segbycols, -1); attno >= 0; - attno = bms_next_member(null_segbycols, attno)) - { - if (!heap_attisnull(uncompressed_tuple, - attno, - RelationGetDescr(uncompressed_chunk_rel))) - { - valid = false; - break; - } - } - if (valid) - { - ExecStoreHeapTuple(uncompressed_tuple, heap_tuple_slot, false); - slot_getallattrs(heap_tuple_slot); - tuplesort_puttupleslot(segment_tuplesortstate, heap_tuple_slot); - /* simple_heap_delete since we don't expect concurrent updates, have exclusive lock on - * the relation */ - simple_heap_delete(uncompressed_chunk_rel, &uncompressed_tuple->t_self); - } - } - ExecDropSingleTupleTableSlot(heap_tuple_slot); - table_endscan(heapScan); - - if (null_segbycols != NULL) - pfree(null_segbycols); - - if (scankey != NULL) - pfree(scankey); -} - /* * Recompress an existing chunk by decompressing the batches * that are affected by the addition of newer data. The existing @@ -1256,7 +977,6 @@ tsl_recompress_chunk_segmentwise(PG_FUNCTION_ARGS) uncompressed_chunk->fd.schema_name.data, uncompressed_chunk->fd.table_name.data); } - /* * only proceed if status in (3, 9, 11) * 1: compressed @@ -1272,30 +992,8 @@ tsl_recompress_chunk_segmentwise(PG_FUNCTION_ARGS) uncompressed_chunk->fd.schema_name.data, uncompressed_chunk->fd.table_name.data); - int i = 0, htcols_listlen; - ListCell *lc; - - /* need it to find the segby cols from the catalog */ - int32 srcht_id = uncompressed_chunk->fd.hypertable_id; Chunk *compressed_chunk = ts_chunk_get_by_id(uncompressed_chunk->fd.compressed_chunk_id, true); - List *htcols_list = ts_hypertable_compression_get(srcht_id); - htcols_listlen = list_length(htcols_list); - /* find segmentby columns (need to know their index) */ - const ColumnCompressionInfo **colinfo_array; - /* convert list to array of pointers for compress_chunk */ - colinfo_array = palloc(sizeof(ColumnCompressionInfo *) * htcols_listlen); - - int nsegmentby_cols = 0; - - foreach (lc, htcols_list) - { - FormData_hypertable_compression *fd = (FormData_hypertable_compression *) lfirst(lc); - colinfo_array[i++] = fd; - if (COMPRESSIONCOL_IS_SEGMENT_BY(fd)) - nsegmentby_cols++; - } - /* new status after recompress should simply be compressed (1) * It is ok to update this early on in the transaction as it keeps a lock * on the updated tuple in the CHUNK table potentially preventing other transaction @@ -1308,274 +1006,10 @@ tsl_recompress_chunk_segmentwise(PG_FUNCTION_ARGS) /* TODO: Take RowExclusive locks instead of AccessExclusive */ LockRelationOid(uncompressed_chunk->table_id, ExclusiveLock); LockRelationOid(compressed_chunk->table_id, ExclusiveLock); - - /************* need this for sorting my tuples *****************/ - const ColumnCompressionInfo **keys; - int n_keys; - int16 *in_column_offsets = compress_chunk_populate_keys(uncompressed_chunk->table_id, - colinfo_array, - htcols_listlen, - &n_keys, - &keys); - - /* XXX: Ideally, we want to take RowExclusiveLock to allow more concurrent operations than - * simply SELECTs */ Relation compressed_chunk_rel = table_open(compressed_chunk->table_id, ExclusiveLock); Relation uncompressed_chunk_rel = table_open(uncompressed_chunk->table_id, ExclusiveLock); - /****** compression statistics ******/ - RelationSize after_size; - - Tuplesortstate *segment_tuplesortstate; - - /*************** tuplesort state *************************/ - TupleDesc compressed_rel_tupdesc = RelationGetDescr(compressed_chunk_rel); - TupleDesc uncompressed_rel_tupdesc = RelationGetDescr(uncompressed_chunk_rel); - - AttrNumber *sort_keys = palloc(sizeof(*sort_keys) * n_keys); - Oid *sort_operators = palloc(sizeof(*sort_operators) * n_keys); - Oid *sort_collations = palloc(sizeof(*sort_collations) * n_keys); - bool *nulls_first = palloc(sizeof(*nulls_first) * n_keys); - - for (int n = 0; n < n_keys; n++) - compress_chunk_populate_sort_info_for_column(RelationGetRelid(uncompressed_chunk_rel), - keys[n], - &sort_keys[n], - &sort_operators[n], - &sort_collations[n], - &nulls_first[n]); - - segment_tuplesortstate = tuplesort_begin_heap(uncompressed_rel_tupdesc, - n_keys, - sort_keys, - sort_operators, - sort_collations, - nulls_first, - maintenance_work_mem, - NULL, - false); - - /******************** row decompressor **************/ - - RowDecompressor decompressor = build_decompressor(compressed_chunk_rel, uncompressed_chunk_rel); - /* do not need the indexes on the uncompressed chunk as we do not write to it anymore */ - ts_catalog_close_indexes(decompressor.indexstate); - /********** row compressor *******************/ - RowCompressor row_compressor; - row_compressor_init(&row_compressor, - uncompressed_rel_tupdesc, - compressed_chunk_rel, - htcols_listlen, - colinfo_array, - in_column_offsets, - compressed_rel_tupdesc->natts, - true /*need_bistate*/, - true /*reset_sequence*/); - - /* create an array of the segmentby column offsets in the compressed chunk */ - int16 *segmentby_column_offsets_compressed = - palloc(sizeof(*segmentby_column_offsets_compressed) * nsegmentby_cols); - int seg_idx = 0; - for (int col = 0; col < decompressor.num_compressed_columns; col++) - { - if (!compressed_chunk_column_is_segmentby(decompressor.per_compressed_cols[col])) - continue; - segmentby_column_offsets_compressed[seg_idx++] = col; - } - - HeapTuple compressed_tuple; - - IndexScanDesc index_scan; - SegmentInfo *segment_info = NULL; - bool changed_segment = false; - /************ current segment **************/ - CompressedSegmentInfo **current_segment = - palloc(sizeof(CompressedSegmentInfo *) * nsegmentby_cols); - - for (int i = 0; i < nsegmentby_cols; i++) - { - current_segment[i] = palloc(sizeof(CompressedSegmentInfo)); - } - bool current_segment_init = false; - - /************** snapshot ****************************/ - Snapshot snapshot = RegisterSnapshot(GetTransactionSnapshot()); - - /* Index scan */ - Relation index_rel = index_open(row_compressor.index_oid, AccessExclusiveLock); - - index_scan = index_beginscan(compressed_chunk_rel, index_rel, snapshot, 0, 0); - TupleTableSlot *slot = table_slot_create(compressed_chunk_rel, NULL); - index_rescan(index_scan, NULL, 0, NULL, 0); - - while (index_getnext_slot(index_scan, ForwardScanDirection, slot)) - { - i = 0; - int col = 0; - slot_getallattrs(slot); - - if (!current_segment_init) - { - current_segment_init = true; - Datum val; - bool is_null; - /* initialize current segment */ - for (col = 0; col < slot->tts_tupleDescriptor->natts; col++) - { - val = slot_getattr(slot, AttrOffsetGetAttrNumber(col), &is_null); - if (compressed_chunk_column_is_segmentby(decompressor.per_compressed_cols[col])) - { - segment_info = segment_info_new(TupleDescAttr(slot->tts_tupleDescriptor, col)); - current_segment[i]->decompressed_chunk_offset = - decompressor.per_compressed_cols[col].decompressed_column_offset; - /* also need to call segment_info_update here to update the val part */ - segment_info_update(segment_info, val, is_null); - current_segment[i]->segment_info = segment_info; - i++; - } - } - } - /* we have a segment already, so compare those */ - changed_segment = decompress_segment_changed_group(current_segment, - slot, - decompressor.per_compressed_cols, - segmentby_column_offsets_compressed, - nsegmentby_cols); - if (!changed_segment) - { - i = 0; - bool should_free; - - compressed_tuple = ExecFetchSlotHeapTuple(slot, false, &should_free); - - heap_deform_tuple(compressed_tuple, - compressed_rel_tupdesc, - decompressor.compressed_datums, - decompressor.compressed_is_nulls); - - row_decompressor_decompress_row(&decompressor, segment_tuplesortstate); - - simple_table_tuple_delete(compressed_chunk_rel, &(slot->tts_tid), snapshot); - - if (should_free) - heap_freetuple(compressed_tuple); - } - else if (changed_segment) - { - fetch_matching_uncompressed_chunk_into_tuplesort(segment_tuplesortstate, - nsegmentby_cols, - uncompressed_chunk_rel, - current_segment); - - tuplesort_performsort(segment_tuplesortstate); - - recompress_segment(segment_tuplesortstate, uncompressed_chunk_rel, &row_compressor); - - /* now any pointers returned will be garbage */ - tuplesort_end(segment_tuplesortstate); - - decompress_segment_update_current_segment(current_segment, - slot, /*slot from compressed chunk*/ - decompressor.per_compressed_cols, - segmentby_column_offsets_compressed, - nsegmentby_cols); - /* reinit tuplesort and add the first tuple of the new segment to it */ - segment_tuplesortstate = tuplesort_begin_heap(uncompressed_rel_tupdesc, - n_keys, - sort_keys, - sort_operators, - sort_collations, - nulls_first, - maintenance_work_mem, - NULL, - false); - - bool should_free; - compressed_tuple = ExecFetchSlotHeapTuple(slot, false, &should_free); - - heap_deform_tuple(compressed_tuple, - compressed_rel_tupdesc, - decompressor.compressed_datums, - decompressor.compressed_is_nulls); - - row_decompressor_decompress_row(&decompressor, segment_tuplesortstate); - - simple_table_tuple_delete(compressed_chunk_rel, &(slot->tts_tid), snapshot); - /* because this is the first tuple of the new segment */ - changed_segment = false; - /* make changes visible */ - CommandCounterIncrement(); - - if (should_free) - heap_freetuple(compressed_tuple); - } - } - - ExecClearTuple(slot); - - /* never changed segment, but still need to perform the tuplesort and add everything to the - * compressed chunk - * the current segment could not be initialized in the case where two recompress operations - * execute concurrently: one blocks on the Exclusive lock but has already read the chunk - * status and determined that there is data in the uncompressed chunk */ - if (!changed_segment && current_segment_init) - { - fetch_matching_uncompressed_chunk_into_tuplesort(segment_tuplesortstate, - nsegmentby_cols, - uncompressed_chunk_rel, - current_segment); - tuplesort_performsort(segment_tuplesortstate); - recompress_segment(segment_tuplesortstate, uncompressed_chunk_rel, &row_compressor); - tuplesort_end(segment_tuplesortstate); - - CommandCounterIncrement(); - } - /* done with the compressed chunk segments that had new entries in the uncompressed - but there could be rows inserted into the uncompressed that don't already have a corresponding - compressed segment, we need to compress those as well */ - segment_tuplesortstate = tuplesort_begin_heap(uncompressed_rel_tupdesc, - n_keys, - sort_keys, - sort_operators, - sort_collations, - nulls_first, - maintenance_work_mem, - NULL, - false); - - bool unmatched_rows_exist = false; - fetch_unmatched_uncompressed_chunk_into_tuplesort(segment_tuplesortstate, - uncompressed_chunk_rel, - &unmatched_rows_exist); - - if (unmatched_rows_exist) - { - tuplesort_performsort(segment_tuplesortstate); - row_compressor_append_sorted_rows(&row_compressor, - segment_tuplesortstate, - RelationGetDescr(uncompressed_chunk_rel)); - tuplesort_end(segment_tuplesortstate); - - /* make changes visible */ - CommandCounterIncrement(); - } - - after_size = ts_relation_size_impl(compressed_chunk->table_id); - /* the compression size statistics we are able to update and accurately report are: - * rowcount pre/post compression, - * compressed chunk sizes */ - compression_chunk_size_catalog_update_recompressed(uncompressed_chunk->fd.id, - compressed_chunk->fd.id, - &after_size, - row_compressor.rowcnt_pre_compression, - row_compressor.num_compressed_rows); - - row_compressor_finish(&row_compressor); - FreeBulkInsertState(decompressor.bistate); - ExecDropSingleTupleTableSlot(slot); - index_endscan(index_scan); - UnregisterSnapshot(snapshot); - index_close(index_rel, AccessExclusiveLock); + recompress_partial_chunks(uncompressed_chunk, compressed_chunk); #if PG14_LT int options = 0; diff --git a/tsl/src/compression/compression.c b/tsl/src/compression/compression.c index e833ff4cc54..d18e1a65ae0 100644 --- a/tsl/src/compression/compression.c +++ b/tsl/src/compression/compression.c @@ -143,11 +143,11 @@ static bool row_compressor_new_row_is_in_new_group(RowCompressor *row_compressor static void row_compressor_append_row(RowCompressor *row_compressor, TupleTableSlot *row); static void row_compressor_flush(RowCompressor *row_compressor, CommandId mycid, bool changed_groups); - static int create_segment_filter_scankey(RowDecompressor *decompressor, char *segment_filter_col_name, StrategyNumber strategy, ScanKeyData *scankeys, int num_scankeys, - Bitmapset **null_columns, Datum value, bool isnull); + Bitmapset **null_columns, Datum value, bool is_null_check); + static void run_analyze_on_chunk(Oid chunk_relid); /******************** @@ -170,7 +170,7 @@ get_compressed_data_header(Datum data) * because the data remains, just in compressed form. Also don't * restart sequences. Use the transactional branch through ExecuteTruncate. */ -static void +void truncate_relation(Oid table_oid) { List *fks = heap_truncate_find_FKs(list_make1_oid(table_oid)); @@ -397,7 +397,8 @@ compress_chunk(Oid in_table, Oid out_table, const ColumnCompressionInfo **column in_column_offsets, out_desc->natts, true /*need_bistate*/, - false /*reset_sequence*/); + false /*reset_sequence*/, + COMPRESS); if (matched_index_rel != NULL) { @@ -834,7 +835,8 @@ void row_compressor_init(RowCompressor *row_compressor, TupleDesc uncompressed_tuple_desc, Relation compressed_table, int num_compression_infos, const ColumnCompressionInfo **column_compression_info, int16 *in_column_offsets, - int16 num_columns_in_compressed_table, bool need_bistate, bool reset_sequence) + int16 num_columns_in_compressed_table, bool need_bistate, bool reset_sequence, + CompressionType compressiontype) { TupleDesc out_desc = RelationGetDescr(compressed_table); int col; @@ -880,7 +882,9 @@ row_compressor_init(RowCompressor *row_compressor, TupleDesc uncompressed_tuple_ .num_compressed_rows = 0, .sequence_num = SEQUENCE_NUM_GAP, .reset_sequence = reset_sequence, + .update_sequence = true, .first_iteration = true, + .compression_type = compressiontype, }; memset(row_compressor->compressed_is_null, 1, sizeof(bool) * num_columns_in_compressed_table); @@ -1000,7 +1004,9 @@ row_compressor_process_ordered_slot(RowCompressor *row_compressor, TupleTableSlo row_compressor->first_iteration = false; } bool changed_groups = row_compressor_new_row_is_in_new_group(row_compressor, slot); - bool compressed_row_is_full = + bool compressed_row_is_full = false; + + compressed_row_is_full = row_compressor->rows_compressed_into_current_value >= MAX_ROWS_PER_COMPRESSION; if (compressed_row_is_full || changed_groups) { @@ -1009,7 +1015,6 @@ row_compressor_process_ordered_slot(RowCompressor *row_compressor, TupleTableSlo if (changed_groups) row_compressor_update_group(row_compressor, slot); } - row_compressor_append_row(row_compressor, slot); MemoryContextSwitchTo(old_ctx); ExecClearTuple(slot); @@ -1051,19 +1056,19 @@ row_compressor_update_group(RowCompressor *row_compressor, TupleTableSlot *row) * many segmentby columns. * */ - if (row_compressor->reset_sequence) - row_compressor->sequence_num = SEQUENCE_NUM_GAP; /* Start sequence from beginning */ - else - row_compressor->sequence_num = - get_sequence_number_for_current_group(row_compressor->compressed_table, - row_compressor->index_oid, - row_compressor - ->uncompressed_col_to_compressed_col, - row_compressor->per_column, - row_compressor->n_input_columns, - AttrOffsetGetAttrNumber( - row_compressor - ->sequence_num_metadata_column_offset)); + if (row_compressor->update_sequence) + { + if (row_compressor->reset_sequence) + row_compressor->sequence_num = SEQUENCE_NUM_GAP; /* Start sequence from beginning */ + else + row_compressor->sequence_num = get_sequence_number_for_current_group( + row_compressor->compressed_table, + row_compressor->index_oid, + row_compressor->uncompressed_col_to_compressed_col, + row_compressor->per_column, + row_compressor->n_input_columns, + AttrOffsetGetAttrNumber(row_compressor->sequence_num_metadata_column_offset)); + } } static bool @@ -1203,7 +1208,10 @@ row_compressor_flush(RowCompressor *row_compressor, CommandId mycid, bool change if (row_compressor->sequence_num > PG_INT32_MAX - SEQUENCE_NUM_GAP) elog(ERROR, "sequence id overflow"); - row_compressor->sequence_num += SEQUENCE_NUM_GAP; + if (row_compressor->compression_type == RECOMPRESS) + row_compressor->sequence_num += 1; + else + row_compressor->sequence_num += SEQUENCE_NUM_GAP; compressed_tuple = heap_form_tuple(RelationGetDescr(row_compressor->compressed_table), row_compressor->compressed_values, @@ -1848,7 +1856,7 @@ compression_get_toast_storage(CompressionAlgorithms algorithm) * Build scankeys for decompression of specific batches. key_columns references the * columns of the uncompressed chunk. */ -static ScanKeyData * +ScanKeyData * build_scankeys(int32 hypertable_id, Oid hypertable_relid, RowDecompressor decompressor, Bitmapset *key_columns, Bitmapset **null_columns, TupleTableSlot *slot, int *num_scankeys) diff --git a/tsl/src/compression/compression.h b/tsl/src/compression/compression.h index fbfd6e49b1d..5e28ee47de0 100644 --- a/tsl/src/compression/compression.h +++ b/tsl/src/compression/compression.h @@ -7,6 +7,7 @@ #define TIMESCALEDB_TSL_COMPRESSION_COMPRESSION_H #include +#include #include #include #include @@ -195,6 +196,13 @@ typedef enum CompressionAlgorithms _MAX_NUM_COMPRESSION_ALGORITHMS = 128, } CompressionAlgorithms; +/* Defines what kind of compression operation it is */ +typedef enum CompressionType +{ + COMPRESS = 0, + RECOMPRESS, +} CompressionType; + typedef struct CompressionStats { int64 rowcnt_pre_compression; @@ -260,8 +268,17 @@ typedef struct RowCompressor int64 num_compressed_rows; /* if recompressing segmentwise, we use this info to reset the sequence number */ bool reset_sequence; + /* + * During recompression, + * set to FALSE when existing compressed segments are being recompressed + * set to TRUE when new compressed segments are being inserted into heap + * This will ensure that sequence number are generated/updated correctly + */ + bool update_sequence; /* flag for checking if we are working on the first tuple */ bool first_iteration; + /* Represents the kind of compression operation */ + CompressionType compression_type; } RowCompressor; /* SegmentFilter is used for filtering segments based on qualifiers */ @@ -351,7 +368,8 @@ extern void row_compressor_init(RowCompressor *row_compressor, TupleDesc uncompr Relation compressed_table, int num_compression_infos, const ColumnCompressionInfo **column_compression_info, int16 *column_offsets, int16 num_columns_in_compressed_table, - bool need_bistate, bool reset_sequence); + bool need_bistate, bool reset_sequence, + CompressionType compressiontype); extern void row_compressor_finish(RowCompressor *row_compressor); extern void populate_per_compressed_columns_from_data(PerCompressedColumn *per_compressed_cols, int16 num_cols, Datum *compressed_datums, @@ -391,6 +409,11 @@ consumeCompressedData(StringInfo si, int bytes) return result; } +void truncate_relation(Oid table_oid); + +ScanKeyData *build_scankeys(int32 hypertable_id, Oid hypertable_relid, RowDecompressor decompressor, + Bitmapset *key_columns, Bitmapset **null_columns, TupleTableSlot *slot, + int *num_scankeys); /* * Normal compression uses 1k rows, but the regression tests use up to 1015. * We use this limit for sanity checks in case the compressed data is corrupt. diff --git a/tsl/src/compression/recompression.c b/tsl/src/compression/recompression.c new file mode 100644 index 00000000000..61348be75fc --- /dev/null +++ b/tsl/src/compression/recompression.c @@ -0,0 +1,1421 @@ +/* + * This file and its contents are licensed under the Timescale License. + * Please see the included NOTICE for copyright information and + * LICENSE-TIMESCALE for a copy of the license. + */ + +#include +#include +#include +#include +#include +#include + +#include "recompression.h" +#include "compression.h" +#include "hypertable_cache.h" +#include "ts_catalog/hypertable_compression.h" +#include "create.h" + +typedef struct CompressTuplesortstateCxt +{ + Tuplesortstate *tuplestore; + AttrNumber *sort_keys; + Oid *sort_operators; + Oid *sort_collations; + bool *nulls_first; +} CompressTuplesortstateCxt; + +/* + * Placeholder to save all context related to compressed chunk + */ +typedef struct CompressedChunkStats +{ + /* total no: of uncompressed rows */ + unsigned long num_rows_pre_compression; + /* total no: of compressed rows */ + unsigned long num_rows_post_compression; + /* total count of gaps between compressed rows */ + unsigned long num_gaps_in_sequences; + /* total count of gaps between compressed rows generated by compress_chunk() API */ + unsigned long num_expected_gaps_in_sequences; + /* total unique segmentby column values */ + unsigned long num_segments; + /* total number of compressed batches per unique segmentby column values */ + unsigned long num_batches_per_segments[MAX_ROWS_PER_COMPRESSION]; +} CompressedChunkStats; + +/* + * Placeholder to save all context related to recompression of partial chunks + */ +typedef struct AffectedSegmentsCxt +{ + /* holds matching compressed tuples */ + HeapTuple *compressed_tuples; + /* holds total number of uncompressed tuples which match the compressed tuple */ + unsigned long *num_matching_uncompressedrows; + /* holds total affected compressed segments */ + unsigned long total_affected_compressed_rows; + /* flag to know if new segments are found in uncompressed chunk */ + bool new_segments_found; + /* holds total number of uncompressed tuples with same segmentby column values */ + unsigned long *num_non_matching_compressedrows; + /* holds total affected new segmentby column values */ + unsigned long total_new_compressed_segments; + /* default size of above arrays */ + unsigned long default_size; + /* default size of above arrays */ + unsigned long new_segments_default_size; +} AffectedSegmentsCxt; + +/* + * Helper method to resize AffectedSegmentsCxt when total affected + * compressed tuples exceeds its default value of MAX_ROWS_PER_COMPRESSION + * This method will double the size of arrays present in AffectedSegmentsCxt + * and moves contents of existing AffectedSegmentsCxt to new context. + */ +static AffectedSegmentsCxt * +resize_affectedsegmentscxt(AffectedSegmentsCxt *as) +{ + AffectedSegmentsCxt *new_cxt = (AffectedSegmentsCxt *) palloc0(sizeof(AffectedSegmentsCxt)); + unsigned long factor = 0; + unsigned long new_size = 0; + + if (!as->new_segments_found) + { + factor = (as->default_size / MAX_ROWS_PER_COMPRESSION) + 1; + new_size = factor * MAX_ROWS_PER_COMPRESSION; + /* copy only affected compressed segments context */ + new_cxt->num_matching_uncompressedrows = palloc0(sizeof(unsigned long) * new_size); + memmove(new_cxt->num_matching_uncompressedrows, + as->num_matching_uncompressedrows, + sizeof(unsigned long) * as->default_size); + pfree(as->num_matching_uncompressedrows); + new_cxt->compressed_tuples = palloc0(sizeof(unsigned long) * new_size); + memmove(new_cxt->compressed_tuples, + as->compressed_tuples, + sizeof(unsigned long) * as->default_size); + pfree(as->compressed_tuples); + new_cxt->num_non_matching_compressedrows = as->num_non_matching_compressedrows; + new_cxt->default_size = new_size; + new_cxt->new_segments_default_size = as->new_segments_default_size; + } + else + { + factor = (as->new_segments_default_size / MAX_ROWS_PER_COMPRESSION) + 1; + new_size = factor * MAX_ROWS_PER_COMPRESSION; + /* copy all new compressed segments context */ + new_cxt->num_non_matching_compressedrows = palloc0(sizeof(unsigned long) * new_size); + memmove(new_cxt->num_non_matching_compressedrows, + as->num_non_matching_compressedrows, + sizeof(unsigned long) * as->new_segments_default_size); + pfree(as->num_non_matching_compressedrows); + new_cxt->num_matching_uncompressedrows = as->num_matching_uncompressedrows; + new_cxt->compressed_tuples = as->compressed_tuples; + new_cxt->new_segments_default_size = new_size; + new_cxt->default_size = as->default_size; + } + /* copy rest of the contents */ + new_cxt->total_affected_compressed_rows = as->total_affected_compressed_rows; + new_cxt->new_segments_found = as->new_segments_found; + new_cxt->total_new_compressed_segments = as->total_new_compressed_segments; + pfree(as); + return new_cxt; +} + +/* Helper method to update current segment which is being recompressed */ +static void +update_current_segment(CompressedSegmentInfo **current_segment, TupleTableSlot *slot, + const int nsegmentby_cols) +{ + Datum val; + bool is_null; + int seg_idx = 0; + for (int i = 0; i < nsegmentby_cols; i++) + { + int16 col_offset = current_segment[i]->decompressed_chunk_offset; + val = slot_getattr(slot, AttrOffsetGetAttrNumber(col_offset), &is_null); + if (!segment_info_datum_is_in_group(current_segment[seg_idx++]->segment_info, val, is_null)) + { + /* new segment, need to do per-segment processing */ + pfree(current_segment[seg_idx - 1]->segment_info); /* because increased previously */ + SegmentInfo *segment_info = + segment_info_new(TupleDescAttr(slot->tts_tupleDescriptor, col_offset)); + segment_info_update(segment_info, val, is_null); + current_segment[seg_idx - 1]->segment_info = segment_info; + current_segment[seg_idx - 1]->decompressed_chunk_offset = col_offset; + } + } +} + +/* Helper method to find if segment being recompressed, has encountered a new segment */ +static bool +is_segment_changed(CompressedSegmentInfo **current_segment, TupleTableSlot *slot, + const int nsegmentby_cols) +{ + Datum val; + bool is_null; + bool changed_segment = false; + for (int i = 0; i < nsegmentby_cols; i++) + { + int16 col_offset = current_segment[i]->decompressed_chunk_offset; + val = slot_getattr(slot, AttrOffsetGetAttrNumber(col_offset), &is_null); + if (!segment_info_datum_is_in_group(current_segment[i++]->segment_info, val, is_null)) + { + changed_segment = true; + break; + } + } + return changed_segment; +} + +/* This is a wrapper around row_compressor_append_sorted_rows. */ +static void +recompress_segment(Tuplesortstate *tuplesortstate, const Relation compressed_chunk_rel, + RowCompressor *row_compressor) +{ + row_compressor_append_sorted_rows(row_compressor, + tuplesortstate, + RelationGetDescr(compressed_chunk_rel)); + /* make changes visible */ + CommandCounterIncrement(); +} + +/* + * This function updates catalog chunk compression statistics + * for an existing compressed chunk after it has been recompressed + * segmentwise in-place (as opposed to creating a new compressed chunk). + * Note that because of this it is not possible to accurately report + * the fields + * uncompressed_chunk_size, uncompressed_index_size, uncompressed_toast_size + * anymore, so these are not updated. + */ +static int +compression_chunk_size_catalog_update_recompressed(int32 uncompressed_chunk_id, + int32 compressed_chunk_id, + const RelationSize *recompressed_size, + int64 rowcnt_pre_compression, + int64 rowcnt_post_compression) +{ + ScanIterator iterator = + ts_scan_iterator_create(COMPRESSION_CHUNK_SIZE, RowExclusiveLock, CurrentMemoryContext); + bool updated = false; + + iterator.ctx.index = + catalog_get_index(ts_catalog_get(), COMPRESSION_CHUNK_SIZE, COMPRESSION_CHUNK_SIZE_PKEY); + ts_scan_iterator_scan_key_init(&iterator, + Anum_compression_chunk_size_pkey_chunk_id, + BTEqualStrategyNumber, + F_INT4EQ, + Int32GetDatum(uncompressed_chunk_id)); + ts_scanner_foreach(&iterator) + { + Datum values[Natts_compression_chunk_size]; + bool replIsnull[Natts_compression_chunk_size] = { false }; + bool repl[Natts_compression_chunk_size] = { false }; + bool should_free; + TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); + HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); + HeapTuple new_tuple = NULL; + heap_deform_tuple(tuple, ts_scanner_get_tupledesc(ti), values, replIsnull); + + /* Only update the information pertaining to the compressed chunk */ + /* these fields are about the compressed chunk so they can be updated */ + values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_heap_size)] = + Int64GetDatum(recompressed_size->heap_size); + repl[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_heap_size)] = true; + + values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_toast_size)] = + Int64GetDatum(recompressed_size->toast_size); + repl[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_toast_size)] = true; + + values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_index_size)] = + Int64GetDatum(recompressed_size->index_size); + repl[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_index_size)] = true; + + values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_numrows_pre_compression)] = + Int64GetDatum(rowcnt_pre_compression); + repl[AttrNumberGetAttrOffset(Anum_compression_chunk_size_numrows_pre_compression)] = true; + + values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_numrows_post_compression)] = + Int64GetDatum(rowcnt_post_compression); + repl[AttrNumberGetAttrOffset(Anum_compression_chunk_size_numrows_post_compression)] = true; + + new_tuple = + heap_modify_tuple(tuple, ts_scanner_get_tupledesc(ti), values, replIsnull, repl); + ts_catalog_update(ti->scanrel, new_tuple); + heap_freetuple(new_tuple); + + if (should_free) + heap_freetuple(tuple); + + updated = true; + break; + } + + ts_scan_iterator_end(&iterator); + ts_scan_iterator_close(&iterator); + return updated; +} + +/* + * Helper method to get statistics of compressed chunk, before recompression. + */ +static void +get_compressed_chunk_statistics(const Relation in_rel, const Oid index_oid, + CompressedChunkStats *stats) +{ + Relation idxrel; + IndexScanDesc index_scan; + TupleTableSlot *heap_tuple_slot = NULL; + AttrNumber metacount_attnum; + AttrNumber seq_attnum; + Datum curr_seqnum_val = SEQUENCE_NUM_GAP; + Datum prev_seqnum_val = SEQUENCE_NUM_GAP; + + unsigned long total_batches_per_segment = 0; + unsigned long total_segments = 0; + + idxrel = index_open(index_oid, AccessShareLock); + index_scan = index_beginscan(in_rel, idxrel, GetTransactionSnapshot(), 0, 0); + index_rescan(index_scan, NULL, 0, NULL, 0); + heap_tuple_slot = table_slot_create(in_rel, NULL); + metacount_attnum = get_attnum(in_rel->rd_id, COMPRESSION_COLUMN_METADATA_COUNT_NAME); + seq_attnum = get_attnum(in_rel->rd_id, COMPRESSION_COLUMN_METADATA_SEQUENCE_NUM_NAME); + + while (index_getnext_slot(index_scan, ForwardScanDirection, heap_tuple_slot)) + { + slot_getallattrs(heap_tuple_slot); + stats->num_rows_pre_compression += heap_tuple_slot->tts_values[metacount_attnum - 1]; + stats->num_rows_post_compression++; + curr_seqnum_val = heap_tuple_slot->tts_values[seq_attnum - 1]; + if (curr_seqnum_val > SEQUENCE_NUM_GAP) + { + stats->num_gaps_in_sequences += ((curr_seqnum_val - prev_seqnum_val) - 1); + stats->num_expected_gaps_in_sequences += 9; + } + + if (prev_seqnum_val > curr_seqnum_val) + { + /* there was change in segments */ + prev_seqnum_val = curr_seqnum_val = SEQUENCE_NUM_GAP; + stats->num_batches_per_segments[total_segments] = total_batches_per_segment; + total_segments++; + total_batches_per_segment = 1; + } + else + { + total_batches_per_segment++; + prev_seqnum_val = curr_seqnum_val; + } + } + index_endscan(index_scan); + index_close(idxrel, AccessShareLock); + ExecDropSingleTupleTableSlot(heap_tuple_slot); + + stats->num_segments = total_segments + 1; + stats->num_batches_per_segments[total_segments] = total_batches_per_segment; +} + +/* Helper method to initial the first segment by column values in current_segment */ +static void +initialize_segment(const RowDecompressor *decompressor, CompressedSegmentInfo **current_segment, + const Chunk *uncompressed_chunk, TupleTableSlot *uncompressed_slot) +{ + int i = 0; + Datum val; + bool is_null; + SegmentInfo *segment_info = NULL; + /* initialize current segment */ + for (int col = 0; col < uncompressed_slot->tts_tupleDescriptor->natts; col++) + { + if (uncompressed_slot->tts_tupleDescriptor->attrs[col].attisdropped) + continue; + val = slot_getattr(uncompressed_slot, AttrOffsetGetAttrNumber(col), &is_null); + Form_pg_attribute decompressed_attr = TupleDescAttr(decompressor->out_desc, col); + char *col_name = NameStr(decompressed_attr->attname); + FormData_hypertable_compression *fd = + ts_hypertable_compression_get_by_pkey(uncompressed_chunk->fd.hypertable_id, col_name); + if (COMPRESSIONCOL_IS_SEGMENT_BY(fd)) + { + segment_info = + segment_info_new(TupleDescAttr(uncompressed_slot->tts_tupleDescriptor, col)); + current_segment[i]->decompressed_chunk_offset = col; + /* also need to call segment_info_update here to update the val part */ + segment_info_update(segment_info, val, is_null); + current_segment[i]->segment_info = segment_info; + i++; + } + } +} + +/* initialize tuplesort state */ +static CompressTuplesortstateCxt * +initialize_tuplestore(const Relation uncompressed_chunk_rel, const ColumnCompressionInfo **keys, + const int n_keys) +{ + CompressTuplesortstateCxt *tuplestorecxt = NULL; + int tuplesortopts = 0; +#if PG15_GE + tuplesortopts = TUPLESORT_NONE | TUPLESORT_RANDOMACCESS; +#else + tuplesortopts = 1; +#endif + tuplestorecxt = (CompressTuplesortstateCxt *) palloc0(sizeof(CompressTuplesortstateCxt)); + tuplestorecxt->sort_keys = palloc(sizeof(AttrNumber) * n_keys); + tuplestorecxt->sort_operators = palloc(sizeof(Oid) * n_keys); + tuplestorecxt->sort_collations = palloc(sizeof(Oid) * n_keys); + tuplestorecxt->nulls_first = palloc(sizeof(bool) * n_keys); + + for (int n = 0; n < n_keys; n++) + compress_chunk_populate_sort_info_for_column(RelationGetRelid(uncompressed_chunk_rel), + keys[n], + &tuplestorecxt->sort_keys[n], + &tuplestorecxt->sort_operators[n], + &tuplestorecxt->sort_collations[n], + &tuplestorecxt->nulls_first[n]); + + tuplestorecxt->tuplestore = tuplesort_begin_heap(RelationGetDescr(uncompressed_chunk_rel), + n_keys, + tuplestorecxt->sort_keys, + tuplestorecxt->sort_operators, + tuplestorecxt->sort_collations, + tuplestorecxt->nulls_first, + maintenance_work_mem, + NULL, + tuplesortopts); + return tuplestorecxt; +} + +/* cleaup tuplesort state */ +static void +cleanup_tuplestorecxt(CompressTuplesortstateCxt *tuplestorecxt) +{ + if (tuplestorecxt) + { + tuplesort_end(tuplestorecxt->tuplestore); + pfree(tuplestorecxt->sort_keys); + pfree(tuplestorecxt->sort_operators); + pfree(tuplestorecxt->sort_collations); + pfree(tuplestorecxt->nulls_first); + pfree(tuplestorecxt); + tuplestorecxt = NULL; + } +} + +/* + * Helper method to fetch all rows from uncompressed chunk + */ +static int +save_uncompressed_rows_in_tuplestore(const Relation uncompressed_chunk_rel, + Tuplesortstate *uncompressed_rows_sortstate) +{ + TableScanDesc heapScan; + TupleTableSlot *heap_tuple_slot = NULL; + int total_uncompressed_rows = 0; + + heapScan = table_beginscan(uncompressed_chunk_rel, GetLatestSnapshot(), 0, NULL); + heap_tuple_slot = table_slot_create(uncompressed_chunk_rel, NULL); + + while (table_scan_getnextslot(heapScan, ForwardScanDirection, heap_tuple_slot)) + { + slot_getallattrs(heap_tuple_slot); + tuplesort_puttupleslot(uncompressed_rows_sortstate, heap_tuple_slot); + total_uncompressed_rows++; + } + truncate_relation(uncompressed_chunk_rel->rd_id); + ExecDropSingleTupleTableSlot(heap_tuple_slot); + table_endscan(heapScan); + return total_uncompressed_rows; +} + +/* + * Helper method used to build scankeys based on segmentby and orderby + * key columns to do lookup into uncompressed chunks. + */ +static ScanKeyData * +build_segment_order_by_scankeys(const RowDecompressor *decompressor, + const Chunk *uncompressed_chunk, TupleTableSlot *slot, + Bitmapset **null_columns_count, int *num_scankeys, + const int key_flags) +{ + ScanKeyData *segment_order_by_keys; + Bitmapset *key_columns = NULL; + Bitmapset *null_columns = NULL; + int num_keys = 0; + for (int i = 0; i < decompressor->out_desc->natts; i++) + { + Form_pg_attribute attr = TupleDescAttr(decompressor->out_desc, i); + if (attr->attisdropped) + continue; + FormData_hypertable_compression *fd = + ts_hypertable_compression_get_by_pkey(uncompressed_chunk->fd.hypertable_id, + attr->attname.data); + if (key_flags & SEGMENTBY_KEYS) + { + if (COMPRESSIONCOL_IS_SEGMENT_BY(fd)) + key_columns = + bms_add_member(key_columns, attr->attnum - FirstLowInvalidHeapAttributeNumber); + } + if (key_flags & ORDERBY_KEYS) + { + if (COMPRESSIONCOL_IS_ORDER_BY(fd)) + key_columns = + bms_add_member(key_columns, attr->attnum - FirstLowInvalidHeapAttributeNumber); + } + } + segment_order_by_keys = build_scankeys(uncompressed_chunk->fd.hypertable_id, + uncompressed_chunk->table_id, + *decompressor, + key_columns, + &null_columns, + slot, + &num_keys); + if (null_columns_count) + *null_columns_count = null_columns; + bms_free(key_columns); + *num_scankeys = num_keys; + return segment_order_by_keys; +} + +/* + * Helper method to check if segmentby columns in compressed/uncompressed + * tuples have NULL or not. + * + * Return true if all nullable segmentby columns have NULL values, else + * return false. + */ +static bool +has_all_nulls(const RowDecompressor *decompressor, const HeapTuple compressed_tuple, + const HeapTuple uncompressed_tuple, const Bitmapset *null_columns) +{ + bool nulls_found = true; + for (int attno = bms_next_member(null_columns, -1); attno >= 0; + attno = bms_next_member(null_columns, attno)) + { + if (!(heap_attisnull(compressed_tuple, attno, decompressor->in_desc))) + { + nulls_found = false; + break; + } + if (!(heap_attisnull(uncompressed_tuple, attno, decompressor->out_desc))) + { + nulls_found = false; + break; + } + } + return nulls_found; +} + +/* + * Helper method to build scankeys to do lookup into compressed chunk index + */ +static ScanKeyData * +build_index_scankeys(const Relation in_rel, const Relation idxrel, + const TupleTableSlot *compressedslot, const TupleTableSlot *uncompressedslot, + int *num_keys, const int key_flags) +{ + ScanKeyData *scankey = NULL; + int indnkeyatts; + int total_keys = 0; + int attoff; + bool isnull; + Datum indclassDatum; + oidvector *opclass; + int2vector *indkey = NULL; + + indkey = &idxrel->rd_index->indkey; + indclassDatum = + SysCacheGetAttr(INDEXRELID, idxrel->rd_indextuple, Anum_pg_index_indclass, &isnull); + Assert(!isnull); + opclass = (oidvector *) DatumGetPointer(indclassDatum); + + indnkeyatts = IndexRelationGetNumberOfKeyAttributes(idxrel); + scankey = palloc0(sizeof(ScanKeyData) * indnkeyatts); + + /* Build scankey for every attribute in the index. */ + for (attoff = 0; attoff < indnkeyatts; attoff++) + { + Oid operator; + Oid opfamily; + RegProcedure regop; + int pkattno = attoff + 1; + int mainattno = indkey->values[attoff]; + Oid optype = get_opclass_input_type(opclass->values[attoff]); + /* + * Load the operator info. We need this to get the equality operator + * function for the scan key. + */ + opfamily = get_opclass_family(opclass->values[attoff]); + operator= get_opfamily_member(opfamily, optype, optype, BTEqualStrategyNumber); + regop = get_opcode(operator); + + if (attoff < indnkeyatts - 1) + { + /* Initialize segmentby scankeys. */ + if (key_flags & SEGMENTBY_KEYS) + { + if (compressedslot) + { + ScanKeyInit(&scankey[attoff], + pkattno, + BTEqualStrategyNumber, + regop, + compressedslot->tts_values[mainattno - 1]); + if (compressedslot->tts_isnull[mainattno - 1]) + scankey[attoff].sk_flags |= (SK_ISNULL | SK_SEARCHNULL); + scankey[attoff].sk_collation = idxrel->rd_indcollation[attoff]; + total_keys++; + } + else if (uncompressedslot) + { + /* get index attribute name */ + Form_pg_attribute attr = TupleDescAttr(idxrel->rd_att, attoff); + AttrNumber attno = get_attnum(in_rel->rd_id, attr->attname.data); + ScanKeyInit(&scankey[attoff], + pkattno, + BTEqualStrategyNumber, + regop, + uncompressedslot->tts_values[attno - 1]); + if (uncompressedslot->tts_isnull[attno - 1]) + scankey[attoff].sk_flags |= (SK_ISNULL | SK_SEARCHNULL); + scankey[attoff].sk_collation = idxrel->rd_indcollation[attoff]; + total_keys++; + } + } + } + else if (attoff == indnkeyatts - 1 && compressedslot) + { + /* Initialize _ts_meta_sequence_num scankeys. */ + if (key_flags & NEXT_SEQNUM_KEYS) + { + ScanKeyInit(&scankey[attoff], + pkattno, + BTEqualStrategyNumber, + regop, + compressedslot->tts_values[mainattno - 1] + SEQUENCE_NUM_GAP); + scankey[attoff].sk_collation = idxrel->rd_indcollation[attoff]; + total_keys++; + } + } + } + *num_keys = total_keys; + return scankey; +} + +/* + * Helper method to fetch next compressed tuple based on segmentby and + * _ts_meta_sequence_num column values from searchslot + */ +static bool +fetch_next_compressed_tuple_using_index(const Relation in_rel, const Oid index_oid, + const TupleTableSlot *searchslot, + HeapTuple *compressed_tuple) +{ + Relation idxrel; + IndexScanDesc index_scan; + ScanKeyData *scankey = NULL; + TupleTableSlot *index_slot = NULL; + int num_scankeys = 0; + bool ret = false; + + idxrel = index_open(index_oid, AccessShareLock); + scankey = build_index_scankeys(in_rel, + idxrel, + searchslot, + NULL, + &num_scankeys, + SEGMENTBY_KEYS | NEXT_SEQNUM_KEYS); + index_scan = index_beginscan(in_rel, idxrel, GetTransactionSnapshot(), num_scankeys, 0); + index_rescan(index_scan, scankey, num_scankeys, NULL, 0); + index_slot = table_slot_create(in_rel, NULL); + if (index_getnext_slot(index_scan, ForwardScanDirection, index_slot)) + { + slot_getallattrs(index_slot); + /* found valid tuple */ + *compressed_tuple = heap_form_tuple(index_slot->tts_tupleDescriptor, + index_slot->tts_values, + index_slot->tts_isnull); + (*compressed_tuple)->t_self = index_slot->tts_tid; + ret = true; + } + else + { + compressed_tuple = NULL; + } + ExecDropSingleTupleTableSlot(index_slot); + index_endscan(index_scan); + index_close(idxrel, AccessShareLock); + return ret; +} + +static bool +fetch_next_compressed_tuple(const RowDecompressor *decompressor, + const RowCompressor *row_compressor, const Chunk *uncompressed_chunk, + const int nsegmentby_cols, TupleTableSlot *compressed_slot, + HeapTuple *next_tuple) +{ + bool tuple_found = false; + HeapTuple next_compressed_tuple = NULL; + if (OidIsValid(row_compressor->index_oid)) + tuple_found = fetch_next_compressed_tuple_using_index(decompressor->in_rel, + row_compressor->index_oid, + compressed_slot, + &next_compressed_tuple); + *next_tuple = next_compressed_tuple; + return tuple_found; +} + +/* + * In a compressed chunk there can be multiple compressed rows with same segment + * by values and incrementing sequence numbers. This method will tell if the + * given tuple (searchslot) is the last tuple with highest sequence number. + * + * Return true if searchslot is the last tuple else return false. + */ +static bool +is_compressed_tuple_in_last_batch(const Relation in_rel, const Oid index_oid, + const TupleTableSlot *searchslot) +{ + Relation idxrel; + IndexScanDesc index_scan; + ScanKeyData *scankey = NULL; + TupleTableSlot *index_slot; + int num_scankeys = 0; + bool ret = false; + + idxrel = index_open(index_oid, AccessShareLock); + /* Build scankey for every attribute in the index. */ + scankey = build_index_scankeys(in_rel, + idxrel, + searchslot, + NULL, + &num_scankeys, + SEGMENTBY_KEYS | NEXT_SEQNUM_KEYS); + index_scan = index_beginscan(in_rel, idxrel, GetTransactionSnapshot(), num_scankeys, 0); + index_rescan(index_scan, scankey, num_scankeys, NULL, 0); + index_slot = table_slot_create(in_rel, NULL); + ret = index_getnext_slot(index_scan, ForwardScanDirection, index_slot); + ExecDropSingleTupleTableSlot(index_slot); + index_endscan(index_scan); + index_close(idxrel, AccessShareLock); + return !ret; +} + +/* + * Helper method to handle all uncompressed rows which has + * segmentby columns with new values. + */ +static void +process_new_segments(const RowDecompressor *decompressor, const Chunk *uncompressed_chunk, + Tuplesortstate *tuplestore, AffectedSegmentsCxt **affectedsegments, + const int nsegmentby_cols) +{ + CompressedSegmentInfo **current_segment = NULL; + TupleTableSlot *uncompressed_slot = NULL; + unsigned int num_non_matching_compressedrows = 1; + unsigned int new_segment_count = 0; + uncompressed_slot = MakeTupleTableSlot(decompressor->out_desc, &TTSOpsMinimalTuple); + + while (tuplesort_gettupleslot(tuplestore, + true /*=forward*/, + false /*=copy*/, + uncompressed_slot, + NULL /*=abbrev*/)) + { + slot_getallattrs(uncompressed_slot); + if (!current_segment) + { + /* initialize segment */ + current_segment = palloc(sizeof(CompressedSegmentInfo *) * nsegmentby_cols); + for (int i = 0; i < nsegmentby_cols; i++) + current_segment[i] = palloc(sizeof(CompressedSegmentInfo)); + initialize_segment(decompressor, + current_segment, + uncompressed_chunk, + uncompressed_slot); + } + num_non_matching_compressedrows++; + /* check if there is change in segmentby value */ + bool is_new_segment = + is_segment_changed(current_segment, uncompressed_slot, nsegmentby_cols); + + if (is_new_segment) + { + /* update new segment */ + update_current_segment(current_segment, uncompressed_slot, nsegmentby_cols); + if (new_segment_count >= (*affectedsegments)->new_segments_default_size) + (*affectedsegments) = resize_affectedsegmentscxt((*affectedsegments)); + (*affectedsegments)->total_new_compressed_segments++; + (*affectedsegments)->num_non_matching_compressedrows[new_segment_count++] = + num_non_matching_compressedrows - 1; + num_non_matching_compressedrows = 1; + } + } + ExecDropSingleTupleTableSlot(uncompressed_slot); + if (new_segment_count >= (*affectedsegments)->new_segments_default_size) + (*affectedsegments) = resize_affectedsegmentscxt((*affectedsegments)); + /* last row present in tuplestore should be handled */ + (*affectedsegments)->total_new_compressed_segments++; + (*affectedsegments)->num_non_matching_compressedrows[new_segment_count++] = + num_non_matching_compressedrows; + num_non_matching_compressedrows = 0; +} + +/* + * Helper method to fetch all affected compressed rows from compressed chunk + * This method will do an indexscan on compressed chunk to fetch the required + * compressed rows. + */ +static void +save_affected_compressed_rows_using_indexscan( + const RowDecompressor *decompressor, const RowCompressor *row_compressor, + const Chunk *compressed_chunk, const Chunk *uncompressed_chunk, + const CompressTuplesortstateCxt *uncompressed_tuplestorecxt, + AffectedSegmentsCxt **affectedsegments, const int nsegmentby_cols) +{ + Relation index_rel; + IndexScanDesc index_scan; + HeapTuple compressed_tuple = NULL; + HeapTuple uncompressed_tuple = NULL; + TupleTableSlot *uncompressed_slot = NULL; + TupleTableSlot *compressed_slot = NULL; + ScanKeyData *segment_by_keys = NULL; + ScanKeyData *segment_order_by_keys = NULL; + + int num_segkeys = 0; + int num_scankeys = 0; + unsigned int num_uncompressedrows = 0; + unsigned int uncompressed_row_count = 0; + + bool compressed_tuple_found = false; + bool segment_exists = false; + + /* Index scan */ + index_rel = index_open(row_compressor->index_oid, AccessShareLock); + index_scan = index_beginscan(decompressor->in_rel, + index_rel, + GetTransactionSnapshot(), + nsegmentby_cols, + 0); + + compressed_slot = table_slot_create(decompressor->in_rel, NULL); + uncompressed_slot = MakeTupleTableSlot(decompressor->out_desc, &TTSOpsMinimalTuple); + + /* + * 1. For every uncompressed row do following: + * A. If fetched compressed tuple (CR) from (iii) exists: + * - If uncompressed row matching CR goto step1. + * - If uncompressed row does not match, save total + * - fetched uncompressed row count and CR in AffectedSegmentsCxt + * B. Scan the compressed index + * i. Fetch the compressed tuple based on segmentby values + * ii. If matching segmentby tuple is found check the orderby values + * iii. Save the matching segmentby/orderby compressed row (CR) + * iv. If segmentby values does not match goto stepB + * v. If none of compressed rows match the segmentby values mark + * new_segments_found = true and process all new segmentby uncompressed + * rows. + * C. Handle all corner cases as part of steps A,B + */ + while (tuplesort_gettupleslot(uncompressed_tuplestorecxt->tuplestore, + true /*=forward*/, + false /*=copy*/, + uncompressed_slot, + NULL /*=abbrev*/)) + { + Bitmapset *null_columns = NULL; + ScanKeyData *idx_scankey = NULL; + int num_idx_scankeys = 0; + + num_uncompressedrows++; + slot_getallattrs(uncompressed_slot); + + if (segment_by_keys) + pfree(segment_by_keys); + if (segment_order_by_keys) + pfree(segment_order_by_keys); + /* build scan keys to do lookup into uncompressed chunks */ + segment_by_keys = build_segment_order_by_scankeys(decompressor, + uncompressed_chunk, + uncompressed_slot, + &null_columns, + &num_segkeys, + SEGMENTBY_KEYS); + segment_order_by_keys = build_segment_order_by_scankeys(decompressor, + uncompressed_chunk, + uncompressed_slot, + NULL, + &num_scankeys, + SEGMENTBY_KEYS | ORDERBY_KEYS); + if (compressed_tuple_found) + { + if (null_columns) + { + /* + * we cannot use HeapKeyTest to check for NULLS, thus perform + * manual checks by calling has_all_nulls() + */ + uncompressed_tuple = ExecCopySlotHeapTuple(uncompressed_slot); + /* all nullable segmentby columns have matching NULL values */ + bool all_nulls = + has_all_nulls(decompressor, compressed_tuple, uncompressed_tuple, null_columns); + heap_freetuple(uncompressed_tuple); + if (all_nulls) + continue; + } + /* reset the flags */ + segment_exists = false; + compressed_tuple_found = false; + if (num_segkeys) + { + HeapKeyTest(compressed_tuple, + RelationGetDescr(decompressor->in_rel), + num_segkeys, + segment_by_keys, + segment_exists); + HeapKeyTest(compressed_tuple, + RelationGetDescr(decompressor->in_rel), + num_scankeys, + segment_order_by_keys, + compressed_tuple_found); + } + if (!compressed_tuple_found) + { + /* + * current uncompressed tuple did not fit into previously fetched + * compressed tuple. + * If current uncompressed tuple belongs to last batch in + * current segment, then append these uncompressed tuples, else + * new batch gets created assuming that these are new segments. + */ + if (!is_compressed_tuple_in_last_batch(decompressor->in_rel, + row_compressor->index_oid, + compressed_slot) || + !segment_exists) + { + if (uncompressed_row_count >= (*affectedsegments)->default_size) + (*affectedsegments) = resize_affectedsegmentscxt((*affectedsegments)); + (*affectedsegments)->num_matching_uncompressedrows[uncompressed_row_count++] = + num_uncompressedrows - 1; + num_uncompressedrows = 1; + (*affectedsegments) + ->compressed_tuples[(*affectedsegments)->total_affected_compressed_rows] = + heap_copytuple(compressed_tuple); + heap_freetuple(compressed_tuple); + (*affectedsegments)->total_affected_compressed_rows++; + } + else + { + compressed_tuple_found = true; + continue; + } + } + else + continue; + } + else if (segment_exists) + { + /* + * In some cases, compressed tuples with current segmentby values exist + * however these tuples do not match the orderby values. ex: + * uncompressed rows: + * ('1999-12-30 15:30:00+05:30'::timestamp with time zone, 1, 2, 3, 22.2, 33.3); + * ('1999-12-31 05:30:00+05:30'::timestamp with time zone, 1, 2, 3, 2.2, 3.3); + * compressed rows: + * 1 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + * 1 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + * 1 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + * 1 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + * + * For these kind of situations we need to append these uncompressed rows to + * compressed tuple with sequence num = 40 + */ + + HeapKeyTest(compressed_tuple, + RelationGetDescr(decompressor->in_rel), + num_segkeys, + segment_by_keys, + segment_exists); + if (!segment_exists) + { + if (uncompressed_row_count >= (*affectedsegments)->default_size) + (*affectedsegments) = resize_affectedsegmentscxt((*affectedsegments)); + (*affectedsegments)->num_matching_uncompressedrows[uncompressed_row_count++] = + num_uncompressedrows - 1; + num_uncompressedrows = 1; + (*affectedsegments) + ->compressed_tuples[(*affectedsegments)->total_affected_compressed_rows] = + heap_copytuple(compressed_tuple); + heap_freetuple(compressed_tuple); + (*affectedsegments)->total_affected_compressed_rows++; + } + else + continue; + } + idx_scankey = build_index_scankeys(decompressor->out_rel, + index_rel, + NULL, + uncompressed_slot, + &num_idx_scankeys, + SEGMENTBY_KEYS); + index_rescan(index_scan, idx_scankey, num_idx_scankeys, NULL, 0); + segment_exists = false; + /* + * For every compressed tuple fetched which matches segmentby values: + * - check if uncompressed tuple matches the compressed tuple + * - If orderby matches, save this compressed tuple + * - If only segmentby matches, move to next compressed tuple + * - If no segmentby matches mark new_segments_found to true + */ + while (index_getnext_slot(index_scan, ForwardScanDirection, compressed_slot)) + { + segment_exists = true; + slot_getallattrs(compressed_slot); + compressed_tuple = ExecCopySlotHeapTuple(compressed_slot); + /* reset the flags */ + compressed_tuple_found = false; + /* + * check if current uncompressed tuples orderby columns match the + * min/max meta columns from this compressed tuple. If matches set + * compressed_tuple_found to true. + */ + HeapKeyTest(compressed_tuple, + RelationGetDescr(decompressor->in_rel), + num_scankeys, + segment_order_by_keys, + compressed_tuple_found); + /* + * Stop index scan for now and for this compressed tuple, find + * all matching uncompressed tuples and save this compressed tuple + * and total matching uncompressed tuple count in affectedsegments + */ + if (compressed_tuple_found) + break; + } + if (!segment_exists) + { + /* new segment found */ + (*affectedsegments)->new_segments_found = true; + break; + } + } + /* process all new segments uncompressed rows here */ + if ((*affectedsegments)->new_segments_found) + { + process_new_segments(decompressor, + uncompressed_chunk, + uncompressed_tuplestorecxt->tuplestore, + affectedsegments, + nsegmentby_cols); + } + else + { + if (uncompressed_row_count >= (*affectedsegments)->default_size) + (*affectedsegments) = resize_affectedsegmentscxt((*affectedsegments)); + /* last row present in tuplestore should be handled */ + (*affectedsegments)->num_matching_uncompressedrows[uncompressed_row_count++] = + num_uncompressedrows; + (*affectedsegments) + ->compressed_tuples[(*affectedsegments)->total_affected_compressed_rows] = + heap_copytuple(compressed_tuple); + heap_freetuple(compressed_tuple); + (*affectedsegments)->total_affected_compressed_rows++; + } + /* cleanup */ + ExecDropSingleTupleTableSlot(uncompressed_slot); + ExecDropSingleTupleTableSlot(compressed_slot); + index_endscan(index_scan); + index_close(index_rel, AccessShareLock); +} + +/* + * Helper method to decompress a compressed tuple. + * This method will return the sequence number of the current + * compressed tuple. + */ +static int +decompress_in_tuplestore(RowDecompressor *decompressor, const HeapTuple compressed_tuple, + Tuplesortstate *merge_tuplestore) +{ + Assert(HeapTupleIsValid(compressed_tuple)); + heap_deform_tuple(compressed_tuple, + decompressor->in_desc, + decompressor->compressed_datums, + decompressor->compressed_is_nulls); + /* fetch sequence number of current compressed tuple */ + AttrNumber seq_attnum = + get_attnum(decompressor->in_rel->rd_id, COMPRESSION_COLUMN_METADATA_SEQUENCE_NUM_NAME); + /* decompress found tuple and delete from compressed chunk */ + row_decompressor_decompress_row(decompressor, merge_tuplestore); + TM_FailureData tmfd; + TM_Result result pg_attribute_unused(); + result = table_tuple_delete(decompressor->in_rel, + &compressed_tuple->t_self, + GetCurrentCommandId(true), + GetTransactionSnapshot(), + InvalidSnapshot, + true, + &tmfd, + false); + Assert(result == TM_Ok); + /* make changes visible */ + CommandCounterIncrement(); + return decompressor->compressed_datums[seq_attnum - 1]; +} + +static void +recompress_affected_segments(RowDecompressor *decompressor, RowCompressor *row_compressor, + AffectedSegmentsCxt *affectedsegments, const Chunk *uncompressed_chunk, + const int nsegmentby_cols, + const CompressTuplesortstateCxt *uncompressed_tuplestorecxt, + const ColumnCompressionInfo **keys, const int n_keys) +{ + TupleTableSlot *compressed_slot = NULL; + TupleTableSlot *uncompressed_slot = NULL; + HeapTuple next_tuple = NULL; + unsigned int total_uncompressed_rows_per_segment = 0; + unsigned int compressed_row_count = 0; + Datum metacount_val; + AttrNumber meta_count; + + compressed_slot = MakeTupleTableSlot(decompressor->in_desc, &TTSOpsHeapTuple); + while (affectedsegments->total_affected_compressed_rows) + { + ExecStoreHeapTuple(affectedsegments->compressed_tuples[compressed_row_count], + compressed_slot, + false); + heap_deform_tuple(affectedsegments->compressed_tuples[compressed_row_count], + decompressor->in_desc, + compressed_slot->tts_values, + compressed_slot->tts_isnull); + meta_count = + get_attnum(decompressor->in_rel->rd_id, COMPRESSION_COLUMN_METADATA_COUNT_NAME); + metacount_val = compressed_slot->tts_values[meta_count - 1]; + + /* + * Calculate number of compressed batches to decompress in case + * there is an overflow: + * For a given compressed batch with sequence# 30, if there is an overflow, + * then we save rest of rows > 1000 in next sequence# which is 31. However + * if total_uncompressed_rows_per_segment is > (10 * MAX_ROWS_PER_COMPRESSION) + * then we run out of sequence numbers from (31 .. 39) thus we + * need to decompress the next segment and repeat the same until we + * see that all uncompressed rows fits in. + * + * (total_uncompressed_rows_per_segment + meta_count) / (10 * MAX_ROWS_PER_COMPRESSION) + * should give us, how many following compressed batches to decompress. + * + * Based on the calculated value, we fetch next compressed batches and perform + * recompression. + */ + total_uncompressed_rows_per_segment = + affectedsegments->num_matching_uncompressedrows[compressed_row_count]; + unsigned int total_batches_to_decompress = + ((total_uncompressed_rows_per_segment + metacount_val) / + (10 * MAX_ROWS_PER_COMPRESSION)); + + CompressTuplesortstateCxt *tuplestorecxt = NULL; + tuplestorecxt = initialize_tuplestore(decompressor->out_rel, keys, n_keys); + uncompressed_slot = MakeTupleTableSlot(decompressor->out_desc, &TTSOpsMinimalTuple); + while (affectedsegments->num_matching_uncompressedrows[compressed_row_count] && + tuplesort_gettupleslot(uncompressed_tuplestorecxt->tuplestore, + true /*=forward*/, + false /*=copy*/, + uncompressed_slot, + NULL /*=abbrev*/)) + { + tuplesort_puttupleslot(tuplestorecxt->tuplestore, uncompressed_slot); + affectedsegments->num_matching_uncompressedrows[compressed_row_count]--; + } + ExecDropSingleTupleTableSlot(uncompressed_slot); + if (next_tuple) + { + /* this compressed tuple might have already been decompressed */ + ItemPointer ptr1 = &(next_tuple)->t_self; + ItemPointer ptr2 = &affectedsegments->compressed_tuples[compressed_row_count]->t_self; + if (ItemPointerCompare(ptr1, ptr2) != 0) + { + /* decompress affected batch */ + row_compressor->sequence_num = + decompress_in_tuplestore(decompressor, + affectedsegments + ->compressed_tuples[compressed_row_count], + tuplestorecxt->tuplestore); + } + else + { + heap_freetuple(next_tuple); + next_tuple = NULL; + } + } + else + { + /* decompress affected batch */ + row_compressor->sequence_num = + decompress_in_tuplestore(decompressor, + affectedsegments->compressed_tuples[compressed_row_count], + tuplestorecxt->tuplestore); + } + while (total_batches_to_decompress) + { + /* fetch next batches which will be affected */ + if (fetch_next_compressed_tuple(decompressor, + row_compressor, + uncompressed_chunk, + nsegmentby_cols, + compressed_slot, + &next_tuple)) + { + /* decompress affected batch */ + decompress_in_tuplestore(decompressor, next_tuple, tuplestorecxt->tuplestore); + } + total_batches_to_decompress--; + } + tuplesort_performsort(tuplestorecxt->tuplestore); + recompress_segment(tuplestorecxt->tuplestore, decompressor->out_rel, row_compressor); + cleanup_tuplestorecxt(tuplestorecxt); + compressed_row_count++; + affectedsegments->total_affected_compressed_rows--; + } + ExecDropSingleTupleTableSlot(compressed_slot); +} + +static void +do_legacy_recompression(RowDecompressor *decompressor, RowCompressor *row_compressor, + const CompressTuplesortstateCxt *uncompressed_tuplestorecxt) +{ + Relation idxrel; + IndexScanDesc index_scan; + TupleTableSlot *compressed_slot = NULL; + + idxrel = index_open(row_compressor->index_oid, AccessShareLock); + index_scan = index_beginscan(decompressor->in_rel, idxrel, GetTransactionSnapshot(), 0, 0); + index_rescan(index_scan, NULL, 0, NULL, 0); + compressed_slot = table_slot_create(decompressor->in_rel, NULL); + + while (index_getnext_slot(index_scan, ForwardScanDirection, compressed_slot)) + { + slot_getallattrs(compressed_slot); + HeapTuple tuple = heap_form_tuple(compressed_slot->tts_tupleDescriptor, + compressed_slot->tts_values, + compressed_slot->tts_isnull); + tuple->t_self = compressed_slot->tts_tid; + decompress_in_tuplestore(decompressor, tuple, uncompressed_tuplestorecxt->tuplestore); + heap_freetuple(tuple); + } + ExecDropSingleTupleTableSlot(compressed_slot); + index_endscan(index_scan); + index_close(idxrel, AccessShareLock); + + tuplesort_performsort(uncompressed_tuplestorecxt->tuplestore); + recompress_segment(uncompressed_tuplestorecxt->tuplestore, + decompressor->out_rel, + row_compressor); +} + +static void +recompress_new_segments(const RowDecompressor *decompressor, RowCompressor *row_compressor, + AffectedSegmentsCxt *affectedsegments, + const CompressTuplesortstateCxt *uncompressed_tuplestorecxt, + const ColumnCompressionInfo **keys, const int n_keys) +{ + TupleTableSlot *uncompressed_slot; + int count = 0; + uncompressed_slot = MakeTupleTableSlot(decompressor->out_desc, &TTSOpsMinimalTuple); + while (affectedsegments->total_new_compressed_segments) + { + CompressTuplesortstateCxt *tuplestorecxt; + tuplestorecxt = initialize_tuplestore(decompressor->out_rel, keys, n_keys); + + while (affectedsegments->num_non_matching_compressedrows[count] && + tuplesort_gettupleslot(uncompressed_tuplestorecxt->tuplestore, + true /*=forward*/, + false /*=copy*/, + uncompressed_slot, + NULL /*=abbrev*/)) + { + tuplesort_puttupleslot(tuplestorecxt->tuplestore, uncompressed_slot); + affectedsegments->num_non_matching_compressedrows[count]--; + } + tuplesort_performsort(tuplestorecxt->tuplestore); + recompress_segment(tuplestorecxt->tuplestore, decompressor->out_rel, row_compressor); + cleanup_tuplestorecxt(tuplestorecxt); + affectedsegments->total_new_compressed_segments--; + count++; + } + ExecDropSingleTupleTableSlot(uncompressed_slot); +} + +void +recompress_partial_chunks(const Chunk *uncompressed_chunk, const Chunk *compressed_chunk) +{ + Relation uncompressed_chunk_rel; + Relation compressed_chunk_rel; + RowDecompressor decompressor; + RowCompressor row_compressor; + const ColumnCompressionInfo **colinfo_array = NULL; + const ColumnCompressionInfo **keys = NULL; + CompressTuplesortstateCxt *uncompressed_tuplestorecxt; + List *htcols_list; + ListCell *lc; + + int htcols_listlen = 0; + int nsegmentby_cols = 0; + int i = 0; + unsigned long total_segments_decompressed = 0; + unsigned long total_new_segments = 0; + unsigned long total_uncompressed_rows_in_chunk = 0; + + int n_keys = 0; + int16 *in_column_offsets = NULL; + + htcols_list = ts_hypertable_compression_get(uncompressed_chunk->fd.hypertable_id); + htcols_listlen = list_length(htcols_list); + + colinfo_array = palloc(sizeof(ColumnCompressionInfo *) * htcols_listlen); + foreach (lc, htcols_list) + { + FormData_hypertable_compression *fd = (FormData_hypertable_compression *) lfirst(lc); + colinfo_array[i++] = fd; + if (COMPRESSIONCOL_IS_SEGMENT_BY(fd)) + nsegmentby_cols++; + } + in_column_offsets = compress_chunk_populate_keys(uncompressed_chunk->table_id, + colinfo_array, + htcols_listlen, + &n_keys, + &keys); + + uncompressed_chunk_rel = RelationIdGetRelation(uncompressed_chunk->table_id); + compressed_chunk_rel = RelationIdGetRelation(compressed_chunk->table_id); + decompressor = build_decompressor(compressed_chunk_rel, uncompressed_chunk_rel); + /* do not need the indexes on the uncompressed chunk as we do not write to it anymore */ + ts_catalog_close_indexes(decompressor.indexstate); + + row_compressor_init(&row_compressor, + RelationGetDescr(uncompressed_chunk_rel), + compressed_chunk_rel, + htcols_listlen, + colinfo_array, + in_column_offsets, + RelationGetDescr(compressed_chunk_rel)->natts, + true /*need_bistate*/, + true /*reset_sequence*/, + RECOMPRESS); + + uncompressed_tuplestorecxt = initialize_tuplestore(uncompressed_chunk_rel, keys, n_keys); + /* save all rows from uncompressed chunk into tuplestore */ + total_uncompressed_rows_in_chunk = + save_uncompressed_rows_in_tuplestore(uncompressed_chunk_rel, + uncompressed_tuplestorecxt->tuplestore); + + if (total_uncompressed_rows_in_chunk) + { + CompressedChunkStats compressedstats = { + .num_rows_pre_compression = 0, + .num_rows_post_compression = 0, + .num_expected_gaps_in_sequences = 0, + .num_gaps_in_sequences = 0, + .num_segments = 0, + .num_batches_per_segments = { 0 }, + }; + + if (!OidIsValid(row_compressor.index_oid)) + ereport(ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("index does not exist"))); + + AffectedSegmentsCxt *affectedsegments = + (AffectedSegmentsCxt *) palloc0(sizeof(AffectedSegmentsCxt)); + affectedsegments->compressed_tuples = + palloc0(sizeof(unsigned long) * MAX_ROWS_PER_COMPRESSION); + affectedsegments->num_matching_uncompressedrows = + palloc0(sizeof(unsigned long) * MAX_ROWS_PER_COMPRESSION); + affectedsegments->num_non_matching_compressedrows = + palloc0(sizeof(unsigned long) * MAX_ROWS_PER_COMPRESSION); + affectedsegments->default_size = MAX_ROWS_PER_COMPRESSION; + affectedsegments->new_segments_default_size = MAX_ROWS_PER_COMPRESSION; + + /* fetch initial statistics of current chunk */ + get_compressed_chunk_statistics(decompressor.in_rel, + row_compressor.index_oid, + &compressedstats); + /* + * If there is 30% gap available in compressed chunk, recompress + * only affected segments, else call legacy method of recompression + * (decompress + compress) + */ + if ((compressedstats.num_expected_gaps_in_sequences * 0.3) <= + compressedstats.num_gaps_in_sequences) + { + /* sort the tuplestore */ + tuplesort_performsort(uncompressed_tuplestorecxt->tuplestore); + save_affected_compressed_rows_using_indexscan(&decompressor, + &row_compressor, + compressed_chunk, + uncompressed_chunk, + uncompressed_tuplestorecxt, + &affectedsegments, + nsegmentby_cols); + + /* mark current scan position to starting point */ + tuplesort_rescan(uncompressed_tuplestorecxt->tuplestore); + total_segments_decompressed = affectedsegments->total_affected_compressed_rows; + if (affectedsegments->total_affected_compressed_rows) + { + row_compressor.update_sequence = false; + recompress_affected_segments(&decompressor, + &row_compressor, + affectedsegments, + uncompressed_chunk, + nsegmentby_cols, + uncompressed_tuplestorecxt, + keys, + n_keys); + } + total_new_segments = affectedsegments->total_new_compressed_segments; + if (affectedsegments->total_new_compressed_segments) + { + row_compressor.update_sequence = true; + row_compressor.reset_sequence = true; + recompress_new_segments(&decompressor, + &row_compressor, + affectedsegments, + uncompressed_tuplestorecxt, + keys, + n_keys); + } + } + else + { + total_segments_decompressed = compressedstats.num_rows_post_compression; + row_compressor.compression_type = COMPRESS; + do_legacy_recompression(&decompressor, &row_compressor, uncompressed_tuplestorecxt); + } + /****** update compression statistics ******/ + RelationSize after_size = ts_relation_size_impl(compressed_chunk->table_id); + /* the compression size statistics we are able to update and accurately report are: + * rowcount pre/post compression, + * compressed chunk sizes */ + row_compressor.rowcnt_pre_compression = + compressedstats.num_rows_pre_compression + total_uncompressed_rows_in_chunk; + row_compressor.num_compressed_rows = + compressedstats.num_rows_post_compression + total_new_segments; + compression_chunk_size_catalog_update_recompressed(uncompressed_chunk->fd.id, + compressed_chunk->fd.id, + &after_size, + row_compressor.rowcnt_pre_compression, + row_compressor.num_compressed_rows); + elog(LOG, "total decompressed segments are: %lu", total_segments_decompressed); + /* freeup AffectedSegmentsCxt */ + pfree(affectedsegments->compressed_tuples); + pfree(affectedsegments->num_matching_uncompressedrows); + pfree(affectedsegments->num_non_matching_compressedrows); + pfree(affectedsegments); + } + cleanup_tuplestorecxt(uncompressed_tuplestorecxt); + row_compressor_finish(&row_compressor); + FreeBulkInsertState(decompressor.bistate); + + RelationClose(uncompressed_chunk_rel); + RelationClose(compressed_chunk_rel); +} diff --git a/tsl/src/compression/recompression.h b/tsl/src/compression/recompression.h new file mode 100644 index 00000000000..ed55486aa0b --- /dev/null +++ b/tsl/src/compression/recompression.h @@ -0,0 +1,21 @@ +/* + * This file and its contents are licensed under the Timescale License. + * Please see the included NOTICE for copyright information and + * LICENSE-TIMESCALE for a copy of the license. + */ +#ifndef TIMESCALEDB_TSL_COMPRESSION_RECOMPRESSION_H +#define TIMESCALEDB_TSL_COMPRESSION_RECOMPRESSION_H + +/* build scankeys for segmentby columns */ +#define SEGMENTBY_KEYS (1 << 1) +/* build scankeys for orderby columns */ +#define ORDERBY_KEYS (1 << 2) +/* build scankeys for _ts_meta_sequence_num columns */ +#define SEQNUM_KEYS (1 << 3) +/* build scankeys for _ts_meta_sequence_num columns with next value */ +#define NEXT_SEQNUM_KEYS (1 << 4) + +typedef struct Chunk Chunk; +void recompress_partial_chunks(const Chunk *uncompressed_chunk, const Chunk *compressed_chunk); + +#endif diff --git a/tsl/test/expected/recompress_chunk_segmentwise-v2.out b/tsl/test/expected/recompress_chunk_segmentwise-v2.out new file mode 100644 index 00000000000..7c999bb9881 --- /dev/null +++ b/tsl/test/expected/recompress_chunk_segmentwise-v2.out @@ -0,0 +1,1390 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +CREATE TABLE metrics_compressed(filler_1 int, filler_2 int, filler_3 int, time timestamptz NOT NULL, device_id int, v0 int, v1 int, v2 float, v3 float); +CREATE INDEX ON metrics_compressed(time); +CREATE INDEX ON metrics_compressed(device_id,time); +SELECT create_hypertable('metrics_compressed','time',create_default_indexes:=false); + create_hypertable +--------------------------------- + (1,public,metrics_compressed,t) +(1 row) + +ALTER TABLE metrics_compressed DROP COLUMN filler_1; +INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, device_id+1, device_id + 2, device_id + 0.5, NULL FROM generate_series('2000-01-01 0:00:00+0'::timestamptz,'2000-01-05 23:55:00+0','2m') gtime(time), generate_series(1,5,1) gdevice(device_id); +ALTER TABLE metrics_compressed DROP COLUMN filler_2; +INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, device_id-1, device_id + 2, device_id + 0.5, NULL FROM generate_series('2000-01-06 0:00:00+0'::timestamptz,'2000-01-12 23:55:00+0','2m') gtime(time), generate_series(1,5,1) gdevice(device_id); +ALTER TABLE metrics_compressed DROP COLUMN filler_3; +INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, device_id, device_id + 2, device_id + 0.5, NULL FROM generate_series('2000-01-13 0:00:00+0'::timestamptz,'2000-01-19 23:55:00+0','2m') gtime(time), generate_series(1,5,1) gdevice(device_id); +ANALYZE metrics_compressed; +-- compress chunks +ALTER TABLE metrics_compressed SET (timescaledb.compress, timescaledb.compress_orderby='time DESC', timescaledb.compress_segmentby='device_id'); +SELECT compress_chunk(c.schema_name|| '.' || c.table_name) +FROM _timescaledb_catalog.chunk c, _timescaledb_catalog.hypertable ht WHERE c.hypertable_id = ht.id and ht.table_name = 'metrics_compressed' and c.compressed_chunk_id IS NULL +ORDER BY c.table_name DESC; + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_1_chunk +(3 rows) + +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 1 | f +(1 row) + +-- case1: 1 segment affected +-- Number of segments decompressed is 1 +BEGIN; +-- affects segment with sequence_num = 10 +INSERT INTO metrics_compressed VALUES ('2000-01-05 20:06:00+05:30'::timestamp with time zone, 1, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-04 23:06:00+05:30'::timestamp with time zone, 1, 2, 3, 222.2, 333.3); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 9 | f +(1 row) + +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id = 1 ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +-----------+-----------------------+----------------+------------------------------+------------------------------ + 1 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 1 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 1 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 1 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST +(4 rows) + +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: statement: CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: total decompressed segments are: 1 +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id = 1 ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +-----------+-----------------------+----------------+------------------------------+------------------------------ + 1 | 10 | 1000 | Tue Jan 04 06:40:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 1 | 11 | 2 | Tue Jan 04 06:36:00 2000 PST | Tue Jan 04 06:38:00 2000 PST + 1 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 1 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 1 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST +(5 rows) + +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 1 | f +(1 row) + +ROLLBACK; +-- case2: 2 segments affected with same segment by values +-- Number of segments decompressed is 2 +BEGIN; +-- affects segment with sequence_num = 20 +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 1, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:56:00+05:30'::timestamp with time zone, 1, 2, 3, 222.2, 333.3); +-- affects segment with sequence_num = 40 +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 1, 2, 3, 22.2, 33.3); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 9 | f +(1 row) + +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id = 1 ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +-----------+-----------------------+----------------+------------------------------+------------------------------ + 1 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 1 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 1 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 1 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST +(4 rows) + +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: statement: CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: total decompressed segments are: 2 +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id = 1 ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +-----------+-----------------------+----------------+------------------------------+------------------------------ + 1 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 1 | 20 | 1000 | Sun Jan 02 21:20:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 1 | 21 | 2 | Sun Jan 02 21:16:00 2000 PST | Sun Jan 02 21:18:00 2000 PST + 1 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 1 | 40 | 599 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST +(5 rows) + +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 1 | f +(1 row) + +ROLLBACK; +-- case3: 2 different segments affected +-- Number of segments decompressed is 2 +BEGIN; +-- affects segment with sequence_num = 20 +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 1, 2, 3, 22.2, 33.3); +-- affects segment with sequence_num = 40 +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 3, 2, 3, 222.2, 333.3); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 9 | f +(1 row) + +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id IN (1, 3) ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +-----------+-----------------------+----------------+------------------------------+------------------------------ + 1 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 1 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 1 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 1 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 3 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 3 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 3 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 3 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST +(8 rows) + +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: statement: CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: total decompressed segments are: 2 +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id IN (1, 3) ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +-----------+-----------------------+----------------+------------------------------+------------------------------ + 1 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 1 | 20 | 1000 | Sun Jan 02 21:18:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 1 | 21 | 1 | Sun Jan 02 21:16:00 2000 PST | Sun Jan 02 21:16:00 2000 PST + 1 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 1 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 3 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 3 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 3 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 3 | 40 | 599 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST +(9 rows) + +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 1 | f +(1 row) + +ROLLBACK; +-- case4: 2 segments affected with same segment by values + new segments created +-- Number of segments decompressed is 2 +BEGIN; +-- affects segment with sequence_num = 20 +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 1, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:56:00+05:30'::timestamp with time zone, 1, 2, 3, 222.2, 333.3); +-- affects segment with sequence_num = 40 +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 1, 2, 3, 22.2, 33.3); +-- create new segment +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 11, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:56:00+05:30'::timestamp with time zone, 11, 2, 3, 222.2, 333.3); +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 12, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 13, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 14, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 15, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 12, 2, 3, 22.2, 33.3); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 9 | f +(1 row) + +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +-----------+-----------------------+----------------+------------------------------+------------------------------ + 1 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 1 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 1 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 1 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 2 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 2 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 2 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 2 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 3 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 3 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 3 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 3 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 4 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 4 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 4 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 4 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 5 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 5 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 5 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 5 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST +(20 rows) + +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: statement: CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: total decompressed segments are: 2 +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +-----------+-----------------------+----------------+------------------------------+------------------------------ + 1 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 1 | 20 | 1000 | Sun Jan 02 21:20:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 1 | 21 | 2 | Sun Jan 02 21:16:00 2000 PST | Sun Jan 02 21:18:00 2000 PST + 1 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 1 | 40 | 599 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 2 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 2 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 2 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 2 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 3 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 3 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 3 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 3 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 4 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 4 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 4 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 4 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 5 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 5 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 5 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 5 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 11 | 10 | 2 | Mon Jan 03 02:16:00 2000 PST | Mon Jan 03 02:26:00 2000 PST + 12 | 10 | 2 | Sat Jan 01 02:00:00 2000 PST | Mon Jan 03 02:16:00 2000 PST + 13 | 10 | 1 | Sat Jan 01 02:00:00 2000 PST | Sat Jan 01 02:00:00 2000 PST + 14 | 10 | 1 | Sat Jan 01 02:00:00 2000 PST | Sat Jan 01 02:00:00 2000 PST + 15 | 10 | 1 | Sat Jan 01 02:00:00 2000 PST | Sat Jan 01 02:00:00 2000 PST +(26 rows) + +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 1 | f +(1 row) + +ROLLBACK; +-- case5: 3 different segments affected with same segment by values + new segments created +-- Number of segments decompressed is 3 +BEGIN; +-- affects segment with sequence_num = 20 +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 1, 2, 3, 2.2, 3.3); +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:56:00+05:30'::timestamp with time zone, 2, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 3, 2, 3, 222.2, 333.3); +-- create new segment +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 11, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:56:00+05:30'::timestamp with time zone, 11, 2, 3, 222.2, 333.3); +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 12, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 13, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 14, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 15, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 12, 2, 3, 22.2, 33.3); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 9 | f +(1 row) + +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +-----------+-----------------------+----------------+------------------------------+------------------------------ + 1 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 1 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 1 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 1 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 2 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 2 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 2 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 2 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 3 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 3 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 3 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 3 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 4 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 4 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 4 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 4 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 5 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 5 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 5 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 5 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST +(20 rows) + +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: statement: CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: total decompressed segments are: 3 +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +-----------+-----------------------+----------------+------------------------------+------------------------------ + 1 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 1 | 20 | 1000 | Sun Jan 02 21:18:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 1 | 21 | 1 | Sun Jan 02 21:16:00 2000 PST | Sun Jan 02 21:16:00 2000 PST + 1 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 1 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 2 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 2 | 20 | 1000 | Sun Jan 02 21:18:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 2 | 21 | 1 | Sun Jan 02 21:16:00 2000 PST | Sun Jan 02 21:16:00 2000 PST + 2 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 2 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 3 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 3 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 3 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 3 | 40 | 599 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 4 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 4 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 4 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 4 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 5 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 5 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 5 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 5 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 11 | 10 | 2 | Mon Jan 03 02:16:00 2000 PST | Mon Jan 03 02:26:00 2000 PST + 12 | 10 | 2 | Sat Jan 01 02:00:00 2000 PST | Mon Jan 03 02:16:00 2000 PST + 13 | 10 | 1 | Sat Jan 01 02:00:00 2000 PST | Sat Jan 01 02:00:00 2000 PST + 14 | 10 | 1 | Sat Jan 01 02:00:00 2000 PST | Sat Jan 01 02:00:00 2000 PST + 15 | 10 | 1 | Sat Jan 01 02:00:00 2000 PST | Sat Jan 01 02:00:00 2000 PST +(27 rows) + +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 1 | f +(1 row) + +ROLLBACK; +-- case6: all segments affected +-- Number of segments decompressed is 5 +BEGIN; +-- affects segment with sequence_num = 20 +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 1, 2, 3, 2.2, 3.3); +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:56:00+05:30'::timestamp with time zone, 2, 2, 3, 22.2, 33.3); +-- affects segment with sequence_num = 40 +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 3, 2, 3, 222.2, 333.3); +-- affects segment with sequence_num = 10 +INSERT INTO metrics_compressed VALUES ('2000-01-04 15:46:00+05:30'::timestamp with time zone, 4, 2, 3, 2222.2, 3333.3); +INSERT INTO metrics_compressed VALUES ('2000-01-05 15:56:00+05:30'::timestamp with time zone, 5, 2, 3, 22222.2, 33333.3); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 9 | f +(1 row) + +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +-----------+-----------------------+----------------+------------------------------+------------------------------ + 1 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 1 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 1 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 1 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 2 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 2 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 2 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 2 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 3 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 3 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 3 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 3 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 4 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 4 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 4 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 4 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 5 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 5 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 5 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 5 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST +(20 rows) + +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: statement: CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: total decompressed segments are: 5 +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +-----------+-----------------------+----------------+------------------------------+------------------------------ + 1 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 1 | 20 | 1000 | Sun Jan 02 21:18:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 1 | 21 | 1 | Sun Jan 02 21:16:00 2000 PST | Sun Jan 02 21:16:00 2000 PST + 1 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 1 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 2 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 2 | 20 | 1000 | Sun Jan 02 21:18:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 2 | 21 | 1 | Sun Jan 02 21:16:00 2000 PST | Sun Jan 02 21:16:00 2000 PST + 2 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 2 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 3 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 3 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 3 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 3 | 40 | 599 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 4 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 4 | 20 | 1000 | Sun Jan 02 21:18:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 4 | 21 | 1 | Sun Jan 02 21:16:00 2000 PST | Sun Jan 02 21:16:00 2000 PST + 4 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 4 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 5 | 10 | 1000 | Tue Jan 04 06:38:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 5 | 11 | 1 | Tue Jan 04 06:36:00 2000 PST | Tue Jan 04 06:36:00 2000 PST + 5 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 5 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 5 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST +(24 rows) + +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 1 | f +(1 row) + +ROLLBACK; +-- case7: all segments affected + new segments created +-- Number of segments decompressed is 5 +BEGIN; +-- affects segment with sequence_num = 10 +INSERT INTO metrics_compressed VALUES ('2000-01-05 15:46:00+05:30'::timestamp with time zone, 1, 2, 3, 2.2, 3.3); +INSERT INTO metrics_compressed VALUES ('2000-01-05 15:56:00+05:30'::timestamp with time zone, 2, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-05 15:30:00+05:30'::timestamp with time zone, 3, 2, 3, 222.2, 333.3); +INSERT INTO metrics_compressed VALUES ('2000-01-05 15:46:00+05:30'::timestamp with time zone, 4, 2, 3, 2222.2, 3333.3); +INSERT INTO metrics_compressed VALUES ('2000-01-05 15:56:00+05:30'::timestamp with time zone, 5, 2, 3, 22222.2, 33333.3); +-- create new segment +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 11, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:56:00+05:30'::timestamp with time zone, 11, 2, 3, 222.2, 333.3); +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 11, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 13, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 14, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 15, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 12, 2, 3, 22.2, 33.3); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 9 | f +(1 row) + +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +-----------+-----------------------+----------------+------------------------------+------------------------------ + 1 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 1 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 1 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 1 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 2 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 2 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 2 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 2 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 3 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 3 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 3 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 3 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 4 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 4 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 4 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 4 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 5 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 5 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 5 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 5 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST +(20 rows) + +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: statement: CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: total decompressed segments are: 5 +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +-----------+-----------------------+----------------+------------------------------+------------------------------ + 1 | 10 | 1000 | Tue Jan 04 06:38:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 1 | 11 | 1 | Tue Jan 04 06:36:00 2000 PST | Tue Jan 04 06:36:00 2000 PST + 1 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 1 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 1 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 2 | 10 | 1000 | Tue Jan 04 06:38:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 2 | 11 | 1 | Tue Jan 04 06:36:00 2000 PST | Tue Jan 04 06:36:00 2000 PST + 2 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 2 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 2 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 3 | 10 | 1000 | Tue Jan 04 06:38:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 3 | 11 | 1 | Tue Jan 04 06:36:00 2000 PST | Tue Jan 04 06:36:00 2000 PST + 3 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 3 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 3 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 4 | 10 | 1000 | Tue Jan 04 06:38:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 4 | 11 | 1 | Tue Jan 04 06:36:00 2000 PST | Tue Jan 04 06:36:00 2000 PST + 4 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 4 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 4 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 5 | 10 | 1000 | Tue Jan 04 06:38:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 5 | 11 | 1 | Tue Jan 04 06:36:00 2000 PST | Tue Jan 04 06:36:00 2000 PST + 5 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 5 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 5 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 11 | 10 | 3 | Mon Jan 03 02:16:00 2000 PST | Mon Jan 03 02:26:00 2000 PST + 12 | 10 | 1 | Sat Jan 01 02:00:00 2000 PST | Sat Jan 01 02:00:00 2000 PST + 13 | 10 | 1 | Sat Jan 01 02:00:00 2000 PST | Sat Jan 01 02:00:00 2000 PST + 14 | 10 | 1 | Sat Jan 01 02:00:00 2000 PST | Sat Jan 01 02:00:00 2000 PST + 15 | 10 | 1 | Sat Jan 01 02:00:00 2000 PST | Sat Jan 01 02:00:00 2000 PST +(30 rows) + +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 1 | f +(1 row) + +ROLLBACK; +-- case8: create new rows in existing segments +BEGIN; +-- does not affects any segments +INSERT INTO metrics_compressed VALUES ('1999-12-31 05:30:00+05:30'::timestamp with time zone, 1, 2, 3, 2.2, 3.3); +INSERT INTO metrics_compressed VALUES ('1999-12-30 15:30:00+05:30'::timestamp with time zone, 1, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('1999-12-31 05:30:00+05:30'::timestamp with time zone, 2, 2, 3, 222.2, 333.3); +INSERT INTO metrics_compressed VALUES ('1999-12-31 05:30:00+05:30'::timestamp with time zone, 3, 2, 3, 2222.2, 3333.3); +INSERT INTO metrics_compressed VALUES ('1999-12-31 05:30:00+05:30'::timestamp with time zone, 4, 2, 3, 22222.2, 33333.3); +INSERT INTO metrics_compressed VALUES ('1999-12-30 15:30:00+05:30'::timestamp with time zone, 4, 2, 3, 22222.2, 33333.3); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 9 | f +(1 row) + +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +-----------+-----------------------+----------------+------------------------------+------------------------------ + 1 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 1 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 1 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 1 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 2 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 2 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 2 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 2 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 3 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 3 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 3 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 3 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 4 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 4 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 4 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 4 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 5 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 5 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 5 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 5 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST +(20 rows) + +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: statement: CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: total decompressed segments are: 4 +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +-----------+-----------------------+----------------+------------------------------+------------------------------ + 1 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 1 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 1 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 1 | 40 | 600 | Thu Dec 30 02:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 2 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 2 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 2 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 2 | 40 | 599 | Thu Dec 30 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 3 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 3 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 3 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 3 | 40 | 599 | Thu Dec 30 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 4 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 4 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 4 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 4 | 40 | 600 | Thu Dec 30 02:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 5 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 5 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 5 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 5 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST +(20 rows) + +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 1 | f +(1 row) + +ROLLBACK; +-- case9: check with UPDATE/DELETE on compressed chunks +BEGIN; +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+--------------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 1 | f + 2 | 1 | _timescaledb_internal | _hyper_1_2_chunk | 5 | f | 1 | f + 3 | 1 | _timescaledb_internal | _hyper_1_3_chunk | 4 | f | 1 | f + 4 | 2 | _timescaledb_internal | compress_hyper_2_4_chunk | | f | 0 | f + 5 | 2 | _timescaledb_internal | compress_hyper_2_5_chunk | | f | 0 | f + 6 | 2 | _timescaledb_internal | compress_hyper_2_6_chunk | | f | 0 | f +(6 rows) + +/* delete device id 4 */ +DELETE FROM metrics_compressed WHERE device_id = 4; +/* update device id 3 */ +UPDATE metrics_compressed SET v0 = 111, v1 = 222, v2 = 333, v3 = 444 WHERE device_id = 3; +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+--------------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 9 | f + 2 | 1 | _timescaledb_internal | _hyper_1_2_chunk | 5 | f | 9 | f + 3 | 1 | _timescaledb_internal | _hyper_1_3_chunk | 4 | f | 9 | f + 4 | 2 | _timescaledb_internal | compress_hyper_2_4_chunk | | f | 0 | f + 5 | 2 | _timescaledb_internal | compress_hyper_2_5_chunk | | f | 0 | f + 6 | 2 | _timescaledb_internal | compress_hyper_2_6_chunk | | f | 0 | f +(6 rows) + +-- INSERT new segments with device_id = 4 +INSERT INTO metrics_compressed VALUES ('1999-12-31 05:30:00+05:30'::timestamp with time zone, 4, 2, 3, 2.2, 3.3); +INSERT INTO metrics_compressed VALUES ('1999-12-30 15:30:00+05:30'::timestamp with time zone, 4, 2, 3, 22.2, 33.3); +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +-----------+-----------------------+----------------+------------------------------+------------------------------ + 1 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 1 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 1 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 1 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 2 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 2 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 2 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 2 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 5 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 5 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 5 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 5 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST +(12 rows) + +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: statement: CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: total decompressed segments are: 0 +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +-----------+-----------------------+----------------+------------------------------+------------------------------ + 1 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 1 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 1 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 1 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 2 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 2 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 2 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 2 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 3 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 3 | 11 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 3 | 12 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 3 | 13 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST + 4 | 10 | 2 | Thu Dec 30 02:00:00 1999 PST | Thu Dec 30 16:00:00 1999 PST + 5 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 5 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 5 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 5 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST +(17 rows) + +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+--------------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 1 | f + 2 | 1 | _timescaledb_internal | _hyper_1_2_chunk | 5 | f | 9 | f + 3 | 1 | _timescaledb_internal | _hyper_1_3_chunk | 4 | f | 9 | f + 4 | 2 | _timescaledb_internal | compress_hyper_2_4_chunk | | f | 0 | f + 5 | 2 | _timescaledb_internal | compress_hyper_2_5_chunk | | f | 0 | f + 6 | 2 | _timescaledb_internal | compress_hyper_2_6_chunk | | f | 0 | f +(6 rows) + +ROLLBACK; +-- case10: gaps between sequence no: are filled up +-- Number of segments decompressed is 2 +BEGIN; +-- affects segment with sequence_num = 20,30 +INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, V+1.5, V + 2.5, V + 0.5, NULL FROM generate_series('2000-01-02 01:26:00+05:30'::timestamptz,'2000-01-04 20:06:00+05:30','22m') gtime(time), generate_series(3,3,1) gdevice(device_id), generate_series(1,100,1) gv(V); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 9 | f +(1 row) + +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id = 3 ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +-----------+-----------------------+----------------+------------------------------+------------------------------ + 3 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 3 | 20 | 1000 | Sun Jan 02 21:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 3 | 30 | 1000 | Sat Jan 01 11:56:00 2000 PST | Sun Jan 02 21:14:00 2000 PST + 3 | 40 | 598 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:54:00 2000 PST +(4 rows) + +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: statement: CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: total decompressed segments are: 2 +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id = 3 ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +-----------+-----------------------+----------------+------------------------------+------------------------------ + 3 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 3 | 20 | 1000 | Tue Jan 04 03:16:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 3 | 21 | 1000 | Mon Jan 03 23:56:00 2000 PST | Tue Jan 04 03:14:00 2000 PST + 3 | 22 | 1000 | Mon Jan 03 20:36:00 2000 PST | Mon Jan 03 23:54:00 2000 PST + 3 | 23 | 1000 | Mon Jan 03 17:16:00 2000 PST | Mon Jan 03 20:34:00 2000 PST + 3 | 24 | 1000 | Mon Jan 03 13:56:00 2000 PST | Mon Jan 03 17:14:00 2000 PST + 3 | 25 | 1000 | Mon Jan 03 10:36:00 2000 PST | Mon Jan 03 13:54:00 2000 PST + 3 | 26 | 1000 | Mon Jan 03 07:16:00 2000 PST | Mon Jan 03 10:34:00 2000 PST + 3 | 27 | 1000 | Mon Jan 03 03:56:00 2000 PST | Mon Jan 03 07:14:00 2000 PST + 3 | 28 | 1000 | Mon Jan 03 00:36:00 2000 PST | Mon Jan 03 03:54:00 2000 PST + 3 | 29 | 1000 | Sun Jan 02 21:18:00 2000 PST | Mon Jan 03 00:36:00 2000 PST + 3 | 30 | 1000 | Sat Jan 01 15:16:00 2000 PST | Sun Jan 02 21:18:00 2000 PST + 3 | 31 | 100 | Sat Jan 01 11:56:00 2000 PST | Sat Jan 01 15:14:00 2000 PST + 3 | 32 | 1000 | Sun Jan 02 17:38:00 2000 PST | Sun Jan 02 20:56:00 2000 PST + 3 | 33 | 1000 | Sun Jan 02 13:58:00 2000 PST | Sun Jan 02 17:16:00 2000 PST + 3 | 34 | 1000 | Sun Jan 02 10:18:00 2000 PST | Sun Jan 02 13:36:00 2000 PST + 3 | 35 | 1000 | Sun Jan 02 06:38:00 2000 PST | Sun Jan 02 09:56:00 2000 PST + 3 | 36 | 1000 | Sun Jan 02 02:58:00 2000 PST | Sun Jan 02 06:16:00 2000 PST + 3 | 37 | 1000 | Sat Jan 01 23:18:00 2000 PST | Sun Jan 02 02:36:00 2000 PST + 3 | 38 | 1000 | Sat Jan 01 19:38:00 2000 PST | Sat Jan 01 22:56:00 2000 PST + 3 | 39 | 1000 | Sat Jan 01 15:58:00 2000 PST | Sat Jan 01 19:16:00 2000 PST + 3 | 40 | 1000 | Sat Jan 01 12:18:00 2000 PST | Sat Jan 01 15:36:00 2000 PST + 3 | 41 | 698 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 11:56:00 2000 PST +(23 rows) + +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 1 | f +(1 row) + +ROLLBACK; +-- case11: no: of new segment added is > 1000, which calls resize_affectedsegmentscxt() +-- Number of segments decompressed is 10 +BEGIN; +-- affects segment with sequence_num = 20,30 +INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, V+1.5, V + 2.5, V + 0.5, NULL FROM generate_series('2000-01-02 01:26:00+05:30'::timestamptz,'2000-01-04 20:06:00+05:30','22m') gtime(time), generate_series(1,1020,1) gdevice(device_id), generate_series(1,1,1) gv(V); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 9 | f +(1 row) + +SELECT count(distinct device_id) FROM _timescaledb_internal.compress_hyper_2_6_chunk; + count +------- + 5 +(1 row) + +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +SELECT count(distinct device_id) FROM _timescaledb_internal.compress_hyper_2_6_chunk; + count +------- + 1020 +(1 row) + +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 1 | f +(1 row) + +ROLLBACK; +-- case12: no: of affected existing segment is > 1000, which calls resize_affectedsegmentscxt() +INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, V+1.5, V + 2.5, V + 0.5, NULL FROM generate_series('2000-01-02 01:26:00+05:30'::timestamptz,'2000-01-04 20:06:00+05:30','22m') gtime(time), generate_series(1,1020,1) gdevice(device_id), generate_series(1,1,1) gv(V); +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +BEGIN; +-- affects segment with sequence_num = 20,30 +INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, V+1.5, V + 2.5, V + 0.5, NULL FROM generate_series('2000-01-02 01:26:00+05:30'::timestamptz,'2000-01-04 20:06:00+05:30','22m') gtime(time), generate_series(1,1020,1) gdevice(device_id), generate_series(1,1,1) gv(V); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 9 | f +(1 row) + +SELECT count(distinct device_id) FROM _timescaledb_internal.compress_hyper_2_6_chunk; + count +------- + 1020 +(1 row) + +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +SELECT count(distinct device_id) FROM _timescaledb_internal.compress_hyper_2_6_chunk; + count +------- + 1020 +(1 row) + +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 1 | f +(1 row) + +ROLLBACK; +-- case13: gaps between sequence no: are filled up, thus call legacy recompression +BEGIN; +-- affects segment with sequence_num = 20,30 +INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, V+1.5, V + 2.5, V + 0.5, NULL FROM generate_series('2000-01-02 01:26:00+05:30'::timestamptz,'2000-01-04 20:06:00+05:30','22m') gtime(time), generate_series(3,3,1) gdevice(device_id), generate_series(1,100,1) gv(V); +INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, V+1.5, V + 2.5, V + 0.5, NULL FROM generate_series('2000-01-02 01:26:00+05:30'::timestamptz,'2000-01-04 20:06:00+05:30','22m') gtime(time), generate_series(3,3,1) gdevice(device_id), generate_series(1,100,1) gv(V); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 9 | f +(1 row) + +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id = 3 ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count +-----------+-----------------------+---------------- + 3 | 10 | 1000 + 3 | 20 | 1000 + 3 | 21 | 91 + 3 | 30 | 1000 + 3 | 31 | 91 + 3 | 40 | 598 +(6 rows) + +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: statement: CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: total decompressed segments are: 4 +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id = 3 ORDER BY 1, 2; +LOG: statement: SELECT device_id, _ts_meta_sequence_num, _ts_meta_count FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id = 3 ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count +-----------+-----------------------+---------------- + 3 | 10 | 1000 + 3 | 20 | 1000 + 3 | 21 | 1000 + 3 | 21 | 1000 + 3 | 22 | 1000 + 3 | 22 | 691 + 3 | 23 | 1000 + 3 | 23 | 1000 + 3 | 24 | 1000 + 3 | 24 | 1000 + 3 | 25 | 1000 + 3 | 25 | 1000 + 3 | 26 | 1000 + 3 | 26 | 1000 + 3 | 27 | 1000 + 3 | 27 | 1000 + 3 | 28 | 1000 + 3 | 28 | 1000 + 3 | 29 | 1000 + 3 | 29 | 1000 + 3 | 30 | 1000 + 3 | 30 | 1000 + 3 | 31 | 1000 + 3 | 31 | 1000 + 3 | 31 | 1000 + 3 | 32 | 1000 + 3 | 32 | 1000 + 3 | 32 | 691 + 3 | 33 | 1000 + 3 | 33 | 1000 + 3 | 34 | 1000 + 3 | 34 | 1000 + 3 | 35 | 1000 + 3 | 35 | 1000 + 3 | 36 | 1000 + 3 | 36 | 1000 + 3 | 37 | 1000 + 3 | 37 | 1000 + 3 | 38 | 600 + 3 | 38 | 1000 + 3 | 39 | 1000 + 3 | 40 | 198 +(42 rows) + +COMMIT; +LOG: statement: COMMIT; +BEGIN; +LOG: statement: BEGIN; +INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, V+1.5, V + 2.5, V + 0.5, NULL FROM generate_series('2000-01-02 01:26:00+05:30'::timestamptz,'2000-01-04 20:06:00+05:30','22m') gtime(time), generate_series(3,3,1) gdevice(device_id), generate_series(1,100,1) gv(V); +LOG: statement: INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, V+1.5, V + 2.5, V + 0.5, NULL FROM generate_series('2000-01-02 01:26:00+05:30'::timestamptz,'2000-01-04 20:06:00+05:30','22m') gtime(time), generate_series(3,3,1) gdevice(device_id), generate_series(1,100,1) gv(V); +-- should invoke legacy recompression +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: statement: CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: total decompressed segments are: 1081 +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id = 3 ORDER BY 1, 2; +LOG: statement: SELECT device_id, _ts_meta_sequence_num, _ts_meta_count FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id = 3 ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count +-----------+-----------------------+---------------- + 3 | 10 | 1000 + 3 | 20 | 1000 + 3 | 30 | 1000 + 3 | 40 | 1000 + 3 | 50 | 1000 + 3 | 60 | 1000 + 3 | 70 | 1000 + 3 | 80 | 1000 + 3 | 90 | 1000 + 3 | 100 | 1000 + 3 | 110 | 1000 + 3 | 120 | 1000 + 3 | 130 | 1000 + 3 | 140 | 1000 + 3 | 150 | 1000 + 3 | 160 | 1000 + 3 | 170 | 1000 + 3 | 180 | 1000 + 3 | 190 | 1000 + 3 | 200 | 1000 + 3 | 210 | 1000 + 3 | 220 | 1000 + 3 | 230 | 1000 + 3 | 240 | 1000 + 3 | 250 | 1000 + 3 | 260 | 1000 + 3 | 270 | 1000 + 3 | 280 | 1000 + 3 | 290 | 1000 + 3 | 300 | 1000 + 3 | 310 | 1000 + 3 | 320 | 1000 + 3 | 330 | 1000 + 3 | 340 | 1000 + 3 | 350 | 1000 + 3 | 360 | 1000 + 3 | 370 | 1000 + 3 | 380 | 1000 + 3 | 390 | 1000 + 3 | 400 | 1000 + 3 | 410 | 1000 + 3 | 420 | 1000 + 3 | 430 | 1000 + 3 | 440 | 1000 + 3 | 450 | 1000 + 3 | 460 | 1000 + 3 | 470 | 1000 + 3 | 480 | 1000 + 3 | 490 | 1000 + 3 | 500 | 1000 + 3 | 510 | 1000 + 3 | 520 | 1000 + 3 | 530 | 1000 + 3 | 540 | 1000 + 3 | 550 | 1000 + 3 | 560 | 1000 + 3 | 570 | 1000 + 3 | 580 | 1000 + 3 | 590 | 380 +(59 rows) + +COMMIT; +LOG: statement: COMMIT; +BEGIN; +LOG: statement: BEGIN; +INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, V+1.5, V + 2.5, V + 0.5, NULL FROM generate_series('2000-01-02 01:26:00+05:30'::timestamptz,'2000-01-04 20:06:00+05:30','22m') gtime(time), generate_series(3,3,1) gdevice(device_id), generate_series(1,100,1) gv(V); +LOG: statement: INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, V+1.5, V + 2.5, V + 0.5, NULL FROM generate_series('2000-01-02 01:26:00+05:30'::timestamptz,'2000-01-04 20:06:00+05:30','22m') gtime(time), generate_series(3,3,1) gdevice(device_id), generate_series(1,100,1) gv(V); +-- should invoke optimized recompression as there are enough gaps in sequence numbers in compressed chunk +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: statement: CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: total decompressed segments are: 57 +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id = 3 ORDER BY 1, 2; +LOG: statement: SELECT device_id, _ts_meta_sequence_num, _ts_meta_count FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id = 3 ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count +-----------+-----------------------+---------------- + 3 | 10 | 1000 + 3 | 20 | 1000 + 3 | 21 | 400 + 3 | 30 | 1000 + 3 | 31 | 300 + 3 | 40 | 1000 + 3 | 41 | 300 + 3 | 50 | 1000 + 3 | 51 | 300 + 3 | 60 | 1000 + 3 | 61 | 300 + 3 | 70 | 1000 + 3 | 71 | 400 + 3 | 80 | 1000 + 3 | 81 | 300 + 3 | 90 | 1000 + 3 | 91 | 300 + 3 | 100 | 1000 + 3 | 101 | 300 + 3 | 110 | 1000 + 3 | 111 | 400 + 3 | 120 | 1000 + 3 | 121 | 300 + 3 | 130 | 1000 + 3 | 131 | 300 + 3 | 140 | 1000 + 3 | 141 | 300 + 3 | 150 | 1000 + 3 | 151 | 300 + 3 | 160 | 1000 + 3 | 161 | 400 + 3 | 170 | 1000 + 3 | 171 | 300 + 3 | 180 | 1000 + 3 | 181 | 300 + 3 | 190 | 1000 + 3 | 191 | 300 + 3 | 200 | 1000 + 3 | 201 | 300 + 3 | 210 | 1000 + 3 | 211 | 400 + 3 | 220 | 1000 + 3 | 221 | 300 + 3 | 230 | 1000 + 3 | 231 | 300 + 3 | 240 | 1000 + 3 | 241 | 300 + 3 | 250 | 1000 + 3 | 251 | 300 + 3 | 260 | 1000 + 3 | 261 | 400 + 3 | 270 | 1000 + 3 | 271 | 300 + 3 | 280 | 1000 + 3 | 281 | 300 + 3 | 290 | 1000 + 3 | 291 | 300 + 3 | 300 | 1000 + 3 | 301 | 300 + 3 | 310 | 1000 + 3 | 311 | 400 + 3 | 320 | 1000 + 3 | 321 | 300 + 3 | 330 | 1000 + 3 | 331 | 300 + 3 | 340 | 1000 + 3 | 341 | 300 + 3 | 350 | 1000 + 3 | 351 | 300 + 3 | 360 | 1000 + 3 | 361 | 400 + 3 | 370 | 1000 + 3 | 371 | 300 + 3 | 380 | 1000 + 3 | 381 | 300 + 3 | 390 | 1000 + 3 | 391 | 300 + 3 | 400 | 1000 + 3 | 401 | 300 + 3 | 410 | 1000 + 3 | 411 | 400 + 3 | 420 | 1000 + 3 | 421 | 300 + 3 | 430 | 1000 + 3 | 431 | 300 + 3 | 440 | 1000 + 3 | 441 | 300 + 3 | 450 | 1000 + 3 | 451 | 300 + 3 | 460 | 1000 + 3 | 461 | 400 + 3 | 470 | 1000 + 3 | 471 | 300 + 3 | 480 | 1000 + 3 | 481 | 300 + 3 | 490 | 1000 + 3 | 491 | 300 + 3 | 500 | 1000 + 3 | 501 | 400 + 3 | 510 | 1000 + 3 | 511 | 300 + 3 | 520 | 1000 + 3 | 521 | 300 + 3 | 530 | 1000 + 3 | 531 | 300 + 3 | 540 | 1000 + 3 | 541 | 300 + 3 | 550 | 1000 + 3 | 551 | 400 + 3 | 560 | 1000 + 3 | 561 | 300 + 3 | 570 | 1000 + 3 | 571 | 300 + 3 | 580 | 1000 + 3 | 581 | 200 + 3 | 590 | 380 +(116 rows) + +COMMIT; +LOG: statement: COMMIT; +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +LOG: statement: SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 1 | f +(1 row) + +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +-- check for heapscan +DROP INDEX _timescaledb_internal.compress_hyper_2_6_chunk__compressed_hypertable_2_device_id__ts; +-- INSERT new segments with device_id = 4 +INSERT INTO metrics_compressed VALUES ('1999-12-31 05:30:00+05:30'::timestamp with time zone, 4, 2, 3, 2.2, 3.3); +INSERT INTO metrics_compressed VALUES ('1999-12-30 15:30:00+05:30'::timestamp with time zone, 4, 2, 3, 22.2, 33.3); +-- does not affects any segments, append to existing segments +INSERT INTO metrics_compressed VALUES ('1999-12-31 05:30:00+05:30'::timestamp with time zone, 1, 2, 3, 2.2, 3.3); +INSERT INTO metrics_compressed VALUES ('1999-12-30 15:30:00+05:30'::timestamp with time zone, 1, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('1999-12-31 05:30:00+05:30'::timestamp with time zone, 2, 2, 3, 222.2, 333.3); +INSERT INTO metrics_compressed VALUES ('1999-12-31 05:30:00+05:30'::timestamp with time zone, 3, 2, 3, 2222.2, 3333.3); +INSERT INTO metrics_compressed VALUES ('1999-12-31 05:30:00+05:30'::timestamp with time zone, 4, 2, 3, 22222.2, 33333.3); +INSERT INTO metrics_compressed VALUES ('1999-12-30 15:30:00+05:30'::timestamp with time zone, 4, 2, 3, 22222.2, 33333.3); +-- affects segment with sequence_num = 30 +INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, device_id+1, device_id + 2, device_id + 0.5, NULL FROM generate_series('2000-01-02 01:26:00+05:30'::timestamptz,'2000-01-03 10:44:00+05:30','12s') gtime(time), generate_series(4,4,1) gdevice(device_id); +-- create new segment +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 11, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:56:00+05:30'::timestamp with time zone, 11, 2, 3, 222.2, 333.3); +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 11, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 13, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 14, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 15, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 12, 2, 3, 22.2, 33.3); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 6 | f | 9 | f +(1 row) + +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id = 4 ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +-----------+-----------------------+----------------+------------------------------+------------------------------ + 4 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 4 | 20 | 1000 | Mon Jan 03 00:02:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 4 | 30 | 1000 | Sat Jan 01 17:28:00 2000 PST | Mon Jan 03 00:00:00 2000 PST + 4 | 40 | 780 | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 17:26:00 2000 PST +(4 rows) + +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +LOG: statement: CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_7_chunk WHERE device_id = 4 ORDER BY 1, 2; + device_id | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +-----------+-----------------------+----------------+------------------------------+------------------------------ + 4 | 10 | 1000 | Tue Jan 04 06:36:00 2000 PST | Wed Jan 05 15:54:00 2000 PST + 4 | 20 | 1000 | Mon Jan 03 00:02:00 2000 PST | Tue Jan 04 06:34:00 2000 PST + 4 | 30 | 1000 | Sun Jan 02 18:30:12 2000 PST | Mon Jan 03 00:00:00 2000 PST + 4 | 40 | 1000 | Sun Jan 02 15:30:00 2000 PST | Sun Jan 02 18:30:00 2000 PST + 4 | 50 | 1000 | Sun Jan 02 12:29:48 2000 PST | Sun Jan 02 15:29:48 2000 PST + 4 | 60 | 1000 | Sun Jan 02 09:29:24 2000 PST | Sun Jan 02 12:29:36 2000 PST + 4 | 70 | 1000 | Sun Jan 02 06:29:00 2000 PST | Sun Jan 02 09:29:12 2000 PST + 4 | 80 | 1000 | Sun Jan 02 03:28:36 2000 PST | Sun Jan 02 06:28:48 2000 PST + 4 | 90 | 1000 | Sun Jan 02 00:28:12 2000 PST | Sun Jan 02 03:28:24 2000 PST + 4 | 100 | 1000 | Sat Jan 01 21:28:00 2000 PST | Sun Jan 02 00:28:00 2000 PST + 4 | 110 | 1000 | Sat Jan 01 18:27:48 2000 PST | Sat Jan 01 21:28:00 2000 PST + 4 | 120 | 1000 | Sat Jan 01 15:27:24 2000 PST | Sat Jan 01 18:27:36 2000 PST + 4 | 130 | 1000 | Sat Jan 01 12:27:00 2000 PST | Sat Jan 01 15:27:12 2000 PST + 4 | 140 | 775 | Thu Dec 30 02:00:00 1999 PST | Sat Jan 01 12:26:48 2000 PST +(14 rows) + +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk +----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- + 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 7 | f | 1 | f +(1 row) + +DROP TABLE metrics_compressed; +-- case11: check with segmentby column being TEXT +SELECT * FROM ( + SELECT 3 priority, 'en_US' "COLLATION" + UNION ALL (SELECT 2, collname FROM pg_collation WHERE collname ilike 'en_us%' ORDER BY collname limit 1) + UNION ALL (SELECT 1, collname FROM pg_collation WHERE collname ilike 'en_us_utf%8%' ORDER BY collname limit 1) +) c +ORDER BY priority limit 1 \gset +CREATE TABLE compressed_collation_ht( + time timestamp, + name text collate :"COLLATION", + value float +); +SELECT create_hypertable('compressed_collation_ht', 'time'); +WARNING: column type "timestamp without time zone" used for "time" does not follow best practices +NOTICE: adding not-null constraint to column "time" + create_hypertable +-------------------------------------- + (3,public,compressed_collation_ht,t) +(1 row) + +ALTER TABLE compressed_collation_ht SET + ( + timescaledb.compress, + timescaledb.compress_segmentby = 'name', + timescaledb.compress_orderby = 'time' + ); +INSERT INTO compressed_collation_ht VALUES + ('2021-01-01 01:01:01', 'á', '1'), + ('2021-01-01 01:01:02', 'b', '2'), + ('2021-01-01 01:01:03', 'ç', '2'); +SELECT compress_chunk(i) FROM show_chunks('compressed_collation_ht') i; + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_3_8_chunk +(1 row) + +BEGIN; +/* below inserts should affect existing chunk and also create new chunks */ +INSERT INTO compressed_collation_ht VALUES + ('2021-01-01 01:01:01', 'á', '1'); +INSERT INTO compressed_collation_ht (time, name, value) +SELECT time, 'á', '1' + FROM + generate_series( + '2021-01-01 01:01:01' :: timestamptz, + '2022-01-05 23:55:00+0', + '2m' + ) gtime(time); +-- check metadata +SELECT name, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 + FROM _timescaledb_internal.compress_hyper_4_9_chunk ORDER BY 1, 2; + name | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +------+-----------------------+----------------+--------------------------+-------------------------- + á | 10 | 1 | Fri Jan 01 01:01:01 2021 | Fri Jan 01 01:01:01 2021 + b | 10 | 1 | Fri Jan 01 01:01:02 2021 | Fri Jan 01 01:01:02 2021 + ç | 10 | 1 | Fri Jan 01 01:01:03 2021 | Fri Jan 01 01:01:03 2021 +(3 rows) + +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_3_8_chunk'); +LOG: statement: CALL recompress_chunk('_timescaledb_internal._hyper_3_8_chunk'); +LOG: total decompressed segments are: 1 +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT name, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 + FROM _timescaledb_internal.compress_hyper_4_9_chunk ORDER BY 1, 2; + name | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +------+-----------------------+----------------+--------------------------+-------------------------- + á | 10 | 1000 | Fri Jan 01 01:01:01 2021 | Sat Jan 02 10:15:01 2021 + á | 11 | 1000 | Sat Jan 02 10:17:01 2021 | Sun Jan 03 19:35:01 2021 + á | 12 | 1000 | Sun Jan 03 19:37:01 2021 | Tue Jan 05 04:55:01 2021 + á | 13 | 1000 | Tue Jan 05 04:57:01 2021 | Wed Jan 06 14:15:01 2021 + á | 14 | 292 | Wed Jan 06 14:17:01 2021 | Wed Jan 06 23:59:01 2021 + b | 10 | 1 | Fri Jan 01 01:01:02 2021 | Fri Jan 01 01:01:02 2021 + ç | 10 | 1 | Fri Jan 01 01:01:03 2021 | Fri Jan 01 01:01:03 2021 +(7 rows) + +ROLLBACK; +DROP TABLE compressed_collation_ht; +-- case14: check segmentby columns with NULL values +CREATE TABLE mytab (time TIMESTAMPTZ NOT NULL, a INT, b INT, c INT); +SELECT table_name FROM create_hypertable('mytab', 'time', chunk_time_interval => interval '10 day'); + table_name +------------ + mytab +(1 row) + +SELECT '2023-10-10 04:33:44.1234+05:30' as start_date \gset +-- set both segmentby columns (a,c) = (value, NULL) +INSERT INTO mytab + SELECT time, + CASE WHEN (:'start_date'::timestamptz - time < interval '1 days') THEN 1 + WHEN (:'start_date'::timestamptz - time < interval '2 days') THEN 2 + WHEN (:'start_date'::timestamptz - time < interval '3 days') THEN 3 ELSE 4 END as a + from generate_series(:'start_date'::timestamptz - interval '30 days', :'start_date'::timestamptz, interval '5 sec') as g1(time); +-- set both segmentby columns (a,c) = (NULL, value) +INSERT INTO mytab + SELECT time, NULL, 3, 4 + from generate_series(:'start_date'::timestamptz - interval '30 days', :'start_date'::timestamptz, interval '5 sec') as g1(time); +-- set both segmentby columns (a,c) = (NULL, NULL) +INSERT INTO mytab + SELECT time, NULL, 5, NULL + from generate_series(:'start_date'::timestamptz - interval '30 days', :'start_date'::timestamptz, interval '5 sec') as g1(time); +-- set both segmentby columns (a,c) = (value, value) +INSERT INTO mytab + SELECT time, 6, 5, 7 + from generate_series(:'start_date'::timestamptz - interval '30 days', :'start_date'::timestamptz, interval '5 sec') as g1(time); +-- enable compression +ALTER TABLE mytab SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'a, c' +); +-- compress chunks +SELECT compress_chunk(c.schema_name|| '.' || c.table_name) +FROM _timescaledb_catalog.chunk c, _timescaledb_catalog.hypertable ht WHERE c.hypertable_id = ht.id and ht.table_name = 'mytab' and c.compressed_chunk_id IS NULL +ORDER BY c.table_name DESC; + compress_chunk +----------------------------------------- + _timescaledb_internal._hyper_5_65_chunk + _timescaledb_internal._hyper_5_64_chunk + _timescaledb_internal._hyper_5_63_chunk + _timescaledb_internal._hyper_5_62_chunk +(4 rows) + +-- check metadata +SELECT a, c, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_6_69_chunk ORDER BY 1, 2; + a | c | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +---+---+-----------------------+----------------+-----------------------------------+----------------------------------- + 4 | | 10 | 676 | Sat Sep 09 16:03:44.1234 2023 PDT | Sat Sep 09 16:59:59.1234 2023 PDT + 6 | 7 | 10 | 676 | Sat Sep 09 16:03:44.1234 2023 PDT | Sat Sep 09 16:59:59.1234 2023 PDT + | 4 | 10 | 676 | Sat Sep 09 16:03:44.1234 2023 PDT | Sat Sep 09 16:59:59.1234 2023 PDT + | | 10 | 676 | Sat Sep 09 16:03:44.1234 2023 PDT | Sat Sep 09 16:59:59.1234 2023 PDT +(4 rows) + +BEGIN; +-- insert new values +-- should affect segments with values (4, NULL) +INSERT INTO mytab VALUES ('2023-09-10 05:00:00.1234+05:30', 4, 8, NULL); +INSERT INTO mytab VALUES ('2023-09-10 05:00:00.1234+05:30', 4, 9, NULL); +-- should affect segments with values (6, 7) +INSERT INTO mytab VALUES ('2023-09-10 05:00:00.1234+05:30', 6, 10, 7); +-- should affect segments with values (NULL, 4) +INSERT INTO mytab VALUES ('2023-09-10 05:00:00.1234+05:30', NULL, 11, 4); +INSERT INTO mytab VALUES ('2023-09-10 05:00:00.1234+05:30', NULL, 12, 4); +INSERT INTO mytab VALUES ('2023-09-10 05:00:00.1234+05:30', NULL, 13, 4); +-- should affect segments with values (NULL, NULL) +INSERT INTO mytab VALUES ('2023-09-10 05:00:00.1234+05:30', NULL, 14, NULL); +INSERT INTO mytab VALUES ('2023-09-10 05:00:00.1234+05:30', NULL, 15, NULL); +INSERT INTO mytab VALUES ('2023-09-10 05:00:00.1234+05:30', NULL, 16, NULL); +INSERT INTO mytab VALUES ('2023-09-10 05:00:00.1234+05:30', NULL, 17, NULL); +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_5_62_chunk'); +LOG: statement: CALL recompress_chunk('_timescaledb_internal._hyper_5_62_chunk'); +LOG: total decompressed segments are: 4 +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +-- check metadata after recompression +SELECT a, c, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_6_69_chunk ORDER BY 1, 2; + a | c | _ts_meta_sequence_num | _ts_meta_count | _ts_meta_min_1 | _ts_meta_max_1 +---+---+-----------------------+----------------+-----------------------------------+----------------------------------- + 4 | | 10 | 678 | Sat Sep 09 16:03:44.1234 2023 PDT | Sat Sep 09 16:59:59.1234 2023 PDT + 6 | 7 | 10 | 677 | Sat Sep 09 16:03:44.1234 2023 PDT | Sat Sep 09 16:59:59.1234 2023 PDT + | 4 | 10 | 679 | Sat Sep 09 16:03:44.1234 2023 PDT | Sat Sep 09 16:59:59.1234 2023 PDT + | | 10 | 680 | Sat Sep 09 16:03:44.1234 2023 PDT | Sat Sep 09 16:59:59.1234 2023 PDT +(4 rows) + +ROLLBACK; +DROP TABLE mytab; diff --git a/tsl/test/expected/recompress_chunk_segmentwise.out b/tsl/test/expected/recompress_chunk_segmentwise.out index b131e35a1d2..f30791a0915 100644 --- a/tsl/test/expected/recompress_chunk_segmentwise.out +++ b/tsl/test/expected/recompress_chunk_segmentwise.out @@ -118,18 +118,18 @@ select * from compression_rowcnt_view where chunk_name = :'chunk_to_compress_2'; (1 row) insert into mytab_twoseg values ('2023-01-01 19:56:20.048355+02'::timestamptz, 2, NULL, 2); -select * from :chunk_to_compress_2; +select * from :chunk_to_compress_2 order by 1; time | a | b | c -------------------------------------+---+---+--- + Sun Jan 01 09:56:20.048355 2023 PST | 2 | | 2 Sun Jan 01 11:56:20.048355 2023 PST | 2 | | 2 - Sun Jan 01 11:57:20.048355 2023 PST | 3 | | 3 Sun Jan 01 11:56:20.048355 2023 PST | 3 | | 3 - Sun Jan 01 09:56:20.048355 2023 PST | 2 | | 2 + Sun Jan 01 11:57:20.048355 2023 PST | 3 | | 3 (4 rows) SELECT compressed_chunk_schema || '.' || compressed_chunk_name as compressed_chunk_name_2 from compressed_chunk_info_view where hypertable_name = 'mytab_twoseg' \gset -select ctid, * from :compressed_chunk_name_2; +select ctid, * from :compressed_chunk_name_2 order by 1; ctid | time | a | b | c | _ts_meta_count | _ts_meta_sequence_num | _ts_meta_min_1 | _ts_meta_max_1 -------+----------------------------------------------------------------------+---+---+---+----------------+-----------------------+-------------------------------------+------------------------------------- (0,1) | BAAAApQ3/0H94wAClDf/Qf3jAAAAAQAAAAEAAAAAAAAADgAFKG/+g/vG | 2 | | 2 | 1 | 10 | Sun Jan 01 11:56:20.048355 2023 PST | Sun Jan 01 11:56:20.048355 2023 PST @@ -143,21 +143,21 @@ select _timescaledb_functions.recompress_chunk_segmentwise(:'chunk_to_compress_2 (1 row) -- verify that metadata count looks good -select ctid, * from :compressed_chunk_name_2; +select ctid, * from :compressed_chunk_name_2 order by 1; ctid | time | a | b | c | _ts_meta_count | _ts_meta_sequence_num | _ts_meta_min_1 | _ts_meta_max_1 -------+----------------------------------------------------------------------+---+---+---+----------------+-----------------------+-------------------------------------+------------------------------------- + (0,2) | BAAAApQ3/0H94//////8bHkAAAAAAgAAAAIAAAAAAAAA7gAFKHAFqwnGAAUocAzSF8U= | 3 | | 3 | 2 | 10 | Sun Jan 01 11:56:20.048355 2023 PST | Sun Jan 01 11:57:20.048355 2023 PST (0,3) | BAAAApQ2Uhq14/////5S2LgAAAAAAgAAAAIAAAAAAAAA7gAFKG/+g/vGAAUoc1jSi8U= | 2 | | 2 | 2 | 10 | Sun Jan 01 09:56:20.048355 2023 PST | Sun Jan 01 11:56:20.048355 2023 PST - (0,4) | BAAAApQ3/0H94//////8bHkAAAAAAgAAAAIAAAAAAAAA7gAFKHAFqwnGAAUocAzSF8U= | 3 | | 3 | 2 | 10 | Sun Jan 01 11:56:20.048355 2023 PST | Sun Jan 01 11:57:20.048355 2023 PST (2 rows) -- verify that initial data is returned as expected -select * from :chunk_to_compress_2; +select * from :chunk_to_compress_2 order by 1; time | a | b | c -------------------------------------+---+---+--- - Sun Jan 01 11:56:20.048355 2023 PST | 2 | | 2 Sun Jan 01 09:56:20.048355 2023 PST | 2 | | 2 - Sun Jan 01 11:57:20.048355 2023 PST | 3 | | 3 + Sun Jan 01 11:56:20.048355 2023 PST | 2 | | 2 Sun Jan 01 11:56:20.048355 2023 PST | 3 | | 3 + Sun Jan 01 11:57:20.048355 2023 PST | 3 | | 3 (4 rows) -- should still have 2 compressed rows @@ -190,7 +190,7 @@ from compressed_chunk_info_view where hypertable_name = 'mytab2' and compressed_chunk_name is not null limit 1 \gset insert into mytab2 values ('2023-01-01 00:00:02+00'::timestamptz, 0, NULL, 0); -- goes into the uncompressed chunk select show_chunks('mytab2') as chunk_to_compress_2 \gset -select ctid, * from :compressed_chunk_name_2; +select ctid, * from :compressed_chunk_name_2 order by 1; ctid | time | a | b | c | _ts_meta_count | _ts_meta_sequence_num | _ts_meta_min_1 | _ts_meta_max_1 -------+----------------------------------------------------------------------------------+---+---+---+----------------+-----------------------+------------------------------+------------------------------ (0,1) | BAAAApQ0bFLXgP/////+NjyAAAAD6AAAAAMAAAAAAAAP7gAFKHbNWYAAAAUodtDtBv8AAD5gAAAAAA== | 0 | | 0 | 1000 | 10 | Sun Jan 01 07:40:30 2023 PST | Sun Jan 01 16:00:00 2023 PST @@ -217,18 +217,18 @@ select _timescaledb_functions.recompress_chunk_segmentwise(:'chunk_to_compress_2 _timescaledb_internal._hyper_5_5_chunk (1 row) -select ctid, * from :compressed_chunk_name_2; +select ctid, * from :compressed_chunk_name_2 order by 1; ctid | time | a | b | c | _ts_meta_count | _ts_meta_sequence_num | _ts_meta_min_1 | _ts_meta_max_1 --------+------------------------------------------------------------------------------------------+---+---+---+----------------+-----------------------+------------------------------+------------------------------ - (0,10) | BAAAApQ0bFLXgP/////+NjyAAAAD6AAAAAMAAAAAAAAP7gAFKHbNWYAAAAUodtDtBv8AAD5gAAAAAA== | 0 | | 0 | 1000 | 10 | Sun Jan 01 07:40:30 2023 PST | Sun Jan 01 16:00:00 2023 PST - (0,11) | BAAAApQtcC8rgP/////+NjyAAAAD6AAAAAMAAAAAAAAP7gAFKGjVEigAAAUoaNilrv8AAD5gAAAAAA== | 0 | | 0 | 1000 | 20 | Sat Dec 31 23:20:30 2022 PST | Sun Jan 01 07:40:00 2023 PST - (0,12) | BAAAApQnSNVgAP//////4XuAAAADcgAAAAQAAAAAAADf7gAFKFrcytAAAAUoWuBeVv8AADbgAAAAAAMZdQAAPQkA | 0 | | 0 | 882 | 30 | Sat Dec 31 16:00:00 2022 PST | Sat Dec 31 23:20:00 2022 PST - (0,13) | BAAAApQ0bFLXgP/////+NjyAAAAD6AAAAAMAAAAAAAAP7gAFKHbNWYAAAAUodtDtBv8AAD5gAAAAAA== | 1 | | 1 | 1000 | 10 | Sun Jan 01 07:40:30 2023 PST | Sun Jan 01 16:00:00 2023 PST - (0,14) | BAAAApQtcC8rgP/////+NjyAAAAD6AAAAAMAAAAAAAAP7gAFKGjVEigAAAUoaNilrv8AAD5gAAAAAA== | 1 | | 1 | 1000 | 20 | Sat Dec 31 23:20:30 2022 PST | Sun Jan 01 07:40:00 2023 PST - (0,15) | BAAAApQnSNVgAP/////+NjyAAAADcQAAAAMAAAAAAAAP7gAFKFrcytAAAAUoWuBeVv8AADbwAAAAAA== | 1 | | 1 | 881 | 30 | Sat Dec 31 16:00:00 2022 PST | Sat Dec 31 23:20:00 2022 PST - (0,16) | BAAAApQ0bFLXgP/////+NjyAAAAD6AAAAAMAAAAAAAAP7gAFKHbNWYAAAAUodtDtBv8AAD5gAAAAAA== | 2 | | 2 | 1000 | 10 | Sun Jan 01 07:40:30 2023 PST | Sun Jan 01 16:00:00 2023 PST - (0,17) | BAAAApQtcC8rgP/////+NjyAAAAD6AAAAAMAAAAAAAAP7gAFKGjVEigAAAUoaNilrv8AAD5gAAAAAA== | 2 | | 2 | 1000 | 20 | Sat Dec 31 23:20:30 2022 PST | Sun Jan 01 07:40:00 2023 PST - (0,18) | BAAAApQnSNVgAP/////+NjyAAAADcQAAAAMAAAAAAAAP7gAFKFrcytAAAAUoWuBeVv8AADbwAAAAAA== | 2 | | 2 | 881 | 30 | Sat Dec 31 16:00:00 2022 PST | Sat Dec 31 23:20:00 2022 PST + (0,1) | BAAAApQ0bFLXgP/////+NjyAAAAD6AAAAAMAAAAAAAAP7gAFKHbNWYAAAAUodtDtBv8AAD5gAAAAAA== | 0 | | 0 | 1000 | 10 | Sun Jan 01 07:40:30 2023 PST | Sun Jan 01 16:00:00 2023 PST + (0,2) | BAAAApQtcC8rgP/////+NjyAAAAD6AAAAAMAAAAAAAAP7gAFKGjVEigAAAUoaNilrv8AAD5gAAAAAA== | 0 | | 0 | 1000 | 20 | Sat Dec 31 23:20:30 2022 PST | Sun Jan 01 07:40:00 2023 PST + (0,4) | BAAAApQ0bFLXgP/////+NjyAAAAD6AAAAAMAAAAAAAAP7gAFKHbNWYAAAAUodtDtBv8AAD5gAAAAAA== | 1 | | 1 | 1000 | 10 | Sun Jan 01 07:40:30 2023 PST | Sun Jan 01 16:00:00 2023 PST + (0,5) | BAAAApQtcC8rgP/////+NjyAAAAD6AAAAAMAAAAAAAAP7gAFKGjVEigAAAUoaNilrv8AAD5gAAAAAA== | 1 | | 1 | 1000 | 20 | Sat Dec 31 23:20:30 2022 PST | Sun Jan 01 07:40:00 2023 PST + (0,6) | BAAAApQnSNVgAP/////+NjyAAAADcQAAAAMAAAAAAAAP7gAFKFrcytAAAAUoWuBeVv8AADbwAAAAAA== | 1 | | 1 | 881 | 30 | Sat Dec 31 16:00:00 2022 PST | Sat Dec 31 23:20:00 2022 PST + (0,7) | BAAAApQ0bFLXgP/////+NjyAAAAD6AAAAAMAAAAAAAAP7gAFKHbNWYAAAAUodtDtBv8AAD5gAAAAAA== | 2 | | 2 | 1000 | 10 | Sun Jan 01 07:40:30 2023 PST | Sun Jan 01 16:00:00 2023 PST + (0,8) | BAAAApQtcC8rgP/////+NjyAAAAD6AAAAAMAAAAAAAAP7gAFKGjVEigAAAUoaNilrv8AAD5gAAAAAA== | 2 | | 2 | 1000 | 20 | Sat Dec 31 23:20:30 2022 PST | Sun Jan 01 07:40:00 2023 PST + (0,9) | BAAAApQnSNVgAP/////+NjyAAAADcQAAAAMAAAAAAAAP7gAFKFrcytAAAAUoWuBeVv8AADbwAAAAAA== | 2 | | 2 | 881 | 30 | Sat Dec 31 16:00:00 2022 PST | Sat Dec 31 23:20:00 2022 PST + (0,10) | BAAAApQnSNVgAP//////4XuAAAADcgAAAAQAAAAAAADf7gAFKFrcytAAAAUoWuBeVv8AADbgAAAAAAMZdQAAPQkA | 0 | | 0 | 882 | 30 | Sat Dec 31 16:00:00 2022 PST | Sat Dec 31 23:20:00 2022 PST (9 rows) -- after recompression @@ -352,9 +352,9 @@ EXPLAIN (COSTS OFF) EXECUTE p1; EXECUTE p1; time | a | b | c ------------------------------+---+---+--- + Sun Jan 01 00:00:00 2023 PST | 2 | 3 | 2 Sun Jan 01 00:00:00 2023 PST | 2 | | 2 Sun Jan 01 00:00:00 2023 PST | 2 | | 2 - Sun Jan 01 00:00:00 2023 PST | 2 | 3 | 2 (3 rows) -- verify segmentwise recompression when index exists, decompress + compress otherwise @@ -498,8 +498,8 @@ select * from :compressed_chunk_name; ----------------------------------------------------------------------+---+------------------------------------------------------------------------------------------+---+----------------+-----------------------+------------------------------+------------------------------ BAAAAneAR/JEAAACd4BH8kQAAAAAAQAAAAEAAAAAAAAADgAE7wCP5IgA | 1 | BAAAAAAAAAAAAQAAAAAAAAABAAAAAQAAAAEAAAAAAAAAAgAAAAAAAAAC | 1 | 1 | 10 | Sat Jan 01 01:00:00 2022 PST | Sat Jan 01 01:00:00 2022 PST BAAAAneAR/JEAAACd4BH8kQAAAAAAQAAAAEAAAAAAAAADgAE7wCP5IgA | 1 | BAAAAAAAAAAAAgAAAAAAAAACAAAAAQAAAAEAAAAAAAAAAwAAAAAAAAAE | 2 | 1 | 10 | Sat Jan 01 01:00:00 2022 PST | Sat Jan 01 01:00:00 2022 PST - BAAAAneAR/JEAAAAAAAAAAAAAAAAAgAAAAIAAAAAAAAA7gAE7wCP5IgAAATvAI/kh/8= | 1 | BAEAAAAAAAAABAAAAAAAAAAEAAAAAQAAAAEAAAAAAAAABAAAAAAAAAAIAAAAAgAAAAEAAAAAAAAAAQAAAAAAAAAC | | 2 | 10 | Sat Jan 01 01:00:00 2022 PST | Sat Jan 01 01:00:00 2022 PST BAAAAneAR/JEAAACd4BH8kQAAAAAAQAAAAEAAAAAAAAADgAE7wCP5IgA | 2 | BAAAAAAAAAAAAgAAAAAAAAACAAAAAQAAAAEAAAAAAAAAAwAAAAAAAAAE | 2 | 1 | 10 | Sat Jan 01 01:00:00 2022 PST | Sat Jan 01 01:00:00 2022 PST BAAAAneAR/JEAAAAAAAAAAAAAAAAAgAAAAIAAAAAAAAA7gAE7wCP5IgAAATvAI/kh/8= | 2 | BAEAAAAAAAAAAwAAAAAAAAADAAAAAQAAAAEAAAAAAAAAAwAAAAAAAAAGAAAAAgAAAAEAAAAAAAAAAQAAAAAAAAAC | 3 | 2 | 10 | Sat Jan 01 01:00:00 2022 PST | Sat Jan 01 01:00:00 2022 PST + BAAAAneAR/JEAAAAAAAAAAAAAAAAAgAAAAIAAAAAAAAA7gAE7wCP5IgAAATvAI/kh/8= | 1 | BAEAAAAAAAAABAAAAAAAAAAEAAAAAQAAAAEAAAAAAAAABAAAAAAAAAAIAAAAAgAAAAEAAAAAAAAAAQAAAAAAAAAB | | 2 | 10 | Sat Jan 01 01:00:00 2022 PST | Sat Jan 01 01:00:00 2022 PST (5 rows) diff --git a/tsl/test/isolation/expected/compression_ddl_iso.out b/tsl/test/isolation/expected/compression_ddl_iso.out index 17900aaa35d..0522faa73fd 100644 --- a/tsl/test/isolation/expected/compression_ddl_iso.out +++ b/tsl/test/isolation/expected/compression_ddl_iso.out @@ -540,12 +540,13 @@ total_chunks|number_compressed_chunks 3| 3 (1 row) -step SA: SELECT * FROM ts_device_table; +step SA: SELECT * FROM ts_device_table ORDER BY time, location; time|device|location|value ----+------+--------+----- 0| 1| 100| 20 - 1| 1| 100| 20 1| 1| 100| 100 + 1| 1| 100| 20 + 1| 1| 200| 100 2| 1| 100| 20 3| 1| 100| 20 4| 1| 100| 20 @@ -554,7 +555,6 @@ time|device|location|value 7| 1| 100| 20 8| 1| 100| 20 9| 1| 100| 20 - 1| 1| 200| 100 10| 1| 100| 20 11| 1| 100| 20 12| 1| 100| 20 @@ -661,12 +661,12 @@ total_chunks|number_compressed_chunks 3| 3 (1 row) -step SA: SELECT * FROM ts_device_table; +step SA: SELECT * FROM ts_device_table ORDER BY time, location; time|device|location|value ----+------+--------+----- 0| 1| 100| 20 - 1| 1| 100| 20 1| 1| 100| 100 + 1| 1| 100| 20 2| 1| 100| 20 3| 1| 100| 20 4| 1| 100| 20 diff --git a/tsl/test/isolation/specs/compression_ddl_iso.spec b/tsl/test/isolation/specs/compression_ddl_iso.spec index d7b9ca40194..24206633871 100644 --- a/tsl/test/isolation/specs/compression_ddl_iso.spec +++ b/tsl/test/isolation/specs/compression_ddl_iso.spec @@ -44,7 +44,7 @@ session "S" step "S1" { SELECT count(*) from ts_device_table; } step "SC1" { SELECT (count_chunktable(ch)).* FROM show_chunks('ts_device_table') AS ch ORDER BY ch::text LIMIT 1; } step "SH" { SELECT total_chunks, number_compressed_chunks from hypertable_compression_stats('ts_device_table'); } -step "SA" { SELECT * FROM ts_device_table; } +step "SA" { SELECT * FROM ts_device_table ORDER BY time, location; } session "LCT" diff --git a/tsl/test/sql/CMakeLists.txt b/tsl/test/sql/CMakeLists.txt index aae0ccaa603..8e8fa4a4ecc 100644 --- a/tsl/test/sql/CMakeLists.txt +++ b/tsl/test/sql/CMakeLists.txt @@ -128,8 +128,9 @@ if((${PG_VERSION_MAJOR} GREATER_EQUAL "14")) if(CMAKE_BUILD_TYPE MATCHES Debug) list(APPEND TEST_FILES chunk_utils_internal.sql) endif() + # UPDATE/DELETE on compressed hypertables is supported on PG version >= 14 list(APPEND TEST_FILES compression.sql compression_update_delete.sql - compression_permissions.sql) + compression_permissions.sql recompress_chunk_segmentwise-v2.sql) endif() if((${PG_VERSION_MAJOR} GREATER_EQUAL "15")) diff --git a/tsl/test/sql/recompress_chunk_segmentwise-v2.sql b/tsl/test/sql/recompress_chunk_segmentwise-v2.sql new file mode 100644 index 00000000000..2ab66c99135 --- /dev/null +++ b/tsl/test/sql/recompress_chunk_segmentwise-v2.sql @@ -0,0 +1,459 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. + +CREATE TABLE metrics_compressed(filler_1 int, filler_2 int, filler_3 int, time timestamptz NOT NULL, device_id int, v0 int, v1 int, v2 float, v3 float); +CREATE INDEX ON metrics_compressed(time); +CREATE INDEX ON metrics_compressed(device_id,time); +SELECT create_hypertable('metrics_compressed','time',create_default_indexes:=false); + +ALTER TABLE metrics_compressed DROP COLUMN filler_1; +INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, device_id+1, device_id + 2, device_id + 0.5, NULL FROM generate_series('2000-01-01 0:00:00+0'::timestamptz,'2000-01-05 23:55:00+0','2m') gtime(time), generate_series(1,5,1) gdevice(device_id); +ALTER TABLE metrics_compressed DROP COLUMN filler_2; +INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, device_id-1, device_id + 2, device_id + 0.5, NULL FROM generate_series('2000-01-06 0:00:00+0'::timestamptz,'2000-01-12 23:55:00+0','2m') gtime(time), generate_series(1,5,1) gdevice(device_id); +ALTER TABLE metrics_compressed DROP COLUMN filler_3; +INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, device_id, device_id + 2, device_id + 0.5, NULL FROM generate_series('2000-01-13 0:00:00+0'::timestamptz,'2000-01-19 23:55:00+0','2m') gtime(time), generate_series(1,5,1) gdevice(device_id); +ANALYZE metrics_compressed; + +-- compress chunks +ALTER TABLE metrics_compressed SET (timescaledb.compress, timescaledb.compress_orderby='time DESC', timescaledb.compress_segmentby='device_id'); +SELECT compress_chunk(c.schema_name|| '.' || c.table_name) +FROM _timescaledb_catalog.chunk c, _timescaledb_catalog.hypertable ht WHERE c.hypertable_id = ht.id and ht.table_name = 'metrics_compressed' and c.compressed_chunk_id IS NULL +ORDER BY c.table_name DESC; + +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + +-- case1: 1 segment affected +-- Number of segments decompressed is 1 +BEGIN; +-- affects segment with sequence_num = 10 +INSERT INTO metrics_compressed VALUES ('2000-01-05 20:06:00+05:30'::timestamp with time zone, 1, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-04 23:06:00+05:30'::timestamp with time zone, 1, 2, 3, 222.2, 333.3); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id = 1 ORDER BY 1, 2; +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +RESET client_min_messages; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id = 1 ORDER BY 1, 2; +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +ROLLBACK; + +-- case2: 2 segments affected with same segment by values +-- Number of segments decompressed is 2 +BEGIN; +-- affects segment with sequence_num = 20 +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 1, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:56:00+05:30'::timestamp with time zone, 1, 2, 3, 222.2, 333.3); +-- affects segment with sequence_num = 40 +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 1, 2, 3, 22.2, 33.3); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id = 1 ORDER BY 1, 2; +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +RESET client_min_messages; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id = 1 ORDER BY 1, 2; +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +ROLLBACK; + +-- case3: 2 different segments affected +-- Number of segments decompressed is 2 +BEGIN; +-- affects segment with sequence_num = 20 +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 1, 2, 3, 22.2, 33.3); +-- affects segment with sequence_num = 40 +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 3, 2, 3, 222.2, 333.3); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id IN (1, 3) ORDER BY 1, 2; +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +RESET client_min_messages; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id IN (1, 3) ORDER BY 1, 2; +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +ROLLBACK; + +-- case4: 2 segments affected with same segment by values + new segments created +-- Number of segments decompressed is 2 +BEGIN; +-- affects segment with sequence_num = 20 +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 1, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:56:00+05:30'::timestamp with time zone, 1, 2, 3, 222.2, 333.3); +-- affects segment with sequence_num = 40 +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 1, 2, 3, 22.2, 33.3); +-- create new segment +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 11, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:56:00+05:30'::timestamp with time zone, 11, 2, 3, 222.2, 333.3); +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 12, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 13, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 14, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 15, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 12, 2, 3, 22.2, 33.3); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk ORDER BY 1, 2; +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +RESET client_min_messages; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk ORDER BY 1, 2; +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +ROLLBACK; + +-- case5: 3 different segments affected with same segment by values + new segments created +-- Number of segments decompressed is 3 +BEGIN; +-- affects segment with sequence_num = 20 +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 1, 2, 3, 2.2, 3.3); +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:56:00+05:30'::timestamp with time zone, 2, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 3, 2, 3, 222.2, 333.3); +-- create new segment +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 11, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:56:00+05:30'::timestamp with time zone, 11, 2, 3, 222.2, 333.3); +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 12, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 13, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 14, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 15, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 12, 2, 3, 22.2, 33.3); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk ORDER BY 1, 2; +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +RESET client_min_messages; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk ORDER BY 1, 2; +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +ROLLBACK; + +-- case6: all segments affected +-- Number of segments decompressed is 5 +BEGIN; +-- affects segment with sequence_num = 20 +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 1, 2, 3, 2.2, 3.3); +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:56:00+05:30'::timestamp with time zone, 2, 2, 3, 22.2, 33.3); +-- affects segment with sequence_num = 40 +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 3, 2, 3, 222.2, 333.3); +-- affects segment with sequence_num = 10 +INSERT INTO metrics_compressed VALUES ('2000-01-04 15:46:00+05:30'::timestamp with time zone, 4, 2, 3, 2222.2, 3333.3); +INSERT INTO metrics_compressed VALUES ('2000-01-05 15:56:00+05:30'::timestamp with time zone, 5, 2, 3, 22222.2, 33333.3); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk ORDER BY 1, 2; +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +RESET client_min_messages; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk ORDER BY 1, 2; +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +ROLLBACK; + +-- case7: all segments affected + new segments created +-- Number of segments decompressed is 5 +BEGIN; +-- affects segment with sequence_num = 10 +INSERT INTO metrics_compressed VALUES ('2000-01-05 15:46:00+05:30'::timestamp with time zone, 1, 2, 3, 2.2, 3.3); +INSERT INTO metrics_compressed VALUES ('2000-01-05 15:56:00+05:30'::timestamp with time zone, 2, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-05 15:30:00+05:30'::timestamp with time zone, 3, 2, 3, 222.2, 333.3); +INSERT INTO metrics_compressed VALUES ('2000-01-05 15:46:00+05:30'::timestamp with time zone, 4, 2, 3, 2222.2, 3333.3); +INSERT INTO metrics_compressed VALUES ('2000-01-05 15:56:00+05:30'::timestamp with time zone, 5, 2, 3, 22222.2, 33333.3); +-- create new segment +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 11, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:56:00+05:30'::timestamp with time zone, 11, 2, 3, 222.2, 333.3); +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 11, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 13, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 14, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 15, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 12, 2, 3, 22.2, 33.3); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk ORDER BY 1, 2; +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +RESET client_min_messages; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk ORDER BY 1, 2; +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +ROLLBACK; + +-- case8: create new rows in existing segments +BEGIN; +-- does not affects any segments +INSERT INTO metrics_compressed VALUES ('1999-12-31 05:30:00+05:30'::timestamp with time zone, 1, 2, 3, 2.2, 3.3); +INSERT INTO metrics_compressed VALUES ('1999-12-30 15:30:00+05:30'::timestamp with time zone, 1, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('1999-12-31 05:30:00+05:30'::timestamp with time zone, 2, 2, 3, 222.2, 333.3); +INSERT INTO metrics_compressed VALUES ('1999-12-31 05:30:00+05:30'::timestamp with time zone, 3, 2, 3, 2222.2, 3333.3); +INSERT INTO metrics_compressed VALUES ('1999-12-31 05:30:00+05:30'::timestamp with time zone, 4, 2, 3, 22222.2, 33333.3); +INSERT INTO metrics_compressed VALUES ('1999-12-30 15:30:00+05:30'::timestamp with time zone, 4, 2, 3, 22222.2, 33333.3); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk ORDER BY 1, 2; +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +RESET client_min_messages; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk ORDER BY 1, 2; +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +ROLLBACK; + +-- case9: check with UPDATE/DELETE on compressed chunks +BEGIN; +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk ORDER BY id; +/* delete device id 4 */ +DELETE FROM metrics_compressed WHERE device_id = 4; +/* update device id 3 */ +UPDATE metrics_compressed SET v0 = 111, v1 = 222, v2 = 333, v3 = 444 WHERE device_id = 3; +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk ORDER BY id; +-- INSERT new segments with device_id = 4 +INSERT INTO metrics_compressed VALUES ('1999-12-31 05:30:00+05:30'::timestamp with time zone, 4, 2, 3, 2.2, 3.3); +INSERT INTO metrics_compressed VALUES ('1999-12-30 15:30:00+05:30'::timestamp with time zone, 4, 2, 3, 22.2, 33.3); +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk ORDER BY 1, 2; +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +RESET client_min_messages; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk ORDER BY 1, 2; +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk ORDER BY id; +ROLLBACK; + +-- case10: gaps between sequence no: are filled up +-- Number of segments decompressed is 2 +BEGIN; +-- affects segment with sequence_num = 20,30 +INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, V+1.5, V + 2.5, V + 0.5, NULL FROM generate_series('2000-01-02 01:26:00+05:30'::timestamptz,'2000-01-04 20:06:00+05:30','22m') gtime(time), generate_series(3,3,1) gdevice(device_id), generate_series(1,100,1) gv(V); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id = 3 ORDER BY 1, 2; +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +RESET client_min_messages; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id = 3 ORDER BY 1, 2; +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +ROLLBACK; + +-- case11: no: of new segment added is > 1000, which calls resize_affectedsegmentscxt() +-- Number of segments decompressed is 10 +BEGIN; +-- affects segment with sequence_num = 20,30 +INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, V+1.5, V + 2.5, V + 0.5, NULL FROM generate_series('2000-01-02 01:26:00+05:30'::timestamptz,'2000-01-04 20:06:00+05:30','22m') gtime(time), generate_series(1,1020,1) gdevice(device_id), generate_series(1,1,1) gv(V); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +SELECT count(distinct device_id) FROM _timescaledb_internal.compress_hyper_2_6_chunk; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +SELECT count(distinct device_id) FROM _timescaledb_internal.compress_hyper_2_6_chunk; +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +ROLLBACK; + +-- case12: no: of affected existing segment is > 1000, which calls resize_affectedsegmentscxt() +INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, V+1.5, V + 2.5, V + 0.5, NULL FROM generate_series('2000-01-02 01:26:00+05:30'::timestamptz,'2000-01-04 20:06:00+05:30','22m') gtime(time), generate_series(1,1020,1) gdevice(device_id), generate_series(1,1,1) gv(V); +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +BEGIN; +-- affects segment with sequence_num = 20,30 +INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, V+1.5, V + 2.5, V + 0.5, NULL FROM generate_series('2000-01-02 01:26:00+05:30'::timestamptz,'2000-01-04 20:06:00+05:30','22m') gtime(time), generate_series(1,1020,1) gdevice(device_id), generate_series(1,1,1) gv(V); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +SELECT count(distinct device_id) FROM _timescaledb_internal.compress_hyper_2_6_chunk; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +SELECT count(distinct device_id) FROM _timescaledb_internal.compress_hyper_2_6_chunk; +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +ROLLBACK; + +-- case13: gaps between sequence no: are filled up, thus call legacy recompression +BEGIN; +-- affects segment with sequence_num = 20,30 +INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, V+1.5, V + 2.5, V + 0.5, NULL FROM generate_series('2000-01-02 01:26:00+05:30'::timestamptz,'2000-01-04 20:06:00+05:30','22m') gtime(time), generate_series(3,3,1) gdevice(device_id), generate_series(1,100,1) gv(V); +INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, V+1.5, V + 2.5, V + 0.5, NULL FROM generate_series('2000-01-02 01:26:00+05:30'::timestamptz,'2000-01-04 20:06:00+05:30','22m') gtime(time), generate_series(3,3,1) gdevice(device_id), generate_series(1,100,1) gv(V); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id = 3 ORDER BY 1, 2; +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id = 3 ORDER BY 1, 2; +COMMIT; +BEGIN; +INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, V+1.5, V + 2.5, V + 0.5, NULL FROM generate_series('2000-01-02 01:26:00+05:30'::timestamptz,'2000-01-04 20:06:00+05:30','22m') gtime(time), generate_series(3,3,1) gdevice(device_id), generate_series(1,100,1) gv(V); +-- should invoke legacy recompression +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id = 3 ORDER BY 1, 2; +COMMIT; +BEGIN; +INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, V+1.5, V + 2.5, V + 0.5, NULL FROM generate_series('2000-01-02 01:26:00+05:30'::timestamptz,'2000-01-04 20:06:00+05:30','22m') gtime(time), generate_series(3,3,1) gdevice(device_id), generate_series(1,100,1) gv(V); +-- should invoke optimized recompression as there are enough gaps in sequence numbers in compressed chunk +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id = 3 ORDER BY 1, 2; +COMMIT; +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +RESET client_min_messages; + +-- check for heapscan +DROP INDEX _timescaledb_internal.compress_hyper_2_6_chunk__compressed_hypertable_2_device_id__ts; +-- INSERT new segments with device_id = 4 +INSERT INTO metrics_compressed VALUES ('1999-12-31 05:30:00+05:30'::timestamp with time zone, 4, 2, 3, 2.2, 3.3); +INSERT INTO metrics_compressed VALUES ('1999-12-30 15:30:00+05:30'::timestamp with time zone, 4, 2, 3, 22.2, 33.3); +-- does not affects any segments, append to existing segments +INSERT INTO metrics_compressed VALUES ('1999-12-31 05:30:00+05:30'::timestamp with time zone, 1, 2, 3, 2.2, 3.3); +INSERT INTO metrics_compressed VALUES ('1999-12-30 15:30:00+05:30'::timestamp with time zone, 1, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('1999-12-31 05:30:00+05:30'::timestamp with time zone, 2, 2, 3, 222.2, 333.3); +INSERT INTO metrics_compressed VALUES ('1999-12-31 05:30:00+05:30'::timestamp with time zone, 3, 2, 3, 2222.2, 3333.3); +INSERT INTO metrics_compressed VALUES ('1999-12-31 05:30:00+05:30'::timestamp with time zone, 4, 2, 3, 22222.2, 33333.3); +INSERT INTO metrics_compressed VALUES ('1999-12-30 15:30:00+05:30'::timestamp with time zone, 4, 2, 3, 22222.2, 33333.3); +-- affects segment with sequence_num = 30 +INSERT INTO metrics_compressed(time,device_id,v0,v1,v2,v3) SELECT time, device_id, device_id+1, device_id + 2, device_id + 0.5, NULL FROM generate_series('2000-01-02 01:26:00+05:30'::timestamptz,'2000-01-03 10:44:00+05:30','12s') gtime(time), generate_series(4,4,1) gdevice(device_id); +-- create new segment +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 11, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:56:00+05:30'::timestamp with time zone, 11, 2, 3, 222.2, 333.3); +INSERT INTO metrics_compressed VALUES ('2000-01-03 15:46:00+05:30'::timestamp with time zone, 11, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 13, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 14, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 15, 2, 3, 22.2, 33.3); +INSERT INTO metrics_compressed VALUES ('2000-01-01 15:30:00+05:30'::timestamp with time zone, 12, 2, 3, 22.2, 33.3); +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_6_chunk WHERE device_id = 4 ORDER BY 1, 2; +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_1_1_chunk'); +RESET client_min_messages; +SELECT device_id, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_2_7_chunk WHERE device_id = 4 ORDER BY 1, 2; +-- check compression status +SELECT * FROM _timescaledb_catalog.chunk WHERE table_name = '_hyper_1_1_chunk' ORDER BY id; + +DROP TABLE metrics_compressed; + +-- case11: check with segmentby column being TEXT +SELECT * FROM ( + SELECT 3 priority, 'en_US' "COLLATION" + UNION ALL (SELECT 2, collname FROM pg_collation WHERE collname ilike 'en_us%' ORDER BY collname limit 1) + UNION ALL (SELECT 1, collname FROM pg_collation WHERE collname ilike 'en_us_utf%8%' ORDER BY collname limit 1) +) c +ORDER BY priority limit 1 \gset + +CREATE TABLE compressed_collation_ht( + time timestamp, + name text collate :"COLLATION", + value float +); + +SELECT create_hypertable('compressed_collation_ht', 'time'); + +ALTER TABLE compressed_collation_ht SET + ( + timescaledb.compress, + timescaledb.compress_segmentby = 'name', + timescaledb.compress_orderby = 'time' + ); + +INSERT INTO compressed_collation_ht VALUES + ('2021-01-01 01:01:01', 'á', '1'), + ('2021-01-01 01:01:02', 'b', '2'), + ('2021-01-01 01:01:03', 'ç', '2'); + +SELECT compress_chunk(i) FROM show_chunks('compressed_collation_ht') i; + +BEGIN; +/* below inserts should affect existing chunk and also create new chunks */ +INSERT INTO compressed_collation_ht VALUES + ('2021-01-01 01:01:01', 'á', '1'); + +INSERT INTO compressed_collation_ht (time, name, value) +SELECT time, 'á', '1' + FROM + generate_series( + '2021-01-01 01:01:01' :: timestamptz, + '2022-01-05 23:55:00+0', + '2m' + ) gtime(time); + +-- check metadata +SELECT name, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 + FROM _timescaledb_internal.compress_hyper_4_9_chunk ORDER BY 1, 2; + +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_3_8_chunk'); +RESET client_min_messages; + +SELECT name, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 + FROM _timescaledb_internal.compress_hyper_4_9_chunk ORDER BY 1, 2; + +ROLLBACK; +DROP TABLE compressed_collation_ht; + +-- case14: check segmentby columns with NULL values +CREATE TABLE mytab (time TIMESTAMPTZ NOT NULL, a INT, b INT, c INT); +SELECT table_name FROM create_hypertable('mytab', 'time', chunk_time_interval => interval '10 day'); +SELECT '2023-10-10 04:33:44.1234+05:30' as start_date \gset +-- set both segmentby columns (a,c) = (value, NULL) +INSERT INTO mytab + SELECT time, + CASE WHEN (:'start_date'::timestamptz - time < interval '1 days') THEN 1 + WHEN (:'start_date'::timestamptz - time < interval '2 days') THEN 2 + WHEN (:'start_date'::timestamptz - time < interval '3 days') THEN 3 ELSE 4 END as a + from generate_series(:'start_date'::timestamptz - interval '30 days', :'start_date'::timestamptz, interval '5 sec') as g1(time); + +-- set both segmentby columns (a,c) = (NULL, value) +INSERT INTO mytab + SELECT time, NULL, 3, 4 + from generate_series(:'start_date'::timestamptz - interval '30 days', :'start_date'::timestamptz, interval '5 sec') as g1(time); + +-- set both segmentby columns (a,c) = (NULL, NULL) +INSERT INTO mytab + SELECT time, NULL, 5, NULL + from generate_series(:'start_date'::timestamptz - interval '30 days', :'start_date'::timestamptz, interval '5 sec') as g1(time); + +-- set both segmentby columns (a,c) = (value, value) +INSERT INTO mytab + SELECT time, 6, 5, 7 + from generate_series(:'start_date'::timestamptz - interval '30 days', :'start_date'::timestamptz, interval '5 sec') as g1(time); + +-- enable compression +ALTER TABLE mytab SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'a, c' +); +-- compress chunks +SELECT compress_chunk(c.schema_name|| '.' || c.table_name) +FROM _timescaledb_catalog.chunk c, _timescaledb_catalog.hypertable ht WHERE c.hypertable_id = ht.id and ht.table_name = 'mytab' and c.compressed_chunk_id IS NULL +ORDER BY c.table_name DESC; + +-- check metadata +SELECT a, c, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_6_69_chunk ORDER BY 1, 2; + +BEGIN; +-- insert new values +-- should affect segments with values (4, NULL) +INSERT INTO mytab VALUES ('2023-09-10 05:00:00.1234+05:30', 4, 8, NULL); +INSERT INTO mytab VALUES ('2023-09-10 05:00:00.1234+05:30', 4, 9, NULL); + +-- should affect segments with values (6, 7) +INSERT INTO mytab VALUES ('2023-09-10 05:00:00.1234+05:30', 6, 10, 7); + +-- should affect segments with values (NULL, 4) +INSERT INTO mytab VALUES ('2023-09-10 05:00:00.1234+05:30', NULL, 11, 4); +INSERT INTO mytab VALUES ('2023-09-10 05:00:00.1234+05:30', NULL, 12, 4); +INSERT INTO mytab VALUES ('2023-09-10 05:00:00.1234+05:30', NULL, 13, 4); + +-- should affect segments with values (NULL, NULL) +INSERT INTO mytab VALUES ('2023-09-10 05:00:00.1234+05:30', NULL, 14, NULL); +INSERT INTO mytab VALUES ('2023-09-10 05:00:00.1234+05:30', NULL, 15, NULL); +INSERT INTO mytab VALUES ('2023-09-10 05:00:00.1234+05:30', NULL, 16, NULL); +INSERT INTO mytab VALUES ('2023-09-10 05:00:00.1234+05:30', NULL, 17, NULL); + +set client_min_messages TO LOG; +CALL recompress_chunk('_timescaledb_internal._hyper_5_62_chunk'); +RESET client_min_messages; + +-- check metadata after recompression +SELECT a, c, _ts_meta_sequence_num, _ts_meta_count, _ts_meta_min_1, _ts_meta_max_1 FROM _timescaledb_internal.compress_hyper_6_69_chunk ORDER BY 1, 2; + +ROLLBACK; +DROP TABLE mytab; diff --git a/tsl/test/sql/recompress_chunk_segmentwise.sql b/tsl/test/sql/recompress_chunk_segmentwise.sql index 660169d8e89..ca0d4ddf5a6 100644 --- a/tsl/test/sql/recompress_chunk_segmentwise.sql +++ b/tsl/test/sql/recompress_chunk_segmentwise.sql @@ -88,20 +88,20 @@ select * from compression_rowcnt_view where chunk_name = :'chunk_to_compress_2'; insert into mytab_twoseg values ('2023-01-01 19:56:20.048355+02'::timestamptz, 2, NULL, 2); -select * from :chunk_to_compress_2; +select * from :chunk_to_compress_2 order by 1; SELECT compressed_chunk_schema || '.' || compressed_chunk_name as compressed_chunk_name_2 from compressed_chunk_info_view where hypertable_name = 'mytab_twoseg' \gset -select ctid, * from :compressed_chunk_name_2; +select ctid, * from :compressed_chunk_name_2 order by 1; select _timescaledb_functions.recompress_chunk_segmentwise(:'chunk_to_compress_2'); -- verify that metadata count looks good -select ctid, * from :compressed_chunk_name_2; +select ctid, * from :compressed_chunk_name_2 order by 1; -- verify that initial data is returned as expected -select * from :chunk_to_compress_2; +select * from :chunk_to_compress_2 order by 1; -- should still have 2 compressed rows select * from compression_rowcnt_view where chunk_name = :'chunk_to_compress_2'; @@ -127,13 +127,13 @@ insert into mytab2 values ('2023-01-01 00:00:02+00'::timestamptz, 0, NULL, 0); - select show_chunks('mytab2') as chunk_to_compress_2 \gset -select ctid, * from :compressed_chunk_name_2; +select ctid, * from :compressed_chunk_name_2 order by 1; -- after compression select * from compression_rowcnt_view where chunk_name = :'chunk_to_compress_2'; select _timescaledb_functions.recompress_chunk_segmentwise(:'chunk_to_compress_2'); -select ctid, * from :compressed_chunk_name_2; +select ctid, * from :compressed_chunk_name_2 order by 1; -- after recompression select * from compression_rowcnt_view where chunk_name = :'chunk_to_compress_2';