Increase max rows per compression to 4096

test/src/adt_tests.c

@@ -22,7 +22,7 @@ TS_FUNCTION_INFO_V1(ts_test_adts);
 /* We have to stub this for the unit tests. */
 #ifndef CheckCompressedData
 #define CheckCompressedData(X) Assert(X)
-#define GLOBAL_MAX_ROWS_PER_COMPRESSION 1015
+#define GLOBAL_MAX_ROWS_PER_COMPRESSION 1087
 #endif
 
 #include <adts/bit_array.h>

tsl/src/compression/compression.h

@@ -31,7 +31,7 @@ typedef struct BulkInsertStateData *BulkInsertState;
 	char vl_len_[4];                                                                               \
 	uint8 compression_algorithm
 
-#define MAX_ROWS_PER_COMPRESSION 1000
+#define MAX_ROWS_PER_COMPRESSION 1087
 /* gap in sequence id between rows, potential for adding rows in gap later */
 #define SEQUENCE_NUM_GAP 10
 #define COMPRESSIONCOL_IS_SEGMENT_BY(col) ((col)->segmentby_column_index > 0)
@@ -402,6 +402,6 @@ consumeCompressedData(StringInfo si, int bytes)
  * Normal compression uses 1k rows, but the regression tests use up to 1015.
  * We use this limit for sanity checks in case the compressed data is corrupt.
  */
-#define GLOBAL_MAX_ROWS_PER_COMPRESSION 1015
+#define GLOBAL_MAX_ROWS_PER_COMPRESSION 1087
 
 #endif

tsl/src/compression/gorilla.c

@@ -850,16 +850,16 @@ unpack_leading_zeros_array(BitArray *bitarray, uint8 *restrict dest)
 	 * We do have to check that the result fits into the maximum number of rows,
 	 * because we get the length from user input.
 	 */
-	const int16 n_bytes_packed = bitarray->buckets.num_elements * sizeof(uint64);
-	const int16 n_lanes = (n_bytes_packed + LANE_INPUTS - 1) / LANE_INPUTS;
-	const int16 n_outputs = n_lanes * LANE_OUTPUTS;
+	const uint16 n_bytes_packed = bitarray->buckets.num_elements * sizeof(uint64);
+	const uint16 n_lanes = (n_bytes_packed + LANE_INPUTS - 1) / LANE_INPUTS;
+	const uint16 n_outputs = n_lanes * LANE_OUTPUTS;
 	CheckCompressedData(n_outputs <= MAX_NUM_LEADING_ZEROS_PADDED_N64);
 
-	for (int lane = 0; lane < n_lanes; lane++)
+	for (uint16 lane = 0; lane < n_lanes; lane++)
 	{
 		uint8 *restrict lane_dest = &dest[lane * LANE_OUTPUTS];
 		const uint8 *restrict lane_src = &((uint8 *) bitarray->buckets.data)[lane * LANE_INPUTS];
-		for (int output_in_lane = 0; output_in_lane < LANE_OUTPUTS; output_in_lane++)
+		for (uint16 output_in_lane = 0; output_in_lane < LANE_OUTPUTS; output_in_lane++)
 		{
 			const int startbit_abs = output_in_lane * BITS_PER_LEADING_ZEROS;
 			const int startbit_rel = startbit_abs % 8;

tsl/src/nodes/decompress_chunk/exec.c

@@ -691,14 +691,17 @@ perform_vectorized_sum_int4(DecompressChunkState *chunk_state, Aggref *aggref)
 			MemoryContextReset(chunk_state->bulk_decompression_context);
 			MemoryContextSwitchTo(context_before_decompression);
 
-			/* A compressed batch consists of 1000 tuples (see MAX_ROWS_PER_COMPRESSION). The
-			 * attribute value is a int32 with a max value of 2^32. Even if all tuples have the max
-			 * value, the max sum is = 1000 * 2^32 < 2^10 * 2^32 = 2^42. This is smaller than 2^64,
-			 * which is the max value of the int64 variable. The same is true for negative values).
-			 * Therefore, we don't need to check for overflows within the loop, which would slow
-			 * down the calculation. */
-			Assert(arrow->length <= MAX_ROWS_PER_COMPRESSION);
-			Assert(MAX_ROWS_PER_COMPRESSION <= 1024);
+			/*
+			 * We accumulate the sum as int64, so we can sum INT_MAX = 2^31 - 1
+			 * at least 2^31 times without incurrint an overflow of the int64
+			 * accumulator. The same is true for negative numbers. The
+			 * compressed batch size is currently capped at 1000 rows, but even
+			 * if it's changed in the future, it's unlikely that we support
+			 * batches larger than 65536 rows, not to mention 2^31. Therefore,
+			 * we don't need to check for overflows within the loop, which would
+			 * slow down the calculation.
+			 */
+			Assert(arrow->length <= INT_MAX);
 
 			int64 batch_sum = 0;
 			for (int i = 0; i < arrow->length; i++)