From 1c908fd8a1f94861369a8126521face5f82f93d6 Mon Sep 17 00:00:00 2001 From: Antony Stubbs Date: Fri, 23 Jul 2021 15:32:49 +0100 Subject: [PATCH] minor: Fix line endings to dos2unix `find ./ -type f -print0 | xargs -0 dos2unix --` --- .../io/confluent/csid/utils/JavaUtils.java | 52 +- .../JStreamParallelStreamProcessor.java | 56 +- .../ParallelConsumerOptions.java | 330 +++---- .../internal/InternalRuntimeError.java | 46 +- .../internal/ProducerManager.java | 656 +++++++------- .../parallelconsumer/internal/State.java | 42 +- .../BitSetEncodingNotSupportedException.java | 24 +- .../EncodingNotSupportedException.java | 20 +- .../offsets/OffsetBitSet.java | 162 ++-- .../offsets/OffsetDecodingError.java | 20 +- .../offsets/OffsetEncoding.java | 122 +-- .../offsets/OffsetMapCodecManager.java | 530 +++++------ .../offsets/OffsetRunLength.java | 304 +++---- .../offsets/OffsetSimpleSerialisation.java | 248 ++--- .../offsets/RunLengthEncoder.java | 298 +++--- .../RunlengthV1EncodingNotSupported.java | 20 +- .../state/WorkMailBoxManager.java | 274 +++--- .../CloseAndOpenOffsetTest.java | 792 ++++++++-------- .../csid/utils/ProgressBarUtils.java | 74 +- .../parallelconsumer/FakeRuntimeError.java | 28 +- .../ParallelEoSStreamProcessorTestBase.java | 852 +++++++++--------- .../offsets/BitsetEncodingTest.java | 122 +-- .../OffsetEncodingBackPressureTest.java | 530 +++++------ .../offsets/OffsetEncodingTests.java | 558 ++++++------ .../offsets/RunLengthEncoderTest.java | 288 +++--- .../parallelconsumer/vertx/WireMockUtils.java | 64 +- 26 files changed, 3256 insertions(+), 3256 deletions(-) diff --git a/parallel-consumer-core/src/main/java/io/confluent/csid/utils/JavaUtils.java b/parallel-consumer-core/src/main/java/io/confluent/csid/utils/JavaUtils.java index c63bf7398..54f9cd1ff 100644 --- a/parallel-consumer-core/src/main/java/io/confluent/csid/utils/JavaUtils.java +++ b/parallel-consumer-core/src/main/java/io/confluent/csid/utils/JavaUtils.java @@ -1,26 +1,26 @@ -package io.confluent.csid.utils; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ - -import io.confluent.parallelconsumer.internal.InternalRuntimeError; - -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -public class JavaUtils { - public static Optional getLast(final List commitHistory) { - if (commitHistory.isEmpty()) return Optional.empty(); - return Optional.of(commitHistory.get(commitHistory.size() - 1)); - } - - public static Optional getOnlyOne(final Map stringMapMap) { - if (stringMapMap.isEmpty()) return Optional.empty(); - Collection values = stringMapMap.values(); - if (values.size() > 1) throw new InternalRuntimeError("More than one element"); - return Optional.of(values.iterator().next()); - } -} +package io.confluent.csid.utils; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ + +import io.confluent.parallelconsumer.internal.InternalRuntimeError; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +public class JavaUtils { + public static Optional getLast(final List commitHistory) { + if (commitHistory.isEmpty()) return Optional.empty(); + return Optional.of(commitHistory.get(commitHistory.size() - 1)); + } + + public static Optional getOnlyOne(final Map stringMapMap) { + if (stringMapMap.isEmpty()) return Optional.empty(); + Collection values = stringMapMap.values(); + if (values.size() > 1) throw new InternalRuntimeError("More than one element"); + return Optional.of(values.iterator().next()); + } +} diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/JStreamParallelStreamProcessor.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/JStreamParallelStreamProcessor.java index 82f99d98f..079e81ea8 100644 --- a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/JStreamParallelStreamProcessor.java +++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/JStreamParallelStreamProcessor.java @@ -1,28 +1,28 @@ -package io.confluent.parallelconsumer; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ -import io.confluent.parallelconsumer.internal.DrainingCloseable; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.producer.ProducerRecord; - -import java.util.List; -import java.util.function.Function; -import java.util.stream.Stream; - -public interface JStreamParallelStreamProcessor extends DrainingCloseable { - - static JStreamParallelStreamProcessor createJStreamEosStreamProcessor(ParallelConsumerOptions options) { - return new JStreamParallelEoSStreamProcessor<>(options); - } - - /** - * Like {@link ParallelEoSStreamProcessor#pollAndProduceMany} but instead of callbacks, streams the results instead, - * after the produce result is ack'd by Kafka. - * - * @return a stream of results of applying the function to the polled records - */ - Stream> pollProduceAndStream(Function, - List>> userFunction); -} +package io.confluent.parallelconsumer; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ +import io.confluent.parallelconsumer.internal.DrainingCloseable; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.producer.ProducerRecord; + +import java.util.List; +import java.util.function.Function; +import java.util.stream.Stream; + +public interface JStreamParallelStreamProcessor extends DrainingCloseable { + + static JStreamParallelStreamProcessor createJStreamEosStreamProcessor(ParallelConsumerOptions options) { + return new JStreamParallelEoSStreamProcessor<>(options); + } + + /** + * Like {@link ParallelEoSStreamProcessor#pollAndProduceMany} but instead of callbacks, streams the results instead, + * after the produce result is ack'd by Kafka. + * + * @return a stream of results of applying the function to the polled records + */ + Stream> pollProduceAndStream(Function, + List>> userFunction); +} diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ParallelConsumerOptions.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ParallelConsumerOptions.java index a3ae3becf..b5e5dc950 100644 --- a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ParallelConsumerOptions.java +++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/ParallelConsumerOptions.java @@ -1,165 +1,165 @@ -package io.confluent.parallelconsumer; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ -import io.confluent.parallelconsumer.state.WorkContainer; -import lombok.Builder; -import lombok.Getter; -import lombok.ToString; -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.clients.producer.Producer; - -import java.time.Duration; -import java.util.Objects; - -import static io.confluent.csid.utils.StringUtils.msg; -import static io.confluent.parallelconsumer.ParallelConsumerOptions.CommitMode.PERIODIC_TRANSACTIONAL_PRODUCER; - -/** - * The options for the {@link ParallelEoSStreamProcessor} system. - * - * @see #builder() - * @see ParallelConsumerOptions.ParallelConsumerOptionsBuilder - */ -@Getter -@Builder(toBuilder = true) -@ToString -public class ParallelConsumerOptions { - - /** - * Required parameter for all use. - */ - private final Consumer consumer; - - /** - * Supplying a producer is only needed if using the produce flows. - * - * @see ParallelStreamProcessor - */ - private final Producer producer; - - /** - * Path to Managed executor service for Java EE - */ - @Builder.Default - private final String managedExecutorService = "java:comp/DefaultManagedExecutorService"; - - /** - * Path to Managed thread factory for Java EE - */ - @Builder.Default - private final String managedThreadFactory = "java:comp/DefaultManagedThreadFactory"; - - /** - * The ordering guarantee to use. - */ - public enum ProcessingOrder { - - /** - * No ordering is guaranteed, not even partition order. Fastest. Concurrency is at most the max number of - * concurrency or max number of uncommitted messages, limited by the max concurrency or uncommitted settings. - */ - UNORDERED, - - /** - * Process messages within a partition in order, but process multiple partitions in parallel. Similar to running - * more consumer for a topic. Concurrency is at most the number of partitions. - */ - PARTITION, - - /** - * Process messages in key order. Concurrency is at most the number of unique keys in a topic, limited by the - * max concurrency or uncommitted settings. - */ - KEY - } - - /** - * The type of commit to be made, with either a transactions configured Producer where messages produced are - * committed back to the Broker along with the offsets they originated from, or with the faster simpler Consumer - * offset system either synchronously or asynchronously - */ - public enum CommitMode { - - /** - * Periodically commits through the Producer using transactions. Slowest of the options, but no duplicates in - * Kafka guaranteed (message replay may cause duplicates in external systems which is unavoidable with Kafka). - *

- * This is separate from using an IDEMPOTENT Producer, which can be used, along with {@link - * CommitMode#PERIODIC_CONSUMER_SYNC} or {@link CommitMode#PERIODIC_CONSUMER_ASYNCHRONOUS}. - */ - PERIODIC_TRANSACTIONAL_PRODUCER, - - /** - * Periodically synchronous commits with the Consumer. Much faster than {@link - * #PERIODIC_TRANSACTIONAL_PRODUCER}. Slower but potentially less duplicates than {@link - * #PERIODIC_CONSUMER_ASYNCHRONOUS} upon replay. - */ - PERIODIC_CONSUMER_SYNC, - - /** - * Periodically commits offsets asynchronously. The fastest option, under normal conditions will have few or no - * duplicates. Under failure recovery may have more duplicates than {@link #PERIODIC_CONSUMER_SYNC}. - */ - PERIODIC_CONSUMER_ASYNCHRONOUS - - } - - /** - * The {@link ProcessingOrder} type to use - */ - @Builder.Default - private final ProcessingOrder ordering = ProcessingOrder.KEY; - - /** - * The {@link CommitMode} to be used - */ - @Builder.Default - private final CommitMode commitMode = CommitMode.PERIODIC_CONSUMER_ASYNCHRONOUS; - - /** - * Controls the maximum degree of concurrency to occur. Used to limit concurrent calls to external systems to a - * maximum to prevent overloading them or to a degree, using up quotas. - *

- * A note on quotas - if your quota is expressed as maximum concurrent calls, this works well. If it's limited in - * total requests / sec, this may still overload the system. See towards the distributed rate limiting feature for - * this to be properly addressed: https://github.com/confluentinc/parallel-consumer/issues/24 Add distributed rate - * limiting support #24. - *

- * In the core module, this sets the number of threads to use in the core's thread pool. - *

- * It's recommended to set this quite high, much higher than core count, as it's expected that these threads will - * spend most of their time blocked waiting for IO. For automatic setting of this variable, look out for issue - * https://github.com/confluentinc/parallel-consumer/issues/21 Dynamic concurrency control with flow control or tcp - * congestion control theory #21. - */ - @Builder.Default - private final int maxConcurrency = 16; - - /** - * When a message fails, how long the system should wait before trying that message again. - */ - @Builder.Default - private final Duration defaultMessageRetryDelay = Duration.ofSeconds(1); - - public void validate() { - Objects.requireNonNull(consumer, "A consumer must be supplied"); - - if (isUsingTransactionalProducer() && producer == null) { - throw new IllegalArgumentException(msg("Wanting to use Transaction Producer mode ({}) without supplying a Producer instance", - commitMode)); - } - - // - WorkContainer.setDefaultRetryDelay(getDefaultMessageRetryDelay()); - } - - public boolean isUsingTransactionalProducer() { - return commitMode.equals(PERIODIC_TRANSACTIONAL_PRODUCER); - } - - public boolean isProducerSupplied() { - return getProducer() != null; - } -} +package io.confluent.parallelconsumer; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ +import io.confluent.parallelconsumer.state.WorkContainer; +import lombok.Builder; +import lombok.Getter; +import lombok.ToString; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.producer.Producer; + +import java.time.Duration; +import java.util.Objects; + +import static io.confluent.csid.utils.StringUtils.msg; +import static io.confluent.parallelconsumer.ParallelConsumerOptions.CommitMode.PERIODIC_TRANSACTIONAL_PRODUCER; + +/** + * The options for the {@link ParallelEoSStreamProcessor} system. + * + * @see #builder() + * @see ParallelConsumerOptions.ParallelConsumerOptionsBuilder + */ +@Getter +@Builder(toBuilder = true) +@ToString +public class ParallelConsumerOptions { + + /** + * Required parameter for all use. + */ + private final Consumer consumer; + + /** + * Supplying a producer is only needed if using the produce flows. + * + * @see ParallelStreamProcessor + */ + private final Producer producer; + + /** + * Path to Managed executor service for Java EE + */ + @Builder.Default + private final String managedExecutorService = "java:comp/DefaultManagedExecutorService"; + + /** + * Path to Managed thread factory for Java EE + */ + @Builder.Default + private final String managedThreadFactory = "java:comp/DefaultManagedThreadFactory"; + + /** + * The ordering guarantee to use. + */ + public enum ProcessingOrder { + + /** + * No ordering is guaranteed, not even partition order. Fastest. Concurrency is at most the max number of + * concurrency or max number of uncommitted messages, limited by the max concurrency or uncommitted settings. + */ + UNORDERED, + + /** + * Process messages within a partition in order, but process multiple partitions in parallel. Similar to running + * more consumer for a topic. Concurrency is at most the number of partitions. + */ + PARTITION, + + /** + * Process messages in key order. Concurrency is at most the number of unique keys in a topic, limited by the + * max concurrency or uncommitted settings. + */ + KEY + } + + /** + * The type of commit to be made, with either a transactions configured Producer where messages produced are + * committed back to the Broker along with the offsets they originated from, or with the faster simpler Consumer + * offset system either synchronously or asynchronously + */ + public enum CommitMode { + + /** + * Periodically commits through the Producer using transactions. Slowest of the options, but no duplicates in + * Kafka guaranteed (message replay may cause duplicates in external systems which is unavoidable with Kafka). + *

+ * This is separate from using an IDEMPOTENT Producer, which can be used, along with {@link + * CommitMode#PERIODIC_CONSUMER_SYNC} or {@link CommitMode#PERIODIC_CONSUMER_ASYNCHRONOUS}. + */ + PERIODIC_TRANSACTIONAL_PRODUCER, + + /** + * Periodically synchronous commits with the Consumer. Much faster than {@link + * #PERIODIC_TRANSACTIONAL_PRODUCER}. Slower but potentially less duplicates than {@link + * #PERIODIC_CONSUMER_ASYNCHRONOUS} upon replay. + */ + PERIODIC_CONSUMER_SYNC, + + /** + * Periodically commits offsets asynchronously. The fastest option, under normal conditions will have few or no + * duplicates. Under failure recovery may have more duplicates than {@link #PERIODIC_CONSUMER_SYNC}. + */ + PERIODIC_CONSUMER_ASYNCHRONOUS + + } + + /** + * The {@link ProcessingOrder} type to use + */ + @Builder.Default + private final ProcessingOrder ordering = ProcessingOrder.KEY; + + /** + * The {@link CommitMode} to be used + */ + @Builder.Default + private final CommitMode commitMode = CommitMode.PERIODIC_CONSUMER_ASYNCHRONOUS; + + /** + * Controls the maximum degree of concurrency to occur. Used to limit concurrent calls to external systems to a + * maximum to prevent overloading them or to a degree, using up quotas. + *

+ * A note on quotas - if your quota is expressed as maximum concurrent calls, this works well. If it's limited in + * total requests / sec, this may still overload the system. See towards the distributed rate limiting feature for + * this to be properly addressed: https://github.com/confluentinc/parallel-consumer/issues/24 Add distributed rate + * limiting support #24. + *

+ * In the core module, this sets the number of threads to use in the core's thread pool. + *

+ * It's recommended to set this quite high, much higher than core count, as it's expected that these threads will + * spend most of their time blocked waiting for IO. For automatic setting of this variable, look out for issue + * https://github.com/confluentinc/parallel-consumer/issues/21 Dynamic concurrency control with flow control or tcp + * congestion control theory #21. + */ + @Builder.Default + private final int maxConcurrency = 16; + + /** + * When a message fails, how long the system should wait before trying that message again. + */ + @Builder.Default + private final Duration defaultMessageRetryDelay = Duration.ofSeconds(1); + + public void validate() { + Objects.requireNonNull(consumer, "A consumer must be supplied"); + + if (isUsingTransactionalProducer() && producer == null) { + throw new IllegalArgumentException(msg("Wanting to use Transaction Producer mode ({}) without supplying a Producer instance", + commitMode)); + } + + // + WorkContainer.setDefaultRetryDelay(getDefaultMessageRetryDelay()); + } + + public boolean isUsingTransactionalProducer() { + return commitMode.equals(PERIODIC_TRANSACTIONAL_PRODUCER); + } + + public boolean isProducerSupplied() { + return getProducer() != null; + } +} diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/internal/InternalRuntimeError.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/internal/InternalRuntimeError.java index 3cbc45da5..b022db75b 100644 --- a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/internal/InternalRuntimeError.java +++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/internal/InternalRuntimeError.java @@ -1,23 +1,23 @@ -package io.confluent.parallelconsumer.internal; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ -public class InternalRuntimeError extends RuntimeException { - - public InternalRuntimeError(final String message) { - super(message); - } - - public InternalRuntimeError(final String message, final Throwable cause) { - super(message, cause); - } - - public InternalRuntimeError(final Throwable cause) { - super(cause); - } - - public InternalRuntimeError(final String message, final Throwable cause, final boolean enableSuppression, final boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - } -} +package io.confluent.parallelconsumer.internal; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ +public class InternalRuntimeError extends RuntimeException { + + public InternalRuntimeError(final String message) { + super(message); + } + + public InternalRuntimeError(final String message, final Throwable cause) { + super(message, cause); + } + + public InternalRuntimeError(final Throwable cause) { + super(cause); + } + + public InternalRuntimeError(final String message, final Throwable cause, final boolean enableSuppression, final boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/internal/ProducerManager.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/internal/ProducerManager.java index 8ac79c094..21b4e2bc8 100644 --- a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/internal/ProducerManager.java +++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/internal/ProducerManager.java @@ -1,328 +1,328 @@ -package io.confluent.parallelconsumer.internal; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ - -import io.confluent.csid.utils.TimeUtils; -import io.confluent.parallelconsumer.ParallelConsumer; -import io.confluent.parallelconsumer.ParallelConsumerOptions; -import io.confluent.parallelconsumer.ParallelEoSStreamProcessor; -import io.confluent.parallelconsumer.ParallelStreamProcessor; -import io.confluent.parallelconsumer.state.WorkManager; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.ConsumerGroupMetadata; -import org.apache.kafka.clients.consumer.OffsetAndMetadata; -import org.apache.kafka.clients.producer.*; -import org.apache.kafka.clients.producer.internals.TransactionManager; -import org.apache.kafka.common.KafkaException; -import org.apache.kafka.common.TopicPartition; - -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.time.Duration; -import java.util.ConcurrentModificationException; -import java.util.Map; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import static io.confluent.csid.utils.StringUtils.msg; - -@Slf4j -public class ProducerManager extends AbstractOffsetCommitter implements OffsetCommitter { - - protected final Producer producer; - - private final ParallelConsumerOptions options; - - private final boolean producerIsConfiguredForTransactions; - - /** - * The {@link KafkaProducer) isn't actually completely thread safe, at least when using it transactionally. We must - * be careful not to send messages to the producer, while we are committing a transaction - "Cannot call send in - * state COMMITTING_TRANSACTION". - */ - private ReentrantReadWriteLock producerTransactionLock; - - // nasty reflection - private Field txManagerField; - private Method txManagerMethodIsCompleting; - private Method txManagerMethodIsReady; - private final long sendTimeoutSeconds = 5L; - - public ProducerManager(final Producer newProducer, final ConsumerManager newConsumer, final WorkManager wm, ParallelConsumerOptions options) { - super(newConsumer, wm); - this.producer = newProducer; - this.options = options; - - producerIsConfiguredForTransactions = setupReflection(); - - initProducer(); - } - - private void initProducer() { - producerTransactionLock = new ReentrantReadWriteLock(true); - - // String transactionIdProp = options.getProducerConfig().getProperty(ProducerConfig.TRANSACTIONAL_ID_CONFIG); - // boolean txIdSupplied = isBlank(transactionIdProp); - if (options.isUsingTransactionalProducer()) { - if (!producerIsConfiguredForTransactions) { - throw new IllegalArgumentException("Using transactional option, yet Producer doesn't have a transaction ID - Producer needs a transaction id"); - } - try { - log.debug("Initialising producer transaction session..."); - producer.initTransactions(); - producer.beginTransaction(); - } catch (KafkaException e) { - log.error("Make sure your producer is setup for transactions - specifically make sure it's {} is set.", ProducerConfig.TRANSACTIONAL_ID_CONFIG, e); - throw e; - } - } else { - if (producerIsConfiguredForTransactions) { - throw new IllegalArgumentException("Using non-transactional producer option, but Producer has a transaction ID - " + - "the Producer must not have a transaction ID for this option. This is because having such an ID forces the " + - "Producer into transactional mode - i.e. you cannot use it without using transactions."); - } - } - } - - /** - * Nasty reflection but better than relying on user supplying their config - * - * @see ParallelEoSStreamProcessor#checkAutoCommitIsDisabled - */ - @SneakyThrows - private boolean getProducerIsTransactional() { - if (producer instanceof MockProducer) { - // can act as both, delegate to user selection - return options.isUsingTransactionalProducer(); - } else { - TransactionManager transactionManager = getTransactionManager(); - if (transactionManager == null) { - return false; - } else { - return transactionManager.isTransactional(); - } - } - } - - @SneakyThrows - private TransactionManager getTransactionManager() { - if (txManagerField == null) return null; - TransactionManager transactionManager = (TransactionManager) txManagerField.get(producer); - return transactionManager; - } - - /** - * Produce a message back to the broker. - *

- * Implementation uses the blocking API, performance upgrade in later versions, is not an issue for the more common - * use case where messages aren't produced. - * - * @see ParallelConsumer#poll - * @see ParallelStreamProcessor#pollAndProduceMany - */ - public RecordMetadata produceMessage(ProducerRecord outMsg) { - // only needed if not using tx - Callback callback = (RecordMetadata metadata, Exception exception) -> { - if (exception != null) { - log.error("Error producing result message", exception); - throw new RuntimeException("Error producing result message", exception); - } - }; - - ReentrantReadWriteLock.ReadLock readLock = producerTransactionLock.readLock(); - readLock.lock(); - Future send; - try { - send = producer.send(outMsg, callback); - } finally { - readLock.unlock(); - } - - // wait on the send results - try { - log.trace("Blocking on produce result"); - RecordMetadata recordMetadata = TimeUtils.time(() -> - send.get(sendTimeoutSeconds, TimeUnit.SECONDS)); - log.trace("Produce result received"); - return recordMetadata; - } catch (Exception e) { - throw new InternalRuntimeError(e); - } - } - - @Override - protected void preAcquireWork() { - acquireCommitLock(); - } - - @Override - protected void postCommit() { - // only release lock when commit successful - if (producerTransactionLock.getWriteHoldCount() > 1) // sanity - throw new ConcurrentModificationException("Lock held too many times, won't be released problem and will cause deadlock"); - - releaseCommitLock(); - } - - @Override - protected void commitOffsets(final Map offsetsToSend, final ConsumerGroupMetadata groupMetadata) { - log.debug("Transactional offset commit starting"); - if (!options.isUsingTransactionalProducer()) { - throw new IllegalStateException("Bug: cannot use if not using transactional producer"); - } - - producer.sendOffsetsToTransaction(offsetsToSend, groupMetadata); - // see {@link KafkaProducer#commit} this can be interrupted and is safe to retry - boolean committed = false; - int retryCount = 0; - int arbitrarilyChosenLimitForArbitraryErrorSituation = 200; - Exception lastErrorSavedForRethrow = null; - while (!committed) { - if (retryCount > arbitrarilyChosenLimitForArbitraryErrorSituation) { - String msg = msg("Retired too many times ({} > limit of {}), giving up. See error above.", retryCount, arbitrarilyChosenLimitForArbitraryErrorSituation); - log.error(msg, lastErrorSavedForRethrow); - throw new RuntimeException(msg, lastErrorSavedForRethrow); - } - try { - if (producer instanceof MockProducer) { - // see bug https://issues.apache.org/jira/browse/KAFKA-10382 - // KAFKA-10382 - MockProducer is not ThreadSafe, ideally it should be as the implementation it mocks is - synchronized (producer) { - producer.commitTransaction(); - producer.beginTransaction(); - } - } else { - - // producer commit lock should already be acquired at this point, before work was retrieved to commit, - // so that more messages don't sneak into this tx block - the consumer records of which won't yet be - // in this offset collection - ensureLockHeld(); - - boolean retrying = retryCount > 0; - if (retrying) { - if (isTransactionCompleting()) { - // try wait again - producer.commitTransaction(); - } - if (isTransactionReady()) { - // tx has completed since we last tried, start a new one - producer.beginTransaction(); - } - boolean ready = lastErrorSavedForRethrow == null || !lastErrorSavedForRethrow.getMessage().contains("Invalid transition attempted from state READY to state COMMITTING_TRANSACTION"); - if (ready) { - // try again - log.error("Transaction was already in READY state - tx completed between interrupt and retry"); - } - } else { - // happy path - producer.commitTransaction(); - producer.beginTransaction(); - } - } - - committed = true; - if (retryCount > 0) { - log.warn("Commit success, but took {} tries.", retryCount); - } - } catch (Exception e) { - log.warn("Commit exception, will retry, have tried {} times (see KafkaProducer#commit)", retryCount, e); - lastErrorSavedForRethrow = e; - retryCount++; - } - } - } - - /** - * @return boolean which shows if we are setup for transactions or now - */ - @SneakyThrows - private boolean setupReflection() { - if (producer instanceof KafkaProducer) { - txManagerField = producer.getClass().getDeclaredField("transactionManager"); - txManagerField.setAccessible(true); - - boolean producerIsConfiguredForTransactions = getProducerIsTransactional(); - if (producerIsConfiguredForTransactions) { - TransactionManager transactionManager = getTransactionManager(); - txManagerMethodIsCompleting = transactionManager.getClass().getDeclaredMethod("isCompleting"); - txManagerMethodIsCompleting.setAccessible(true); - - txManagerMethodIsReady = transactionManager.getClass().getDeclaredMethod("isReady"); - txManagerMethodIsReady.setAccessible(true); - } - return producerIsConfiguredForTransactions; - } else if (producer instanceof MockProducer) { - // can act as both, delegate to user selection - return options.isUsingTransactionalProducer(); - } else { - // unknown - return false; - } - } - - /** - * TODO talk about alternatives to this brute force approach for retrying committing transactions - */ - @SneakyThrows - private boolean isTransactionCompleting() { - if (producer instanceof MockProducer) return false; - return (boolean) txManagerMethodIsCompleting.invoke(getTransactionManager()); - } - - /** - * TODO talk about alternatives to this brute force approach for retrying committing transactions - */ - @SneakyThrows - private boolean isTransactionReady() { - if (producer instanceof MockProducer) return true; - return (boolean) txManagerMethodIsReady.invoke(getTransactionManager()); - } - - /** - * Assumes the system is drained at this point, or draining is not desired. - */ - public void close(final Duration timeout) { - log.debug("Closing producer, assuming no more in flight..."); - if (options.isUsingTransactionalProducer() && !isTransactionReady()) { - acquireCommitLock(); - try { - // close started after tx began, but before work was done, otherwise a tx wouldn't have been started - producer.abortTransaction(); - } finally { - releaseCommitLock(); - } - } - producer.close(timeout); - } - - private void acquireCommitLock() { - if (producerTransactionLock.getWriteHoldCount() > 0) - throw new ConcurrentModificationException("Lock already held"); - ReentrantReadWriteLock.WriteLock writeLock = producerTransactionLock.writeLock(); - if (producerTransactionLock.isWriteLocked() && !producerTransactionLock.isWriteLockedByCurrentThread()) { - throw new ConcurrentModificationException(this.getClass().getSimpleName() + " is not safe for multi-threaded access"); - } - writeLock.lock(); - } - - private void releaseCommitLock() { - log.trace("Release commit lock"); - ReentrantReadWriteLock.WriteLock writeLock = producerTransactionLock.writeLock(); - if (!producerTransactionLock.isWriteLockedByCurrentThread()) - throw new IllegalStateException("Not held be me"); - writeLock.unlock(); - } - - private void ensureLockHeld() { - if (!producerTransactionLock.isWriteLockedByCurrentThread()) - throw new IllegalStateException("Expected commit lock to be held"); - } - - public boolean isTransactionInProgress() { - return producerTransactionLock.isWriteLocked(); - } -} +package io.confluent.parallelconsumer.internal; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ + +import io.confluent.csid.utils.TimeUtils; +import io.confluent.parallelconsumer.ParallelConsumer; +import io.confluent.parallelconsumer.ParallelConsumerOptions; +import io.confluent.parallelconsumer.ParallelEoSStreamProcessor; +import io.confluent.parallelconsumer.ParallelStreamProcessor; +import io.confluent.parallelconsumer.state.WorkManager; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerGroupMetadata; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.producer.*; +import org.apache.kafka.clients.producer.internals.TransactionManager; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.TopicPartition; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.time.Duration; +import java.util.ConcurrentModificationException; +import java.util.Map; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import static io.confluent.csid.utils.StringUtils.msg; + +@Slf4j +public class ProducerManager extends AbstractOffsetCommitter implements OffsetCommitter { + + protected final Producer producer; + + private final ParallelConsumerOptions options; + + private final boolean producerIsConfiguredForTransactions; + + /** + * The {@link KafkaProducer) isn't actually completely thread safe, at least when using it transactionally. We must + * be careful not to send messages to the producer, while we are committing a transaction - "Cannot call send in + * state COMMITTING_TRANSACTION". + */ + private ReentrantReadWriteLock producerTransactionLock; + + // nasty reflection + private Field txManagerField; + private Method txManagerMethodIsCompleting; + private Method txManagerMethodIsReady; + private final long sendTimeoutSeconds = 5L; + + public ProducerManager(final Producer newProducer, final ConsumerManager newConsumer, final WorkManager wm, ParallelConsumerOptions options) { + super(newConsumer, wm); + this.producer = newProducer; + this.options = options; + + producerIsConfiguredForTransactions = setupReflection(); + + initProducer(); + } + + private void initProducer() { + producerTransactionLock = new ReentrantReadWriteLock(true); + + // String transactionIdProp = options.getProducerConfig().getProperty(ProducerConfig.TRANSACTIONAL_ID_CONFIG); + // boolean txIdSupplied = isBlank(transactionIdProp); + if (options.isUsingTransactionalProducer()) { + if (!producerIsConfiguredForTransactions) { + throw new IllegalArgumentException("Using transactional option, yet Producer doesn't have a transaction ID - Producer needs a transaction id"); + } + try { + log.debug("Initialising producer transaction session..."); + producer.initTransactions(); + producer.beginTransaction(); + } catch (KafkaException e) { + log.error("Make sure your producer is setup for transactions - specifically make sure it's {} is set.", ProducerConfig.TRANSACTIONAL_ID_CONFIG, e); + throw e; + } + } else { + if (producerIsConfiguredForTransactions) { + throw new IllegalArgumentException("Using non-transactional producer option, but Producer has a transaction ID - " + + "the Producer must not have a transaction ID for this option. This is because having such an ID forces the " + + "Producer into transactional mode - i.e. you cannot use it without using transactions."); + } + } + } + + /** + * Nasty reflection but better than relying on user supplying their config + * + * @see ParallelEoSStreamProcessor#checkAutoCommitIsDisabled + */ + @SneakyThrows + private boolean getProducerIsTransactional() { + if (producer instanceof MockProducer) { + // can act as both, delegate to user selection + return options.isUsingTransactionalProducer(); + } else { + TransactionManager transactionManager = getTransactionManager(); + if (transactionManager == null) { + return false; + } else { + return transactionManager.isTransactional(); + } + } + } + + @SneakyThrows + private TransactionManager getTransactionManager() { + if (txManagerField == null) return null; + TransactionManager transactionManager = (TransactionManager) txManagerField.get(producer); + return transactionManager; + } + + /** + * Produce a message back to the broker. + *

+ * Implementation uses the blocking API, performance upgrade in later versions, is not an issue for the more common + * use case where messages aren't produced. + * + * @see ParallelConsumer#poll + * @see ParallelStreamProcessor#pollAndProduceMany + */ + public RecordMetadata produceMessage(ProducerRecord outMsg) { + // only needed if not using tx + Callback callback = (RecordMetadata metadata, Exception exception) -> { + if (exception != null) { + log.error("Error producing result message", exception); + throw new RuntimeException("Error producing result message", exception); + } + }; + + ReentrantReadWriteLock.ReadLock readLock = producerTransactionLock.readLock(); + readLock.lock(); + Future send; + try { + send = producer.send(outMsg, callback); + } finally { + readLock.unlock(); + } + + // wait on the send results + try { + log.trace("Blocking on produce result"); + RecordMetadata recordMetadata = TimeUtils.time(() -> + send.get(sendTimeoutSeconds, TimeUnit.SECONDS)); + log.trace("Produce result received"); + return recordMetadata; + } catch (Exception e) { + throw new InternalRuntimeError(e); + } + } + + @Override + protected void preAcquireWork() { + acquireCommitLock(); + } + + @Override + protected void postCommit() { + // only release lock when commit successful + if (producerTransactionLock.getWriteHoldCount() > 1) // sanity + throw new ConcurrentModificationException("Lock held too many times, won't be released problem and will cause deadlock"); + + releaseCommitLock(); + } + + @Override + protected void commitOffsets(final Map offsetsToSend, final ConsumerGroupMetadata groupMetadata) { + log.debug("Transactional offset commit starting"); + if (!options.isUsingTransactionalProducer()) { + throw new IllegalStateException("Bug: cannot use if not using transactional producer"); + } + + producer.sendOffsetsToTransaction(offsetsToSend, groupMetadata); + // see {@link KafkaProducer#commit} this can be interrupted and is safe to retry + boolean committed = false; + int retryCount = 0; + int arbitrarilyChosenLimitForArbitraryErrorSituation = 200; + Exception lastErrorSavedForRethrow = null; + while (!committed) { + if (retryCount > arbitrarilyChosenLimitForArbitraryErrorSituation) { + String msg = msg("Retired too many times ({} > limit of {}), giving up. See error above.", retryCount, arbitrarilyChosenLimitForArbitraryErrorSituation); + log.error(msg, lastErrorSavedForRethrow); + throw new RuntimeException(msg, lastErrorSavedForRethrow); + } + try { + if (producer instanceof MockProducer) { + // see bug https://issues.apache.org/jira/browse/KAFKA-10382 + // KAFKA-10382 - MockProducer is not ThreadSafe, ideally it should be as the implementation it mocks is + synchronized (producer) { + producer.commitTransaction(); + producer.beginTransaction(); + } + } else { + + // producer commit lock should already be acquired at this point, before work was retrieved to commit, + // so that more messages don't sneak into this tx block - the consumer records of which won't yet be + // in this offset collection + ensureLockHeld(); + + boolean retrying = retryCount > 0; + if (retrying) { + if (isTransactionCompleting()) { + // try wait again + producer.commitTransaction(); + } + if (isTransactionReady()) { + // tx has completed since we last tried, start a new one + producer.beginTransaction(); + } + boolean ready = lastErrorSavedForRethrow == null || !lastErrorSavedForRethrow.getMessage().contains("Invalid transition attempted from state READY to state COMMITTING_TRANSACTION"); + if (ready) { + // try again + log.error("Transaction was already in READY state - tx completed between interrupt and retry"); + } + } else { + // happy path + producer.commitTransaction(); + producer.beginTransaction(); + } + } + + committed = true; + if (retryCount > 0) { + log.warn("Commit success, but took {} tries.", retryCount); + } + } catch (Exception e) { + log.warn("Commit exception, will retry, have tried {} times (see KafkaProducer#commit)", retryCount, e); + lastErrorSavedForRethrow = e; + retryCount++; + } + } + } + + /** + * @return boolean which shows if we are setup for transactions or now + */ + @SneakyThrows + private boolean setupReflection() { + if (producer instanceof KafkaProducer) { + txManagerField = producer.getClass().getDeclaredField("transactionManager"); + txManagerField.setAccessible(true); + + boolean producerIsConfiguredForTransactions = getProducerIsTransactional(); + if (producerIsConfiguredForTransactions) { + TransactionManager transactionManager = getTransactionManager(); + txManagerMethodIsCompleting = transactionManager.getClass().getDeclaredMethod("isCompleting"); + txManagerMethodIsCompleting.setAccessible(true); + + txManagerMethodIsReady = transactionManager.getClass().getDeclaredMethod("isReady"); + txManagerMethodIsReady.setAccessible(true); + } + return producerIsConfiguredForTransactions; + } else if (producer instanceof MockProducer) { + // can act as both, delegate to user selection + return options.isUsingTransactionalProducer(); + } else { + // unknown + return false; + } + } + + /** + * TODO talk about alternatives to this brute force approach for retrying committing transactions + */ + @SneakyThrows + private boolean isTransactionCompleting() { + if (producer instanceof MockProducer) return false; + return (boolean) txManagerMethodIsCompleting.invoke(getTransactionManager()); + } + + /** + * TODO talk about alternatives to this brute force approach for retrying committing transactions + */ + @SneakyThrows + private boolean isTransactionReady() { + if (producer instanceof MockProducer) return true; + return (boolean) txManagerMethodIsReady.invoke(getTransactionManager()); + } + + /** + * Assumes the system is drained at this point, or draining is not desired. + */ + public void close(final Duration timeout) { + log.debug("Closing producer, assuming no more in flight..."); + if (options.isUsingTransactionalProducer() && !isTransactionReady()) { + acquireCommitLock(); + try { + // close started after tx began, but before work was done, otherwise a tx wouldn't have been started + producer.abortTransaction(); + } finally { + releaseCommitLock(); + } + } + producer.close(timeout); + } + + private void acquireCommitLock() { + if (producerTransactionLock.getWriteHoldCount() > 0) + throw new ConcurrentModificationException("Lock already held"); + ReentrantReadWriteLock.WriteLock writeLock = producerTransactionLock.writeLock(); + if (producerTransactionLock.isWriteLocked() && !producerTransactionLock.isWriteLockedByCurrentThread()) { + throw new ConcurrentModificationException(this.getClass().getSimpleName() + " is not safe for multi-threaded access"); + } + writeLock.lock(); + } + + private void releaseCommitLock() { + log.trace("Release commit lock"); + ReentrantReadWriteLock.WriteLock writeLock = producerTransactionLock.writeLock(); + if (!producerTransactionLock.isWriteLockedByCurrentThread()) + throw new IllegalStateException("Not held be me"); + writeLock.unlock(); + } + + private void ensureLockHeld() { + if (!producerTransactionLock.isWriteLockedByCurrentThread()) + throw new IllegalStateException("Expected commit lock to be held"); + } + + public boolean isTransactionInProgress() { + return producerTransactionLock.isWriteLocked(); + } +} diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/internal/State.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/internal/State.java index 911b8a42b..40670ddd7 100644 --- a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/internal/State.java +++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/internal/State.java @@ -1,21 +1,21 @@ -package io.confluent.parallelconsumer.internal; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ - -/** - * The run state of the controller. - */ -public enum State { - unused, - running, - /** - * When draining, the system will stop polling for more records, but will attempt to process all already downloaded - * records. Note that if you choose to close without draining, records already processed will still be committed - * first before closing. - */ - draining, - closing, - closed; -} +package io.confluent.parallelconsumer.internal; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ + +/** + * The run state of the controller. + */ +public enum State { + unused, + running, + /** + * When draining, the system will stop polling for more records, but will attempt to process all already downloaded + * records. Note that if you choose to close without draining, records already processed will still be committed + * first before closing. + */ + draining, + closing, + closed; +} diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/BitSetEncodingNotSupportedException.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/BitSetEncodingNotSupportedException.java index 6601e3fa5..0a0e1c340 100644 --- a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/BitSetEncodingNotSupportedException.java +++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/BitSetEncodingNotSupportedException.java @@ -1,12 +1,12 @@ -package io.confluent.parallelconsumer.offsets; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ -public class BitSetEncodingNotSupportedException extends EncodingNotSupportedException { - - public BitSetEncodingNotSupportedException(String msg) { - super(msg); - } - -} +package io.confluent.parallelconsumer.offsets; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ +public class BitSetEncodingNotSupportedException extends EncodingNotSupportedException { + + public BitSetEncodingNotSupportedException(String msg) { + super(msg); + } + +} diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/EncodingNotSupportedException.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/EncodingNotSupportedException.java index 1446bfb5a..c17edd3d0 100644 --- a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/EncodingNotSupportedException.java +++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/EncodingNotSupportedException.java @@ -1,10 +1,10 @@ -package io.confluent.parallelconsumer.offsets; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ -public class EncodingNotSupportedException extends Exception { - public EncodingNotSupportedException(final String message) { - super(message); - } -} +package io.confluent.parallelconsumer.offsets; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ +public class EncodingNotSupportedException extends Exception { + public EncodingNotSupportedException(final String message) { + super(message); + } +} diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/OffsetBitSet.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/OffsetBitSet.java index f6d503547..7e2cbc2fa 100644 --- a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/OffsetBitSet.java +++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/OffsetBitSet.java @@ -1,81 +1,81 @@ -package io.confluent.parallelconsumer.offsets; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ -import io.confluent.parallelconsumer.internal.InternalRuntimeError; -import io.confluent.parallelconsumer.offsets.OffsetMapCodecManager.HighestOffsetAndIncompletes; -import lombok.extern.slf4j.Slf4j; - -import java.nio.ByteBuffer; -import java.util.BitSet; -import java.util.HashSet; -import java.util.Set; - -import static io.confluent.csid.utils.Range.range; - -/** - * Deserialisation tools for {@link BitsetEncoder}. - *

- * todo unify or refactor with {@link BitsetEncoder}. Why was it ever seperate? - * - * @see BitsetEncoder - */ -@Slf4j -public class OffsetBitSet { - - static String deserialiseBitSetWrap(ByteBuffer wrap, OffsetEncoding.Version version) { - wrap.rewind(); - - int originalBitsetSize = switch (version) { - case v1 -> (int) wrap.getShort(); // up cast ok - case v2 -> wrap.getInt(); - }; - - ByteBuffer slice = wrap.slice(); - return deserialiseBitSet(originalBitsetSize, slice); - } - - static String deserialiseBitSet(int originalBitsetSize, ByteBuffer s) { - BitSet bitSet = BitSet.valueOf(s); - - StringBuilder result = new StringBuilder(bitSet.size()); - for (var offset : range(originalBitsetSize)) { - if (bitSet.get(offset)) { - result.append('x'); - } else { - result.append('o'); - } - } - - return result.toString(); - } - - static HighestOffsetAndIncompletes deserialiseBitSetWrapToIncompletes(OffsetEncoding encoding, long baseOffset, ByteBuffer wrap) { - wrap.rewind(); - int originalBitsetSize = switch (encoding) { - case BitSet -> wrap.getShort(); - case BitSetV2 -> wrap.getInt(); - default -> throw new InternalRuntimeError("Invalid state"); - }; - ByteBuffer slice = wrap.slice(); - Set incompletes = deserialiseBitSetToIncompletes(baseOffset, originalBitsetSize, slice); - long highestSeenOffset = baseOffset + originalBitsetSize - 1; - return HighestOffsetAndIncompletes.of(highestSeenOffset, incompletes); - } - - static Set deserialiseBitSetToIncompletes(long baseOffset, int originalBitsetSize, ByteBuffer inputBuffer) { - BitSet bitSet = BitSet.valueOf(inputBuffer); - int numberOfIncompletes = originalBitsetSize - bitSet.cardinality(); - var incompletes = new HashSet(numberOfIncompletes); - for (var relativeOffset : range(originalBitsetSize)) { - long offset = baseOffset + relativeOffset; - if (bitSet.get(relativeOffset)) { - log.trace("Ignoring completed offset {}", relativeOffset); - } else { - incompletes.add(offset); - } - } - return incompletes; - } -} +package io.confluent.parallelconsumer.offsets; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ +import io.confluent.parallelconsumer.internal.InternalRuntimeError; +import io.confluent.parallelconsumer.offsets.OffsetMapCodecManager.HighestOffsetAndIncompletes; +import lombok.extern.slf4j.Slf4j; + +import java.nio.ByteBuffer; +import java.util.BitSet; +import java.util.HashSet; +import java.util.Set; + +import static io.confluent.csid.utils.Range.range; + +/** + * Deserialisation tools for {@link BitsetEncoder}. + *

+ * todo unify or refactor with {@link BitsetEncoder}. Why was it ever seperate? + * + * @see BitsetEncoder + */ +@Slf4j +public class OffsetBitSet { + + static String deserialiseBitSetWrap(ByteBuffer wrap, OffsetEncoding.Version version) { + wrap.rewind(); + + int originalBitsetSize = switch (version) { + case v1 -> (int) wrap.getShort(); // up cast ok + case v2 -> wrap.getInt(); + }; + + ByteBuffer slice = wrap.slice(); + return deserialiseBitSet(originalBitsetSize, slice); + } + + static String deserialiseBitSet(int originalBitsetSize, ByteBuffer s) { + BitSet bitSet = BitSet.valueOf(s); + + StringBuilder result = new StringBuilder(bitSet.size()); + for (var offset : range(originalBitsetSize)) { + if (bitSet.get(offset)) { + result.append('x'); + } else { + result.append('o'); + } + } + + return result.toString(); + } + + static HighestOffsetAndIncompletes deserialiseBitSetWrapToIncompletes(OffsetEncoding encoding, long baseOffset, ByteBuffer wrap) { + wrap.rewind(); + int originalBitsetSize = switch (encoding) { + case BitSet -> wrap.getShort(); + case BitSetV2 -> wrap.getInt(); + default -> throw new InternalRuntimeError("Invalid state"); + }; + ByteBuffer slice = wrap.slice(); + Set incompletes = deserialiseBitSetToIncompletes(baseOffset, originalBitsetSize, slice); + long highestSeenOffset = baseOffset + originalBitsetSize - 1; + return HighestOffsetAndIncompletes.of(highestSeenOffset, incompletes); + } + + static Set deserialiseBitSetToIncompletes(long baseOffset, int originalBitsetSize, ByteBuffer inputBuffer) { + BitSet bitSet = BitSet.valueOf(inputBuffer); + int numberOfIncompletes = originalBitsetSize - bitSet.cardinality(); + var incompletes = new HashSet(numberOfIncompletes); + for (var relativeOffset : range(originalBitsetSize)) { + long offset = baseOffset + relativeOffset; + if (bitSet.get(relativeOffset)) { + log.trace("Ignoring completed offset {}", relativeOffset); + } else { + incompletes.add(offset); + } + } + return incompletes; + } +} diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/OffsetDecodingError.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/OffsetDecodingError.java index 8c78af824..6dfd81ae0 100644 --- a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/OffsetDecodingError.java +++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/OffsetDecodingError.java @@ -1,10 +1,10 @@ -package io.confluent.parallelconsumer.offsets; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ -public class OffsetDecodingError extends Exception { - public OffsetDecodingError(final String s, final IllegalArgumentException a) { - super(s, a); - } -} +package io.confluent.parallelconsumer.offsets; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ +public class OffsetDecodingError extends Exception { + public OffsetDecodingError(final String s, final IllegalArgumentException a) { + super(s, a); + } +} diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/OffsetEncoding.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/OffsetEncoding.java index 6eef62d24..c8c685bce 100644 --- a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/OffsetEncoding.java +++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/OffsetEncoding.java @@ -1,61 +1,61 @@ -package io.confluent.parallelconsumer.offsets; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import lombok.ToString; - -import java.util.Arrays; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static io.confluent.parallelconsumer.offsets.OffsetEncoding.Version.v1; -import static io.confluent.parallelconsumer.offsets.OffsetEncoding.Version.v2; - -@ToString -@RequiredArgsConstructor -public enum OffsetEncoding { - ByteArray(v1, (byte) 'L'), - ByteArrayCompressed(v1, (byte) 'î'), - BitSet(v1, (byte) 'l'), - BitSetCompressed(v1, (byte) 'a'), - RunLength(v1, (byte) 'n'), - RunLengthCompressed(v1, (byte) 'J'), - /** - * switch from encoding bitset length as a short to an integer (length of 32,000 was reasonable too short) - */ - BitSetV2(v2, (byte) 'o'), - BitSetV2Compressed(v2, (byte) 's'), - /** - * switch from encoding run lengths as Shorts to Integers - */ - RunLengthV2(v2, (byte) 'e'), - RunLengthV2Compressed(v2, (byte) 'p'); - - enum Version { - v1, v2 - } - - public final Version version; - - @Getter - public final byte magicByte; - - private static final Map magicMap = Arrays.stream(values()).collect(Collectors.toMap(OffsetEncoding::getMagicByte, Function.identity())); - - public static OffsetEncoding decode(byte magic) { - OffsetEncoding encoding = magicMap.get(magic); - if (encoding == null) { - throw new RuntimeException("Unexpected magic: " + magic); - } else { - return encoding; - } - } - - public String description() { - return name() + ":" + version; - } -} +package io.confluent.parallelconsumer.offsets; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +import java.util.Arrays; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static io.confluent.parallelconsumer.offsets.OffsetEncoding.Version.v1; +import static io.confluent.parallelconsumer.offsets.OffsetEncoding.Version.v2; + +@ToString +@RequiredArgsConstructor +public enum OffsetEncoding { + ByteArray(v1, (byte) 'L'), + ByteArrayCompressed(v1, (byte) 'î'), + BitSet(v1, (byte) 'l'), + BitSetCompressed(v1, (byte) 'a'), + RunLength(v1, (byte) 'n'), + RunLengthCompressed(v1, (byte) 'J'), + /** + * switch from encoding bitset length as a short to an integer (length of 32,000 was reasonable too short) + */ + BitSetV2(v2, (byte) 'o'), + BitSetV2Compressed(v2, (byte) 's'), + /** + * switch from encoding run lengths as Shorts to Integers + */ + RunLengthV2(v2, (byte) 'e'), + RunLengthV2Compressed(v2, (byte) 'p'); + + enum Version { + v1, v2 + } + + public final Version version; + + @Getter + public final byte magicByte; + + private static final Map magicMap = Arrays.stream(values()).collect(Collectors.toMap(OffsetEncoding::getMagicByte, Function.identity())); + + public static OffsetEncoding decode(byte magic) { + OffsetEncoding encoding = magicMap.get(magic); + if (encoding == null) { + throw new RuntimeException("Unexpected magic: " + magic); + } else { + return encoding; + } + } + + public String description() { + return name() + ":" + version; + } +} diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/OffsetMapCodecManager.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/OffsetMapCodecManager.java index 321a43699..2a434182e 100644 --- a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/OffsetMapCodecManager.java +++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/OffsetMapCodecManager.java @@ -1,265 +1,265 @@ -package io.confluent.parallelconsumer.offsets; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ - -import io.confluent.parallelconsumer.internal.InternalRuntimeError; -import io.confluent.parallelconsumer.state.PartitionState; -import lombok.Value; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.OffsetAndMetadata; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.errors.WakeupException; - -import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.util.*; - -import static io.confluent.csid.utils.Range.range; -import static io.confluent.csid.utils.StringUtils.msg; -import static java.nio.charset.StandardCharsets.UTF_8; - -/** - * Uses multiple encodings to compare, when decided, can refactor other options out for analysis only - {@link - * #encodeOffsetsCompressed} - *

- * TODO: consider IO exception management - question sneaky throws usage? - *

- * TODO: enforce max uncommitted {@literal <} encoding length (Short.MAX) - *

- * Bitset serialisation format: - *

    - *
  • byte1: magic - *
  • byte2-3: Short: bitset size - *
  • byte4-n: serialised {@link BitSet} - *
- */ -@Slf4j -public class OffsetMapCodecManager { - - /** - * Used to prevent tests running in parallel that depends on setting static state in this class. Manipulation of - * static state in tests needs to be removed to this isn't necessary. - *

- * todo remove static state manipulation from tests (make non static) - */ - public static final String METADATA_DATA_SIZE_RESOURCE_LOCK = "Value doesn't matter, just needs a constant"; - - /** - * Maximum size of the commit offset metadata - * - * @see OffsetConfig#DefaultMaxMetadataSize - * @see "kafka.coordinator.group.OffsetConfig#DefaultMaxMetadataSize" - */ - public static int DefaultMaxMetadataSize = 4096; - - public static final Charset CHARSET_TO_USE = UTF_8; - - org.apache.kafka.clients.consumer.Consumer consumer; - - /** - * Decoding result for encoded offsets - */ - @Value - public static class HighestOffsetAndIncompletes { - - /** - * The highest represented offset in this result. - */ - Long highestSeenOffset; - - /** - * Of the offsets encoded, the incomplete ones. - */ - Set incompleteOffsets; - - public static HighestOffsetAndIncompletes of(Long highestSeenOffset) { - return new HighestOffsetAndIncompletes(highestSeenOffset, new HashSet<>()); - } - - public static HighestOffsetAndIncompletes of(long highestSeenOffset, Set incompleteOffsets) { - return new HighestOffsetAndIncompletes(highestSeenOffset, incompleteOffsets); - } - - public static HighestOffsetAndIncompletes of() { - return HighestOffsetAndIncompletes.of(null); - } - } - - /** - * Forces the use of a specific codec, instead of choosing the most efficient one. Useful for testing. - */ - public static Optional forcedCodec = Optional.empty(); - - public OffsetMapCodecManager(final org.apache.kafka.clients.consumer.Consumer consumer) { - this.consumer = consumer; - } - - /** - * Load all the previously completed offsets that were not committed - * - * @return - */ - // todo make package private? - // todo rename - public Map> loadOffsetMapForPartition(final Set assignment) { - // load last committed state / metadata from consumer - // todo this should be controlled for - improve consumer management so that this can't happen - Map lastCommittedOffsets = null; - int attempts = 0; - while (lastCommittedOffsets == null) { - WakeupException lastWakeupException = null; - try { - lastCommittedOffsets = consumer.committed(assignment); - } catch (WakeupException exception) { - log.warn("Woken up trying to get assignment", exception); - lastWakeupException = exception; - } - attempts++; - if (attempts > 10) // shouldn't need more than 1 ever - throw new InternalRuntimeError("Failed to get partition assignment - continuously woken up.", lastWakeupException); - } - - var states = new HashMap>(); - lastCommittedOffsets.forEach((tp, offsetAndMeta) -> { - if (offsetAndMeta != null) { - long nextExpectedOffset = offsetAndMeta.offset(); - String metadata = offsetAndMeta.metadata(); - try { - // todo rename - PartitionState incompletes = decodeIncompletes(nextExpectedOffset, tp, metadata); - states.put(tp, incompletes); - } catch (OffsetDecodingError offsetDecodingError) { - log.error("Error decoding offsets from assigned partition, dropping offset map (will replay previously completed messages - partition: {}, data: {})", - tp, offsetAndMeta, offsetDecodingError); - } - } - - }); - - // for each assignment which isn't now added in the states to return, enter a default entry. Catches multiple other cases. - assignment.stream() - .filter(x -> !states.containsKey(x)) - .forEach(x -> - states.put(x, new PartitionState<>(x, HighestOffsetAndIncompletes.of()))); - - return states; - } - - static HighestOffsetAndIncompletes deserialiseIncompleteOffsetMapFromBase64(long committedOffsetForPartition, String base64EncodedOffsetPayload) throws OffsetDecodingError { - byte[] decodedBytes; - try { - decodedBytes = OffsetSimpleSerialisation.decodeBase64(base64EncodedOffsetPayload); - } catch (IllegalArgumentException a) { - throw new OffsetDecodingError(msg("Error decoding offset metadata, input was: {}", base64EncodedOffsetPayload), a); - } - return decodeCompressedOffsets(committedOffsetForPartition, decodedBytes); - } - - // todo rename - PartitionState decodeIncompletes(long nextExpectedOffset, TopicPartition tp, String offsetMetadataPayload) throws OffsetDecodingError { - HighestOffsetAndIncompletes incompletes = deserialiseIncompleteOffsetMapFromBase64(nextExpectedOffset, offsetMetadataPayload); - log.debug("Loaded incomplete offsets from offset payload {}", incompletes); - return new PartitionState(tp, incompletes); - } - - public String makeOffsetMetadataPayload(long finalOffsetForPartition, PartitionState state) throws EncodingNotSupportedException { - String offsetMap = serialiseIncompleteOffsetMapToBase64(finalOffsetForPartition, state); - return offsetMap; - } - - String serialiseIncompleteOffsetMapToBase64(long finalOffsetForPartition, PartitionState state) throws EncodingNotSupportedException { - byte[] compressedEncoding = encodeOffsetsCompressed(finalOffsetForPartition, state); - String b64 = OffsetSimpleSerialisation.base64(compressedEncoding); - return b64; - } - - /** - * Print out all the offset status into a String, and use X to effectively do run length encoding compression on the - * string. - *

- * Include the magic byte in the returned array. - *

- * Can remove string encoding in favour of the boolean array for the `BitSet` if that's how things settle. - */ - byte[] encodeOffsetsCompressed(long finalOffsetForPartition, PartitionState partition) throws EncodingNotSupportedException { - Long nextExpectedOffset = partition.getOffsetHighestSeen() + 1; - TopicPartition tp = partition.getTp(); - Set incompleteOffsets = partition.getIncompleteOffsets(); - log.debug("Encoding partition {} incomplete offsets {}", tp, incompleteOffsets); - OffsetSimultaneousEncoder simultaneousEncoder = new OffsetSimultaneousEncoder(finalOffsetForPartition, nextExpectedOffset, incompleteOffsets).invoke(); - if (forcedCodec.isPresent()) { - var forcedOffsetEncoding = forcedCodec.get(); - log.debug("Forcing use of {}, for testing", forcedOffsetEncoding); - Map encodingMap = simultaneousEncoder.getEncodingMap(); - byte[] bytes = encodingMap.get(forcedOffsetEncoding); - if (bytes == null) - throw new EncodingNotSupportedException(msg("Can't force an encoding that hasn't been run: {}", forcedOffsetEncoding)); - return simultaneousEncoder.packEncoding(new EncodedOffsetPair(forcedOffsetEncoding, ByteBuffer.wrap(bytes))); - } else { - return simultaneousEncoder.packSmallest(); - } - } - - /** - * Print out all the offset status into a String, and potentially use zstd to effectively do run length encoding - * compression - * - * @return Set of offsets which are not complete, and the highest offset encoded. - */ - static HighestOffsetAndIncompletes decodeCompressedOffsets(long nextExpectedOffset, byte[] decodedBytes) { - - // if no offset bitmap data - if (decodedBytes.length == 0) { - // in this case, as there is no encoded offset data in the matadata, the highest we previously saw must be - // the offset before the committed offset - long highestSeenOffsetIsThen = nextExpectedOffset - 1; - return HighestOffsetAndIncompletes.of(highestSeenOffsetIsThen); - } else { - var result = EncodedOffsetPair.unwrap(decodedBytes); - - HighestOffsetAndIncompletes incompletesTuple = result.getDecodedIncompletes(nextExpectedOffset); - - Set incompletes = incompletesTuple.getIncompleteOffsets(); - long highWater = incompletesTuple.getHighestSeenOffset(); - - return HighestOffsetAndIncompletes.of(highWater, incompletes); - } - } - - String incompletesToBitmapString(long finalOffsetForPartition, PartitionState state) { - var runLengthString = new StringBuilder(); - Long lowWaterMark = finalOffsetForPartition; - Long highWaterMark = state.getOffsetHighestSeen(); - long end = highWaterMark - lowWaterMark; - for (final var relativeOffset : range(end)) { - long offset = lowWaterMark + relativeOffset; - if (state.getIncompleteOffsets().contains(offset)) { - runLengthString.append("o"); - } else { - runLengthString.append("x"); - } - } - return runLengthString.toString(); - } - - static Set bitmapStringToIncomplete(final long baseOffset, final String inputBitmapString) { - final Set incompleteOffsets = new HashSet<>(); - - final long longLength = inputBitmapString.length(); - range(longLength).forEach(i -> { - var bit = inputBitmapString.charAt(i); - if (bit == 'o') { - incompleteOffsets.add(baseOffset + i); - } else if (bit == 'x') { - log.trace("Dropping completed offset"); - } else { - throw new IllegalArgumentException("Invalid encoding - unexpected char: " + bit); - } - }); - - return incompleteOffsets; - } - -} +package io.confluent.parallelconsumer.offsets; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ + +import io.confluent.parallelconsumer.internal.InternalRuntimeError; +import io.confluent.parallelconsumer.state.PartitionState; +import lombok.Value; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.WakeupException; + +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.*; + +import static io.confluent.csid.utils.Range.range; +import static io.confluent.csid.utils.StringUtils.msg; +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Uses multiple encodings to compare, when decided, can refactor other options out for analysis only - {@link + * #encodeOffsetsCompressed} + *

+ * TODO: consider IO exception management - question sneaky throws usage? + *

+ * TODO: enforce max uncommitted {@literal <} encoding length (Short.MAX) + *

+ * Bitset serialisation format: + *

    + *
  • byte1: magic + *
  • byte2-3: Short: bitset size + *
  • byte4-n: serialised {@link BitSet} + *
+ */ +@Slf4j +public class OffsetMapCodecManager { + + /** + * Used to prevent tests running in parallel that depends on setting static state in this class. Manipulation of + * static state in tests needs to be removed to this isn't necessary. + *

+ * todo remove static state manipulation from tests (make non static) + */ + public static final String METADATA_DATA_SIZE_RESOURCE_LOCK = "Value doesn't matter, just needs a constant"; + + /** + * Maximum size of the commit offset metadata + * + * @see OffsetConfig#DefaultMaxMetadataSize + * @see "kafka.coordinator.group.OffsetConfig#DefaultMaxMetadataSize" + */ + public static int DefaultMaxMetadataSize = 4096; + + public static final Charset CHARSET_TO_USE = UTF_8; + + org.apache.kafka.clients.consumer.Consumer consumer; + + /** + * Decoding result for encoded offsets + */ + @Value + public static class HighestOffsetAndIncompletes { + + /** + * The highest represented offset in this result. + */ + Long highestSeenOffset; + + /** + * Of the offsets encoded, the incomplete ones. + */ + Set incompleteOffsets; + + public static HighestOffsetAndIncompletes of(Long highestSeenOffset) { + return new HighestOffsetAndIncompletes(highestSeenOffset, new HashSet<>()); + } + + public static HighestOffsetAndIncompletes of(long highestSeenOffset, Set incompleteOffsets) { + return new HighestOffsetAndIncompletes(highestSeenOffset, incompleteOffsets); + } + + public static HighestOffsetAndIncompletes of() { + return HighestOffsetAndIncompletes.of(null); + } + } + + /** + * Forces the use of a specific codec, instead of choosing the most efficient one. Useful for testing. + */ + public static Optional forcedCodec = Optional.empty(); + + public OffsetMapCodecManager(final org.apache.kafka.clients.consumer.Consumer consumer) { + this.consumer = consumer; + } + + /** + * Load all the previously completed offsets that were not committed + * + * @return + */ + // todo make package private? + // todo rename + public Map> loadOffsetMapForPartition(final Set assignment) { + // load last committed state / metadata from consumer + // todo this should be controlled for - improve consumer management so that this can't happen + Map lastCommittedOffsets = null; + int attempts = 0; + while (lastCommittedOffsets == null) { + WakeupException lastWakeupException = null; + try { + lastCommittedOffsets = consumer.committed(assignment); + } catch (WakeupException exception) { + log.warn("Woken up trying to get assignment", exception); + lastWakeupException = exception; + } + attempts++; + if (attempts > 10) // shouldn't need more than 1 ever + throw new InternalRuntimeError("Failed to get partition assignment - continuously woken up.", lastWakeupException); + } + + var states = new HashMap>(); + lastCommittedOffsets.forEach((tp, offsetAndMeta) -> { + if (offsetAndMeta != null) { + long nextExpectedOffset = offsetAndMeta.offset(); + String metadata = offsetAndMeta.metadata(); + try { + // todo rename + PartitionState incompletes = decodeIncompletes(nextExpectedOffset, tp, metadata); + states.put(tp, incompletes); + } catch (OffsetDecodingError offsetDecodingError) { + log.error("Error decoding offsets from assigned partition, dropping offset map (will replay previously completed messages - partition: {}, data: {})", + tp, offsetAndMeta, offsetDecodingError); + } + } + + }); + + // for each assignment which isn't now added in the states to return, enter a default entry. Catches multiple other cases. + assignment.stream() + .filter(x -> !states.containsKey(x)) + .forEach(x -> + states.put(x, new PartitionState<>(x, HighestOffsetAndIncompletes.of()))); + + return states; + } + + static HighestOffsetAndIncompletes deserialiseIncompleteOffsetMapFromBase64(long committedOffsetForPartition, String base64EncodedOffsetPayload) throws OffsetDecodingError { + byte[] decodedBytes; + try { + decodedBytes = OffsetSimpleSerialisation.decodeBase64(base64EncodedOffsetPayload); + } catch (IllegalArgumentException a) { + throw new OffsetDecodingError(msg("Error decoding offset metadata, input was: {}", base64EncodedOffsetPayload), a); + } + return decodeCompressedOffsets(committedOffsetForPartition, decodedBytes); + } + + // todo rename + PartitionState decodeIncompletes(long nextExpectedOffset, TopicPartition tp, String offsetMetadataPayload) throws OffsetDecodingError { + HighestOffsetAndIncompletes incompletes = deserialiseIncompleteOffsetMapFromBase64(nextExpectedOffset, offsetMetadataPayload); + log.debug("Loaded incomplete offsets from offset payload {}", incompletes); + return new PartitionState(tp, incompletes); + } + + public String makeOffsetMetadataPayload(long finalOffsetForPartition, PartitionState state) throws EncodingNotSupportedException { + String offsetMap = serialiseIncompleteOffsetMapToBase64(finalOffsetForPartition, state); + return offsetMap; + } + + String serialiseIncompleteOffsetMapToBase64(long finalOffsetForPartition, PartitionState state) throws EncodingNotSupportedException { + byte[] compressedEncoding = encodeOffsetsCompressed(finalOffsetForPartition, state); + String b64 = OffsetSimpleSerialisation.base64(compressedEncoding); + return b64; + } + + /** + * Print out all the offset status into a String, and use X to effectively do run length encoding compression on the + * string. + *

+ * Include the magic byte in the returned array. + *

+ * Can remove string encoding in favour of the boolean array for the `BitSet` if that's how things settle. + */ + byte[] encodeOffsetsCompressed(long finalOffsetForPartition, PartitionState partition) throws EncodingNotSupportedException { + Long nextExpectedOffset = partition.getOffsetHighestSeen() + 1; + TopicPartition tp = partition.getTp(); + Set incompleteOffsets = partition.getIncompleteOffsets(); + log.debug("Encoding partition {} incomplete offsets {}", tp, incompleteOffsets); + OffsetSimultaneousEncoder simultaneousEncoder = new OffsetSimultaneousEncoder(finalOffsetForPartition, nextExpectedOffset, incompleteOffsets).invoke(); + if (forcedCodec.isPresent()) { + var forcedOffsetEncoding = forcedCodec.get(); + log.debug("Forcing use of {}, for testing", forcedOffsetEncoding); + Map encodingMap = simultaneousEncoder.getEncodingMap(); + byte[] bytes = encodingMap.get(forcedOffsetEncoding); + if (bytes == null) + throw new EncodingNotSupportedException(msg("Can't force an encoding that hasn't been run: {}", forcedOffsetEncoding)); + return simultaneousEncoder.packEncoding(new EncodedOffsetPair(forcedOffsetEncoding, ByteBuffer.wrap(bytes))); + } else { + return simultaneousEncoder.packSmallest(); + } + } + + /** + * Print out all the offset status into a String, and potentially use zstd to effectively do run length encoding + * compression + * + * @return Set of offsets which are not complete, and the highest offset encoded. + */ + static HighestOffsetAndIncompletes decodeCompressedOffsets(long nextExpectedOffset, byte[] decodedBytes) { + + // if no offset bitmap data + if (decodedBytes.length == 0) { + // in this case, as there is no encoded offset data in the matadata, the highest we previously saw must be + // the offset before the committed offset + long highestSeenOffsetIsThen = nextExpectedOffset - 1; + return HighestOffsetAndIncompletes.of(highestSeenOffsetIsThen); + } else { + var result = EncodedOffsetPair.unwrap(decodedBytes); + + HighestOffsetAndIncompletes incompletesTuple = result.getDecodedIncompletes(nextExpectedOffset); + + Set incompletes = incompletesTuple.getIncompleteOffsets(); + long highWater = incompletesTuple.getHighestSeenOffset(); + + return HighestOffsetAndIncompletes.of(highWater, incompletes); + } + } + + String incompletesToBitmapString(long finalOffsetForPartition, PartitionState state) { + var runLengthString = new StringBuilder(); + Long lowWaterMark = finalOffsetForPartition; + Long highWaterMark = state.getOffsetHighestSeen(); + long end = highWaterMark - lowWaterMark; + for (final var relativeOffset : range(end)) { + long offset = lowWaterMark + relativeOffset; + if (state.getIncompleteOffsets().contains(offset)) { + runLengthString.append("o"); + } else { + runLengthString.append("x"); + } + } + return runLengthString.toString(); + } + + static Set bitmapStringToIncomplete(final long baseOffset, final String inputBitmapString) { + final Set incompleteOffsets = new HashSet<>(); + + final long longLength = inputBitmapString.length(); + range(longLength).forEach(i -> { + var bit = inputBitmapString.charAt(i); + if (bit == 'o') { + incompleteOffsets.add(baseOffset + i); + } else if (bit == 'x') { + log.trace("Dropping completed offset"); + } else { + throw new IllegalArgumentException("Invalid encoding - unexpected char: " + bit); + } + }); + + return incompleteOffsets; + } + +} diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/OffsetRunLength.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/OffsetRunLength.java index 3537939f1..92fe44cc6 100644 --- a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/OffsetRunLength.java +++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/OffsetRunLength.java @@ -1,152 +1,152 @@ -package io.confluent.parallelconsumer.offsets; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ -import io.confluent.parallelconsumer.offsets.OffsetMapCodecManager.HighestOffsetAndIncompletes; -import lombok.experimental.UtilityClass; -import lombok.extern.slf4j.Slf4j; - -import java.nio.BufferUnderflowException; -import java.nio.ByteBuffer; -import java.nio.IntBuffer; -import java.nio.ShortBuffer; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Supplier; - -@Slf4j -@UtilityClass -public class OffsetRunLength { - - /** - * @return run length encoding, always starting with an 'o' count - */ - static List runLengthEncode(final String in) { - final AtomicInteger length = new AtomicInteger(); - final AtomicBoolean previous = new AtomicBoolean(false); - final List encoding = new ArrayList<>(); - in.chars().forEachOrdered(bit -> { - final boolean current = switch (bit) { - case 'o' -> false; - case 'x' -> true; - default -> throw new IllegalArgumentException(bit + " in " + in); - }; - if (previous.get() == current) { - length.getAndIncrement(); - } else { - previous.set(current); - encoding.add(length.get()); - length.set(1); - } - }); - encoding.add(length.get()); // add tail - return encoding; - } - - /** - * @see #runLengthEncode - */ - static String runLengthDecodeToString(final List in) { - final StringBuilder sb = new StringBuilder(in.size()); - boolean current = false; - for (final Integer i : in) { - for (int x = 0; x < i; x++) { - if (current) { - sb.append('x'); - } else { - sb.append('o'); - } - } - current = !current; // toggle - } - return sb.toString(); - } - - - /** - * @see #runLengthEncode - */ - static HighestOffsetAndIncompletes runLengthDecodeToIncompletes(OffsetEncoding encoding, final long baseOffset, final ByteBuffer in) { - in.rewind(); - final ShortBuffer v1ShortBuffer = in.asShortBuffer(); - final IntBuffer v2IntegerBuffer = in.asIntBuffer(); - - final var incompletes = new HashSet(); // we don't know the capacity yet - - long highestSeenOffset = 0L; - - Supplier hasRemainingTest = () -> { - return switch (encoding.version) { - case v1 -> v1ShortBuffer.hasRemaining(); - case v2 -> v2IntegerBuffer.hasRemaining(); - }; - }; - if (log.isTraceEnabled()) { - // print out all run lengths - var runlengths = new ArrayList(); - try { - while (hasRemainingTest.get()) { - Number runLength = switch (encoding.version) { - case v1 -> v1ShortBuffer.get(); - case v2 -> v2IntegerBuffer.get(); - }; - runlengths.add(runLength); - } - } catch (BufferUnderflowException u) { - log.error("Error decoding offsets", u); - } - log.debug("Unrolled runlengths: {}", runlengths); - v1ShortBuffer.rewind(); - v2IntegerBuffer.rewind(); - } - - // decodes incompletes - boolean currentRunLengthIsComplete = false; - long currentOffset = baseOffset; - while (hasRemainingTest.get()) { - try { - Number runLength = switch (encoding.version) { - case v1 -> v1ShortBuffer.get(); - case v2 -> v2IntegerBuffer.get(); - }; - - if (currentRunLengthIsComplete) { - log.trace("Ignoring {} completed offset(s) (offset:{})", runLength, currentOffset); - currentOffset += runLength.longValue(); - highestSeenOffset = currentOffset - 1; - } else { - log.trace("Adding {} incomplete offset(s) (starting with offset:{})", runLength, currentOffset); - for (int relativeOffset = 0; relativeOffset < runLength.longValue(); relativeOffset++) { - incompletes.add(currentOffset); - highestSeenOffset = currentOffset; - currentOffset++; - } - } - log.trace("Highest seen: {}", highestSeenOffset); - } catch (BufferUnderflowException u) { - log.error("Error decoding offsets", u); - throw u; - } - currentRunLengthIsComplete = !currentRunLengthIsComplete; // toggle - } - return HighestOffsetAndIncompletes.of(highestSeenOffset, incompletes); - } - - static List runLengthDeserialise(final ByteBuffer in) { - // view as short buffer - in.rewind(); - final ShortBuffer shortBuffer = in.asShortBuffer(); - - // - final List results = new ArrayList<>(shortBuffer.capacity()); - while (shortBuffer.hasRemaining()) { - results.add((int) shortBuffer.get()); - } - return results; - } - -} +package io.confluent.parallelconsumer.offsets; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ +import io.confluent.parallelconsumer.offsets.OffsetMapCodecManager.HighestOffsetAndIncompletes; +import lombok.experimental.UtilityClass; +import lombok.extern.slf4j.Slf4j; + +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.nio.IntBuffer; +import java.nio.ShortBuffer; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; + +@Slf4j +@UtilityClass +public class OffsetRunLength { + + /** + * @return run length encoding, always starting with an 'o' count + */ + static List runLengthEncode(final String in) { + final AtomicInteger length = new AtomicInteger(); + final AtomicBoolean previous = new AtomicBoolean(false); + final List encoding = new ArrayList<>(); + in.chars().forEachOrdered(bit -> { + final boolean current = switch (bit) { + case 'o' -> false; + case 'x' -> true; + default -> throw new IllegalArgumentException(bit + " in " + in); + }; + if (previous.get() == current) { + length.getAndIncrement(); + } else { + previous.set(current); + encoding.add(length.get()); + length.set(1); + } + }); + encoding.add(length.get()); // add tail + return encoding; + } + + /** + * @see #runLengthEncode + */ + static String runLengthDecodeToString(final List in) { + final StringBuilder sb = new StringBuilder(in.size()); + boolean current = false; + for (final Integer i : in) { + for (int x = 0; x < i; x++) { + if (current) { + sb.append('x'); + } else { + sb.append('o'); + } + } + current = !current; // toggle + } + return sb.toString(); + } + + + /** + * @see #runLengthEncode + */ + static HighestOffsetAndIncompletes runLengthDecodeToIncompletes(OffsetEncoding encoding, final long baseOffset, final ByteBuffer in) { + in.rewind(); + final ShortBuffer v1ShortBuffer = in.asShortBuffer(); + final IntBuffer v2IntegerBuffer = in.asIntBuffer(); + + final var incompletes = new HashSet(); // we don't know the capacity yet + + long highestSeenOffset = 0L; + + Supplier hasRemainingTest = () -> { + return switch (encoding.version) { + case v1 -> v1ShortBuffer.hasRemaining(); + case v2 -> v2IntegerBuffer.hasRemaining(); + }; + }; + if (log.isTraceEnabled()) { + // print out all run lengths + var runlengths = new ArrayList(); + try { + while (hasRemainingTest.get()) { + Number runLength = switch (encoding.version) { + case v1 -> v1ShortBuffer.get(); + case v2 -> v2IntegerBuffer.get(); + }; + runlengths.add(runLength); + } + } catch (BufferUnderflowException u) { + log.error("Error decoding offsets", u); + } + log.debug("Unrolled runlengths: {}", runlengths); + v1ShortBuffer.rewind(); + v2IntegerBuffer.rewind(); + } + + // decodes incompletes + boolean currentRunLengthIsComplete = false; + long currentOffset = baseOffset; + while (hasRemainingTest.get()) { + try { + Number runLength = switch (encoding.version) { + case v1 -> v1ShortBuffer.get(); + case v2 -> v2IntegerBuffer.get(); + }; + + if (currentRunLengthIsComplete) { + log.trace("Ignoring {} completed offset(s) (offset:{})", runLength, currentOffset); + currentOffset += runLength.longValue(); + highestSeenOffset = currentOffset - 1; + } else { + log.trace("Adding {} incomplete offset(s) (starting with offset:{})", runLength, currentOffset); + for (int relativeOffset = 0; relativeOffset < runLength.longValue(); relativeOffset++) { + incompletes.add(currentOffset); + highestSeenOffset = currentOffset; + currentOffset++; + } + } + log.trace("Highest seen: {}", highestSeenOffset); + } catch (BufferUnderflowException u) { + log.error("Error decoding offsets", u); + throw u; + } + currentRunLengthIsComplete = !currentRunLengthIsComplete; // toggle + } + return HighestOffsetAndIncompletes.of(highestSeenOffset, incompletes); + } + + static List runLengthDeserialise(final ByteBuffer in) { + // view as short buffer + in.rewind(); + final ShortBuffer shortBuffer = in.asShortBuffer(); + + // + final List results = new ArrayList<>(shortBuffer.capacity()); + while (shortBuffer.hasRemaining()) { + results.add((int) shortBuffer.get()); + } + return results; + } + +} diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/OffsetSimpleSerialisation.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/OffsetSimpleSerialisation.java index 7e489aebb..7f9d4ae4a 100644 --- a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/OffsetSimpleSerialisation.java +++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/OffsetSimpleSerialisation.java @@ -1,124 +1,124 @@ -package io.confluent.parallelconsumer.offsets; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ -import com.github.luben.zstd.ZstdInputStream; -import com.github.luben.zstd.ZstdOutputStream; -import lombok.SneakyThrows; -import lombok.experimental.UtilityClass; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.common.utils.ByteBufferInputStream; -import org.xerial.snappy.SnappyInputStream; -import org.xerial.snappy.SnappyOutputStream; - -import java.io.*; -import java.nio.ByteBuffer; -import java.util.Base64; -import java.util.Set; -import java.util.TreeSet; -import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; - -import static io.confluent.csid.utils.BackportUtils.readFully; - -@UtilityClass -@Slf4j -public class OffsetSimpleSerialisation { - - @SneakyThrows - static String encodeAsJavaObjectStream(final Set incompleteOffsets) { - final ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (final ObjectOutputStream os = new ObjectOutputStream(baos)) { - os.writeObject(incompleteOffsets); - } - return Base64.getEncoder().encodeToString(baos.toByteArray()); - } - - private static TreeSet deserialiseJavaWriteObject(final byte[] decode) throws IOException, ClassNotFoundException { - final Set raw; - try (final ObjectInputStream objectInputStream = new ObjectInputStream(new ByteArrayInputStream(decode))) { - raw = (Set) objectInputStream.readObject(); - } - return new TreeSet<>(raw); - } - - @SneakyThrows - static byte[] compressSnappy(final byte[] bytes) { - try (final var out = new ByteArrayOutputStream(); - final var stream = new SnappyOutputStream(out)) { - stream.write(bytes); - return out.toByteArray(); - } - } - - static ByteBuffer decompressSnappy(final ByteBuffer input) throws IOException { - try (final var snappy = new SnappyInputStream(new ByteBufferInputStream(input))) { - byte[] bytes = readFully(snappy); - return ByteBuffer.wrap(bytes); - } - } - - static String base64(final ByteArrayOutputStream out) { - final byte[] src = out.toByteArray(); - return base64(src); - } - - static byte[] compressZstd(final byte[] bytes) throws IOException { - final var out = new ByteArrayOutputStream(); - try (final var zstream = new ZstdOutputStream(out)) { - zstream.write(bytes); - } - return out.toByteArray(); - } - - static byte[] compressGzip(final byte[] bytes) throws IOException { - final var out = new ByteArrayOutputStream(); - try (final var zstream = new GZIPOutputStream(out)) { - zstream.write(bytes); - } - return out.toByteArray(); - } - - static String base64(final byte[] src) { - final byte[] encode = Base64.getEncoder().encode(src); - final String out = new String(encode, OffsetMapCodecManager.CHARSET_TO_USE); - log.trace("Final b64 size: {}", out.length()); - return out; - } - - static byte[] decodeBase64(final String b64) { - final byte[] bytes = b64.getBytes(OffsetMapCodecManager.CHARSET_TO_USE); - return Base64.getDecoder().decode(bytes); - } - - static ByteBuffer decompressZstd(final ByteBuffer input) throws IOException { - try (final var zstream = new ZstdInputStream(new ByteBufferInputStream(input))) { - final byte[] bytes = readFully(zstream); - return ByteBuffer.wrap(bytes); - } - } - - static byte[] decompressGzip(final ByteBuffer input) throws IOException { - try (final var gstream = new GZIPInputStream(new ByteBufferInputStream(input))) { - return readFully(gstream); - } - } - - /** - * @see OffsetEncoding#ByteArray - */ - static String deserialiseByteArrayToBitMapString(final ByteBuffer data) { - data.rewind(); - final StringBuilder sb = new StringBuilder(data.capacity()); - while (data.hasRemaining()) { - final byte b = data.get(); - if (b == 1) { - sb.append('x'); - } else { - sb.append('o'); - } - } - return sb.toString(); - } -} +package io.confluent.parallelconsumer.offsets; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ +import com.github.luben.zstd.ZstdInputStream; +import com.github.luben.zstd.ZstdOutputStream; +import lombok.SneakyThrows; +import lombok.experimental.UtilityClass; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.common.utils.ByteBufferInputStream; +import org.xerial.snappy.SnappyInputStream; +import org.xerial.snappy.SnappyOutputStream; + +import java.io.*; +import java.nio.ByteBuffer; +import java.util.Base64; +import java.util.Set; +import java.util.TreeSet; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +import static io.confluent.csid.utils.BackportUtils.readFully; + +@UtilityClass +@Slf4j +public class OffsetSimpleSerialisation { + + @SneakyThrows + static String encodeAsJavaObjectStream(final Set incompleteOffsets) { + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (final ObjectOutputStream os = new ObjectOutputStream(baos)) { + os.writeObject(incompleteOffsets); + } + return Base64.getEncoder().encodeToString(baos.toByteArray()); + } + + private static TreeSet deserialiseJavaWriteObject(final byte[] decode) throws IOException, ClassNotFoundException { + final Set raw; + try (final ObjectInputStream objectInputStream = new ObjectInputStream(new ByteArrayInputStream(decode))) { + raw = (Set) objectInputStream.readObject(); + } + return new TreeSet<>(raw); + } + + @SneakyThrows + static byte[] compressSnappy(final byte[] bytes) { + try (final var out = new ByteArrayOutputStream(); + final var stream = new SnappyOutputStream(out)) { + stream.write(bytes); + return out.toByteArray(); + } + } + + static ByteBuffer decompressSnappy(final ByteBuffer input) throws IOException { + try (final var snappy = new SnappyInputStream(new ByteBufferInputStream(input))) { + byte[] bytes = readFully(snappy); + return ByteBuffer.wrap(bytes); + } + } + + static String base64(final ByteArrayOutputStream out) { + final byte[] src = out.toByteArray(); + return base64(src); + } + + static byte[] compressZstd(final byte[] bytes) throws IOException { + final var out = new ByteArrayOutputStream(); + try (final var zstream = new ZstdOutputStream(out)) { + zstream.write(bytes); + } + return out.toByteArray(); + } + + static byte[] compressGzip(final byte[] bytes) throws IOException { + final var out = new ByteArrayOutputStream(); + try (final var zstream = new GZIPOutputStream(out)) { + zstream.write(bytes); + } + return out.toByteArray(); + } + + static String base64(final byte[] src) { + final byte[] encode = Base64.getEncoder().encode(src); + final String out = new String(encode, OffsetMapCodecManager.CHARSET_TO_USE); + log.trace("Final b64 size: {}", out.length()); + return out; + } + + static byte[] decodeBase64(final String b64) { + final byte[] bytes = b64.getBytes(OffsetMapCodecManager.CHARSET_TO_USE); + return Base64.getDecoder().decode(bytes); + } + + static ByteBuffer decompressZstd(final ByteBuffer input) throws IOException { + try (final var zstream = new ZstdInputStream(new ByteBufferInputStream(input))) { + final byte[] bytes = readFully(zstream); + return ByteBuffer.wrap(bytes); + } + } + + static byte[] decompressGzip(final ByteBuffer input) throws IOException { + try (final var gstream = new GZIPInputStream(new ByteBufferInputStream(input))) { + return readFully(gstream); + } + } + + /** + * @see OffsetEncoding#ByteArray + */ + static String deserialiseByteArrayToBitMapString(final ByteBuffer data) { + data.rewind(); + final StringBuilder sb = new StringBuilder(data.capacity()); + while (data.hasRemaining()) { + final byte b = data.get(); + if (b == 1) { + sb.append('x'); + } else { + sb.append('o'); + } + } + return sb.toString(); + } +} diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/RunLengthEncoder.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/RunLengthEncoder.java index 39fc70988..768a7104b 100644 --- a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/RunLengthEncoder.java +++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/RunLengthEncoder.java @@ -1,149 +1,149 @@ -package io.confluent.parallelconsumer.offsets; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ - -import io.confluent.csid.utils.Range; -import lombok.Getter; - -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - -import static io.confluent.csid.utils.StringUtils.msg; -import static io.confluent.parallelconsumer.offsets.OffsetEncoding.*; - -/** - * RunLength encoder that leverages the nature of this system. - *

- * One such nature is that gaps between completed offsets get encoded as succeeded offsets. This doesn't matter because - * they don't exist and we'll neve see them (they no longer exist in the source partition). - */ -class RunLengthEncoder extends OffsetEncoder { - - private int currentRunLengthCount = 0; - private boolean previousRunLengthState = false; - - @Getter - private final List runLengthEncodingIntegers; - - private Optional encodedBytes = Optional.empty(); - - private final Version version; // default to new version - - private static final Version DEFAULT_VERSION = Version.v2; - - public RunLengthEncoder(OffsetSimultaneousEncoder offsetSimultaneousEncoder, Version newVersion) { - super(offsetSimultaneousEncoder); - // run length setup - runLengthEncodingIntegers = new ArrayList<>(); - version = newVersion; - } - - @Override - protected OffsetEncoding getEncodingType() { - return switch (version) { - case v1 -> RunLength; - case v2 -> RunLengthV2; - }; - } - - @Override - protected OffsetEncoding getEncodingTypeCompressed() { - return switch (version) { - case v1 -> RunLengthCompressed; - case v2 -> RunLengthV2Compressed; - }; - } - - @Override - public void encodeIncompleteOffset(final int rangeIndex) { - encodeRunLength(false, rangeIndex); - } - - @Override - public void encodeCompletedOffset(final int rangeIndex) { - encodeRunLength(true, rangeIndex); - } - - @Override - public byte[] serialise() throws EncodingNotSupportedException { - addTail(); - - int entryWidth = switch (version) { - case v1 -> Short.BYTES; - case v2 -> Integer.BYTES; - }; - ByteBuffer runLengthEncodedByteBuffer = ByteBuffer.allocate(runLengthEncodingIntegers.size() * entryWidth); - - for (final Integer runLength : runLengthEncodingIntegers) { - switch (version) { - case v1 -> { - final short shortCastRunlength = runLength.shortValue(); - if (runLength != shortCastRunlength) - throw new RunlengthV1EncodingNotSupported(msg("Runlength too long for Short ({} cast to {})", runLength, shortCastRunlength)); - runLengthEncodedByteBuffer.putShort(shortCastRunlength); - } - case v2 -> { - runLengthEncodedByteBuffer.putInt(runLength); - } - } - } - - byte[] array = runLengthEncodedByteBuffer.array(); - encodedBytes = Optional.of(array); - return array; - } - - void addTail() { - runLengthEncodingIntegers.add(currentRunLengthCount); - } - - @Override - public int getEncodedSize() { - return encodedBytes.get().length; - } - - @Override - protected byte[] getEncodedBytes() { - return encodedBytes.get(); - } - - int previousRangeIndex = -1; - - private void encodeRunLength(final boolean currentIsComplete, final int rangeIndex) { - // run length - boolean currentOffsetMatchesOurRunLengthState = previousRunLengthState == currentIsComplete; - if (currentOffsetMatchesOurRunLengthState) { - int delta = rangeIndex - previousRangeIndex; - currentRunLengthCount += delta; - } else { - previousRunLengthState = currentIsComplete; - runLengthEncodingIntegers.add(currentRunLengthCount); - currentRunLengthCount = 1; // reset to 1 - } - previousRangeIndex = rangeIndex; - } - - /** - * @return the offsets which are succeeded - */ - public List calculateSucceededActualOffsets(long originalBaseOffset) { - List successfulOffsets = new ArrayList<>(); - boolean succeeded = false; - long offsetPosition = originalBaseOffset; - for (final int run : runLengthEncodingIntegers) { - if (succeeded) { - for (final Integer integer : Range.range(run)) { - long newGoodOffset = offsetPosition + integer; - successfulOffsets.add(newGoodOffset); - } - } - offsetPosition += run; - succeeded = !succeeded; - } - return successfulOffsets; - } -} +package io.confluent.parallelconsumer.offsets; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ + +import io.confluent.csid.utils.Range; +import lombok.Getter; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import static io.confluent.csid.utils.StringUtils.msg; +import static io.confluent.parallelconsumer.offsets.OffsetEncoding.*; + +/** + * RunLength encoder that leverages the nature of this system. + *

+ * One such nature is that gaps between completed offsets get encoded as succeeded offsets. This doesn't matter because + * they don't exist and we'll neve see them (they no longer exist in the source partition). + */ +class RunLengthEncoder extends OffsetEncoder { + + private int currentRunLengthCount = 0; + private boolean previousRunLengthState = false; + + @Getter + private final List runLengthEncodingIntegers; + + private Optional encodedBytes = Optional.empty(); + + private final Version version; // default to new version + + private static final Version DEFAULT_VERSION = Version.v2; + + public RunLengthEncoder(OffsetSimultaneousEncoder offsetSimultaneousEncoder, Version newVersion) { + super(offsetSimultaneousEncoder); + // run length setup + runLengthEncodingIntegers = new ArrayList<>(); + version = newVersion; + } + + @Override + protected OffsetEncoding getEncodingType() { + return switch (version) { + case v1 -> RunLength; + case v2 -> RunLengthV2; + }; + } + + @Override + protected OffsetEncoding getEncodingTypeCompressed() { + return switch (version) { + case v1 -> RunLengthCompressed; + case v2 -> RunLengthV2Compressed; + }; + } + + @Override + public void encodeIncompleteOffset(final int rangeIndex) { + encodeRunLength(false, rangeIndex); + } + + @Override + public void encodeCompletedOffset(final int rangeIndex) { + encodeRunLength(true, rangeIndex); + } + + @Override + public byte[] serialise() throws EncodingNotSupportedException { + addTail(); + + int entryWidth = switch (version) { + case v1 -> Short.BYTES; + case v2 -> Integer.BYTES; + }; + ByteBuffer runLengthEncodedByteBuffer = ByteBuffer.allocate(runLengthEncodingIntegers.size() * entryWidth); + + for (final Integer runLength : runLengthEncodingIntegers) { + switch (version) { + case v1 -> { + final short shortCastRunlength = runLength.shortValue(); + if (runLength != shortCastRunlength) + throw new RunlengthV1EncodingNotSupported(msg("Runlength too long for Short ({} cast to {})", runLength, shortCastRunlength)); + runLengthEncodedByteBuffer.putShort(shortCastRunlength); + } + case v2 -> { + runLengthEncodedByteBuffer.putInt(runLength); + } + } + } + + byte[] array = runLengthEncodedByteBuffer.array(); + encodedBytes = Optional.of(array); + return array; + } + + void addTail() { + runLengthEncodingIntegers.add(currentRunLengthCount); + } + + @Override + public int getEncodedSize() { + return encodedBytes.get().length; + } + + @Override + protected byte[] getEncodedBytes() { + return encodedBytes.get(); + } + + int previousRangeIndex = -1; + + private void encodeRunLength(final boolean currentIsComplete, final int rangeIndex) { + // run length + boolean currentOffsetMatchesOurRunLengthState = previousRunLengthState == currentIsComplete; + if (currentOffsetMatchesOurRunLengthState) { + int delta = rangeIndex - previousRangeIndex; + currentRunLengthCount += delta; + } else { + previousRunLengthState = currentIsComplete; + runLengthEncodingIntegers.add(currentRunLengthCount); + currentRunLengthCount = 1; // reset to 1 + } + previousRangeIndex = rangeIndex; + } + + /** + * @return the offsets which are succeeded + */ + public List calculateSucceededActualOffsets(long originalBaseOffset) { + List successfulOffsets = new ArrayList<>(); + boolean succeeded = false; + long offsetPosition = originalBaseOffset; + for (final int run : runLengthEncodingIntegers) { + if (succeeded) { + for (final Integer integer : Range.range(run)) { + long newGoodOffset = offsetPosition + integer; + successfulOffsets.add(newGoodOffset); + } + } + offsetPosition += run; + succeeded = !succeeded; + } + return successfulOffsets; + } +} diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/RunlengthV1EncodingNotSupported.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/RunlengthV1EncodingNotSupported.java index 3ac83bfb4..56360a57c 100644 --- a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/RunlengthV1EncodingNotSupported.java +++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/offsets/RunlengthV1EncodingNotSupported.java @@ -1,10 +1,10 @@ -package io.confluent.parallelconsumer.offsets; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ -public class RunlengthV1EncodingNotSupported extends EncodingNotSupportedException { - public RunlengthV1EncodingNotSupported(final String msg) { - super(msg); - } -} +package io.confluent.parallelconsumer.offsets; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ +public class RunlengthV1EncodingNotSupported extends EncodingNotSupportedException { + public RunlengthV1EncodingNotSupported(final String msg) { + super(msg); + } +} diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/state/WorkMailBoxManager.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/state/WorkMailBoxManager.java index 3b1a8e1c4..e08089d1c 100644 --- a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/state/WorkMailBoxManager.java +++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/state/WorkMailBoxManager.java @@ -1,137 +1,137 @@ -package io.confluent.parallelconsumer.state; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ - -import io.confluent.parallelconsumer.internal.BrokerPollSystem; -import io.confluent.parallelconsumer.internal.CountingCRLinkedList; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.common.TopicPartition; - -import java.util.Collection; -import java.util.LinkedList; -import java.util.Queue; -import java.util.concurrent.LinkedBlockingQueue; - -import static io.confluent.csid.utils.KafkaUtils.toTP; - -/** - * Handles the incoming mail for {@link WorkManager}. - */ -@Slf4j -public class WorkMailBoxManager { - - /** - * The number of nested {@link ConsumerRecord} entries in the shared blocking mail box. Cached for performance. - */ - private int sharedBoxNestedRecordCount; - - /** - * The shared mail box. Doesn't need to be thread safe as we already need synchronize on it. - */ - private final LinkedBlockingQueue> workInbox = new LinkedBlockingQueue<>(); - - /** - * Mail box where mail is transferred to immediately. - */ - private final CountingCRLinkedList internalBatchMailQueue = new CountingCRLinkedList<>(); - - /** - * Queue of records flattened from the {@link #internalBatchMailQueue}. - *

- * This is needed because {@link java.util.concurrent.BlockingQueue#drainTo(Collection)} must drain to a collection - * of the same type. We could have {@link BrokerPollSystem} do the flattening, but that would require many calls to - * the Concurrent queue, where this only needs one. Also as we don't expect there to be that many elements in these - * collections (as they contain large batches of records), the overhead will be small. - */ - private final Queue> internalFlattenedMailQueue = new LinkedList<>(); - - /** - * @return amount of work queued in the mail box, awaiting processing into shards, not exact - */ - Integer getWorkQueuedInMailboxCount() { - return sharedBoxNestedRecordCount + - internalBatchMailQueue.getNestedCount() + - internalFlattenedMailQueue.size(); - } - - /** - * Work must be registered in offset order - *

- * Thread safe for use by control and broker poller thread. - * - * @see WorkManager#onSuccess - * @see WorkManager#raisePartitionHighWaterMark - */ - public void registerWork(final ConsumerRecords records) { - synchronized (workInbox) { - sharedBoxNestedRecordCount += records.count(); - workInbox.add(records); - } - } - - - /** - * Must synchronise to keep sharedBoxNestedRecordCount in lock step with the inbox. Register is easy, but drain you - * need to run through an intermediary collection and then count the nested elements, to know how many to subtract - * from the Atomic nested count. - *

- * Plus registering work is relatively infrequent, so shouldn't worry about a little synchronized here - makes it - * much simpler. - */ - private void drainSharedMailbox() { - synchronized (workInbox) { - workInbox.drainTo(internalBatchMailQueue); - sharedBoxNestedRecordCount = 0; - } - } - - /** - * Take our inbound messages from the {@link BrokerPollSystem} and add them to our registry. - */ - private synchronized void flattenBatchQueue() { - drainSharedMailbox(); - - // flatten - while (!internalBatchMailQueue.isEmpty()) { - ConsumerRecords consumerRecords = internalBatchMailQueue.poll(); - log.debug("Flattening {} records", consumerRecords.count()); - for (final ConsumerRecord consumerRecord : consumerRecords) { - internalFlattenedMailQueue.add(consumerRecord); - } - } - } - - /** - * Remove revoked work from the mailbox - */ - public synchronized void onPartitionsRemoved(final Collection removedPartitions) { - log.debug("Removing stale work from inbox queues"); - flattenBatchQueue(); - internalFlattenedMailQueue.removeIf(rec -> - removedPartitions.contains(toTP(rec)) - ); - } - - public synchronized boolean internalFlattenedMailQueueIsEmpty() { - return internalFlattenedMailQueue.isEmpty(); - } - - /** - * @return the next element in our outbound queue, or null if empty - */ - public synchronized ConsumerRecord internalFlattenedMailQueuePoll() { - if (internalBatchMailQueue.isEmpty()) { - // flatten the batch queue in batches when needed - flattenBatchQueue(); - } - return internalFlattenedMailQueue.poll(); - } - - public int internalFlattenedMailQueueSize() { - return internalFlattenedMailQueue.size(); - } -} +package io.confluent.parallelconsumer.state; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ + +import io.confluent.parallelconsumer.internal.BrokerPollSystem; +import io.confluent.parallelconsumer.internal.CountingCRLinkedList; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.common.TopicPartition; + +import java.util.Collection; +import java.util.LinkedList; +import java.util.Queue; +import java.util.concurrent.LinkedBlockingQueue; + +import static io.confluent.csid.utils.KafkaUtils.toTP; + +/** + * Handles the incoming mail for {@link WorkManager}. + */ +@Slf4j +public class WorkMailBoxManager { + + /** + * The number of nested {@link ConsumerRecord} entries in the shared blocking mail box. Cached for performance. + */ + private int sharedBoxNestedRecordCount; + + /** + * The shared mail box. Doesn't need to be thread safe as we already need synchronize on it. + */ + private final LinkedBlockingQueue> workInbox = new LinkedBlockingQueue<>(); + + /** + * Mail box where mail is transferred to immediately. + */ + private final CountingCRLinkedList internalBatchMailQueue = new CountingCRLinkedList<>(); + + /** + * Queue of records flattened from the {@link #internalBatchMailQueue}. + *

+ * This is needed because {@link java.util.concurrent.BlockingQueue#drainTo(Collection)} must drain to a collection + * of the same type. We could have {@link BrokerPollSystem} do the flattening, but that would require many calls to + * the Concurrent queue, where this only needs one. Also as we don't expect there to be that many elements in these + * collections (as they contain large batches of records), the overhead will be small. + */ + private final Queue> internalFlattenedMailQueue = new LinkedList<>(); + + /** + * @return amount of work queued in the mail box, awaiting processing into shards, not exact + */ + Integer getWorkQueuedInMailboxCount() { + return sharedBoxNestedRecordCount + + internalBatchMailQueue.getNestedCount() + + internalFlattenedMailQueue.size(); + } + + /** + * Work must be registered in offset order + *

+ * Thread safe for use by control and broker poller thread. + * + * @see WorkManager#onSuccess + * @see WorkManager#raisePartitionHighWaterMark + */ + public void registerWork(final ConsumerRecords records) { + synchronized (workInbox) { + sharedBoxNestedRecordCount += records.count(); + workInbox.add(records); + } + } + + + /** + * Must synchronise to keep sharedBoxNestedRecordCount in lock step with the inbox. Register is easy, but drain you + * need to run through an intermediary collection and then count the nested elements, to know how many to subtract + * from the Atomic nested count. + *

+ * Plus registering work is relatively infrequent, so shouldn't worry about a little synchronized here - makes it + * much simpler. + */ + private void drainSharedMailbox() { + synchronized (workInbox) { + workInbox.drainTo(internalBatchMailQueue); + sharedBoxNestedRecordCount = 0; + } + } + + /** + * Take our inbound messages from the {@link BrokerPollSystem} and add them to our registry. + */ + private synchronized void flattenBatchQueue() { + drainSharedMailbox(); + + // flatten + while (!internalBatchMailQueue.isEmpty()) { + ConsumerRecords consumerRecords = internalBatchMailQueue.poll(); + log.debug("Flattening {} records", consumerRecords.count()); + for (final ConsumerRecord consumerRecord : consumerRecords) { + internalFlattenedMailQueue.add(consumerRecord); + } + } + } + + /** + * Remove revoked work from the mailbox + */ + public synchronized void onPartitionsRemoved(final Collection removedPartitions) { + log.debug("Removing stale work from inbox queues"); + flattenBatchQueue(); + internalFlattenedMailQueue.removeIf(rec -> + removedPartitions.contains(toTP(rec)) + ); + } + + public synchronized boolean internalFlattenedMailQueueIsEmpty() { + return internalFlattenedMailQueue.isEmpty(); + } + + /** + * @return the next element in our outbound queue, or null if empty + */ + public synchronized ConsumerRecord internalFlattenedMailQueuePoll() { + if (internalBatchMailQueue.isEmpty()) { + // flatten the batch queue in batches when needed + flattenBatchQueue(); + } + return internalFlattenedMailQueue.poll(); + } + + public int internalFlattenedMailQueueSize() { + return internalFlattenedMailQueue.size(); + } +} diff --git a/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/CloseAndOpenOffsetTest.java b/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/CloseAndOpenOffsetTest.java index ad998aed7..8296505bf 100644 --- a/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/CloseAndOpenOffsetTest.java +++ b/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/CloseAndOpenOffsetTest.java @@ -1,396 +1,396 @@ -package io.confluent.parallelconsumer.integrationTests; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ - -import io.confluent.csid.utils.Range; -import io.confluent.parallelconsumer.FakeRuntimeError; -import io.confluent.parallelconsumer.ParallelConsumerOptions; -import io.confluent.parallelconsumer.ParallelEoSStreamProcessor; -import io.confluent.parallelconsumer.integrationTests.utils.KafkaClientUtils; -import io.confluent.parallelconsumer.offsets.OffsetEncoding; -import io.confluent.parallelconsumer.offsets.OffsetMapCodecManager; -import io.confluent.parallelconsumer.offsets.OffsetSimultaneousEncoder; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; -import org.junit.jupiter.api.parallel.ResourceLock; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.EnumSource; -import org.testcontainers.shaded.org.apache.commons.lang.math.RandomUtils; -import pl.tlinkowski.unij.api.UniLists; -import pl.tlinkowski.unij.api.UniSets; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ConcurrentSkipListSet; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.stream.Collectors; - -import static io.confluent.parallelconsumer.ParallelConsumerOptions.CommitMode.PERIODIC_TRANSACTIONAL_PRODUCER; -import static io.confluent.parallelconsumer.ParallelConsumerOptions.ProcessingOrder.UNORDERED; -import static java.time.Duration.ofMillis; -import static java.time.Duration.ofSeconds; -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; -import static org.junit.jupiter.api.Assumptions.assumeFalse; -import static org.junit.jupiter.api.parallel.ResourceAccessMode.READ; - -/** - * Series of tests that check when we close a PC with incompletes encoded, when we open a new one, the correct messages - * are skipped. - * - * @see OffsetMapCodecManager - */ -@Timeout(value = 60) -@Slf4j -public class CloseAndOpenOffsetTest extends BrokerIntegrationTest { - - Duration normalTimeout = ofSeconds(5); - Duration debugTimeout = Duration.ofMinutes(1); - - // use debug timeout while debugging -// Duration timeoutToUse = debugTimeout; - Duration timeoutToUse = normalTimeout; - - String rebalanceTopic; - - @BeforeEach - void setup() { - rebalanceTopic = "close-and-open-" + RandomUtils.nextInt(); - } - - /** - * Publish some messages, some fail, shutdown, startup again, consume again - check we only consume the failed - * messages - *

- * Test with different encodings to make sure each encoding can be used to reload - *

- * Sometimes fails as 5 is not committed in the first run and comes out in the 2nd - *

- * NB: messages 4 and 2 are made to fail - */ - @Timeout(value = 60) - @SneakyThrows - @ParameterizedTest - @EnumSource() - @ResourceLock(value = OffsetMapCodecManager.METADATA_DATA_SIZE_RESOURCE_LOCK, mode = READ) - void offsetsOpenClose(OffsetEncoding encoding) { - var skip = UniLists.of(OffsetEncoding.ByteArray, OffsetEncoding.ByteArrayCompressed); - assumeFalse(skip.contains(encoding)); - - // todo remove - not even relevant to this test? smelly - OffsetMapCodecManager.forcedCodec = Optional.of(encoding); - OffsetSimultaneousEncoder.compressionForced = true; - - // 2 partition topic - try { - ensureTopic(rebalanceTopic, 1); - } catch (Exception e) { - log.warn(e.getMessage(), e); - } - - // - KafkaConsumer newConsumerOne = kcu.createNewConsumer(); - KafkaProducer producerOne = kcu.createNewProducer(true); - var options = ParallelConsumerOptions.builder() - .ordering(UNORDERED) - .commitMode(PERIODIC_TRANSACTIONAL_PRODUCER) - .consumer(newConsumerOne) - .producer(producerOne) - .build(); - kcu.props.put(ConsumerConfig.CLIENT_ID_CONFIG, "ONE-my-client"); - - // first client - { - // - var asyncOne = new ParallelEoSStreamProcessor(options); - - // - asyncOne.subscribe(UniLists.of(rebalanceTopic)); - - // read some messages - var successfullInOne = new ConcurrentLinkedQueue>(); - asyncOne.poll(x -> { - log.info("Read by consumer ONE: {}", x); - if (x.value().equals("4")) { - log.info("Throwing fake error for message 4"); - throw new FakeRuntimeError("Fake error - Message 4"); - } - if (x.value().equals("2")) { - log.info("Throwing fake error for message 2"); - throw new FakeRuntimeError("Fake error - Message 2"); - } - successfullInOne.add(x); - }); - - // wait for initial 0 commit - Thread.sleep(500); - - // - send(rebalanceTopic, 0, 0); - send(rebalanceTopic, 0, 1); - send(rebalanceTopic, 0, 2); - send(rebalanceTopic, 0, 3); - send(rebalanceTopic, 0, 4); - send(rebalanceTopic, 0, 5); - - // all are processed except msg 2 and 4, which holds up the queue - await().alias("check all except 2 and 4 are processed").atMost(normalTimeout).untilAsserted(() -> { - ArrayList> copy = new ArrayList<>(successfullInOne); - assertThat(copy.stream() - .map(x -> x.value()).collect(Collectors.toList())) - .containsOnly("0", "1", "3", "5"); - } - ); - - // wait until all expected records have been processed and committed - // need to wait for final message processing's offset data to be committed - // TODO test for event/trigger instead - could consume offsets topic but have to decode the binary - // could listen to a produce topic, but currently it doesn't use the produce flow - // could add a commit listener to the api, but that's heavy just for this? - // could use Consumer#committed to check and decode, but it's not thread safe - // sleep is lazy but much much simpler - Thread.sleep(500); - - // commit what we've done so far, don't wait for failing messages to be retried (message 4) - log.info("Closing consumer, committing offset map"); - asyncOne.closeDontDrainFirst(); - - await().alias("check all except 2 and 4 are processed") - .atMost(normalTimeout) - .untilAsserted(() -> - assertThat(successfullInOne.stream() - .map(x -> x.value()).collect(Collectors.toList())) - .containsOnly("0", "1", "3", "5")); - - assertThat(asyncOne.getFailureCause()).isNull(); - } - - // second client - { - // - kcu.props.put(ConsumerConfig.CLIENT_ID_CONFIG, "THREE-my-client"); - KafkaConsumer newConsumerThree = kcu.createNewConsumer(); - KafkaProducer producerThree = kcu.createNewProducer(true); - var optionsThree = options.toBuilder().consumer(newConsumerThree).producer(producerThree).build(); - try (var asyncThree = new ParallelEoSStreamProcessor(optionsThree)) { - asyncThree.subscribe(UniLists.of(rebalanceTopic)); - - // read what we're given - var processedByThree = new ConcurrentLinkedQueue>(); - asyncThree.poll(x -> { - log.info("Read by consumer THREE: {}", x.value()); - processedByThree.add(x); - }); - - // - await().alias("only 2 and 4 should be delivered again, as everything else was processed successfully") - .atMost(timeoutToUse) - .untilAsserted(() -> - assertThat(processedByThree).extracting(ConsumerRecord::value) - .containsExactlyInAnyOrder("2", "4")); - } - } - - OffsetMapCodecManager.forcedCodec = Optional.empty(); - OffsetSimultaneousEncoder.compressionForced = false; - } - - private void send(String topic, int partition, Integer value) throws InterruptedException, ExecutionException { - RecordMetadata recordMetadata = kcu.producer.send(new ProducerRecord<>(topic, partition, value.toString(), value.toString())).get(); - } - - private void send(int quantity, String topic, int partition) throws InterruptedException, ExecutionException { - log.debug("Sending {} messages to {}", quantity, topic); - var futures = new ArrayList>(); - // async - for (Integer index : Range.range(quantity)) { - Future send = kcu.producer.send(new ProducerRecord<>(topic, partition, index.toString(), index.toString())); - futures.add(send); - } - // block until finished - for (Future future : futures) { - future.get(); - } - log.debug("Finished sending {} messages", quantity); - } - - - /** - * Make sure we commit a basic offset correctly - send a single message, read, commit, close, open, read - should be - * nothing - */ - @Test - void correctOffsetVerySimple() { - setupTopic(); - - // send a single message - kcu.producer.send(new ProducerRecord<>(topic, "0", "0")); - - KafkaConsumer consumer = kcu.createNewConsumer(); - KafkaProducer producerOne = kcu.createNewProducer(true); - var options = ParallelConsumerOptions.builder() - .ordering(UNORDERED) - .consumer(consumer) - .producer(producerOne) - .commitMode(PERIODIC_TRANSACTIONAL_PRODUCER) - .build(); - - try (var asyncOne = new ParallelEoSStreamProcessor(options)) { - - asyncOne.subscribe(UniLists.of(topic)); - - var readByOne = new ArrayList>(); - asyncOne.poll(msg -> { - log.debug("Reading {}", msg); - readByOne.add(msg); - }); - - // the single message is processed - await().untilAsserted(() -> assertThat(readByOne) - .extracting(ConsumerRecord::value) - .containsExactly("0")); - - } finally { - log.debug("asyncOne closed"); - } - - // - log.debug("Starting up new client"); - kcu.props.put(ConsumerConfig.CLIENT_ID_CONFIG, "THREE-my-client"); - KafkaConsumer newConsumerThree = kcu.createNewConsumer(); - KafkaProducer producerThree = kcu.createNewProducer(true); - ParallelConsumerOptions optionsThree = options.toBuilder() - .consumer(newConsumerThree) - .producer(producerThree) - .build(); - try (var asyncThree = new ParallelEoSStreamProcessor(optionsThree)) { - asyncThree.subscribe(UniLists.of(topic)); - - // read what we're given - var readByThree = new ArrayList>(); - asyncThree.poll(x -> { - log.info("Three read: {}", x.value()); - readByThree.add(x); - }); - - // for at least normalTimeout, nothing should be read back (will be long enough to be sure it never is) - await().alias("nothing should be read back (will be long enough to be sure it never is)") - .pollDelay(ofSeconds(1)) - .atMost(ofSeconds(2)) - .atLeast(ofSeconds(1)) - .untilAsserted(() -> { - assertThat(readByThree).as("Nothing should be read into the collection") - .extracting(ConsumerRecord::value) - .isEmpty(); - } - ); - } - } - - /** - * @see KafkaClientUtils#MAX_POLL_RECORDS - */ - @SneakyThrows - @Test - void largeNumberOfMessagesSmallOffsetBitmap() { - setupTopic(); - - int quantity = 10_000; - assertThat(quantity).as("Test expects to process all the produced messages in a single poll") - .isLessThanOrEqualTo(KafkaClientUtils.MAX_POLL_RECORDS); - send(quantity, topic, 0); - - var baseOptions = ParallelConsumerOptions.builder() - .ordering(UNORDERED) - .commitMode(PERIODIC_TRANSACTIONAL_PRODUCER) - .build(); - - Set failingMessages = UniSets.of("123", "2345", "8765"); - int numberOfFailingMessages = failingMessages.size(); - - // step 1 - { - KafkaConsumer consumer = kcu.createNewConsumer(); - KafkaProducer producerOne = kcu.createNewProducer(true); - var options = baseOptions.toBuilder() - .consumer(consumer) - .producer(producerOne) - .build(); - var asyncOne = new ParallelEoSStreamProcessor(options); - - asyncOne.subscribe(UniLists.of(topic)); - - var readByOne = new ConcurrentSkipListSet(); - asyncOne.poll(x -> { - String value = x.value(); - if (failingMessages.contains(value)) { - throw new FakeRuntimeError("Fake error for message " + value); - } - readByOne.add(value); - }); - - // the single message is not processed - await().atMost(ofSeconds(10)).untilAsserted(() -> assertThat(readByOne.size()) - .isEqualTo(quantity - numberOfFailingMessages)); - - // - // TODO: fatal vs retriable exceptions. Retry limits particularly for draining state? - asyncOne.closeDontDrainFirst(); - - // sanity - post close - assertThat(readByOne.size()).isEqualTo(quantity - numberOfFailingMessages); - } - - // step 2 - { - // - kcu.props.put(ConsumerConfig.CLIENT_ID_CONFIG, "THREE-my-client"); - KafkaConsumer newConsumerThree = kcu.createNewConsumer(); - KafkaProducer producerThree = kcu.createNewProducer(true); - var optionsThree = baseOptions.toBuilder() - .consumer(newConsumerThree) - .producer(producerThree) - .build(); - try (var asyncThree = new ParallelEoSStreamProcessor(optionsThree)) { - asyncThree.subscribe(UniLists.of(topic)); - - // read what we're given - var readByThree = new ConcurrentSkipListSet(); - asyncThree.poll(x -> { - log.info("Three read: {}", x.value()); - readByThree.add(x.value()); - }); - - await().alias("Only the one remaining failing message should be submitted for processing") - .pollDelay(ofMillis(1000)) - .atLeast(ofMillis(500)) - .untilAsserted(() -> { - assertThat(readByThree) - .as("Contains only previously failed messages") - .hasSize(numberOfFailingMessages); - } - ); - - // - assertThat(readByThree).hasSize(numberOfFailingMessages); // double check after closing - } - } - } - - -} +package io.confluent.parallelconsumer.integrationTests; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ + +import io.confluent.csid.utils.Range; +import io.confluent.parallelconsumer.FakeRuntimeError; +import io.confluent.parallelconsumer.ParallelConsumerOptions; +import io.confluent.parallelconsumer.ParallelEoSStreamProcessor; +import io.confluent.parallelconsumer.integrationTests.utils.KafkaClientUtils; +import io.confluent.parallelconsumer.offsets.OffsetEncoding; +import io.confluent.parallelconsumer.offsets.OffsetMapCodecManager; +import io.confluent.parallelconsumer.offsets.OffsetSimultaneousEncoder; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.parallel.ResourceLock; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.testcontainers.shaded.org.apache.commons.lang.math.RandomUtils; +import pl.tlinkowski.unij.api.UniLists; +import pl.tlinkowski.unij.api.UniSets; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.stream.Collectors; + +import static io.confluent.parallelconsumer.ParallelConsumerOptions.CommitMode.PERIODIC_TRANSACTIONAL_PRODUCER; +import static io.confluent.parallelconsumer.ParallelConsumerOptions.ProcessingOrder.UNORDERED; +import static java.time.Duration.ofMillis; +import static java.time.Duration.ofSeconds; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assumptions.assumeFalse; +import static org.junit.jupiter.api.parallel.ResourceAccessMode.READ; + +/** + * Series of tests that check when we close a PC with incompletes encoded, when we open a new one, the correct messages + * are skipped. + * + * @see OffsetMapCodecManager + */ +@Timeout(value = 60) +@Slf4j +public class CloseAndOpenOffsetTest extends BrokerIntegrationTest { + + Duration normalTimeout = ofSeconds(5); + Duration debugTimeout = Duration.ofMinutes(1); + + // use debug timeout while debugging +// Duration timeoutToUse = debugTimeout; + Duration timeoutToUse = normalTimeout; + + String rebalanceTopic; + + @BeforeEach + void setup() { + rebalanceTopic = "close-and-open-" + RandomUtils.nextInt(); + } + + /** + * Publish some messages, some fail, shutdown, startup again, consume again - check we only consume the failed + * messages + *

+ * Test with different encodings to make sure each encoding can be used to reload + *

+ * Sometimes fails as 5 is not committed in the first run and comes out in the 2nd + *

+ * NB: messages 4 and 2 are made to fail + */ + @Timeout(value = 60) + @SneakyThrows + @ParameterizedTest + @EnumSource() + @ResourceLock(value = OffsetMapCodecManager.METADATA_DATA_SIZE_RESOURCE_LOCK, mode = READ) + void offsetsOpenClose(OffsetEncoding encoding) { + var skip = UniLists.of(OffsetEncoding.ByteArray, OffsetEncoding.ByteArrayCompressed); + assumeFalse(skip.contains(encoding)); + + // todo remove - not even relevant to this test? smelly + OffsetMapCodecManager.forcedCodec = Optional.of(encoding); + OffsetSimultaneousEncoder.compressionForced = true; + + // 2 partition topic + try { + ensureTopic(rebalanceTopic, 1); + } catch (Exception e) { + log.warn(e.getMessage(), e); + } + + // + KafkaConsumer newConsumerOne = kcu.createNewConsumer(); + KafkaProducer producerOne = kcu.createNewProducer(true); + var options = ParallelConsumerOptions.builder() + .ordering(UNORDERED) + .commitMode(PERIODIC_TRANSACTIONAL_PRODUCER) + .consumer(newConsumerOne) + .producer(producerOne) + .build(); + kcu.props.put(ConsumerConfig.CLIENT_ID_CONFIG, "ONE-my-client"); + + // first client + { + // + var asyncOne = new ParallelEoSStreamProcessor(options); + + // + asyncOne.subscribe(UniLists.of(rebalanceTopic)); + + // read some messages + var successfullInOne = new ConcurrentLinkedQueue>(); + asyncOne.poll(x -> { + log.info("Read by consumer ONE: {}", x); + if (x.value().equals("4")) { + log.info("Throwing fake error for message 4"); + throw new FakeRuntimeError("Fake error - Message 4"); + } + if (x.value().equals("2")) { + log.info("Throwing fake error for message 2"); + throw new FakeRuntimeError("Fake error - Message 2"); + } + successfullInOne.add(x); + }); + + // wait for initial 0 commit + Thread.sleep(500); + + // + send(rebalanceTopic, 0, 0); + send(rebalanceTopic, 0, 1); + send(rebalanceTopic, 0, 2); + send(rebalanceTopic, 0, 3); + send(rebalanceTopic, 0, 4); + send(rebalanceTopic, 0, 5); + + // all are processed except msg 2 and 4, which holds up the queue + await().alias("check all except 2 and 4 are processed").atMost(normalTimeout).untilAsserted(() -> { + ArrayList> copy = new ArrayList<>(successfullInOne); + assertThat(copy.stream() + .map(x -> x.value()).collect(Collectors.toList())) + .containsOnly("0", "1", "3", "5"); + } + ); + + // wait until all expected records have been processed and committed + // need to wait for final message processing's offset data to be committed + // TODO test for event/trigger instead - could consume offsets topic but have to decode the binary + // could listen to a produce topic, but currently it doesn't use the produce flow + // could add a commit listener to the api, but that's heavy just for this? + // could use Consumer#committed to check and decode, but it's not thread safe + // sleep is lazy but much much simpler + Thread.sleep(500); + + // commit what we've done so far, don't wait for failing messages to be retried (message 4) + log.info("Closing consumer, committing offset map"); + asyncOne.closeDontDrainFirst(); + + await().alias("check all except 2 and 4 are processed") + .atMost(normalTimeout) + .untilAsserted(() -> + assertThat(successfullInOne.stream() + .map(x -> x.value()).collect(Collectors.toList())) + .containsOnly("0", "1", "3", "5")); + + assertThat(asyncOne.getFailureCause()).isNull(); + } + + // second client + { + // + kcu.props.put(ConsumerConfig.CLIENT_ID_CONFIG, "THREE-my-client"); + KafkaConsumer newConsumerThree = kcu.createNewConsumer(); + KafkaProducer producerThree = kcu.createNewProducer(true); + var optionsThree = options.toBuilder().consumer(newConsumerThree).producer(producerThree).build(); + try (var asyncThree = new ParallelEoSStreamProcessor(optionsThree)) { + asyncThree.subscribe(UniLists.of(rebalanceTopic)); + + // read what we're given + var processedByThree = new ConcurrentLinkedQueue>(); + asyncThree.poll(x -> { + log.info("Read by consumer THREE: {}", x.value()); + processedByThree.add(x); + }); + + // + await().alias("only 2 and 4 should be delivered again, as everything else was processed successfully") + .atMost(timeoutToUse) + .untilAsserted(() -> + assertThat(processedByThree).extracting(ConsumerRecord::value) + .containsExactlyInAnyOrder("2", "4")); + } + } + + OffsetMapCodecManager.forcedCodec = Optional.empty(); + OffsetSimultaneousEncoder.compressionForced = false; + } + + private void send(String topic, int partition, Integer value) throws InterruptedException, ExecutionException { + RecordMetadata recordMetadata = kcu.producer.send(new ProducerRecord<>(topic, partition, value.toString(), value.toString())).get(); + } + + private void send(int quantity, String topic, int partition) throws InterruptedException, ExecutionException { + log.debug("Sending {} messages to {}", quantity, topic); + var futures = new ArrayList>(); + // async + for (Integer index : Range.range(quantity)) { + Future send = kcu.producer.send(new ProducerRecord<>(topic, partition, index.toString(), index.toString())); + futures.add(send); + } + // block until finished + for (Future future : futures) { + future.get(); + } + log.debug("Finished sending {} messages", quantity); + } + + + /** + * Make sure we commit a basic offset correctly - send a single message, read, commit, close, open, read - should be + * nothing + */ + @Test + void correctOffsetVerySimple() { + setupTopic(); + + // send a single message + kcu.producer.send(new ProducerRecord<>(topic, "0", "0")); + + KafkaConsumer consumer = kcu.createNewConsumer(); + KafkaProducer producerOne = kcu.createNewProducer(true); + var options = ParallelConsumerOptions.builder() + .ordering(UNORDERED) + .consumer(consumer) + .producer(producerOne) + .commitMode(PERIODIC_TRANSACTIONAL_PRODUCER) + .build(); + + try (var asyncOne = new ParallelEoSStreamProcessor(options)) { + + asyncOne.subscribe(UniLists.of(topic)); + + var readByOne = new ArrayList>(); + asyncOne.poll(msg -> { + log.debug("Reading {}", msg); + readByOne.add(msg); + }); + + // the single message is processed + await().untilAsserted(() -> assertThat(readByOne) + .extracting(ConsumerRecord::value) + .containsExactly("0")); + + } finally { + log.debug("asyncOne closed"); + } + + // + log.debug("Starting up new client"); + kcu.props.put(ConsumerConfig.CLIENT_ID_CONFIG, "THREE-my-client"); + KafkaConsumer newConsumerThree = kcu.createNewConsumer(); + KafkaProducer producerThree = kcu.createNewProducer(true); + ParallelConsumerOptions optionsThree = options.toBuilder() + .consumer(newConsumerThree) + .producer(producerThree) + .build(); + try (var asyncThree = new ParallelEoSStreamProcessor(optionsThree)) { + asyncThree.subscribe(UniLists.of(topic)); + + // read what we're given + var readByThree = new ArrayList>(); + asyncThree.poll(x -> { + log.info("Three read: {}", x.value()); + readByThree.add(x); + }); + + // for at least normalTimeout, nothing should be read back (will be long enough to be sure it never is) + await().alias("nothing should be read back (will be long enough to be sure it never is)") + .pollDelay(ofSeconds(1)) + .atMost(ofSeconds(2)) + .atLeast(ofSeconds(1)) + .untilAsserted(() -> { + assertThat(readByThree).as("Nothing should be read into the collection") + .extracting(ConsumerRecord::value) + .isEmpty(); + } + ); + } + } + + /** + * @see KafkaClientUtils#MAX_POLL_RECORDS + */ + @SneakyThrows + @Test + void largeNumberOfMessagesSmallOffsetBitmap() { + setupTopic(); + + int quantity = 10_000; + assertThat(quantity).as("Test expects to process all the produced messages in a single poll") + .isLessThanOrEqualTo(KafkaClientUtils.MAX_POLL_RECORDS); + send(quantity, topic, 0); + + var baseOptions = ParallelConsumerOptions.builder() + .ordering(UNORDERED) + .commitMode(PERIODIC_TRANSACTIONAL_PRODUCER) + .build(); + + Set failingMessages = UniSets.of("123", "2345", "8765"); + int numberOfFailingMessages = failingMessages.size(); + + // step 1 + { + KafkaConsumer consumer = kcu.createNewConsumer(); + KafkaProducer producerOne = kcu.createNewProducer(true); + var options = baseOptions.toBuilder() + .consumer(consumer) + .producer(producerOne) + .build(); + var asyncOne = new ParallelEoSStreamProcessor(options); + + asyncOne.subscribe(UniLists.of(topic)); + + var readByOne = new ConcurrentSkipListSet(); + asyncOne.poll(x -> { + String value = x.value(); + if (failingMessages.contains(value)) { + throw new FakeRuntimeError("Fake error for message " + value); + } + readByOne.add(value); + }); + + // the single message is not processed + await().atMost(ofSeconds(10)).untilAsserted(() -> assertThat(readByOne.size()) + .isEqualTo(quantity - numberOfFailingMessages)); + + // + // TODO: fatal vs retriable exceptions. Retry limits particularly for draining state? + asyncOne.closeDontDrainFirst(); + + // sanity - post close + assertThat(readByOne.size()).isEqualTo(quantity - numberOfFailingMessages); + } + + // step 2 + { + // + kcu.props.put(ConsumerConfig.CLIENT_ID_CONFIG, "THREE-my-client"); + KafkaConsumer newConsumerThree = kcu.createNewConsumer(); + KafkaProducer producerThree = kcu.createNewProducer(true); + var optionsThree = baseOptions.toBuilder() + .consumer(newConsumerThree) + .producer(producerThree) + .build(); + try (var asyncThree = new ParallelEoSStreamProcessor(optionsThree)) { + asyncThree.subscribe(UniLists.of(topic)); + + // read what we're given + var readByThree = new ConcurrentSkipListSet(); + asyncThree.poll(x -> { + log.info("Three read: {}", x.value()); + readByThree.add(x.value()); + }); + + await().alias("Only the one remaining failing message should be submitted for processing") + .pollDelay(ofMillis(1000)) + .atLeast(ofMillis(500)) + .untilAsserted(() -> { + assertThat(readByThree) + .as("Contains only previously failed messages") + .hasSize(numberOfFailingMessages); + } + ); + + // + assertThat(readByThree).hasSize(numberOfFailingMessages); // double check after closing + } + } + } + + +} diff --git a/parallel-consumer-core/src/test/java/io/confluent/csid/utils/ProgressBarUtils.java b/parallel-consumer-core/src/test/java/io/confluent/csid/utils/ProgressBarUtils.java index 1c338b270..920aaa5f6 100644 --- a/parallel-consumer-core/src/test/java/io/confluent/csid/utils/ProgressBarUtils.java +++ b/parallel-consumer-core/src/test/java/io/confluent/csid/utils/ProgressBarUtils.java @@ -1,37 +1,37 @@ -package io.confluent.csid.utils; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ - -import lombok.experimental.UtilityClass; -import me.tongfei.progressbar.DelegatingProgressBarConsumer; -import me.tongfei.progressbar.ProgressBar; -import me.tongfei.progressbar.ProgressBarBuilder; -import org.slf4j.Logger; - -@UtilityClass -public class ProgressBarUtils { - - public static ProgressBar getNewMessagesBar(Logger log, int initialMax) { - return getNewMessagesBar(null, log, initialMax); - } - - public static ProgressBar getNewMessagesBar(String name, Logger log, int initialMax) { - DelegatingProgressBarConsumer delegatingProgressBarConsumer = new DelegatingProgressBarConsumer(log::info); - - int max = 100; - String usedName = "progress"; - if (name != null) - usedName = name; - - ProgressBar build = new ProgressBarBuilder() - .setConsumer(delegatingProgressBarConsumer) - .setInitialMax(initialMax) - .showSpeed() - .setTaskName(usedName) - .setUnit("msg", 1) - .build(); - return build; - } -} +package io.confluent.csid.utils; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ + +import lombok.experimental.UtilityClass; +import me.tongfei.progressbar.DelegatingProgressBarConsumer; +import me.tongfei.progressbar.ProgressBar; +import me.tongfei.progressbar.ProgressBarBuilder; +import org.slf4j.Logger; + +@UtilityClass +public class ProgressBarUtils { + + public static ProgressBar getNewMessagesBar(Logger log, int initialMax) { + return getNewMessagesBar(null, log, initialMax); + } + + public static ProgressBar getNewMessagesBar(String name, Logger log, int initialMax) { + DelegatingProgressBarConsumer delegatingProgressBarConsumer = new DelegatingProgressBarConsumer(log::info); + + int max = 100; + String usedName = "progress"; + if (name != null) + usedName = name; + + ProgressBar build = new ProgressBarBuilder() + .setConsumer(delegatingProgressBarConsumer) + .setInitialMax(initialMax) + .showSpeed() + .setTaskName(usedName) + .setUnit("msg", 1) + .build(); + return build; + } +} diff --git a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/FakeRuntimeError.java b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/FakeRuntimeError.java index 4ffda8bfa..55bcb3acd 100644 --- a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/FakeRuntimeError.java +++ b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/FakeRuntimeError.java @@ -1,14 +1,14 @@ -package io.confluent.parallelconsumer; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ - -/** - * Used for testing error handling - easier to identify than a plan exception. - */ -public class FakeRuntimeError extends RuntimeException { - public FakeRuntimeError(String msg) { - super(msg); - } -} +package io.confluent.parallelconsumer; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ + +/** + * Used for testing error handling - easier to identify than a plan exception. + */ +public class FakeRuntimeError extends RuntimeException { + public FakeRuntimeError(String msg) { + super(msg); + } +} diff --git a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/ParallelEoSStreamProcessorTestBase.java b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/ParallelEoSStreamProcessorTestBase.java index 378572e3f..975a32851 100644 --- a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/ParallelEoSStreamProcessorTestBase.java +++ b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/ParallelEoSStreamProcessorTestBase.java @@ -1,426 +1,426 @@ -package io.confluent.parallelconsumer; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ - -import io.confluent.csid.utils.KafkaTestUtils; -import io.confluent.csid.utils.LongPollingMockConsumer; -import io.confluent.parallelconsumer.ParallelConsumerOptions.ProcessingOrder; -import io.confluent.parallelconsumer.state.WorkContainer; -import io.confluent.parallelconsumer.state.WorkManager; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.*; -import org.apache.kafka.clients.producer.MockProducer; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.serialization.Serdes; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import pl.tlinkowski.unij.api.UniLists; -import pl.tlinkowski.unij.api.UniMaps; - -import java.time.Duration; -import java.util.*; -import java.util.concurrent.Callable; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; - -import static io.confluent.csid.utils.KafkaTestUtils.trimAllGeneisOffset; -import static io.confluent.csid.utils.LatchTestUtils.awaitLatch; -import static io.confluent.csid.utils.StringUtils.msg; -import static io.confluent.parallelconsumer.ParallelConsumerOptions.CommitMode.*; -import static io.confluent.parallelconsumer.ParallelConsumerOptions.ProcessingOrder.UNORDERED; -import static java.time.Duration.ofMillis; -import static java.time.Duration.ofSeconds; -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; -import static org.awaitility.Awaitility.waitAtMost; -import static org.mockito.Mockito.*; -import static pl.tlinkowski.unij.api.UniLists.of; - -@Slf4j -public class ParallelEoSStreamProcessorTestBase { - - public String INPUT_TOPIC; - public String OUTPUT_TOPIC; - public String CONSUMER_GROUP_ID; - - public ConsumerGroupMetadata DEFAULT_GROUP_METADATA; - - /** - * The frequency with which we pretend to poll the broker for records - actually the pretend long poll timeout. A - * lower value shouldn't affect test speed much unless many different batches of messages are "published". - * - * @see LongPollingMockConsumer#poll(Duration) - */ - public static final int DEFAULT_BROKER_POLL_FREQUENCY_MS = 100; - - /** - * The commit interval for the main {@link ParallelEoSStreamProcessor} control thread. Actually the timeout that we - * poll the {@link LinkedBlockingQueue} for. A lower value will increase the frequency of control loop cycles, - * making our test waiting go faster. - * - * @see ParallelEoSStreamProcessor#workMailBox - * @see ParallelEoSStreamProcessor#processWorkCompleteMailBox - */ - public static final int DEFAULT_COMMIT_INTERVAL_MAX_MS = 100; - - protected LongPollingMockConsumer consumerSpy; - protected MockProducer producerSpy; - - protected ParallelEoSStreamProcessor parallelConsumer; - - public static int defaultTimeoutSeconds = 10; - - public static Duration defaultTimeout = ofSeconds(defaultTimeoutSeconds); - protected static long defaultTimeoutMs = defaultTimeout.toMillis(); - protected static Duration effectivelyInfiniteTimeout = Duration.ofMinutes(20); - - ParallelEoSStreamProcessorTest.MyAction myRecordProcessingAction; - - ConsumerRecord firstRecord; - ConsumerRecord secondRecord; - - protected KafkaTestUtils ktu; - - protected AtomicReference loopCountRef; - - volatile CountDownLatch loopLatchV = new CountDownLatch(0); - volatile CountDownLatch controlLoopPauseLatch = new CountDownLatch(0); - protected AtomicReference loopCount; - - /** - * Time to wait to verify some assertion types - */ - long verificationWaitDelay; - protected TopicPartition topicPartition; - - /** - * Unique topic names for each test method - */ - public void setupTopicNames() { - INPUT_TOPIC = "input-" + Math.random(); - OUTPUT_TOPIC = "output-" + Math.random(); - CONSUMER_GROUP_ID = "my-group" + Math.random(); - topicPartition = new TopicPartition(INPUT_TOPIC, 0); - DEFAULT_GROUP_METADATA = new ConsumerGroupMetadata(CONSUMER_GROUP_ID); - } - - @BeforeEach - public void setupAsyncConsumerTestBase() { - setupTopicNames(); - - ParallelConsumerOptions options = getOptions(); - setupParallelConsumerInstance(options); - } - - protected ParallelConsumerOptions getOptions() { - ParallelConsumerOptions options = getDefaultOptions() - .build(); - return options; - } - - protected ParallelConsumerOptions.ParallelConsumerOptionsBuilder getDefaultOptions() { - return ParallelConsumerOptions.builder() - .commitMode(PERIODIC_CONSUMER_SYNC) - .ordering(UNORDERED); - } - - @AfterEach - public void close() { - // don't try to close if error'd (at least one test purposefully creates an error to tests error handling) - we - // don't want to bubble up an error here that we expect from here. - if (!parallelConsumer.isClosedOrFailed()) { - parallelConsumer.close(); - } - } - - protected void injectWorkSuccessListener(WorkManager wm, List> customSuccessfulWork) { - wm.getSuccessfulWorkListeners().add((work) -> { - log.debug("Test work listener heard some successful work: {}", work); - synchronized (customSuccessfulWork) { - customSuccessfulWork.add(work); - } - }); - } - - protected void primeFirstRecord() { - firstRecord = ktu.makeRecord("key-0", "v0"); - consumerSpy.addRecord(firstRecord); - } - - protected MockConsumer setupClients() { - instantiateConsumerProducer(); - - ktu = new KafkaTestUtils(INPUT_TOPIC, CONSUMER_GROUP_ID, consumerSpy); - - return consumerSpy; - } - - protected void instantiateConsumerProducer() { - LongPollingMockConsumer consumer = new LongPollingMockConsumer<>(OffsetResetStrategy.EARLIEST); - MockProducer producer = new MockProducer<>(true, - Serdes.String().serializer(), Serdes.String().serializer()); - - this.producerSpy = spy(producer); - this.consumerSpy = spy(consumer); - myRecordProcessingAction = mock(ParallelEoSStreamProcessorTest.MyAction.class); - - when(consumerSpy.groupMetadata()).thenReturn(DEFAULT_GROUP_METADATA); - } - - /** - * Need to make sure we only use {@link ParallelEoSStreamProcessor#subscribe} methods, and not do manual assignment, - * otherwise rebalance listeners don't fire (because there are never rebalances). - */ - protected void subscribeParallelConsumerAndMockConsumerTo(String topic) { - List of = of(topic); - parallelConsumer.subscribe(of); - consumerSpy.subscribeWithRebalanceAndAssignment(of, 2); - } - - protected void setupParallelConsumerInstance(ProcessingOrder order) { - setupParallelConsumerInstance(ParallelConsumerOptions.builder().ordering(order).build()); - } - - protected void setupParallelConsumerInstance(ParallelConsumerOptions parallelConsumerOptions) { - setupClients(); - - var optionsWithClients = parallelConsumerOptions.toBuilder() - .consumer(consumerSpy) - .producer(producerSpy) - .build(); - - parallelConsumer = initAsyncConsumer(optionsWithClients); - - subscribeParallelConsumerAndMockConsumerTo(INPUT_TOPIC); - - parallelConsumer.setLongPollTimeout(ofMillis(DEFAULT_BROKER_POLL_FREQUENCY_MS)); - parallelConsumer.setTimeBetweenCommits(ofMillis(DEFAULT_COMMIT_INTERVAL_MAX_MS)); - - verificationWaitDelay = parallelConsumer.getTimeBetweenCommits().multipliedBy(2).toMillis(); - - loopCountRef = attachLoopCounter(parallelConsumer); - } - - protected ParallelEoSStreamProcessor initAsyncConsumer(ParallelConsumerOptions parallelConsumerOptions) { - parallelConsumer = new ParallelEoSStreamProcessor<>(parallelConsumerOptions); - - return parallelConsumer; - } - - protected void sendSecondRecord(MockConsumer consumer) { - secondRecord = ktu.makeRecord("key-0", "v1"); - consumer.addRecord(secondRecord); - } - - protected AtomicReference attachLoopCounter(ParallelEoSStreamProcessor parallelConsumer) { - final AtomicReference currentLoop = new AtomicReference<>(0); - parallelConsumer.addLoopEndCallBack(() -> { - Integer currentNumber = currentLoop.get(); - int newLoopNumber = currentNumber + 1; - currentLoop.compareAndSet(currentNumber, newLoopNumber); - log.trace("Counting down latch from {}", loopLatchV.getCount()); - loopLatchV.countDown(); - log.trace("Loop latch remaining: {}", loopLatchV.getCount()); - if (controlLoopPauseLatch.getCount() > 0) { - log.debug("Waiting on pause latch ({})...", controlLoopPauseLatch.getCount()); - try { - controlLoopPauseLatch.await(); - } catch (InterruptedException e) { - log.error(e.getMessage(), e); - } - log.trace("Completed waiting on pause latch"); - } - log.trace("Loop count {}", currentLoop.get()); - }); - return currentLoop; - } - - /** - * Pauses the control loop by awaiting this injected countdown lunch - */ - protected void pauseControlLoop() { - log.trace("Pause loop"); - controlLoopPauseLatch = new CountDownLatch(1); - } - - /** - * Resume is the controller by decrementing the injected countdown latch - */ - protected void resumeControlLoop() { - log.trace("Resume loop"); - controlLoopPauseLatch.countDown(); - } - - protected void waitForOneLoopCycle() { - waitForSomeLoopCycles(1); - } - - protected void waitForSomeLoopCycles(int thisManyMore) { - log.debug("Waiting for {} more iterations of the control loop.", thisManyMore); - blockingLoopLatchTrigger(thisManyMore); - log.debug("Completed waiting on {} loop(s)", thisManyMore); - } - - protected void waitUntilTrue(Callable booleanCallable) { - waitAtMost(defaultTimeout).until(booleanCallable); - } - - /** - * Make sure the latch is attached, if this times out unexpectedly - */ - @SneakyThrows - private void blockingLoopLatchTrigger(int waitForCount) { - log.debug("Waiting on {} cycles on loop latch...", waitForCount); - loopLatchV = new CountDownLatch(waitForCount); - boolean timeout = !loopLatchV.await(defaultTimeoutSeconds, SECONDS); - if (timeout) - throw new TimeoutException(msg("Timeout of {}, waiting for {} counts, on latch with {} left", defaultTimeoutSeconds, waitForCount, loopLatchV.getCount())); - } - - @SneakyThrows - private void waitForLoopCount(int waitForCount) { - log.debug("Waiting on {} cycles on loop latch...", waitForCount); - waitAtMost(defaultTimeout.multipliedBy(100)).until(() -> loopCount.get() > waitForCount); - } - - protected void waitForCommitExact(int offset) { - log.debug("Waiting for commit offset {}", offset); - await().untilAsserted(() -> assertCommits(of(offset))); - } - - - protected void waitForCommitExact(int partition, int offset) { - log.debug("Waiting for commit offset {} on partition {}", offset, partition); - var expectedOffset = new OffsetAndMetadata(offset, ""); - TopicPartition partitionNumber = new TopicPartition(INPUT_TOPIC, partition); - var expectedOffsetMap = UniMaps.of(partitionNumber, expectedOffset); - verify(producerSpy, timeout(defaultTimeoutMs).times(1)).sendOffsetsToTransaction(argThat( - (offsetMap) -> offsetMap.equals(expectedOffsetMap)), - any(ConsumerGroupMetadata.class)); - } - - public void assertCommits(List offsets, String description) { - assertCommits(offsets, Optional.of(description)); - } - - /** - * Flattens the offsets of all partitions into a single sequential list - */ - public void assertCommits(List offsets, Optional description) { - if (isUsingTransactionalProducer()) { - ktu.assertCommits(producerSpy, offsets, description); - assertThat(extractAllPartitionsOffsetsSequentially()).isEmpty(); - } else { - List collect = extractAllPartitionsOffsetsSequentially(); - collect = trimAllGeneisOffset(collect); - // duplicates are ok - // is there a nicer optional way? - // {@link Optional#ifPresentOrElse} only @since 9 - if (description.isPresent()) { - assertThat(collect).as(description.get()).hasSameElementsAs(offsets); - } else { - try { - assertThat(collect).hasSameElementsAs(offsets); - } catch (AssertionError e) { - throw e; - } - } - - ktu.assertCommits(producerSpy, UniLists.of(), Optional.of("Empty")); - } - } - - /** - * Flattens the offsets of all partitions into a single sequential list - */ - protected List extractAllPartitionsOffsetsSequentially() { - var result = new ArrayList(); - // copy the list for safe concurrent access - List> history = new ArrayList<>(consumerSpy.getCommitHistoryInt()); - return history.stream() - .flatMap(commits -> - { - Collection values = new ArrayList<>(commits.values()); - return values.stream().map(meta -> (int) meta.offset()); - } - ).collect(Collectors.toList()); - } - - - protected List extractAllPartitionsOffsetsAndMetadataSequentially() { - var result = new ArrayList(); - // copy the list for safe concurrent access - List> history = new ArrayList<>(consumerSpy.getCommitHistoryInt()); - return history.stream() - .flatMap(commits -> - { - Collection values = new ArrayList<>(commits.values()); - return values.stream(); - } - ).collect(Collectors.toList()); - } - - public void assertCommits(List offsets) { - assertCommits(offsets, Optional.empty()); - } - - /** - * Checks a list of commits of a list of partitions - outer list is partition, inner list is commits - */ - public void assertCommitLists(List> offsets) { - if (isUsingTransactionalProducer()) { - ktu.assertCommitLists(producerSpy, offsets, Optional.empty()); - } else { - List>> commitHistoryWithGropuId = consumerSpy.getCommitHistoryWithGropuId(); - ktu.assertCommitLists(commitHistoryWithGropuId, offsets, Optional.empty()); - } - } - - protected List>> getCommitHistory() { - if (isUsingTransactionalProducer()) { - return producerSpy.consumerGroupOffsetsHistory(); - } else { - return consumerSpy.getCommitHistoryWithGropuId(); - } - } - - protected boolean isUsingTransactionalProducer() { - ParallelConsumerOptions.CommitMode commitMode = parallelConsumer.getWm().getOptions().getCommitMode(); - return commitMode.equals(PERIODIC_TRANSACTIONAL_PRODUCER); - } - - protected boolean isUsingAsyncCommits() { - ParallelConsumerOptions.CommitMode commitMode = parallelConsumer.getWm().getOptions().getCommitMode(); - return commitMode.equals(PERIODIC_CONSUMER_ASYNCHRONOUS); - } - - - protected void releaseAndWait(List locks, List lockIndexes) { - for (Integer i : lockIndexes) { - log.debug("Releasing {}...", i); - locks.get(i).countDown(); - } - waitForSomeLoopCycles(1); - } - - protected void releaseAndWait(List locks, int lockIndex) { - log.debug("Releasing {}...", lockIndex); - locks.get(lockIndex).countDown(); - waitForSomeLoopCycles(1); - } - - protected void pauseControlToAwaitForLatch(CountDownLatch latch) { - pauseControlLoop(); - awaitLatch(latch); - resumeControlLoop(); - waitForOneLoopCycle(); - } - -} +package io.confluent.parallelconsumer; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ + +import io.confluent.csid.utils.KafkaTestUtils; +import io.confluent.csid.utils.LongPollingMockConsumer; +import io.confluent.parallelconsumer.ParallelConsumerOptions.ProcessingOrder; +import io.confluent.parallelconsumer.state.WorkContainer; +import io.confluent.parallelconsumer.state.WorkManager; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.*; +import org.apache.kafka.clients.producer.MockProducer; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.serialization.Serdes; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import pl.tlinkowski.unij.api.UniLists; +import pl.tlinkowski.unij.api.UniMaps; + +import java.time.Duration; +import java.util.*; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import static io.confluent.csid.utils.KafkaTestUtils.trimAllGeneisOffset; +import static io.confluent.csid.utils.LatchTestUtils.awaitLatch; +import static io.confluent.csid.utils.StringUtils.msg; +import static io.confluent.parallelconsumer.ParallelConsumerOptions.CommitMode.*; +import static io.confluent.parallelconsumer.ParallelConsumerOptions.ProcessingOrder.UNORDERED; +import static java.time.Duration.ofMillis; +import static java.time.Duration.ofSeconds; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.awaitility.Awaitility.waitAtMost; +import static org.mockito.Mockito.*; +import static pl.tlinkowski.unij.api.UniLists.of; + +@Slf4j +public class ParallelEoSStreamProcessorTestBase { + + public String INPUT_TOPIC; + public String OUTPUT_TOPIC; + public String CONSUMER_GROUP_ID; + + public ConsumerGroupMetadata DEFAULT_GROUP_METADATA; + + /** + * The frequency with which we pretend to poll the broker for records - actually the pretend long poll timeout. A + * lower value shouldn't affect test speed much unless many different batches of messages are "published". + * + * @see LongPollingMockConsumer#poll(Duration) + */ + public static final int DEFAULT_BROKER_POLL_FREQUENCY_MS = 100; + + /** + * The commit interval for the main {@link ParallelEoSStreamProcessor} control thread. Actually the timeout that we + * poll the {@link LinkedBlockingQueue} for. A lower value will increase the frequency of control loop cycles, + * making our test waiting go faster. + * + * @see ParallelEoSStreamProcessor#workMailBox + * @see ParallelEoSStreamProcessor#processWorkCompleteMailBox + */ + public static final int DEFAULT_COMMIT_INTERVAL_MAX_MS = 100; + + protected LongPollingMockConsumer consumerSpy; + protected MockProducer producerSpy; + + protected ParallelEoSStreamProcessor parallelConsumer; + + public static int defaultTimeoutSeconds = 10; + + public static Duration defaultTimeout = ofSeconds(defaultTimeoutSeconds); + protected static long defaultTimeoutMs = defaultTimeout.toMillis(); + protected static Duration effectivelyInfiniteTimeout = Duration.ofMinutes(20); + + ParallelEoSStreamProcessorTest.MyAction myRecordProcessingAction; + + ConsumerRecord firstRecord; + ConsumerRecord secondRecord; + + protected KafkaTestUtils ktu; + + protected AtomicReference loopCountRef; + + volatile CountDownLatch loopLatchV = new CountDownLatch(0); + volatile CountDownLatch controlLoopPauseLatch = new CountDownLatch(0); + protected AtomicReference loopCount; + + /** + * Time to wait to verify some assertion types + */ + long verificationWaitDelay; + protected TopicPartition topicPartition; + + /** + * Unique topic names for each test method + */ + public void setupTopicNames() { + INPUT_TOPIC = "input-" + Math.random(); + OUTPUT_TOPIC = "output-" + Math.random(); + CONSUMER_GROUP_ID = "my-group" + Math.random(); + topicPartition = new TopicPartition(INPUT_TOPIC, 0); + DEFAULT_GROUP_METADATA = new ConsumerGroupMetadata(CONSUMER_GROUP_ID); + } + + @BeforeEach + public void setupAsyncConsumerTestBase() { + setupTopicNames(); + + ParallelConsumerOptions options = getOptions(); + setupParallelConsumerInstance(options); + } + + protected ParallelConsumerOptions getOptions() { + ParallelConsumerOptions options = getDefaultOptions() + .build(); + return options; + } + + protected ParallelConsumerOptions.ParallelConsumerOptionsBuilder getDefaultOptions() { + return ParallelConsumerOptions.builder() + .commitMode(PERIODIC_CONSUMER_SYNC) + .ordering(UNORDERED); + } + + @AfterEach + public void close() { + // don't try to close if error'd (at least one test purposefully creates an error to tests error handling) - we + // don't want to bubble up an error here that we expect from here. + if (!parallelConsumer.isClosedOrFailed()) { + parallelConsumer.close(); + } + } + + protected void injectWorkSuccessListener(WorkManager wm, List> customSuccessfulWork) { + wm.getSuccessfulWorkListeners().add((work) -> { + log.debug("Test work listener heard some successful work: {}", work); + synchronized (customSuccessfulWork) { + customSuccessfulWork.add(work); + } + }); + } + + protected void primeFirstRecord() { + firstRecord = ktu.makeRecord("key-0", "v0"); + consumerSpy.addRecord(firstRecord); + } + + protected MockConsumer setupClients() { + instantiateConsumerProducer(); + + ktu = new KafkaTestUtils(INPUT_TOPIC, CONSUMER_GROUP_ID, consumerSpy); + + return consumerSpy; + } + + protected void instantiateConsumerProducer() { + LongPollingMockConsumer consumer = new LongPollingMockConsumer<>(OffsetResetStrategy.EARLIEST); + MockProducer producer = new MockProducer<>(true, + Serdes.String().serializer(), Serdes.String().serializer()); + + this.producerSpy = spy(producer); + this.consumerSpy = spy(consumer); + myRecordProcessingAction = mock(ParallelEoSStreamProcessorTest.MyAction.class); + + when(consumerSpy.groupMetadata()).thenReturn(DEFAULT_GROUP_METADATA); + } + + /** + * Need to make sure we only use {@link ParallelEoSStreamProcessor#subscribe} methods, and not do manual assignment, + * otherwise rebalance listeners don't fire (because there are never rebalances). + */ + protected void subscribeParallelConsumerAndMockConsumerTo(String topic) { + List of = of(topic); + parallelConsumer.subscribe(of); + consumerSpy.subscribeWithRebalanceAndAssignment(of, 2); + } + + protected void setupParallelConsumerInstance(ProcessingOrder order) { + setupParallelConsumerInstance(ParallelConsumerOptions.builder().ordering(order).build()); + } + + protected void setupParallelConsumerInstance(ParallelConsumerOptions parallelConsumerOptions) { + setupClients(); + + var optionsWithClients = parallelConsumerOptions.toBuilder() + .consumer(consumerSpy) + .producer(producerSpy) + .build(); + + parallelConsumer = initAsyncConsumer(optionsWithClients); + + subscribeParallelConsumerAndMockConsumerTo(INPUT_TOPIC); + + parallelConsumer.setLongPollTimeout(ofMillis(DEFAULT_BROKER_POLL_FREQUENCY_MS)); + parallelConsumer.setTimeBetweenCommits(ofMillis(DEFAULT_COMMIT_INTERVAL_MAX_MS)); + + verificationWaitDelay = parallelConsumer.getTimeBetweenCommits().multipliedBy(2).toMillis(); + + loopCountRef = attachLoopCounter(parallelConsumer); + } + + protected ParallelEoSStreamProcessor initAsyncConsumer(ParallelConsumerOptions parallelConsumerOptions) { + parallelConsumer = new ParallelEoSStreamProcessor<>(parallelConsumerOptions); + + return parallelConsumer; + } + + protected void sendSecondRecord(MockConsumer consumer) { + secondRecord = ktu.makeRecord("key-0", "v1"); + consumer.addRecord(secondRecord); + } + + protected AtomicReference attachLoopCounter(ParallelEoSStreamProcessor parallelConsumer) { + final AtomicReference currentLoop = new AtomicReference<>(0); + parallelConsumer.addLoopEndCallBack(() -> { + Integer currentNumber = currentLoop.get(); + int newLoopNumber = currentNumber + 1; + currentLoop.compareAndSet(currentNumber, newLoopNumber); + log.trace("Counting down latch from {}", loopLatchV.getCount()); + loopLatchV.countDown(); + log.trace("Loop latch remaining: {}", loopLatchV.getCount()); + if (controlLoopPauseLatch.getCount() > 0) { + log.debug("Waiting on pause latch ({})...", controlLoopPauseLatch.getCount()); + try { + controlLoopPauseLatch.await(); + } catch (InterruptedException e) { + log.error(e.getMessage(), e); + } + log.trace("Completed waiting on pause latch"); + } + log.trace("Loop count {}", currentLoop.get()); + }); + return currentLoop; + } + + /** + * Pauses the control loop by awaiting this injected countdown lunch + */ + protected void pauseControlLoop() { + log.trace("Pause loop"); + controlLoopPauseLatch = new CountDownLatch(1); + } + + /** + * Resume is the controller by decrementing the injected countdown latch + */ + protected void resumeControlLoop() { + log.trace("Resume loop"); + controlLoopPauseLatch.countDown(); + } + + protected void waitForOneLoopCycle() { + waitForSomeLoopCycles(1); + } + + protected void waitForSomeLoopCycles(int thisManyMore) { + log.debug("Waiting for {} more iterations of the control loop.", thisManyMore); + blockingLoopLatchTrigger(thisManyMore); + log.debug("Completed waiting on {} loop(s)", thisManyMore); + } + + protected void waitUntilTrue(Callable booleanCallable) { + waitAtMost(defaultTimeout).until(booleanCallable); + } + + /** + * Make sure the latch is attached, if this times out unexpectedly + */ + @SneakyThrows + private void blockingLoopLatchTrigger(int waitForCount) { + log.debug("Waiting on {} cycles on loop latch...", waitForCount); + loopLatchV = new CountDownLatch(waitForCount); + boolean timeout = !loopLatchV.await(defaultTimeoutSeconds, SECONDS); + if (timeout) + throw new TimeoutException(msg("Timeout of {}, waiting for {} counts, on latch with {} left", defaultTimeoutSeconds, waitForCount, loopLatchV.getCount())); + } + + @SneakyThrows + private void waitForLoopCount(int waitForCount) { + log.debug("Waiting on {} cycles on loop latch...", waitForCount); + waitAtMost(defaultTimeout.multipliedBy(100)).until(() -> loopCount.get() > waitForCount); + } + + protected void waitForCommitExact(int offset) { + log.debug("Waiting for commit offset {}", offset); + await().untilAsserted(() -> assertCommits(of(offset))); + } + + + protected void waitForCommitExact(int partition, int offset) { + log.debug("Waiting for commit offset {} on partition {}", offset, partition); + var expectedOffset = new OffsetAndMetadata(offset, ""); + TopicPartition partitionNumber = new TopicPartition(INPUT_TOPIC, partition); + var expectedOffsetMap = UniMaps.of(partitionNumber, expectedOffset); + verify(producerSpy, timeout(defaultTimeoutMs).times(1)).sendOffsetsToTransaction(argThat( + (offsetMap) -> offsetMap.equals(expectedOffsetMap)), + any(ConsumerGroupMetadata.class)); + } + + public void assertCommits(List offsets, String description) { + assertCommits(offsets, Optional.of(description)); + } + + /** + * Flattens the offsets of all partitions into a single sequential list + */ + public void assertCommits(List offsets, Optional description) { + if (isUsingTransactionalProducer()) { + ktu.assertCommits(producerSpy, offsets, description); + assertThat(extractAllPartitionsOffsetsSequentially()).isEmpty(); + } else { + List collect = extractAllPartitionsOffsetsSequentially(); + collect = trimAllGeneisOffset(collect); + // duplicates are ok + // is there a nicer optional way? + // {@link Optional#ifPresentOrElse} only @since 9 + if (description.isPresent()) { + assertThat(collect).as(description.get()).hasSameElementsAs(offsets); + } else { + try { + assertThat(collect).hasSameElementsAs(offsets); + } catch (AssertionError e) { + throw e; + } + } + + ktu.assertCommits(producerSpy, UniLists.of(), Optional.of("Empty")); + } + } + + /** + * Flattens the offsets of all partitions into a single sequential list + */ + protected List extractAllPartitionsOffsetsSequentially() { + var result = new ArrayList(); + // copy the list for safe concurrent access + List> history = new ArrayList<>(consumerSpy.getCommitHistoryInt()); + return history.stream() + .flatMap(commits -> + { + Collection values = new ArrayList<>(commits.values()); + return values.stream().map(meta -> (int) meta.offset()); + } + ).collect(Collectors.toList()); + } + + + protected List extractAllPartitionsOffsetsAndMetadataSequentially() { + var result = new ArrayList(); + // copy the list for safe concurrent access + List> history = new ArrayList<>(consumerSpy.getCommitHistoryInt()); + return history.stream() + .flatMap(commits -> + { + Collection values = new ArrayList<>(commits.values()); + return values.stream(); + } + ).collect(Collectors.toList()); + } + + public void assertCommits(List offsets) { + assertCommits(offsets, Optional.empty()); + } + + /** + * Checks a list of commits of a list of partitions - outer list is partition, inner list is commits + */ + public void assertCommitLists(List> offsets) { + if (isUsingTransactionalProducer()) { + ktu.assertCommitLists(producerSpy, offsets, Optional.empty()); + } else { + List>> commitHistoryWithGropuId = consumerSpy.getCommitHistoryWithGropuId(); + ktu.assertCommitLists(commitHistoryWithGropuId, offsets, Optional.empty()); + } + } + + protected List>> getCommitHistory() { + if (isUsingTransactionalProducer()) { + return producerSpy.consumerGroupOffsetsHistory(); + } else { + return consumerSpy.getCommitHistoryWithGropuId(); + } + } + + protected boolean isUsingTransactionalProducer() { + ParallelConsumerOptions.CommitMode commitMode = parallelConsumer.getWm().getOptions().getCommitMode(); + return commitMode.equals(PERIODIC_TRANSACTIONAL_PRODUCER); + } + + protected boolean isUsingAsyncCommits() { + ParallelConsumerOptions.CommitMode commitMode = parallelConsumer.getWm().getOptions().getCommitMode(); + return commitMode.equals(PERIODIC_CONSUMER_ASYNCHRONOUS); + } + + + protected void releaseAndWait(List locks, List lockIndexes) { + for (Integer i : lockIndexes) { + log.debug("Releasing {}...", i); + locks.get(i).countDown(); + } + waitForSomeLoopCycles(1); + } + + protected void releaseAndWait(List locks, int lockIndex) { + log.debug("Releasing {}...", lockIndex); + locks.get(lockIndex).countDown(); + waitForSomeLoopCycles(1); + } + + protected void pauseControlToAwaitForLatch(CountDownLatch latch) { + pauseControlLoop(); + awaitLatch(latch); + resumeControlLoop(); + waitForOneLoopCycle(); + } + +} diff --git a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/offsets/BitsetEncodingTest.java b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/offsets/BitsetEncodingTest.java index 360f886fe..4600cf073 100644 --- a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/offsets/BitsetEncodingTest.java +++ b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/offsets/BitsetEncodingTest.java @@ -1,61 +1,61 @@ -package io.confluent.parallelconsumer.offsets; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ - -import lombok.SneakyThrows; -import org.junit.jupiter.api.Test; -import pl.tlinkowski.unij.api.UniLists; -import pl.tlinkowski.unij.api.UniSets; - -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import static io.confluent.parallelconsumer.offsets.OffsetEncoding.Version.v2; -import static org.assertj.core.api.Assertions.assertThat; - -class BitsetEncodingTest { - - @SneakyThrows - @Test - void basic() { - Set incompletes = UniSets.of(0, 4, 6, 7, 8, 10).stream().map(x -> (long) x).collect(Collectors.toSet()); // lol - DRY! - List completes = UniLists.of(1, 2, 3, 5, 9).stream().map(x -> (long) x).collect(Collectors.toList()); // lol - DRY! - OffsetSimultaneousEncoder offsetSimultaneousEncoder = new OffsetSimultaneousEncoder(-1, 0L, incompletes); - int length = 11; - BitsetEncoder bs = new BitsetEncoder(length, offsetSimultaneousEncoder, v2); - - bs.encodeIncompleteOffset(0); - bs.encodeCompletedOffset(1); - bs.encodeCompletedOffset(2); - bs.encodeCompletedOffset(3); - bs.encodeIncompleteOffset(4); - bs.encodeCompletedOffset(5); - bs.encodeIncompleteOffset(6); - bs.encodeIncompleteOffset(7); - bs.encodeIncompleteOffset(8); - bs.encodeCompletedOffset(9); - bs.encodeIncompleteOffset(10); - - // before serialisation - { - assertThat(bs.getBitSet().stream().toArray()).containsExactly(1, 2, 3, 5, 9); - } - - // after serialisation - { - byte[] raw = bs.serialise(); - - byte[] wrapped = offsetSimultaneousEncoder.packEncoding(new EncodedOffsetPair(OffsetEncoding.BitSetV2, ByteBuffer.wrap(raw))); - - OffsetMapCodecManager.HighestOffsetAndIncompletes result = OffsetMapCodecManager.decodeCompressedOffsets(0, wrapped); - - assertThat(result.getHighestSeenOffset()).isEqualTo(10); - - assertThat(result.getIncompleteOffsets()).containsExactlyInAnyOrderElementsOf(incompletes); - } - } -} +package io.confluent.parallelconsumer.offsets; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ + +import lombok.SneakyThrows; +import org.junit.jupiter.api.Test; +import pl.tlinkowski.unij.api.UniLists; +import pl.tlinkowski.unij.api.UniSets; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static io.confluent.parallelconsumer.offsets.OffsetEncoding.Version.v2; +import static org.assertj.core.api.Assertions.assertThat; + +class BitsetEncodingTest { + + @SneakyThrows + @Test + void basic() { + Set incompletes = UniSets.of(0, 4, 6, 7, 8, 10).stream().map(x -> (long) x).collect(Collectors.toSet()); // lol - DRY! + List completes = UniLists.of(1, 2, 3, 5, 9).stream().map(x -> (long) x).collect(Collectors.toList()); // lol - DRY! + OffsetSimultaneousEncoder offsetSimultaneousEncoder = new OffsetSimultaneousEncoder(-1, 0L, incompletes); + int length = 11; + BitsetEncoder bs = new BitsetEncoder(length, offsetSimultaneousEncoder, v2); + + bs.encodeIncompleteOffset(0); + bs.encodeCompletedOffset(1); + bs.encodeCompletedOffset(2); + bs.encodeCompletedOffset(3); + bs.encodeIncompleteOffset(4); + bs.encodeCompletedOffset(5); + bs.encodeIncompleteOffset(6); + bs.encodeIncompleteOffset(7); + bs.encodeIncompleteOffset(8); + bs.encodeCompletedOffset(9); + bs.encodeIncompleteOffset(10); + + // before serialisation + { + assertThat(bs.getBitSet().stream().toArray()).containsExactly(1, 2, 3, 5, 9); + } + + // after serialisation + { + byte[] raw = bs.serialise(); + + byte[] wrapped = offsetSimultaneousEncoder.packEncoding(new EncodedOffsetPair(OffsetEncoding.BitSetV2, ByteBuffer.wrap(raw))); + + OffsetMapCodecManager.HighestOffsetAndIncompletes result = OffsetMapCodecManager.decodeCompressedOffsets(0, wrapped); + + assertThat(result.getHighestSeenOffset()).isEqualTo(10); + + assertThat(result.getIncompleteOffsets()).containsExactlyInAnyOrderElementsOf(incompletes); + } + } +} diff --git a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/offsets/OffsetEncodingBackPressureTest.java b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/offsets/OffsetEncodingBackPressureTest.java index 959e611f9..909d3e987 100644 --- a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/offsets/OffsetEncodingBackPressureTest.java +++ b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/offsets/OffsetEncodingBackPressureTest.java @@ -1,265 +1,265 @@ -package io.confluent.parallelconsumer.offsets; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ - -import io.confluent.csid.utils.TrimListRepresentation; -import io.confluent.parallelconsumer.FakeRuntimeError; -import io.confluent.parallelconsumer.ParallelEoSStreamProcessorTestBase; -import io.confluent.parallelconsumer.offsets.OffsetMapCodecManager.HighestOffsetAndIncompletes; -import io.confluent.parallelconsumer.state.PartitionState; -import io.confluent.parallelconsumer.state.WorkContainer; -import io.confluent.parallelconsumer.state.WorkManager; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.OffsetAndMetadata; -import org.apache.kafka.common.TopicPartition; -import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.parallel.Isolated; -import org.junit.jupiter.api.parallel.ResourceAccessMode; -import org.junit.jupiter.api.parallel.ResourceLock; - -import java.time.Duration; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicInteger; - -import static io.confluent.csid.utils.JavaUtils.getLast; -import static io.confluent.csid.utils.JavaUtils.getOnlyOne; -import static io.confluent.csid.utils.LatchTestUtils.awaitLatch; -import static io.confluent.csid.utils.ThreadUtils.sleepQuietly; -import static java.time.Duration.ofMillis; -import static java.time.Duration.ofSeconds; -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; -import static org.awaitility.Awaitility.waitAtMost; - -/** - * Writes to static state to perform test - needs to be run in isolation. However it runs very fast, so it doesn't slow - * down parallel test suite much. - *

- * Runs in isolation regardless of the resource lock read/write setting, because actually various tests depend - * indirectly on the behaviour of the metadata size, even if not so explicitly. - *

- * See {@link OffsetMapCodecManager#METADATA_DATA_SIZE_RESOURCE_LOCK} - */ -@Isolated // messes with static state - breaks other tests running in parallel -@Slf4j -class OffsetEncodingBackPressureTest extends ParallelEoSStreamProcessorTestBase { - - /** - * Tests that when required space for encoding offset becomes too large, back pressure is put into the system so - * that no further messages for the given partitions can be taken for processing, until more messages complete. - */ - @Test - // needed due to static accessors in parallel tests - @ResourceLock(value = OffsetMapCodecManager.METADATA_DATA_SIZE_RESOURCE_LOCK, mode = ResourceAccessMode.READ_WRITE) - void backPressureShouldPreventTooManyMessagesBeingQueuedForProcessing() throws OffsetDecodingError { - // mock messages downloaded for processing > MAX_TO_QUEUE - // make sure work manager doesn't queue more than MAX_TO_QUEUE -// final int numRecords = 1_000_0; - final int numRecords = 1_00; - parallelConsumer.setLongPollTimeout(ofMillis(200)); -// parallelConsumer.setTimeBetweenCommits(); - - // todo - very smelly - store for restoring - var realMax = OffsetMapCodecManager.DefaultMaxMetadataSize; - - // todo don't use static public accessors to change things - makes parallel testing harder and is smelly - OffsetMapCodecManager.DefaultMaxMetadataSize = 40; // reduce available to make testing easier - OffsetMapCodecManager.forcedCodec = Optional.of(OffsetEncoding.BitSetV2); // force one that takes a predictable large amount of space - - ktu.send(consumerSpy, ktu.generateRecords(numRecords)); - - AtomicInteger processedCount = new AtomicInteger(0); - CountDownLatch msgLock = new CountDownLatch(1); - CountDownLatch msgLockTwo = new CountDownLatch(1); - AtomicInteger attempts = new AtomicInteger(0); - parallelConsumer.poll((rec) -> { - - // block the partition to create bigger and bigger offset encoding blocks - if (rec.offset() == 0) { - int attemptNumber = attempts.incrementAndGet(); - if (attemptNumber == 1) { - log.debug("force first message to 'never' complete, causing a large offset encoding (lots of messages completing above the low water mark"); - awaitLatch(msgLock, 60); - log.debug("very slow message awoken, throwing exception"); - throw new FakeRuntimeError("Fake error"); - } else { - log.debug("Second attempt, sleeping"); - awaitLatch(msgLockTwo, 60); - log.debug("Second attempt, unlocked, succeeding"); - } - } else { - sleepQuietly(1); - } - processedCount.getAndIncrement(); - }); - - try { - - // wait for all pre-produced messages to be processed and produced - Assertions.useRepresentation(new TrimListRepresentation()); - waitAtMost(ofSeconds(120)) - // dynamic reason support still waiting https://github.com/awaitility/awaitility/pull/193#issuecomment-873116199 - .failFast("PC died - check logs", parallelConsumer::isClosedOrFailed) - //, () -> parallelConsumer.getFailureCause()) // requires https://github.com/awaitility/awaitility/issues/178#issuecomment-734769761 - .pollInterval(1, SECONDS) - .untilAsserted(() -> { - assertThat(processedCount.get()).isEqualTo(99L); - }); - - // assert commit ok - nothing blocked - { - // - waitForSomeLoopCycles(1); - parallelConsumer.requestCommitAsap(); - waitForSomeLoopCycles(1); - - // - List offsetAndMetadataList = extractAllPartitionsOffsetsAndMetadataSequentially(); - assertThat(offsetAndMetadataList).isNotEmpty(); - OffsetAndMetadata offsetAndMetadata = offsetAndMetadataList.get(offsetAndMetadataList.size() - 1); - assertThat(offsetAndMetadata.offset()).isZero(); - - // - String metadata = offsetAndMetadata.metadata(); - HighestOffsetAndIncompletes decodedOffsetPayload = OffsetMapCodecManager.deserialiseIncompleteOffsetMapFromBase64(0, metadata); - Long highestSeenOffset = decodedOffsetPayload.getHighestSeenOffset(); - Set incompletes = decodedOffsetPayload.getIncompleteOffsets(); - assertThat(incompletes).isNotEmpty().contains(0L).doesNotContain(1L, 50L, 99L); - assertThat(highestSeenOffset).isEqualTo(99L); - } - - WorkManager wm = parallelConsumer.getWm(); - - // partition not blocked - { - boolean partitionBlocked = wm.getPm().isBlocked(topicPartition); - assertThat(partitionBlocked).isFalse(); - } - - // feed more messages in order to threshold block - int extraRecordsToBlockWithThresholdBlocks = numRecords / 2; - int expectedMsgsProcessedUponThresholdBlock = numRecords + (extraRecordsToBlockWithThresholdBlocks / 2); - { - ktu.send(consumerSpy, ktu.generateRecords(extraRecordsToBlockWithThresholdBlocks)); - waitForOneLoopCycle(); - assertThat(wm.getPm().isAllowedMoreRecords(topicPartition)).isTrue(); // should initially be not blocked - await().untilAsserted(() -> - assertThat(processedCount.get()).isGreaterThan(expectedMsgsProcessedUponThresholdBlock) // some new message processed - ); - waitForOneLoopCycle(); - } - - // assert partition now blocked from threshold - int expectedMsgsProcessedBeforePartitionBlocks = numRecords + numRecords / 4; - { - Long partitionOffsetHighWaterMarks = wm.getPm().getHighestSeenOffset(topicPartition); - assertThat(partitionOffsetHighWaterMarks) - .isGreaterThan(expectedMsgsProcessedBeforePartitionBlocks); // high water mark is beyond our expected processed count upon blocking - PartitionState state = wm.getPm().getState(topicPartition); - assertThat(wm.getPm().isBlocked(topicPartition)) - .as("Partition SHOULD be blocked due to back pressure") - .isTrue(); // blocked - - // assert blocked, but can still write payload - // assert the committed offset metadata contains a payload - await().untilAsserted(() -> - { - OffsetAndMetadata partitionCommit = getLastCommit(); - // - assertThat(partitionCommit.offset()).isZero(); - // - String meta = partitionCommit.metadata(); - HighestOffsetAndIncompletes incompletes = OffsetMapCodecManager - .deserialiseIncompleteOffsetMapFromBase64(0L, meta); - assertThat(incompletes.getIncompleteOffsets()).containsOnly(0L); - assertThat(incompletes.getHighestSeenOffset()).isEqualTo(processedCount.get()); - } - ); - } - - // test max payload exceeded, payload dropped - { - // force system to allow more records (i.e. the actual system attempts to never allow the payload to grow this big) - wm.getPm().setUSED_PAYLOAD_THRESHOLD_MULTIPLIER(2); - - // - ktu.send(consumerSpy, ktu.generateRecords(expectedMsgsProcessedBeforePartitionBlocks)); - - // - await().atMost(ofSeconds(5)).untilAsserted(() -> - assertThat(processedCount.get()).isGreaterThan(expectedMsgsProcessedBeforePartitionBlocks) // some new message processed - ); - // assert payload missing from commit now - await().untilAsserted(() -> { - OffsetAndMetadata partitionCommit = getLastCommit(); - assertThat(partitionCommit.offset()).isZero(); - assertThat(partitionCommit.metadata()).isBlank(); // missing offset encoding as too large - }); - } - - // test failed messages can retry - { - Duration aggressiveDelay = ofMillis(100); - WorkContainer.setDefaultRetryDelay(aggressiveDelay); // more aggressive retry - - // release message that was blocking partition progression - // fail the message - msgLock.countDown(); - - // wait for the retry - waitForOneLoopCycle(); - sleepQuietly(aggressiveDelay.toMillis()); - await().until(() -> attempts.get() >= 2); - - // assert partition still blocked - waitForOneLoopCycle(); - await().untilAsserted(() -> assertThat(wm.getPm().isAllowedMoreRecords(topicPartition)).isFalse()); - - // release the message for the second time, allowing it to succeed - msgLockTwo.countDown(); - } - - // assert partition is now not blocked - { - waitForOneLoopCycle(); - await().untilAsserted(() -> assertThat(wm.getPm().isAllowedMoreRecords(topicPartition)).isTrue()); - } - - // assert all committed, nothing blocked- next expected offset is now 1+ the offset of the final message we sent (numRecords*2) - { - int nextExpectedOffsetAfterSubmittedWork = numRecords * 2; - await().untilAsserted(() -> { - List offsets = extractAllPartitionsOffsetsSequentially(); - assertThat(offsets).contains(processedCount.get()); - }); - await().untilAsserted(() -> assertThat(wm.getPm().isAllowedMoreRecords(topicPartition)).isTrue()); - } - } finally { - // make sure to unlock threads - speeds up failed tests, instead of waiting for latch or close timeouts - msgLock.countDown(); - msgLockTwo.countDown(); - - // todo restore static defaults - lazy way to override settings at runtime but causes bugs by allowing them to be statically changeable - OffsetMapCodecManager.DefaultMaxMetadataSize = realMax; // todo wow this is smelly, but convenient - OffsetMapCodecManager.forcedCodec = Optional.empty(); - } - - - } - - private OffsetAndMetadata getLastCommit() { - List>> commitHistory = getCommitHistory(); - Map> lastCommit = getLast(commitHistory).get(); - Map allPartitionCommits = getOnlyOne(lastCommit).get(); - return allPartitionCommits.get(topicPartition); - } - -} +package io.confluent.parallelconsumer.offsets; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ + +import io.confluent.csid.utils.TrimListRepresentation; +import io.confluent.parallelconsumer.FakeRuntimeError; +import io.confluent.parallelconsumer.ParallelEoSStreamProcessorTestBase; +import io.confluent.parallelconsumer.offsets.OffsetMapCodecManager.HighestOffsetAndIncompletes; +import io.confluent.parallelconsumer.state.PartitionState; +import io.confluent.parallelconsumer.state.WorkContainer; +import io.confluent.parallelconsumer.state.WorkManager; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.common.TopicPartition; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Isolated; +import org.junit.jupiter.api.parallel.ResourceAccessMode; +import org.junit.jupiter.api.parallel.ResourceLock; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; + +import static io.confluent.csid.utils.JavaUtils.getLast; +import static io.confluent.csid.utils.JavaUtils.getOnlyOne; +import static io.confluent.csid.utils.LatchTestUtils.awaitLatch; +import static io.confluent.csid.utils.ThreadUtils.sleepQuietly; +import static java.time.Duration.ofMillis; +import static java.time.Duration.ofSeconds; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.awaitility.Awaitility.waitAtMost; + +/** + * Writes to static state to perform test - needs to be run in isolation. However it runs very fast, so it doesn't slow + * down parallel test suite much. + *

+ * Runs in isolation regardless of the resource lock read/write setting, because actually various tests depend + * indirectly on the behaviour of the metadata size, even if not so explicitly. + *

+ * See {@link OffsetMapCodecManager#METADATA_DATA_SIZE_RESOURCE_LOCK} + */ +@Isolated // messes with static state - breaks other tests running in parallel +@Slf4j +class OffsetEncodingBackPressureTest extends ParallelEoSStreamProcessorTestBase { + + /** + * Tests that when required space for encoding offset becomes too large, back pressure is put into the system so + * that no further messages for the given partitions can be taken for processing, until more messages complete. + */ + @Test + // needed due to static accessors in parallel tests + @ResourceLock(value = OffsetMapCodecManager.METADATA_DATA_SIZE_RESOURCE_LOCK, mode = ResourceAccessMode.READ_WRITE) + void backPressureShouldPreventTooManyMessagesBeingQueuedForProcessing() throws OffsetDecodingError { + // mock messages downloaded for processing > MAX_TO_QUEUE + // make sure work manager doesn't queue more than MAX_TO_QUEUE +// final int numRecords = 1_000_0; + final int numRecords = 1_00; + parallelConsumer.setLongPollTimeout(ofMillis(200)); +// parallelConsumer.setTimeBetweenCommits(); + + // todo - very smelly - store for restoring + var realMax = OffsetMapCodecManager.DefaultMaxMetadataSize; + + // todo don't use static public accessors to change things - makes parallel testing harder and is smelly + OffsetMapCodecManager.DefaultMaxMetadataSize = 40; // reduce available to make testing easier + OffsetMapCodecManager.forcedCodec = Optional.of(OffsetEncoding.BitSetV2); // force one that takes a predictable large amount of space + + ktu.send(consumerSpy, ktu.generateRecords(numRecords)); + + AtomicInteger processedCount = new AtomicInteger(0); + CountDownLatch msgLock = new CountDownLatch(1); + CountDownLatch msgLockTwo = new CountDownLatch(1); + AtomicInteger attempts = new AtomicInteger(0); + parallelConsumer.poll((rec) -> { + + // block the partition to create bigger and bigger offset encoding blocks + if (rec.offset() == 0) { + int attemptNumber = attempts.incrementAndGet(); + if (attemptNumber == 1) { + log.debug("force first message to 'never' complete, causing a large offset encoding (lots of messages completing above the low water mark"); + awaitLatch(msgLock, 60); + log.debug("very slow message awoken, throwing exception"); + throw new FakeRuntimeError("Fake error"); + } else { + log.debug("Second attempt, sleeping"); + awaitLatch(msgLockTwo, 60); + log.debug("Second attempt, unlocked, succeeding"); + } + } else { + sleepQuietly(1); + } + processedCount.getAndIncrement(); + }); + + try { + + // wait for all pre-produced messages to be processed and produced + Assertions.useRepresentation(new TrimListRepresentation()); + waitAtMost(ofSeconds(120)) + // dynamic reason support still waiting https://github.com/awaitility/awaitility/pull/193#issuecomment-873116199 + .failFast("PC died - check logs", parallelConsumer::isClosedOrFailed) + //, () -> parallelConsumer.getFailureCause()) // requires https://github.com/awaitility/awaitility/issues/178#issuecomment-734769761 + .pollInterval(1, SECONDS) + .untilAsserted(() -> { + assertThat(processedCount.get()).isEqualTo(99L); + }); + + // assert commit ok - nothing blocked + { + // + waitForSomeLoopCycles(1); + parallelConsumer.requestCommitAsap(); + waitForSomeLoopCycles(1); + + // + List offsetAndMetadataList = extractAllPartitionsOffsetsAndMetadataSequentially(); + assertThat(offsetAndMetadataList).isNotEmpty(); + OffsetAndMetadata offsetAndMetadata = offsetAndMetadataList.get(offsetAndMetadataList.size() - 1); + assertThat(offsetAndMetadata.offset()).isZero(); + + // + String metadata = offsetAndMetadata.metadata(); + HighestOffsetAndIncompletes decodedOffsetPayload = OffsetMapCodecManager.deserialiseIncompleteOffsetMapFromBase64(0, metadata); + Long highestSeenOffset = decodedOffsetPayload.getHighestSeenOffset(); + Set incompletes = decodedOffsetPayload.getIncompleteOffsets(); + assertThat(incompletes).isNotEmpty().contains(0L).doesNotContain(1L, 50L, 99L); + assertThat(highestSeenOffset).isEqualTo(99L); + } + + WorkManager wm = parallelConsumer.getWm(); + + // partition not blocked + { + boolean partitionBlocked = wm.getPm().isBlocked(topicPartition); + assertThat(partitionBlocked).isFalse(); + } + + // feed more messages in order to threshold block + int extraRecordsToBlockWithThresholdBlocks = numRecords / 2; + int expectedMsgsProcessedUponThresholdBlock = numRecords + (extraRecordsToBlockWithThresholdBlocks / 2); + { + ktu.send(consumerSpy, ktu.generateRecords(extraRecordsToBlockWithThresholdBlocks)); + waitForOneLoopCycle(); + assertThat(wm.getPm().isAllowedMoreRecords(topicPartition)).isTrue(); // should initially be not blocked + await().untilAsserted(() -> + assertThat(processedCount.get()).isGreaterThan(expectedMsgsProcessedUponThresholdBlock) // some new message processed + ); + waitForOneLoopCycle(); + } + + // assert partition now blocked from threshold + int expectedMsgsProcessedBeforePartitionBlocks = numRecords + numRecords / 4; + { + Long partitionOffsetHighWaterMarks = wm.getPm().getHighestSeenOffset(topicPartition); + assertThat(partitionOffsetHighWaterMarks) + .isGreaterThan(expectedMsgsProcessedBeforePartitionBlocks); // high water mark is beyond our expected processed count upon blocking + PartitionState state = wm.getPm().getState(topicPartition); + assertThat(wm.getPm().isBlocked(topicPartition)) + .as("Partition SHOULD be blocked due to back pressure") + .isTrue(); // blocked + + // assert blocked, but can still write payload + // assert the committed offset metadata contains a payload + await().untilAsserted(() -> + { + OffsetAndMetadata partitionCommit = getLastCommit(); + // + assertThat(partitionCommit.offset()).isZero(); + // + String meta = partitionCommit.metadata(); + HighestOffsetAndIncompletes incompletes = OffsetMapCodecManager + .deserialiseIncompleteOffsetMapFromBase64(0L, meta); + assertThat(incompletes.getIncompleteOffsets()).containsOnly(0L); + assertThat(incompletes.getHighestSeenOffset()).isEqualTo(processedCount.get()); + } + ); + } + + // test max payload exceeded, payload dropped + { + // force system to allow more records (i.e. the actual system attempts to never allow the payload to grow this big) + wm.getPm().setUSED_PAYLOAD_THRESHOLD_MULTIPLIER(2); + + // + ktu.send(consumerSpy, ktu.generateRecords(expectedMsgsProcessedBeforePartitionBlocks)); + + // + await().atMost(ofSeconds(5)).untilAsserted(() -> + assertThat(processedCount.get()).isGreaterThan(expectedMsgsProcessedBeforePartitionBlocks) // some new message processed + ); + // assert payload missing from commit now + await().untilAsserted(() -> { + OffsetAndMetadata partitionCommit = getLastCommit(); + assertThat(partitionCommit.offset()).isZero(); + assertThat(partitionCommit.metadata()).isBlank(); // missing offset encoding as too large + }); + } + + // test failed messages can retry + { + Duration aggressiveDelay = ofMillis(100); + WorkContainer.setDefaultRetryDelay(aggressiveDelay); // more aggressive retry + + // release message that was blocking partition progression + // fail the message + msgLock.countDown(); + + // wait for the retry + waitForOneLoopCycle(); + sleepQuietly(aggressiveDelay.toMillis()); + await().until(() -> attempts.get() >= 2); + + // assert partition still blocked + waitForOneLoopCycle(); + await().untilAsserted(() -> assertThat(wm.getPm().isAllowedMoreRecords(topicPartition)).isFalse()); + + // release the message for the second time, allowing it to succeed + msgLockTwo.countDown(); + } + + // assert partition is now not blocked + { + waitForOneLoopCycle(); + await().untilAsserted(() -> assertThat(wm.getPm().isAllowedMoreRecords(topicPartition)).isTrue()); + } + + // assert all committed, nothing blocked- next expected offset is now 1+ the offset of the final message we sent (numRecords*2) + { + int nextExpectedOffsetAfterSubmittedWork = numRecords * 2; + await().untilAsserted(() -> { + List offsets = extractAllPartitionsOffsetsSequentially(); + assertThat(offsets).contains(processedCount.get()); + }); + await().untilAsserted(() -> assertThat(wm.getPm().isAllowedMoreRecords(topicPartition)).isTrue()); + } + } finally { + // make sure to unlock threads - speeds up failed tests, instead of waiting for latch or close timeouts + msgLock.countDown(); + msgLockTwo.countDown(); + + // todo restore static defaults - lazy way to override settings at runtime but causes bugs by allowing them to be statically changeable + OffsetMapCodecManager.DefaultMaxMetadataSize = realMax; // todo wow this is smelly, but convenient + OffsetMapCodecManager.forcedCodec = Optional.empty(); + } + + + } + + private OffsetAndMetadata getLastCommit() { + List>> commitHistory = getCommitHistory(); + Map> lastCommit = getLast(commitHistory).get(); + Map allPartitionCommits = getOnlyOne(lastCommit).get(); + return allPartitionCommits.get(topicPartition); + } + +} diff --git a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/offsets/OffsetEncodingTests.java b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/offsets/OffsetEncodingTests.java index 76b503278..79dc927c5 100644 --- a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/offsets/OffsetEncodingTests.java +++ b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/offsets/OffsetEncodingTests.java @@ -1,279 +1,279 @@ -package io.confluent.parallelconsumer.offsets; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ - -import io.confluent.csid.utils.KafkaTestUtils; -import io.confluent.parallelconsumer.ParallelConsumerOptions; -import io.confluent.parallelconsumer.ParallelEoSStreamProcessorTestBase; -import io.confluent.parallelconsumer.state.PartitionState; -import io.confluent.parallelconsumer.state.WorkContainer; -import io.confluent.parallelconsumer.state.WorkManager; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.OffsetAndMetadata; -import org.apache.kafka.common.TopicPartition; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.parallel.ResourceLock; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.EnumSource; -import org.junit.jupiter.params.provider.ValueSource; -import pl.tlinkowski.unij.api.UniSets; - -import java.nio.ByteBuffer; -import java.util.*; -import java.util.stream.Collectors; - -import static io.confluent.parallelconsumer.offsets.OffsetEncoding.ByteArray; -import static io.confluent.parallelconsumer.offsets.OffsetEncoding.ByteArrayCompressed; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.hamcrest.Matchers.in; -import static org.hamcrest.Matchers.not; -import static org.junit.Assume.assumeThat; -import static org.junit.jupiter.api.parallel.ResourceAccessMode.READ; -import static org.junit.jupiter.api.parallel.ResourceAccessMode.READ_WRITE; -import static pl.tlinkowski.unij.api.UniLists.of; - -@Slf4j -public class OffsetEncodingTests extends ParallelEoSStreamProcessorTestBase { - - @Test - void runLengthDeserialise() { - var sb = ByteBuffer.allocate(3); - sb.put((byte) 0); // magic byte place holder, can ignore - sb.putShort((short) 1); - byte[] array = new byte[2]; - sb.rewind(); - sb.get(array); - ByteBuffer wrap = ByteBuffer.wrap(array); - byte b = wrap.get(); // simulate reading magic byte - ByteBuffer slice = wrap.slice(); - List integers = OffsetRunLength.runLengthDeserialise(slice); - assertThat(integers).isEmpty(); - } - - /** - * Triggers Short shortfall in BitSet encoder and tests encodable range of RunLength encoding - system should - * gracefully drop runlength if it has Short overflows (too hard to measure every runlength of incoming records - * before accepting?) - *

- * https://github.com/confluentinc/parallel-consumer/issues/37 Support BitSet encoding lengths longer than - * Short.MAX_VALUE #37 - *

- * https://github.com/confluentinc/parallel-consumer/issues/35 RuntimeException when running with very high options - * in 0.2.0.0 (Bitset too long to encode) #35 - *

- */ - @SneakyThrows - @ParameterizedTest - @ValueSource(longs = { - 10_000L, - 100_000L, - 100_000_0L, -// 100_000_000L, // very~ slow - }) - @ResourceLock(value = OffsetSimultaneousEncoder.COMPRESSION_FORCED_RESOURCE_LOCK, mode = READ_WRITE) - void largeIncompleteOffsetValues(long nextExpectedOffset) { - var incompletes = new HashSet(); - long lowWaterMark = 123L; - incompletes.addAll(UniSets.of(lowWaterMark, 2345L, 8765L)); - - OffsetSimultaneousEncoder encoder = new OffsetSimultaneousEncoder(lowWaterMark, nextExpectedOffset, incompletes); - OffsetSimultaneousEncoder.compressionForced = true; - - // - encoder.invoke(); - Map encodingMap = encoder.getEncodingMap(); - - // - byte[] smallestBytes = encoder.packSmallest(); - EncodedOffsetPair unwrap = EncodedOffsetPair.unwrap(smallestBytes); - OffsetMapCodecManager.HighestOffsetAndIncompletes decodedIncompletes = unwrap.getDecodedIncompletes(lowWaterMark); - assertThat(decodedIncompletes.getIncompleteOffsets()).containsExactlyInAnyOrderElementsOf(incompletes); - - // - for (OffsetEncoding encodingToUse : OffsetEncoding.values()) { - log.info("Testing {}", encodingToUse); - byte[] bitsetBytes = encodingMap.get(encodingToUse); - if (bitsetBytes != null) { - EncodedOffsetPair bitsetUnwrap = EncodedOffsetPair.unwrap(encoder.packEncoding(new EncodedOffsetPair(encodingToUse, ByteBuffer.wrap(bitsetBytes)))); - OffsetMapCodecManager.HighestOffsetAndIncompletes decodedBitsets = bitsetUnwrap.getDecodedIncompletes(lowWaterMark); - assertThat(decodedBitsets.getIncompleteOffsets()) - .as(encodingToUse.toString()) - .containsExactlyInAnyOrderElementsOf(incompletes); - } else { - log.info("Encoding not performed: " + encodingToUse); - } - } - - OffsetSimultaneousEncoder.compressionForced = false; - } - - /** - * There's no guarantee that offsets are always sequential. The most obvious case is with a compacted topic - there - * will always be offsets missing. - * - * @see #ensureEncodingGracefullyWorksWhenOffsetsArentSequentialTwo - */ - @SneakyThrows - @ParameterizedTest - @EnumSource(OffsetEncoding.class) - // needed due to static accessors in parallel tests - @ResourceLock(value = OffsetMapCodecManager.METADATA_DATA_SIZE_RESOURCE_LOCK, mode = READ) - // depends on OffsetMapCodecManager#DefaultMaxMetadataSize - @ResourceLock(value = OffsetSimultaneousEncoder.COMPRESSION_FORCED_RESOURCE_LOCK, mode = READ_WRITE) - void ensureEncodingGracefullyWorksWhenOffsetsAreVeryLargeAndNotSequential(OffsetEncoding encoding) { - assumeThat("Codec skipped, not applicable", encoding, - not(in(of(ByteArray, ByteArrayCompressed)))); // byte array not currently used - - // todo don't use static public accessors to change things - makes parallel testing harder and is smelly - OffsetMapCodecManager.forcedCodec = Optional.of(encoding); - OffsetSimultaneousEncoder.compressionForced = true; - - var records = new ArrayList>(); - records.add(new ConsumerRecord<>(INPUT_TOPIC, 0, 0, "akey", "avalue")); // will complete - records.add(new ConsumerRecord<>(INPUT_TOPIC, 0, 1, "akey", "avalue")); - records.add(new ConsumerRecord<>(INPUT_TOPIC, 0, 4, "akey", "avalue")); - records.add(new ConsumerRecord<>(INPUT_TOPIC, 0, 5, "akey", "avalue")); - records.add(new ConsumerRecord<>(INPUT_TOPIC, 0, 69, "akey", "avalue")); // will complete - records.add(new ConsumerRecord<>(INPUT_TOPIC, 0, 100, "akey", "avalue")); - records.add(new ConsumerRecord<>(INPUT_TOPIC, 0, 1_000, "akey", "avalue")); - records.add(new ConsumerRecord<>(INPUT_TOPIC, 0, 20_000, "akey", "avalue")); // near upper limit of Short.MAX_VALUE - records.add(new ConsumerRecord<>(INPUT_TOPIC, 0, 25_000, "akey", "avalue")); // will complete, near upper limit of Short.MAX_VALUE - records.add(new ConsumerRecord<>(INPUT_TOPIC, 0, 30_000, "akey", "avalue")); // near upper limit of Short.MAX_VALUE - - // Extremely large tests for v2 encoders - records.add(new ConsumerRecord<>(INPUT_TOPIC, 0, 40_000, "akey", "avalue")); // higher than Short.MAX_VALUE - int avoidOffByOne = 2; - records.add(new ConsumerRecord<>(INPUT_TOPIC, 0, 40_000 + Short.MAX_VALUE + avoidOffByOne, "akey", "avalue")); // runlength higher than Short.MAX_VALUE - - var firstSucceededRecordRemoved = new ArrayList<>(records); - firstSucceededRecordRemoved.remove(firstSucceededRecordRemoved.stream().filter(x -> x.offset() == 0).findFirst().get()); - firstSucceededRecordRemoved.remove(firstSucceededRecordRemoved.stream().filter(x -> x.offset() == 69).findFirst().get()); - firstSucceededRecordRemoved.remove(firstSucceededRecordRemoved.stream().filter(x -> x.offset() == 25_000).findFirst().get()); - - // - ktu.send(consumerSpy, records); - - // - ParallelConsumerOptions options = parallelConsumer.getWm().getOptions(); - HashMap>> recordsMap = new HashMap<>(); - TopicPartition tp = new TopicPartition(INPUT_TOPIC, 0); - recordsMap.put(tp, records); - ConsumerRecords crs = new ConsumerRecords<>(recordsMap); - - // write offsets - Map completedEligibleOffsetsAndRemove; - { - WorkManager wmm = new WorkManager<>(options, consumerSpy); - wmm.onPartitionsAssigned(UniSets.of(new TopicPartition(INPUT_TOPIC, 0))); - wmm.registerWork(crs); - - List> work = wmm.maybeGetWork(); - assertThat(work).hasSameSizeAs(records); - - KafkaTestUtils.completeWork(wmm, work, 0); - - KafkaTestUtils.completeWork(wmm, work, 69); - - KafkaTestUtils.completeWork(wmm, work, 25_000); - - completedEligibleOffsetsAndRemove = wmm.findCompletedEligibleOffsetsAndRemove(); - - // check for graceful fall back to the smallest available encoder - OffsetMapCodecManager om = new OffsetMapCodecManager<>(consumerSpy); - Set collect = firstSucceededRecordRemoved.stream().map(x -> x.offset()).collect(Collectors.toSet()); - OffsetMapCodecManager.forcedCodec = Optional.empty(); // turn off forced - var state = new PartitionState(tp, new OffsetMapCodecManager.HighestOffsetAndIncompletes(25_000L, collect)); - String bestPayload = om.makeOffsetMetadataPayload(1, state); - assertThat(bestPayload).isNotEmpty(); - } - consumerSpy.commitSync(completedEligibleOffsetsAndRemove); - - // read offsets - { - WorkManager newWm = new WorkManager<>(options, consumerSpy); - newWm.onPartitionsAssigned(UniSets.of(tp)); - newWm.registerWork(crs); -// ThreadUtils.sleepQuietly(1); - List> workRetrieved = newWm.maybeGetWork(); - switch (encoding) { - case BitSet, BitSetCompressed, // BitSetV1 both get a short overflow due to the length being too long - BitSetV2, // BitSetv2 uncompressed is too large to fit in metadata payload - RunLength, RunLengthCompressed // RunLength V1 max runlength is Short.MAX_VALUE - -> { - assertThatThrownBy(() -> - assertThat(workRetrieved).extracting(WorkContainer::getCr) - .containsExactlyElementsOf(firstSucceededRecordRemoved)) - .hasMessageContaining("but some elements were not") - .hasMessageContaining("offset = 25000"); - } - default -> { - assertThat(workRetrieved).extracting(WorkContainer::getCr) - .containsExactlyElementsOf(firstSucceededRecordRemoved); - } - } - } - - OffsetSimultaneousEncoder.compressionForced = false; - } - - /** - * This version of non sequential test just test the encoder directly, and is only half the story, as at the - * encoding stage they don't know which offsets have never been seen, and assume simply working with continuous - * ranges. - *

- * See more info in the class javadoc of {@link BitsetEncoder}. - * - * @see BitsetEncoder - * @see #ensureEncodingGracefullyWorksWhenOffsetsAreVeryLargeAndNotSequential - */ - @SneakyThrows - @Test - @ResourceLock(value = OffsetSimultaneousEncoder.COMPRESSION_FORCED_RESOURCE_LOCK, mode = READ_WRITE) - void ensureEncodingGracefullyWorksWhenOffsetsArentSequentialTwo() { - long nextExpectedOffset = 101; - long lowWaterMark = 0; - var incompletes = new HashSet<>(UniSets.of(1L, 4L, 5L, 100L)); - - OffsetSimultaneousEncoder encoder = new OffsetSimultaneousEncoder(lowWaterMark, nextExpectedOffset, incompletes); - OffsetSimultaneousEncoder.compressionForced = true; - - // - encoder.invoke(); - Map encodingMap = encoder.getEncodingMap(); - - // - byte[] smallestBytes = encoder.packSmallest(); - EncodedOffsetPair unwrap = EncodedOffsetPair.unwrap(smallestBytes); - OffsetMapCodecManager.HighestOffsetAndIncompletes decodedIncompletes = unwrap.getDecodedIncompletes(lowWaterMark); - assertThat(decodedIncompletes.getIncompleteOffsets()).containsExactlyInAnyOrderElementsOf(incompletes); - - if (nextExpectedOffset - lowWaterMark > BitsetEncoder.MAX_LENGTH_ENCODABLE) - assertThat(encodingMap.keySet()).as("Gracefully ignores that Bitset can't be supported").doesNotContain(OffsetEncoding.BitSet); - else - assertThat(encodingMap.keySet()).contains(OffsetEncoding.BitSet); - - // - for (OffsetEncoding encodingToUse : OffsetEncoding.values()) { - log.info("Testing {}", encodingToUse); - byte[] bitsetBytes = encodingMap.get(encodingToUse); - if (bitsetBytes != null) { - EncodedOffsetPair bitsetUnwrap = EncodedOffsetPair.unwrap(encoder.packEncoding(new EncodedOffsetPair(encodingToUse, ByteBuffer.wrap(bitsetBytes)))); - OffsetMapCodecManager.HighestOffsetAndIncompletes decodedBitsets = bitsetUnwrap.getDecodedIncompletes(lowWaterMark); - assertThat(decodedBitsets.getIncompleteOffsets()) - .as(encodingToUse.toString()) - .containsExactlyInAnyOrderElementsOf(incompletes); - } else { - log.info("Encoding not performed: " + encodingToUse); - } - } - - OffsetSimultaneousEncoder.compressionForced = false; - } - -} +package io.confluent.parallelconsumer.offsets; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ + +import io.confluent.csid.utils.KafkaTestUtils; +import io.confluent.parallelconsumer.ParallelConsumerOptions; +import io.confluent.parallelconsumer.ParallelEoSStreamProcessorTestBase; +import io.confluent.parallelconsumer.state.PartitionState; +import io.confluent.parallelconsumer.state.WorkContainer; +import io.confluent.parallelconsumer.state.WorkManager; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.common.TopicPartition; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.ResourceLock; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; +import pl.tlinkowski.unij.api.UniSets; + +import java.nio.ByteBuffer; +import java.util.*; +import java.util.stream.Collectors; + +import static io.confluent.parallelconsumer.offsets.OffsetEncoding.ByteArray; +import static io.confluent.parallelconsumer.offsets.OffsetEncoding.ByteArrayCompressed; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.hamcrest.Matchers.in; +import static org.hamcrest.Matchers.not; +import static org.junit.Assume.assumeThat; +import static org.junit.jupiter.api.parallel.ResourceAccessMode.READ; +import static org.junit.jupiter.api.parallel.ResourceAccessMode.READ_WRITE; +import static pl.tlinkowski.unij.api.UniLists.of; + +@Slf4j +public class OffsetEncodingTests extends ParallelEoSStreamProcessorTestBase { + + @Test + void runLengthDeserialise() { + var sb = ByteBuffer.allocate(3); + sb.put((byte) 0); // magic byte place holder, can ignore + sb.putShort((short) 1); + byte[] array = new byte[2]; + sb.rewind(); + sb.get(array); + ByteBuffer wrap = ByteBuffer.wrap(array); + byte b = wrap.get(); // simulate reading magic byte + ByteBuffer slice = wrap.slice(); + List integers = OffsetRunLength.runLengthDeserialise(slice); + assertThat(integers).isEmpty(); + } + + /** + * Triggers Short shortfall in BitSet encoder and tests encodable range of RunLength encoding - system should + * gracefully drop runlength if it has Short overflows (too hard to measure every runlength of incoming records + * before accepting?) + *

+ * https://github.com/confluentinc/parallel-consumer/issues/37 Support BitSet encoding lengths longer than + * Short.MAX_VALUE #37 + *

+ * https://github.com/confluentinc/parallel-consumer/issues/35 RuntimeException when running with very high options + * in 0.2.0.0 (Bitset too long to encode) #35 + *

+ */ + @SneakyThrows + @ParameterizedTest + @ValueSource(longs = { + 10_000L, + 100_000L, + 100_000_0L, +// 100_000_000L, // very~ slow + }) + @ResourceLock(value = OffsetSimultaneousEncoder.COMPRESSION_FORCED_RESOURCE_LOCK, mode = READ_WRITE) + void largeIncompleteOffsetValues(long nextExpectedOffset) { + var incompletes = new HashSet(); + long lowWaterMark = 123L; + incompletes.addAll(UniSets.of(lowWaterMark, 2345L, 8765L)); + + OffsetSimultaneousEncoder encoder = new OffsetSimultaneousEncoder(lowWaterMark, nextExpectedOffset, incompletes); + OffsetSimultaneousEncoder.compressionForced = true; + + // + encoder.invoke(); + Map encodingMap = encoder.getEncodingMap(); + + // + byte[] smallestBytes = encoder.packSmallest(); + EncodedOffsetPair unwrap = EncodedOffsetPair.unwrap(smallestBytes); + OffsetMapCodecManager.HighestOffsetAndIncompletes decodedIncompletes = unwrap.getDecodedIncompletes(lowWaterMark); + assertThat(decodedIncompletes.getIncompleteOffsets()).containsExactlyInAnyOrderElementsOf(incompletes); + + // + for (OffsetEncoding encodingToUse : OffsetEncoding.values()) { + log.info("Testing {}", encodingToUse); + byte[] bitsetBytes = encodingMap.get(encodingToUse); + if (bitsetBytes != null) { + EncodedOffsetPair bitsetUnwrap = EncodedOffsetPair.unwrap(encoder.packEncoding(new EncodedOffsetPair(encodingToUse, ByteBuffer.wrap(bitsetBytes)))); + OffsetMapCodecManager.HighestOffsetAndIncompletes decodedBitsets = bitsetUnwrap.getDecodedIncompletes(lowWaterMark); + assertThat(decodedBitsets.getIncompleteOffsets()) + .as(encodingToUse.toString()) + .containsExactlyInAnyOrderElementsOf(incompletes); + } else { + log.info("Encoding not performed: " + encodingToUse); + } + } + + OffsetSimultaneousEncoder.compressionForced = false; + } + + /** + * There's no guarantee that offsets are always sequential. The most obvious case is with a compacted topic - there + * will always be offsets missing. + * + * @see #ensureEncodingGracefullyWorksWhenOffsetsArentSequentialTwo + */ + @SneakyThrows + @ParameterizedTest + @EnumSource(OffsetEncoding.class) + // needed due to static accessors in parallel tests + @ResourceLock(value = OffsetMapCodecManager.METADATA_DATA_SIZE_RESOURCE_LOCK, mode = READ) + // depends on OffsetMapCodecManager#DefaultMaxMetadataSize + @ResourceLock(value = OffsetSimultaneousEncoder.COMPRESSION_FORCED_RESOURCE_LOCK, mode = READ_WRITE) + void ensureEncodingGracefullyWorksWhenOffsetsAreVeryLargeAndNotSequential(OffsetEncoding encoding) { + assumeThat("Codec skipped, not applicable", encoding, + not(in(of(ByteArray, ByteArrayCompressed)))); // byte array not currently used + + // todo don't use static public accessors to change things - makes parallel testing harder and is smelly + OffsetMapCodecManager.forcedCodec = Optional.of(encoding); + OffsetSimultaneousEncoder.compressionForced = true; + + var records = new ArrayList>(); + records.add(new ConsumerRecord<>(INPUT_TOPIC, 0, 0, "akey", "avalue")); // will complete + records.add(new ConsumerRecord<>(INPUT_TOPIC, 0, 1, "akey", "avalue")); + records.add(new ConsumerRecord<>(INPUT_TOPIC, 0, 4, "akey", "avalue")); + records.add(new ConsumerRecord<>(INPUT_TOPIC, 0, 5, "akey", "avalue")); + records.add(new ConsumerRecord<>(INPUT_TOPIC, 0, 69, "akey", "avalue")); // will complete + records.add(new ConsumerRecord<>(INPUT_TOPIC, 0, 100, "akey", "avalue")); + records.add(new ConsumerRecord<>(INPUT_TOPIC, 0, 1_000, "akey", "avalue")); + records.add(new ConsumerRecord<>(INPUT_TOPIC, 0, 20_000, "akey", "avalue")); // near upper limit of Short.MAX_VALUE + records.add(new ConsumerRecord<>(INPUT_TOPIC, 0, 25_000, "akey", "avalue")); // will complete, near upper limit of Short.MAX_VALUE + records.add(new ConsumerRecord<>(INPUT_TOPIC, 0, 30_000, "akey", "avalue")); // near upper limit of Short.MAX_VALUE + + // Extremely large tests for v2 encoders + records.add(new ConsumerRecord<>(INPUT_TOPIC, 0, 40_000, "akey", "avalue")); // higher than Short.MAX_VALUE + int avoidOffByOne = 2; + records.add(new ConsumerRecord<>(INPUT_TOPIC, 0, 40_000 + Short.MAX_VALUE + avoidOffByOne, "akey", "avalue")); // runlength higher than Short.MAX_VALUE + + var firstSucceededRecordRemoved = new ArrayList<>(records); + firstSucceededRecordRemoved.remove(firstSucceededRecordRemoved.stream().filter(x -> x.offset() == 0).findFirst().get()); + firstSucceededRecordRemoved.remove(firstSucceededRecordRemoved.stream().filter(x -> x.offset() == 69).findFirst().get()); + firstSucceededRecordRemoved.remove(firstSucceededRecordRemoved.stream().filter(x -> x.offset() == 25_000).findFirst().get()); + + // + ktu.send(consumerSpy, records); + + // + ParallelConsumerOptions options = parallelConsumer.getWm().getOptions(); + HashMap>> recordsMap = new HashMap<>(); + TopicPartition tp = new TopicPartition(INPUT_TOPIC, 0); + recordsMap.put(tp, records); + ConsumerRecords crs = new ConsumerRecords<>(recordsMap); + + // write offsets + Map completedEligibleOffsetsAndRemove; + { + WorkManager wmm = new WorkManager<>(options, consumerSpy); + wmm.onPartitionsAssigned(UniSets.of(new TopicPartition(INPUT_TOPIC, 0))); + wmm.registerWork(crs); + + List> work = wmm.maybeGetWork(); + assertThat(work).hasSameSizeAs(records); + + KafkaTestUtils.completeWork(wmm, work, 0); + + KafkaTestUtils.completeWork(wmm, work, 69); + + KafkaTestUtils.completeWork(wmm, work, 25_000); + + completedEligibleOffsetsAndRemove = wmm.findCompletedEligibleOffsetsAndRemove(); + + // check for graceful fall back to the smallest available encoder + OffsetMapCodecManager om = new OffsetMapCodecManager<>(consumerSpy); + Set collect = firstSucceededRecordRemoved.stream().map(x -> x.offset()).collect(Collectors.toSet()); + OffsetMapCodecManager.forcedCodec = Optional.empty(); // turn off forced + var state = new PartitionState(tp, new OffsetMapCodecManager.HighestOffsetAndIncompletes(25_000L, collect)); + String bestPayload = om.makeOffsetMetadataPayload(1, state); + assertThat(bestPayload).isNotEmpty(); + } + consumerSpy.commitSync(completedEligibleOffsetsAndRemove); + + // read offsets + { + WorkManager newWm = new WorkManager<>(options, consumerSpy); + newWm.onPartitionsAssigned(UniSets.of(tp)); + newWm.registerWork(crs); +// ThreadUtils.sleepQuietly(1); + List> workRetrieved = newWm.maybeGetWork(); + switch (encoding) { + case BitSet, BitSetCompressed, // BitSetV1 both get a short overflow due to the length being too long + BitSetV2, // BitSetv2 uncompressed is too large to fit in metadata payload + RunLength, RunLengthCompressed // RunLength V1 max runlength is Short.MAX_VALUE + -> { + assertThatThrownBy(() -> + assertThat(workRetrieved).extracting(WorkContainer::getCr) + .containsExactlyElementsOf(firstSucceededRecordRemoved)) + .hasMessageContaining("but some elements were not") + .hasMessageContaining("offset = 25000"); + } + default -> { + assertThat(workRetrieved).extracting(WorkContainer::getCr) + .containsExactlyElementsOf(firstSucceededRecordRemoved); + } + } + } + + OffsetSimultaneousEncoder.compressionForced = false; + } + + /** + * This version of non sequential test just test the encoder directly, and is only half the story, as at the + * encoding stage they don't know which offsets have never been seen, and assume simply working with continuous + * ranges. + *

+ * See more info in the class javadoc of {@link BitsetEncoder}. + * + * @see BitsetEncoder + * @see #ensureEncodingGracefullyWorksWhenOffsetsAreVeryLargeAndNotSequential + */ + @SneakyThrows + @Test + @ResourceLock(value = OffsetSimultaneousEncoder.COMPRESSION_FORCED_RESOURCE_LOCK, mode = READ_WRITE) + void ensureEncodingGracefullyWorksWhenOffsetsArentSequentialTwo() { + long nextExpectedOffset = 101; + long lowWaterMark = 0; + var incompletes = new HashSet<>(UniSets.of(1L, 4L, 5L, 100L)); + + OffsetSimultaneousEncoder encoder = new OffsetSimultaneousEncoder(lowWaterMark, nextExpectedOffset, incompletes); + OffsetSimultaneousEncoder.compressionForced = true; + + // + encoder.invoke(); + Map encodingMap = encoder.getEncodingMap(); + + // + byte[] smallestBytes = encoder.packSmallest(); + EncodedOffsetPair unwrap = EncodedOffsetPair.unwrap(smallestBytes); + OffsetMapCodecManager.HighestOffsetAndIncompletes decodedIncompletes = unwrap.getDecodedIncompletes(lowWaterMark); + assertThat(decodedIncompletes.getIncompleteOffsets()).containsExactlyInAnyOrderElementsOf(incompletes); + + if (nextExpectedOffset - lowWaterMark > BitsetEncoder.MAX_LENGTH_ENCODABLE) + assertThat(encodingMap.keySet()).as("Gracefully ignores that Bitset can't be supported").doesNotContain(OffsetEncoding.BitSet); + else + assertThat(encodingMap.keySet()).contains(OffsetEncoding.BitSet); + + // + for (OffsetEncoding encodingToUse : OffsetEncoding.values()) { + log.info("Testing {}", encodingToUse); + byte[] bitsetBytes = encodingMap.get(encodingToUse); + if (bitsetBytes != null) { + EncodedOffsetPair bitsetUnwrap = EncodedOffsetPair.unwrap(encoder.packEncoding(new EncodedOffsetPair(encodingToUse, ByteBuffer.wrap(bitsetBytes)))); + OffsetMapCodecManager.HighestOffsetAndIncompletes decodedBitsets = bitsetUnwrap.getDecodedIncompletes(lowWaterMark); + assertThat(decodedBitsets.getIncompleteOffsets()) + .as(encodingToUse.toString()) + .containsExactlyInAnyOrderElementsOf(incompletes); + } else { + log.info("Encoding not performed: " + encodingToUse); + } + } + + OffsetSimultaneousEncoder.compressionForced = false; + } + +} diff --git a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/offsets/RunLengthEncoderTest.java b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/offsets/RunLengthEncoderTest.java index c4d09cb2b..8a941a9cc 100644 --- a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/offsets/RunLengthEncoderTest.java +++ b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/offsets/RunLengthEncoderTest.java @@ -1,144 +1,144 @@ -package io.confluent.parallelconsumer.offsets; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ - -import io.confluent.parallelconsumer.offsets.OffsetMapCodecManager.HighestOffsetAndIncompletes; -import lombok.SneakyThrows; -import org.junit.jupiter.api.Test; -import pl.tlinkowski.unij.api.UniLists; -import pl.tlinkowski.unij.api.UniSets; - -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import static io.confluent.parallelconsumer.offsets.OffsetEncoding.Version.v2; -import static org.assertj.core.api.Assertions.assertThat; - -class RunLengthEncoderTest { - - /** - * Check that run length supports gaps in the source partition - i.e. compacted topics where offsets aren't strictly - * sequential - */ - @SneakyThrows - @Test - void noGaps() { - Set incompletes = UniSets.of(0, 4, 6, 7, 8, 10).stream().map(x -> (long) x).collect(Collectors.toSet()); // lol - DRY! - Set completes = UniSets.of(1, 2, 3, 5, 9).stream().map(x -> (long) x).collect(Collectors.toSet()); // lol - DRY! - List runs = UniLists.of(1, 3, 1, 1, 3, 1, 1); - OffsetSimultaneousEncoder offsetSimultaneousEncoder = new OffsetSimultaneousEncoder(-1, 0L, incompletes); - - { - RunLengthEncoder rl = new RunLengthEncoder(offsetSimultaneousEncoder, v2); - - rl.encodeIncompleteOffset(0); // 1 - rl.encodeCompletedOffset(1); // 3 - rl.encodeCompletedOffset(2); - rl.encodeCompletedOffset(3); - rl.encodeIncompleteOffset(4); // 1 - rl.encodeCompletedOffset(5); // 1 - rl.encodeIncompleteOffset(6); // 3 - rl.encodeIncompleteOffset(7); - rl.encodeIncompleteOffset(8); - rl.encodeCompletedOffset(9); // 1 - rl.encodeIncompleteOffset(10); // 1 - - rl.addTail(); - - // before serialisation - { - assertThat(rl.getRunLengthEncodingIntegers()).containsExactlyElementsOf(runs); - - List calculatedCompletedOffsets = rl.calculateSucceededActualOffsets(0); - - assertThat(calculatedCompletedOffsets).containsExactlyElementsOf(completes); - } - } - } - - - /** - * Check that run length supports gaps in the source partition - i.e. compacted topics where offsets aren't strictly - * sequential - */ - @SneakyThrows - @Test - void noGapsSerialisation() { - Set incompletes = UniSets.of(0, 4, 6, 7, 8, 10).stream().map(x -> (long) x).collect(Collectors.toSet()); // lol - DRY! - Set completes = UniSets.of(1, 2, 3, 5, 9).stream().map(x -> (long) x).collect(Collectors.toSet()); // lol - DRY! - List runs = UniLists.of(1, 3, 1, 1, 3, 1, 1); - OffsetSimultaneousEncoder offsetSimultaneousEncoder = new OffsetSimultaneousEncoder(-1, 0L, incompletes); - - { - RunLengthEncoder rl = new RunLengthEncoder(offsetSimultaneousEncoder, v2); - - rl.encodeIncompleteOffset(0); // 1 - rl.encodeCompletedOffset(1); // 3 - rl.encodeCompletedOffset(2); - rl.encodeCompletedOffset(3); - rl.encodeIncompleteOffset(4); // 1 - rl.encodeCompletedOffset(5); // 1 - rl.encodeIncompleteOffset(6); // 3 - rl.encodeIncompleteOffset(7); - rl.encodeIncompleteOffset(8); - rl.encodeCompletedOffset(9); // 1 - rl.encodeIncompleteOffset(10); // 1 - - // after serialisation - { - byte[] raw = rl.serialise(); - - byte[] wrapped = offsetSimultaneousEncoder.packEncoding(new EncodedOffsetPair(OffsetEncoding.RunLengthV2, ByteBuffer.wrap(raw))); - - HighestOffsetAndIncompletes result = OffsetMapCodecManager.decodeCompressedOffsets(0, wrapped); - - assertThat(result.getHighestSeenOffset()).isEqualTo(10); - - assertThat(result.getIncompleteOffsets()).containsExactlyElementsOf(incompletes); - } - } - } - - /** - * Check that run length supports gaps in the source partition - i.e. compacted topics where offsets aren't strictly - * sequential. - */ - @SneakyThrows - @Test - void gapsInOffsetsWork() { - Set incompletes = UniSets.of(0, 6, 10).stream().map(x -> (long) x).collect(Collectors.toSet()); // lol - DRY! - - // NB: gaps between completed offsets get encoded as succeeded offsets. This doesn't matter because they don't exist and we'll neve see them. - Set completes = UniSets.of(1, 2, 3, 4, 5, 9).stream().map(x -> (long) x).collect(Collectors.toSet()); // lol - DRY! - List runs = UniLists.of(1, 5, 3, 1, 1); - OffsetSimultaneousEncoder offsetSimultaneousEncoder = new OffsetSimultaneousEncoder(-1, 0L, incompletes); - - { - RunLengthEncoder rl = new RunLengthEncoder(offsetSimultaneousEncoder, v2); - - rl.encodeIncompleteOffset(0); - rl.encodeCompletedOffset(1); - // gap completes at 2 - rl.encodeCompletedOffset(3); - rl.encodeCompletedOffset(4); - rl.encodeCompletedOffset(5); - rl.encodeIncompleteOffset(6); - // gap incompletes at 7 - rl.encodeIncompleteOffset(8); - rl.encodeCompletedOffset(9); - rl.encodeIncompleteOffset(10); - - rl.addTail(); - - assertThat(rl.getRunLengthEncodingIntegers()).containsExactlyElementsOf(runs); - - List calculatedCompletedOffsets = rl.calculateSucceededActualOffsets(0); - - assertThat(calculatedCompletedOffsets).containsExactlyElementsOf(completes); - } - } -} +package io.confluent.parallelconsumer.offsets; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ + +import io.confluent.parallelconsumer.offsets.OffsetMapCodecManager.HighestOffsetAndIncompletes; +import lombok.SneakyThrows; +import org.junit.jupiter.api.Test; +import pl.tlinkowski.unij.api.UniLists; +import pl.tlinkowski.unij.api.UniSets; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static io.confluent.parallelconsumer.offsets.OffsetEncoding.Version.v2; +import static org.assertj.core.api.Assertions.assertThat; + +class RunLengthEncoderTest { + + /** + * Check that run length supports gaps in the source partition - i.e. compacted topics where offsets aren't strictly + * sequential + */ + @SneakyThrows + @Test + void noGaps() { + Set incompletes = UniSets.of(0, 4, 6, 7, 8, 10).stream().map(x -> (long) x).collect(Collectors.toSet()); // lol - DRY! + Set completes = UniSets.of(1, 2, 3, 5, 9).stream().map(x -> (long) x).collect(Collectors.toSet()); // lol - DRY! + List runs = UniLists.of(1, 3, 1, 1, 3, 1, 1); + OffsetSimultaneousEncoder offsetSimultaneousEncoder = new OffsetSimultaneousEncoder(-1, 0L, incompletes); + + { + RunLengthEncoder rl = new RunLengthEncoder(offsetSimultaneousEncoder, v2); + + rl.encodeIncompleteOffset(0); // 1 + rl.encodeCompletedOffset(1); // 3 + rl.encodeCompletedOffset(2); + rl.encodeCompletedOffset(3); + rl.encodeIncompleteOffset(4); // 1 + rl.encodeCompletedOffset(5); // 1 + rl.encodeIncompleteOffset(6); // 3 + rl.encodeIncompleteOffset(7); + rl.encodeIncompleteOffset(8); + rl.encodeCompletedOffset(9); // 1 + rl.encodeIncompleteOffset(10); // 1 + + rl.addTail(); + + // before serialisation + { + assertThat(rl.getRunLengthEncodingIntegers()).containsExactlyElementsOf(runs); + + List calculatedCompletedOffsets = rl.calculateSucceededActualOffsets(0); + + assertThat(calculatedCompletedOffsets).containsExactlyElementsOf(completes); + } + } + } + + + /** + * Check that run length supports gaps in the source partition - i.e. compacted topics where offsets aren't strictly + * sequential + */ + @SneakyThrows + @Test + void noGapsSerialisation() { + Set incompletes = UniSets.of(0, 4, 6, 7, 8, 10).stream().map(x -> (long) x).collect(Collectors.toSet()); // lol - DRY! + Set completes = UniSets.of(1, 2, 3, 5, 9).stream().map(x -> (long) x).collect(Collectors.toSet()); // lol - DRY! + List runs = UniLists.of(1, 3, 1, 1, 3, 1, 1); + OffsetSimultaneousEncoder offsetSimultaneousEncoder = new OffsetSimultaneousEncoder(-1, 0L, incompletes); + + { + RunLengthEncoder rl = new RunLengthEncoder(offsetSimultaneousEncoder, v2); + + rl.encodeIncompleteOffset(0); // 1 + rl.encodeCompletedOffset(1); // 3 + rl.encodeCompletedOffset(2); + rl.encodeCompletedOffset(3); + rl.encodeIncompleteOffset(4); // 1 + rl.encodeCompletedOffset(5); // 1 + rl.encodeIncompleteOffset(6); // 3 + rl.encodeIncompleteOffset(7); + rl.encodeIncompleteOffset(8); + rl.encodeCompletedOffset(9); // 1 + rl.encodeIncompleteOffset(10); // 1 + + // after serialisation + { + byte[] raw = rl.serialise(); + + byte[] wrapped = offsetSimultaneousEncoder.packEncoding(new EncodedOffsetPair(OffsetEncoding.RunLengthV2, ByteBuffer.wrap(raw))); + + HighestOffsetAndIncompletes result = OffsetMapCodecManager.decodeCompressedOffsets(0, wrapped); + + assertThat(result.getHighestSeenOffset()).isEqualTo(10); + + assertThat(result.getIncompleteOffsets()).containsExactlyElementsOf(incompletes); + } + } + } + + /** + * Check that run length supports gaps in the source partition - i.e. compacted topics where offsets aren't strictly + * sequential. + */ + @SneakyThrows + @Test + void gapsInOffsetsWork() { + Set incompletes = UniSets.of(0, 6, 10).stream().map(x -> (long) x).collect(Collectors.toSet()); // lol - DRY! + + // NB: gaps between completed offsets get encoded as succeeded offsets. This doesn't matter because they don't exist and we'll neve see them. + Set completes = UniSets.of(1, 2, 3, 4, 5, 9).stream().map(x -> (long) x).collect(Collectors.toSet()); // lol - DRY! + List runs = UniLists.of(1, 5, 3, 1, 1); + OffsetSimultaneousEncoder offsetSimultaneousEncoder = new OffsetSimultaneousEncoder(-1, 0L, incompletes); + + { + RunLengthEncoder rl = new RunLengthEncoder(offsetSimultaneousEncoder, v2); + + rl.encodeIncompleteOffset(0); + rl.encodeCompletedOffset(1); + // gap completes at 2 + rl.encodeCompletedOffset(3); + rl.encodeCompletedOffset(4); + rl.encodeCompletedOffset(5); + rl.encodeIncompleteOffset(6); + // gap incompletes at 7 + rl.encodeIncompleteOffset(8); + rl.encodeCompletedOffset(9); + rl.encodeIncompleteOffset(10); + + rl.addTail(); + + assertThat(rl.getRunLengthEncodingIntegers()).containsExactlyElementsOf(runs); + + List calculatedCompletedOffsets = rl.calculateSucceededActualOffsets(0); + + assertThat(calculatedCompletedOffsets).containsExactlyElementsOf(completes); + } + } +} diff --git a/parallel-consumer-vertx/src/test/java/io/confluent/parallelconsumer/vertx/WireMockUtils.java b/parallel-consumer-vertx/src/test/java/io/confluent/parallelconsumer/vertx/WireMockUtils.java index 10cb4aff3..35e2f44e5 100644 --- a/parallel-consumer-vertx/src/test/java/io/confluent/parallelconsumer/vertx/WireMockUtils.java +++ b/parallel-consumer-vertx/src/test/java/io/confluent/parallelconsumer/vertx/WireMockUtils.java @@ -1,32 +1,32 @@ -package io.confluent.parallelconsumer.vertx; - -/*- - * Copyright (C) 2020-2021 Confluent, Inc. - */ - -import com.github.tomakehurst.wiremock.WireMockServer; -import com.github.tomakehurst.wiremock.client.MappingBuilder; -import com.github.tomakehurst.wiremock.core.WireMockConfiguration; - -import static com.github.tomakehurst.wiremock.client.WireMock.*; -import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; - -public class WireMockUtils { - - protected static final String stubResponse = "Good times."; - - public WireMockServer setupWireMock() { - WireMockServer stubServer; - WireMockConfiguration options = wireMockConfig().dynamicPort(); - stubServer = new WireMockServer(options); - MappingBuilder mappingBuilder = get(urlPathEqualTo("/")) - .willReturn(aResponse() - .withBody(stubResponse)); - stubServer.stubFor(mappingBuilder); - stubServer.stubFor(get(urlPathEqualTo("/api")). - willReturn(aResponse().withBody(stubResponse))); - stubServer.start(); - return stubServer; - } - -} +package io.confluent.parallelconsumer.vertx; + +/*- + * Copyright (C) 2020-2021 Confluent, Inc. + */ + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.client.MappingBuilder; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; + +public class WireMockUtils { + + protected static final String stubResponse = "Good times."; + + public WireMockServer setupWireMock() { + WireMockServer stubServer; + WireMockConfiguration options = wireMockConfig().dynamicPort(); + stubServer = new WireMockServer(options); + MappingBuilder mappingBuilder = get(urlPathEqualTo("/")) + .willReturn(aResponse() + .withBody(stubResponse)); + stubServer.stubFor(mappingBuilder); + stubServer.stubFor(get(urlPathEqualTo("/api")). + willReturn(aResponse().withBody(stubResponse))); + stubServer.start(); + return stubServer; + } + +}