diff --git a/java/e2e-v4/pom.xml b/java/e2e-v4/pom.xml index 68b61b3..1d56a06 100644 --- a/java/e2e-v4/pom.xml +++ b/java/e2e-v4/pom.xml @@ -88,6 +88,11 @@ logback-core 1.3.0-beta0 + + org.awaitility + awaitility + 4.0.3 + @@ -144,7 +149,7 @@ maven-surefire-plugin 3.0.0-M5 - -Xmx1024m -XX:MaxPermSize=256m + -Xmx1024m 1 false diff --git a/java/e2e-v4/src/main/java/org/apache/rocketmq/client/rmq/RMQNormalConsumer.java b/java/e2e-v4/src/main/java/org/apache/rocketmq/client/rmq/RMQNormalConsumer.java index 4d464dd..d242bc6 100644 --- a/java/e2e-v4/src/main/java/org/apache/rocketmq/client/rmq/RMQNormalConsumer.java +++ b/java/e2e-v4/src/main/java/org/apache/rocketmq/client/rmq/RMQNormalConsumer.java @@ -17,45 +17,117 @@ package org.apache.rocketmq.client.rmq; +import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; +import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; import org.apache.rocketmq.client.consumer.MessageSelector; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.AbstractMQConsumer; import org.apache.rocketmq.listener.AbstractListener; +import org.apache.rocketmq.listener.rmq.concurrent.RMQIdempotentListener; import org.apache.rocketmq.listener.rmq.concurrent.RMQNormalListener; import org.apache.rocketmq.listener.rmq.concurrent.RMQOrderListener; import org.apache.rocketmq.utils.TestUtils; +import org.junit.jupiter.api.Assertions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class RMQNormalConsumer extends AbstractMQConsumer { private static Logger logger = LoggerFactory.getLogger(RMQNormalConsumer.class); - private DefaultMQPushConsumer consumer; + private DefaultMQPushConsumer pushConsumer; + private DefaultLitePullConsumer litePullConsumer; + private DefaultMQPullConsumer pullConsumer; private AbstractListener listener = null; public RMQNormalConsumer(DefaultMQPushConsumer consumer) { - this.consumer = consumer; + this.pushConsumer = consumer; + } + + public RMQNormalConsumer(DefaultLitePullConsumer consumer) { + this.litePullConsumer = consumer; + } + + public RMQNormalConsumer(DefaultMQPullConsumer consumer) { + this.pullConsumer = consumer; } public void subscribeAndStart(String topic, String tag, RMQNormalListener listener) { + Assertions.assertNotNull(pushConsumer); this.listener = listener; try { - consumer.subscribe(topic, tag); - consumer.setMessageListener(listener); - consumer.start(); + pushConsumer.subscribe(topic, tag); + pushConsumer.setMessageListener(listener); + pushConsumer.start(); } catch (MQClientException e) { logger.info("Start DefaultMQPushConsumer failed, {}", e.getMessage()); } logger.info("DefaultMQPushConsumer started - topic: {}, tag: {}", topic, tag); } + public void subscribeAndStart(String topic, String tag, RMQIdempotentListener listener) { + Assertions.assertNotNull(pushConsumer); + this.listener = listener; + try { + pushConsumer.subscribe(topic, tag); + pushConsumer.setMessageListener(listener); + pushConsumer.start(); + } catch (MQClientException e) { + logger.info("Start DefaultMQPushConsumer failed, {}", e.getMessage()); + } + logger.info("DefaultMQPushConsumer started - topic: {}, tag: {}", topic, tag); + } + + public void subscribeAndStartLitePull(String topic, String tag) { + Assertions.assertNotNull(litePullConsumer); + try { + litePullConsumer.subscribe(topic, tag); + litePullConsumer.start(); + } catch (MQClientException e) { + logger.info("Start DefaultMQLitePullConsumer failed, {}", e.getMessage()); + } + logger.info("DefaultMQLitePullConsumer started - topic: {}, tag: {}", topic, tag); + } + + public void startLitePullAssignMode() { + Assertions.assertNotNull(litePullConsumer); + try { + litePullConsumer.setAutoCommit(false); + litePullConsumer.start(); + } catch (MQClientException e) { + logger.info("Start DefaultMQLitePullConsumer failed, {}", e.getMessage()); + } + logger.info("DefaultMQLitePullConsumer Assign Mode started"); + } + + public void startDefaultPull() { + Assertions.assertNotNull(pullConsumer); + try { + pullConsumer.start(); + } catch (MQClientException e) { + logger.info("Start DefaultMQPullConsumer failed, {}", e.getMessage()); + } + logger.info("DefaultMQPullConsumer started"); + } + + public void subscribeAndStartLitePull(String topic, MessageSelector messageSelector) { + Assertions.assertNotNull(litePullConsumer); + try { + litePullConsumer.subscribe(topic, messageSelector); + litePullConsumer.start(); + } catch (MQClientException e) { + logger.info("Start DefaultMQPushConsumer failed, {}", e.getMessage()); + } + logger.info("DefaultMQLitePullConsumer started - topic: {}, sql: {}", topic, messageSelector.getExpression()); + } + public void subscribeAndStart(String topic, String tag, RMQOrderListener listener) { + Assertions.assertNotNull(pushConsumer); this.listener = listener; try { - consumer.subscribe(topic, tag); - consumer.setMessageListener(listener); - consumer.start(); + pushConsumer.subscribe(topic, tag); + pushConsumer.setMessageListener(listener); + pushConsumer.start(); } catch (MQClientException e) { logger.info("Start DefaultMQPushConsumer failed, {}", e.getMessage()); } @@ -63,32 +135,38 @@ public void subscribeAndStart(String topic, String tag, RMQOrderListener listene } public void subscribeAndStart(String topic, MessageSelector messageSelector, RMQNormalListener listener) { + Assertions.assertNotNull(pushConsumer); this.listener = listener; try { - consumer.subscribe(topic, messageSelector); - consumer.setMessageListener(listener); - consumer.start(); + pushConsumer.subscribe(topic, messageSelector); + pushConsumer.setMessageListener(listener); + pushConsumer.start(); } catch (MQClientException e) { logger.info("Start DefaultMQPushConsumer failed, {}", e.getMessage()); } - logger.info("DefaultMQPushConsumer started - topic: {}, messageSelector: {}", topic, messageSelector.getExpression()); + logger.info("DefaultMQPushConsumer started - topic: {}, messageSelector: {}", topic, + messageSelector.getExpression()); TestUtils.waitForSeconds(5); } @Override public void shutdown() { - if (consumer != null) { - consumer.shutdown(); + if (pushConsumer != null) { + pushConsumer.shutdown(); logger.info("DefaultMQPushConsumer shutdown !!!"); } + if (litePullConsumer != null) { + litePullConsumer.shutdown(); + logger.info("DefaultLitePullConsumer shutdown !!!"); + } } - public DefaultMQPushConsumer getConsumer() { - return consumer; + public DefaultMQPushConsumer getPushConsumer() { + return pushConsumer; } - public void setConsumer(DefaultMQPushConsumer consumer) { - this.consumer = consumer; + public void setPushConsumer(DefaultMQPushConsumer consumer) { + this.pushConsumer = consumer; } public AbstractListener getListener() { @@ -98,4 +176,20 @@ public AbstractListener getListener() { public void setListener(AbstractListener listener) { this.listener = listener; } + + public DefaultLitePullConsumer getLitePullConsumer() { + return litePullConsumer; + } + + public void setLitePullConsumer(DefaultLitePullConsumer consumer) { + this.litePullConsumer = consumer; + } + + public DefaultMQPullConsumer getPullConsumer() { + return pullConsumer; + } + + public void setPullConsumer(DefaultMQPullConsumer consumer) { + this.pullConsumer = consumer; + } } diff --git a/java/e2e-v4/src/main/java/org/apache/rocketmq/client/rmq/RMQNormalProducer.java b/java/e2e-v4/src/main/java/org/apache/rocketmq/client/rmq/RMQNormalProducer.java index 60688d6..59ec091 100644 --- a/java/e2e-v4/src/main/java/org/apache/rocketmq/client/rmq/RMQNormalProducer.java +++ b/java/e2e-v4/src/main/java/org/apache/rocketmq/client/rmq/RMQNormalProducer.java @@ -107,6 +107,58 @@ public void sendWithQueue(List mqs, int messageNum) { logger.info("Producer send messages finished"); } + /** + * 向哪些Queue发送顺序消息 + * + * @param mqs Queue列表 + * @param messageNum 每个Queue发送的消息数量 + */ + public void sendWithQueue(List mqs, String tag, int messageNum) { + logger.info("Producer start to send messages"); + for (MessageQueue mq : mqs) { + for (int i = 0; i < messageNum; i++) { + Message message = MessageFactory.buildOneMessageWithTagAndBody(mq.getTopic(), tag, String.valueOf(i)); + try { + SendResult sendResult = producer.send(message, mq); + MessageExt messageExt = new MessageExt(); + messageExt.setMsgId(sendResult.getMsgId()); + messageExt.setBody(message.getBody()); + logger.info("{}, index: {}, tag: {}", sendResult, i, tag); + this.enqueueMessages.addData(messageExt); + } catch (Exception e) { + logger.error("DefaultMQProducer send message failed"); + } + } + } + logger.info("Producer send messages finished"); + } + + /** + * 向Queue发送顺序消息 + * + * @param mqs Queue列表 + * @param messageNum 每个Queue发送的消息数量 + */ + public void sendWithQueue(List mqs, int messageNum, String tag) { + logger.info("Producer start to send messages"); + for (MessageQueue mq : mqs) { + for (int i = 0; i < messageNum; i++) { + Message message = MessageFactory.buildOneMessageWithTagAndBody(mq.getTopic(), tag, String.valueOf(i)); + try { + SendResult sendResult = producer.send(message, mq); + MessageExt messageExt = new MessageExt(); + messageExt.setMsgId(sendResult.getMsgId()); + messageExt.setBody(message.getBody()); + logger.info("{}, index: {}, tag: {}", sendResult, i, tag); + this.enqueueMessages.addData(messageExt); + } catch (Exception e) { + logger.error("DefaultMQProducer send message failed"); + } + } + } + logger.info("Producer send messages finished"); + } + /** * 发送指定正常的properties的普通消息l * @@ -132,7 +184,7 @@ public void sendWithTagAndBody(String topic, String tag, String messageBody, int e.printStackTrace(); } logger.info(sendResult.toString()); - this.enqueueMessages.addData((MessageExt)message); + this.enqueueMessages.addData((MessageExt) message); } Assertions.assertEquals(messageNum, enqueueMessages.getAllData().size(), "消息没有全部发送成功！"); logger.info("Producer send messages finished"); @@ -169,24 +221,26 @@ public void sendAsync(String topic, RMQSendCallBack callBack, int messageNum) { } // - //public void sendAsyncWithTagAndBody(String topic, String tag, String messageBody, OnsSendCallBack callBack, int messageNum) { - // logger.info("Producer start to async send messages"); - // for (int i = 0; i < messageNum; i++) { - // Message message = MessageFactory.buildOneMessageWithTagAndBody(topic, tag, messageBody); - // producer.sendAsync(message, callBack); - // callBack.waitResponse(); - // if (callBack.isBSuccessResponse()) { - // this.enqueueMessages.addData(message); - // } - // if (callBack.isBFailResponse()) { - // this.enqueueFailedMessages.addData(message); - // } - // } - // logger.info("Producer async send messages finished"); - // if (enqueueFailedMessages.getAllData().size() > 0) { - // logger.warn("send failed messages: {}", enqueueFailedMessages.getAllData()); - // } - //} + // public void sendAsyncWithTagAndBody(String topic, String tag, String + // messageBody, OnsSendCallBack callBack, int messageNum) { + // logger.info("Producer start to async send messages"); + // for (int i = 0; i < messageNum; i++) { + // Message message = MessageFactory.buildOneMessageWithTagAndBody(topic, tag, + // messageBody); + // producer.sendAsync(message, callBack); + // callBack.waitResponse(); + // if (callBack.isBSuccessResponse()) { + // this.enqueueMessages.addData(message); + // } + // if (callBack.isBFailResponse()) { + // this.enqueueFailedMessages.addData(message); + // } + // } + // logger.info("Producer async send messages finished"); + // if (enqueueFailedMessages.getAllData().size() > 0) { + // logger.warn("send failed messages: {}", enqueueFailedMessages.getAllData()); + // } + // } // public void sendAsync(String topic, String tag, RMQSendCallBack callBack, int messageNum) { logger.info("Producer start to async send messages"); @@ -218,6 +272,34 @@ public void sendAsync(String topic, String tag, RMQSendCallBack callBack, int me } } + public void sendAsync(Message message, RMQSendCallBack callBack) { + logger.info("Producer start to async send messages"); + + MessageExt messageExt = null; + try { + producer.send(message, callBack); + callBack.waitResponse(); + if (callBack.isbSuccessResponse()) { + messageExt = new MessageExt(); + messageExt.setMsgId(callBack.getMessageId()); + this.enqueueMessages.addData(messageExt); + } + if (callBack.isbFailResponse()) { + this.enqueueFailedMessages.addData(messageExt); + } + } catch (MQClientException e) { + e.printStackTrace(); + } catch (RemotingException e) { + e.printStackTrace(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + logger.info("Producer async send messages finished"); + if (enqueueFailedMessages.getAllData().size() > 0) { + logger.warn("send failed messages: {}", enqueueFailedMessages.getAllData()); + } + } + public void sendOneWay(String topic, String tag, int messageNum) { logger.info("Producer start to OneWay send message"); for (int i = 0; i < messageNum; i++) { @@ -225,7 +307,7 @@ public void sendOneWay(String topic, String tag, int messageNum) { MessageExt messageExt = null; try { producer.sendOneway(message); - //this.enqueueMessages.addData(messageExt); + // this.enqueueMessages.addData(messageExt); } catch (MQClientException e) { e.printStackTrace(); } catch (RemotingException e) { @@ -251,8 +333,9 @@ public void sendOnewayWithTagAndBody(String topic, String tag, String messageBod } catch (InterruptedException e) { e.printStackTrace(); } - //logger.info("topic: {}, msgId: {}, index: {}, tag: {}, body:{}", topic, message.getMsgID(), i, tag, messageBody); - this.enqueueMessages.addData((MessageExt)message); + // logger.info("topic: {}, msgId: {}, index: {}, tag: {}, body:{}", topic, + // message.getMsgID(), i, tag, messageBody); + this.enqueueMessages.addData((MessageExt) message); } logger.info("Producer send messages finished"); } @@ -290,103 +373,142 @@ public void sendDelay(String topic, int delayLevel, int messageNum) { logger.info("Producer send delay messages finished"); } - ///** + /// ** // * 发送延迟消息 // * - // * @param topic topic名称 + // * @param topic topic名称 // * @param delaySecondTime 延迟时间，单位:秒 - // * @param messageNum 消息条数 + // * @param messageNum 消息条数 // */ - //public void sendDelayWithTag(String topic, String tag, int delaySecondTime, int messageNum) { - // logger.info("Producer start to send delay messages"); - // for (int i = 0; i < messageNum; i++) { - // Message message = MessageFactory.buildOneMessageWithTagAndStartDeliverTime(topic, tag, - // System.currentTimeMillis() + delaySecondTime * 1000L); - // SendResult sendResult = producer.send(message); - // logger.info(sendResult.toString()); - // this.enqueueMessages.addData(message); - // } - // logger.info("Producer send delay messages finished"); - //} + // public void sendDelayWithTag(String topic, String tag, int delaySecondTime, + /// int messageNum) { + // logger.info("Producer start to send delay messages"); + // for (int i = 0; i < messageNum; i++) { + // Message message = + /// MessageFactory.buildOneMessageWithTagAndStartDeliverTime(topic, tag, + // System.currentTimeMillis() + delaySecondTime * 1000L); + // SendResult sendResult = producer.send(message); + // logger.info(sendResult.toString()); + // this.enqueueMessages.addData(message); + // } + // logger.info("Producer send delay messages finished"); + // } // - ///** + /// ** // * 发送定时消息 // * - // * @param topic topic名称 - // * @param time 定时消息的时间戳 + // * @param topic topic名称 + // * @param time 定时消息的时间戳 // * @param messageNum 消息条数 // */ - //public void sendTimingWithTag(String topic, String tag, long time, int messageNum) { - // logger.info("Producer start to send delay messages"); - // for (int i = 0; i < messageNum; i++) { - // String deliverTime = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(System.currentTimeMillis() + time * 1000L)); - // Message message = MessageFactory.buildOneTimingMessageWithTag(topic, tag, UUID.randomUUID().toString(), deliverTime); - // SendResult sendResult = producer.send(message); - // logger.info(sendResult.toString()); - // this.enqueueMessages.addData(message); - // } - // logger.info("Producer send delay messages finished"); - //} + // public void sendTimingWithTag(String topic, String tag, long time, int + /// messageNum) { + // logger.info("Producer start to send delay messages"); + // for (int i = 0; i < messageNum; i++) { + // String deliverTime = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new + /// Date(System.currentTimeMillis() + time * 1000L)); + // Message message = MessageFactory.buildOneTimingMessageWithTag(topic, tag, + /// UUID.randomUUID().toString(), deliverTime); + // SendResult sendResult = producer.send(message); + // logger.info(sendResult.toString()); + // this.enqueueMessages.addData(message); + // } + // logger.info("Producer send delay messages finished"); + // } // - ///** + /// ** // * 发送定时消息 // * - // * @param topic topic名称 - // * @param time 定时消息的时间戳 + // * @param topic topic名称 + // * @param time 定时消息的时间戳 // * @param messageNum 消息条数 // */ - //public void sendTimingWithTagAndBody(String topic, String tag, String body, long time, int messageNum) { - // logger.info("Producer start to send delay messages"); - // for (int i = 0; i < messageNum; i++) { - // String deliverTime = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(System.currentTimeMillis() + time * 1000L)); - // Message message = MessageFactory.buildOneTimingMessageWithTag(topic, tag, body, deliverTime); - // SendResult sendResult = producer.send(message); - // logger.info(sendResult.toString()); - // this.enqueueMessages.addData(message); - // } - // logger.info("Producer send delay messages finished"); - //} + // public void sendTimingWithTagAndBody(String topic, String tag, String body, + /// long time, int messageNum) { + // logger.info("Producer start to send delay messages"); + // for (int i = 0; i < messageNum; i++) { + // String deliverTime = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new + /// Date(System.currentTimeMillis() + time * 1000L)); + // Message message = MessageFactory.buildOneTimingMessageWithTag(topic, tag, + /// body, deliverTime); + // SendResult sendResult = producer.send(message); + // logger.info(sendResult.toString()); + // this.enqueueMessages.addData(message); + // } + // logger.info("Producer send delay messages finished"); + // } // - ///** - // * @param topic 发送topic - // * @param tag 消息tag + /// ** + // * @param topic 发送topic + // * @param tag 消息tag // * @param messageNum 消息数量 - // * @param userProps 消息属性 + // * @param userProps 消息属性 // */ - //public void sendWithTagAndUserProps(String topic, String tag, int messageNum, HashMap userProps) { - // logger.info("Producer start to send messages"); - // for (int i = 0; i < messageNum; i++) { - // Message message = MessageFactory.buildOneMessageWithTagAndUserProps(topic, tag, UUID.randomUUID().toString(), userProps); - // SendResult sendResult = producer.send(message); - // logger.info(sendResult.toString()); - // this.enqueueMessages.addData(message); - // } - // logger.info("Producer send messages finished"); - //} + // public void sendWithTagAndUserProps(String topic, String tag, int messageNum, + /// HashMap userProps) { + // logger.info("Producer start to send messages"); + // for (int i = 0; i < messageNum; i++) { + // Message message = MessageFactory.buildOneMessageWithTagAndUserProps(topic, + /// tag, UUID.randomUUID().toString(), userProps); + // SendResult sendResult = producer.send(message); + // logger.info(sendResult.toString()); + // this.enqueueMessages.addData(message); + // } + // logger.info("Producer send messages finished"); + // } // - ///** - // * @param topic 发送topic - // * @param tag 消息tag + /// ** + // * @param topic 发送topic + // * @param tag 消息tag // * @param delaySeconds 消息延迟 单位：s - // * @param messageNum 消息数量 - // * @param userProps 消息属性 + // * @param messageNum 消息数量 + // * @param userProps 消息属性 // */ - //public void sendDelayWithTagAndUserProps(String topic, String tag, int delaySeconds, int messageNum, HashMap userProps) { - // logger.info("Producer start to send messages"); - // for (int i = 0; i < messageNum; i++) { - // Message message = MessageFactory.buildOneDelayMessageWithTagAndUserProps(topic, tag, System.currentTimeMillis() + delaySeconds * 1000L, - // UUID.randomUUID().toString(), userProps); - // SendResult sendResult = producer.send(message); - // logger.info(sendResult.toString()); - // this.enqueueMessages.addData(message); - // } - // logger.info("Producer send messages finished"); - //} + // public void sendDelayWithTagAndUserProps(String topic, String tag, int + /// delaySeconds, int messageNum, HashMap userProps) { + // logger.info("Producer start to send messages"); + // for (int i = 0; i < messageNum; i++) { + // Message message = + /// MessageFactory.buildOneDelayMessageWithTagAndUserProps(topic, tag, + /// System.currentTimeMillis() + delaySeconds * 1000L, + // UUID.randomUUID().toString(), userProps); + // SendResult sendResult = producer.send(message); + // logger.info(sendResult.toString()); + // this.enqueueMessages.addData(message); + // } + // logger.info("Producer send messages finished"); + // } + + public void send(String topic, String tag, String body) { + logger.info("Producer start to send messages"); + + Message message = null; + try { + message = MessageFactory.buildMessage(topic, tag, body); + } catch (Exception e) { + logger.error("build message failed"); + } + try { + SendResult sendResult = producer.send(message); + MessageExt messageExt = new MessageExt(); + messageExt.setMsgId(sendResult.getMsgId()); + logger.info("{}, tag: {}", sendResult, tag); + this.enqueueMessages.addData(messageExt); + } catch (Exception e) { + logger.error("DefaultMQProducer send message failed"); + } + + logger.info("Producer send messages finished"); + } public void send(Message message) { SendResult sendResult = null; + MessageExt messageExt = null; try { sendResult = producer.send(message); + messageExt = new MessageExt(); + messageExt.setMsgId(sendResult.getMsgId()); + logger.info("{}", sendResult); } catch (MQClientException e) { e.printStackTrace(); } catch (RemotingException e) { @@ -396,8 +518,7 @@ public void send(Message message) { } catch (InterruptedException e) { e.printStackTrace(); } - logger.info(sendResult.toString()); - this.enqueueMessages.addData((MessageExt)message); + this.enqueueMessages.addData(messageExt); } public List fetchPublishMessageQueues(String topic) { diff --git a/java/e2e-v4/src/main/java/org/apache/rocketmq/client/rmq/RMQTransactionProducer.java b/java/e2e-v4/src/main/java/org/apache/rocketmq/client/rmq/RMQTransactionProducer.java index bf7ab03..7d6a878 100644 --- a/java/e2e-v4/src/main/java/org/apache/rocketmq/client/rmq/RMQTransactionProducer.java +++ b/java/e2e-v4/src/main/java/org/apache/rocketmq/client/rmq/RMQTransactionProducer.java @@ -38,9 +38,10 @@ public RMQTransactionProducer(TransactionMQProducer producer) { this.producer = producer; } // - //public RMQTransactionProducer(String nsAddr, String topic, TransactionListener transactionListener) { - // this(nsAddr, topic, false, transactionListener); - //} + // public RMQTransactionProducer(String nsAddr, String topic, + // TransactionListener transactionListener) { + // this(nsAddr, topic, false, transactionListener); + // } @Override public void shutdown() { @@ -50,7 +51,8 @@ public void shutdown() { public void send(String topic, String tag, int messageNum) { logger.info("Producer start to send messages"); for (int i = 0; i < messageNum; i++) { - //Message message = MessageFactory.buildOneMessageWithTagAndBody(topic, tag, String.valueOf(i)); + // Message message = MessageFactory.buildOneMessageWithTagAndBody(topic, tag, + // String.valueOf(i)); Message message = MessageFactory.buildOneMessageWithTag(topic, tag); SendResult sendResult = null; MessageExt messageExt = null; @@ -73,4 +75,25 @@ public void send(String topic, String tag, int messageNum) { logger.info("Producer send messages finished"); } + public void sendTrans(String topic, String tag, int messageNum) { + logger.info("Producer start to send transaction messages"); + for (int i = 0; i < messageNum; i++) { + // Message message = MessageFactory.buildOneMessageWithTagAndBody(topic, tag, + // String.valueOf(i)); + Message message = MessageFactory.buildNormalMessage(topic, tag, String.valueOf(i)); + SendResult sendResult = null; + MessageExt messageExt = null; + try { + sendResult = producer.sendMessageInTransaction(message, null); + messageExt = new MessageExt(); + messageExt.setMsgId(sendResult.getMsgId()); + logger.info("{}, index: {}, tag: {}", sendResult, i, tag); + this.enqueueMessages.addData(messageExt); + } catch (MQClientException e) { + e.printStackTrace(); + } + } + logger.info("Producer send messages finished"); + } + } diff --git a/java/e2e-v4/src/main/java/org/apache/rocketmq/common/MQCollector.java b/java/e2e-v4/src/main/java/org/apache/rocketmq/common/MQCollector.java index c794907..8dc8539 100644 --- a/java/e2e-v4/src/main/java/org/apache/rocketmq/common/MQCollector.java +++ b/java/e2e-v4/src/main/java/org/apache/rocketmq/common/MQCollector.java @@ -40,15 +40,17 @@ public abstract class MQCollector { * 全部出队的消息体（去重） */ protected DataCollector dequeueUndupMessageBody = null; - //消息发送响应时间收集器 + // 消息发送响应时间收集器 protected DataCollector msgRTs = null; public MQCollector() { enqueueMessages = DataCollectorManager.getInstance().fetchListDataCollector(RandomUtils.getStringByUUID()); - enqueueFailedMessages = DataCollectorManager.getInstance().fetchListDataCollector(RandomUtils.getStringByUUID()); + enqueueFailedMessages = DataCollectorManager.getInstance() + .fetchListDataCollector(RandomUtils.getStringByUUID()); dequeueMessages = DataCollectorManager.getInstance().fetchListDataCollector(RandomUtils.getStringByUUID()); dequeueAllMessages = DataCollectorManager.getInstance().fetchListDataCollector(RandomUtils.getStringByUUID()); - dequeueUndupMessageBody = DataCollectorManager.getInstance().fetchListDataCollector(RandomUtils.getStringByUUID()); + dequeueUndupMessageBody = DataCollectorManager.getInstance() + .fetchListDataCollector(RandomUtils.getStringByUUID()); msgRTs = DataCollectorManager.getInstance().fetchListDataCollector(RandomUtils.getStringByUUID()); } diff --git a/java/e2e-v4/src/main/java/org/apache/rocketmq/enums/TESTSET.java b/java/e2e-v4/src/main/java/org/apache/rocketmq/enums/TESTSET.java index 2e59e7c..ba72d8a 100644 --- a/java/e2e-v4/src/main/java/org/apache/rocketmq/enums/TESTSET.java +++ b/java/e2e-v4/src/main/java/org/apache/rocketmq/enums/TESTSET.java @@ -38,16 +38,34 @@ public class TESTSET { public static final String TRANSACTION = "transaction"; public static final String PULL = "pull"; + public static final String BATCH_CONSUME = "batchConsume"; + public static final String DEADLETTER = "deadletter"; + public static final String REBALANCE = "rebalance"; + public static final String CLIENT = "client"; + public static final String RETRY = "retry"; + public static final String ASNYC = "async"; + public static final String TAG = "tag"; + public static final String SQL = "sql"; + public static final String CLUSTER = "cluster"; + public static final String BROADCAST = "broadcast"; + public static final String ACL = "acl"; - public static final String SIMPLE = "Simple"; + + public static final String MODEL = "model"; + + public static final String LOAD_BALANCING = "loadBalancing"; + + public static final String BATCHPRODUCER = "batchProducer"; + + public static final String OFFSET = "offset"; } diff --git a/java/e2e-v4/src/main/java/org/apache/rocketmq/factory/ConsumerFactory.java b/java/e2e-v4/src/main/java/org/apache/rocketmq/factory/ConsumerFactory.java index 0a37860..d3959a4 100644 --- a/java/e2e-v4/src/main/java/org/apache/rocketmq/factory/ConsumerFactory.java +++ b/java/e2e-v4/src/main/java/org/apache/rocketmq/factory/ConsumerFactory.java @@ -17,9 +17,13 @@ package org.apache.rocketmq.factory; +import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy; +import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; +import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; import org.apache.rocketmq.client.rmq.RMQNormalConsumer; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.utils.RandomUtils; @@ -45,4 +49,98 @@ public static RMQNormalConsumer getRMQNormalConsumer(String nsAddr, String consu consumer.setNamesrvAddr(nsAddr); return new RMQNormalConsumer(consumer); } + + public static RMQNormalConsumer getRMQBroadCastConsumer(String nsAddr, String consumerGroup, RPCHook rpcHook) { + DefaultMQPushConsumer consumer; + if (aclEnable) { + consumer = new DefaultMQPushConsumer(consumerGroup, rpcHook, new AllocateMessageQueueAveragely()); + } else { + consumer = new DefaultMQPushConsumer(consumerGroup); + } + consumer.setInstanceName(RandomUtils.getStringByUUID()); + consumer.setMessageModel(MessageModel.BROADCASTING); + consumer.setNamesrvAddr(nsAddr); + return new RMQNormalConsumer(consumer); + } + + public static RMQNormalConsumer getRMQClusterConsumer(String nsAddr, String consumerGroup, RPCHook rpcHook) { + DefaultMQPushConsumer consumer; + if (aclEnable) { + consumer = new DefaultMQPushConsumer(consumerGroup, rpcHook, new AllocateMessageQueueAveragely()); + } else { + consumer = new DefaultMQPushConsumer(consumerGroup); + } + consumer.setInstanceName(RandomUtils.getStringByUUID()); + consumer.setMessageModel(MessageModel.CLUSTERING); + consumer.setNamesrvAddr(nsAddr); + return new RMQNormalConsumer(consumer); + } + + public static RMQNormalConsumer getRMQClusterConsumer(String nsAddr, String consumerGroup, RPCHook rpcHook, + AllocateMessageQueueStrategy allocateMessageQueueStrategy) { + DefaultMQPushConsumer consumer; + if (aclEnable) { + consumer = new DefaultMQPushConsumer(consumerGroup, rpcHook, allocateMessageQueueStrategy); + } else { + consumer = new DefaultMQPushConsumer(consumerGroup); + } + consumer.setInstanceName(RandomUtils.getStringByUUID()); + consumer.setMessageModel(MessageModel.CLUSTERING); + consumer.setNamesrvAddr(nsAddr); + return new RMQNormalConsumer(consumer); + } + + public static RMQNormalConsumer getRMQNormalConsumer(String nsAddr, String consumerGroup, RPCHook rpcHook, + AllocateMessageQueueStrategy allocateMessageQueueStrategy) { + DefaultMQPushConsumer consumer; + if (aclEnable) { + consumer = new DefaultMQPushConsumer(consumerGroup, rpcHook, allocateMessageQueueStrategy); + } else { + consumer = new DefaultMQPushConsumer(consumerGroup); + } + consumer.setInstanceName(RandomUtils.getStringByUUID()); + consumer.setNamesrvAddr(nsAddr); + return new RMQNormalConsumer(consumer); + } + + public static RMQNormalConsumer getRMQLitePullConsumer(String nsAddr, String consumerGroup, RPCHook rpcHook) { + DefaultLitePullConsumer consumer; + if (aclEnable) { + consumer = new DefaultLitePullConsumer(consumerGroup, rpcHook); + } else { + consumer = new DefaultLitePullConsumer(consumerGroup); + } + consumer.setNamesrvAddr(nsAddr); + consumer.setInstanceName(RandomUtils.getStringByUUID()); + consumer.setConsumerPullTimeoutMillis(5000); + return new RMQNormalConsumer(consumer); + } + + public static RMQNormalConsumer getRMQLitePullConsumer(String nsAddr, String consumerGroup, RPCHook rpcHook, + int batchSize) { + DefaultLitePullConsumer consumer; + if (aclEnable) { + consumer = new DefaultLitePullConsumer(consumerGroup, rpcHook); + } else { + consumer = new DefaultLitePullConsumer(consumerGroup); + } + consumer.setNamesrvAddr(nsAddr); + consumer.setInstanceName(RandomUtils.getStringByUUID()); + consumer.setConsumerPullTimeoutMillis(5000); + consumer.setPullBatchSize(batchSize); + return new RMQNormalConsumer(consumer); + } + + public static RMQNormalConsumer getRMQPullConsumer(String nsAddr, String consumerGroup, RPCHook rpcHook) { + DefaultMQPullConsumer consumer; + if (aclEnable) { + consumer = new DefaultMQPullConsumer(consumerGroup, rpcHook); + } else { + consumer = new DefaultMQPullConsumer(consumerGroup); + } + consumer.setInstanceName(RandomUtils.getStringByUUID()); + consumer.setNamesrvAddr(nsAddr); + consumer.setInstanceName(RandomUtils.getStringByUUID()); + return new RMQNormalConsumer(consumer); + } } diff --git a/java/e2e-v4/src/main/java/org/apache/rocketmq/factory/MessageFactory.java b/java/e2e-v4/src/main/java/org/apache/rocketmq/factory/MessageFactory.java index e1b32f3..04335d5 100644 --- a/java/e2e-v4/src/main/java/org/apache/rocketmq/factory/MessageFactory.java +++ b/java/e2e-v4/src/main/java/org/apache/rocketmq/factory/MessageFactory.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Map; public class MessageFactory { @@ -61,9 +62,50 @@ public static Collection getRandomMessageListByTag(String topic, String return msgList; } + /** + * Generate a message, customize the topic, tags, body, detect the body is null throws an exception problem. + * + * @param topic topic + * @return message + */ + public static Message buildMessage(String topic, String tags, String body) throws Exception{ + return new Message(topic, tags, body.getBytes()); + } /** - * 生成一条消息，tag不设置， body使用随机字符串 + * Generate a message, customize the topic, tags, body. + * + * @param topic topic + * @return message + */ + public static Message buildNormalMessage(String topic, String tags, String body){ + return new Message(topic, tags, body.getBytes()); + } + + public static Message buildMessageWithProperty(String topic, Map userProperties, String body){ + Message msg = new Message(topic, body.getBytes()); + for (Map.Entry entry : userProperties.entrySet()) { + msg.putUserProperty(entry.getKey(), entry.getValue()); + } + return msg; + } + + /** + * Generate a message, customize the topic and properties. + * + * @param topic topic + * @return message + */ + public static Message buildMessageWithProperty(String topic, Map userProperties){ + Message msg = new Message(topic, RandomUtils.getStringByUUID().getBytes()); + for (Map.Entry entry : userProperties.entrySet()) { + msg.putUserProperty(entry.getKey(), entry.getValue()); + } + return msg; + } + + /** + * Generate a message with no tag and a random string for the body. * * @param topic topic * @return message @@ -73,7 +115,7 @@ public static Message buildOneMessage(String topic) { } /** - * 根据tag生成一条消息， body使用随机字符串 + * A message is generated from the tag, with the body using a random string. * * @param topic topic * @param tag tag @@ -84,10 +126,10 @@ public static Message buildOneMessageWithTag(String topic, String tag) { } /** - * 根据消息体生成一条消息，tag不设置 + * A message is generated based on the message body. The tag is not set. * * @param topic topic - * @param messageBody 消息体 + * @param messageBody message body * @return message */ public static Message buildOneMessageWithBody(String topic, String messageBody) { @@ -95,11 +137,11 @@ public static Message buildOneMessageWithBody(String topic, String messageBody) } /** - * 构建一条指定tag，消息body的message + * Builds a message specifying the tag, message body * - * @param topic 发送topic - * @param tag 发送tag - * @param body 消息体 + * @param topic topic + * @param tag tag + * @param body message body * @return Message */ public static Message buildOneMessageWithTagAndBody(String topic, String tag, String body) { @@ -107,10 +149,10 @@ public static Message buildOneMessageWithTagAndBody(String topic, String tag, St } ///** - // * 构建指定一条指定properties，消息body的message + // * The build specifies a message that specifies properties, the body of the message // * - // * @param topic 发送topic - // * @param properties 自定义属性 + // * @param topic topic + // * @param properties custom properties // * @return Message // */ //public static Message buildMessageWithUserProperties(String topic, HashMap properties) { @@ -122,8 +164,8 @@ public static Message buildOneMessageWithTagAndBody(String topic, String tag, St //} // /** - * 延时消息，单位毫秒（ms），在指定延迟时间（当前时间之后）进行投递，例如消息在3秒后投递 - * 定时消息，单位毫秒（ms），在指定时间戳（当前时间之后）进行投递，例如2016-03-07 16:21:00投递。如果被设置成当前时间戳之前的某个时刻，消息将立即被投递给消费者。 + * A delayed message, in milliseconds (ms), is delivered after the specified delay time (after the current time), for example, the message is delivered after 3 seconds + * An order message, in milliseconds (ms), is delivered in a specified timestamp (after the current time), such as 2016-03-07 16:21:00 delivery. If it is set to a time before the current timestamp, the message will be delivered to the consumer immediately. * * @param topic * @param level @@ -137,16 +179,16 @@ public static Message buildOneMessageWithDelayTimeLevel(String topic, int level) // //public static Message buildOneMessageWithTagAndStartDeliverTime(String topic, String tag, long time) { // Message msg = new Message(topic, tag, RandomUtils.getStringByUUID().getBytes()); - // //发送时间 + // //sending time // msg.putUserProperties("sendTime", String.valueOf(System.currentTimeMillis())); - // //推送下发消息时间 + // //push message delivery time // msg.putUserProperties("deliverTime", String.valueOf(time)); // msg.setStartDeliverTime(time); // return msg; //} // ///** - // * 定时消息，单位毫秒（ms），在指定时间戳（当前时间之后）进行投递，例如2016-03-07 16:21:00投递。如果被设置成当前时间戳之前的某个时刻，消息将立即被投递给消费者。 + // * An order message, in milliseconds (ms), is delivered in a specified timestamp (after the current time), such as 2016-03-07 16:21:00 delivery. If it is set to a time before the current timestamp, the message will be delivered to the consumer immediately. // * // * @param topic // * @param tag diff --git a/java/e2e-v4/src/main/java/org/apache/rocketmq/frame/BaseOperate.java b/java/e2e-v4/src/main/java/org/apache/rocketmq/frame/BaseOperate.java index d7e84f1..d3c8ce9 100644 --- a/java/e2e-v4/src/main/java/org/apache/rocketmq/frame/BaseOperate.java +++ b/java/e2e-v4/src/main/java/org/apache/rocketmq/frame/BaseOperate.java @@ -18,6 +18,8 @@ package org.apache.rocketmq.frame; import org.apache.rocketmq.utils.MQAdmin; +import org.apache.rocketmq.utils.RandomUtils; +import org.junit.jupiter.api.Assertions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,4 +34,18 @@ public void run() { } }); } + + protected static String getTopic(String methodName) { + String topic = String.format("topic_%s_%s", methodName, RandomUtils.getStringWithCharacter(6)); + logger.info("[Topic] topic:{}, methodName:{}", topic, methodName); + boolean result = MQAdmin.createTopic(namesrvAddr,cluster, topic, 8); + Assertions.assertTrue(result, String.format("Create topic:%s failed", topic)); + return topic; + } + + protected static String getGroupId(String methodName) { + String groupId = String.format("GID_%s_%s", methodName, RandomUtils.getStringWithCharacter(6)); + logger.info("[ConsumerGroupId] groupId:{}, methodName:{}", groupId, methodName); + return groupId; + } } diff --git a/java/e2e-v4/src/main/java/org/apache/rocketmq/listener/rmq/concurrent/MessageIdStore.java b/java/e2e-v4/src/main/java/org/apache/rocketmq/listener/rmq/concurrent/MessageIdStore.java new file mode 100644 index 0000000..d80d85e --- /dev/null +++ b/java/e2e-v4/src/main/java/org/apache/rocketmq/listener/rmq/concurrent/MessageIdStore.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.listener.rmq.concurrent; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +public class MessageIdStore { + private Set consumedMessageIds; + private static MessageIdStore instance; + + private MessageIdStore() { + consumedMessageIds = Collections.newSetFromMap(new ConcurrentHashMap<>()); + } + + public static MessageIdStore getInstance() { + if (instance == null) { + synchronized (MessageIdStore.class) { + if (instance == null) { + instance = new MessageIdStore(); + } + } + } + return instance; + } + + public boolean isMessageConsumed(String messageId) { + return consumedMessageIds.contains(messageId); + } + + public void markMessageAsConsumed(String messageId) { + consumedMessageIds.add(messageId); + } + + public void clear() { + consumedMessageIds.clear(); + } +} diff --git a/java/e2e-v4/src/main/java/org/apache/rocketmq/listener/rmq/concurrent/RMQIdempotentListener.java b/java/e2e-v4/src/main/java/org/apache/rocketmq/listener/rmq/concurrent/RMQIdempotentListener.java new file mode 100644 index 0000000..5671062 --- /dev/null +++ b/java/e2e-v4/src/main/java/org/apache/rocketmq/listener/rmq/concurrent/RMQIdempotentListener.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.listener.rmq.concurrent; + +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.listener.AbstractListener; +import org.apache.rocketmq.utils.RandomUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +public class RMQIdempotentListener extends AbstractListener implements MessageListenerConcurrently { + private static Logger logger = LoggerFactory.getLogger(RMQNormalListener.class); + private ConsumeConcurrentlyStatus consumeStatus = ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + private AtomicInteger msgIndex = new AtomicInteger(0); + private String listenerName; + private MessageIdStore messageIdStore = MessageIdStore.getInstance(); + + public RMQIdempotentListener() { + this.listenerName = RandomUtils.getStringByUUID(); + logger.info("Start listening:{}", listenerName); + } + + public RMQIdempotentListener(String listenerName) { + this.listenerName = listenerName; + logger.info("Start listening:{}", listenerName); + } + + public RMQIdempotentListener(ConsumeConcurrentlyStatus consumeStatus) { + this.consumeStatus = consumeStatus; + this.listenerName = RandomUtils.getStringByUUID(); + logger.info("Start listening:{}", listenerName); + } + + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext consumeConcurrentlyContext) { + for (MessageExt message : msgs) { + boolean isConsumed = messageIdStore.isMessageConsumed(message.getMsgId()); + if (isConsumed) { + return consumeStatus; + } else { + msgIndex.getAndIncrement(); + message.putUserProperty("startDeliverTime", String.valueOf(System.currentTimeMillis())); + this.dequeueAllMessages.addData(message); + this.dequeueMessages.addData(message); + logger.info("{} - MessageId:{}, ReconsumeTimes:{}, Body:{}, tag:{}, recvIndex:{}, action:{}", + listenerName, message.getMsgId(), + message.getReconsumeTimes(), new String(message.getBody()), message.getTags(), + msgIndex.getAndIncrement() + 1, consumeStatus); + messageIdStore.markMessageAsConsumed(message.getMsgId()); + } + } + return consumeStatus; + } +} diff --git a/java/e2e-v4/src/main/java/org/apache/rocketmq/listener/rmq/concurrent/RMQNormalListener.java b/java/e2e-v4/src/main/java/org/apache/rocketmq/listener/rmq/concurrent/RMQNormalListener.java index 91b703c..eec5fa6 100644 --- a/java/e2e-v4/src/main/java/org/apache/rocketmq/listener/rmq/concurrent/RMQNormalListener.java +++ b/java/e2e-v4/src/main/java/org/apache/rocketmq/listener/rmq/concurrent/RMQNormalListener.java @@ -37,13 +37,18 @@ public class RMQNormalListener extends AbstractListener implements MessageListen public RMQNormalListener() { this.listenerName = RandomUtils.getStringByUUID(); - logger.info("启动监听:{}", listenerName); + logger.info("Start listening:{}", listenerName); + } + + public RMQNormalListener(String listenerName) { + this.listenerName = listenerName; + logger.info("Start listening:{}", listenerName); } public RMQNormalListener(ConsumeConcurrentlyStatus consumeStatus) { this.consumeStatus = consumeStatus; this.listenerName = RandomUtils.getStringByUUID(); - logger.info("启动监听:{}", listenerName); + logger.info("Start listening:{}", listenerName); } public ConsumeConcurrentlyStatus consumeMessage(List msgs, ConsumeConcurrentlyContext consumeConcurrentlyContext) { diff --git a/java/e2e-v4/src/main/java/org/apache/rocketmq/listener/rmq/concurrent/RMQOrderListener.java b/java/e2e-v4/src/main/java/org/apache/rocketmq/listener/rmq/concurrent/RMQOrderListener.java index ead6bf8..76a149f 100644 --- a/java/e2e-v4/src/main/java/org/apache/rocketmq/listener/rmq/concurrent/RMQOrderListener.java +++ b/java/e2e-v4/src/main/java/org/apache/rocketmq/listener/rmq/concurrent/RMQOrderListener.java @@ -37,13 +37,13 @@ public class RMQOrderListener extends AbstractListener implements MessageListene public RMQOrderListener() { this.listenerName = RandomUtils.getStringByUUID(); - logger.info("启动监听:{}", listenerName); + logger.info("Start listening:{}", listenerName); } public RMQOrderListener(ConsumeOrderlyStatus consumeStatus) { this.consumeStatus = consumeStatus; this.listenerName = RandomUtils.getStringByUUID(); - logger.info("启动监听:{}", listenerName); + logger.info("Start listening:{}", listenerName); } @Override diff --git a/java/e2e-v4/src/main/java/org/apache/rocketmq/listener/rmq/concurrent/TransactionListenerImpl.java b/java/e2e-v4/src/main/java/org/apache/rocketmq/listener/rmq/concurrent/TransactionListenerImpl.java index 926f083..77008c0 100644 --- a/java/e2e-v4/src/main/java/org/apache/rocketmq/listener/rmq/concurrent/TransactionListenerImpl.java +++ b/java/e2e-v4/src/main/java/org/apache/rocketmq/listener/rmq/concurrent/TransactionListenerImpl.java @@ -40,60 +40,60 @@ public class TransactionListenerImpl extends AbstractListener implements Transac private ConcurrentHashMap localTrans = new ConcurrentHashMap<>(); - - public TransactionListenerImpl() { this.checkerName = RandomUtils.getStringByUUID(); - logger.info("启动监听:{}", checkerName); + logger.info("Start listening:{}", checkerName); } public TransactionListenerImpl(LocalTransactionState checker, LocalTransactionState executor) { this.checkerName = RandomUtils.getStringByUUID(); this.checker = checker; this.executor = executor; - logger.info("启动监听:{}", checkerName); + logger.info("Start listening:{}", checkerName); } @Override public LocalTransactionState executeLocalTransaction(Message message, Object arg) { logger.warn("TransactionId:{}, transactionStatus:{}", message.getTransactionId(), executor.name()); - //int value = transactionIndex.getAndIncrement(); - //int status = value % 3; - //localTrans.put(message.getTransactionId(), status); + // int value = transactionIndex.getAndIncrement(); + // int status = value % 3; + // localTrans.put(message.getTransactionId(), status); return executor; } @Override public LocalTransactionState checkLocalTransaction(MessageExt message) { - //if (checkTimes == 1) { - // //消息 ID（有可能消息体一样，但消息 ID 不一样，当前消息属于半事务消息，所以消息 ID 在控制台无法查询） - // String msgId = message.getMsgId(); - // logger.info("checker:{}, MessageId:{}, transactionStatus:{}", checkerName, msgId, checker.name()); - // return checker; - //} - //if (new String(message.getBody()).equals(new String(sendMsg.getBody()))) { - // checkNum++; - // //check次数达到预期，则提交 - // if (checkNum == checkTimes) { - // checker = LocalTransactionState.COMMIT_MESSAGE; - // } - // logger.info("checkerName:{}, MessageId:{}, status:{}, checkedTimes:{}, expectCheckNum:{}", checkerName, message.getMsgId(), checker, checkTimes, - // checkNum); - //} - //Integer status = localTrans.get(message.getTransactionId()); - //if (null != status) { - // switch (status) { - // case 0: - // return LocalTransactionState.UNKNOW; - // case 1: - // return LocalTransactionState.COMMIT_MESSAGE; - // case 2: - // return LocalTransactionState.ROLLBACK_MESSAGE; - // } - //} - //本地事务已成功则提交消息 + // if (checkTimes == 1) { + // //消息 ID（有可能消息体一样，但消息 ID 不一样，当前消息属于半事务消息，所以消息 ID 在控制台无法查询） + // String msgId = message.getMsgId(); + // logger.info("checker:{}, MessageId:{}, transactionStatus:{}", checkerName, + // msgId, checker.name()); + // return checker; + // } + // if (new String(message.getBody()).equals(new String(sendMsg.getBody()))) { + // checkNum++; + // //check次数达到预期，则提交 + // if (checkNum == checkTimes) { + // checker = LocalTransactionState.COMMIT_MESSAGE; + // } + // logger.info("checkerName:{}, MessageId:{}, status:{}, checkedTimes:{}, + // expectCheckNum:{}", checkerName, message.getMsgId(), checker, checkTimes, + // checkNum); + // } + // Integer status = localTrans.get(message.getTransactionId()); + // if (null != status) { + // switch (status) { + // case 0: + // return LocalTransactionState.UNKNOW; + // case 1: + // return LocalTransactionState.COMMIT_MESSAGE; + // case 2: + // return LocalTransactionState.ROLLBACK_MESSAGE; + // } + // } + // 本地事务已成功则提交消息 return checker; } } diff --git a/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/AssertUtils.java b/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/AssertUtils.java new file mode 100644 index 0000000..89c7922 --- /dev/null +++ b/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/AssertUtils.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.utils; + +import org.junit.jupiter.api.Assertions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +public class AssertUtils { + private static final Logger log = LoggerFactory.getLogger(AssertUtils.class); +// final ExecutorService threadPool = Executors.newFixedThreadPool(numThreads); + + public static void assertConcurrent(final String message, final List runnables, + final int maxTimeoutSeconds) throws InterruptedException { + final int numThreads = runnables.size(); + final List exceptions = Collections.synchronizedList(new ArrayList()); + final ExecutorService threadPool = Executors.newFixedThreadPool(numThreads); + try { + final CountDownLatch allExecutorThreadsReady = new CountDownLatch(numThreads); + final CountDownLatch afterInitBlocker = new CountDownLatch(1); + final CountDownLatch allDone = new CountDownLatch(numThreads); + for (final Runnable submittedTestRunnable : runnables) { + threadPool.submit(new Runnable() { + public void run() { + allExecutorThreadsReady.countDown(); + try { + afterInitBlocker.await(); + submittedTestRunnable.run(); + } catch (final Throwable e) { + exceptions.add(e); + } finally { + allDone.countDown(); + } + } + }); + } + // wait until all threads are ready + Assertions.assertTrue(allExecutorThreadsReady.await(runnables.size() * 10, TimeUnit.MILLISECONDS), "Timeout initializing threads! Perform long lasting initializations before passing runnables to assertConcurrent"); + // start all test runners + afterInitBlocker.countDown(); + Assertions.assertTrue(allDone.await(maxTimeoutSeconds, TimeUnit.SECONDS), message + " timeout! More than " + maxTimeoutSeconds + " seconds"); + } finally { + log.info("exit sub thread"); + threadPool.shutdown(); + } + Assertions.assertTrue(exceptions.isEmpty(), message + " failed with exception(s)" + exceptions); + } + + public static void assertConcurrentAtMost(final String message, final List runnables, + final int maxTimeoutSeconds) throws InterruptedException { + final int numThreads = runnables.size(); + final List exceptions = Collections.synchronizedList(new ArrayList()); + final ExecutorService threadPool = Executors.newFixedThreadPool(numThreads); + try { + final CountDownLatch allExecutorThreadsReady = new CountDownLatch(numThreads); + final CountDownLatch afterInitBlocker = new CountDownLatch(1); + final CountDownLatch allDone = new CountDownLatch(numThreads); + for (final Runnable submittedTestRunnable : runnables) { + threadPool.submit(new Runnable() { + public void run() { + allExecutorThreadsReady.countDown(); + try { + afterInitBlocker.await(); + submittedTestRunnable.run(); + } catch (final Throwable e) { + exceptions.add(e); + } finally { + allDone.countDown(); + } + } + }); + } + // wait until all threads are ready + Assertions.assertTrue(allExecutorThreadsReady.await(runnables.size() * 10, TimeUnit.MILLISECONDS), "Timeout initializing threads! Perform long lasting initializations before passing runnables to assertConcurrent"); + // start all test runners + afterInitBlocker.countDown(); + allDone.await(maxTimeoutSeconds, TimeUnit.SECONDS); +// Assertions.assertTrue(allDone.await(maxTimeoutSeconds, TimeUnit.SECONDS), message + " timeout! More than " + maxTimeoutSeconds + " seconds"); + } finally { + log.info("exit sub thread"); + threadPool.shutdown(); + } + Assertions.assertTrue(exceptions.isEmpty(), message + " failed with exception(s)" + exceptions); + } +} diff --git a/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/DateUtils.java b/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/DateUtils.java index 0752557..275a2ef 100644 --- a/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/DateUtils.java +++ b/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/DateUtils.java @@ -27,9 +27,9 @@ public class DateUtils { /** - * 获取当前时间的秒级偏移时间 - * offset=10 10秒后 offset=-10 10秒前 - * @param offset 偏移时间 + * Gets the second-level offset time of the current time + * offset=10 (10 seconds later) offset=-10 (10 seconds before) + * @param offset Offset time * @return */ public static long getCurrentTimeOffsetBySecond(int offset) { @@ -47,9 +47,9 @@ public static long getCurrentTimeOffsetBySecond(int offset) { } /** - * 获取当前时间的分钟级偏移时间 + * Gets the minute-level offset time of the current time * - * @param offset 偏移时间 + * @param offset Offset time * @return */ public static long getCurrentTimeOffsetByMin(int offset) { @@ -66,9 +66,9 @@ public static long getCurrentTimeOffsetByMin(int offset) { } /** - * 获取当前时间的小时级偏移时间 + * Gets the hour level offset time of the current time * - * @param offset 偏移时间 + * @param offset Offset time * @return */ public static long getCurrentTimeOffsetByHour(int offset) { @@ -85,9 +85,9 @@ public static long getCurrentTimeOffsetByHour(int offset) { } /** - * 获取当前时间的天级偏移时间 + * Gets the day offset time of the current time * - * @param offset 偏移时间 + * @param offset Offset time * @return */ public static long getCurrentTimeOffsetByDay(int offset) { @@ -104,9 +104,9 @@ public static long getCurrentTimeOffsetByDay(int offset) { } /** - * 格式化时间戳 + * Formatting timestamp * - * @param time 偏移时间 + * @param time Offset time * @return */ public static String format(long time) { diff --git a/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/FilterUtils.java b/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/FilterUtils.java new file mode 100644 index 0000000..6efce2d --- /dev/null +++ b/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/FilterUtils.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.utils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashSet; +import java.util.Set; + +public class FilterUtils { + private static final Logger log = LoggerFactory.getLogger(AssertUtils.class); + + // Custom used to filter tag for pullconsumer + public static boolean inTags(String msgTag, String tags) { + if (null == tags || tags.equals("*") || tags.length() == 0) { + return true; + } else { + // partition + String[] tag_splits = tags.split("\\|\\|"); + Set tagSet = new HashSet<>(); + if (tag_splits.length > 0) { + for (String tag : tag_splits) { + if (tag.length() > 0) { + if ("*".equals(tag)) + return true; + String trimString = tag.trim(); + if (trimString.length() > 0) { + tagSet.add(trimString.hashCode()); + } + } + if (tagSet.contains(msgTag.hashCode())) { + return true; + } + } + } + } + return false; + } +} diff --git a/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/NameUtils.java b/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/NameUtils.java index 45d3b7d..4979dbe 100644 --- a/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/NameUtils.java +++ b/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/NameUtils.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.utils; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.RandomStringUtils; import java.util.Map; @@ -61,4 +62,29 @@ public synchronized static String getTagName() { } + public synchronized static String getRandomTagName() { + while (true) { + String tag = "tag-server-" + RandomStringUtils.randomAlphanumeric(20); + String used = alreadyUsed.putIfAbsent(tag, tag); + if (used == null) { + return tag; + } + } + } + + public synchronized static String getRandomGroupName() { + while (true) { + String gid = "GID-server-" + RandomStringUtils.randomAlphanumeric(20); + String used = alreadyUsed.putIfAbsent(gid, gid); + if (used == null) { + return gid; + } + } + } + + protected static String getMD5Sum(String className, String methodName) { + String completeName = String.format("%s-%s", className, methodName); + return methodName + "-" + DigestUtils.md5Hex(completeName).substring(0, 6); + } + } diff --git a/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/TestUtils.java b/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/TestUtils.java index b28fcc1..a988189 100644 --- a/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/TestUtils.java +++ b/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/TestUtils.java @@ -106,7 +106,8 @@ public static void waitForInput(String keyWord, String info) { byte[] b = new byte[1024]; int n = System.in.read(b); - for (String s = (new String(b, 0, n - 1)).replace("\r", "").replace("\n", ""); !s.equals(keyWord); s = new String(b, 0, n - 1)) { + for (String s = (new String(b, 0, n - 1)).replace("\r", "").replace("\n", ""); !s + .equals(keyWord); s = new String(b, 0, n - 1)) { n = System.in.read(b); } @@ -121,14 +122,14 @@ public static > Map sortByValue(Map>() { @Override public int compare(Map.Entry o1, Map.Entry o2) { - return ((Comparable)o1.getValue()).compareTo(o2.getValue()); + return ((Comparable) o1.getValue()).compareTo(o2.getValue()); } }); Map result = new LinkedHashMap(); Iterator var3 = list.iterator(); while (var3.hasNext()) { - Map.Entry entry = (Map.Entry)var3.next(); + Map.Entry entry = (Map.Entry) var3.next(); result.put(entry.getKey(), entry.getValue()); } diff --git a/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/VerifyUtils.java b/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/VerifyUtils.java index 88e032f..c363c69 100644 --- a/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/VerifyUtils.java +++ b/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/VerifyUtils.java @@ -17,40 +17,48 @@ package org.apache.rocketmq.utils; +import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; +import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.rmq.DelayConf; +import org.apache.rocketmq.client.rmq.RMQNormalProducer; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.utils.data.collect.DataCollector; import org.junit.jupiter.api.Assertions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; +import java.util.stream.Collectors; public class VerifyUtils { private static Logger logger = LoggerFactory.getLogger(VerifyUtils.class); + private static AtomicInteger receivedIndex = new AtomicInteger(0); private static final int TIMEOUT = 60; + private static int defaultSimpleThreadNums = 4; /** - * 检验顺序消息 - * 校验点：1. 每个shardingkey中的消息都是顺序的 + * Check order message + * Check point: 1. The messages in each shardingkey are sequential * * @param receivedMessage * @return */ public static boolean checkOrderMessage(ConcurrentHashMap> receivedMessage) { for (Map.Entry> stringLinkedListEntry : receivedMessage.entrySet()) { - StringBuilder sb = new StringBuilder(String.format("shardingKey %s,message order: ", stringLinkedListEntry.getKey())); + StringBuilder sb = new StringBuilder( + String.format("shardingKey %s,message order: ", stringLinkedListEntry.getKey())); int preNode = -1; LinkedList messages = stringLinkedListEntry.getValue(); String tag = messages.getFirst().getTags(); @@ -71,83 +79,150 @@ public static boolean checkOrderMessage(ConcurrentHashMap enqueueMessages, DataCollector dequeueMessages) { - Collection unConsumedMessages = waitForMessageConsume(enqueueMessages, dequeueMessages, TIMEOUT * 1000L, 1); + public static void verifyNormalMessage(DataCollector enqueueMessages, + DataCollector dequeueMessages) { + Collection unConsumedMessages = waitForMessageConsume(enqueueMessages, dequeueMessages, + TIMEOUT * 1000L, 1); if (unConsumedMessages.size() > 0) { - Assertions.fail(String.format("以下%s条消息未被消费: %s", unConsumedMessages.size(), unConsumedMessages)); + Assertions.fail(String.format("The following %s messages are not consumed: %s", unConsumedMessages.size(), + unConsumedMessages)); } } /** - * 普通消息检查器 + * Check normal message * - * @param enqueueMessages 发送入队消息集合 - * @param dequeueMessages 消费出队消息集合 + * @param enqueueMessages collection of enqueued messages sent + * @param dequeueMessages collection of dequeued messages consumed */ - public static void verifyNormalMessage(DataCollector enqueueMessages, DataCollector dequeueMessages, int timeout) { - Collection unConsumedMessages = waitForMessageConsume(enqueueMessages, dequeueMessages, timeout * 1000L, 1); + public static void verifyNormalMessage(DataCollector enqueueMessages, + DataCollector dequeueMessages, int timeout) { + Collection unConsumedMessages = waitForMessageConsume(enqueueMessages, dequeueMessages, + timeout * 1000L, 1); if (unConsumedMessages.size() > 0) { - Assertions.fail(String.format("以下%s条消息未被消费: %s", unConsumedMessages.size(), unConsumedMessages)); + Assertions.fail(String.format("The following %s messages are not consumed: %s", unConsumedMessages.size(), + unConsumedMessages)); } } + public static void verifyNormalMessage(DataCollector enqueueMessages, + DataCollector dequeueMessages, Set unconsumedMsgIds, int timeout) { + Collection unConsumedMessages = waitForMessageConsume(enqueueMessages, dequeueMessages, + timeout * 1000L, 1); + Set unConsumedMessagesCopy = new HashSet<>(unConsumedMessages); + System.out.println(unConsumedMessagesCopy.size()); + Set finalUnconsumedMsgIds = unconsumedMsgIds; + unConsumedMessagesCopy = unConsumedMessagesCopy.stream() + .filter(msgExt -> !finalUnconsumedMsgIds.contains(msgExt.getMsgId())).collect(Collectors.toSet()); + System.out.println(unConsumedMessagesCopy.size()); + StringBuilder sb = new StringBuilder(); + boolean allInUnConsumedMessages = true; + for (String unconsumedMsgId : unconsumedMsgIds) { + boolean check = false; + for (MessageExt unConsumedMessage : unConsumedMessages) { + if (unConsumedMessage.getMsgId().equals(unconsumedMsgId)) { + check = true; + } + } + if (!check) { + allInUnConsumedMessages = false; + break; + } + } + if (!allInUnConsumedMessages) { + unconsumedMsgIds = unconsumedMsgIds.stream().filter(msgId -> { + for (MessageExt unConsumedMessage : unConsumedMessages) { + if (unConsumedMessage.getMsgId().equals(msgId)) { + return false; + } + } + return true; + }).collect(Collectors.toSet()); + logger.info(unconsumedMsgIds.size() + "messages are consumed:" + unconsumedMsgIds.size()); + sb.append("The following ").append(unconsumedMsgIds.size()).append(" messages are consumed:") + .append(unconsumedMsgIds); + Assertions.fail(sb.toString()); + } + if (unConsumedMessagesCopy.size() > 0) { + logger.info(unConsumedMessagesCopy.size() + "messages are not consumed:" + unConsumedMessagesCopy); + MessageExt messageExt = dequeueMessages.getFirstElement(); + sb.append(messageExt.getTopic()).append(" The following").append(unConsumedMessagesCopy.size()) + .append("messages are not consumed:").append(unConsumedMessagesCopy); + Assertions.fail(sb.toString()); + } + + } + /** - * 普通消息检查器，订阅消息是否与发送消息内容一致 + * Check normal message whether the subscribed message is consistent with the + * sent message content * - * @param enqueueMessages 发送入队消息集合 - * @param dequeueMessages 消费出队消息集合 - * @param messageBody 消息体 + * @param enqueueMessages collection of enqueued messages sent + * @param dequeueMessages collection of dequeued messages consumed + * @param messageBody message body */ - public static void verifyNormalMessageWithBody(DataCollector enqueueMessages, DataCollector dequeueMessages, String messageBody) { - Collection unConsumedMessages = waitForMessageConsume(enqueueMessages, dequeueMessages, TIMEOUT * 1000L, 1); + public static void verifyNormalMessageWithBody(DataCollector enqueueMessages, + DataCollector dequeueMessages, String messageBody) { + Collection unConsumedMessages = waitForMessageConsume(enqueueMessages, dequeueMessages, + TIMEOUT * 1000L, 1); if (unConsumedMessages.size() > 0) { - Assertions.fail(String.format("以下%s条消息未被消费: %s", unConsumedMessages.size(), unConsumedMessages)); + Assertions.fail(String.format("The following %s messages are not consumed: %s", unConsumedMessages.size(), + unConsumedMessages)); } Collection receivedMessages = dequeueMessages.getAllData(); List messages = new ArrayList<>(receivedMessages); for (Message message : messages) { - Assertions.assertEquals(messageBody, new String(message.getBody()), "订阅到的messageBody与期望不符"); + Assertions.assertEquals(messageBody, new String(message.getBody()), + "The messageBody subscribed didn't match expectations"); } } /** - * 顺序消息检查器 + * Check order message * - * @param enqueueMessages 发送入队消息集合 - * @param dequeueMessages 消费出队消息集合 + * @param enqueueMessages collection of enqueued messages sent + * @param dequeueMessages collection of dequeued messages consumed */ - public static void verifyOrderMessage(DataCollector enqueueMessages, DataCollector dequeueMessages) { - //检查是否消费完成 - Collection unConsumedMessages = waitForMessageConsume(enqueueMessages, dequeueMessages, TIMEOUT * 1000L, 1); + public static void verifyOrderMessage(DataCollector enqueueMessages, + DataCollector dequeueMessages) { + // Check whether consumption is completed + Collection unConsumedMessages = waitForMessageConsume(enqueueMessages, dequeueMessages, + TIMEOUT * 1000L, 1); if (unConsumedMessages.size() > 0) { - Assertions.fail(String.format("以下%s条消息未被消费: %s", unConsumedMessages.size(), unConsumedMessages)); + Assertions.fail(String.format("The following %s messages are not consumed: %s", unConsumedMessages.size(), + unConsumedMessages)); } - //检查是否消费顺序性 logger.warn(String.format("以下%s条消息未被消费: %s", unConsumedMessages.size(), unConsumedMessages)); - Assertions.assertTrue(checkOrder(dequeueMessages), "消息非顺序"); + // Check whether the consumption is sequential logger.warn(String.format("The + // following %s messages are not consumed: %s", unConsumedMessages.size(), + // unConsumedMessages)); + Assertions.assertTrue(checkOrder(dequeueMessages), "Messages are not in order"); } /** - * 延迟、定时消息检查器 + * Check delay message * - * @param enqueueMessages 发送入队消息集合 - * @param dequeueMessages 消费出队消息集合 - * @param delayLevel 预计需要的消费时间 + * @param enqueueMessages collection of enqueued messages sent + * @param dequeueMessages collection of dequeued messages consumed + * @param delayLevel delay level */ - public static void verifyDelayMessage(DataCollector enqueueMessages, DataCollector dequeueMessages, int delayLevel) { - //检查是否消费完成 - Collection unConsumedMessages = waitForMessageConsume(enqueueMessages, dequeueMessages, (TIMEOUT + DelayConf.DELAY_LEVEL[delayLevel - 1]) * 1000L, 1); + public static void verifyDelayMessage(DataCollector enqueueMessages, + DataCollector dequeueMessages, int delayLevel) { + // Check whether the consumption is complete + Collection unConsumedMessages = waitForMessageConsume(enqueueMessages, dequeueMessages, + (TIMEOUT + DelayConf.DELAY_LEVEL[delayLevel - 1]) * 1000L, 1); if (unConsumedMessages.size() > 0) { - Assertions.fail(String.format("以下%s条消息未被消费: %s", unConsumedMessages.size(), unConsumedMessages)); + Assertions.fail(String.format("The following %s messages are not consumed: %s", unConsumedMessages.size(), + unConsumedMessages)); } - //检查是否消费延迟性 + // Check for consumption delay HashMap delayUnExcept = checkDelay(dequeueMessages, 5); StringBuilder sb = new StringBuilder(); - sb.append("以下消息不符合延迟要求 \n"); + sb.append("The following messages do not meet the delay requirements \n"); for (String msg : delayUnExcept.keySet()) { sb.append(msg).append(" , interval:").append(delayUnExcept.get(msg)).append("\n"); } @@ -155,36 +230,42 @@ public static void verifyDelayMessage(DataCollector enqueueMessages, } /** - * @param enqueueMessages 发送入队消息集合 - * @param dequeueMessages 消费出队消息集合 - * @param delayTime 延迟时间 - * @param count 不被消费的数量 + * @param enqueueMessages collection of enqueued messages sent + * @param dequeueMessages collection of dequeued messages consumed + * @param delayTime delay level + * @param count the amount of messages that are not consumed */ - public static void verifyDelayMessageWithUnConsumeCount(DataCollector enqueueMessages, DataCollector dequeueMessages, int delayTime, int count) { - //检查是否消费完成 - Collection unConsumedMessages = waitForMessageConsume(enqueueMessages, dequeueMessages, (TIMEOUT + delayTime) * 1000L, 1); + public static void verifyDelayMessageWithUnConsumeCount(DataCollector enqueueMessages, + DataCollector dequeueMessages, int delayTime, int count) { + // Check whether consumption is completed + Collection unConsumedMessages = waitForMessageConsume(enqueueMessages, dequeueMessages, + (TIMEOUT + delayTime) * 1000L, 1); if (unConsumedMessages.size() > count) { - Assertions.fail(String.format("以下%s条消息未被消费: %s", unConsumedMessages.size(), unConsumedMessages)); + Assertions.fail(String.format("The following %s messages are not consumed: %s", unConsumedMessages.size(), + unConsumedMessages)); } - //检查是否消费延迟性 + // Check whether consumption is delayed HashMap delayUnExcept = checkDelay(dequeueMessages, TIMEOUT + 5); StringBuilder sb = new StringBuilder(); - sb.append("以下消息不符合延迟要求 \n"); - //时间戳格式化 + sb.append("The following messages do not meet the delay requirements.\n"); + // 时间戳格式化 SimpleDateFormat date = new SimpleDateFormat("ss"); for (String msg : delayUnExcept.keySet()) { - sb.append(msg).append(" , interval:").append("相差" + date.format(new Date(Long.parseLong(String.valueOf(delayUnExcept.get(msg))))) + "秒").append("\n"); + sb.append(msg).append(" , interval:").append( + date.format(new Date(Long.parseLong(String.valueOf(delayUnExcept.get(msg))))) + "second difference") + .append("\n"); } Assertions.assertEquals(0, delayUnExcept.size(), sb.toString()); } /** - * @param enqueueMessages 发送入队消息集合 - * @param dequeueMessages 消费出队消息集合 - * @param delayTime 延迟时间 - * @param reconsumeTime 重试次数 + * @param enqueueMessages collection of enqueued messages sent + * @param dequeueMessages collection of dequeued messages consumed + * @param delayTime delay time + * @param reconsumeTime reconsume times */ - public static void verifyDelayMessageWithReconsumeTimes(DataCollector enqueueMessages, DataCollector dequeueMessages, int delayTime, int reconsumeTime) { + public static void verifyDelayMessageWithReconsumeTimes(DataCollector enqueueMessages, + DataCollector dequeueMessages, int delayTime, int reconsumeTime) { int flexibleTime = TIMEOUT; if (reconsumeTime == 1) { flexibleTime = flexibleTime + 10; @@ -195,22 +276,25 @@ public static void verifyDelayMessageWithReconsumeTimes(DataCollector unConsumedMessages = waitForMessageConsume(enqueueMessages, dequeueMessages, (flexibleTime + delayTime) * 1000L, 1); + // Check whether consumption is completed + Collection unConsumedMessages = waitForMessageConsume(enqueueMessages, dequeueMessages, + (flexibleTime + delayTime) * 1000L, 1); if (unConsumedMessages.size() > 0) { - Assertions.fail(String.format("以下%s条消息未被消费: %s", unConsumedMessages.size(), unConsumedMessages)); + Assertions.fail(String.format("The following %s messages are not consumed: %s", unConsumedMessages.size(), + unConsumedMessages)); } - //检查是否消费延迟性 + // Check whether consumption is delayed HashMap delayUnExcept = checkDelay(dequeueMessages, 5 + flexibleTime - 30); StringBuilder sb = new StringBuilder(); - sb.append("以下消息不符合延迟要求 \n"); + sb.append("The following messages do not meet the delay requirements \n"); for (String msg : delayUnExcept.keySet()) { sb.append(msg).append(" , interval:").append(delayUnExcept.get(msg)).append("\n"); } Assertions.assertEquals(0, delayUnExcept.size(), sb.toString()); } - public static void verifyNormalMessageWithReconsumeTimes(DataCollector enqueueMessages, DataCollector dequeueMessages, int reconsumeTime) { + public static void verifyNormalMessageWithReconsumeTimes(DataCollector enqueueMessages, + DataCollector dequeueMessages, int reconsumeTime) { int flexibleTime = TIMEOUT; if (reconsumeTime == 1) { flexibleTime = flexibleTime + 10; @@ -221,47 +305,59 @@ public static void verifyNormalMessageWithReconsumeTimes(DataCollector unConsumedMessages = waitForMessageConsume(enqueueMessages, dequeueMessages, flexibleTime * 1000L, 1); + // Check whether consumption is completed + Collection unConsumedMessages = waitForMessageConsume(enqueueMessages, dequeueMessages, + flexibleTime * 1000L, 1); if (unConsumedMessages.size() > 0) { - Assertions.fail(String.format("以下%s条消息未被消费: %s", unConsumedMessages.size(), unConsumedMessages)); + Assertions.fail(String.format("The following %s messages are not consumed: %s", unConsumedMessages.size(), + unConsumedMessages)); } } /** * 校验消息重试消费 * - * @param enqueueMessages 发送的消息列表 - * @param dequeueAllMessages 消费的消息列表 - * @param consumedTimes 重复消费的次数 + * @param enqueueMessages collection of enqueued messages sent + * @param dequeueAllMessages collection of dequeued messages consumed + * @param consumedTimes times of repeated consumption */ - public static void verifyRetryConsume(DataCollector enqueueMessages, DataCollector dequeueAllMessages, int consumedTimes) { - Collection unConsumedMessages = waitForMessageConsume(enqueueMessages, dequeueAllMessages, TIMEOUT * 1000L, consumedTimes); + public static void verifyRetryConsume(DataCollector enqueueMessages, + DataCollector dequeueAllMessages, int consumedTimes) { + Collection unConsumedMessages = waitForMessageConsume(enqueueMessages, dequeueAllMessages, + TIMEOUT * 1000L, consumedTimes); if (unConsumedMessages.size() > 0) { - Assertions.fail(String.format("以下%s条消息未被消费: %s", unConsumedMessages.size(), unConsumedMessages)); + Assertions.fail(String.format("The following %s messages are not consumed: %s", unConsumedMessages.size(), + unConsumedMessages)); } } /** - * 事务消息检查器 + * Check transaction message * - * @param enqueueMessages 发送入队消息集合 - * @param dequeueMessages 消费出队消息集合 + * @param enqueueMessages collection of enqueued messages sent + * @param dequeueMessages collection of dequeued messages consumed */ - public static void checkTransactionMessage(DataCollector enqueueMessages, DataCollector dequeueMessages) { - Collection unConsumedMessages = waitForMessageConsume(enqueueMessages, dequeueMessages, TIMEOUT * 1000L, 1); + public static void checkTransactionMessage(DataCollector enqueueMessages, + DataCollector dequeueMessages) { + Collection unConsumedMessages = waitForMessageConsume(enqueueMessages, dequeueMessages, + TIMEOUT * 1000L, 1); if (unConsumedMessages.size() > 0) { - Assertions.fail(String.format("以下%s条消息未被消费: %s", unConsumedMessages.size(), unConsumedMessages)); + Assertions.fail(String.format("The following %s messages are not consumed: %s", unConsumedMessages.size(), + unConsumedMessages)); } } - public static void verifyConsumeFailed(DataCollector enqueueMessages, DataCollector dequeueMessages, Integer reconsumeTimes) { - Collection unConsumedMessages = waitForMessageConsume(enqueueMessages, dequeueMessages, TIMEOUT * 1000L, reconsumeTimes + 1); + public static void verifyConsumeFailed(DataCollector enqueueMessages, + DataCollector dequeueMessages, Integer reconsumeTimes) { + Collection unConsumedMessages = waitForMessageConsume(enqueueMessages, dequeueMessages, + TIMEOUT * 1000L, reconsumeTimes + 1); if (unConsumedMessages.size() > 0) { - //Assertions.fail(String.format("以下%s条消息未被消费: %s", unConsumedMessages.size(), unConsumedMessages)); - logger.warn(String.format("以下%s条消息未被消费: %s", unConsumedMessages.size(), unConsumedMessages)); + // Assertions.fail(String.format("The following %s messages are not consumed: + // %s", unConsumedMessages.size(), unConsumedMessages)); + logger.warn(String.format("The following %s messages are not consumed: %s", unConsumedMessages.size(), + unConsumedMessages)); } else { - Assertions.fail("消息全部被消费"); + Assertions.fail("All messages are consumed"); } } @@ -286,25 +382,26 @@ public static void verifyBatchSize(List everyConsumeResult, int batchCo } } } - Assertions.assertTrue(result, "批量消费校验失败"); + Assertions.assertTrue(result, "Batch consumption verification failed"); } /** - * 校验负载均衡 + * Check load balancing * - * @param msgSize 消费到的消息条数 - * @param recvSize 每个客户端消费到的消息条数 + * @param msgSize number of messages consumed + * @param recvSize the number of messages consumed by each client */ public static void verifyBalance(int msgSize, long... recvSize) { Assertions.assertTrue(verifyBalance(msgSize, 0.1f, recvSize), "客户端负载不均衡 " + Arrays.toString(recvSize)); - //return verifyBalance(msgSize, 0.1f, recvSize); + // return verifyBalance(msgSize, 0.1f, recvSize); } private static boolean verifyBalance(int msgSize, float error, long... recvSize) { boolean balance = true; - int evenSize = msgSize / recvSize.length; //平均值 + int evenSize = msgSize / recvSize.length; // average value for (long size : recvSize) { - //如果消费到的消息比平均值大于误差则不算均衡 + // If the message consumed is greater than the average error, it is not + // considered balanced. if (Math.abs(size - evenSize) > error * evenSize) { balance = false; logger.error("msgSize:{}, recvSize:{}, not balance!", msgSize, recvSize); @@ -319,10 +416,12 @@ private static HashMap checkDelay(DataCollector dequeu Collection receivedMessages = dequeueMessages.getAllData(); for (MessageExt receivedMessage : receivedMessages) { long startDeliverTime = Long.parseLong(receivedMessage.getUserProperty("startDeliverTime")); - //判断当前时间跟分发时间，如果相差在5s内，则满足要求 + // Determine the current time and distribution time. If the difference is within + // 5 seconds, the requirement is met. long bornTimestamp = receivedMessage.getBornTimestamp(); - //if () - if (Math.abs(startDeliverTime - bornTimestamp) / 1000 > DelayConf.DELAY_LEVEL[receivedMessage.getDelayTimeLevel() - 1] + offset) { + + if (Math.abs(startDeliverTime - bornTimestamp) + / 1000 > DelayConf.DELAY_LEVEL[receivedMessage.getDelayTimeLevel() - 1] + offset) { map.put(receivedMessage.getMsgId(), (startDeliverTime - bornTimestamp) / 1000); } } @@ -330,10 +429,10 @@ private static HashMap checkDelay(DataCollector dequeu } /** - * 检查消息的顺序性 + * Check the order of messages * - * @param dequeueMessages 收到的消息集合 - * @return 是否分区顺序 + * @param dequeueMessages collection of received messages + * @return partition order or not */ private static boolean checkOrder(DataCollector dequeueMessages) { Collection receivedMessages = dequeueMessages.getAllData(); @@ -354,22 +453,24 @@ private static boolean checkOrder(DataCollector dequeueMessages) { } /** - * 检查发送的消息是否都被消费 + * Check whether all sent messages have been consumed * - * @param enqueueMessages 发送入队消息集合 - * @param dequeueMessages 消费出队消息集合 - * @param timeoutMills 检查超时时间 + * @param enqueueMessages collection of enqueued messages sent + * @param dequeueMessages collection of dequeued messages consumed + * @param timeoutMills check timeout * @param consumedTimes - * @return 未被消费的消息集合 + * @return collection of unconsumed messages */ - private static Collection waitForMessageConsume(DataCollector enqueueMessages, DataCollector dequeueMessages, Long timeoutMills, Integer consumedTimes) { + private static Collection waitForMessageConsume(DataCollector enqueueMessages, + DataCollector dequeueMessages, Long timeoutMills, Integer consumedTimes) { logger.info("Set timeout: {}ms", timeoutMills); Collection sendMessages = enqueueMessages.getAllData(); long currentTime = System.currentTimeMillis(); while (!sendMessages.isEmpty()) { - //logger.info("param1:{}, param2:{}", enqueueMessages.getDataSize(), dequeueMessages.getDataSize()); + // logger.info("param1:{}, param2:{}", enqueueMessages.getDataSize(), + // dequeueMessages.getDataSize()); List receivedMessagesCopy = new ArrayList<>(dequeueMessages.getAllData()); Iterator iter = sendMessages.iterator(); while (iter.hasNext()) { @@ -377,8 +478,10 @@ private static Collection waitForMessageConsume(DataCollector { - if (msg.getUserProperty("UNIQ_KEY") != null && !msg.getUserProperty("UNIQ_KEY").equals(msg.getMsgId())) { - return msg.getUserProperty("UNIQ_KEY").equals(message.getMsgId()) || msg.getMsgId().equals(message.getMsgId()); + if (msg.getUserProperty("UNIQ_KEY") != null + && !msg.getUserProperty("UNIQ_KEY").equals(msg.getMsgId())) { + return msg.getUserProperty("UNIQ_KEY").equals(message.getMsgId()) + || msg.getMsgId().equals(message.getMsgId()); } return msg.getMsgId().equals(message.getMsgId()); }) @@ -386,15 +489,19 @@ private static Collection waitForMessageConsume(DataCollector 0 && getRepeatedTimes(receivedMessagesCopy, message) == consumedTimes) { iter.remove(); } else if (getRepeatedTimes(receivedMessagesCopy, message) > consumedTimes) { - Assertions.fail(String.format("消费到的重试消息多于预期（包含一条原始消息），Except:%s, Actual:%s, MsgId:%s", consumedTimes, getRepeatedTimes(receivedMessagesCopy, message), message.getMsgId())); - //logger.error("消费到的重试消息多于预期，Except:{}, Actual:{}", consumedTimes, getRepeatedTimes(receivedMessagesCopy, message)); + Assertions.fail(String.format( + "More retry messages than expected were consumed (including one original message)，Except:%s, Actual:%s, MsgId:%s", + consumedTimes, getRepeatedTimes(receivedMessagesCopy, message), message.getMsgId())); + // logger.error("More retry messages than expected were consumed，Except:{}, + // Actual:{}", consumedTimes, getRepeatedTimes(receivedMessagesCopy, message)); } } if (sendMessages.isEmpty()) { break; } if (System.currentTimeMillis() - currentTime >= timeoutMills) { - logger.error("Timeout but not received all send messages, not received msg: {}\n received msg:{}\n", sendMessages, receivedMessagesCopy); + logger.error("Timeout but not received all send messages, not received msg: {}\n received msg:{}\n", + sendMessages, receivedMessagesCopy); break; } TestUtils.waitForMoment(500L); @@ -405,7 +512,9 @@ private static Collection waitForMessageConsume(DataCollector recvMsgs, MessageExt msg) { int count = 0; for (MessageExt recvMsg : recvMsgs) { - if (recvMsg.getUserProperty("UNIQ_KEY") != null && !recvMsg.getUserProperty("UNIQ_KEY").equals(recvMsg.getMsgId()) && !recvMsg.getMsgId().equals(msg.getMsgId())) { + if (recvMsg.getUserProperty("UNIQ_KEY") != null + && !recvMsg.getUserProperty("UNIQ_KEY").equals(recvMsg.getMsgId()) + && !recvMsg.getMsgId().equals(msg.getMsgId())) { if (recvMsg.getUserProperty("UNIQ_KEY").equals(msg.getMsgId())) { count++; } @@ -417,19 +526,433 @@ private static synchronized int getRepeatedTimes(Collection recvMsgs } /** - * 校验一段时间内没有消费 + * Verify that there is no consumption within a certain period of time * * @param receivedMessages - * @param timeout 时间 + * @param timeout time */ public static void waitForConsumeFailed(DataCollector receivedMessages, int timeout) { long currentTime = System.currentTimeMillis(); while (currentTime + timeout * 1000L > System.currentTimeMillis()) { if (receivedMessages.getDataSize() > 0) { - Assertions.fail("消费到消息"); + Assertions.fail("Consume the message"); break; } TestUtils.waitForSeconds(5); } } + + public static void tryReceiveOnce(DefaultLitePullConsumer consumer) { + tryReceiveOnce(consumer, false, false); + } + + public static void tryReceiveOnce(DefaultMQPullConsumer consumer, String topic, String tag, int maxNums) { + tryReceiveOnce(consumer, topic, tag, maxNums, false, false); + } + + public static void tryReceiveOnce(DefaultMQPullConsumer consumer, String topic, String tag, int maxNums, + Boolean useExistTopic, Boolean useExistGid) { + Set messageQueues = null; + try { + messageQueues = consumer.fetchSubscribeMessageQueues(topic); + } catch (MQClientException e) { + Assertions.fail("Fail to fetchSubscribeMessageQueues"); + } + + long start = System.currentTimeMillis(); + if (!useExistTopic || !useExistGid) { + for (int i = 0; i < 5; i++) { + logger.info("Try pulling a message once"); + Set finalMessageQueues = messageQueues; + CompletableFuture[] futures = new CompletableFuture[messageQueues.size()]; + int mqCount = 0; + for (MessageQueue mq : finalMessageQueues) { + CompletableFuture future = CompletableFuture.supplyAsync(() -> { + try { + long offset = consumer.fetchConsumeOffset(mq, false); + if (offset < 0) + return null; + boolean shouldContinue = true; + while (shouldContinue) { + PullResult pullResult = consumer.pull(mq, tag, offset, maxNums); + switch (pullResult.getPullStatus()) { + case FOUND: + List messages = pullResult.getMsgFoundList(); + for (MessageExt message : messages) { + receivedIndex.getAndIncrement(); + logger.info("MessageId:{}, Body:{}, Property:{}, Retry:{}", + message.getMsgId(), + StandardCharsets.UTF_8.decode(ByteBuffer.wrap(message.getBody())), + message.getProperties(), message.getReconsumeTimes()); + } + offset = pullResult.getNextBeginOffset(); + consumer.updateConsumeOffset(mq, offset); + break; + case NO_MATCHED_MSG: + shouldContinue = false; // Exit the loop when there is no matching message + break; + case NO_NEW_MSG: + shouldContinue = false; // Exit the loop when there are no new messages + break; + case OFFSET_ILLEGAL: + shouldContinue = false; // Exit loop when offset is illegal + break; + default: + break; + } + } + } catch (MQBrokerException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } catch (RemotingException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } catch (InterruptedException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } catch (MQClientException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } + return null; + }); + futures[mqCount++] = future; + } + try { + CompletableFuture.allOf(futures).get(6, TimeUnit.SECONDS); + } catch (Exception e) { + e.printStackTrace(); + Assertions.fail("receive response count not match"); + } + } + } + logger.info("receive server response, cost={}ms", System.currentTimeMillis() - start); + } + + public static void tryReceiveOnce(DefaultLitePullConsumer consumer, Boolean useExistTopic, Boolean useExistGid) { + long start = System.currentTimeMillis(); + if (!useExistTopic || !useExistGid) { + CompletableFuture[] cfs = new CompletableFuture[5]; + for (int i = 0; i < 5; i++) { + logger.info("Try pulling a message once"); + int finalI = i; + CompletableFuture future = CompletableFuture.supplyAsync(() -> consumer.poll(2000)) + .thenAcceptAsync(extList -> { + if (extList.size() > 0) { + for (MessageExt ext : extList) { + receivedIndex.getAndIncrement(); + logger.info("MessageId:{}, Body:{}, Property:{}, Index:{}, Retry:{}", + ext.getMsgId(), + StandardCharsets.UTF_8.decode(ByteBuffer.wrap(ext.getBody())), + ext.getProperties(), finalI, ext.getReconsumeTimes()); + } + } + }); + cfs[i] = future; + } + try { + CompletableFuture.allOf(cfs).get(30, TimeUnit.SECONDS); + logger.info("receive server response, cost={}ms", System.currentTimeMillis() - start); + } catch (Exception e) { + e.printStackTrace(); + Assertions.fail("receive response count not match"); + } + } + } + + private static synchronized int getRepeatedTimes(Collection recvMsgs, String enqueueMessageId) { + int count = 0; + for (MessageExt recvMsg : recvMsgs) { + if (recvMsg.getMsgId().equals(enqueueMessageId)) { + count++; + } + } + return count; + } + + /** + * Verifying Cluster Consumption + * + * @param enqueueMessages All messages sent + * @param dequeueAllMessages Multiple consumer end, consumption of all messages + */ + @SafeVarargs + public static void verifyClusterConsume(DataCollector enqueueMessages, + DataCollector... dequeueAllMessages) { + long currentTime = System.currentTimeMillis(); + List sendMessagesCopy = new ArrayList<>(enqueueMessages.getAllData()); + + while (!sendMessagesCopy.isEmpty()) { + Collection noDupMsgs = new ArrayList<>(); + for (DataCollector messages : dequeueAllMessages) { + noDupMsgs.addAll(messages.getAllData()); + logger.info("consumer received message: {}", messages.getDataSize()); + } + logger.info("sendMessagesCopy left: {}", sendMessagesCopy.size()); + + List receivedMessagesCopy = new ArrayList<>(noDupMsgs); + Iterator iter = sendMessagesCopy.iterator(); + while (iter.hasNext()) { + MessageExt messageExt = iter.next(); + String messageId = messageExt.getMsgId(); + long msgCount = receivedMessagesCopy.stream().filter(msg -> msg.getMsgId().equals(messageId)).count(); + if (msgCount > 0 && getRepeatedTimes(receivedMessagesCopy, messageId) == 1) { + iter.remove(); + } + } + if (sendMessagesCopy.isEmpty()) { + break; + } + if (System.currentTimeMillis() - currentTime >= 60000L) { + logger.error("Timeout but not received all send messages, not received msg: {}\n received msg:{}\n", + sendMessagesCopy, receivedMessagesCopy); + break; + } + TestUtils.waitForMoment(500L); + } + + Assertions.assertEquals(0, sendMessagesCopy.size(), String + .format("The following %s messages are not consumed: %s", sendMessagesCopy.size(), sendMessagesCopy)); + + } + + /** + * Validation of sql attribute filtering + * + * @param enqueueMessages A message sent + * @param dequeueMessages News of consumption + * @param props The desired attribute condition is not met + */ + public static void verifyNormalMessageWithUserProperties(DataCollector enqueueMessages, + DataCollector dequeueMessages, HashMap props, int expectedUnrecvMsgNum) { + Collection unConsumedMessages = waitForMessageConsume(enqueueMessages, dequeueMessages, + TIMEOUT * 1000L, 1); + Collection recvMsgs = dequeueMessages.getAllData(); + for (MessageExt unConsumedMessage : recvMsgs) { + for (Map.Entry entry : props.entrySet()) { + Map msgProperties = unConsumedMessage.getProperties(); + for (Map.Entry property : msgProperties.entrySet()) { + if (property.getKey().equals(entry.getKey()) && property.getValue().equals(entry.getValue())) { + Assertions.fail( + "sql attribute filtering is not in effect, consuming messages to other attributes," + + unConsumedMessage.getProperties().toString()); + } + } + } + } + if (unConsumedMessages.size() != expectedUnrecvMsgNum) { + Assertions.fail("Failed to consume all the sent data by sql filter"); + } + } + + public static void waitLitePullReceiveThenAck(RMQNormalProducer producer, DefaultLitePullConsumer consumer, + String topic, String tag) { + Assertions.assertFalse(consumer.isAutoCommit()); + ArrayList assignList = null; + try { + assignList = new ArrayList<>(consumer.fetchMessageQueues(topic)); + } catch (MQClientException e) { + Assertions.fail("PullConsumer fetchMessageQueues error"); + } + Assertions.assertNotNull(assignList); + consumer.assign(assignList); + + long endTime = System.currentTimeMillis() + TIMEOUT * 1000; + Collection sendCollection = producer.getEnqueueMessages().getAllData(); + try { + while (endTime > System.currentTimeMillis()) { + final List extList = consumer.poll(); + if (extList.size() > 0) { + for (MessageExt messageExt : extList) { + receivedIndex.getAndIncrement(); + String tags = messageExt.getTags(); + FilterUtils.inTags(tags, tag); + logger.info("MessageId:{}, Body:{}, tag:{}, Property:{}, Index:{}", messageExt.getMsgId(), + StandardCharsets.UTF_8.decode(ByteBuffer.wrap(messageExt.getBody())), + messageExt.getTags(), messageExt.getProperties(), receivedIndex.get()); + sendCollection + .removeIf(sendMessageExt -> sendMessageExt.getMsgId().equals(messageExt.getMsgId())); + } + } + consumer.commitSync(); + logger.info("Pull message: {} bar, remaining unconsumed message: {} bar", extList.size(), + sendCollection.size()); + if (sendCollection.size() == 0) { + break; + } + } + Assertions.assertTrue(sendCollection.size() == 0, String.format("Remaining [%s] unconsumed messages: %s", + sendCollection.size(), Arrays.toString(sendCollection.toArray()))); + } catch (Exception e) { + Assertions.fail(e.getMessage()); + } + } + + public static void waitPullReceiveThenAck(RMQNormalProducer producer, DefaultMQPullConsumer consumer, String topic, + String tag, int maxNums) { + Set messageQueues = null; + try { + messageQueues = consumer.fetchSubscribeMessageQueues(topic); + } catch (MQClientException e) { + Assertions.fail("Fail to fetchSubscribeMessageQueues"); + } + + // long endTime = System.currentTimeMillis() + TIMEOUT * 1000; + Collection sendCollection = Collections + .synchronizedCollection(producer.getEnqueueMessages().getAllData()); + Set finalMessageQueues = messageQueues; + CompletableFuture[] futures = new CompletableFuture[messageQueues.size()]; + int mqCount = 0; + for (MessageQueue mq : finalMessageQueues) { + CompletableFuture future = CompletableFuture.supplyAsync(() -> { + try { + long offset = consumer.fetchConsumeOffset(mq, false); + if (offset < 0) + return null; + boolean shouldContinue = true; + while (shouldContinue) { + PullResult pullResult = consumer.pull(mq, tag, offset, maxNums); + switch (pullResult.getPullStatus()) { + case FOUND: + List messages = pullResult.getMsgFoundList(); + for (MessageExt message : messages) { + receivedIndex.getAndIncrement(); + logger.info("MessageId:{}, Body:{}, Property:{}, Retry:{}", message.getMsgId(), + StandardCharsets.UTF_8.decode(ByteBuffer.wrap(message.getBody())), + message.getProperties(), message.getReconsumeTimes()); + offset = message.getQueueOffset() + 1; + consumer.updateConsumeOffset(mq, offset); + sendCollection + .removeIf(messageExt -> messageExt.getMsgId().equals(message.getMsgId())); + } + break; + case NO_MATCHED_MSG: + shouldContinue = false; // Exit the loop when there is no matching + break; + case NO_NEW_MSG: + shouldContinue = false; // Exit the loop when there are no new messages + break; + case OFFSET_ILLEGAL: + shouldContinue = false; // Exit loop when offset is illegal + break; + default: + break; + } + } + } catch (MQBrokerException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } catch (RemotingException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } catch (InterruptedException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } catch (MQClientException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } + return null; + }); + futures[mqCount++] = future; + } + try { + CompletableFuture.allOf(futures).get(6, TimeUnit.SECONDS); + } catch (Exception e) { + e.printStackTrace(); + Assertions.fail("receive response count not match"); + } + } + + public static void waitFIFOReceiveThenAck(RMQNormalProducer producer, DefaultMQPullConsumer consumer, String topic, + String tag, int maxNums) { + Set messageQueues = null; + try { + messageQueues = consumer.fetchSubscribeMessageQueues(topic); + } catch (MQClientException e) { + Assertions.fail("Fail to fetchSubscribeMessageQueues"); + } + + long endTime = System.currentTimeMillis() + TIMEOUT * 1000; + Collection sendCollection = producer.getEnqueueMessages().getAllData(); + ConcurrentHashMap> map = new ConcurrentHashMap<>(); + + Set finalMessageQueues = messageQueues; + CompletableFuture[] futures = new CompletableFuture[messageQueues.size()]; + int mqCount = 0; + for (MessageQueue mq : finalMessageQueues) { + CompletableFuture future = CompletableFuture.supplyAsync(() -> { + try { + long offset = consumer.fetchConsumeOffset(mq, false); + if (offset < 0) + return null; + boolean shouldContinue = true; + while (shouldContinue) { + PullResult pullResult = consumer.pull(mq, tag, offset, maxNums); + switch (pullResult.getPullStatus()) { + case FOUND: + List messages = pullResult.getMsgFoundList(); + for (MessageExt message : messages) { + receivedIndex.getAndIncrement(); + logger.info("MessageId:{}, Body:{}, Property:{}, Retry:{}", message.getMsgId(), + StandardCharsets.UTF_8.decode(ByteBuffer.wrap(message.getBody())), + message.getProperties(), message.getReconsumeTimes()); + offset = message.getQueueOffset() + 1; + consumer.updateConsumeOffset(mq, offset); + sendCollection + .removeIf(messageExt -> messageExt.getMsgId().equals(message.getMsgId())); + String shardingKey = String.valueOf(mq.getQueueId()); + LinkedList messagesList; + if (map.containsKey(shardingKey)) { + messagesList = map.get(shardingKey); + messagesList.add(message); + } else { + messagesList = new LinkedList<>(); + messagesList.add(message); + map.put(shardingKey, messagesList); + } + if (sendCollection.size() == 0) { + Assertions.assertTrue(checkOrderMessage(map), "Consumption is not sequential"); + } + } + break; + case NO_MATCHED_MSG: + shouldContinue = false; // Exit the loop when there is no matching message + break; + case NO_NEW_MSG: + shouldContinue = false; // Exit the loop when there are no new messages + break; + case OFFSET_ILLEGAL: + shouldContinue = false; // Exit loop when offset is illegal + break; + default: + break; + } + } + } catch (MQBrokerException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } catch (RemotingException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } catch (InterruptedException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } catch (MQClientException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } + return null; + }); + futures[mqCount++] = future; + } + try { + CompletableFuture.allOf(futures).get(6, TimeUnit.SECONDS); + } catch (Exception e) { + e.printStackTrace(); + Assertions.fail("receive response count not match"); + } + Assertions.assertTrue(sendCollection.size() == 0, String.format("Remaining [%s] unconsumed messages: %s", + sendCollection.size(), Arrays.toString(sendCollection.toArray()))); + } } diff --git a/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/data/collect/DataCollector.java b/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/data/collect/DataCollector.java index d991ad4..8f8d51c 100644 --- a/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/data/collect/DataCollector.java +++ b/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/data/collect/DataCollector.java @@ -20,73 +20,73 @@ import java.util.Collection; public interface DataCollector { /** - * 计数归零 + * reset datas' size to zero */ void resetData(); /** - * 取得所有的收集数据，包含重复的数据 + * Get all collected data, including duplicate data * * @return */ Collection getAllData(); /** - * 取得收集的非重复数据 + * Get collected non-duplicate data * * @return */ Collection getAllDataWithoutDuplicate(); /** - * 增加数据 + * Add data */ void addData(T data); /** - * 取得非重复数据的总个数 + * Gets the total number of non-duplicate data * * @return */ long getDataSizeWithoutDuplicate(); /** - * 取得所有数据的总个数，包括重复数据 + * Gets the total number of all data, including duplicates * * @return */ long getDataSize(); /** - * 验证一条数据是否存在重复 + * Verify that a piece of data is duplicated * * @return */ boolean isRepeatedData(T data); /** - * 取得一条数据重复出现的次数 + * Gets the number of times a piece of data is repeated * * @return */ int getRepeatedTimeForData(T data); /** - * 移除一条数据 + * Remove a piece of data * * @return */ void removeData(T data); /** - * 锁定计数器，使其不再增加，但是可以删除 + * Lock the counter so that it no longer increases, but can be deleted * * @return */ void lockIncrement(); /** - * 接触计数器锁定，使其可以增加和删除 + * The contact counter locks so that it can be added and removed * * @return */ diff --git a/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/data/collect/DataCollectorManager.java b/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/data/collect/DataCollectorManager.java index 818728d..5b9405b 100644 --- a/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/data/collect/DataCollectorManager.java +++ b/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/data/collect/DataCollectorManager.java @@ -17,7 +17,6 @@ package org.apache.rocketmq.utils.data.collect; - import org.apache.rocketmq.utils.data.collect.impl.ListDataCollectorImpl; import org.apache.rocketmq.utils.data.collect.impl.MapDataCollectorImpl; diff --git a/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/data/collect/impl/ListDataCollectorImpl.java b/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/data/collect/impl/ListDataCollectorImpl.java index f0d480d..996f2e3 100644 --- a/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/data/collect/impl/ListDataCollectorImpl.java +++ b/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/data/collect/impl/ListDataCollectorImpl.java @@ -24,7 +24,8 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; - +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; public class ListDataCollectorImpl implements DataCollector { diff --git a/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/data/collect/impl/MapDataCollectorImpl.java b/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/data/collect/impl/MapDataCollectorImpl.java index de037dd..2284e79 100644 --- a/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/data/collect/impl/MapDataCollectorImpl.java +++ b/java/e2e-v4/src/main/java/org/apache/rocketmq/utils/data/collect/impl/MapDataCollectorImpl.java @@ -17,7 +17,6 @@ package org.apache.rocketmq.utils.data.collect.impl; - import org.apache.rocketmq.utils.data.collect.DataCollector; import java.util.ArrayList; diff --git a/java/e2e-v4/src/test/java/org/apache/rocketmq/client/consumer/ConsumerGroupTest.java b/java/e2e-v4/src/test/java/org/apache/rocketmq/client/consumer/ConsumerGroupTest.java new file mode 100644 index 0000000..999f0d3 --- /dev/null +++ b/java/e2e-v4/src/test/java/org/apache/rocketmq/client/consumer/ConsumerGroupTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.consumer; + +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.enums.TESTSET; +import org.apache.rocketmq.frame.BaseOperate; +import org.junit.jupiter.api.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +@Tag(TESTSET.CLIENT) +public class ConsumerGroupTest extends BaseOperate { + private static final Logger log = LoggerFactory.getLogger(ConsumerGroupTest.class); + private static String topic; + private static String className; + private DefaultLitePullConsumer consumer; + + @BeforeAll + public static void setUpAll() { + className = ConsumerGroupTest.class.getName(); + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + topic = getTopic(methodName); + } + + @AfterEach + public void tearDown() { + if (consumer != null) { + consumer.shutdown(); + } + } + + @Test + @DisplayName("Use the built-in ConsumerGroup[DEFAULT_CONSUMER] to consume messages and expect consume failed") + public void testSystemInnerConsumerGroup() { + String groupId = "DEFAULT_CONSUMER"; + assertThrows(Exception.class, () -> { + consumer = new DefaultLitePullConsumer(groupId, rpcHook); + consumer.setNamesrvAddr(namesrvAddr); + consumer.subscribe(topic, "*"); + consumer.setPullBatchSize(20); + consumer.start(); + while (true) { + List messageExts = consumer.poll(); + log.info("MessageExt: {}", messageExts); + } + }, "Expected Start [SimpleConsumer] Exception to throw, but it didn't"); + } + +} diff --git a/java/e2e-v4/src/test/java/org/apache/rocketmq/client/consumer/PullConsumerInitTest.java b/java/e2e-v4/src/test/java/org/apache/rocketmq/client/consumer/PullConsumerInitTest.java new file mode 100644 index 0000000..dededa6 --- /dev/null +++ b/java/e2e-v4/src/test/java/org/apache/rocketmq/client/consumer/PullConsumerInitTest.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.consumer; + +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.enums.TESTSET; +import org.apache.rocketmq.frame.BaseOperate; +import org.junit.jupiter.api.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +@Tag(TESTSET.CLIENT) +public class PullConsumerInitTest extends BaseOperate { + private static final Logger log = LoggerFactory.getLogger(PullConsumerInitTest.class); + private static String topic; + private static String groupId; + + @BeforeAll + public static void setUpAll() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + topic = getTopic(methodName); + groupId = getGroupId(methodName); + } + + @BeforeEach + public void setUp() { + + } + + @AfterEach + public void tearDown() { + } + + @AfterAll + public static void tearDownAll() { + } + + @Test + @DisplayName("PullConsumer all parameters are set properly, expect start success") + public void testNormalSetting() { + try { + DefaultLitePullConsumer pullConsumer = new DefaultLitePullConsumer(groupId, rpcHook); + pullConsumer.setNamesrvAddr(namesrvAddr); + pullConsumer.subscribe(topic, "*"); + pullConsumer.setConsumerPullTimeoutMillis(10 * 1000); + pullConsumer.start(); + pullConsumer.shutdown(); + } catch (Exception e) { + Assertions.fail("Start [PullConsumer] failed, expected success, message: " + e.getMessage()); + } + } + + @Test + @DisplayName("Without setting 'Endpoint Configuration' of the consumer client, expect start failed") + public void testNoClientConfiguration() { + assertThrows(Exception.class, () -> { + DefaultLitePullConsumer pullConsumer = new DefaultLitePullConsumer(groupId); + pullConsumer.subscribe(topic, "*"); + pullConsumer.setConsumerPullTimeoutMillis(10 * 1000); + pullConsumer.start(); + }, "Expected Start [PullConsumer] Exception to throw, but it didn't"); + } + + @Test + @DisplayName("Without setting 'ConsumerGroup' of the consumer client, expect start failed") + public void testNoConsumerGroup() { + assertThrows(Exception.class, () -> { + DefaultLitePullConsumer pullConsumer = new DefaultLitePullConsumer(rpcHook); + pullConsumer.setNamesrvAddr(namesrvAddr); + pullConsumer.subscribe(topic, "*"); + pullConsumer.setConsumerPullTimeoutMillis(10 * 1000); + pullConsumer.start(); + }, "Expected Start [PullConsumer] Exception to throw, but it didn't"); + } + + @Disabled + @DisplayName("Without setting 'Subscription' of the consumer client, expect start failed") + public void testNoSubscription() { + assertThrows(Exception.class, () -> { + DefaultLitePullConsumer pullConsumer = new DefaultLitePullConsumer(groupId, rpcHook); + pullConsumer.setNamesrvAddr(namesrvAddr); + pullConsumer.setConsumerPullTimeoutMillis(10 * 1000); + pullConsumer.start(); + }, "Expected Start [PullConsumer] Exception to throw, but it didn't"); + } + + @Test + @DisplayName("Error setting 'SubscriptionExpressions' empty of the consumer client, except start failed") + public void testEmptySubscription() { + assertThrows(Exception.class, () -> { + DefaultLitePullConsumer pullConsumer = new DefaultLitePullConsumer(groupId, rpcHook); + pullConsumer.setNamesrvAddr(namesrvAddr); + String var1 = null; + String var2 = null; + pullConsumer.subscribe(var1, var2); + pullConsumer.start(); + }, "Expected Start [PullConsumer] Exception to throw, but it didn't"); + } + + @Disabled + @DisplayName("Error setting 'ConsumerPullTimeoutMillis=0' of the consumer client, except start failed") + public void testConsumerPullTimeoutMillisIs0s() { + DefaultLitePullConsumer pullConsumer = null; + try { + pullConsumer = new DefaultLitePullConsumer(groupId, rpcHook); + pullConsumer.setNamesrvAddr(namesrvAddr); + pullConsumer.subscribe(topic, "*"); + pullConsumer.setConsumerPullTimeoutMillis(0 * 1000); + pullConsumer.start(); + pullConsumer.poll(); + } catch (Exception e) { + e.printStackTrace(); + Assertions.fail("PullConsumer start failed"); + } + } + + @Test + @DisplayName("Setting 'Wait Duration = 10s',Expect the empty pull message request to return between 10s and 20s") + public void testAwaitDuration() { + DefaultLitePullConsumer pullConsumer = null; + try { + pullConsumer = new DefaultLitePullConsumer(groupId, rpcHook); + pullConsumer.setNamesrvAddr(namesrvAddr); + pullConsumer.subscribe(topic, "*"); + pullConsumer.setConsumerPullTimeoutMillis(10 * 1000); + pullConsumer.start(); + long startTime = System.currentTimeMillis(); + log.info("startTime: {}", startTime); + pullConsumer.poll(10 * 1000); + long endTime = System.currentTimeMillis(); + log.info("endTime: {}", endTime); + pullConsumer.shutdown(); + Assertions.assertTrue((endTime - startTime) > 10000 && (endTime - startTime) < 20000, + String.format("invoke method 'receive()' exception, startTime:%s, endTime:%s, intervalTime:%s", + startTime, endTime, endTime - startTime)); + } catch (Exception e) { + Assertions.fail("PullConsumer start exception"); + } + } +} diff --git a/java/e2e-v4/src/test/java/org/apache/rocketmq/client/consumer/PushConsumerInitTest.java b/java/e2e-v4/src/test/java/org/apache/rocketmq/client/consumer/PushConsumerInitTest.java new file mode 100644 index 0000000..fa555c6 --- /dev/null +++ b/java/e2e-v4/src/test/java/org/apache/rocketmq/client/consumer/PushConsumerInitTest.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.consumer; + +import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.enums.TESTSET; +import org.apache.rocketmq.frame.BaseOperate; +import org.apache.rocketmq.listener.rmq.concurrent.RMQNormalListener; +import org.apache.rocketmq.utils.TestUtils; +import org.junit.jupiter.api.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * PushConsumer client initialization use case + */ +@Tag(TESTSET.CLIENT) +public class PushConsumerInitTest extends BaseOperate { + private static final Logger log = LoggerFactory.getLogger(PushConsumerInitTest.class); + private static String topic; + private static String groupId; + + @BeforeAll + public static void setUpAll() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + topic = getTopic(methodName); + groupId = getGroupId(methodName); + TestUtils.waitForSeconds(2); + } + + @BeforeEach + public void setUp() { + + } + + @AfterEach + public void tearDown() { + } + + @AfterAll + public static void tearDownAll() { + } + + @Test + @DisplayName("PushConsumer all parameters are set properly, expect start success") + public void testNormalSetting() { + try { + DefaultMQPushConsumer pushConsumer = new DefaultMQPushConsumer(groupId, rpcHook, + new AllocateMessageQueueAveragely()); + pushConsumer.setNamesrvAddr(namesrvAddr); + pushConsumer.subscribe(topic, "*"); + pushConsumer.setConsumeThreadMax(20); + pushConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); + pushConsumer.setMessageListener(new RMQNormalListener()); + pushConsumer.start(); + pushConsumer.shutdown(); + } catch (Exception e) { + Assertions.fail("Start [PushConsumer] failed, expected success, message: " + e.getMessage()); + } + } + + @Test + @Tag(TESTSET.ACL) + @DisplayName("Error setting the 'EndPoint' of the consumer client,expect start failed") + public void testErrorNameserver() { + assertThrows(Exception.class, () -> { + DefaultMQPushConsumer pushConsumer = new DefaultMQPushConsumer(groupId, rpcHook, + new AllocateMessageQueueAveragely()); + pushConsumer.setNamesrvAddr("https://www.aliyun.com"); + pushConsumer.subscribe(topic, "*"); + pushConsumer.setConsumeThreadMax(20); + pushConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); + pushConsumer.setMessageListener(new RMQNormalListener()); + pushConsumer.start(); + pushConsumer.shutdown(); + }, "Expected Start [PushConsumer] ClientException to throw, but it didn't"); + } + + @Disabled + @DisplayName("Set the consumer client's topic error, expecting a message receiving failure to throw an Exception") + public void testErrorTopic() { + assertThrows(Exception.class, () -> { + DefaultMQPushConsumer pushConsumer = new DefaultMQPushConsumer(groupId, rpcHook, + new AllocateMessageQueueAveragely()); + pushConsumer.setNamesrvAddr(namesrvAddr); + pushConsumer.subscribe("topicNotExist", "*"); + pushConsumer.setConsumeThreadMax(20); + pushConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); + pushConsumer.setMessageListener(new RMQNormalListener()); + pushConsumer.start(); + pushConsumer.shutdown(); + }, "Expected Start [PushConsumer] ClientException to throw, but it didn't"); + } + + @Test + @DisplayName("Without setting ConsumerGroup, expect PushConsumer NPE exception to start") + public void testNoGroupId() { + assertThrows(Exception.class, () -> { + DefaultMQPushConsumer pushConsumer = new DefaultMQPushConsumer(rpcHook); + pushConsumer.setNamesrvAddr(namesrvAddr); + pushConsumer.subscribe(topic, "*"); + pushConsumer.setConsumeThreadMax(20); + pushConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); + pushConsumer.setMessageListener(new RMQNormalListener()); + pushConsumer.start(); + pushConsumer.shutdown(); + }, "Expected Start [PushConsumer] ClientException to throw, but it didn't"); + } + + @Disabled + @DisplayName("The 'Subscription' is not set, expect PushConsumer IllegalArgumentException exception to start") + public void testNoSubscription() { + assertThrows(Exception.class, () -> { + DefaultMQPushConsumer pushConsumer = new DefaultMQPushConsumer(groupId, rpcHook, + new AllocateMessageQueueAveragely()); + pushConsumer.setNamesrvAddr(namesrvAddr); + pushConsumer.setConsumeThreadMax(20); + pushConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); + pushConsumer.setMessageListener(new RMQNormalListener()); + pushConsumer.start(); + pushConsumer.shutdown(); + }, "Expected Start [PushConsumer] ClientException to throw, but it didn't"); + } + + @Test + @DisplayName("Set an empty Subscription, expecting PushConsumer IllegalArgumentException to be raised") + public void testEmptySubscription() { + assertThrows(Exception.class, () -> { + DefaultMQPushConsumer pushConsumer = new DefaultMQPushConsumer(groupId, rpcHook, + new AllocateMessageQueueAveragely()); + pushConsumer.setNamesrvAddr(namesrvAddr); + String var1 = null; + String var2 = null; + pushConsumer.subscribe(var1, var2); + pushConsumer.setConsumeThreadMax(20); + pushConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); + pushConsumer.setMessageListener(new RMQNormalListener()); + pushConsumer.start(); + pushConsumer.shutdown(); + }, "Expected Start [PushConsumer] ClientException to throw, but it didn't"); + } + + @Test + @DisplayName("The 'Endpoint Configuration' is not set. PushConsumer IllegalState exception is expected") + public void testNoClientConfiguration() { + assertThrows(Exception.class, () -> { + DefaultMQPushConsumer pushConsumer = new DefaultMQPushConsumer(groupId); + pushConsumer.subscribe(topic, "*"); + pushConsumer.setConsumeThreadMax(20); + pushConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); + pushConsumer.setMessageListener(new RMQNormalListener()); + pushConsumer.start(); + pushConsumer.shutdown(); + }, "Expected Start [PushConsumer] ClientException to throw, but it didn't"); + } + + @Test + @DisplayName("The 'MessageListener' is not set. PushConsumer MQClient exception is expected") + public void testNoListener() { + assertThrows(MQClientException.class, () -> { + DefaultMQPushConsumer pushConsumer = new DefaultMQPushConsumer(groupId, rpcHook, + new AllocateMessageQueueAveragely()); + pushConsumer.setNamesrvAddr(namesrvAddr); + pushConsumer.subscribe(topic, "*"); + pushConsumer.setConsumeThreadMax(20); + pushConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); + pushConsumer.start(); + pushConsumer.shutdown(); + }, "Expected Start [PushConsumer] ClientException to throw, but it didn't"); + } +} diff --git a/java/e2e-v4/src/test/java/org/apache/rocketmq/client/message/MessageAbnormalTest.java b/java/e2e-v4/src/test/java/org/apache/rocketmq/client/message/MessageAbnormalTest.java new file mode 100644 index 0000000..000d7a7 --- /dev/null +++ b/java/e2e-v4/src/test/java/org/apache/rocketmq/client/message/MessageAbnormalTest.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.message; + +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.enums.TESTSET; +import org.apache.rocketmq.factory.MessageFactory; +import org.apache.rocketmq.frame.BaseOperate; +import org.apache.rocketmq.utils.NameUtils; +import org.apache.rocketmq.utils.RandomUtils; +import org.junit.jupiter.api.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Test message properties + */ +@Tag(TESTSET.CLIENT) +public class MessageAbnormalTest extends BaseOperate { + private static final Logger log = LoggerFactory.getLogger(MessageAbnormalTest.class); + private static DefaultMQProducer producer; + private static String topic; + private String tag; + + @BeforeAll + public static void setUpAll() { + String className = MessageAbnormalTest.class.getName(); + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + topic = getTopic(methodName); + try { + producer = new DefaultMQProducer("MessageAbnormalTest", rpcHook); + producer.setNamesrvAddr(namesrvAddr); + producer.start(); + } catch (MQClientException e) { + throw new RuntimeException(e); + } + } + + @BeforeEach + public void setUp() { + tag = NameUtils.getRandomTagName(); + } + + @AfterEach + public void tearDown() { + } + + @AfterAll + public static void tearDownAll() { + + } + + // TODO + @Disabled + @DisplayName("producer invoke send(messageBody=\"\"), expect throw exception") + public void sendMsgBodyIsEmpty() { + assertThrows(Exception.class, () -> { + Message message = MessageFactory.buildMessage(topic, tag, ""); + producer.send(message); + }, "Send messages with a null character message, Expected send() to throw exception, but it didn't"); + } + + @Test + @DisplayName("producer invoke send(messageBody=null), expect build message throw exception") + public void sendMsgBodyIsNull() { + assertThrows(Exception.class, () -> { + Message message = MessageFactory.buildMessage(topic, tag, null); + producer.send(message); + }, "Send messages with a null character message, Expected build() to throw exception, but it didn't"); + } + + @Test + @DisplayName("producer invoke send(topic=\"\"), expect throw exception") + public void sendMsgTopicIsEmpty() { + assertThrows(Exception.class, () -> { + Message message = MessageFactory.buildMessage("", tag, RandomUtils.getStringByUUID()); + producer.send(message); + }, "Topic does not exist, Expected build() to throw exception, but it didn't"); + } + + @Test + @DisplayName("producer invoke send(topic=null), expect throw exception") + public void sendMsgTopicIsNull() { + assertThrows(Exception.class, () -> { + Message message = MessageFactory.buildMessage(null, tag, RandomUtils.getStringByUUID()); + producer.send(message); + }, "Topic does not exist,Expected build() to throw exception, but it didn't"); + } + + @Disabled + @DisplayName("producer invoke send(tag=null), expect build message throw exception") + public void sendMsgTagIsNull() { + assertThrows(Exception.class, () -> { + Message message = MessageFactory.buildMessage(topic, null, RandomUtils.getStringByUUID()); + producer.send(message); + }, "tag is null, Expected build() to throw exception, but it didn't"); + } + + @Disabled + @DisplayName("producer invoke send(tag=\"\"), expect build message throw exception") + public void sendMsgTagIsEmpty() { + assertThrows(Exception.class, () -> { + Message message = MessageFactory.buildMessage(topic, "", RandomUtils.getStringByUUID()); + producer.send(message); + }, "tag is blank, Expected build() to throw exception, but it didn't"); + } +} diff --git a/java/e2e-v4/src/test/java/org/apache/rocketmq/client/message/MessageBodyContentTest.java b/java/e2e-v4/src/test/java/org/apache/rocketmq/client/message/MessageBodyContentTest.java new file mode 100644 index 0000000..f340eeb --- /dev/null +++ b/java/e2e-v4/src/test/java/org/apache/rocketmq/client/message/MessageBodyContentTest.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.message; + +import org.apache.rocketmq.client.rmq.RMQNormalConsumer; +import org.apache.rocketmq.client.rmq.RMQNormalProducer; +import org.apache.rocketmq.enums.TESTSET; +import org.apache.rocketmq.factory.ConsumerFactory; +import org.apache.rocketmq.factory.ProducerFactory; +import org.apache.rocketmq.frame.BaseOperate; +import org.apache.rocketmq.listener.rmq.concurrent.RMQNormalListener; +import org.apache.rocketmq.utils.NameUtils; +import org.apache.rocketmq.utils.VerifyUtils; +import org.junit.jupiter.api.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Test message body + */ +@Tag(TESTSET.CLIENT) +public class MessageBodyContentTest extends BaseOperate { + private static final Logger log = LoggerFactory.getLogger(MessageBodyContentTest.class); + private String tag; + private static String topic; + private RMQNormalProducer producer; + private RMQNormalConsumer pushConsumer; + private RMQNormalConsumer pullConsumer; + + @BeforeAll + public static void setUpAll() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + topic = getTopic(methodName); + } + + @BeforeEach + public void setUp() { + tag = NameUtils.getRandomTagName(); + } + + @AfterEach + public void tearDown() { + if (producer != null) { + producer.shutdown(); + } + if (pushConsumer != null) { + pushConsumer.shutdown(); + } + if (pullConsumer != null) { + pullConsumer.shutdown(); + } + } + + @Test + @DisplayName("Send normal message, setting message body with space character, expect consume success") + public void testMessageBodyContentIsSpace() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String groupId = getGroupId(methodName); + + pushConsumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer.subscribeAndStart(topic, tag, new RMQNormalListener()); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + + String body = " "; + producer.send(topic, tag, body); + + Assertions.assertEquals(1, producer.getEnqueueMessages().getDataSize(), "send message failed"); + VerifyUtils.verifyNormalMessageWithBody(producer.getEnqueueMessages(), + pushConsumer.getListener().getDequeueMessages(), body); + } + + @Test + @DisplayName("Send normal message, setting message body with chinese character, expect consume success") + public void testMessageBodyContentIsChinese() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String groupId = getGroupId(methodName); + + pushConsumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer.subscribeAndStart(topic, tag, new RMQNormalListener()); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + + String body = "中文字符"; + producer.send(topic, tag, body); + + Assertions.assertEquals(1, producer.getEnqueueMessages().getDataSize(), "send message failed"); + VerifyUtils.verifyNormalMessageWithBody(producer.getEnqueueMessages(), + pushConsumer.getListener().getDequeueMessages(), body); + } + + @Test + @DisplayName("Send normal message, setting message body with emoji(😱) character, expect consume success ") + public void testMessageBodyContentIsEmoji() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String groupId = getGroupId(methodName); + + pushConsumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer.subscribeAndStart(topic, tag, new RMQNormalListener()); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + + String body = "😱"; + producer.send(topic, tag, body); + + Assertions.assertEquals(1, producer.getEnqueueMessages().getDataSize(), "send message failed"); + VerifyUtils.verifyNormalMessageWithBody(producer.getEnqueueMessages(), + pushConsumer.getListener().getDequeueMessages(), body); + } +} diff --git a/java/e2e-v4/src/test/java/org/apache/rocketmq/client/message/MessageKeyTest.java b/java/e2e-v4/src/test/java/org/apache/rocketmq/client/message/MessageKeyTest.java new file mode 100644 index 0000000..c3568ed --- /dev/null +++ b/java/e2e-v4/src/test/java/org/apache/rocketmq/client/message/MessageKeyTest.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.message; + +import org.apache.commons.lang3.RandomStringUtils; + +import org.apache.rocketmq.client.rmq.RMQNormalConsumer; +import org.apache.rocketmq.client.rmq.RMQNormalProducer; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.enums.TESTSET; +import org.apache.rocketmq.factory.ConsumerFactory; +import org.apache.rocketmq.factory.ProducerFactory; +import org.apache.rocketmq.frame.BaseOperate; +import org.apache.rocketmq.utils.NameUtils; +import org.apache.rocketmq.utils.RandomUtils; +import org.apache.rocketmq.utils.VerifyUtils; +import org.apache.rocketmq.listener.rmq.concurrent.RMQNormalListener; +import org.junit.jupiter.api.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Test message key + */ +@Tag(TESTSET.CLIENT) +public class MessageKeyTest extends BaseOperate { + private static final Logger log = LoggerFactory.getLogger(MessageKeyTest.class); + private static String topic; + private RMQNormalProducer producer; + private RMQNormalConsumer pushConsumer; + private RMQNormalConsumer pullConsumer; + + @BeforeAll + public static void setUpAll() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + topic = getTopic(methodName); + } + + @AfterEach + public void tearDown() { + if (producer != null) { + producer.shutdown(); + } + if (pushConsumer != null) { + pushConsumer.shutdown(); + } + if (pullConsumer != null) { + pullConsumer.shutdown(); + } + } + + @Disabled + @DisplayName("Message Key beyond 16KB, expect throw exception") + public void testMessageKeyBeyond16KB() { + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + String body = RandomStringUtils.randomAlphabetic(64); + String key = RandomStringUtils.randomAlphabetic(16 * 1024 + 1); + + Assertions.assertNotNull(producer); + assertThrows(Exception.class, () -> { + Message message = new Message(topic, "*", key, body.getBytes()); + producer.getProducer().send(message); + }, " message key beyond 16KB , expect throw exception but it didn't"); + } + + @Disabled + @DisplayName("Message Key equals 16KB, expect send success") + public void testMessageKeyEquals16KB() { + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + String body = RandomStringUtils.randomAlphabetic(64); + String key = RandomStringUtils.randomAlphabetic(16 * 1024); + Assertions.assertNotNull(producer); + + Message message = new Message(topic, "*", key, body.getBytes()); + producer.send(message); + } + + @Disabled + @DisplayName("Message Key contains invisible characters \u0000 ,expect throw exception") + public void testMessageKeyWithInvisibleCharacter() { + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + String body = RandomStringUtils.randomAlphabetic(64); + String key = "\u0000"; + + Assertions.assertNotNull(producer); + assertThrows(Exception.class, () -> { + Message message = new Message(topic, "*", key, body.getBytes()); + producer.getProducer().send(message); + }, " message key contains invisible character ,expect throw exception but it didn't"); + } + + @Test + @DisplayName("Message key contains Chinese, expect send and consume success") + public void testMessageKeyContentWithChinese() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String groupId = getGroupId(methodName); + + String key = "中文字符"; + String tag = NameUtils.getRandomTagName(); + String body = RandomStringUtils.randomAlphabetic(64); + + pushConsumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer.subscribeAndStart(topic, tag, new RMQNormalListener()); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer); + + Message message = new Message(topic, tag, key, body.getBytes()); + producer.send(message); + + Assertions.assertEquals(1, producer.getEnqueueMessages().getDataSize(), "send message failed"); + VerifyUtils.verifyNormalMessage(producer.getEnqueueMessages(), pushConsumer.getListener().getDequeueMessages()); + } + + @Test + @DisplayName("The message contains multiple keys, expect send and consume success") + public void testMessageWithMultiKey() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String groupId = getGroupId(methodName); + + String key1 = RandomStringUtils.randomAlphabetic(64); + String key2 = RandomStringUtils.randomAlphabetic(64); + String tag = NameUtils.getRandomTagName(); + String body = RandomStringUtils.randomAlphabetic(64); + + pushConsumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer.subscribeAndStart(topic, tag, new RMQNormalListener()); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer); + + Message message = new Message(topic, tag, RandomUtils.getStringByUUID(), body.getBytes()); + message.setKeys(Arrays.asList(key1, key2)); + producer.send(message); + + Assertions.assertEquals(1, producer.getEnqueueMessages().getDataSize(), "send message failed"); + VerifyUtils.verifyNormalMessage(producer.getEnqueueMessages(), pushConsumer.getListener().getDequeueMessages()); + } +} diff --git a/java/e2e-v4/src/test/java/org/apache/rocketmq/client/message/MessageTagTest.java b/java/e2e-v4/src/test/java/org/apache/rocketmq/client/message/MessageTagTest.java new file mode 100644 index 0000000..b0222a8 --- /dev/null +++ b/java/e2e-v4/src/test/java/org/apache/rocketmq/client/message/MessageTagTest.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.message; + +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.rocketmq.client.rmq.RMQNormalConsumer; +import org.apache.rocketmq.client.rmq.RMQNormalProducer; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.enums.TESTSET; +import org.apache.rocketmq.factory.ConsumerFactory; +import org.apache.rocketmq.factory.ProducerFactory; +import org.apache.rocketmq.frame.BaseOperate; +import org.apache.rocketmq.listener.rmq.concurrent.RMQNormalListener; +import org.apache.rocketmq.utils.RandomUtils; +import org.apache.rocketmq.utils.VerifyUtils; +import org.junit.jupiter.api.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.charset.StandardCharsets; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Test message tag + */ +@Tag(TESTSET.CLIENT) +public class MessageTagTest extends BaseOperate { + private static final Logger log = LoggerFactory.getLogger(MessageTagTest.class); + private static String topic; + private RMQNormalProducer producer; + private RMQNormalConsumer pushConsumer; + private RMQNormalConsumer pullConsumer; + + @BeforeAll + public static void setUpAll() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + topic = getTopic(methodName); + } + + @AfterEach + public void tearDown() { + if (producer != null) { + producer.shutdown(); + } + if (pushConsumer != null) { + pushConsumer.shutdown(); + } + if (pullConsumer != null) { + pullConsumer.shutdown(); + } + } + + @Disabled + @DisplayName("Message Tag beyond 16KB, expect throw exception") + public void testMessageTagBeyond16KB() { + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + String tag = RandomStringUtils.randomAlphabetic(16 * 1024 + 1); + String body = RandomStringUtils.randomAlphabetic(64); + + Assertions.assertNotNull(producer); + assertThrows(Exception.class, () -> { + Message message = new Message(topic, tag, RandomUtils.getStringByUUID(), body.getBytes()); + producer.getProducer().send(message); + }, " message tag beyond 16KB ,expect throw exception but it didn't"); + } + + @Disabled + @DisplayName("Message Tag equals 16KB, expect send success") + public void testMessageTagEquals16KB() { + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + String tag = RandomStringUtils.randomAlphabetic(16 * 1024); + String body = RandomStringUtils.randomAlphabetic(64); + + Message message = new Message(topic, tag, RandomUtils.getStringByUUID(), body.getBytes()); + producer.send(message); + } + + @Disabled + @DisplayName("Message Tag contains invisible characters \u0000 ,expect throw exception") + public void testMessageTagWithInvisibleCharacter() { + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + + String tag = "\u0000"; + + Assertions.assertNotNull(producer); + assertThrows(Exception.class, () -> { + Message message = new Message(topic, tag, RandomUtils.getStringByUUID(), + RandomUtils.getStringByUUID().getBytes(StandardCharsets.UTF_8)); + producer.getProducer().send(message); + }, " message tag contains invisible character ,expect throw exception but it didn't"); + } + + @Disabled + @DisplayName("Message Tag contains | , expect throw exception") + public void testMessageTagContentWith() { + String tag = "tag|"; + String body = RandomStringUtils.randomAlphabetic(64); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + + assertThrows(Exception.class, () -> { + Message message = new Message(topic, tag, RandomUtils.getStringByUUID(), + body.getBytes(StandardCharsets.UTF_8)); + producer.getProducer().send(message); + }, " message tag contains | , expect throw exception but it didn't"); + + } + + @Test + @DisplayName("Message Tag contains Chinese, expect send and consume success") + public void testMessageTagContentWithChinese() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String groupId = getGroupId(methodName); + + String tag = "中文字符"; + String body = RandomStringUtils.randomAlphabetic(64); + + pushConsumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer.subscribeAndStart(topic, tag, new RMQNormalListener()); + + pullConsumer = ConsumerFactory.getRMQLitePullConsumer(namesrvAddr, groupId, rpcHook); + pullConsumer.subscribeAndStartLitePull(topic, tag); + VerifyUtils.tryReceiveOnce(pullConsumer.getLitePullConsumer()); + pullConsumer.shutdown(); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer); + + Message message = new Message(topic, tag, RandomUtils.getStringByUUID(), body.getBytes(StandardCharsets.UTF_8)); + producer.send(message); + + Assertions.assertEquals(1, producer.getEnqueueMessages().getDataSize(), "send message failed"); + VerifyUtils.verifyNormalMessage(producer.getEnqueueMessages(), pushConsumer.getListener().getDequeueMessages()); + } +} diff --git a/java/e2e-v4/src/test/java/org/apache/rocketmq/client/message/MessageUserPropertyTest.java b/java/e2e-v4/src/test/java/org/apache/rocketmq/client/message/MessageUserPropertyTest.java new file mode 100644 index 0000000..bd52846 --- /dev/null +++ b/java/e2e-v4/src/test/java/org/apache/rocketmq/client/message/MessageUserPropertyTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.message; + +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.rocketmq.client.rmq.RMQNormalProducer; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.enums.TESTSET; +import org.apache.rocketmq.factory.ProducerFactory; +import org.apache.rocketmq.frame.BaseOperate; +import org.apache.rocketmq.utils.RandomUtils; +import org.junit.jupiter.api.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Test user property + */ +@Tag(TESTSET.CLIENT) +public class MessageUserPropertyTest extends BaseOperate { + private static final Logger log = LoggerFactory.getLogger(MessageUserPropertyTest.class); + private static String topic; + private static RMQNormalProducer producer; + + @BeforeAll + public static void setUpAll() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + topic = getTopic(methodName); + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + } + + @AfterAll + public static void tearDownAll() { + if (producer != null) { + producer.shutdown(); + } + } + + @Disabled + @DisplayName("Message user property equals limit 31KB, expect send success") + public void testMessageUserPropertyEquals16KB() { + String key = RandomStringUtils.randomAlphabetic(31 * 512); + String value = RandomStringUtils.randomAlphabetic(31 * 512); + HashMap userProperty = new HashMap<>(); + userProperty.put(key, value); + + Message message = new Message(topic, RandomUtils.getStringByUUID().getBytes()); + for (Map.Entry entry : userProperty.entrySet()) { + message.putUserProperty(entry.getKey(), entry.getValue()); + } + producer.send(message); + } + + @Disabled + @DisplayName("Message user property limit 32KB ,expect throw exception") + public void testMessageUserPropertyBeyond16KB() { + String key = RandomStringUtils.randomAlphabetic(16 * 1024); + String value = RandomStringUtils.randomAlphabetic(16 * 1024); + HashMap userProperty = new HashMap<>(); + userProperty.put(key, value); + + assertThrows(Exception.class, () -> { + Message message = new Message(topic, RandomUtils.getStringByUUID().getBytes()); + for (Map.Entry entry : userProperty.entrySet()) { + message.putUserProperty(entry.getKey(), entry.getValue()); + } + producer.getProducer().send(message); + }, " message user property beyond 16KB ,expect throw exception but it didn't"); + } + +} diff --git a/java/e2e-v4/src/test/java/org/apache/rocketmq/client/message/NormalMessageSizeTest.java b/java/e2e-v4/src/test/java/org/apache/rocketmq/client/message/NormalMessageSizeTest.java new file mode 100644 index 0000000..eef292f --- /dev/null +++ b/java/e2e-v4/src/test/java/org/apache/rocketmq/client/message/NormalMessageSizeTest.java @@ -0,0 +1,376 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.message; + +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.producer.LocalTransactionState; +import org.apache.rocketmq.client.producer.TransactionMQProducer; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.enums.TESTSET; +import org.apache.rocketmq.factory.MessageFactory; +import org.apache.rocketmq.frame.BaseOperate; +import org.apache.rocketmq.listener.rmq.concurrent.TransactionListenerImpl; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.utils.NameUtils; +import org.apache.rocketmq.utils.RandomUtils; +import org.junit.jupiter.api.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.*; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +@Tag(TESTSET.NORMAL) +@DisplayName("Test cases that send messages") +public class NormalMessageSizeTest extends BaseOperate { + private static final Logger log = LoggerFactory.getLogger(NormalMessageSizeTest.class); + private static String normalTopic; + private static String transTopic; + private static String fifoTopic; + private static String delayTopic; + private static DefaultMQProducer producer; + + @BeforeAll + public static void setUpAll() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + normalTopic = getTopic(methodName); + transTopic = getTopic(methodName); + delayTopic = getTopic(methodName); + fifoTopic = getTopic(methodName); + } + + @AfterAll + public static void tearDownAll() { + if (producer != null) { + producer.shutdown(); + } + } + + @Test + @DisplayName("Send normal messages synchronously with the body size of 4M+1, expect send failed") + public void testNormalMsgSize4MAdd1() { + producer = new DefaultMQProducer(RandomUtils.getStringByUUID(), rpcHook); + producer.setInstanceName(UUID.randomUUID().toString()); + producer.setNamesrvAddr(namesrvAddr); + try { + producer.start(); + } catch (MQClientException e) { + log.info("Start DefaultMQProducer failed, {}", e.getMessage()); + } + + String messageBody = RandomStringUtils.randomAlphabetic(4 * 1024 * 1024 + 1); + String tag = NameUtils.getRandomTagName(); + assertThrows(Exception.class, () -> { + Message message = new Message(normalTopic, tag, messageBody.getBytes()); + producer.send(message); + }); + } + + @Test + @DisplayName("Send normal messages synchronously with the body size of 4M, expect send success") + public void testNormalMsgSize4M() { + producer = new DefaultMQProducer(RandomUtils.getStringByUUID(), rpcHook); + producer.setInstanceName(UUID.randomUUID().toString()); + producer.setNamesrvAddr(namesrvAddr); + try { + producer.start(); + } catch (MQClientException e) { + log.info("Start DefaultMQProducer failed, {}", e.getMessage()); + } + String messageBody = RandomStringUtils.randomAlphabetic(4 * 1024 * 1024); + String tag = NameUtils.getRandomTagName(); + Message message = new Message(normalTopic, tag, messageBody.getBytes()); + try { + producer.send(message); + } catch (Exception e) { + e.printStackTrace(); + Assertions.fail("Send message failed, expected success"); + } + } + + @Test + @DisplayName("Send delay messages synchronously with the body size of 4M+1, expect send failed") + public void testDelayMsgSize4MAdd1() { + producer = new DefaultMQProducer(RandomUtils.getStringByUUID(), rpcHook); + producer.setInstanceName(UUID.randomUUID().toString()); + producer.setNamesrvAddr(namesrvAddr); + try { + producer.start(); + } catch (MQClientException e) { + log.info("Start DefaultMQProducer failed, {}", e.getMessage()); + } + String messageBody = RandomStringUtils.randomAlphabetic(4 * 1024 * 1024 + 1); + String tag = NameUtils.getRandomTagName(); + assertThrows(Exception.class, () -> { + Message message = new Message(delayTopic, tag, messageBody.getBytes()); + message.setDelayTimeLevel(3); + producer.send(message); + }); + } + + @Test + @DisplayName("Send delay messages synchronously with the body size of 4M, expect send success") + public void testDelayMsgSize4M() { + producer = new DefaultMQProducer(RandomUtils.getStringByUUID(), rpcHook); + producer.setInstanceName(UUID.randomUUID().toString()); + producer.setNamesrvAddr(namesrvAddr); + try { + producer.start(); + } catch (MQClientException e) { + log.info("Start DefaultMQProducer failed, {}", e.getMessage()); + } + String messageBody = RandomStringUtils.randomAlphabetic(4 * 1024 * 1024); + String tag = NameUtils.getRandomTagName(); + Message message = new Message(delayTopic, tag, messageBody.getBytes()); + message.setDelayTimeLevel(3); + try { + producer.send(message); + } catch (MQClientException e) { + Assertions.fail("Send message failed, expected success, message:" + e.getMessage()); + } catch (MQBrokerException e) { + Assertions.fail("Send message failed, expected success, message:" + e.getMessage()); + } catch (RemotingException e) { + Assertions.fail("Send message failed, expected success, message:" + e.getMessage()); + } catch (InterruptedException e) { + Assertions.fail("Send message failed, expected success, message:" + e.getMessage()); + } + } + + @Test + @DisplayName("Send transaction messages synchronously with the body size of 4M+1, expect send failed") + public void testTransMsgSize4MAdd1() { + TransactionMQProducer producer = new TransactionMQProducer(RandomUtils.getStringByUUID(), rpcHook); + producer.setInstanceName(UUID.randomUUID().toString()); + producer.setNamesrvAddr(namesrvAddr); + TransactionListenerImpl transactionListener = new TransactionListenerImpl(LocalTransactionState.COMMIT_MESSAGE, + LocalTransactionState.COMMIT_MESSAGE); + ExecutorService executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, + new ArrayBlockingQueue(2000), new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + Thread thread = new Thread(r); + thread.setName("client-transaction-msg-check-thread"); + return thread; + } + }); + try { + if (executorService != null) { + producer.setExecutorService(executorService); + } + producer.setTransactionListener(transactionListener); + producer.start(); + } catch (MQClientException e) { + log.info("Start TransactionMQProducer failed, {}", e.getMessage()); + } + String messageBody = RandomStringUtils.randomAlphabetic(4 * 1024 * 1024 + 1); + String tag = NameUtils.getRandomTagName(); + assertThrows(Exception.class, () -> { + Message message = new Message(transTopic, tag, messageBody.getBytes()); + producer.sendMessageInTransaction(message, null); + }); + producer.shutdown(); + } + + @Test + @DisplayName("Send transaction messages synchronously with the body size of 4M, expect send success") + public void testTransMsgSize4M() { + TransactionMQProducer producer = new TransactionMQProducer(RandomUtils.getStringByUUID(), rpcHook); + producer.setInstanceName(UUID.randomUUID().toString()); + producer.setNamesrvAddr(namesrvAddr); + TransactionListenerImpl transactionListener = new TransactionListenerImpl(LocalTransactionState.COMMIT_MESSAGE, + LocalTransactionState.COMMIT_MESSAGE); + ExecutorService executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, + new ArrayBlockingQueue(2000), new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + Thread thread = new Thread(r); + thread.setName("client-transaction-msg-check-thread"); + return thread; + } + }); + try { + if (executorService != null) { + producer.setExecutorService(executorService); + } + producer.setTransactionListener(transactionListener); + producer.start(); + } catch (MQClientException e) { + log.info("Start TransactionMQProducer failed, {}", e.getMessage()); + } + String messageBody = RandomStringUtils.randomAlphabetic(4 * 1024 * 1024); + String tag = NameUtils.getRandomTagName(); + Message message = new Message(transTopic, tag, messageBody.getBytes()); + try { + producer.sendMessageInTransaction(message, null); + } catch (MQClientException e) { + Assertions.fail("Send message failed, expected success, message:" + e.getMessage()); + } + producer.shutdown(); + } + + @Test + @DisplayName("Send FIFO messages synchronously with the body size of 4M+1, expect send failed") + public void testFifoMsgSize4MAdd1() { + producer = new DefaultMQProducer(RandomUtils.getStringByUUID(), rpcHook); + producer.setInstanceName(UUID.randomUUID().toString()); + producer.setNamesrvAddr(namesrvAddr); + try { + producer.start(); + } catch (MQClientException e) { + log.info("Start DefaultMQProducer failed, {}", e.getMessage()); + } + List messageQueues = null; + try { + messageQueues = producer.fetchPublishMessageQueues(fifoTopic); + } catch (MQClientException e) { + Assertions.assertNotNull(messageQueues); + } + String messageBody = RandomStringUtils.randomAlphabetic(4 * 1024 * 1024 + 1); + String tag = NameUtils.getRandomTagName(); + List finalMessageQueues = messageQueues; + assertThrows(Exception.class, () -> { + Message message = new Message(fifoTopic, tag, messageBody.getBytes()); + if (finalMessageQueues.size() > 0) { + producer.send(message, finalMessageQueues.get(0)); + } + producer.send(message); + }); + } + + @Test + @DisplayName("Send FIFO messages synchronously with the body size of 4M, expect send success") + public void testFifoMsgSize4M() { + producer = new DefaultMQProducer(RandomUtils.getStringByUUID(), rpcHook); + producer.setInstanceName(UUID.randomUUID().toString()); + producer.setNamesrvAddr(namesrvAddr); + try { + producer.start(); + } catch (MQClientException e) { + log.info("Start DefaultMQProducer failed, {}", e.getMessage()); + } + List messageQueues = null; + try { + messageQueues = producer.fetchPublishMessageQueues(fifoTopic); + } catch (MQClientException e) { + Assertions.assertNotNull(messageQueues); + } + String messageBody = RandomStringUtils.randomAlphabetic(4 * 1024 * 1024); + String tag = NameUtils.getRandomTagName(); + Message message = new Message(fifoTopic, tag, messageBody.getBytes()); + List finalMessageQueues = messageQueues; + try { + if (finalMessageQueues.size() > 0) { + producer.send(message, finalMessageQueues.get(0)); + } + } catch (MQBrokerException e) { + Assertions.fail("Send message failed, expected success, message:" + e.getMessage()); + } catch (RemotingException e) { + Assertions.fail("Send message failed, expected success, message:" + e.getMessage()); + } catch (InterruptedException e) { + Assertions.fail("Send message failed, expected success, message:" + e.getMessage()); + } catch (MQClientException e) { + Assertions.fail("Send message failed, expected success, message:" + e.getMessage()); + } + } + + @Test + @DisplayName("Send normal messages synchronously with the body size of 4M and the user property size of 16KB, expect send success") + public void testNormalMsgSize4MAndUserProperty16KB() { + producer = new DefaultMQProducer(RandomUtils.getStringByUUID(), rpcHook); + producer.setInstanceName(UUID.randomUUID().toString()); + producer.setNamesrvAddr(namesrvAddr); + try { + producer.start(); + } catch (MQClientException e) { + log.info("Start DefaultMQProducer failed, {}", e.getMessage()); + } + String messageBody = RandomStringUtils.randomAlphabetic(4 * 1024 * 1024); + String key = RandomStringUtils.randomAlphabetic(8 * 1024); + String value = RandomStringUtils.randomAlphabetic(8 * 1024); + HashMap userProperty = new HashMap<>(); + userProperty.put(key, value); + try { + Message message = new Message(normalTopic, messageBody.getBytes()); + for (Map.Entry entry : userProperty.entrySet()) { + message.putUserProperty(entry.getKey(), entry.getValue()); + } + producer.send(message); + } catch (MQBrokerException e) { + Assertions.fail("Send message failed, expected success, message:" + e.getMessage()); + } catch (RemotingException e) { + Assertions.fail("Send message failed, expected success, message:" + e.getMessage()); + } catch (InterruptedException e) { + Assertions.fail("Send message failed, expected success, message:" + e.getMessage()); + } catch (MQClientException e) { + Assertions.fail("Send message failed, expected success, message:" + e.getMessage()); + } + } + + @Test + @DisplayName("Send FIFO messages synchronously with the body size of 4M and the user property size of 16KB, expect send success") + public void testFifoMsgSize4MAndUserProperty16KB() { + producer = new DefaultMQProducer(RandomUtils.getStringByUUID(), rpcHook); + producer.setInstanceName(UUID.randomUUID().toString()); + producer.setNamesrvAddr(namesrvAddr); + try { + producer.start(); + } catch (MQClientException e) { + log.info("Start DefaultMQProducer failed, {}", e.getMessage()); + } + List messageQueues = null; + try { + messageQueues = producer.fetchPublishMessageQueues(fifoTopic); + } catch (MQClientException e) { + Assertions.assertNotNull(messageQueues); + } + String messageBody = RandomStringUtils.randomAlphabetic(4 * 1024 * 1024); + String key = RandomStringUtils.randomAlphabetic(8 * 1024); + String value = RandomStringUtils.randomAlphabetic(8 * 1024); + HashMap userProperty = new HashMap<>(); + userProperty.put(key, value); + List finalMessageQueues = messageQueues; + try { + Message message = new Message(fifoTopic, messageBody.getBytes()); + for (Map.Entry entry : userProperty.entrySet()) { + message.putUserProperty(entry.getKey(), entry.getValue()); + } + if (finalMessageQueues.size() > 0) { + producer.send(message, finalMessageQueues.get(0)); + } + } catch (MQBrokerException e) { + Assertions.fail("Send message failed, expected success, message:" + e.getMessage()); + } catch (RemotingException e) { + Assertions.fail("Send message failed, expected success, message:" + e.getMessage()); + } catch (InterruptedException e) { + Assertions.fail("Send message failed, expected success, message:" + e.getMessage()); + } catch (MQClientException e) { + Assertions.fail("Send message failed, expected success, message:" + e.getMessage()); + } + } + +} diff --git a/java/e2e-v4/src/test/java/org/apache/rocketmq/client/producer/ProducerInitTest.java b/java/e2e-v4/src/test/java/org/apache/rocketmq/client/producer/ProducerInitTest.java new file mode 100644 index 0000000..58bd6c3 --- /dev/null +++ b/java/e2e-v4/src/test/java/org/apache/rocketmq/client/producer/ProducerInitTest.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.producer; + +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.enums.TESTSET; +import org.apache.rocketmq.factory.AclClient; +import org.apache.rocketmq.frame.BaseOperate; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.utils.RandomUtils; +import org.junit.jupiter.api.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +@Tag(TESTSET.CLIENT) +public class ProducerInitTest extends BaseOperate { + private static final Logger log = LoggerFactory.getLogger(ProducerInitTest.class); + private static String topic; + + @BeforeAll + public static void setUpAll() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + topic = getTopic(methodName); + } + + @BeforeEach + public void setUp() { + } + + @AfterEach + public void tearDown() { + } + + @Test + @DisplayName("Producer is normally set,expected success") + public void testNormalSetting() { + try { + DefaultMQProducer producer = new DefaultMQProducer(RandomUtils.getStringByUUID(), rpcHook); + producer.setNamesrvAddr(namesrvAddr); + producer.setInstanceName(UUID.randomUUID().toString()); + producer.start(); + producer.shutdown(); + } catch (MQClientException e) { + Assertions.fail("Send message failed, expected success, message:" + e.getMessage()); + } + } + + @Disabled + @DisplayName("The NAMESRV_ADDR setting of the Producer failed, expect ONSClientException to throw") + public void testErrorNameSrvAddr() { + assertThrows(Exception.class, () -> { + DefaultMQProducer producer = new DefaultMQProducer(RandomUtils.getStringByUUID(), rpcHook); + producer.setNamesrvAddr("https://www.aliyun.com"); + producer.setInstanceName(UUID.randomUUID().toString()); + producer.start(); + producer.shutdown(); + }, "Expected ClientException to throw, but it didn't"); + } + + @Test + @DisplayName("The Producer does not set the AccessKey, expect an exception occurs when the client start") + public void testUnsetAK() { + assertThrows(Exception.class, () -> { + RPCHook aclRPCHook = AclClient.getAclRPCHook(null, account.getSecretKey()); + DefaultMQProducer producer = new DefaultMQProducer(RandomUtils.getStringByUUID(), aclRPCHook); + producer.setNamesrvAddr(namesrvAddr); + producer.setInstanceName(UUID.randomUUID().toString()); + producer.start(); + producer.shutdown(); + }, "Expected ClientException to throw, but it didn't"); + } + + @Test + @DisplayName("The Producer does not set the SecretKey, expect an exception occurs when the client start") + public void testUnsetSK() { + assertThrows(Exception.class, () -> { + RPCHook aclRPCHook = AclClient.getAclRPCHook(account.getAccessKey(), ""); + DefaultMQProducer producer = new DefaultMQProducer(RandomUtils.getStringByUUID(), aclRPCHook); + producer.setNamesrvAddr(namesrvAddr); + producer.setInstanceName(UUID.randomUUID().toString()); + producer.start(); + producer.shutdown(); + }, "Expected ClientException to throw, but it didn't"); + } + + @Disabled + @DisplayName("The Producer does not set the Properties, expect an exception occurs when the client start") + public void testUnsetProperties() { + assertThrows(Exception.class, () -> { + DefaultMQProducer producer = new DefaultMQProducer(RandomUtils.getStringByUUID(), (RPCHook) null); + producer.setNamesrvAddr(namesrvAddr); + producer.setInstanceName(UUID.randomUUID().toString()); + producer.start(); + producer.shutdown(); + }, "Expected ClientException to throw, but it didn't"); + } + + @Test + @DisplayName("The Producer sets the maximum retry times to 0, expect the client start success") + public void testSet0MaxAttempts() { + try { + DefaultMQProducer producer = new DefaultMQProducer(RandomUtils.getStringByUUID(), rpcHook); + producer.setNamesrvAddr(namesrvAddr); + producer.setInstanceName(UUID.randomUUID().toString()); + producer.setRetryTimesWhenSendFailed(0); + producer.start(); + producer.shutdown(); + } catch (Exception e) { + Assertions.fail("Expected the client to start successfully, but it didn't"); + } + } + + @Disabled + @DisplayName("The Producer sets the maximum retry times to -1, expect the client start failed") + public void testSetLess0MaxAttempts() { + assertThrows(Exception.class, () -> { + DefaultMQProducer producer = new DefaultMQProducer(RandomUtils.getStringByUUID(), rpcHook); + producer.setNamesrvAddr(namesrvAddr); + producer.setInstanceName(UUID.randomUUID().toString()); + producer.setRetryTimesWhenSendFailed(-1); + producer.start(); + producer.shutdown(); + }, "Expected ClientException to throw, but it didn't"); + } +} diff --git a/java/e2e-v4/src/test/java/org/apache/rocketmq/cluster/ClusterTest.java b/java/e2e-v4/src/test/java/org/apache/rocketmq/cluster/ClusterTest.java new file mode 100644 index 0000000..07aeda2 --- /dev/null +++ b/java/e2e-v4/src/test/java/org/apache/rocketmq/cluster/ClusterTest.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.cluster; + +import org.apache.rocketmq.client.rmq.RMQNormalConsumer; +import org.apache.rocketmq.client.rmq.RMQNormalProducer; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.enums.TESTSET; +import org.apache.rocketmq.factory.ConsumerFactory; +import org.apache.rocketmq.factory.ProducerFactory; +import org.apache.rocketmq.frame.BaseOperate; +import org.apache.rocketmq.listener.rmq.concurrent.RMQNormalListener; +import org.apache.rocketmq.utils.NameUtils; +import org.apache.rocketmq.utils.RandomUtils; +import org.apache.rocketmq.utils.VerifyUtils; +import org.junit.jupiter.api.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Tag(TESTSET.MODEL) +public class ClusterTest extends BaseOperate { + private final Logger log = LoggerFactory.getLogger(ClusterTest.class); + private String tag; + private final static int SEND_NUM = 100; + private RMQNormalConsumer pushConsumer01; + private RMQNormalConsumer pushConsumer02; + private RMQNormalConsumer pushConsumer03; + private RMQNormalConsumer pullConsumer; + private RMQNormalProducer producer; + + @BeforeEach + public void setUp() { + tag = NameUtils.getRandomTagName(); + } + + @BeforeEach + public void tearDown() { + if (pushConsumer01 != null) { + pushConsumer01.shutdown(); + } + if (pushConsumer02 != null) { + pushConsumer02.shutdown(); + } + if (pushConsumer03 != null) { + pushConsumer03.shutdown(); + } + if (pullConsumer != null) { + pullConsumer.shutdown(); + } + if (producer != null) { + producer.shutdown(); + } + } + + @Test + @DisplayName("Send 100 normal messages synchronously, start three consumers on different GroupId, and expect each client to consume up to 100 messages") + public void testBroadcastConsume() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId01 = getGroupId(methodName + 1); + String groupId02 = getGroupId(methodName + 2); + String groupId03 = getGroupId(methodName + 3); + + RMQNormalListener listenerA = new RMQNormalListener("ListenerA"); + RMQNormalListener listenerB = new RMQNormalListener("ListenerB"); + RMQNormalListener listenerC = new RMQNormalListener("ListenerC"); + pushConsumer01 = ConsumerFactory.getRMQBroadCastConsumer(namesrvAddr, groupId01, rpcHook); + pushConsumer02 = ConsumerFactory.getRMQBroadCastConsumer(namesrvAddr, groupId02, rpcHook); + pushConsumer03 = ConsumerFactory.getRMQBroadCastConsumer(namesrvAddr, groupId03, rpcHook); + pushConsumer01.subscribeAndStart(topic,tag, listenerA); + pushConsumer02.subscribeAndStart(topic,tag, listenerB); + pushConsumer03.subscribeAndStart(topic,tag, listenerC); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get Producer failed"); + for (int i = 0; i < SEND_NUM; i++) { + Message message = new Message(topic, tag, RandomUtils.getStringByUUID().getBytes()); + producer.send(message); + } + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + VerifyUtils.verifyNormalMessage(producer.getEnqueueMessages(), listenerA.getDequeueMessages()); + VerifyUtils.verifyNormalMessage(producer.getEnqueueMessages(), listenerB.getDequeueMessages()); + VerifyUtils.verifyNormalMessage(producer.getEnqueueMessages(), listenerC.getDequeueMessages()); + } + + @Test + @DisplayName("Send 100 normal messages synchronously, start three consumers on same GroupId, and expect each client to consume up to 100 messages") + public void testBroadcastConsumeWithSameGroupId() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + RMQNormalListener listenerA = new RMQNormalListener("ListenerA"); + RMQNormalListener listenerB = new RMQNormalListener("ListenerB"); + RMQNormalListener listenerC = new RMQNormalListener("ListenerC"); + pushConsumer01 = ConsumerFactory.getRMQBroadCastConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer02 = ConsumerFactory.getRMQBroadCastConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer03 = ConsumerFactory.getRMQBroadCastConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer01.subscribeAndStart(topic,tag, listenerA); + pushConsumer02.subscribeAndStart(topic,tag, listenerB); + pushConsumer03.subscribeAndStart(topic,tag, listenerC); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get Producer failed"); + for (int i = 0; i < SEND_NUM; i++) { + Message message = new Message(topic, tag, RandomUtils.getStringByUUID().getBytes()); + producer.send(message); + } + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + VerifyUtils.verifyNormalMessage(producer.getEnqueueMessages(), listenerA.getDequeueMessages()); + VerifyUtils.verifyNormalMessage(producer.getEnqueueMessages(), listenerB.getDequeueMessages()); + VerifyUtils.verifyNormalMessage(producer.getEnqueueMessages(), listenerC.getDequeueMessages()); + } + + @Test + @DisplayName("Send 100 normal messages synchronously, start 3 consumers on the same GroupId, expect 3 clients to consume a total of 100 messages") + public void testClusterConsume() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + RMQNormalListener listenerA = new RMQNormalListener("ListenerA"); + RMQNormalListener listenerB = new RMQNormalListener("ListenerB"); + RMQNormalListener listenerC = new RMQNormalListener("ListenerC"); + pushConsumer01 = ConsumerFactory.getRMQClusterConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer02 = ConsumerFactory.getRMQClusterConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer03 = ConsumerFactory.getRMQClusterConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer01.subscribeAndStart(topic,tag, listenerA); + pushConsumer02.subscribeAndStart(topic,tag, listenerB); + pushConsumer03.subscribeAndStart(topic,tag, listenerC); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + for (int i = 0; i < SEND_NUM; i++) { + Message message = new Message(topic, tag, String.valueOf(i).getBytes()); + producer.send(message); + } + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + VerifyUtils.verifyClusterConsume(producer.getEnqueueMessages(), listenerA.getDequeueMessages(), listenerB.getDequeueMessages(), listenerC.getDequeueMessages()); + } +} + diff --git a/java/e2e-v4/src/test/java/org/apache/rocketmq/cluster/LoadBalancingTest.java b/java/e2e-v4/src/test/java/org/apache/rocketmq/cluster/LoadBalancingTest.java new file mode 100644 index 0000000..dc4aa76 --- /dev/null +++ b/java/e2e-v4/src/test/java/org/apache/rocketmq/cluster/LoadBalancingTest.java @@ -0,0 +1,304 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.cluster; + +import org.apache.rocketmq.frame.BaseOperate; +import org.apache.rocketmq.listener.rmq.concurrent.RMQIdempotentListener; +import org.apache.rocketmq.listener.rmq.concurrent.RMQNormalListener; +import org.apache.rocketmq.listener.rmq.concurrent.RMQOrderListener; +import org.apache.rocketmq.utils.NameUtils; +import org.apache.rocketmq.utils.VerifyUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; + +import org.apache.rocketmq.client.consumer.MessageSelector; +import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; +import org.apache.rocketmq.client.producer.MessageQueueSelector; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.rmq.RMQNormalConsumer; +import org.apache.rocketmq.client.rmq.RMQNormalProducer; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.enums.TESTSET; +import org.apache.rocketmq.factory.ConsumerFactory; +import org.apache.rocketmq.factory.MessageFactory; +import org.apache.rocketmq.factory.ProducerFactory; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Tag(TESTSET.LOAD_BALANCING) +public class LoadBalancingTest extends BaseOperate { + private final Logger log = LoggerFactory.getLogger(LoadBalancingTest.class); + private String tag; + private final static int SEND_NUM = 10; + + @BeforeEach + public void setUp() { + tag = NameUtils.getRandomTagName(); + } + + @Test + @DisplayName("Normal message load balancing, start 4 consumers, send 240 messages, expect 4 consumers to consume load balancing, each consume 1/4, then shutdown 2 of them, send 240 messages again, still load balancing, each consume half, and start 2 new consumers. Another 240 messages are sent, still load balanced, each consuming 1/4") + public void testLoadBalancing_normal_message() { + int messageSize = 240; + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + + RMQNormalConsumer consumer1 = ConsumerFactory.getRMQClusterConsumer(namesrvAddr, groupId, rpcHook,new AllocateMessageQueueAveragely()); + RMQNormalConsumer consumer2 = ConsumerFactory.getRMQClusterConsumer(namesrvAddr, groupId, rpcHook,new AllocateMessageQueueAveragely()); + RMQNormalConsumer consumer3 = ConsumerFactory.getRMQClusterConsumer(namesrvAddr, groupId, rpcHook,new AllocateMessageQueueAveragely()); + RMQNormalConsumer consumer4 = ConsumerFactory.getRMQClusterConsumer(namesrvAddr, groupId, rpcHook,new AllocateMessageQueueAveragely()); + consumer1.subscribeAndStart(topic, tag, new RMQIdempotentListener()); + consumer2.subscribeAndStart(topic, tag, new RMQIdempotentListener()); + consumer3.subscribeAndStart(topic, tag, new RMQIdempotentListener()); + consumer4.subscribeAndStart(topic, tag, new RMQIdempotentListener()); + + Assertions.assertNotNull(producer); + producer.send(topic, tag, messageSize); + + Assertions.assertEquals(messageSize, producer.getEnqueueMessages().getDataSize(), "send message failed"); + + await().atMost(120, SECONDS).until(new Callable() { + @Override + public Boolean call() throws Exception { + return consumer1.getListener().getDequeueMessages().getDataSize() + + consumer2.getListener().getDequeueMessages().getDataSize() + + consumer3.getListener().getDequeueMessages().getDataSize() + + consumer4.getListener().getDequeueMessages().getDataSize() == messageSize; + } + }); + + VerifyUtils.verifyBalance(messageSize, consumer1.getListener().getDequeueMessages().getDataSize(), + consumer2.getListener().getDequeueMessages().getDataSize(), + consumer3.getListener().getDequeueMessages().getDataSize(), + consumer4.getListener().getDequeueMessages().getDataSize()); + + consumer1.getListener().clearMsg(); + consumer2.getListener().clearMsg(); + + consumer3.shutdown(); + consumer4.shutdown(); + + producer.send(topic, tag, messageSize); + + await().atMost(120, SECONDS).until(new Callable() { + @Override + public Boolean call() throws Exception { + return consumer1.getListener().getDequeueMessages().getDataSize() + + consumer2.getListener().getDequeueMessages().getDataSize() == messageSize; + } + }); + + VerifyUtils.verifyBalance(messageSize, consumer1.getListener().getDequeueMessages().getDataSize(), + consumer2.getListener().getDequeueMessages().getDataSize()); + + consumer1.getListener().clearMsg(); + consumer2.getListener().clearMsg(); + + RMQNormalConsumer consumer5 = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook,new AllocateMessageQueueAveragely()); + RMQNormalConsumer consumer6 = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook,new AllocateMessageQueueAveragely()); + consumer5.subscribeAndStart(topic, tag, new RMQIdempotentListener()); + consumer6.subscribeAndStart(topic, tag, new RMQIdempotentListener()); + + producer.send(topic, tag, messageSize); + + await().atMost(120, SECONDS).until(new Callable() { + @Override + public Boolean call() throws Exception { + return consumer1.getListener().getDequeueMessages().getDataSize() + + consumer2.getListener().getDequeueMessages().getDataSize() + + consumer5.getListener().getDequeueMessages().getDataSize() + + consumer6.getListener().getDequeueMessages().getDataSize() == messageSize; + } + }); + + VerifyUtils.verifyBalance(messageSize, consumer1.getListener().getDequeueMessages().getDataSize(), + consumer2.getListener().getDequeueMessages().getDataSize(), + consumer5.getListener().getDequeueMessages().getDataSize(), + consumer6.getListener().getDequeueMessages().getDataSize()); + + producer.shutdown(); + consumer1.shutdown(); + consumer2.shutdown(); + consumer5.shutdown(); + consumer6.shutdown(); + } + + @Test + @DisplayName("Global sequential message load balancing: Start 2 consumers, send 30 messages, expect only 1 Consumer to consume the message, and the other Consumer to consume 0, then shutdown 1 Consumer with message consumption, send another 30 messages, expect the idling Consumer to pull the message") + public void testLoadBalancing_global_sequential_message(){ + int messageSize = 30; + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + + RMQNormalConsumer pullConsumer = ConsumerFactory.getRMQLitePullConsumer(namesrvAddr, groupId, rpcHook,1); + pullConsumer.subscribeAndStartLitePull(topic,MessageSelector.byTag(tag)); + VerifyUtils.tryReceiveOnce(pullConsumer.getLitePullConsumer()); + pullConsumer.shutdown(); + + RMQNormalConsumer consumer1 = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook,new AllocateMessageQueueAveragely()); + RMQNormalConsumer consumer2 = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook,new AllocateMessageQueueAveragely()); + consumer1.subscribeAndStart(topic, tag, new RMQOrderListener()); + consumer2.subscribeAndStart(topic, tag, new RMQOrderListener()); + + Assertions.assertNotNull(producer); + List msgQueues = producer.fetchPublishMessageQueues(topic); + List msgQueue = new ArrayList<>(); + msgQueue.add(msgQueues.get(0)); + producer.sendWithQueue(msgQueue,tag,messageSize); + + await().atMost(120, SECONDS).until(new Callable() { + @Override + public Boolean call() throws Exception { + return consumer1.getListener().getDequeueMessages().getDataSize() + + consumer2.getListener().getDequeueMessages().getDataSize() == messageSize; + } + }); + Assertions.assertTrue(consumer1.getListener().getDequeueMessages().getDataSize() == 0 || consumer2.getListener().getDequeueMessages().getDataSize() == 0, "global sequential message load balancing is not satisfied"); + + consumer1.shutdown(); + + consumer2.getListener().clearMsg(); + + producer.sendWithQueue(msgQueue,tag,messageSize); + + await().atMost(120, SECONDS).until(new Callable() { + @Override + public Boolean call() throws Exception { + return consumer2.getListener().getDequeueMessages().getDataSize() == messageSize; + } + }); + + Assertions.assertTrue(consumer2.getListener().getDequeueMessages().getDataSize() == messageSize, "global sequential message load balancing is not satisfied"); + + producer.shutdown(); + consumer2.shutdown(); + } + + @Test + @DisplayName("Partition sequential message load balancing, start 4 consumers, send 120 messages, shardingkey=8, expect 4 consumers to consume load balancing, each consumes 1/4 Partition, shutdown 2 of the consumers, send 120 messages again. The load is still balanced, and half of each Partition is consumed") + public void testLoadBalancing_partition_sequential_message(){ + int messageSize = 120; + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + + RMQNormalConsumer pullConsumer = ConsumerFactory.getRMQLitePullConsumer(namesrvAddr, groupId, rpcHook,1); + pullConsumer.subscribeAndStartLitePull(topic,MessageSelector.byTag(tag)); + VerifyUtils.tryReceiveOnce(pullConsumer.getLitePullConsumer()); + pullConsumer.shutdown(); + + RMQNormalConsumer consumer1 = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook,new AllocateMessageQueueAveragely()); + RMQNormalConsumer consumer2 = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook,new AllocateMessageQueueAveragely()); + RMQNormalConsumer consumer3 = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook,new AllocateMessageQueueAveragely()); + RMQNormalConsumer consumer4 = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook,new AllocateMessageQueueAveragely()); + consumer1.subscribeAndStart(topic, tag, new RMQOrderListener()); + consumer2.subscribeAndStart(topic, tag, new RMQOrderListener()); + consumer3.subscribeAndStart(topic, tag, new RMQOrderListener()); + consumer4.subscribeAndStart(topic, tag, new RMQOrderListener()); + + Assertions.assertNotNull(producer); + for (int i = 0; i < messageSize; i++) { + Message message = MessageFactory.buildOneMessageWithTagAndBody(topic, tag, String.valueOf(i)); + try { + SendResult sendResult = producer.getProducer().send(message, new MessageQueueSelector(){ + + @Override + public MessageQueue select(List mqs, Message msg, Object arg) { + Integer index = (Integer) arg; + return mqs.get(index % mqs.size()); + } + + },i); + log.info("{}, index: {}, tag: {}", sendResult, i, tag); + } catch (Exception e) { + Assertions.fail("DefaultMQProducer send message failed"); + } + } + + await().atMost(120, SECONDS).until(new Callable() { + @Override + public Boolean call() throws Exception { + return consumer1.getListener().getDequeueMessages().getDataSize() + + consumer2.getListener().getDequeueMessages().getDataSize() + + consumer3.getListener().getDequeueMessages().getDataSize() + + consumer4.getListener().getDequeueMessages().getDataSize() == messageSize; + } + }); + + Assertions.assertTrue(messageSize/4==consumer1.getListener().getDequeueMessages().getDataSize(), "consumer1: first load balancing is not satisfied"); + Assertions.assertTrue(messageSize/4==consumer2.getListener().getDequeueMessages().getDataSize(), "consumer2: first load balancing is not satisfied"); + Assertions.assertTrue(messageSize/4==consumer3.getListener().getDequeueMessages().getDataSize(), "consumer3: first load balancing is not satisfied"); + Assertions.assertTrue(messageSize/4==consumer4.getListener().getDequeueMessages().getDataSize(), "consumer4: first load balancing is not satisfied"); + + consumer3.shutdown(); + consumer4.shutdown(); + + consumer1.getListener().clearMsg(); + consumer2.getListener().clearMsg(); + + for (int i = 0; i < messageSize; i++) { + Message message = MessageFactory.buildOneMessageWithTagAndBody(topic, tag, String.valueOf(i)); + try { + SendResult sendResult = producer.getProducer().send(message, new MessageQueueSelector(){ + + @Override + public MessageQueue select(List mqs, Message msg, Object arg) { + Integer index = (Integer) arg; + return mqs.get(index % mqs.size()); + } + + },i); + log.info("{}, index: {}, tag: {}", sendResult, i, tag); + } catch (Exception e) { + Assertions.fail("DefaultMQProducer send message failed"); + } + } + + await().atMost(120, SECONDS).until(new Callable() { + @Override + public Boolean call() throws Exception { + return consumer1.getListener().getDequeueMessages().getDataSize() + + consumer2.getListener().getDequeueMessages().getDataSize() == messageSize; + } + }); + + Assertions.assertTrue(messageSize/2==consumer1.getListener().getDequeueMessages().getDataSize(), "consumer1: second load balancing is not satisfied"); + Assertions.assertTrue(messageSize/2==consumer2.getListener().getDequeueMessages().getDataSize(), "consumer2: second load balancing is not satisfied"); + + producer.shutdown(); + consumer1.shutdown(); + consumer2.shutdown(); + } +} diff --git a/java/e2e-v4/src/test/java/org/apache/rocketmq/filter/SqlFilterTest.java b/java/e2e-v4/src/test/java/org/apache/rocketmq/filter/SqlFilterTest.java new file mode 100644 index 0000000..046992b --- /dev/null +++ b/java/e2e-v4/src/test/java/org/apache/rocketmq/filter/SqlFilterTest.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.filter; + +import org.apache.rocketmq.client.consumer.MessageSelector; +import org.apache.rocketmq.client.rmq.RMQNormalConsumer; +import org.apache.rocketmq.client.rmq.RMQNormalProducer; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.enums.TESTSET; +import org.apache.rocketmq.factory.ConsumerFactory; +import org.apache.rocketmq.factory.MessageFactory; +import org.apache.rocketmq.factory.ProducerFactory; +import org.apache.rocketmq.frame.BaseOperate; +import org.apache.rocketmq.listener.rmq.concurrent.RMQNormalListener; +import org.apache.rocketmq.utils.TestUtils; +import org.apache.rocketmq.utils.VerifyUtils; +import org.junit.jupiter.api.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; + +@Tag(TESTSET.SQL) +public class SqlFilterTest extends BaseOperate { + private final Logger log = LoggerFactory.getLogger(SqlFilterTest.class); + private final static int SEND_NUM = 10; + private RMQNormalProducer producer; + private RMQNormalConsumer pushConsumer; + private RMQNormalConsumer pullConsumer; + + @BeforeEach + public void setUp() { + } + + @AfterEach + public void tearDown() { + if (producer != null) { + producer.shutdown(); + } + if (pushConsumer != null) { + pushConsumer.shutdown(); + } + if (pullConsumer != null) { + pullConsumer.shutdown(); + } + } + + @Test + @DisplayName("10 messages are sent synchronously, without any attribute filtering, and expected to be consumed to the 10 messages sent") + public void testSendWithTagAndPropsRecvWithOutFilter() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + HashMap userProps = new HashMap<>(); + userProps.put("regionId", "cn-hangzhou"); + userProps.put("price", "30"); + String subExpression = "TRUE"; + + pushConsumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer.subscribeAndStart(topic, MessageSelector.bySql(subExpression), new RMQNormalListener()); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + + for (int i = 0; i < SEND_NUM; i++) { + Message message = MessageFactory.buildMessageWithProperty(topic, userProps); + producer.send(message); + } + VerifyUtils.verifyNormalMessage(producer.getEnqueueMessages(), pushConsumer.getListener().getDequeueMessages()); + } + + @Test + @DisplayName("Send 10 messages with the attribute price=10 and 10 messages with the attribute price=30. Set the filtering rule to price>20 and expect only 10 messages to be consumed") + public void testSqlSendTwoProps_SubFilterOne() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + HashMap userProps1 = new HashMap<>(); + userProps1.put("price", "10"); + HashMap userProps2 = new HashMap<>(); + userProps2.put("price", "30"); + String subExpression = "price>20"; + + pushConsumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer.subscribeAndStart(topic, MessageSelector.bySql(subExpression), new RMQNormalListener()); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + + for (int i = 0; i < SEND_NUM; i++) { + Message message = MessageFactory.buildMessageWithProperty(topic, userProps1); + producer.send(message); + } + for (int i = 0; i < SEND_NUM; i++) { + Message message = MessageFactory.buildMessageWithProperty(topic, userProps2); + producer.send(message); + } + VerifyUtils.verifyNormalMessageWithUserProperties(producer.getEnqueueMessages(), + pushConsumer.getListener().getDequeueMessages(), userProps1, 10); + } + + @Test + @DisplayName("Send 10 messages synchronously, using the attribute between{a,b} filter, expect to consume 10 messages sent") + public void testSendWithTagAndPropsRecvWithBetween() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + HashMap userProps = new HashMap<>(); + userProps.put("regionId", "cn-hangzhou"); + userProps.put("price", "30"); + String subExpression = "(price BETWEEN 10 AND 100) AND regionId IS NOT NUll"; + + pushConsumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer.subscribeAndStart(topic, MessageSelector.bySql(subExpression), new RMQNormalListener()); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + + for (int i = 0; i < SEND_NUM; i++) { + Message message = MessageFactory.buildMessageWithProperty(topic, userProps); + producer.send(message); + } + + VerifyUtils.verifyNormalMessage(producer.getEnqueueMessages(), pushConsumer.getListener().getDequeueMessages()); + } + + @Test + @DisplayName("Send 10 messages synchronously, filter messages using unknown attributes, expect to consume up to 0 messages") + public void testSendWithTagAndPropsRecvWithUnknownProps() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + HashMap userProps = new HashMap<>(); + userProps.put("regionId", "cn-hangzhou"); + userProps.put("price", "30"); + String subExpression = "product = 'MQ'"; + + pushConsumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer.subscribeAndStart(topic, MessageSelector.bySql(subExpression), new RMQNormalListener()); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + + for (int i = 0; i < SEND_NUM; i++) { + Message message = MessageFactory.buildMessageWithProperty(topic, userProps); + producer.send(message); + } + TestUtils.waitForSeconds(20); + Assertions.assertEquals(0, pushConsumer.getListener().getDequeueMessages().getDataSize()); + } + +} diff --git a/java/e2e-v4/src/test/java/org/apache/rocketmq/filter/TagFilterTest.java b/java/e2e-v4/src/test/java/org/apache/rocketmq/filter/TagFilterTest.java new file mode 100644 index 0000000..52ca522 --- /dev/null +++ b/java/e2e-v4/src/test/java/org/apache/rocketmq/filter/TagFilterTest.java @@ -0,0 +1,347 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.filter; + +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.rocketmq.client.consumer.MessageSelector; +import org.apache.rocketmq.client.rmq.RMQNormalConsumer; +import org.apache.rocketmq.client.rmq.RMQNormalProducer; +import org.apache.rocketmq.enums.TESTSET; +import org.apache.rocketmq.factory.ConsumerFactory; +import org.apache.rocketmq.factory.ProducerFactory; +import org.apache.rocketmq.frame.BaseOperate; +import org.apache.rocketmq.listener.rmq.concurrent.RMQNormalListener; +import org.apache.rocketmq.utils.NameUtils; +import org.apache.rocketmq.utils.TestUtils; +import org.apache.rocketmq.utils.VerifyUtils; +import org.junit.jupiter.api.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@DisplayName("TAG filtering test: Use PushConsumer for consumption") +@Tag(TESTSET.TAG) +public class TagFilterTest extends BaseOperate { + private final Logger log = LoggerFactory.getLogger(TagFilterTest.class); + private final static int SEND_NUM = 10; + RMQNormalProducer producer; + RMQNormalConsumer pushConsumer; + RMQNormalConsumer pullConsumer; + + @BeforeEach + public void setUp() { + + } + + @AfterEach + public void tearDown() { + if (producer != null) { + producer.shutdown(); + } + if (pushConsumer != null) { + pushConsumer.shutdown(); + } + if (pullConsumer != null) { + pullConsumer.shutdown(); + } + } + + @Test + @DisplayName("Using tagA sent 10 messages, the use of tagA | | tagB filter messages, expect consumption to send 10 messages") + public void testSendTagA_SubTagAorTagB() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + String sendTag = NameUtils.getRandomTagName(); + String receiveTag = sendTag + "||TagB"; + + pushConsumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer.subscribeAndStart(topic, MessageSelector.byTag(receiveTag), new RMQNormalListener()); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + + producer.send(topic, sendTag, SEND_NUM); + + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + VerifyUtils.verifyNormalMessage(producer.getEnqueueMessages(), pushConsumer.getListener().getDequeueMessages()); + } + + @Test + @DisplayName("Use tagA sent 10 messages first, after using tagB sent 10 messages, use tagA | | tagB filter messages, expect consumption to send 20 messages") + public void testSndTagATagB_SubTagATagB() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + String sendTagA = NameUtils.getRandomTagName(); + String sendTagB = NameUtils.getRandomTagName(); + String receiveTag = sendTagA + "||" + sendTagB; + + pushConsumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer.subscribeAndStart(topic, MessageSelector.byTag(receiveTag), new RMQNormalListener()); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + + producer.send(topic, sendTagA, SEND_NUM); + producer.send(topic, sendTagB, SEND_NUM); + + Assertions.assertEquals(SEND_NUM * 2, producer.getEnqueueMessages().getDataSize(), "send message failed"); + VerifyUtils.verifyNormalMessage(producer.getEnqueueMessages(), pushConsumer.getListener().getDequeueMessages()); + } + + @Test + @DisplayName("The tagA is used to send 10 messages, then the tagB is used to send 10 messages, and the * is used to filter the messages, expecting to consume 20 messages sent") + public void testSendTagAAndTagB_SubAll() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + String sendTagA = NameUtils.getRandomTagName(); + String sendTagB = NameUtils.getRandomTagName(); + String receiveTag = "*"; + + pushConsumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer.subscribeAndStart(topic, MessageSelector.byTag(receiveTag), new RMQNormalListener()); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + + producer.send(topic, sendTagA, SEND_NUM); + producer.send(topic, sendTagB, SEND_NUM); + + Assertions.assertEquals(SEND_NUM * 2, producer.getEnqueueMessages().getDataSize(), "send message failed"); + VerifyUtils.verifyNormalMessage(producer.getEnqueueMessages(), pushConsumer.getListener().getDequeueMessages()); + } + + @Test + @DisplayName("Send 10 tagA messages, subscribe to tagB messages, expect to consume up to 0 messages") + public void testSendTagA_SubTagB() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + String sendTagA = NameUtils.getRandomTagName(); + String receiveTag = NameUtils.getRandomTagName(); + + pushConsumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer.subscribeAndStart(topic, MessageSelector.byTag(receiveTag), new RMQNormalListener()); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + + producer.send(topic, sendTagA, SEND_NUM); + + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + TestUtils.waitForSeconds(20); + Assertions.assertEquals(0, pushConsumer.getListener().getDequeueMessages().getDataSize()); + } + + @Test + @DisplayName("Send 10 tagA messages, subscribe to tagA messages, expect to consume up to 10 messages") + public void testSendTagA_SubTagA() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + String sendTag = NameUtils.getRandomTagName(); + + pushConsumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer.subscribeAndStart(topic, MessageSelector.byTag(sendTag), new RMQNormalListener()); + + pullConsumer = ConsumerFactory.getRMQLitePullConsumer(namesrvAddr, groupId, rpcHook); + pullConsumer.subscribeAndStartLitePull(topic, MessageSelector.byTag(sendTag)); + VerifyUtils.tryReceiveOnce(pullConsumer.getLitePullConsumer()); + pullConsumer.shutdown(); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + + Assertions.assertNotNull(producer); + producer.send(topic, sendTag, SEND_NUM); + + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + VerifyUtils.verifyNormalMessage(producer.getEnqueueMessages(), pushConsumer.getListener().getDequeueMessages()); + } + + @Test + @DisplayName("Consumption uses a very long tagA, sending 10 messages, expecting to consume 10 tagA messages") + public void testLongTagSize() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + String sendTag = RandomStringUtils.randomAlphanumeric(1024 * 10); + + pushConsumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer.subscribeAndStart(topic, MessageSelector.byTag(sendTag), new RMQNormalListener()); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + + producer.send(topic, sendTag, SEND_NUM); + + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + VerifyUtils.verifyNormalMessage(producer.getEnqueueMessages(), pushConsumer.getListener().getDequeueMessages()); + } + + @Test + @DisplayName("The consumption uses a space-spaced tag, and two tags are used to send 10 messages each, with the expectation of consuming up to 20 messages") + public void testSubTagWithSpace() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + String sendTagA = NameUtils.getRandomTagName(); + String sendTagB = NameUtils.getRandomTagName(); + String receiveTag = " " + sendTagA + " || " + sendTagB + " "; + + pushConsumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer.subscribeAndStart(topic, MessageSelector.byTag(receiveTag), new RMQNormalListener()); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + + producer.send(topic, sendTagA, SEND_NUM); + producer.send(topic, sendTagB, SEND_NUM); + + Assertions.assertEquals(SEND_NUM * 2, producer.getEnqueueMessages().getDataSize(), "send message failed"); + VerifyUtils.verifyNormalMessage(producer.getEnqueueMessages(), pushConsumer.getListener().getDequeueMessages()); + } + + @Disabled + @DisplayName("Send 10 tag = '@ | | | @' news, expect to send an exception is thrown, the tag is not allowed to include |") + public void testTagWithSpecialSymbol01() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + + Assertions.assertThrows(Exception.class, () -> { + producer.send(topic, "|@", SEND_NUM); + }, "Send messages with tag \"|@\", Expected send() to throw exception, but it didn't"); + } + + @Test + @DisplayName("Send 10 messages with tag='*', subscribe to messages with tag='*', expect to consume the message") + public void testTagWithSpecialSymbol02() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + pushConsumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer.subscribeAndStart(topic, MessageSelector.byTag("*"), new RMQNormalListener()); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + producer.send(topic, "*", SEND_NUM); + + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + VerifyUtils.verifyNormalMessage(producer.getEnqueueMessages(), pushConsumer.getListener().getDequeueMessages()); + } + + @Test + @DisplayName("Consumer use | | separators between the tag, respectively using two tag each 10 messages sent, and expect consumption to 20 messages") + public void testTagWithSpecialSymbol03() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + String sendTagA = NameUtils.getRandomTagName(); + String sendTagB = NameUtils.getRandomTagName(); + String receiveTag = sendTagA + "||||" + sendTagB; + + pushConsumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer.subscribeAndStart(topic, MessageSelector.byTag(receiveTag), new RMQNormalListener()); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + producer.send(topic, sendTagA, SEND_NUM); + producer.send(topic, sendTagB, SEND_NUM); + + Assertions.assertEquals(SEND_NUM * 2, producer.getEnqueueMessages().getDataSize(), "send message failed"); + VerifyUtils.verifyNormalMessage(producer.getEnqueueMessages(), pushConsumer.getListener().getDequeueMessages()); + } + + @Disabled + @DisplayName("Send 10 messages each using the whitespace characters tag\"\" and \"\", expecting the send to throw an exception") + public void testTagWithBlankSymbol() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + + String sendTagA = ""; + String sendTagB = " "; + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + Assertions.assertThrows(Exception.class, () -> { + producer.send(topic, sendTagA, SEND_NUM); + }, "Send messages with blank tag \"\", Expected send() to throw exception, but it didn't"); + Assertions.assertThrows(Exception.class, () -> { + producer.send(topic, sendTagB, SEND_NUM); + }, "Send messages with blank tag \" \", Expected send() to throw exception, but it didn't"); + } + + @Test + @DisplayName("The sent tag uses two strings with the same hash value, and the consumed tag uses BB, expecting to consume messages with tag=BB") + public void testSendTagWithSameHashCode_SubWithOne() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + String sendTagA = "BB"; + String sendTagB = "Aa"; + String receiveTag = "BB"; + pushConsumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer.subscribeAndStart(topic, MessageSelector.byTag(receiveTag), new RMQNormalListener()); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + producer.send(topic, sendTagA, SEND_NUM); + VerifyUtils.verifyNormalMessage(producer.getEnqueueMessages(), pushConsumer.getListener().getDequeueMessages()); + pushConsumer.getListener().clearMsg(); + producer.send(topic, sendTagB, SEND_NUM); + TestUtils.waitForSeconds(10); + Assertions.assertEquals(0, pushConsumer.getListener().getDequeueAllMessages().getDataSize()); + } + + @Test + @DisplayName("Send 10 messages with tag=BB, 10 messages with tag=bb, subscribe with tag=BB, expect case-sensitive messages to be consumed to tag=BB") + public void testTagCaseSensitive() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + String sendTagA = "BB"; + String sendTagB = "bb"; + String receiveTag = "BB"; + pushConsumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer.subscribeAndStart(topic, MessageSelector.byTag(receiveTag), new RMQNormalListener()); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + producer.send(topic, sendTagA, SEND_NUM); + VerifyUtils.verifyNormalMessage(producer.getEnqueueMessages(), pushConsumer.getListener().getDequeueMessages()); + pushConsumer.getListener().clearMsg(); + producer.send(topic, sendTagB, SEND_NUM); + TestUtils.waitForSeconds(10); + Assertions.assertEquals(0, pushConsumer.getListener().getDequeueAllMessages().getDataSize()); + } +} diff --git a/java/e2e-v4/src/test/java/org/apache/rocketmq/offset/OffsetTest.java b/java/e2e-v4/src/test/java/org/apache/rocketmq/offset/OffsetTest.java new file mode 100644 index 0000000..455305e --- /dev/null +++ b/java/e2e-v4/src/test/java/org/apache/rocketmq/offset/OffsetTest.java @@ -0,0 +1,311 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.offset; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.Map.Entry; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.consumer.MessageSelector; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.rmq.RMQNormalConsumer; +import org.apache.rocketmq.client.rmq.RMQNormalProducer; +import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.enums.TESTSET; +import org.apache.rocketmq.factory.ConsumerFactory; +import org.apache.rocketmq.factory.ProducerFactory; +import org.apache.rocketmq.frame.BaseOperate; +import org.apache.rocketmq.listener.rmq.concurrent.RMQNormalListener; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.server.batch.BatchProducerTest; +import org.apache.rocketmq.utils.NameUtils; +import org.apache.rocketmq.utils.RandomUtils; +import org.apache.rocketmq.utils.TestUtils; +import org.apache.rocketmq.utils.VerifyUtils; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.awaitility.Awaitility.await; + +@Tag(TESTSET.OFFSET) +public class OffsetTest extends BaseOperate { + private final Logger log = LoggerFactory.getLogger(OffsetTest.class); + private String tag; + private final static int SEND_NUM = 10; + + @BeforeEach + public void setUp() { + tag = NameUtils.getRandomTagName(); + } + + @Test + @DisplayName("Send 10 messages, set other groupid's pushconsumer consumption from first, expect to accept all messages again") + public void testConsumeFromFisrtOffset() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId1 = getGroupId(methodName); + String groupId2 = getGroupId(methodName); + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + + RMQNormalConsumer pullConsumer = ConsumerFactory.getRMQLitePullConsumer(namesrvAddr, groupId1, rpcHook, 1); + pullConsumer.subscribeAndStartLitePull(topic, MessageSelector.byTag(tag)); + VerifyUtils.tryReceiveOnce(pullConsumer.getLitePullConsumer()); + pullConsumer.shutdown(); + + DefaultMQPushConsumer pushConsumer; + ConcurrentLinkedDeque deque = new ConcurrentLinkedDeque<>(); + try { + pushConsumer = new DefaultMQPushConsumer(groupId1, rpcHook, new AllocateMessageQueueAveragely()); + pushConsumer.setInstanceName(RandomUtils.getStringByUUID()); + pushConsumer.setNamesrvAddr(namesrvAddr); + pushConsumer.subscribe(topic, tag); + pushConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); + pushConsumer.setMessageListener(new MessageListenerConcurrently() { + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + for (MessageExt message : msgs) { + log.info("receive message:{}", message); + deque.add(message); + } + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + pushConsumer.start(); + } catch (MQClientException e) { + throw new RuntimeException(e); + } + + Assertions.assertNotNull(producer); + + producer.send(topic, tag, SEND_NUM); + + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + + await().atMost(30, SECONDS).until(new Callable() { + @Override + public Boolean call() throws Exception { + return deque.size() == SEND_NUM; + } + }); + + pushConsumer.shutdown(); + deque.clear(); + + DefaultMQPushConsumer reConsumer; + try { + reConsumer = new DefaultMQPushConsumer(groupId2, rpcHook, new AllocateMessageQueueAveragely()); + reConsumer.setInstanceName(RandomUtils.getStringByUUID()); + reConsumer.setNamesrvAddr(namesrvAddr); + reConsumer.subscribe(topic, tag); + reConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); + reConsumer.setMessageListener(new MessageListenerConcurrently() { + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + for (MessageExt message : msgs) { + log.info("reconsumer received message:{}", message); + deque.add(message); + } + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + reConsumer.start(); + } catch (MQClientException e) { + throw new RuntimeException(e); + } + + TestUtils.waitForSeconds(30); + + Assertions.assertEquals(SEND_NUM, deque.size(), "reconsumer receive message failed"); + + reConsumer.shutdown(); + producer.shutdown(); + + } + + @Test + @DisplayName("Backlog 100 messages, start the consumer, and set the pull message from the LAST, expect to consume 100 messages") + public void testConsumeFromLastOffset() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + + Assertions.assertNotNull(producer); + + producer.send(topic, tag, 100); + + Assertions.assertEquals(100, producer.getEnqueueMessages().getDataSize(), "send message failed"); + + DefaultMQPushConsumer pushConsumer; + ConcurrentLinkedDeque deque = new ConcurrentLinkedDeque<>(); + try { + pushConsumer = new DefaultMQPushConsumer(groupId, rpcHook, new AllocateMessageQueueAveragely()); + pushConsumer.setInstanceName(RandomUtils.getStringByUUID()); + pushConsumer.setNamesrvAddr(namesrvAddr); + pushConsumer.subscribe(topic, tag); + pushConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); + pushConsumer.setMessageListener(new MessageListenerConcurrently() { + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + for (MessageExt message : msgs) { + log.info("receive message:{}", message); + deque.add(message); + } + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + pushConsumer.start(); + } catch (MQClientException e) { + throw new RuntimeException(e); + } + + TestUtils.waitForSeconds(30); + Assertions.assertEquals(100, deque.size(), "consumer receive message failed"); + + pushConsumer.shutdown(); + producer.shutdown(); + + } + + @Test + @DisplayName("send 10 messages, PullConsumer normally receives messages, but does not update messages offset, expect the messages are receive again") + public void test_pull_receive_nack() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + RMQNormalConsumer consumer = ConsumerFactory.getRMQPullConsumer(namesrvAddr, groupId, rpcHook); + consumer.startDefaultPull(); + VerifyUtils.tryReceiveOnce(consumer.getPullConsumer(), topic, tag, 32); + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + + producer.send(topic, tag, SEND_NUM); + + TestUtils.waitForSeconds(1); + + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + + Set receiveMessageQueues = null; + try { + receiveMessageQueues = consumer.getPullConsumer().fetchSubscribeMessageQueues(topic); + } catch (MQClientException e) { + Assertions.fail("Fail to fetchSubscribeMessageQueues"); + } + + Collection sendCollection = Collections + .synchronizedCollection(producer.getEnqueueMessages().getAllData()); + Set finalMessageQueues = receiveMessageQueues; + CompletableFuture[] futures = new CompletableFuture[receiveMessageQueues.size()]; + ConcurrentHashMap messageMap = new ConcurrentHashMap<>(); + int mqCount = 0; + for (MessageQueue mq : finalMessageQueues) { + CompletableFuture future = CompletableFuture.supplyAsync(() -> { + try { + long offset = consumer.getPullConsumer().fetchConsumeOffset(mq, false); + if (offset < 0) + return null; + long startTime = System.currentTimeMillis(); + while (System.currentTimeMillis() < startTime + 5000) { + PullResult pullResult = consumer.getPullConsumer().pull(mq, tag, offset, SEND_NUM); + switch (pullResult.getPullStatus()) { + case FOUND: + List messages = pullResult.getMsgFoundList(); + for (MessageExt message : messages) { + log.info("MessageId:{}, Body:{}, Property:{}, Retry:{}", message.getMsgId(), + StandardCharsets.UTF_8.decode(ByteBuffer.wrap(message.getBody())), + message.getProperties(), message.getReconsumeTimes()); + sendCollection + .removeIf(messageExt -> messageExt.getMsgId().equals(message.getMsgId())); + if (messageMap.containsKey(message.getMsgId())) { + messageMap.get(message.getMsgId()).incrementAndGet(); + } else { + messageMap.put(message.getMsgId(), new AtomicInteger(1)); + } + } + break; + case NO_MATCHED_MSG: + break; + case NO_NEW_MSG: + break; + case OFFSET_ILLEGAL: + break; + default: + break; + } + } + } catch (MQBrokerException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } catch (RemotingException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } catch (InterruptedException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } catch (MQClientException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } + return null; + }); + futures[mqCount++] = future; + } + try { + CompletableFuture.allOf(futures).get(60, TimeUnit.SECONDS); + } catch (Exception e) { + e.printStackTrace(); + Assertions.fail("receive response count not match"); + } + log.info("A total of {} messages were received", messageMap.size()); + for (Entry entry : messageMap.entrySet()) { + Assertions.assertTrue(entry.getValue().get() >= 1, "message receive count not match"); + } + } +} diff --git a/java/e2e-v4/src/test/java/org/apache/rocketmq/pull/PullAckTest.java b/java/e2e-v4/src/test/java/org/apache/rocketmq/pull/PullAckTest.java new file mode 100644 index 0000000..46b1659 --- /dev/null +++ b/java/e2e-v4/src/test/java/org/apache/rocketmq/pull/PullAckTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.pull; + +import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.client.rmq.RMQNormalConsumer; +import org.apache.rocketmq.client.rmq.RMQNormalProducer; +import org.apache.rocketmq.enums.TESTSET; +import org.apache.rocketmq.factory.ConsumerFactory; +import org.apache.rocketmq.factory.MessageFactory; +import org.apache.rocketmq.factory.ProducerFactory; +import org.apache.rocketmq.frame.BaseOperate; +import org.apache.rocketmq.utils.NameUtils; +import org.apache.rocketmq.utils.TestUtils; +import org.apache.rocketmq.utils.VerifyUtils; +import org.junit.jupiter.api.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Tag(TESTSET.NORMAL) +@Tag(TESTSET.PULL) +public class PullAckTest extends BaseOperate { + private final Logger log = LoggerFactory.getLogger(PullAckTest.class); + private String tag; + private final static int SEND_NUM = 20; + private RMQNormalProducer producer; + + @BeforeAll + public static void setUpAll() { + } + + @BeforeEach + public void setUp() { + tag = NameUtils.getRandomTagName(); + log.info("tag:{}", tag); + } + + @AfterEach + public void tearDown() { + if (producer != null) { + producer.shutdown(); + } + } + + @Test + @Timeout(180) + @DisplayName("Send 20 normal messages synchronously and expect lite pull consumer to consume with receive and ack messages successful") + public void testNormal_lite_pull_receive_ack() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + RMQNormalConsumer pullConsumer = ConsumerFactory.getRMQLitePullConsumer(namesrvAddr, groupId, rpcHook, 1); + pullConsumer.startLitePullAssignMode(); + VerifyUtils.tryReceiveOnce(pullConsumer.getLitePullConsumer()); + + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + + for (int i = 0; i < SEND_NUM; i++) { + Message message = MessageFactory.buildNormalMessage(topic, tag, tag + "-" + i); + producer.send(message); + } + TestUtils.waitForSeconds(1); + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + VerifyUtils.waitLitePullReceiveThenAck(producer, pullConsumer.getLitePullConsumer(), topic, tag); + } + + @Test + @Timeout(180) + @DisplayName("Send 20 normal messages synchronously and expect pull consumer to consume with receive and ack messages successful") + public void testNormal_pull_receive_ack() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + RMQNormalConsumer consumer = ConsumerFactory.getRMQPullConsumer(namesrvAddr, groupId, rpcHook); + consumer.startDefaultPull(); + VerifyUtils.tryReceiveOnce(consumer.getPullConsumer(), topic, tag, 32); + producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + for (int i = 0; i < SEND_NUM; i++) { + Message message = MessageFactory.buildNormalMessage(topic, tag, tag + "-" + i); + producer.send(message); + } + TestUtils.waitForSeconds(1); + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + VerifyUtils.waitPullReceiveThenAck(producer, consumer.getPullConsumer(), topic, tag, 32); + } + +} diff --git a/java/e2e-v4/src/test/java/org/apache/rocketmq/pull/PullOrderParamTest.java b/java/e2e-v4/src/test/java/org/apache/rocketmq/pull/PullOrderParamTest.java new file mode 100644 index 0000000..ce248bb --- /dev/null +++ b/java/e2e-v4/src/test/java/org/apache/rocketmq/pull/PullOrderParamTest.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.pull; + +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.rmq.RMQNormalConsumer; +import org.apache.rocketmq.client.rmq.RMQNormalProducer; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.enums.TESTSET; +import org.apache.rocketmq.factory.ConsumerFactory; +import org.apache.rocketmq.factory.MessageFactory; +import org.apache.rocketmq.factory.ProducerFactory; +import org.apache.rocketmq.frame.BaseOperate; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.utils.NameUtils; +import org.apache.rocketmq.utils.TestUtils; +import org.apache.rocketmq.utils.VerifyUtils; +import org.junit.jupiter.api.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +@Tag(TESTSET.PULL) +public class PullOrderParamTest extends BaseOperate { + private final Logger log = LoggerFactory.getLogger(PullOrderParamTest.class); + private String tag; + private String groupId; + private final static int SEND_NUM = 20; + + @BeforeEach + public void setUp() { + tag = NameUtils.getRandomTagName(); + groupId = NameUtils.getRandomGroupName(); + log.info("tag:{}, groupId:{}", tag, groupId); + } + + @AfterEach + public void tearDown() { + } + + @Test + @DisplayName("When sending 20 sequential messages synchronously using the same MessageQueue, PullConsumer normally receives messages, but does not ack messages, and keeps the sequence; the messages are stuck at the first") + public void testFIFO_pull_receive_nack() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + RMQNormalConsumer consumer = ConsumerFactory.getRMQPullConsumer(namesrvAddr, groupId, rpcHook); + consumer.startDefaultPull(); + VerifyUtils.tryReceiveOnce(consumer.getPullConsumer(), topic, tag, 32); + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + + List messageQueues = producer.fetchPublishMessageQueues(topic); + List messageGroup = new ArrayList<>(); + messageGroup.add(messageQueues.get(0)); + producer.sendWithQueue(messageGroup, tag, SEND_NUM); + + TestUtils.waitForSeconds(1); + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + + Set receiveMessageQueues = null; + try { + receiveMessageQueues = consumer.getPullConsumer().fetchSubscribeMessageQueues(topic); + } catch (MQClientException e) { + Assertions.fail("Fail to fetchSubscribeMessageQueues"); + } + + Collection sendCollection = Collections + .synchronizedCollection(producer.getEnqueueMessages().getAllData()); + Set finalMessageQueues = receiveMessageQueues; + CompletableFuture[] futures = new CompletableFuture[receiveMessageQueues.size()]; + List receivedMessage = new ArrayList<>(); + int mqCount = 0; + for (MessageQueue mq : finalMessageQueues) { + CompletableFuture future = CompletableFuture.supplyAsync(() -> { + try { + long offset = consumer.getPullConsumer().fetchConsumeOffset(mq, false); + if (offset < 0) + return null; + long startTime = System.currentTimeMillis(); + while (System.currentTimeMillis() < startTime + 30000) { + PullResult pullResult = consumer.getPullConsumer().pull(mq, tag, offset, 1); + switch (pullResult.getPullStatus()) { + case FOUND: + List messages = pullResult.getMsgFoundList(); + for (MessageExt message : messages) { + log.info("MessageId:{}, Body:{}, Property:{}, Retry:{}", message.getMsgId(), + StandardCharsets.UTF_8.decode(ByteBuffer.wrap(message.getBody())), + message.getProperties(), message.getReconsumeTimes()); + sendCollection + .removeIf(messageExt -> messageExt.getMsgId().equals(message.getMsgId())); + receivedMessage.add(message); + } + break; + case NO_MATCHED_MSG: + break; + case NO_NEW_MSG: + break; + case OFFSET_ILLEGAL: + break; + default: + break; + } + } + } catch (MQBrokerException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } catch (RemotingException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } catch (InterruptedException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } catch (MQClientException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } + return null; + }); + futures[mqCount++] = future; + } + try { + CompletableFuture.allOf(futures).get(60, TimeUnit.SECONDS); + } catch (Exception e) { + e.printStackTrace(); + Assertions.fail("receive response count not match"); + } + log.info("A total of {} messages were received", receivedMessage.size()); + for (MessageExt ext : receivedMessage) { + if (!StandardCharsets.UTF_8.decode(ByteBuffer.wrap(ext.getBody())).toString().equals("0")) { + Assertions.fail(String.format("Consumption out of order, expected :Body=%s Actual :Body=%s", 0, + StandardCharsets.UTF_8.decode(ByteBuffer.wrap(ext.getBody())))); + } + } + } +} diff --git a/java/e2e-v4/src/test/java/org/apache/rocketmq/pull/PullOrderTest.java b/java/e2e-v4/src/test/java/org/apache/rocketmq/pull/PullOrderTest.java new file mode 100644 index 0000000..5d25fca --- /dev/null +++ b/java/e2e-v4/src/test/java/org/apache/rocketmq/pull/PullOrderTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.pull; + +import org.apache.rocketmq.client.rmq.RMQNormalConsumer; +import org.apache.rocketmq.client.rmq.RMQNormalProducer; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.enums.TESTSET; +import org.apache.rocketmq.factory.ConsumerFactory; +import org.apache.rocketmq.factory.MessageFactory; +import org.apache.rocketmq.factory.ProducerFactory; +import org.apache.rocketmq.frame.BaseOperate; +import org.apache.rocketmq.utils.NameUtils; +import org.apache.rocketmq.utils.TestUtils; +import org.apache.rocketmq.utils.VerifyUtils; +import org.junit.jupiter.api.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +@Tag(TESTSET.PULL) +public class PullOrderTest extends BaseOperate { + private final Logger log = LoggerFactory.getLogger(PullOrderTest.class); + private String tag; + private String groupId; + private final static int SEND_NUM = 20; + + @BeforeEach + public void setUp() { + tag = NameUtils.getRandomTagName(); + groupId = NameUtils.getRandomGroupName(); + log.info("tag:{}, groupId:{}", tag, groupId); + } + + @Test + @DisplayName("Send 20 sequential messages synchronously, and expect PullConsumer to receive and ack messages properly and maintain the sequence") + public void testFIFO_simple_receive_ack() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + RMQNormalConsumer consumer = ConsumerFactory.getRMQPullConsumer(namesrvAddr, groupId, rpcHook); + consumer.startDefaultPull(); + VerifyUtils.tryReceiveOnce(consumer.getPullConsumer(), topic, tag, 32); + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + + List messageQueues = producer.fetchPublishMessageQueues(topic); + List messageGroup = new ArrayList<>(); + messageGroup.add(messageQueues.get(0)); + producer.sendWithQueue(messageGroup, tag, SEND_NUM); + TestUtils.waitForSeconds(1); + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + VerifyUtils.waitFIFOReceiveThenAck(producer, consumer.getPullConsumer(), topic, tag, 5); + } + +} diff --git a/java/e2e-v4/src/test/java/org/apache/rocketmq/pull/PullParamTest.java b/java/e2e-v4/src/test/java/org/apache/rocketmq/pull/PullParamTest.java new file mode 100644 index 0000000..8607e64 --- /dev/null +++ b/java/e2e-v4/src/test/java/org/apache/rocketmq/pull/PullParamTest.java @@ -0,0 +1,294 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.pull; + +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.rmq.RMQNormalConsumer; +import org.apache.rocketmq.client.rmq.RMQNormalProducer; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.enums.TESTSET; +import org.apache.rocketmq.factory.ConsumerFactory; +import org.apache.rocketmq.factory.MessageFactory; +import org.apache.rocketmq.factory.ProducerFactory; +import org.apache.rocketmq.frame.BaseOperate; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.utils.NameUtils; +import org.apache.rocketmq.utils.RandomUtils; +import org.apache.rocketmq.utils.TestUtils; +import org.apache.rocketmq.utils.VerifyUtils; +import org.apache.rocketmq.utils.data.collect.DataCollector; +import org.apache.rocketmq.utils.data.collect.DataCollectorManager; +import org.junit.jupiter.api.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +@Tag(TESTSET.PULL) +public class PullParamTest extends BaseOperate { + private final Logger log = LoggerFactory.getLogger(PullParamTest.class); + private String tag; + private String groupId; + + @BeforeEach + public void setUp() { + tag = NameUtils.getRandomTagName(); + groupId = NameUtils.getRandomGroupName(); + log.info("tag:{}, groupId:{}", tag, groupId); + } + + @AfterEach + public void tearDown() { + } + + @Test + @DisplayName("Send 300 normal messages synchronously, and after using PullConsumer receive 30 messages, ack them after consuming them, expecting each receive to be less than or equal to 32 messages, and never receive the ack messages again") + public void testNormal_pull_receive_maxsize_sync() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + int sendNum = 300; + RMQNormalConsumer consumer = ConsumerFactory.getRMQPullConsumer(namesrvAddr, groupId, rpcHook); + consumer.startDefaultPull(); + VerifyUtils.tryReceiveOnce(consumer.getPullConsumer(), topic, tag, 32); + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + + for (int i = 0; i < sendNum; i++) { + Message message = MessageFactory.buildNormalMessage(topic, tag, RandomUtils.getStringByUUID()); + producer.send(message); + } + Assertions.assertEquals(sendNum, producer.getEnqueueMessages().getDataSize(), "send message failed"); + + Set receiveMessageQueues = null; + try { + receiveMessageQueues = consumer.getPullConsumer().fetchSubscribeMessageQueues(topic); + } catch (MQClientException e) { + Assertions.fail("Fail to fetchSubscribeMessageQueues"); + } + + Set finalMessageQueues = receiveMessageQueues; + CompletableFuture[] futures = new CompletableFuture[receiveMessageQueues.size()]; + Map recvMsgs = new ConcurrentHashMap<>(); + int mqCount = 0; + for (MessageQueue mq : finalMessageQueues) { + CompletableFuture future = CompletableFuture.supplyAsync(() -> { + try { + long offset = consumer.getPullConsumer().fetchConsumeOffset(mq, false); + if (offset < 0) + return null; + boolean shouldContinue = true; + while (shouldContinue) { + PullResult pullResult = consumer.getPullConsumer().pull(mq, tag, offset, 50); + switch (pullResult.getPullStatus()) { + case FOUND: + List messages = pullResult.getMsgFoundList(); + Assertions.assertTrue(messages.size() <= 32); + for (MessageExt message : messages) { + log.info("MessageId:{}, Body:{}, Property:{}, Retry:{}", message.getMsgId(), + StandardCharsets.UTF_8.decode(ByteBuffer.wrap(message.getBody())), + message.getProperties(), message.getReconsumeTimes()); + if (recvMsgs.containsKey(message.getMsgId())) { + Assertions.fail("Consume an ack message: " + message.getMsgId()); + } else { + recvMsgs.put(message.getMsgId(), message); + } + } + offset = pullResult.getNextBeginOffset(); + consumer.getPullConsumer().updateConsumeOffset(mq, offset); + break; + case NO_MATCHED_MSG: + shouldContinue = false; + break; + case NO_NEW_MSG: + shouldContinue = false; + break; + case OFFSET_ILLEGAL: + shouldContinue = false; + break; + default: + break; + } + } + } catch (MQBrokerException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } catch (RemotingException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } catch (InterruptedException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } catch (MQClientException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } + return null; + }); + futures[mqCount++] = future; + } + try { + CompletableFuture.allOf(futures).get(60, TimeUnit.SECONDS); + } catch (Exception e) { + e.printStackTrace(); + Assertions.fail("receive response count not match"); + } + DataCollector dequeueMessages = DataCollectorManager.getInstance() + .fetchListDataCollector(RandomUtils.getStringByUUID()); + for (MessageExt messageExt : recvMsgs.values()) { + dequeueMessages.addData(messageExt); + } + log.info("{} messages are received", dequeueMessages.getDataSize()); + VerifyUtils.verifyNormalMessage(producer.getEnqueueMessages(), dequeueMessages, 10); + } + + @Test + @DisplayName("Twenty ordinary messages are sent synchronously, and receive(50) messages are received in batch. All the pulled messages are ack() messages except the last one. expected the ack messages will not be consumed repeatedly") + public void testNormal_simple_receive_multi_nack() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + int sendNum = 20; + RMQNormalConsumer consumer = ConsumerFactory.getRMQPullConsumer(namesrvAddr, groupId, rpcHook); + consumer.startDefaultPull(); + VerifyUtils.tryReceiveOnce(consumer.getPullConsumer(), topic, tag, 32); + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + + for (int i = 0; i < sendNum; i++) { + Message message = MessageFactory.buildNormalMessage(topic, tag, String.valueOf(i)); + producer.send(message); + } + Assertions.assertEquals(sendNum, producer.getEnqueueMessages().getDataSize(), "send message failed"); + + Set receiveMessageQueues = null; + try { + receiveMessageQueues = consumer.getPullConsumer().fetchSubscribeMessageQueues(topic); + } catch (MQClientException e) { + Assertions.fail("Fail to fetchSubscribeMessageQueues"); + } + + Map recvMsgs = new ConcurrentHashMap<>(); + Set unconsumedMsgIds = new HashSet<>(); + boolean[] flag = { true }; + Set finalMessageQueues = receiveMessageQueues; + CompletableFuture[] futures = new CompletableFuture[receiveMessageQueues.size()]; + int mqCount = 0; + for (MessageQueue mq : finalMessageQueues) { + CompletableFuture future = CompletableFuture.supplyAsync(() -> { + try { + long offset = consumer.getPullConsumer().fetchConsumeOffset(mq, false); + if (offset < 0) + return null; + long startTime = System.currentTimeMillis(); + while (System.currentTimeMillis() < startTime + 40000) { + PullResult pullResult = consumer.getPullConsumer().pull(mq, tag, offset, 50); + switch (pullResult.getPullStatus()) { + case FOUND: + List messages = pullResult.getMsgFoundList(); + Assertions.assertTrue(messages.size() <= 32); + for (MessageExt message : messages) { + log.info("MessageId:{}, Body:{}, Property:{}, Retry:{}", message.getMsgId(), + StandardCharsets.UTF_8.decode(ByteBuffer.wrap(message.getBody())), + message.getProperties(), message.getReconsumeTimes()); + int msgId = Integer.parseInt(String.valueOf( + StandardCharsets.UTF_8.decode(ByteBuffer.wrap(message.getBody())))); + if (msgId == 19 && flag[0]) { + flag[0] = false; + unconsumedMsgIds.add(message.getMsgId()); + log.info("nack message:{} {}", + StandardCharsets.UTF_8.decode(ByteBuffer.wrap(message.getBody())), + message); + } else { + if (msgId == 19) { + unconsumedMsgIds.add(message.getMsgId()); + } else { + consumer.getPullConsumer().updateConsumeOffset(mq, ++offset); + if (recvMsgs.containsKey(message.getMsgId())) { + Assertions.fail("Consume an ack message"); + } else { + recvMsgs.put(message.getMsgId(), message); + } + } + } + } + if (messages.size() != 1 || Integer.parseInt(String.valueOf(StandardCharsets.UTF_8 + .decode(ByteBuffer.wrap(messages.get(0).getBody())))) != 19) { + return null; + } + break; + case NO_MATCHED_MSG: + break; + case NO_NEW_MSG: + break; + case OFFSET_ILLEGAL: + break; + default: + break; + } + if (recvMsgs.size() == sendNum) { + break; + } + } + } catch (MQBrokerException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } catch (RemotingException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } catch (InterruptedException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } catch (MQClientException e) { + e.printStackTrace(); + Assertions.fail("Pull fetch message error"); + } + return null; + }); + futures[mqCount++] = future; + } + try { + CompletableFuture.allOf(futures).get(60, TimeUnit.SECONDS); + } catch (Exception e) { + e.printStackTrace(); + Assertions.fail("receive response count not match"); + } + + DataCollector dequeueMessages = DataCollectorManager.getInstance() + .fetchListDataCollector(RandomUtils.getStringByUUID()); + for (MessageExt messageExt : recvMsgs.values()) { + dequeueMessages.addData(messageExt); + } + + VerifyUtils.verifyNormalMessage(producer.getEnqueueMessages(), dequeueMessages, unconsumedMsgIds, 10); + + } + +} diff --git a/java/e2e-v4/src/test/java/org/apache/rocketmq/server/abnormal/PushConsumerRetryTest.java b/java/e2e-v4/src/test/java/org/apache/rocketmq/server/abnormal/PushConsumerRetryTest.java new file mode 100644 index 0000000..09ec0a2 --- /dev/null +++ b/java/e2e-v4/src/test/java/org/apache/rocketmq/server/abnormal/PushConsumerRetryTest.java @@ -0,0 +1,826 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.server.abnormal; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.Vector; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.consumer.MessageSelector; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus; +import org.apache.rocketmq.client.consumer.listener.MessageListener; +import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly; +import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.rmq.RMQNormalConsumer; +import org.apache.rocketmq.client.rmq.RMQNormalProducer; +import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.enums.TESTSET; +import org.apache.rocketmq.factory.ConsumerFactory; +import org.apache.rocketmq.factory.MessageFactory; +import org.apache.rocketmq.factory.ProducerFactory; +import org.apache.rocketmq.frame.BaseOperate; +import org.apache.rocketmq.listener.rmq.concurrent.RMQNormalListener; +import org.apache.rocketmq.utils.NameUtils; +import org.apache.rocketmq.utils.RandomUtils; +import org.apache.rocketmq.utils.VerifyUtils; +import org.apache.rocketmq.utils.TestUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Tag(TESTSET.RETRY) +public class PushConsumerRetryTest extends BaseOperate { + private final Logger log = LoggerFactory.getLogger(PushConsumerRetryTest.class); + private String tag; + private final static int SEND_NUM = 5; + + @BeforeEach + public void setUp() { + tag = NameUtils.getRandomTagName(); + } + + @Test + @Timeout(value = 180, unit = TimeUnit.SECONDS) + @DisplayName("Send normal messages, set the maximum number of retries and set the received messages to RECONSUME_LATER. The expected retry time is about 10s for the first time and about 30s for the second time") + public void testNormalMessageReconsumeTime() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + + Map msgsReConsumeTime = new ConcurrentHashMap<>(); + Map msgsReachedTwoReconsumeTimes = new ConcurrentHashMap<>(); + Lock lock = new ReentrantLock(); + + DefaultMQPushConsumer pushConsumer = null; + try { + pushConsumer = new DefaultMQPushConsumer(groupId, rpcHook, new AllocateMessageQueueAveragely()); + pushConsumer.setInstanceName(RandomUtils.getStringByUUID()); + pushConsumer.setNamesrvAddr(namesrvAddr); + pushConsumer.subscribe(topic, tag); + pushConsumer.setMessageModel(MessageModel.CLUSTERING); + pushConsumer.setMaxReconsumeTimes(2); + pushConsumer.setConsumeTimeout(60000); + pushConsumer.setMessageListener(new MessageListenerConcurrently() { + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + for (MessageExt msg : msgs) { + if (msg.getReconsumeTimes() == 0) { + msgsReConsumeTime.putIfAbsent(msg.getMsgId(), Instant.now()); + msgsReachedTwoReconsumeTimes.putIfAbsent(msg.getMsgId(), false); + log.info(String.format("recv msg(first) %s ", msg)); + } else { + Instant nowTime = Instant.now(); + if (msgsReConsumeTime.containsKey(msg.getMsgId())) { + lock.lock(); + try { + Instant lastTime = msgsReConsumeTime.get(msg.getMsgId()); + Duration duration = Duration.between(lastTime, nowTime); + if (msg.getReconsumeTimes() == 1) { + log.info("first retry time is: {}", duration.getSeconds()); + Assertions.assertTrue(duration.getSeconds() < 20); + } else { + log.info("second retry time is: {}", duration.getSeconds()); + Assertions.assertTrue(duration.getSeconds() < 60); + msgsReachedTwoReconsumeTimes.put(msg.getMsgId(), true); + } + msgsReConsumeTime.put(msg.getMsgId(), nowTime); + } finally { + lock.unlock(); + } + } else { + msgsReConsumeTime.putIfAbsent(msg.getMsgId(), Instant.now()); + } + + log.info(String.format("recv msgid(reconsume later) %s ", msg.getMsgId())); + } + } + for (MessageExt msg : msgs) { + if (msg.getReconsumeTimes() != 2) { + return ConsumeConcurrentlyStatus.RECONSUME_LATER; + } + } + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + + }); + pushConsumer.start(); + } catch (MQClientException e) { + Assertions.fail(e.getMessage()); + } + + Assertions.assertNotNull(producer, "Get Producer Failed"); + for (int i = 0; i < SEND_NUM; i++) { + Message message = MessageFactory.buildNormalMessage(topic, tag, String.valueOf(i)); + producer.send(message); + } + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + + await().atMost(120, SECONDS).until(new Callable() { + @Override + public Boolean call() throws Exception { + boolean flag = true; + for (Map.Entry entry : msgsReachedTwoReconsumeTimes.entrySet()) { + if (!entry.getValue()) { + flag = false; + break; + } + } + return msgsReachedTwoReconsumeTimes.size() == SEND_NUM && flag; + } + }); + + producer.shutdown(); + pushConsumer.shutdown(); + } + + @Test + @Timeout(value = 60, unit = TimeUnit.SECONDS) + @DisplayName("Send order messages, set the maximum number of retries and set the received messages to SUSPEND_CURRENT_QUEUE_A_MOMENT. The expected retry time is about 1s") + public void testOrderMessageReconsumeTime() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + + Map msgsReConsumeTime = new ConcurrentHashMap<>(); + Map msgsReachedTwoReconsumeTimes = new ConcurrentHashMap<>(); + Lock lock = new ReentrantLock(); + + DefaultMQPushConsumer pushConsumer = null; + try { + pushConsumer = new DefaultMQPushConsumer(groupId, rpcHook, new AllocateMessageQueueAveragely()); + pushConsumer.setInstanceName(RandomUtils.getStringByUUID()); + pushConsumer.setNamesrvAddr(namesrvAddr); + pushConsumer.subscribe(topic, tag); + pushConsumer.setMessageModel(MessageModel.CLUSTERING); + pushConsumer.setMaxReconsumeTimes(2); + pushConsumer.setConsumeTimeout(60000); + pushConsumer.setMessageListener(new MessageListenerOrderly() { + @Override + public ConsumeOrderlyStatus consumeMessage(List msgs, + ConsumeOrderlyContext context) { + for (MessageExt msg : msgs) { + if (msg.getReconsumeTimes() == 0) { + msgsReConsumeTime.putIfAbsent(msg.getMsgId(), Instant.now()); + msgsReachedTwoReconsumeTimes.putIfAbsent(msg.getMsgId(), false); + log.info(String.format("recv msg(first) %s ", msg)); + } else { + Instant nowTime = Instant.now(); + if (msgsReConsumeTime.containsKey(msg.getMsgId())) { + lock.lock(); + try { + Instant lastTime = msgsReConsumeTime.get(msg.getMsgId()); + Duration duration = Duration.between(lastTime, nowTime); + if (msg.getReconsumeTimes() == 1) { + log.info("first retry time is: {}", duration.getSeconds()); + Assertions.assertTrue(duration.getSeconds() < 2); + } else { + log.info("second retry time is: {}", duration.getSeconds()); + Assertions.assertTrue(duration.getSeconds() < 2); + msgsReachedTwoReconsumeTimes.put(msg.getMsgId(), true); + } + msgsReConsumeTime.put(msg.getMsgId(), nowTime); + } finally { + lock.unlock(); + } + } else { + msgsReConsumeTime.putIfAbsent(msg.getMsgId(), Instant.now()); + } + + log.info(String.format("recv msgid(reconsume later) %s ", msg.getMsgId())); + } + } + for (MessageExt msg : msgs) { + if (msg.getReconsumeTimes() != 2) { + return ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT; + } + } + return ConsumeOrderlyStatus.SUCCESS; + } + + }); + pushConsumer.start(); + } catch (MQClientException e) { + Assertions.fail(e.getMessage()); + } + + Assertions.assertNotNull(producer, "Get Producer Failed"); + List mqs = producer.fetchPublishMessageQueues(topic); + List sendMqs = new ArrayList<>(); + sendMqs.add(mqs.get(0)); + producer.sendWithQueue(sendMqs, SEND_NUM, tag); + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + + await().atMost(120, SECONDS).until(new Callable() { + @Override + public Boolean call() throws Exception { + boolean flag = true; + for (Map.Entry entry : msgsReachedTwoReconsumeTimes.entrySet()) { + if (!entry.getValue()) { + flag = false; + break; + } + } + return msgsReachedTwoReconsumeTimes.size() == SEND_NUM && flag; + } + }); + + producer.shutdown(); + pushConsumer.shutdown(); + } + + @Test + @Timeout(value = 180, unit = TimeUnit.SECONDS) + @DisplayName("Send normal messages, set the maximum consumption to 0(The first retry is 10 seconds, and the setting is 15 seconds. Then check whether the retry occurs), and set the message reception to RECONUME_LATER, expecting that the retry will not occur") + public void testNormalMessageRetryTimesSetting() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + + Set msgsRecv = new ConcurrentSkipListSet<>(); + + DefaultMQPushConsumer pushConsumer = null; + try { + pushConsumer = new DefaultMQPushConsumer(groupId, rpcHook, new AllocateMessageQueueAveragely()); + pushConsumer.setInstanceName(RandomUtils.getStringByUUID()); + pushConsumer.setNamesrvAddr(namesrvAddr); + pushConsumer.subscribe(topic, tag); + pushConsumer.setMessageModel(MessageModel.CLUSTERING); + pushConsumer.setMaxReconsumeTimes(0); + pushConsumer.setMessageListener(new MessageListenerConcurrently() { + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + for (MessageExt msg : msgs) { + if (msg.getReconsumeTimes() > 0) { + Assertions.fail("retry times is not equal to maxReconsumeTimes"); + } else { + log.info("retry times is equal to maxReconsumeTimes: {}", msg.getMsgId()); + msgsRecv.add(msg.getMsgId()); + log.info("recv msgid(reconsume later) {} ", msg.getMsgId()); + } + } + return ConsumeConcurrentlyStatus.RECONSUME_LATER; + } + + }); + pushConsumer.start(); + } catch (MQClientException e) { + Assertions.fail(e.getMessage()); + } + + Assertions.assertNotNull(producer, "Get Producer Failed"); + for (int i = 0; i < SEND_NUM; i++) { + Message message = MessageFactory.buildNormalMessage(topic, tag, String.valueOf(i)); + producer.send(message); + } + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + + TestUtils.waitForSeconds(15); + + Assertions.assertEquals(SEND_NUM, msgsRecv.size(), + "retry message size is not equal to send message size"); + + producer.shutdown(); + pushConsumer.shutdown(); + } + + @Test + @Timeout(value = 60, unit = TimeUnit.SECONDS) + @DisplayName("Send one order message, set the maximum consumption to 0(The retry time of each message is 1s. Then check whether the retry occurs), and set the message reception to SUSPEND_CURRENT_QUEUE_A_MOMENT, expecting that the retry will not occur") + public void testOrderMessageRetryTimesSetting() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + + Map msgsReConsumeTimes = new ConcurrentHashMap<>(); + + DefaultMQPushConsumer pushConsumer = null; + try { + pushConsumer = new DefaultMQPushConsumer(groupId, rpcHook, new AllocateMessageQueueAveragely()); + pushConsumer.setInstanceName(RandomUtils.getStringByUUID()); + pushConsumer.setNamesrvAddr(namesrvAddr); + pushConsumer.subscribe(topic, tag); + pushConsumer.setMessageModel(MessageModel.CLUSTERING); + pushConsumer.setMaxReconsumeTimes(30); + pushConsumer.setMessageListener(new MessageListenerOrderly() { + @Override + public ConsumeOrderlyStatus consumeMessage(List msgs, + ConsumeOrderlyContext context) { + for (MessageExt msg : msgs) { + if (msg.getReconsumeTimes() > 30) { + Assertions.fail("retry times is not equal to maxReconsumeTimes"); + } else { + msgsReConsumeTimes.put(msg.getMsgId(), msg.getReconsumeTimes()); + log.info("recv msgid(reconsume later) %s, reconsumer times is {}", msg.getMsgId()); + } + } + return ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT; + } + + }); + pushConsumer.start(); + } catch (MQClientException e) { + Assertions.fail(e.getMessage()); + } + + Assertions.assertNotNull(producer, "Get Producer Failed"); + List mqs = producer.fetchPublishMessageQueues(topic); + List sendMqs = new ArrayList<>(); + sendMqs.add(mqs.get(0)); + producer.sendWithQueue(sendMqs, 1, tag); + Assertions.assertEquals(1, producer.getEnqueueMessages().getDataSize(), "send message failed"); + + TestUtils.waitForSeconds(35); + + Assertions.assertEquals(1, msgsReConsumeTimes.size(), + "retry message size is not equal to send message size"); + for (Map.Entry entry : msgsReConsumeTimes.entrySet()) { + Assertions.assertEquals(30, entry.getValue(), "retry times is not equal to maxReconsumeTimes"); + } + + producer.shutdown(); + pushConsumer.shutdown(); + } + + @Test + @Timeout(value = 180, unit = TimeUnit.SECONDS) + @DisplayName("Send order messages, set the maximum consumption to 30, and set the message reception to SUSPEND_CURRENT_QUEUE_A_MOMENT, expecting that the received messages's reconsume time will be equal to 30 in 2 minutes") + public void testOrderMessageRetryTimesWith30() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + + Map msgsReConsumeTimes = new ConcurrentHashMap<>(); + + DefaultMQPushConsumer pushConsumer = null; + try { + pushConsumer = new DefaultMQPushConsumer(groupId, rpcHook, new AllocateMessageQueueAveragely()); + pushConsumer.setInstanceName(RandomUtils.getStringByUUID()); + pushConsumer.setNamesrvAddr(namesrvAddr); + pushConsumer.subscribe(topic, tag); + pushConsumer.setMessageModel(MessageModel.CLUSTERING); + pushConsumer.setMaxReconsumeTimes(30); + pushConsumer.setMessageListener(new MessageListenerOrderly() { + @Override + public ConsumeOrderlyStatus consumeMessage(List msgs, + ConsumeOrderlyContext context) { + for (MessageExt msg : msgs) { + log.info(String.format("recv msgid(reconsume later) %s, reconsume time is %s ", msg.getMsgId(), + msg.getReconsumeTimes())); + msgsReConsumeTimes.put(msg.getMsgId(), msg.getReconsumeTimes()); + } + return ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT; + } + + }); + pushConsumer.start(); + } catch (MQClientException e) { + Assertions.fail(e.getMessage()); + } + + Assertions.assertNotNull(producer, "Get Producer Failed"); + List mqs = producer.fetchPublishMessageQueues(topic); + List sendMqs = new ArrayList<>(); + sendMqs.add(mqs.get(0)); + producer.sendWithQueue(sendMqs, 2, tag); + Assertions.assertEquals(2, producer.getEnqueueMessages().getDataSize(), "send message failed"); + + TestUtils.waitForMinutes(2); + + Assertions.assertEquals(2, msgsReConsumeTimes.size(), + "retry message size is not equal to send message size"); + for (Map.Entry entry : msgsReConsumeTimes.entrySet()) { + Assertions.assertTrue(30 == entry.getValue(), "retry times is not equal to maxReconsumeTimes(30)"); + } + + producer.shutdown(); + pushConsumer.shutdown(); + } + + @Test + @Timeout(value = 60, unit = TimeUnit.SECONDS) + @DisplayName("To send order messages, set SUSPEND_CURRENT_QUEUE_A_MOMENT for the first message. The next sequential message will not be consumed until the first message is in the dead letter queue. All messages except the first message are expected to be received") + public void testOrderMessageRetryTimesWithMaxReconsumeimes() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + + Map msgsReConsumeTimes = new ConcurrentHashMap<>(); + + DefaultMQPushConsumer pushConsumer = null; + try { + pushConsumer = new DefaultMQPushConsumer(groupId, rpcHook, new AllocateMessageQueueAveragely()); + pushConsumer.setInstanceName(RandomUtils.getStringByUUID()); + pushConsumer.setNamesrvAddr(namesrvAddr); + pushConsumer.subscribe(topic, tag); + pushConsumer.setMessageModel(MessageModel.CLUSTERING); + pushConsumer.setMaxReconsumeTimes(2); + pushConsumer.setMessageListener(new MessageListenerOrderly() { + @Override + public ConsumeOrderlyStatus consumeMessage(List msgs, + ConsumeOrderlyContext context) { + for (MessageExt msg : msgs) { + String body = String.valueOf(StandardCharsets.UTF_8.decode(ByteBuffer.wrap(msg.getBody()))); + if ("0".equals(body)) { + log.info(String.format("recv msgid(first message) %s, reconsume time is %s ", + msg.getMsgId(), msg.getReconsumeTimes())); + return ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT; + } else { + msgsReConsumeTimes.putIfAbsent(msg.getMsgId(), msg.getReconsumeTimes()); + log.info("recv msgid: {}", msg.getMsgId()); + } + } + return ConsumeOrderlyStatus.SUCCESS; + } + + }); + pushConsumer.start(); + } catch (MQClientException e) { + Assertions.fail(e.getMessage()); + } + + Assertions.assertNotNull(producer, "Get Producer Failed"); + List mqs = producer.fetchPublishMessageQueues(topic); + List sendMqs = new ArrayList<>(); + sendMqs.add(mqs.get(0)); + producer.sendWithQueue(sendMqs, SEND_NUM, tag); + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + + TestUtils.waitForSeconds(30); + + Assertions.assertEquals(SEND_NUM - 1, msgsReConsumeTimes.size(), + "retry message size is not equal to send message size"); + + producer.shutdown(); + pushConsumer.shutdown(); + } + + @Test + @Timeout(value = 180, unit = TimeUnit.SECONDS) + @DisplayName("Simulate pushconsumer consumption fail, expect that the original message was not received, and capture all messages after message retry") + public void testExceptionConsumption() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + + Map firstMsgs = new ConcurrentHashMap<>(); + Map retryMsgs = new ConcurrentHashMap<>(); + + DefaultMQPushConsumer pushConsumer = null; + try { + pushConsumer = new DefaultMQPushConsumer(groupId, rpcHook, new AllocateMessageQueueAveragely()); + pushConsumer.setInstanceName(RandomUtils.getStringByUUID()); + pushConsumer.setNamesrvAddr(namesrvAddr); + pushConsumer.subscribe(topic, tag); + pushConsumer.setMessageModel(MessageModel.CLUSTERING); + pushConsumer.setMaxReconsumeTimes(2); + pushConsumer.setMessageListener(new MessageListenerConcurrently() { + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + for (MessageExt msg : msgs) { + if (msg.getReconsumeTimes() == 2) { + retryMsgs.putIfAbsent(msg.getMsgId(), msg); + log.info("consume success: {}", msg); + } else { + // Simulate consuming operations + log.info("{}", "Simulate consuming operations fail"); + int i = 1 / 0; + log.info("{}", "Simulate consuming operations fail end"); + firstMsgs.putIfAbsent(msg.getMsgId(), msg); + log.info(String.format("recv msg(fail) %s ", msg)); + } + + } + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + + }); + pushConsumer.start(); + } catch (MQClientException e) { + Assertions.fail(e.getMessage()); + } + Assertions.assertNotNull(producer, "Get Producer Failed"); + for (int i = 0; i < SEND_NUM; i++) { + Message message = MessageFactory.buildNormalMessage(topic, tag, String.valueOf(i)); + producer.send(message); + } + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + + await().atMost(120, SECONDS).until(new Callable() { + @Override + public Boolean call() throws Exception { + return retryMsgs.size() == SEND_NUM && firstMsgs.size() == 0; + } + }); + + producer.shutdown(); + pushConsumer.shutdown(); + } + + @Test + @Timeout(value = 180, unit = TimeUnit.SECONDS) + @DisplayName("Simulate pushconsumer consumption return null, expect that the original message was not received, and capture all messages after message retry") + public void testNullConsumption() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + + Map firstMsgs = new ConcurrentHashMap<>(); + Map retryMsgs = new ConcurrentHashMap<>(); + + DefaultMQPushConsumer pushConsumer = null; + try { + pushConsumer = new DefaultMQPushConsumer(groupId, rpcHook, new AllocateMessageQueueAveragely()); + pushConsumer.setInstanceName(RandomUtils.getStringByUUID()); + pushConsumer.setNamesrvAddr(namesrvAddr); + pushConsumer.subscribe(topic, tag); + pushConsumer.setMessageModel(MessageModel.CLUSTERING); + pushConsumer.setMaxReconsumeTimes(2); + pushConsumer.setMessageListener(new MessageListenerConcurrently() { + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + for (MessageExt msg : msgs) { + if (msg.getReconsumeTimes() == 2) { + retryMsgs.putIfAbsent(msg.getMsgId(), msg); + log.info("consume success: {}", msg); + } else { + // Simulate consuming operations + log.info("{}", "Simulate consuming operations return null"); + firstMsgs.putIfAbsent(msg.getMsgId(), msg); + log.info(String.format("recv msg(null) %s ", msg)); + return null; + } + + } + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + pushConsumer.start(); + } catch (MQClientException e) { + Assertions.fail(e.getMessage()); + } + Assertions.assertNotNull(producer, "Get Producer Failed"); + for (int i = 0; i < SEND_NUM; i++) { + Message message = MessageFactory.buildNormalMessage(topic, tag, String.valueOf(i)); + producer.send(message); + } + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + + await().atMost(120, SECONDS).until(new Callable() { + @Override + public Boolean call() throws Exception { + return retryMsgs.size() == SEND_NUM && firstMsgs.size() == SEND_NUM; + } + }); + + producer.shutdown(); + pushConsumer.shutdown(); + } + + @Disabled + @Test + @Timeout(value = 360, unit = TimeUnit.SECONDS) + @DisplayName("Simulate pushconsumer consumption timeout, expect message retry") + public void testTimeoutConsumption() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + + Map firstMsgs = new ConcurrentHashMap<>(); + Map retryMsgs = new ConcurrentHashMap<>(); + + DefaultMQPushConsumer pushConsumer = null; + try { + pushConsumer = new DefaultMQPushConsumer(groupId, rpcHook, new AllocateMessageQueueAveragely()); + pushConsumer.setInstanceName(RandomUtils.getStringByUUID()); + pushConsumer.setNamesrvAddr(namesrvAddr); + pushConsumer.subscribe(topic, tag); + pushConsumer.setMessageModel(MessageModel.CLUSTERING); + pushConsumer.setMaxReconsumeTimes(3); + pushConsumer.setConsumeTimeout(60000); + pushConsumer.setMessageListener(new MessageListenerConcurrently() { + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + try { + for (MessageExt msg : msgs) { + if (msg.getReconsumeTimes() == 0) { + // Simulate time-consuming operations + log.info("{}", "Simulate time-consuming operations"); + Thread.sleep(65000); // Assume that message processing takes 20 seconds + log.info("{}", "Simulate time-consuming operations end"); + firstMsgs.putIfAbsent(msg.getMsgId(), msg); + log.info(String.format("recv msg(timeout) %s ", msg)); + } else { + retryMsgs.putIfAbsent(msg.getMsgId(), msg); + log.info("consume success: {}", msg); + } + } + } catch (InterruptedException e) { + e.printStackTrace(); + return ConsumeConcurrentlyStatus.RECONSUME_LATER; + } + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + pushConsumer.start(); + } catch (MQClientException e) { + Assertions.fail(e.getMessage()); + } + Assertions.assertNotNull(producer, "Get Producer Failed"); + for (int i = 0; i < SEND_NUM; i++) { + Message message = MessageFactory.buildNormalMessage(topic, tag, String.valueOf(i)); + producer.send(message); + } + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + + TestUtils.waitForMinutes(5); + Assertions.assertTrue(retryMsgs.size() == SEND_NUM, "retry message size is not equal to send message size"); + + producer.shutdown(); + pushConsumer.shutdown(); + } + + @Test + @Timeout(value = 300, unit = TimeUnit.SECONDS) + @DisplayName("The normal message is sent, and after the PushConsumer retry, the retry message is expected to be consumed") + public void testNormalTopicPushConsumerRetry() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Map firstMsgs = new ConcurrentHashMap<>(); + Map retryMsgs = new ConcurrentHashMap<>(); + + DefaultMQPushConsumer pushConsumer = null; + try { + pushConsumer = new DefaultMQPushConsumer(groupId, rpcHook, new AllocateMessageQueueAveragely()); + pushConsumer.setInstanceName(RandomUtils.getStringByUUID()); + pushConsumer.setNamesrvAddr(namesrvAddr); + pushConsumer.subscribe(topic, tag); + pushConsumer.setMessageListener(new MessageListenerConcurrently() { + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + for (int i = 0; i < msgs.size(); i++) { + MessageExt messageExt = msgs.get(i); + if (messageExt.getReconsumeTimes() == 0) { + log.info(String.format("first normal msg %s ", messageExt)); + firstMsgs.putIfAbsent(messageExt.getMsgId().toString(), messageExt); + return ConsumeConcurrentlyStatus.RECONSUME_LATER; + } else if (messageExt.getReconsumeTimes() > 0) { + log.info(String.format("retry normal msg %s ", messageExt)); + retryMsgs.putIfAbsent(messageExt.getMsgId().toString(), messageExt); + } + } + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + + }); + pushConsumer.start(); + } catch (MQClientException e) { + Assertions.fail(e.getMessage()); + } + Assertions.assertNotNull(producer, "Get Producer Failed"); + for (int i = 0; i < SEND_NUM; i++) { + Message message = MessageFactory.buildNormalMessage(topic, tag, String.valueOf(i)); + producer.send(message); + } + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + // All messages are consumed. + await().atMost(240, SECONDS).until(new Callable() { + @Override + public Boolean call() throws Exception { + return firstMsgs.size() == retryMsgs.size() && retryMsgs.size() == SEND_NUM; + } + }); + for (MessageExt messageExt : producer.getEnqueueMessages().getAllData()) { + Assertions.assertTrue(firstMsgs.containsKey(messageExt.getMsgId().toString()) + || retryMsgs.containsKey(messageExt.getMsgId().toString())); + } + producer.shutdown(); + pushConsumer.shutdown(); + } + + @Test + @Timeout(value = 300, unit = TimeUnit.SECONDS) + @DisplayName("The send order message, after the PushConsumer retry, is expected to consume the retry message, and the message consumption order and send order") + public void testFiFoTopicPushConsumerRetry() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + Assertions.assertNotNull(producer, "Get producer failed"); + Vector recvMessages = new Vector<>(); + + DefaultMQPushConsumer pushConsumer = null; + try { + pushConsumer = new DefaultMQPushConsumer(groupId, rpcHook, new AllocateMessageQueueAveragely()); + pushConsumer.setInstanceName(RandomUtils.getStringByUUID()); + pushConsumer.setNamesrvAddr(namesrvAddr); + pushConsumer.subscribe(topic, tag); + pushConsumer.setSuspendCurrentQueueTimeMillis(1000); + pushConsumer.setMessageListener(new MessageListenerOrderly() { + + @Override + public ConsumeOrderlyStatus consumeMessage(List msgs, ConsumeOrderlyContext context) { + for (int i = 0; i < msgs.size(); i++) { + MessageExt messageExt = msgs.get(i); + if (messageExt.getReconsumeTimes() == 0) { + log.info(String.format("first normal msg %s ", messageExt)); + return ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT; + } else if (messageExt.getReconsumeTimes() > 0) { + log.info(String.format("retry normal msg %s ", messageExt)); + recvMessages.add(messageExt); + } + } + return ConsumeOrderlyStatus.SUCCESS; + } + + }); + pushConsumer.start(); + } catch (MQClientException e) { + Assertions.fail(e.getMessage()); + } + Assertions.assertNotNull(producer, "Get Producer Failed"); + List messageQueues = producer.fetchPublishMessageQueues(topic); + List sendQueues = new ArrayList<>(); + sendQueues.add(messageQueues.get(0)); + producer.sendWithQueue(sendQueues, tag, SEND_NUM); + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + // All messages are consumed. + await().atMost(240, SECONDS).until(new Callable() { + @Override + public Boolean call() throws Exception { + return recvMessages.size() == SEND_NUM; + } + }); + for (int i = 0; i < SEND_NUM; i++) { + Assertions.assertEquals(i, + Integer.parseInt(String + .valueOf(StandardCharsets.UTF_8.decode(ByteBuffer.wrap(recvMessages.get(i).getBody())))), + "recv message failed"); + } + } +} diff --git a/java/e2e-v4/src/test/java/org/apache/rocketmq/server/batch/BatchProducerTest.java b/java/e2e-v4/src/test/java/org/apache/rocketmq/server/batch/BatchProducerTest.java new file mode 100644 index 0000000..08ad64b --- /dev/null +++ b/java/e2e-v4/src/test/java/org/apache/rocketmq/server/batch/BatchProducerTest.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.server.batch; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.rocketmq.client.consumer.MessageSelector; +import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.rmq.RMQNormalConsumer; +import org.apache.rocketmq.client.rmq.RMQNormalProducer; +import org.apache.rocketmq.cluster.LoadBalancingTest; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.enums.TESTSET; +import org.apache.rocketmq.factory.ConsumerFactory; +import org.apache.rocketmq.factory.MessageFactory; +import org.apache.rocketmq.factory.ProducerFactory; +import org.apache.rocketmq.frame.BaseOperate; +import org.apache.rocketmq.listener.rmq.concurrent.RMQNormalListener; +import org.apache.rocketmq.utils.NameUtils; +import org.apache.rocketmq.utils.TestUtils; +import org.apache.rocketmq.utils.VerifyUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Tag(TESTSET.BATCHPRODUCER) +public class BatchProducerTest extends BaseOperate { + private final Logger log = LoggerFactory.getLogger(BatchProducerTest.class); + private String tag; + private final static int SEND_NUM = 10; + + @BeforeEach + public void setUp() { + tag = NameUtils.getRandomTagName(); + } + + @Test + @DisplayName("Send 10 messages in batch, expect pushconsumer to accept them all") + public void testBatchProducer() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + + RMQNormalConsumer consumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook, + new AllocateMessageQueueAveragely()); + consumer.subscribeAndStart(topic, tag, new RMQNormalListener()); + + Assertions.assertNotNull(producer); + List messages = new ArrayList<>(); + for (int i = 0; i < SEND_NUM; i++) { + Message message = MessageFactory.buildNormalMessage(topic, tag, String.valueOf(i)); + messages.add(message); + } + + try { + producer.getProducer().send(messages); + } catch (Exception e) { + Assertions.fail(e.getMessage()); + } + + TestUtils.waitForSeconds(5); + + Assertions.assertEquals(SEND_NUM, consumer.getListener().getDequeueMessages().getDataSize()); + + producer.shutdown(); + consumer.shutdown(); + } + + @Test + @DisplayName("Send 10 messages to a queue in batch , expect pushconsumer to accept them all") + public void testBatchProducer_queue() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + + RMQNormalConsumer consumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook, + new AllocateMessageQueueAveragely()); + consumer.subscribeAndStart(topic, tag, new RMQNormalListener()); + + Assertions.assertNotNull(producer); + List messages = new ArrayList<>(); + for (int i = 0; i < SEND_NUM; i++) { + Message message = MessageFactory.buildNormalMessage(topic, tag, String.valueOf(i)); + messages.add(message); + } + + List msgQueues = producer.fetchPublishMessageQueues(topic); + try { + producer.getProducer().send(messages, msgQueues.get(0)); + } catch (Exception e) { + Assertions.fail(e.getMessage()); + } + + TestUtils.waitForSeconds(5); + + Assertions.assertEquals(SEND_NUM, consumer.getListener().getDequeueMessages().getDataSize()); + + producer.shutdown(); + consumer.shutdown(); + } + +} diff --git a/java/e2e-v4/src/test/java/org/apache/rocketmq/server/delay/DelayMessageTest.java b/java/e2e-v4/src/test/java/org/apache/rocketmq/server/delay/DelayMessageTest.java index 3e4a369..2fd9f26 100644 --- a/java/e2e-v4/src/test/java/org/apache/rocketmq/server/delay/DelayMessageTest.java +++ b/java/e2e-v4/src/test/java/org/apache/rocketmq/server/delay/DelayMessageTest.java @@ -17,17 +17,25 @@ package org.apache.rocketmq.server.delay; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.client.rmq.RMQNormalConsumer; import org.apache.rocketmq.client.rmq.RMQNormalProducer; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.enums.TESTSET; import org.apache.rocketmq.factory.ConsumerFactory; import org.apache.rocketmq.factory.ProducerFactory; import org.apache.rocketmq.frame.BaseOperate; import org.apache.rocketmq.listener.rmq.concurrent.RMQNormalListener; +import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.utils.MQAdmin; import org.apache.rocketmq.utils.NameUtils; +import org.apache.rocketmq.utils.RandomUtils; import org.apache.rocketmq.utils.VerifyUtils; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Tag; @@ -35,11 +43,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.junit.jupiter.api.Assertions.assertThrows; + @Tag(TESTSET.SMOKE) @Tag(TESTSET.DELAY) public class DelayMessageTest extends BaseOperate { private final Logger logger = LoggerFactory.getLogger(DelayMessageTest.class); - //private RMQOrderConsumer consumer; + // private RMQOrderConsumer consumer; private String tag; private String topic; private String groupId; @@ -65,10 +75,11 @@ public void testDelayLevel1() { int delayLevel = 1; RMQNormalConsumer consumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); consumer.subscribeAndStart(topic, "*", new RMQNormalListener()); - RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr,rpcHook); + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); producer.sendDelay(topic, delayLevel, SEND_NUM); - VerifyUtils.verifyDelayMessage(producer.getEnqueueMessages(), consumer.getListener().getDequeueMessages(), delayLevel); + VerifyUtils.verifyDelayMessage(producer.getEnqueueMessages(), consumer.getListener().getDequeueMessages(), + delayLevel); producer.shutdown(); consumer.shutdown(); @@ -80,12 +91,47 @@ public void testDelayLevel4() { int delayLevel = 4; RMQNormalConsumer consumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); consumer.subscribeAndStart(topic, "*", new RMQNormalListener()); - RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr,rpcHook); + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); producer.sendDelay(topic, delayLevel, SEND_NUM); - VerifyUtils.verifyDelayMessage(producer.getEnqueueMessages(), consumer.getListener().getDequeueMessages(), delayLevel); + VerifyUtils.verifyDelayMessage(producer.getEnqueueMessages(), consumer.getListener().getDequeueMessages(), + delayLevel); producer.shutdown(); consumer.shutdown(); } + + @Disabled + @Test + @DisplayName("Send one delay message and set the delay test negative delay level, expecting message building wrong") + public void testNegativeDelayLevel() { + int delayLevel = -1; + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + + assertThrows(Exception.class, () -> { + Message msg = new Message(topic, "*", RandomUtils.getStringByUUID().getBytes()); + msg.setDelayTimeLevel(delayLevel); + SendResult sendResult = producer.getProducer().send(msg); + logger.info(sendResult.toString()); + }, "Send messages with a negative delay level, Expected send() to throw exception, but it didn't"); + + producer.shutdown(); + } + + @Disabled + @Test + @DisplayName("Send one delay message and set the delay test delay level=19, expecting message building wrong") + public void testDelayLevelWith19() { + int delayLevel = 19; + RMQNormalProducer producer = ProducerFactory.getRMQProducer(namesrvAddr, rpcHook); + + assertThrows(Exception.class, () -> { + Message msg = new Message(topic, "*", RandomUtils.getStringByUUID().getBytes()); + msg.setDelayTimeLevel(delayLevel); + SendResult sendResult = producer.getProducer().send(msg); + logger.info(sendResult.toString()); + }, "Send messages with delay level=19, Expected send() to throw exception, but it didn't"); + + producer.shutdown(); + } } diff --git a/java/e2e-v4/src/test/java/org/apache/rocketmq/server/transaction/TransactionMessageTest.java b/java/e2e-v4/src/test/java/org/apache/rocketmq/server/transaction/TransactionMessageTest.java index fd1638e..22bf0be 100644 --- a/java/e2e-v4/src/test/java/org/apache/rocketmq/server/transaction/TransactionMessageTest.java +++ b/java/e2e-v4/src/test/java/org/apache/rocketmq/server/transaction/TransactionMessageTest.java @@ -17,18 +17,25 @@ package org.apache.rocketmq.server.transaction; +import org.apache.rocketmq.client.consumer.MessageSelector; import org.apache.rocketmq.client.producer.LocalTransactionState; +import org.apache.rocketmq.client.producer.TransactionListener; import org.apache.rocketmq.client.rmq.RMQNormalConsumer; import org.apache.rocketmq.client.rmq.RMQTransactionProducer; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.enums.TESTSET; import org.apache.rocketmq.factory.ConsumerFactory; +import org.apache.rocketmq.factory.MessageFactory; import org.apache.rocketmq.factory.ProducerFactory; import org.apache.rocketmq.frame.BaseOperate; import org.apache.rocketmq.listener.rmq.concurrent.RMQNormalListener; import org.apache.rocketmq.listener.rmq.concurrent.TransactionListenerImpl; import org.apache.rocketmq.utils.MQAdmin; import org.apache.rocketmq.utils.NameUtils; +import org.apache.rocketmq.utils.TestUtils; import org.apache.rocketmq.utils.VerifyUtils; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Tag; @@ -36,12 +43,19 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.awaitility.Awaitility.await; @Tag(TESTSET.TRANSACTION) @Tag(TESTSET.SMOKE) @@ -67,21 +81,187 @@ public void testConsumeNormalMessage() { RMQNormalConsumer consumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); consumer.subscribeAndStart(topic, tag, new RMQNormalListener()); - ExecutorService executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, new ArrayBlockingQueue(2000), new ThreadFactory() { - @Override - public Thread newThread(Runnable r) { - Thread thread = new Thread(r); - thread.setName("client-transaction-msg-check-thread"); - return thread; - } - }); + ExecutorService executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, + new ArrayBlockingQueue(2000), new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + Thread thread = new Thread(r); + thread.setName("client-transaction-msg-check-thread"); + return thread; + } + }); - RMQTransactionProducer producer = ProducerFactory.getTransProducer(namesrvAddr, executorService, new TransactionListenerImpl(LocalTransactionState.COMMIT_MESSAGE, LocalTransactionState.COMMIT_MESSAGE), rpcHook); - producer.send(topic, tag, SEND_NUM); + RMQTransactionProducer producer = ProducerFactory.getTransProducer(namesrvAddr, executorService, + new TransactionListenerImpl(LocalTransactionState.COMMIT_MESSAGE, LocalTransactionState.COMMIT_MESSAGE), + rpcHook); + producer.sendTrans(topic, tag, SEND_NUM); VerifyUtils.verifyNormalMessage(producer.getEnqueueMessages(), consumer.getListener().getDequeueMessages()); producer.shutdown(); consumer.shutdown(); } + @Test + @DisplayName("Send 10 transaction messages and rollback directly (Checker does commit), expecting that these 10 messages cannot be consumed by PushConsumer") + public void testTrans_SendRollback_PushConsume() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + RMQNormalConsumer pushConsumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer.subscribeAndStart(topic, MessageSelector.byTag(tag), new RMQNormalListener()); + + ExecutorService executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, + new ArrayBlockingQueue(2000), new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + Thread thread = new Thread(r); + thread.setName("client-transaction-msg-check-thread"); + return thread; + } + }); + + RMQTransactionProducer producer = ProducerFactory.getTransProducer(namesrvAddr, executorService, + new TransactionListenerImpl(LocalTransactionState.COMMIT_MESSAGE, + LocalTransactionState.ROLLBACK_MESSAGE), + rpcHook); + producer.sendTrans(topic, tag, SEND_NUM); + // Wait for the callback, expecting not to commit the already rolled back + // message + TestUtils.waitForSeconds(60); + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + Assertions.assertEquals(0, pushConsumer.getListener().getDequeueMessages().getDataSize()); + producer.shutdown(); + pushConsumer.shutdown(); + } + + @Test + @DisplayName("Send 10 transaction messages and COMMIT the transaction by Checker (perform COMMIT), expecting the 10 messages to be consumed by PushConsumer") + public void testTrans_SendCheckerCommit_PushConsume() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + RMQNormalConsumer pushConsumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer.subscribeAndStart(topic, MessageSelector.byTag(tag), new RMQNormalListener()); + + ExecutorService executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, + new ArrayBlockingQueue(2000), new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + Thread thread = new Thread(r); + thread.setName("client-transaction-msg-check-thread"); + return thread; + } + }); + + RMQTransactionProducer producer = ProducerFactory.getTransProducer(namesrvAddr, executorService, + new TransactionListenerImpl(LocalTransactionState.COMMIT_MESSAGE, LocalTransactionState.UNKNOW), + rpcHook); + producer.sendTrans(topic, tag, SEND_NUM); + + // Wait for the callback to execute commit + TestUtils.waitForSeconds(60); + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + VerifyUtils.verifyNormalMessage(producer.getEnqueueMessages(), pushConsumer.getListener().getDequeueMessages()); + producer.shutdown(); + pushConsumer.shutdown(); + } + + @Test + @DisplayName("Send 10 transaction messages and roll back the transaction by Checker (performing ROLLBACK), expecting that the 10 messages will not be consumed by PushConsumer") + public void testTrans_CheckerRollback() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + RMQNormalConsumer pushConsumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer.subscribeAndStart(topic, MessageSelector.byTag(tag), new RMQNormalListener()); + + ExecutorService executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, + new ArrayBlockingQueue(2000), new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + Thread thread = new Thread(r); + thread.setName("client-transaction-msg-check-thread"); + return thread; + } + }); + + RMQTransactionProducer producer = ProducerFactory.getTransProducer(namesrvAddr, executorService, + new TransactionListenerImpl(LocalTransactionState.ROLLBACK_MESSAGE, LocalTransactionState.UNKNOW), + rpcHook); + producer.sendTrans(topic, tag, SEND_NUM); + // Wait for the rollback and execute rollback + TestUtils.waitForSeconds(60); + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + Assertions.assertEquals(0, pushConsumer.getListener().getDequeueMessages().getDataSize()); + producer.shutdown(); + pushConsumer.shutdown(); + } + + @Test + @DisplayName("Send 10 transactional messages and commit them by checking back (Checker commits for partial messages), and the expected committed messages can be consumed by PushConsumer") + public void testTrans_SendCheckerPartionCommit() { + String methodName = Thread.currentThread().getStackTrace()[1].getMethodName(); + + String topic = getTopic(methodName); + String groupId = getGroupId(methodName); + + RMQNormalConsumer pushConsumer = ConsumerFactory.getRMQNormalConsumer(namesrvAddr, groupId, rpcHook); + pushConsumer.subscribeAndStart(topic, MessageSelector.byTag(tag), new RMQNormalListener()); + + AtomicInteger commitMsgNum = new AtomicInteger(0); + AtomicInteger rollbackMsgNum = new AtomicInteger(0); + + ExecutorService executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, + new ArrayBlockingQueue(2000), new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + Thread thread = new Thread(r); + thread.setName("client-transaction-msg-check-thread"); + return thread; + } + }); + + RMQTransactionProducer producer = ProducerFactory.getTransProducer(namesrvAddr, executorService, + new TransactionListener() { + + @Override + public LocalTransactionState executeLocalTransaction(Message msg, Object arg) { + return LocalTransactionState.UNKNOW; + } + + @Override + public LocalTransactionState checkLocalTransaction(MessageExt msg) { + if (Integer + .parseInt(String.valueOf(StandardCharsets.UTF_8.decode(ByteBuffer.wrap(msg.getBody())))) + % 2 == 0) { + commitMsgNum.getAndIncrement(); + return LocalTransactionState.COMMIT_MESSAGE; + } else { + rollbackMsgNum.getAndIncrement(); + return LocalTransactionState.ROLLBACK_MESSAGE; + } + } + + }, rpcHook); + producer.sendTrans(topic, tag, SEND_NUM); + Assertions.assertNotNull(producer); + + await().atMost(90, SECONDS).until(new Callable() { + @Override + public Boolean call() { + return rollbackMsgNum.get() == commitMsgNum.get() && commitMsgNum.get() == SEND_NUM / 2; + } + }); + // Wait for the rollback and execute commit/rollback + TestUtils.waitForSeconds(60); + Assertions.assertEquals(SEND_NUM, producer.getEnqueueMessages().getDataSize(), "send message failed"); + Assertions.assertEquals(5, pushConsumer.getListener().getDequeueMessages().getDataSize()); + producer.shutdown(); + pushConsumer.shutdown(); + } + } diff --git a/java/e2e-v4/src/test/resources/logback-test.xml b/java/e2e-v4/src/test/resources/logback-test.xml index ce2a862..3be8f8a 100644 --- a/java/e2e-v4/src/test/resources/logback-test.xml +++ b/java/e2e-v4/src/test/resources/logback-test.xml @@ -19,7 +19,7 @@ logback - +