MQ
MQ architecture:
The Message contains:
RocketMQ client coordinates:
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-client</artifactId>
<version>4.5.2</version>
</dependency>
Messaging & consumption patterns
Note: when the producer reports an error message that the topic cannot be found, check whether it is caused by the firewall policy of the server.
One to one (single producer, single consumer)
producer
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.common.message.Message;
public class Producer {
public static void main(String[] args) throws Exception {
// 1. Create an object Producer that sends messages
DefaultMQProducer producer = new DefaultMQProducer("group1"); // The input parameter is a custom group name
// 2. Set the sending naming server address
producer.setNamesrvAddr("192.168.3.244:9876");
// Set the timeout for sending messages (3000 by default)
producer.setSendMsgTimeout(60000);
// 3. Start sending service
producer.start();
// 4. Create a message object to send, and specify topic and body
Message msg = new Message("topic1", "hello rocketmq".getBytes("UTF-8"));
// 5. Send a single message
SendResult result = producer.send(msg);
// Print return message
System.out.println("Return result:"+result);
// 6. Close the connection
producer.shutdown();
}
}
Operation results:
Return result: SendResult [sendStatus=SEND_OK, msgId=A9FE135E658458644D464D61A1F80000, offsetMsgId=C0A803DE00002A9F0000000000000000, messageQueue=MessageQueue [topic=topic1, brokerName=broker-a, queueId=3], queueOffset=0]
consumer
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.common.message.MessageExt;
import java.util.List;
public class Consumer {
public static void main(String[] args) throws Exception {
// 1. Create an object Consumer that receives messages
DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("group1"); // The input parameter is a custom group name
// 2. Set the received naming server address
consumer.setNamesrvAddr("192.168.3.244:9876");
// 3. Set the topic corresponding to the received message, and the corresponding sub tag is any*
consumer.subscribe("topic1", "*");
// 4. Enable listening to receive messages
consumer.registerMessageListener(new MessageListenerConcurrently() {
public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> list, ConsumeConcurrentlyContext consumeConcurrentlyContext) {
// Traversal message
for(MessageExt msg : list){
// System.out.println("received message:" + msg);
System.out.println("Message:" + new String(msg.getBody()));
}
// After successful processing, mq receives the tag, and the same message will not be sent to the consumer again
return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
}
});
// 5. Start the service for receiving messages
consumer.start(); // Turn on the multi-threaded monitoring message and it will run continuously
System.out.println("The receive message service is up and running");
}
}
Operation results:
The receive message service is up and running Message: hello rocketmq
One to many
producer
package onetoone;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.common.message.Message;
public class Producer {
public static void main(String[] args) throws Exception {
// 1. Create an object Producer that sends messages
DefaultMQProducer producer = new DefaultMQProducer("group1"); // The input parameter is a custom group name
// 2. Set the sending naming server address
producer.setNamesrvAddr("192.168.3.222:9876");
// Set the timeout for sending messages (3000 by default)
producer.setSendMsgTimeout(60000);
// 3. Start sending service
producer.start();
// Create a message object to send, and specify topic and body
// Message msg = new Message("topic1", "hello rocketmq".getBytes("UTF-8"));
// Send a single message
// SendResult result = producer.send(msg);
// 4. Send multiple messages
for (int i = 1; i <= 10; i++) {
Message msg = new Message("topic1", ("producer: hello rocketmq "+i).getBytes("UTF-8"));
SendResult result = producer.send(msg);
System.out.println("Return result:" + result);
}
// 5. Close the connection
producer.shutdown();
}
}
Operation results:
Return result: SendResult [sendStatus=SEND_OK, msgId=A9FE135E7DB058644D464D70EF300000, offsetMsgId=C0A803DE00002A9F0000000000000E79, messageQueue=MessageQueue [topic=topic1, brokerName=broker-a, queueId=2], queueOffset=4] Return result: SendResult [sendStatus=SEND_OK, msgId=A9FE135E7DB058644D464D70EF7A0001, offsetMsgId=C0A803DE00002A9F0000000000000F2A, messageQueue=MessageQueue [topic=topic1, brokerName=broker-a, queueId=3], queueOffset=6] Return result: SendResult [sendStatus=SEND_OK, msgId=A9FE135E7DB058644D464D70EF980002, offsetMsgId=C0A803DE00002A9F0000000000000FDB, messageQueue=MessageQueue [topic=topic1, brokerName=broker-a, queueId=0], queueOffset=6] Return result: SendResult [sendStatus=SEND_OK, msgId=A9FE135E7DB058644D464D70EFAB0003, offsetMsgId=C0A803DE00002A9F000000000000108C, messageQueue=MessageQueue [topic=topic1, brokerName=broker-a, queueId=1], queueOffset=5] Return result: SendResult [sendStatus=SEND_OK, msgId=A9FE135E7DB058644D464D70EFB70004, offsetMsgId=C0A803DE00002A9F000000000000113D, messageQueue=MessageQueue [topic=topic1, brokerName=broker-a, queueId=2], queueOffset=5] Return result: SendResult [sendStatus=SEND_OK, msgId=A9FE135E7DB058644D464D70EFCC0005, offsetMsgId=C0A803DE00002A9F00000000000011EE, messageQueue=MessageQueue [topic=topic1, brokerName=broker-a, queueId=3], queueOffset=7] Return result: SendResult [sendStatus=SEND_OK, msgId=A9FE135E7DB058644D464D70EFDF0006, offsetMsgId=C0A803DE00002A9F000000000000129F, messageQueue=MessageQueue [topic=topic1, brokerName=broker-a, queueId=0], queueOffset=7] Return result: SendResult [sendStatus=SEND_OK, msgId=A9FE135E7DB058644D464D70EFEB0007, offsetMsgId=C0A803DE00002A9F0000000000001350, messageQueue=MessageQueue [topic=topic1, brokerName=broker-a, queueId=1], queueOffset=6] Return result: SendResult [sendStatus=SEND_OK, msgId=A9FE135E7DB058644D464D70EFF20008, offsetMsgId=C0A803DE00002A9F0000000000001401, messageQueue=MessageQueue [topic=topic1, brokerName=broker-a, queueId=2], queueOffset=6] Return result: SendResult [sendStatus=SEND_OK, msgId=A9FE135E7DB058644D464D70EFFD0009, offsetMsgId=C0A803DE00002A9F00000000000014B2, messageQueue=MessageQueue [topic=topic1, brokerName=broker-a, queueId=3], queueOffset=8]
consumer
Load balancing mode
Load balancing mode: that is, multiple messages (quantities) produced will be evenly distributed to each consumer.
package onetoone;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
import java.util.List;
public class Consumer {
public static void main(String[] args) throws Exception {
// 1. Create an object Consumer that receives messages
DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("group1"); // The input parameter is a custom group name
// 2. Set the received naming server address
consumer.setNamesrvAddr("192.168.3.222:9876");
// 3. Set the topic corresponding to the received message, and the corresponding sub tag is any*
consumer.subscribe("topic1", "*");
// Set the current consumer's consumption mode: load balancing (also the default mode)
consumer.setMessageModel(MessageModel.CLUSTERING);
// 4. Enable listening to receive messages
consumer.registerMessageListener(new MessageListenerConcurrently() {
public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> list, ConsumeConcurrentlyContext consumeConcurrentlyContext) {
// Traversal message
for(MessageExt msg : list){
// System.out.println("received message:" + msg);
System.out.println("Message:" + new String(msg.getBody()));
}
// After successful processing, mq receives the tag, and the same message will not be sent to the consumer again
return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
}
});
// 5. Start the service for receiving messages
consumer.start(); // Turn on the multi-threaded monitoring message and it will run continuously
System.out.println("The receive message service is up and running");
}
}
Operation results:
Broadcast mode
Broadcast mode: that is, the same message produced will be received by each consumer.
Phenomena of broadcast mode:
- If the message is produced first and then consumed, the message can only be consumed once.
- If multiple consumers start (broadcast mode) first and then produce, the effect of broadcasting can only be achieved at this time.
- Conclusion: only by starting consumers and then producers can the effect of broadcasting be achieved.
package onetoone;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
import java.util.List;
public class Consumer {
public static void main(String[] args) throws Exception {
// 1. Create an object Consumer that receives messages
DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("group1"); // The input parameter is a custom group name
// 2. Set the received naming server address
consumer.setNamesrvAddr("192.168.3.222:9876");
// 3. Set the topic corresponding to the received message, and the corresponding sub tag is any*
consumer.subscribe("topic1", "*");
// Set the current consumer's consumption mode to broadcast mode: the messages received by all clients are the same
consumer.setMessageModel(MessageModel.BROADCASTING);
// 4. Enable listening to receive messages
consumer.registerMessageListener(new MessageListenerConcurrently() {
public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> list, ConsumeConcurrentlyContext consumeConcurrentlyContext) {
// Traversal message
for(MessageExt msg : list){
// System.out.println("received message:" + msg);
System.out.println("Message:" + new String(msg.getBody()));
}
// After successful processing, mq receives the tag, and the same message will not be sent to the consumer again
return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
}
});
// 5. Start the service for receiving messages
consumer.start(); // Turn on the multi-threaded monitoring message and it will run continuously
System.out.println("The receive message service is up and running");
}
}
Operation results:
Many to many
Messages generated by multiple producers can be consumed by the same consumer or by multiple consumers.
(demonstration: just start multiple instances of the producer of the above example).
Message category
Synchronization message
Features: messages with strong immediacy, which are usually important and must have a receipt, such as SMS and notification (successful transfer).
Code implementation:
SendResult result = producer.send(msg);
Asynchronous message
Features: messages with weak immediacy but receipt, such as some information in the order.
Code implementation:
// Note: the callback processing result must be executed before the end of the producer process, otherwise the callback cannot be executed correctly
producer.send(msg, new SendCallback() {
// Indicates that the result is returned successfully
public void onSuccess(SendResult sendResult) {
System.out.println(sendResult);
}
// Indicates that sending the message failed
public void onException(Throwable t) {
System.out.println(t);
}
});
// Be careful not to shut down the connection immediately later. Make sure you have time to receive asynchronous messages
One way message
Features: messages without receipt, such as log messages.
Code implementation:
producer.sendOneway(msg);
Delay message
Delayed message: when a message is sent, it is not directly sent to the message server, but arrives according to the set waiting time, which plays the role of buffer for delayed arrival.
Message msg = new Message("topic3", ("Delay message: hello rocketmq "+i).getBytes("UTF-8"));
// Set the delay effect of the current message
msg.setDelayTimeLevel(3); // The input parameter represents subscript 3 and represents 30s (refer to the table below)
SendResult result = producer.send(msg);
System.out.println("Return result:"+result);
Currently supported message times:
- Second level: 1, 5, 10, 30
- Classification: 1 ~ 10, 20, 30
- Time level: 1, 2
- Meaning of subscript: 1s, 5s, 10s, 30s, 1m, 2m, 3m, 4m, 5m, 6m, 7m, 8m, 9m, 10m, 20m, 30m, 1h, 2h
Batch message
// Create a message collection, create a message object to send, and specify topic and body
ArrayList<Message> messageList = new ArrayList<>();
Message msg1 = new Message("topic1", "hello rocketmq 1".getBytes("UTF-8"));
Message msg2 = new Message("topic1", "hello rocketmq 2".getBytes("UTF-8"));
Message msg3 = new Message("topic1", "hello rocketmq 3".getBytes("UTF-8"));
messageList.add(msg1);
messageList.add(msg2);
messageList.add(msg3);
// Send batch messages (once)
SendResult result = producer.send(messageList);
System.out.println("Send results:"+result);
Operation results:
Send results: SendResult [sendStatus=SEND_OK, msgId=A9FE135E681458644D4650530BBF0000,A9FE135E681458644D4650530BBF0001,A9FE135E681458644D4650530BBF0002, offsetMsgId=C0A803DE00002A9F0000000000001E3E,C0A803DE00002A9F0000000000001EE3,C0A803DE00002A9F0000000000001F88, messageQueue=MessageQueue [topic=topic1, brokerName=broker-a, queueId=2], queueOffset=13]
Message filtering
Classification filtering
- Producer: specify tag when sending
Message msg = new Message("topic6", "tag2", ("Message filtering by tag: hello rocketmq 2").getBytes("UTF-8"));
- Consumer: consume the message of the specified tag
// When receiving a message, you can specify not only topic, but also the received tag (* for any tag, | | for or)
consumer.subscribe("topic6", "tag1 || tag2");
Attribute filtering (SQL filtering)
producer:
// Add properties to the message
msg.putUserProperty("vip", "1");
msg.putUserProperty("age", "20");
consumer:
// The message selector is used to filter the corresponding attributes. The syntax format is SQL like syntax
consumer.subscribe("topic1", MessageSelector.bySql("age >= 18"));
to configure:
- Note: SQL filtering depends on the function support of the server. In the broker Add corresponding function items to the conf configuration file and turn on the corresponding functions.
enablePropertyFilter=true
- Start the server to enable the corresponding profile:
sh mqbroker -n localhost:9876 -c ../conf/broker.conf
Message order
Message out of order
-
By default, MQ opens multiple queues. If multiple messages are sent at the same time, it is uncertain which queue to send to. At the same time, when the consumer reads a message, one thread is started for each message, which can not guarantee the sequence of messages.
-
To ensure the order of messages, you need to specify the queue when sending messages. At the same time, consumers should open a thread to receive a queue, rather than a thread to receive a message.
Sequential message
producer:
// Set the message to enter the specified message queue
for(final Order order : orderList){
Message msg = new Message("orderTopic", order.toString().getBytes());
// Specify the corresponding message queue selector when sending
SendResult result = producer.send(msg, new MessageQueueSelector() {
// Set which message queue to use when sending the current message
public MessageQueue select(List<MessageQueue> list, Message message, Object o) {
// Select different message queues according to different information sent
// Select an object of the message queue according to the id (take the module with the hash value of the id)
int mqIndex = order.getId().hashCode() % list.size();
return list.get(mqIndex);
}
}, null);
System.out.println(result);
}
consumer:
// The single thread mode is used to fetch data from the message queue, and one thread binds to one message queue
consumer.registerMessageListener(new MessageListenerOrderly() {
// After using the MessageListenerOrderly interface,
// The processing of message queue is transformed from a message queue with multiple thread services to a message queue with one thread service
public ConsumeOrderlyStatus consumeMessage(List<MessageExt> list, ConsumeOrderlyContext consumeOrderlyContext) {
for(MessageExt msg : list){
System.out.println("Message:"+new String(msg.getBody()));
}
return ConsumeOrderlyStatus.SUCCESS;
}
});
Transaction message
Transaction message process
Transaction message status
-
Submission status: it is allowed to enter the queue. This message is no different from non transaction messages.
-
Rollback status: it is not allowed to enter the queue. This message is equivalent to not sent.
-
Intermediate status: the sending of the half message is completed, and no secondary status confirmation is performed on MQ.
Note: transaction messages are only related to producers, not consumers.
Transaction message implementation
// The producer used by transaction messages is TransactionMQProducer
TransactionMQProducer producer = new TransactionMQProducer("group1");
producer.setNamesrvAddr("192.168.184.128:9876");
// Add listener corresponding to local transaction
producer.setTransactionListener(new TransactionListener() {
// Normal transaction process
public LocalTransactionState executeLocalTransaction(Message message, Object o) {
// Submission status
return LocalTransactionState.COMMIT_MESSAGE;
// Rollback status
return LocalTransactionState.ROLLBACK_MESSAGE;
// Intermediate state
return LocalTransactionState.UNKNOW;
}
// Transaction compensation process
public LocalTransactionState checkLocalTransaction(MessageExt messageExt) {
// The transaction compensation process must ensure that the server is running, otherwise normal transaction compensation cannot be performed
return LocalTransactionState.COMMIT_MESSAGE;
// return null;
}
});
producer.start();
Message msg = new Message("topic8", ("Transaction message: hello rocketmq ").getBytes("UTF-8"));
SendResult result = producer.sendMessageInTransaction(msg,null);
System.out.println("Return result:"+result);
producer.shutdown();