Ali Cloud Internet of Things Platform Device Data Forwarding to Message Queue RocketMQ Full Link Test

Posted by kaitan on Tue, 10 Dec 2019 07:08:30 +0100

Summary

You can use the rule engine to forward IOT platform data to a message queue (RocketMQ) for storage.This enables the full-link, highly reliable transmission of messages from devices, Internet of Things platforms, RocketMQ to application servers.Starting with the creation of products and devices on the Internet of Things platform, the text introduces the complete implementation of the whole link step by step.

Operation steps

1. Create Internet of Things products and devices

Reference resources Ali Cloud Internet of Things Platform Qucik Start Create products and devices quickly.

2,RocketMQ Console Create instances, Topic s, and Group s, which choose to create resources in the network area to facilitate local testing of consumer MQ messages

_

3. Configure Rule Engine

a. Overview of configuration

b. Processing data

c. Forwarding data

d. Turn on Rule Engine after configuration

Relevant references:
SQL statement reference
Data Forwarding to Message Queue RocketMQ

Device Report Property Message
import com.alibaba.taro.AliyunIoTSignUtil;
import org.eclipse.paho.client.mqttv3.*;
import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence;

import java.util.HashMap;
import java.util.Map;

public class IoTDemoPubSubDemo1 {
    public static String productKey = "*********";
    public static String deviceName = "*********";
    public static String deviceSecret = "*********";
    public static String regionId = "cn-shanghai";

    // Material Model-Property Upload topic
    private static String pubTopic = "/sys/" + productKey + "/" + deviceName + "/thing/event/property/post";

    private static MqttClient mqttClient;

    public static void main(String [] args){

        // Initialize Mqtt Client object
        initAliyunIoTClient();
        // Reporting Properties
        postDeviceProperties();

    }

    /**
     * Initialize Client Object
     */
    private static void initAliyunIoTClient() {

        try {
            // Parameters required to construct a connection
            String clientId = "java" + System.currentTimeMillis();
            Map<String, String> params = new HashMap<>(16);
            params.put("productKey", productKey);
            params.put("deviceName", deviceName);
            params.put("clientId", clientId);
            String timestamp = String.valueOf(System.currentTimeMillis());
            params.put("timestamp", timestamp);
            // cn-shanghai
            String targetServer = "tcp://" + productKey + ".iot-as-mqtt."+regionId+".aliyuncs.com:1883";

            String mqttclientId = clientId + "|securemode=3,signmethod=hmacsha1,timestamp=" + timestamp + "|";
            String mqttUsername = deviceName + "&" + productKey;
            String mqttPassword = AliyunIoTSignUtil.sign(params, deviceSecret, "hmacsha1");

            connectMqtt(targetServer, mqttclientId, mqttUsername, mqttPassword);

        } catch (Exception e) {
            System.out.println("initAliyunIoTClient error " + e.getMessage());
        }
    }

    public static void connectMqtt(String url, String clientId, String mqttUsername, String mqttPassword) throws Exception {

        MemoryPersistence persistence = new MemoryPersistence();
        mqttClient = new MqttClient(url, clientId, persistence);
        MqttConnectOptions connOpts = new MqttConnectOptions();
        // MQTT 3.1.1
        connOpts.setMqttVersion(4);
        connOpts.setAutomaticReconnect(false);
//        connOpts.setCleanSession(true);
        connOpts.setCleanSession(false);

        connOpts.setUserName(mqttUsername);
        connOpts.setPassword(mqttPassword.toCharArray());
        connOpts.setKeepAliveInterval(60);

        mqttClient.connect(connOpts);
    }

    /**
     * Reporting Properties
     */
    private static void postDeviceProperties() {

        try {
            //Report data
            //Advanced layout model - property reporting payload
            System.out.println("Report property value");
            String payloadJson = "{\"params\":{\"Status\":1,\"Data\":\"33\"}}";
            MqttMessage message = new MqttMessage(payloadJson.getBytes("utf-8"));
            message.setQos(1);
            mqttClient.publish(pubTopic, message);
        } catch (Exception e) {
            System.out.println(e.getMessage());
        }
    }
}
Running state display

Reference link: Connecting Ali Cloud IoT Based on Open Source JAVA MQTT Client

MQ Topic Message Subscription

1,pom.xml


    <dependencies>
        <dependency>
            <groupId>com.aliyun.openservices</groupId>
            <artifactId>ons-client</artifactId>
            <version>1.8.0.Final</version>
        </dependency>
    </dependencies>

2,Code Sample

import com.aliyun.openservices.ons.api.*;
import java.util.Properties;

public class ConsumerTest {
    public static void main(String[] args) {
        Properties properties = new Properties();
        // Group ID you created in the console
        properties.put(PropertyKeyConst.GROUP_ID, "GID_****");
        // AccessKey Ali Cloud Authentication, created in Ali Cloud Server Management Console
        properties.put(PropertyKeyConst.AccessKey,"********");
        // SecretKey Ali Cloud Authentication, created in Ali Cloud Server Management Console
        properties.put(PropertyKeyConst.SecretKey, "********");

        properties.put(PropertyKeyConst.ConsumeThreadNums,50);
        // Set up the TCP access domain name and view in the Get Access Point Information area of the instance management page of the console
        properties.put(PropertyKeyConst.NAMESRV_ADDR,
                "http://MQ_INST_184821781661****_BaQU****.mq-internet-access.mq-internet.aliyuncs.com:80");

        Consumer consumer = ONSFactory.createConsumer(properties);
        consumer.subscribe("Topic_MQDemo", "*", new MessageListener() { //Subscribe to multiple Tag s
            public Action consume(Message message, ConsumeContext context) {
                System.out.println("Receive: " + message);
                System.out.println("Message : " + new String(message.getBody()));//Print Output Message
                return Action.CommitMessage;
            }
        });
        consumer.start();
        System.out.println("Consumer Started");
    }
}

3. Test results

Receive: Message [topic=Topic_MQDemo, systemProperties={KEYS=1202850299555940352, __KEY=1202850299555940352, __RECONSUMETIMES=0, __BORNHOST=/11.115.104.187:51935, __MSGID=0B7368BB19AF531D72CA1D0A9B540077, MIN_OFFSET=520, __BORNTIMESTAMP=1575616834388, MAX_OFFSET=525}, userProperties={UNIQ_KEY=0B7368BB19AF531D72CA1D0A9B540077, MSG_REGION=cn-qingdao-publictest, eagleTraceId=0bc5f2c215756168339094214d0a1b, TRACE_ON=true, eagleData=s3bef5fb6, CONSUME_START_TIME=1575616834470, eagleRpcId=0.1.11.10.10.1.1.1.1}, body=38]
Message : {"data1":"33","deviceName":"MQDevice"}

Log queries (to facilitate tracking, locating, and troubleshooting of problems)

1. Internet of Things Platform - "Operations and Maintenance Monitoring -"Log Service - "Up Log

2. Message Queue RocketMQ -"Message Query

AMQP Server-side Subscription

1. Manage Portal Configuration


2. Code Subscription, Reference link

3,Code Sample

import java.net.URI;
import java.util.Hashtable;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import javax.jms.Connection;
import javax.jms.ConnectionFactory;
import javax.jms.Destination;
import javax.jms.Message;
import javax.jms.MessageConsumer;
import javax.jms.MessageListener;
import javax.jms.MessageProducer;
import javax.jms.Session;
import javax.naming.Context;
import javax.naming.InitialContext;
import org.apache.commons.codec.binary.Base64;
import org.apache.qpid.jms.JmsConnection;
import org.apache.qpid.jms.JmsConnectionListener;
import org.apache.qpid.jms.message.JmsInboundMessageDispatch;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class AmqpJavaClientDemo {

    private final static Logger logger = LoggerFactory.getLogger(AmqpJavaClientDemo.class);

    public static void main(String[] args) throws Exception {
        //Refer to the previous document: AMQP Client Access Instructions for parameter descriptions.
        String accessKey = "******";
        String accessSecret = "******";
        String consumerGroupId = "******";
        long timeStamp = System.currentTimeMillis();
        //Signature method: supports hmacmd5, hmacsha1, and hmacsha256
        String signMethod = "hmacsha1";
        //The clientId parameter is displayed in the Client ID column of the Consumer Group Status Page in the console service-side subscription.
        //It is recommended to use machine UUID, MAC address, IP and other unique identities as clientId.It is easy for you to distinguish and identify different clients.
        String clientId = "yutaodemotest";

logger.debug("demo");

        logger.error("error","this is my test error info.");
        logger.info("info");
        logger.error("error");

        //UserName assembly method, see previous document: AMQP client access instructions.
        String userName = clientId + "|authMode=aksign"
                + ",signMethod=" + signMethod
                + ",timestamp=" + timeStamp
                + ",authId=" + accessKey
                + ",consumerGroupId=" + consumerGroupId
                + "|";
        //For the password assembly method, see the previous document: AMQP Client Access Instructions.
        String signContent = "authId=" + accessKey + "&timestamp=" + timeStamp;
        String password = doSign(signContent,accessSecret, signMethod);
        //Assemble the connection URL according to the qpid-jms specification.
        String connectionUrl = "failover:(amqps://******.iot-amqp.cn-shanghai.aliyuncs.com?amqp.idleTimeout=80000)"
                + "?failover.maxReconnectAttempts=10&failover.reconnectDelay=30";

        Hashtable<String, String> hashtable = new Hashtable<>();
        hashtable.put("connectionfactory.SBCF",connectionUrl);
        hashtable.put("queue.QUEUE", "default");
        hashtable.put(Context.INITIAL_CONTEXT_FACTORY, "org.apache.qpid.jms.jndi.JmsInitialContextFactory");
        Context context = new InitialContext(hashtable);
        ConnectionFactory cf = (ConnectionFactory)context.lookup("SBCF");
        Destination queue = (Destination)context.lookup("QUEUE");
        // Create Connection
        Connection connection = cf.createConnection(userName, password);
        ((JmsConnection) connection).addConnectionListener(myJmsConnectionListener);
        // Create Session
        // Session.CLIENT_ACKNOWLEDGE: After receiving the message, you need to call message.acknowledge() manually
        // Session.AUTO_ACKNOWLEDGE: SDK Automatic ACK (recommended)
        Session session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE);
        connection.start();
        // Create Receiver Link
        MessageConsumer consumer = session.createConsumer(queue);
        consumer.setMessageListener(messageListener);
    }

    private static MessageListener messageListener = new MessageListener() {
        @Override
        public void onMessage(Message message) {
            try {
                byte[] body = message.getBody(byte[].class);
                String content = new String(body);
                String topic = message.getStringProperty("topic");
                String messageId = message.getStringProperty("messageId");
                System.out.println("Content:" + content);
                logger.info("receive message"
                        + ", topic = " + topic
                        + ", messageId = " + messageId
                        + ", content = " + content);
                //If Session is created using Session.CLIENT_ACKNOWLEDGE, a manual ACK is required.
                //message.acknowledge();
                //If you want to process incoming messages in a time-consuming manner, process them asynchronously to ensure that there is no time-consuming logic here.
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    };

    private static JmsConnectionListener myJmsConnectionListener = new JmsConnectionListener() {
        /**
         * The connection was successfully established.
         */
        @Override
        public void onConnectionEstablished(URI remoteURI) {
            logger.info("onConnectionEstablished, remoteUri:{}", remoteURI);
        }

        /**
         * After trying the maximum number of retries, the connection failed.
         */
        @Override
        public void onConnectionFailure(Throwable error) {
            logger.error("onConnectionFailure, {}", error.getMessage());
        }

        /**
         * Connection broken.
         */
        @Override
        public void onConnectionInterrupted(URI remoteURI) {
            logger.info("onConnectionInterrupted, remoteUri:{}", remoteURI);
        }

        /**
         * Connection is automatically reconnected after disconnection.
         */
        @Override
        public void onConnectionRestored(URI remoteURI) {
            logger.info("onConnectionRestored, remoteUri:{}", remoteURI);
        }

        @Override
        public void onInboundMessage(JmsInboundMessageDispatch envelope) {}

        @Override
        public void onSessionClosed(Session session, Throwable cause) {}

        @Override
        public void onConsumerClosed(MessageConsumer consumer, Throwable cause) {}

        @Override
        public void onProducerClosed(MessageProducer producer, Throwable cause) {}
    };

    /**
     * password For a signature calculation method, see the previous document: AMQP Client Access Instructions.
     */
    private static String doSign(String toSignString, String secret, String signMethod) throws Exception {
        SecretKeySpec signingKey = new SecretKeySpec(secret.getBytes(), signMethod);
        Mac mac = Mac.getInstance(signMethod);
        mac.init(signingKey);
        byte[] rawHmac = mac.doFinal(toSignString.getBytes());
        return Base64.encodeBase64String(rawHmac);
    }
}

4. Test results


Content:{"deviceType":"None","iotId":"QyLOC9FsRiJePV*********","requestId":"null","productKey":"a1QVZRPkS5g","gmtCreate":1575632021248,"deviceName":"MQDevice","items":{"Status":{"value":1,"time":1575632021255},"Data":{"value":"33","time":1575632021255}}}

5. Server Subscription Portal Monitoring

More References

Console Configuration AMQP Server-side Subscription

Rocket MQ Subscription Message

Topics: Java Session Mac Apache