Watson IOT device data - Cloudant and back again
How many times have you found examples of how easy it is to store Watson IOT data straight into Cloudant with Node-RED?
Compare that with how often you struggle to get the data back out again (assuming you're in the majority of developers that aren't yet a Cloudant wiz).
A common use-case is to be able to view the current values of sensors being reported from a device :- many common devices report subsets of data in each event, dependent on the sampling/reporting frequency appropriate for the sensor - accelerometer vs humidity, for example. So not all current values are reported at the same time, in the same event.
This flow shows a relatively simple way to create registers to hold the most recent value of any sensor value, for multiple devices. Each register is a single Cloudant document with predictable identifier (_id) - sensorName@deviceId.
In addition, the historical values are stored in similarly named documents - sensorName@deviceId-###, allowing for easier retrieval through a minimal index.
Note: this does not prevent you from also taking the "save it all" approach by dumping the incoming events straight to Cloudant.
There is a navigator API to access the registers, their recent values, and view individual record details:
/register will show a list of sensor/device registers, with links to register recent history
/register/:reg will show a list of recent values, and simple chart to visualise
register:/reg/detail shows record-level values
To make use of the flows, update the IBM IOT App in node(s) with the deviceId of your IOT device, and update the Sensor values flow to set registers for the sensor values you want to track - note, not all sensor values need to be registered, just those you are interested in accessing more easily.
The charts and display pages are for proof-of-concpet purposes -- use/modify/discard as you like.
[{"id":"fade88e1.25a438","type":"ibmiot in","z":"d7f42138.8884d","authentication":"quickstart","apiKey":"","inputType":"evt","deviceId":"dc4f8a341086","applicationId":"","deviceType":"+","eventType":"+","commandType":"","format":"json","name":"IBM IoT dc4f8a341086","service":"quickstart","allDevices":"","allApplications":"","allDeviceTypes":true,"allEvents":true,"allCommands":"","allFormats":"","qos":0,"x":120,"y":203,"wires":[["92823421.698a3"]]},{"id":"f2492763.52141","type":"cloudant out","z":"d7f42138.8884d","name":"store reg","cloudant":"","database":"iottest","service":"rdc-gb-whereismypi-cloudantNoSQLDB","payonly":true,"operation":"insert","x":900,"y":403,"wires":[]},{"id":"9d29f869.710e08","type":"debug","z":"d7f42138.8884d","name":"","active":false,"console":"false","complete":"true","x":890,"y":203,"wires":[]},{"id":"67c68b68.da732c","type":"function","z":"d7f42138.8884d","name":"update register","func":"msg.dataset._rev = msg.cloudant._rev;\nreturn msg;","outputs":1,"noerr":0,"x":509,"y":409,"wires":[["abe3e022.16dad"]]},{"id":"bb4c2e9d.52bbf","type":"cloudant in","z":"d7f42138.8884d","name":"reg check","cloudant":"","database":"iottest","service":"rdc-gb-whereismypi-cloudantNoSQLDB","search":"_id_","design":"","index":"","x":700,"y":297,"wires":[["ebca9acc.5fd0f","461fadc8.302074"]]},{"id":"dbb8c28c.9729b8","type":"switch","z":"d7f42138.8884d","name":"check _rev","property":"cloudant._rev","propertyType":"msg","rules":[{"t":"null"},{"t":"else"}],"checkall":"true","outputs":2,"x":281,"y":402,"wires":[["1a7c310e.3aa467"],["e02cb71a.5d2ec8","67c68b68.da732c"]]},{"id":"ebca9acc.5fd0f","type":"debug","z":"d7f42138.8884d","name":"","active":false,"console":"false","complete":"true","x":890,"y":297,"wires":[]},{"id":"b9ab6bf.f7b6998","type":"change","z":"d7f42138.8884d","name":"set search id","rules":[{"t":"set","p":"payload","pt":"msg","to":"_id","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":505,"y":297,"wires":[["bb4c2e9d.52bbf"]]},{"id":"e8d59175.3416","type":"change","z":"d7f42138.8884d","name":"save payload","rules":[{"t":"set","p":"dataset","pt":"msg","to":"payload","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":130,"y":297,"wires":[["ab50310a.84c848"]]},{"id":"e02cb71a.5d2ec8","type":"function","z":"d7f42138.8884d","name":"& save history","func":"var seqnum = msg.cloudant._rev.split(\"-\")[0];\nmsg.dataset._id += \"-\" + seqnum;\nmsg.dataset.seqnum = seqnum;\nreturn msg;","outputs":1,"noerr":0,"x":507,"y":455,"wires":[["abe3e022.16dad"]]},{"id":"461fadc8.302074","type":"function","z":"d7f42138.8884d","name":"","func":"if(!msg.hasOwnProperty(\"cloudant\")) {\n msg.cloudant = '';\n msg.cloudant._rev = '';\n}\n//delete msg.dataset.d;\nreturn msg;","outputs":1,"noerr":0,"x":110,"y":403,"wires":[["dbb8c28c.9729b8"]]},{"id":"1a7c310e.3aa467","type":"function","z":"d7f42138.8884d","name":"add register","func":"\nreturn msg;","outputs":1,"noerr":0,"x":499,"y":366,"wires":[["abe3e022.16dad"]]},{"id":"abe3e022.16dad","type":"change","z":"d7f42138.8884d","name":"restore payload","rules":[{"t":"set","p":"payload","pt":"msg","to":"dataset","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":720,"y":403,"wires":[["f2492763.52141"]]},{"id":"d7dbf27c.6beb1","type":"comment","z":"d7f42138.8884d","name":"check for existing register value","info":"if the register has already been created,\nretrieve the *_rev* value needed to allow\nthe register to be updated","x":670,"y":260,"wires":[]},{"id":"27dc9109.540136","type":"comment","z":"d7f42138.8884d","name":"create/update register value","info":"if there is no *_rev* value present, the \nregister will be created/initialized","x":760,"y":363,"wires":[]},{"id":"261bc7e9.9048b","type":"comment","z":"d7f42138.8884d","name":"save history","info":"update the existing register value, and create\na corresponding history record","x":710,"y":443,"wires":[]},{"id":"d5b6be75.079d38","type":"http in","z":"d7f42138.8884d","name":"","url":"/register/:reg","method":"get","upload":false,"swaggerDoc":"","x":120,"y":678,"wires":[["a8bcdffb.f9df18","d2b4f900.86ba38"]]},{"id":"115d3c3e.5a75ec","type":"cloudant in","z":"d7f42138.8884d","name":"","cloudant":"","database":"iottest","service":"rdc-gb-whereismypi-cloudantNoSQLDB","search":"_idx_","design":"history","index":"history","x":490,"y":678,"wires":[["2541e6be.41e312","6d09eda6.36803c"]]},{"id":"a8bcdffb.f9df18","type":"function","z":"d7f42138.8884d","name":"x most recent","func":"var recent = global.get(\"recent\");\nmsg.payload = { query:\"name:\" + msg.req.params.reg + \"*\",\n \"limit\": recent,\n \"sort\": \"-seqnum\"\n };\nreturn msg;","outputs":1,"noerr":0,"x":310,"y":678,"wires":[["115d3c3e.5a75ec"]]},{"id":"b05f7e07.3ced48","type":"template","z":"d7f42138.8884d","name":"chart","field":"payload","fieldType":"msg","format":"handlebars","syntax":"mustache","template":"<script src=\"https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.4.0/Chart.min.js\"></script>\n<h2>{{req.params.reg}} values:</h2>\n<table>\n<tr>\n <td>\n <table>\n <tr><th>record-id</th><th>reading</th></tr>\n {{#payload}}\n <tr> <td><a href=/register/{{_id}}/detail>{{_id}}</a></td><td>{{value}}</td></tr>\n {{/payload}}\n </table>\n </td>\n <td style=\"vertical-align:top\">\n <canvas id=\"plot\"></canvas>\n </td>\n</tr>\n</table>\n<script>\n var ctx = document.getElementById('plot').getContext('2d');\n var chart = new Chart(ctx, {\n type: 'scatter',\n data: {\n// labels: [\n// {{#payload}}\n// \"{{order}}\",\n// {{/payload}}\n// ],\n datasets: [{\n label: \"{{req.params.reg}}\",\n backgroundColor: 'rgb(40,170,40)',\n borderColor: 'rgb(255, 0, 130)',\n fill: false,\n data: [\n {{#payload}}\n { x:{{order}}, y:{{value}} },\n {{/payload}}\n ],\n }]\n },\n\n // Configuration options go here\n options: {}\n});\n</script>\n","output":"str","x":771,"y":678,"wires":[["717c9c2.c4dac64"]]},{"id":"717c9c2.c4dac64","type":"http response","z":"d7f42138.8884d","name":"","statusCode":"","headers":{},"x":890,"y":678,"wires":[]},{"id":"2541e6be.41e312","type":"debug","z":"d7f42138.8884d","name":"","active":false,"console":"false","complete":"false","x":610,"y":738,"wires":[]},{"id":"d2b4f900.86ba38","type":"debug","z":"d7f42138.8884d","name":"","active":false,"console":"false","complete":"true","x":250,"y":738,"wires":[]},{"id":"2a7423b5.731b54","type":"comment","z":"d7f42138.8884d","name":"Create current value registers and historical IOT data","info":"There is a frequent need store IOT data streams into\nCloudant database, and then interrogate the stored data.\n\nA common use-case is to retrieve the \n\"current value\" of each attribute of interest - \nthese values may not necessarily be present \nin the most recent data logged from the device.\n\nThis example shows how to store the most recent\nattribute value in a document of its own, \nand generate a histoy trail for trending analysis, etc\n\n","x":210,"y":163,"wires":[]},{"id":"ab50310a.84c848","type":"function","z":"d7f42138.8884d","name":"registers","func":"var reg = global.get(\"registers\");\nvar sensreg = [];\nvar now = Date.now();\nvar dev = msg.topic.split('/')[4];\nfor (var r of reg){\n if (msg.dataset.d.hasOwnProperty(r)){\n var did = r + \"\\@\" + dev;\n sensreg.push(\n { _id: did, \n dataset: \n { _id: did,\n value: msg.dataset.d[r],\n d: msg.dataset.d,\n ts: now\n }\n }\n );\n }\n}\nreturn [ sensreg ];\n\n","outputs":1,"noerr":0,"x":300,"y":297,"wires":[["b9ab6bf.f7b6998"]]},{"id":"2e46a8c7.f703b8","type":"comment","z":"d7f42138.8884d","name":"select register values","info":"often, devices will send subsets of sensor \ndata at different frequencies - this stage\nchecks and captures particular values, if\navailable, in the incoming event","x":380,"y":260,"wires":[]},{"id":"6ceaa153.affc8","type":"http in","z":"d7f42138.8884d","name":"","url":"/register/:reg/detail","method":"get","upload":false,"swaggerDoc":"","x":140,"y":798,"wires":[["9ff9afd4.cf1ab8","4d38042f.60c8f4"]]},{"id":"a4e1f0de.a023c8","type":"cloudant in","z":"d7f42138.8884d","name":"","cloudant":"","database":"iottest","service":"rdc-gb-whereismypi-cloudantNoSQLDB","search":"_id_","design":"history","index":"history","x":490,"y":798,"wires":[["1bffa9b6.d26d86","11ea1e61.3802b2"]]},{"id":"9ff9afd4.cf1ab8","type":"function","z":"d7f42138.8884d","name":"","func":"msg.payload = msg.req.params.reg;\nreturn msg;","outputs":1,"noerr":0,"x":350,"y":798,"wires":[["a4e1f0de.a023c8"]]},{"id":"51d79abb.2905b4","type":"template","z":"d7f42138.8884d","name":"display","field":"payload","fieldType":"msg","format":"handlebars","syntax":"mustache","template":"<h2>{{req.params.reg}} details: </h2>\n<br>\nValue: {{payload.value}}\n<br>\nSequence: {{payload.seqnum}}\n<br>\nTimestamp: {{payload.ts}}\n<p>\n{{#display}}\n<table>\n<tr><th>Key</th><th>Value</th></tr>\n{{#d}}\n <tr><td>{{key}}</td><td>{{value}}</td></tr>\n{{/d}}\n</table>\n{{/display}}\n","output":"str","x":760,"y":798,"wires":[["4a75b2f8.666fbc"]]},{"id":"4a75b2f8.666fbc","type":"http response","z":"d7f42138.8884d","name":"","statusCode":"","headers":{},"x":890,"y":798,"wires":[]},{"id":"1bffa9b6.d26d86","type":"debug","z":"d7f42138.8884d","name":"","active":false,"console":"false","complete":"false","x":611,"y":854,"wires":[]},{"id":"4d38042f.60c8f4","type":"debug","z":"d7f42138.8884d","name":"","active":false,"console":"false","complete":"true","x":250,"y":858,"wires":[]},{"id":"11ea1e61.3802b2","type":"function","z":"d7f42138.8884d","name":"details","func":"var display={ 'd' : [] };\n\n// build list of properties as \n// k/v pairs for easy mustache display\nvar o = msg.payload.d;\nfor (var prop in o){\n if (o.hasOwnProperty(prop)){\n display['d'].push({\n 'key' : prop,\n 'value' : o[prop]\n });\n }\n}\nmsg.display = display;\nreturn msg;","outputs":1,"noerr":0,"x":630,"y":798,"wires":[["51d79abb.2905b4"]]},{"id":"8a4fd55e.2c82e8","type":"comment","z":"d7f42138.8884d","name":"Register API","info":"|URL| result |\n|--|--|\n|**/registers**|retrieve current values for all monitored sensor values |\n|**/register/:reg**| retrieve 20 most recent values for specific register|\n|**/register/:reg/detail**| display detail of original sensor event|","x":110,"y":503,"wires":[]},{"id":"acaadca5.325aa8","type":"inject","z":"d7f42138.8884d","name":"Sensor values","topic":"","payload":"","payloadType":"date","repeat":"","crontab":"","once":true,"x":140,"y":80,"wires":[["75d117a5.3af968"]]},{"id":"5a9b4bc1.310eac","type":"debug","z":"d7f42138.8884d","name":"","active":false,"console":"false","complete":"true","x":890,"y":120,"wires":[]},{"id":"2ea1a98b.253096","type":"http in","z":"d7f42138.8884d","name":"","url":"/registers","method":"get","upload":false,"swaggerDoc":"","x":110,"y":558,"wires":[["f96e3760.8bd9c8","20715eed.2c7a7a"]]},{"id":"e00734d2.8599d","type":"http response","z":"d7f42138.8884d","name":"","statusCode":"","headers":{},"x":890,"y":558,"wires":[]},{"id":"ed405d1c.7f2fc","type":"cloudant in","z":"d7f42138.8884d","name":"","cloudant":"","database":"iottest","service":"rdc-gb-whereismypi-cloudantNoSQLDB","search":"_id_","design":"","index":"","x":490,"y":558,"wires":[["fdad2825.d1d04","5b54c777.d21ef"]]},{"id":"e4b080ae.e99c","type":"template","z":"d7f42138.8884d","name":"display","field":"payload","fieldType":"msg","format":"handlebars","syntax":"mustache","template":"<h2>Current registers:</h2> \n<br>\n <table>\n {{#payload}}\n <tr><td><a href=/register/{{_id}}>{{_id}}</a></td><td>{{value}}</td></tr>\n {{/payload}}\n </table>","output":"str","x":760,"y":558,"wires":[["e00734d2.8599d"]]},{"id":"f96e3760.8bd9c8","type":"function","z":"d7f42138.8884d","name":"list","func":"//get list of currently registered values\nvar regs = global.get(\"registers\");\nvar devs = global.get(\"devices\");\nvar chk = [];\nfor( var r of regs){\n for ( var d of devs) {\n chk.push(r + \"@\" + d);\n }\n}\nnode.warn(devs);\nmsg.payload = chk;\nreturn msg;","outputs":1,"noerr":0,"x":250,"y":558,"wires":[["c7e1afef.7d457"]]},{"id":"c7e1afef.7d457","type":"split","z":"d7f42138.8884d","name":"","splt":"\\n","spltType":"str","arraySplt":1,"arraySpltType":"len","stream":false,"addname":"","x":370,"y":558,"wires":[["ed405d1c.7f2fc"]]},{"id":"fdad2825.d1d04","type":"join","z":"d7f42138.8884d","name":"","mode":"auto","build":"string","property":"payload","propertyType":"msg","key":"topic","joiner":"\\n","joinerType":"str","accumulate":"false","timeout":"","count":"","x":615,"y":558,"wires":[["e4b080ae.e99c"]]},{"id":"5b54c777.d21ef","type":"debug","z":"d7f42138.8884d","name":"","active":false,"console":"false","complete":"false","x":610,"y":618,"wires":[]},{"id":"20715eed.2c7a7a","type":"debug","z":"d7f42138.8884d","name":"","active":false,"console":"false","complete":"true","x":250,"y":618,"wires":[]},{"id":"2e02d5fb.6cd8aa","type":"inject","z":"d7f42138.8884d","name":"","topic":"","payload":"humidity","payloadType":"str","repeat":"","crontab":"","once":false,"x":120,"y":1014,"wires":[["e1e61d1.42d9ae"]]},{"id":"ce72519b.cde39","type":"cloudant in","z":"d7f42138.8884d","name":"","cloudant":"","database":"iottest","service":"rdc-gb-whereismypi-cloudantNoSQLDB","search":"_id_","design":"","index":"","x":470,"y":1014,"wires":[["84ed938f.040b18"]]},{"id":"84ed938f.040b18","type":"debug","z":"d7f42138.8884d","name":"","active":true,"console":"false","complete":"false","x":890,"y":1014,"wires":[]},{"id":"85b870cf.da96","type":"inject","z":"d7f42138.8884d","name":"","topic":"","payload":"objectTemp","payloadType":"str","repeat":"","crontab":"","once":false,"x":130,"y":1054,"wires":[["e1e61d1.42d9ae"]]},{"id":"c9fb5849.a31a2","type":"inject","z":"d7f42138.8884d","name":"","topic":"","payload":"temp","payloadType":"str","repeat":"","crontab":"","once":false,"x":110,"y":1094,"wires":[["e1e61d1.42d9ae"]]},{"id":"ae27f481.2c9268","type":"cloudant in","z":"d7f42138.8884d","name":"","cloudant":"","database":"iottest","service":"rdc-gb-whereismypi-cloudantNoSQLDB","search":"_idx_","design":"history","index":"history","x":690,"y":1114,"wires":[["240f48cd.ace1b8"]]},{"id":"e1e61d1.42d9ae","type":"function","z":"d7f42138.8884d","name":"","func":"\nreturn msg;","outputs":1,"noerr":0,"x":290,"y":1054,"wires":[["ce72519b.cde39","6ee2cb9f.01d87c"]]},{"id":"240f48cd.ace1b8","type":"debug","z":"d7f42138.8884d","name":"","active":true,"console":"false","complete":"true","x":870,"y":1114,"wires":[]},{"id":"6ee2cb9f.01d87c","type":"function","z":"d7f42138.8884d","name":"20 most recent","func":"msg.payload = { query:\"name:\" + msg.payload + \"*\",\n \"limit\": \"20\",\n \"sort\": \"-seqnum\"\n };\nreturn msg;","outputs":1,"noerr":0,"x":500,"y":1114,"wires":[["ae27f481.2c9268"]]},{"id":"a72fb319.ff6c68","type":"comment","z":"d7f42138.8884d","name":"test queries","info":"","x":110,"y":974,"wires":[]},{"id":"8705b35f.0dd4","type":"comment","z":"d7f42138.8884d","name":"current register","info":"","x":620,"y":1034,"wires":[]},{"id":"6e0eb339.1baf64","type":"comment","z":"d7f42138.8884d","name":"history array","info":"array of most recent register values,\nin descending order","x":610,"y":1074,"wires":[]},{"id":"e818d3e0.5e689","type":"comment","z":"d7f42138.8884d","name":"Names of WIOT sensor values to monitor","info":"","x":194.5,"y":38.399993896484375,"wires":[]},{"id":"a936cdb8.1961","type":"change","z":"d7f42138.8884d","name":"create index","rules":[{"t":"set","p":"payload","pt":"msg","to":"{\"_id\":\"_design/history\",\"views\":{},\"language\":\"javascript\",\"indexes\":{\"history\":{\"analyzer\":\"keyword\",\"index\":\"function (doc) {\\n index(\\\"name\\\", doc._id);\\n index(\\\"seqnum\\\", doc.seqnum);\\n}\"}}}","tot":"json"}],"action":"","property":"","from":"","to":"","reg":false,"x":690,"y":80,"wires":[["5a9b4bc1.310eac","d1979a36.12ba5"]]},{"id":"d1979a36.12ba5","type":"cloudant out","z":"d7f42138.8884d","name":"","cloudant":"","database":"iottest","service":"rdc-gb-whereismypi-cloudantNoSQLDB","payonly":true,"operation":"insert","x":890,"y":80,"wires":[]},{"id":"92823421.698a3","type":"delay","z":"d7f42138.8884d","name":"","pauseType":"delay","timeout":"0","timeoutUnits":"seconds","rate":"1","nbRateUnits":"30","rateUnits":"second","randomFirst":"1","randomLast":"5","randomUnits":"seconds","drop":true,"x":380,"y":203,"wires":[["e8d59175.3416","85e47dd7.5952a8"]]},{"id":"6d09eda6.36803c","type":"function","z":"d7f42138.8884d","name":"chart order","func":"var order = global.get(\"recent\");\nfor(var o of msg.payload){\n o.order = order--;\n}\nreturn msg;","outputs":1,"noerr":0,"x":632,"y":678,"wires":[["b05f7e07.3ced48"]]},{"id":"75d117a5.3af968","type":"change","z":"d7f42138.8884d","name":"global.registers","rules":[{"t":"set","p":"registers","pt":"global","to":"[\"objectTemp\",\"temp\",\"humidity\"]","tot":"json"},{"t":"set","p":"devices","pt":"global","to":"[]","tot":"json"}],"action":"","property":"","from":"","to":"","reg":false,"x":320,"y":80,"wires":[["8fa0a197.041b08"]]},{"id":"8fa0a197.041b08","type":"change","z":"d7f42138.8884d","name":"global.recent","rules":[{"t":"set","p":"recent","pt":"global","to":"20","tot":"num"}],"action":"","property":"","from":"","to":"","reg":false,"x":510,"y":80,"wires":[["a936cdb8.1961"]]},{"id":"6a5b8ef8.407d7","type":"http in","z":"d7f42138.8884d","name":"","url":"/set/recent/:limit","method":"get","upload":false,"swaggerDoc":"","x":130,"y":900,"wires":[["1ca974a.84f530b"]]},{"id":"1ca974a.84f530b","type":"change","z":"d7f42138.8884d","name":"update global.recent","rules":[{"t":"set","p":"recent","pt":"global","to":"req.params.limit","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":420,"y":900,"wires":[["c77654f4.81f37"]]},{"id":"c77654f4.81f37","type":"http response","z":"d7f42138.8884d","name":"","statusCode":"","headers":{},"x":890,"y":900,"wires":[]},{"id":"85e47dd7.5952a8","type":"function","z":"d7f42138.8884d","name":"Devices","func":"// keep track of reporting devices\nvar devs = global.get(\"devices\") || [];\nvar dev = msg.topic.split('/')[4];\nvar newdev = true;\n\nfor (var d of devs){\n if (d === dev){\n newdev = false;\n } \n}\nif (newdev){\n devs.push(dev);\n global.set(\"devices\",devs);\n}\nmsg.devs=devs;\nreturn msg;","outputs":1,"noerr":0,"x":600,"y":203,"wires":[["9d29f869.710e08"]]},{"id":"41ccb828.a4a188","type":"ibmiot in","z":"d7f42138.8884d","authentication":"quickstart","apiKey":"","inputType":"evt","deviceId":"df8351be8a3f","applicationId":"","deviceType":"+","eventType":"+","commandType":"","format":"json","name":"IBM IoT df8351be8a3f","service":"quickstart","allDevices":"","allApplications":"","allDeviceTypes":true,"allEvents":true,"allCommands":"","allFormats":"","qos":0,"x":120,"y":240,"wires":[["92823421.698a3"]]}]
