COSMOS data ingestion flow from Meteo.gr

The flow is used to get data from the weather site www.meteo.gr, regarding weather conditions on the Aegean sea. Polling is performed every 6 hours, when the update to the website data is performed. Data are stored locally in a csv file (you need to specify the path to local file).

[{"id":"49aa173a.3cee5","type":"comment","name":"README","info":"Needs to have installed in settings.js file\nthe node.js package \"time\"\n\nNeeds to be manually triggered once in order\nto start the periodic data collection\n\nNeeds to configure the local path to file for\nstoring of the data","x":635.1388778686523,"y":271.80553245544434,"z":"98225c68.c4dbd8","wires":[]},{"id":"3b95eb2e.5507b4","type":"http request","name":"","method":"GET","ret":"txt","url":"","x":500.08331298828125,"y":176.08334350585938,"z":"98225c68.c4dbd8","wires":[["f4cd0840.f90dd8","71d3c80e.5128c"]]},{"id":"d780ac27.b55f78","type":"inject","name":"TRIGGER ONCE, POLLING EVERY 6 HOURS","topic":"","payload":"","payloadType":"date","repeat":"21600","crontab":"","once":false,"x":220.3333282470703,"y":37.08335876464844,"z":"98225c68.c4dbd8","wires":[["b00b822c.fb1918"]]},{"id":"b00b822c.fb1918","type":"function","name":"Configure calls","func":"context.global.timestamp={};\ncontext.global.timestamp=msg.payload;\n//variable to indicate if this is the last message\n//and the flow should stop\nmsg.final=false;\n\n//configure base url\nvar baseurl='http://www.meteo.gr/meteoplus/Sailing/templates/waves_sp1JSON.cfm?SailMapID=';\n\nvar extension='&FTime=3';\n//needs a for loop for every tile from 1 to 16\n//needs node.send for every tile request\n\nfor (i = 1; i < 17; i++) {\n    //text += \"The number is \" + i + \"<br>\";\n    var overall_url=baseurl+i+extension;\n    //here we need to check how it is passed to the \n    //http request node\n    msg.url=overall_url;\n    node.send(msg);\n}\nmsg.final=true;\nreturn msg;\n\n\n","outputs":1,"valid":true,"x":134.0833282470703,"y":165.08334350585938,"z":"98225c68.c4dbd8","wires":[["129aec27.a5f904"]]},{"id":"129aec27.a5f904","type":"switch","name":"Final tile?","property":"final","rules":[{"t":"false"},{"t":"true"}],"checkall":"true","outputs":2,"x":134.7976245880127,"y":250.5119113922119,"z":"98225c68.c4dbd8","wires":[["bfbf584a.dfa59","bfe9ae5b.2519c"],["eb37181e.76e468"]]},{"id":"eb37181e.76e468","type":"debug","name":"DO NOOP","active":true,"console":"false","complete":"true","x":195.08333206176758,"y":367.0833206176758,"z":"98225c68.c4dbd8","wires":[]},{"id":"bfbf584a.dfa59","type":"debug","name":"","active":true,"console":"false","complete":"url","x":411.5117988586426,"y":397.36906242370605,"z":"98225c68.c4dbd8","wires":[]},{"id":"da99f010.349188","type":"file","name":"/home/wp6/fablab/data","filename":"/home/wp6/fablab/data","appendNewline":true,"overwriteFile":"false","x":1053.511863708496,"y":56.6547737121582,"z":"98225c68.c4dbd8","wires":[]},{"id":"767ab8f0.25dfc8","type":"mysql","name":"","x":1026.7499465942383,"y":140.7500171661377,"z":"98225c68.c4dbd8","wires":[[]]},{"id":"a4a89771.8d3d2","type":"function","name":"Filter fields and add ts","func":"msg.payload.timestamp=context.global.timestamp;\ndelete msg.payload.COLUMNS;\nreturn msg;","outputs":1,"valid":true,"x":772.0832633972168,"y":57.94045448303223,"z":"98225c68.c4dbd8","wires":[["e94cb0b0.8d652"]]},{"id":"f4cd0840.f90dd8","type":"json","name":"","x":613.1428184509277,"y":118.28570747375488,"z":"98225c68.c4dbd8","wires":[["a4a89771.8d3d2"]]},{"id":"63c591ca.0a41a8","type":"comment","name":"PENDING POINTS: OUTPUT FORMAT WRAPPING, which values, wrap arrays into lines, DUPLICATE VALUES","info":"","x":590.9999389648438,"y":723.3332672119141,"z":"98225c68.c4dbd8","wires":[]},{"id":"71d3c80e.5128c","type":"debug","name":"","active":false,"console":"false","complete":"false","x":979.9999999999998,"y":257.99999999999994,"z":"98225c68.c4dbd8","wires":[]},{"id":"bfe9ae5b.2519c","type":"delay","name":"","pauseType":"rate","timeout":"2","timeoutUnits":"seconds","rate":"0.5","rateUnits":"second","randomFirst":"1","randomLast":"5","randomUnits":"seconds","drop":false,"x":461.4285714285714,"y":88.57142857142857,"z":"98225c68.c4dbd8","wires":[["3b95eb2e.5507b4"]]},{"id":"e94cb0b0.8d652","type":"switch","name":"Null DATA","property":"payload.DATA.length","rules":[{"t":"gt","v":"0"},{"t":"eq","v":"0"}],"checkall":"true","outputs":2,"x":866.805549621582,"y":135.1388931274414,"z":"98225c68.c4dbd8","wires":[["da99f010.349188"],["71d3c80e.5128c"]]}]
COSMOSFP7

Flow Info

created 10 months, 2 weeks ago

Node Types

Core
  • comment (x2)
  • debug (x3)
  • delay (x1)
  • file (x1)
  • function (x2)
  • http request (x1)
  • inject (x1)
  • json (x1)
  • switch (x2)
Other

Tags

  • weather
  • data
  • greece
  • cosmos
Copy this flow JSON to your clipboard and then import into Node-RED using the Import From > Clipboard (Ctrl-I) menu option