Auto Logger: log data in daily CSV files
A very simple flow to store your Node-Red data in CSV file, new file every day. The flow will generate a new file for every day automatically. There is also an archiving flow that moves data older than 10 days from your PI to NAS for long term storage. Check the comment nodes for further details.
More about this flow in this video: https://youtu.be/E2aBIqssQLM
[{"id":"782b06fb.cc1df8","type":"comment","z":"74f191ff.db063","name":"Auto Logger archiving","info":"Generate file name that are to be archived.\n\nIdea is that this generates a list of file names for let's say between current day-5 days to current day-10 days, so all files that are 5-10 days old will get archived. And this can be executed daily or weekly, and the interval allows some overlap (e.g. the program fails, Node-Red was down, etc.)\n\nInject node should contain a JSON:\n{ \n \"start\": 20, \n \"end\": 10\n}\n\nThis will generate file names D-20 to D-10 days.\nStart should be greather than end!\n","x":140,"y":3600,"wires":[]},{"id":"f14e2f71.1ad87","type":"function","z":"74f191ff.db063","name":"Maplin Filename generator","func":"let output = [];\nfor (var i=msg.payload.end;i<msg.payload.start;i++) {\n \n // calculate the date\n let now = new Date();\n now.setTime(now.getTime() - 1000*60*60*24*i);\n let yyyy = now.getFullYear();\n let mm = now.getMonth() < 9 ? \"0\" + (now.getMonth() + 1) : (now.getMonth() + 1); // getMonth() is zero-based\n let dd = now.getDate() < 10 ? \"0\" + now.getDate() : now.getDate();\n let hh = now.getHours() < 10 ? \"0\" + now.getHours() : now.getHours();\n let mmm = now.getMinutes() < 10 ? \"0\" + now.getMinutes() : now.getMinutes();\n let ss = now.getSeconds() < 10 ? \"0\" + now.getSeconds() : now.getSeconds();\n \n let newfile = { \"topic\": \"archive\", \"payload\": \"\"};\n // Generate out file name pattern\n newfile.fname = \"maplin_\"+ yyyy + mm + dd + \".csv\";\n // Full filename with path \n newfile.filename = \"/home/pi/datalog/\"+ newfile.fname;\n \n // Shell script only needs the filename without the path\n newfile.payload = newfile.fname;\n output.push(newfile);\n\n}\n\nreturn [output];","outputs":1,"noerr":0,"initialize":"","finalize":"","x":400,"y":3680,"wires":[["5ba61ce5.11d8a4"]]},{"id":"b57ab78e.cf1658","type":"function","z":"74f191ff.db063","name":"Weather Station Filename generator","func":"let output = [];\nfor (var i=msg.payload.end;i<msg.payload.start;i++) {\n \n // calculate the date\n let now = new Date();\n now.setTime(now.getTime() - 1000*60*60*24*i);\n let yyyy = now.getFullYear();\n let mm = now.getMonth() < 9 ? \"0\" + (now.getMonth() + 1) : (now.getMonth() + 1); // getMonth() is zero-based\n let dd = now.getDate() < 10 ? \"0\" + now.getDate() : now.getDate();\n let hh = now.getHours() < 10 ? \"0\" + now.getHours() : now.getHours();\n let mmm = now.getMinutes() < 10 ? \"0\" + now.getMinutes() : now.getMinutes();\n let ss = now.getSeconds() < 10 ? \"0\" + now.getSeconds() : now.getSeconds();\n \n let newfile = { \"topic\": \"archive\", \"payload\": \"\"};\n // Generate out file name pattern\n newfile.fname = \"weather_sensor_\"+ yyyy + mm + dd + \".csv\";\n // Full filename with path \n newfile.filename = \"/home/pi/datalog/\"+ newfile.fname;\n \n // Shell script only needs the filename without the path\n newfile.payload = newfile.fname;\n output.push(newfile);\n\n}\n\nreturn [output];","outputs":1,"noerr":0,"initialize":"","finalize":"","x":430,"y":3740,"wires":[["5ba61ce5.11d8a4"]]},{"id":"5e47f3b4.7c779c","type":"inject","z":"74f191ff.db063","name":"Start","props":[{"p":"payload","v":"{\"start\":20,\"end\":10}","vt":"json"},{"p":"topic","v":"","vt":"string"}],"repeat":"","crontab":"40 02 * * *","once":false,"onceDelay":0.1,"topic":"","payload":"{\"start\":20,\"end\":10}","payloadType":"json","x":130,"y":3700,"wires":[["f14e2f71.1ad87","b57ab78e.cf1658"]]},{"id":"ad87fee.c53d5","type":"exec","z":"74f191ff.db063","command":"/home/pi/log_upload.sh","addpay":true,"append":"","useSpawn":"false","timer":"","oldrc":false,"name":"File upload","x":730,"y":3680,"wires":[[],[],["5f0f09f4.a09208"]]},{"id":"5f0f09f4.a09208","type":"switch","z":"74f191ff.db063","name":"Return code","property":"payload.code","propertyType":"msg","rules":[{"t":"neq","v":"0","vt":"num"},{"t":"eq","v":"0","vt":"num"}],"checkall":"true","repair":false,"outputs":2,"x":1010,"y":3780,"wires":[[],["639dcab5.1028b4"]]},{"id":"49210be0.0dbbe4","type":"exec","z":"74f191ff.db063","command":"sudo rm ","addpay":true,"append":"","useSpawn":"false","timer":"","oldrc":false,"name":"File delete","x":1150,"y":3680,"wires":[[],[],["1fee1de5.695e32"]]},{"id":"639dcab5.1028b4","type":"change","z":"74f191ff.db063","name":"Get filename","rules":[{"t":"set","p":"payload","pt":"msg","to":"filename","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":970,"y":3680,"wires":[["49210be0.0dbbe4"]]},{"id":"1fee1de5.695e32","type":"switch","z":"74f191ff.db063","name":"Return code","property":"payload.code","propertyType":"msg","rules":[{"t":"neq","v":"0","vt":"num"},{"t":"eq","v":"0","vt":"num"}],"checkall":"true","outputs":2,"x":1370,"y":3700,"wires":[[],[]]},{"id":"5ba61ce5.11d8a4","type":"delay","z":"74f191ff.db063","name":"","pauseType":"rate","timeout":"5","timeoutUnits":"seconds","rate":"1","nbRateUnits":"5","rateUnits":"second","randomFirst":"1","randomLast":"5","randomUnits":"seconds","drop":false,"x":660,"y":3880,"wires":[["ad87fee.c53d5"]]},{"id":"9da5c02.66c844","type":"comment","z":"74f191ff.db063","name":"log_upload.sh","info":"ftp -inv 192.168.1.x << EOF\n user <username> <password>\n binary\n cd backup/logs\n lcd /home/pi/datalog\n put $1\nEOF","x":710,"y":3620,"wires":[]},{"id":"f79c29b7.055068","type":"comment","z":"74f191ff.db063","name":"Auto Logger","info":"","x":150,"y":3960,"wires":[]},{"id":"4c8ae605.33be88","type":"function","z":"74f191ff.db063","name":"Set data","func":"var now = new Date();\n\nmsg.payload = {\n \"timestamp\" : now.getTime(),\n \"temperature\" : msg.msg433.TEMP,\n \"humidity\" : msg.msg433.HUM\n}\nreturn msg;","outputs":1,"noerr":0,"x":240,"y":4020,"wires":[["f46f569e.ff1a08"]]},{"id":"f46f569e.ff1a08","type":"function","z":"74f191ff.db063","name":"Filename generator","func":"// Get the current time and convert it to text\nvar now = new Date();\nvar yyyy = now.getFullYear();\nvar mm = now.getMonth() < 9 ? \"0\" + (now.getMonth() + 1) : (now.getMonth() + 1); // getMonth() is zero-based\nvar dd = now.getDate() < 10 ? \"0\" + now.getDate() : now.getDate();\nvar hh = now.getHours() < 10 ? \"0\" + now.getHours() : now.getHours();\nvar mmm = now.getMinutes() < 10 ? \"0\" + now.getMinutes() : now.getMinutes();\nvar ss = now.getSeconds() < 10 ? \"0\" + now.getSeconds() : now.getSeconds();\n\n// Generate out file name pattern\nmsg.fname = \"maplin_\"+ yyyy + mm + dd + \".csv\";\n// Full filename with path for the file node later\nmsg.filename = \"/home/pi/datalog/\"+ msg.fname;\n\n// We save the current payload into a different place on the msg object\nmsg.filecontent = msg.payload;\n\n// We are passing the file name search pattern to fs node to tell us if the file exists or not\nmsg.payload = {\"pattern\":msg.fname};\n\nnode.status({fill:\"blue\",shape:\"ring\",text:msg.fname});\nreturn msg;","outputs":1,"noerr":0,"x":210,"y":4140,"wires":[["fa6936f.7758dc8"]]},{"id":"fa6936f.7758dc8","type":"fs-file-lister","z":"74f191ff.db063","name":"","start":"/home/pi/datalog/","pattern":"","folders":"*","hidden":true,"lstype":"files","path":true,"single":true,"depth":0,"stat":true,"showWarnings":false,"x":400,"y":4140,"wires":[["a96494a5.b0f4c8","3db97537.b889ea"]]},{"id":"a96494a5.b0f4c8","type":"debug","z":"74f191ff.db063","name":"","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","x":570,"y":4060,"wires":[]},{"id":"3db97537.b889ea","type":"switch","z":"74f191ff.db063","name":"","property":"$count(msg.payload)","propertyType":"jsonata","rules":[{"t":"eq","v":"0","vt":"num"},{"t":"else"}],"checkall":"true","repair":false,"outputs":2,"x":550,"y":4140,"wires":[["3cb887e8.e533a8"],["f3ba1c7a.54ef5"]]},{"id":"fe6f74d5.674ec8","type":"csv","z":"74f191ff.db063","name":"","sep":",","hdrin":"","hdrout":true,"multi":"one","ret":"\\n","temp":"timestamp,temperature,humidity","skip":"0","strings":true,"x":890,"y":4120,"wires":[["3a8ebdf3.772cc2"]]},{"id":"3a8ebdf3.772cc2","type":"file","z":"74f191ff.db063","name":"","filename":"","appendNewline":false,"createDir":true,"overwriteFile":"false","encoding":"none","x":1050,"y":4120,"wires":[[]]},{"id":"f226a86d.9bdfb8","type":"csv","z":"74f191ff.db063","name":"","sep":",","hdrin":"","hdrout":false,"multi":"one","ret":"\\n","temp":"timestamp,temperature,humidity","skip":"0","strings":true,"x":890,"y":4160,"wires":[["3a8ebdf3.772cc2"]]},{"id":"3cb887e8.e533a8","type":"change","z":"74f191ff.db063","name":"Get file content","rules":[{"t":"set","p":"payload","pt":"msg","to":"filecontent","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":720,"y":4120,"wires":[["fe6f74d5.674ec8"]]},{"id":"f3ba1c7a.54ef5","type":"change","z":"74f191ff.db063","name":"Get file content","rules":[{"t":"set","p":"payload","pt":"msg","to":"filecontent","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":720,"y":4160,"wires":[["f226a86d.9bdfb8"]]}]
