getUserMedia video + audio with Multipart forms + Visual Recognition
This flow demonstrates how to capture video/audio streams and file blobs (an image in this example) in a client application and pass them to a Node-RED flow via a multipart form + the fetch API.
Images posted to the Node-RED flow can be passed through Watson Visual Recognition for quick analysis. Otherwise, the files posted can manipulated as the developer desires.
[{"id":"66d82650.1f4618","type":"httpInMultipart","z":"59091b6d.99c4b4","name":"/analyse","url":"/analyse","method":"post","fields":"[ { \"name\" : \"image\"}, { \"name\" : \"video\" }, { \"name\" : \"audio\" } ]","swaggerDoc":"","x":75,"y":150,"wires":[["99296515.24cdc8","eccd9f01.0c201"]]},{"id":"99296515.24cdc8","type":"debug","z":"59091b6d.99c4b4","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"true","x":215,"y":195,"wires":[]},{"id":"3f17b5d9.56444a","type":"http in","z":"59091b6d.99c4b4","name":"","url":"/demo/image","method":"get","upload":false,"swaggerDoc":"","x":105,"y":465,"wires":[["e989f438.843f68"]]},{"id":"ccb0c592.09a798","type":"http response","z":"59091b6d.99c4b4","name":"","statusCode":"","headers":{},"x":695,"y":465,"wires":[]},{"id":"e989f438.843f68","type":"template","z":"59091b6d.99c4b4","name":"Styles","field":"payload.css","fieldType":"msg","format":"css","syntax":"mustache","template":"html, body{\n padding : 0;\n width : 100%;\n height : 100%;\n margin: 0;\n font-family: sans-serif;\n}\n\nbody{\n display: flex;\n flex-direction: column;\n align-items: center;\n}\n\nbody[data-recording=\"true\"] #start-recording, body[data-recording=\"false\"] #stop-recording{\n display: none;\n}\n\nvideo{\n position: fixed;\n top:100%;\n}\n\nbutton {\n display: inline-block;\n padding: 1em 1.2em;\n border-radius: 5px;\n border: 2px solid;\n font-weight: 800;\n cursor: pointer;\n outline: transparent;\n margin-top: 1em;\n}\n\nbutton:hover{\n background: black;\n color: white;\n}\n\n#response-message{\n position: fixed;\n top: 75%;\n left: 0;\n width: 100%;\n padding: 1em;\n color: white;\n text-align: center;\n font-weight: 800;\n box-shadow: 0 1px 1px black;\n text-shadow: 0 1px 1px black;\n box-sizing: border-box;\n display: none;\n}\n\n#response-message[data-success=\"true\"]{\n background-color: green;\n}\n\n#response-message[data-success=\"false\"]{\n background-color: red;\n}","output":"str","x":275.5,"y":465,"wires":[["baa0e73c.8ac2e8"]]},{"id":"baa0e73c.8ac2e8","type":"template","z":"59091b6d.99c4b4","name":"JavaScript","field":"payload.javascript","fieldType":"msg","format":"javascript","syntax":"mustache","template":"(function(){\n \n 'use strict';\n \n console.log('Hello!');\n \n const formStatus = document.querySelector('#response-message');\n const video = document.querySelector('#camera-capture');\n const canvas = document.querySelector('#capture-canvas');\n const ctx = canvas.getContext('2d');\n \n const sendImageToServerBtn = document.querySelector('#snap-image');\n \n const constraints = {\n video : true,\n audio : false\n };\n \n navigator.mediaDevices.getUserMedia(constraints)\n .then(function(stream) {\n console.log(stream);\n \n let hideResult;\n\n video.addEventListener('canplay', function(){\n this.play();\n\n canvas.width = video.offsetWidth;\n canvas.height = video.offsetHeight;\n\n });\n\n const vidURL = window.URL.createObjectURL(stream);\n video.src = vidURL;\n \n function drawVideoToCanvas(){\n ctx.drawImage(video, 0, 0);\n window.requestAnimationFrame(drawVideoToCanvas);\n }\n \n drawVideoToCanvas();\n \n sendImageToServerBtn.addEventListener('click', function(){\n console.log('Click!');\n \n canvas.toBlob(function(blob){\n \n console.log(blob);\n const form = new FormData();\n \n form.append('image', blob, `${Date.now() / 1000 | 0}.png`);\n \n console.log( form.get('image') );\n \n fetch('/analyse', {\n method : 'post',\n body : form\n })\n .then(function(res){\n if(res.ok){\n formStatus.textContent = 'Image saved!';\n formStatus.dataset.success = \"true\";\n formStatus.style.display = \"block\";\n \n clearTimeout(hideResult);\n\n hideResult = setTimeout(function(){\n formStatus.style.display = \"none\";\n }, 3000);\n\n return res.json();\n } else {\n throw res;\n }\n })\n .then(function(data){\n console.log(data);\n })\n .catch(function(err){\n formStatus.textContent = 'Could not save image :(';\n formStatus.dataset.success = \"false\";\n formStatus.style.display = \"block\";\n\n clearTimeout(hideResult);\n\n hideResult = setTimeout(function(){\n formStatus.style.display = \"none\";\n }, 3000);\n\n console.log('fetch err:', err);\n })\n ;\n \n \n }, 'image/png', 100);\n \n }, false);\n \n })\n .catch(function(err) {\n console.log('gUM Error:', err);\n })\n ;\n \n})();","output":"str","x":430,"y":465,"wires":[["e18ca9eb.9f04b8"]]},{"id":"e18ca9eb.9f04b8","type":"template","z":"59091b6d.99c4b4","name":"HTML","field":"payload","fieldType":"msg","format":"handlebars","syntax":"mustache","template":"<html>\n <head>\n <style>\n {{{payload.css}}}\n </style>\n </head>\n <body>\n <h1>Smile!</h1>\n <p>A demo app for sending an image to a Node-red server (with Fetch, FormData, and Multipart)</p>\n \n <video id=\"camera-capture\"></video>\n <canvas id=\"capture-canvas\"></canvas>\n \n <button id=\"snap-image\">Send image to server</button>\n \n <div id=\"response-message\"></div>\n \n <script>{{{payload.javascript}}}</script>\n </body>\n \n</html>","output":"str","x":575,"y":465,"wires":[["ccb0c592.09a798"]]},{"id":"cfefcd35.d85b3","type":"comment","z":"59091b6d.99c4b4","name":"Analyse the image.","info":"","x":105,"y":90,"wires":[]},{"id":"d295e43f.4db3b8","type":"comment","z":"59091b6d.99c4b4","name":"Load the demo image app","info":"","x":124.5,"y":420,"wires":[]},{"id":"620bd33e.a1e79c","type":"http in","z":"59091b6d.99c4b4","name":"","url":"/demo/video","method":"get","upload":false,"swaggerDoc":"","x":95,"y":585,"wires":[["e2ea473e.545d58"]]},{"id":"e63b0bd5.e11038","type":"http response","z":"59091b6d.99c4b4","name":"","statusCode":"","headers":{},"x":695,"y":585,"wires":[]},{"id":"e2ea473e.545d58","type":"template","z":"59091b6d.99c4b4","name":"Styles","field":"payload.css","fieldType":"msg","format":"css","syntax":"mustache","template":"html, body{\n padding : 0;\n width : 100%;\n height : 100%;\n margin: 0;\n font-family: sans-serif;\n}\n\nbody{\n display: flex;\n flex-direction: column;\n align-items: center;\n}\n\nbody[data-recording=\"true\"] #start-recording, body[data-recording=\"false\"] #stop-recording{\n display: none;\n}\n\n.btnContainer{\n margin-top: 1em;\n}\n\nbutton {\n display: inline-block;\n padding: 1em 1.2em;\n border-radius: 5px;\n border: 2px solid;\n font-weight: 800;\n cursor: pointer;\n outline: transparent;\n}\n\n#start-recording{\n color: green;\n border-color: green;\n}\n\n#start-recording:hover{\n background-color: green;\n color: white;\n}\n\n#stop-recording{\n color: red;\n border-color: red;\n}\n\n#stop-recording:hover{\n background-color: red;\n color: white;\n}\n\n#response-message{\n position: fixed;\n top: 75%;\n left: 0;\n width: 100%;\n padding: 1em;\n color: white;\n text-align: center;\n font-weight: 800;\n box-shadow: 0 1px 1px black;\n text-shadow: 0 1px 1px black;\n box-sizing: border-box;\n display: none;\n}\n\n#response-message[data-success=\"true\"]{\n background-color: green;\n}\n\n#response-message[data-success=\"false\"]{\n background-color: red;\n}","output":"str","x":275.5,"y":585,"wires":[["e213f1c7.0121"]]},{"id":"e213f1c7.0121","type":"template","z":"59091b6d.99c4b4","name":"JavaScript","field":"payload.javascript","fieldType":"msg","format":"javascript","syntax":"mustache","template":"(function(){\n \n 'use strict';\n \n console.log('Hello!');\n \n const formStatus = document.querySelector('#response-message');\n const video = document.querySelector('#camera-capture');\n \n const startRecordingBtn = document.querySelector('#start-recording');\n const stopRecordingBtn = document.querySelector('#stop-recording');\n \n const constraints = {\n video : true,\n audio : false\n };\n \n navigator.mediaDevices.getUserMedia(constraints)\n .then(function(stream) {\n console.log(stream);\n\n let hideResult;\n \n video.addEventListener('canplay', function(){\n this.play(); \n });\n\n const vidURL = window.URL.createObjectURL(stream);\n video.src = vidURL;\n \n let streamRecorder\n const capturedChunks = []\n\n startRecordingBtn.addEventListener('click', function(){\n document.body.dataset.recording = \"true\";\n streamRecorder = new MediaRecorder(stream);\n \n streamRecorder.ondataavailable = function(e){\n console.log(e);\n capturedChunks.push(e.data);\n console.log(capturedChunks);\n };\n \n streamRecorder.start(100);\n }, false);\n\n stopRecordingBtn.addEventListener('click', function(){\n document.body.dataset.recording = \"false\";\n streamRecorder.stop();\n const mediaFile = new Blob(capturedChunks, { 'type' : 'video/webm; codecs=vp8' } );\n\n const form = new FormData();\n form.append('video', mediaFile, Date.now() / 1000 | 0 );\n\n capturedChunks.length = 0\n\n fetch('/receive', {\n method : 'post',\n body : form\n })\n .then(function(res){\n if(res.ok){\n formStatus.textContent = 'Media saved!';\n formStatus.dataset.success = \"true\";\n formStatus.style.display = \"block\";\n\n clearTimeout(hideResult);\n\n hideResult = setTimeout(function(){\n formStatus.style.display = \"none\";\n }, 3000);\n\n return res.text();\n } else {\n throw res;\n }\n })\n .then(function(response){\n console.log('response');\n })\n .catch(function(err){\n console.log('Fetch err:', err);\n formStatus.textContent = 'Could not save video';\n formStatus.dataset.success = \"true\";\n formStatus.style.display = \"block\";\n\n clearTimeout(hideResult);\n\n hideResult = setTimeout(function(){\n formStatus.style.display = \"none\";\n }, 3000);\n\n })\n ;\n\n }, false);\n \n })\n .catch(function(err) {\n console.log('gUM Error:', err);\n })\n ;\n \n})();","output":"str","x":430,"y":585,"wires":[["13e43fab.a05df"]]},{"id":"13e43fab.a05df","type":"template","z":"59091b6d.99c4b4","name":"HTML","field":"payload","fieldType":"msg","format":"handlebars","syntax":"mustache","template":"<html>\n <head>\n <style>\n {{{payload.css}}}\n </style>\n </head>\n <body data-recording=\"false\">\n <h1>Smile!</h1>\n <p>A demo app for sending a captured video stream to a Node-red server (with Fetch, FormData, and Multipart)</p>\n \n <video id=\"camera-capture\"></video>\n \n <div class=\"btnContainer\">\n <button id=\"start-recording\">Start recording</button>\n <button id=\"stop-recording\">Stop recording</button>\n </div>\n \n <div id=\"response-message\"></div>\n \n <script>{{{payload.javascript}}}</script>\n </body>\n \n</html>","output":"str","x":575,"y":585,"wires":[["e63b0bd5.e11038"]]},{"id":"f0f8a67d.6b89b8","type":"comment","z":"59091b6d.99c4b4","name":"Load the demo video/audio app","info":"","x":144.5,"y":540,"wires":[]},{"id":"1c63ba83.5bc8f5","type":"file-buffer","z":"59091b6d.99c4b4","name":"","mode":"asBuffer","x":390,"y":150,"wires":[["ec03acee.0d1f1","438957fc.9565d8"]]},{"id":"ec03acee.0d1f1","type":"debug","z":"59091b6d.99c4b4","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","x":550,"y":175,"wires":[]},{"id":"438957fc.9565d8","type":"visual-recognition-v3","z":"59091b6d.99c4b4","name":"","apikey":"__PWRD__","image-feature":"detectFaces","lang":"en","x":570,"y":135,"wires":[["4cddd730.039f18","17c93225.a2270e"]]},{"id":"4cddd730.039f18","type":"debug","z":"59091b6d.99c4b4","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"true","x":740,"y":120,"wires":[]},{"id":"a67a9f44.536ef","type":"camera","z":"59091b6d.99c4b4","name":"","x":400,"y":105,"wires":[["438957fc.9565d8"]]},{"id":"eccd9f01.0c201","type":"change","z":"59091b6d.99c4b4","name":"","rules":[{"t":"set","p":"payload","pt":"msg","to":"req.files.image[0].path","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":230,"y":150,"wires":[["1c63ba83.5bc8f5"]]},{"id":"1dde9bef.abbfe4","type":"http response","z":"59091b6d.99c4b4","name":"","statusCode":"","headers":{},"x":1070,"y":255,"wires":[]},{"id":"17c93225.a2270e","type":"switch","z":"59091b6d.99c4b4","name":"Was HTTP Request?","property":"res","propertyType":"msg","rules":[{"t":"nnull"}],"checkall":"true","repair":false,"outputs":1,"x":790,"y":165,"wires":[["ad97cdcd.7557a"]]},{"id":"ad97cdcd.7557a","type":"change","z":"59091b6d.99c4b4","name":"","rules":[{"t":"set","p":"payload","pt":"msg","to":"result","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":965,"y":210,"wires":[["1dde9bef.abbfe4"]]},{"id":"773defba.7d4de","type":"httpInMultipart","z":"59091b6d.99c4b4","name":"/receive","url":"/receive","method":"post","fields":"[ { \"name\" : \"image\"}, { \"name\" : \"video\" }, { \"name\" : \"audio\" } ]","swaggerDoc":"","x":65,"y":330,"wires":[["8f7f8764.2c5a68","614137d.22690c8"]]},{"id":"e75344a2.ad1aa8","type":"comment","z":"59091b6d.99c4b4","name":"Only receive a file","info":"","x":95,"y":270,"wires":[]},{"id":"8f7f8764.2c5a68","type":"http response","z":"59091b6d.99c4b4","name":"","statusCode":"","headers":{},"x":201,"y":310,"wires":[]},{"id":"614137d.22690c8","type":"debug","z":"59091b6d.99c4b4","name":"","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"true","x":200,"y":345,"wires":[]}]
