Dynamic Word Cloud from IBM Connections Search Data

Forked from dceejay's twitter version. Converted to use IBM Connections instead of Twitter. "Intelligent" stop list added. Changed to use the on-line D3 source as opposed to a local copy (needs v3 to work. v4 does not work with this code).

This flow generatates a dynamic Word Cloud type graphic. It takes the output of an IBM Connections Search node as input, and uses d3 to generate a Word Cloud based on the relative frequency of words found.

It includes a stop list to exclude common words and exclude the search term and words in the search term your are searching on.

As well as the flow you also need to download and save this web page

https://gist.githubusercontent.com/dceejay/1d9b0959cf7cc36f9f8c/raw/3bb9263537d996031d2337fd64582a323a5e9168/cloud.html

and the d3 word cloud library from here

https://github.com/jasondavies/d3-cloud

You just need the d3.layout.cloud.js (and the license :-)

Place these 2 (or 3) files into the node-red/public directory then fire up the flow below. Then browse to http://localhost:1880/cloud.html

Copy[{"id":"d7ff7cf5.975e48","type":"function","z":"16feff23.748181","name":"Count words in array","func":"var index = {};\n\n//var toIgnore = [ \"and\", \"for\", \"but\", \"the\", \"&lt\", \"&gt\", \"his\", \"her\", \"pre\", \"are\", \"&amp\", \"with\"];\n\nvar toIgnore = flow.get('stopWords')||\"\";\n\n\nconsole.log(toIgnore);\n\nfunction countWords(sentence) {\n\twords = sentence\n\t\t.replace(/[.,?!;()\"'-]/g, \" \")\n\t\t.replace(/\\s+/g, \" \")\n\t\t//.toLowerCase()\n\t\t.split(\" \");\n\t\n\twords.forEach(function (word) {\n\t\tif ((word.length > 2)&&( isNaN(Number(word)) )&&( toIgnore.indexOf(word.toLowerCase()) == -1 )) {\n\t\t    if (!(index.hasOwnProperty(word))) {\n\t\t        index[word] = 0;\n\t\t    }\n\t\t    index[word]++;\n\t\t}\n\t});\n\t\n\n}\n\nfor (var i in msg.payload) {\n\tcountWords(msg.payload[i]);\n}\n\nmsg.payload = JSON.stringify(index);\nmsg.stopWords = toIgnore;\nreturn msg;","outputs":1,"noerr":0,"x":1476.0000343322754,"y":699.666576385498,"wires":[["4c875e90.27911","a5cd3214.ab9688","ad7f768f.758cc"]]},{"id":"4c875e90.27911","type":"websocket out","z":"16feff23.748181","name":"","server":"acf0a873.d12ed","x":1774.999912261963,"y":698.3332281112671,"wires":[]},{"id":"c375b720.e42f8","type":"inject","z":"16feff23.748181","name":"","topic":"","payload":"\"tim minter\"","payloadType":"str","repeat":"","crontab":"","once":false,"x":246.66668701171875,"y":378.3332796096802,"wires":[["1df07a5c.61e33e","694379cd.423428"]]},{"id":"1df07a5c.61e33e","type":"function","z":"16feff23.748181","name":"Translate msg.payload to msg.query","func":"msg.query = msg.payload;\nreturn msg;","outputs":1,"noerr":0,"x":338.50001525878906,"y":456.99993228912354,"wires":[["8a924470.987ef"]]},{"id":"8a924470.987ef","type":"SimpleSearch","z":"16feff23.748181","name":"Search All Status Updates","server":"","query":"","theScope":"status_updates","sortKey":"date","sortOrder":"desc","maxResults":"50","limitCB":true,"sinceDate":"07/08/2016","sinceCB":false,"untilDate":"06/04/2017","untilCB":false,"mytags":"","x":648.5,"y":610.3332328796387,"wires":[["818ece04.a37978"]]},{"id":"818ece04.a37978","type":"splitter","z":"16feff23.748181","name":"","property":"payload","x":859.2500228881836,"y":610.4165840148926,"wires":[["1848d520.bc02fb"]]},{"id":"1848d520.bc02fb","type":"function","z":"16feff23.748181","name":"Get Title","func":"msg.payload = msg.payload.title;\nreturn msg;","outputs":1,"noerr":0,"x":869.0000343322754,"y":699.9166975021362,"wires":[["5f57099c.6f5e3"]]},{"id":"a5cd3214.ab9688","type":"debug","z":"16feff23.748181","name":"count","active":true,"console":"false","complete":"payload","x":1735.8332862854004,"y":766.6665849685669,"wires":[]},{"id":"3710da36.e222ce","type":"function","z":"16feff23.748181","name":"Build Array","func":"\nvar entries = context.entries || [\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\"];\n\n\tvar foo = msg.payload;\n\tif (foo.indexOf(\"http\") != -1) {\n\t\tfoo = (foo.split(\"http\"))[0];\n\t}\n\t\n\tif (foo.toLowerCase().indexOf(\"job\") != -1) return null;\n\tfoo = foo.replace(/[^\\x00-\\x7F]/g, \"\");\n\t\n\tif (entries.indexOf(foo) == -1) {\n\t\tentries.shift();\n\t\tentries.push(foo);\n\t\tcontext.entries = entries;\n\t\treturn {payload:entries};\n\t}\n\t\nreturn null;\n","outputs":1,"noerr":0,"x":1236.6665229797363,"y":699.999927520752,"wires":[["d7ff7cf5.975e48"]]},{"id":"5f57099c.6f5e3","type":"badwords","z":"16feff23.748181","name":"","x":1044.1666641235352,"y":699.9999465942383,"wires":[["3710da36.e222ce"]]},{"id":"694379cd.423428","type":"function","z":"16feff23.748181","name":"stopWords (with search term added)","func":"\nvar stopWords = [\n\"a\",\n\"about\",\n\"above\",\n\"after\",\n\"again\",\n\"against\",\n\"all\",\n\"am\",\n\"an\",\n\"and\",\n\"any\",\n\"are\",\n\"aren't\",\n\"as\",\n\"at\",\n\"be\",\n\"because\",\n\"been\",\n\"before\",\n\"being\",\n\"below\",\n\"between\",\n\"both\",\n\"but\",\n\"by\",\n\"can't\",\n\"cannot\",\n\"could\",\n\"couldn't\",\n\"did\",\n\"didn't\",\n\"do\",\n\"does\",\n\"doesn't\",\n\"doing\",\n\"don't\",\n\"down\",\n\"during\",\n\"each\",\n\"few\",\n\"for\",\n\"from\",\n\"further\",\n\"had\",\n\"hadn't\",\n\"has\",\n\"hasn't\",\n\"have\",\n\"haven't\",\n\"having\",\n\"he\",\n\"he'd\",\n\"he'll\",\n\"he's\",\n\"her\",\n\"here\",\n\"here's\",\n\"hers\",\n\"herself\",\n\"him\",\n\"himself\",\n\"his\",\n\"how\",\n\"how's\",\n\"i\",\n\"i'd\",\n\"i'll\",\n\"i'm\",\n\"i've\",\n\"if\",\n\"in\",\n\"into\",\n\"is\",\n\"isn't\",\n\"it\",\n\"it's\",\n\"its\",\n\"itself\",\n\"let's\",\n\"me\",\n\"more\",\n\"most\",\n\"mustn't\",\n\"my\",\n\"myself\",\n\"no\",\n\"nor\",\n\"not\",\n\"of\",\n\"off\",\n\"on\",\n\"once\",\n\"only\",\n\"or\",\n\"other\",\n\"ought\",\n\"our\",\n\"ours\",\n\"ourselves\",\n\"out\",\n\"over\",\n\"own\",\n\"same\",\n\"shan't\",\n\"she\",\n\"she'd\",\n\"she'll\",\n\"she's\",\n\"should\",\n\"shouldn't\",\n\"so\",\n\"some\",\n\"such\",\n\"than\",\n\"that\",\n\"that's\",\n\"the\",\n\"their\",\n\"theirs\",\n\"them\",\n\"themselves\",\n\"then\",\n\"there\",\n\"there's\",\n\"these\",\n\"they\",\n\"they'd\",\n\"they'll\",\n\"they're\",\n\"they've\",\n\"this\",\n\"those\",\n\"through\",\n\"to\",\n\"too\",\n\"under\",\n\"until\",\n\"up\",\n\"very\",\n\"was\",\n\"wasn't\",\n\"we\",\n\"we'd\",\n\"we'll\",\n\"we're\",\n\"we've\",\n\"were\",\n\"weren't\",\n\"what\",\n\"what's\",\n\"when\",\n\"when's\",\n\"where\",\n\"where's\",\n\"which\",\n\"while\",\n\"who\",\n\"who's\",\n\"whom\",\n\"why\",\n\"why's\",\n\"with\",\n\"won't\",\n\"would\",\n\"wouldn't\",\n\"you\",\n\"you'd\",\n\"you'll\",\n\"you're\",\n\"you've\",\n\"your\",\n\"yours\",\n\"yourself\",\n\"yourselves\"];\n\n\nvar searchTerm = msg.payload;\nvar cleanedSearchTerm = searchTerm.replace(/[\"']/g, \"\");\nstopWords.push(cleanedSearchTerm);\n\nvar splitSearchTerm = cleanedSearchTerm.split(\" \");\nsplitSearchTerm.forEach(function(item, index, array) {\n  stopWords.push(item);\n});\n\nconsole.log(stopWords);\nflow.set('stopWords',stopWords);\n    ","outputs":1,"noerr":0,"x":694.1666641235352,"y":378.33331871032715,"wires":[[]]},{"id":"ad7f768f.758cc","type":"debug","z":"16feff23.748181","name":"","active":true,"console":"false","complete":"stopWords","x":1764.166648864746,"y":626.6666507720947,"wires":[]},{"id":"6e00a3b7.210b2c","type":"comment","z":"16feff23.748181","name":"The search term and the individual words in the search term are added to the stop list","info":"","x":840.8333587646484,"y":333.33333587646484,"wires":[]},{"id":"1a3334cc.5d3773","type":"comment","z":"16feff23.748181","name":"If nothing is found, the error \"Missing <ENTRY> element\" is returned","info":"","x":772.5000152587891,"y":558.3333530426025,"wires":[]},{"id":"5b680203.e8d60c","type":"comment","z":"16feff23.748181","name":"Edit the stop list here","info":"","x":640.8333206176758,"y":288.3333339691162,"wires":[]},{"id":"acf0a873.d12ed","type":"websocket-listener","path":"/ws/wordcloud","wholemsg":"false"}]