Merge pull request #1986 from colinl/1977_categorical_xaxis_remove_old_data

Improve historical chart data management

Author: Steve-Mcl

nodes/store/data.js

@@ -102,6 +102,22 @@ const setters = {
             }
             data[node.id].push(config.RED.util.cloneMessage(msg))
         }
+    },
+    /**
+     * Fast filtering of existing array data (skips cloning and save checks for fast data cleanup)
+     * @param {*} base - the base node
+     * @param {*} node - the owner node
+     * @param {(msg) => Boolean} filterFunction
+     */
+    filter (base, node, filterFunction) {
+        const currentData = data[node.id]
+        if (filterFunction && Array.isArray(currentData) && currentData.length) {
+            const filteredMessages = currentData.filter(filterFunction)
+            if (filteredMessages.length !== currentData.length) {
+                // no need for save operation to process messages - just apply them
+                data[node.id] = filteredMessages
+            }
+        }
     }
 }
 
@@ -111,5 +127,6 @@ module.exports = {
     setConfig: setters.setConfig,
     save: setters.save,
     append: setters.append,
+    filter: setters.filter,
     clear: setters.clear
 }

nodes/widgets/ui_chart.js

@@ -61,19 +61,36 @@ module.exports = function (RED) {
             if (removeOlder > 0) {
                 const removeOlderUnit = parseFloat(config.removeOlderUnit)
                 const ago = (removeOlder * removeOlderUnit) * 1000 // milliseconds ago
-                const cutoff = (new Date()).getTime() - ago
-                const _msg = datastore.get(node.id).filter((msg) => {
+                const cutOff = (new Date()).getTime() - ago
+                const filterFn = (msg) => {
                     let timestamp = msg._datapoint.x
                     // is x already a millisecond timestamp?
                     if (typeof (msg._datapoint.x) === 'string') {
                         timestamp = (new Date(msg._datapoint.x)).getTime()
                     }
-                    return timestamp > cutoff
-                })
-                datastore.save(base, node, _msg)
+                    return timestamp > cutOff
+                }
+                datastore.filter(base, node, filterFn)
             }
         }
 
+        /**
+         * For categorical xaxis and types other than histogram then only keep the latest data point for
+         * each category in each series
+         */
+        function clearOldCategoricalPoints () {
+            const points = datastore.get(node.id)
+            const latestSet = {}
+            for (const item of points) {
+                const { category, x } = item._datapoint
+                const key = JSON.stringify([category, x]) // a unique key for each category/series combination
+                latestSet[key] = item
+            }
+
+            const filtered = Object.values(latestSet)
+            datastore.save(base, node, filtered)
+        }
+
         // ensure sane defaults
         if (!['msg', 'str', 'property', 'timestamp'].includes(config.xAxisPropertyType)) {
             config.xAxisPropertyType = 'timestamp' // default to 'timestamp'
@@ -269,6 +286,10 @@ module.exports = function (RED) {
                         if (config.xAxisType === 'time' && config.removeOlder && config.removeOlderUnit) {
                             // remove any points older than the specified time
                             clearOldPoints()
+                        } else if (config.xAxisType === 'category' && config.chartType !== 'histogram') {
+                            // for categorical xaxis and types other than histogram then only keep the latest data point for
+                            // each category in each series
+                            clearOldCategoricalPoints()
                         }
                     }
                 }

ui/src/widgets/ui-chart/UIChart.vue

@@ -550,7 +550,9 @@ export default {
                     console.log('have no payload')
                 }
             }
-            if (this.chartType === 'line' || this.chartType === 'area' || this.chartType === 'scatter') {
+            if (this.xAxisType === 'category' && this.props.chartType !== 'histogram') {
+                this.clearOldCategoricalPoints(options)
+            } else if (this.chartType === 'line' || this.chartType === 'area' || this.chartType === 'scatter') {
                 this.limitDataSize(options)
             }
             this.updateChart(options)
@@ -770,7 +772,6 @@ export default {
                 // remove older points
                 points = parseInt(this.props.removeOlderPoints)
             }
-
             // apply data limitations to the chart
             const series = options.series
             if ((cutoff || points) && series.length > 0) {
@@ -789,13 +790,24 @@ export default {
                     }
                 }
             }
-            // apply data limtations to the vuex store
+            // apply data limitations to the vuex store
             this.$store.commit('data/restrict', {
                 widgetId: this.id,
                 min: cutoff,
                 points
             })
         },
+
+        /**
+         * For categorical xaxis and types other than histogram then only keep the latest data point for
+         * each category in each series
+         * @param {Object} options - existing eChart options object
+         */
+        clearOldCategoricalPoints (options) {
+            // There is no need to remove old data from the chart itself as, for categorical xAxis,
+            // the chart only retains one value for each category
+        },
+
         calculateBins () {
             if (this.props.chartType !== 'histogram') {
                 return []