[feature] Added organization filter to timeseries charts #532

- Added option to define "summary_query" for metrics
- Added functionality to handle "GROUP by tags" clause to InfluxDB client
- Added support to override metrics added  with register_metric method
- Added setting to set the DEFAULT_CHART_TIME
- Added option to specify trace labels

Related to #532

README.rst

@@ -2088,6 +2088,19 @@ In case you just want to change the colors used in a chart here's how to do it:
         }
     }
 
+``OPENWISP_MONITORING_DEFAULT_CHART_TIME``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
++---------------------+---------------------------------------------+
+| **type**:           | ``str``                                     |
++---------------------+---------------------------------------------+
+| **default**:        | ``7d``                                      |
++---------------------+---------------------------------------------+
+| **possible values** | ``1d``, ``3d``, ``7d``, ``30d`` or ``365d`` |
++---------------------+---------------------------------------------+
+
+Allows to set the default time period of the time series charts.
+
 ``OPENWISP_MONITORING_AUTO_CLEAR_MANAGEMENT_IP``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

openwisp_monitoring/db/backends/influxdb/client.py

@@ -256,7 +256,24 @@ def read(self, key, fields, tags, **kwargs):
         return list(self.query(q, precision='s').get_points())
 
     def get_list_query(self, query, precision='s'):
-        return list(self.query(query, precision=precision).get_points())
+        result = self.query(query, precision=precision)
+        if not len(result.keys()) or result.keys()[0][1] is None:
+            return list(result.get_points())
+        # Handles query which contains "GROUP BY TAG" clause
+        result_points = {}
+        for (measurement, tag), group_points in result.items():
+            tag_suffix = '_'.join(tag.values())
+            for point in group_points:
+                values = {}
+                for key, value in point.items():
+                    if key != 'time':
+                        values[tag_suffix] = value
+                values['time'] = point['time']
+                try:
+                    result_points[values['time']].update(values)
+                except KeyError:
+                    result_points[values['time']] = values
+        return list(result_points.values())
 
     @retry
     def get_list_retention_policies(self):
@@ -329,7 +346,12 @@ def get_query(
             query = f'{query} LIMIT 1'
         return f"{query} tz('{timezone}')"
 
-    _group_by_regex = re.compile(r'GROUP BY time\(\w+\)', flags=re.IGNORECASE)
+    _group_by_time_tag_regex = re.compile(
+        r'GROUP BY ((time\(\w+\))(?:,\s+\w+)?)', flags=re.IGNORECASE
+    )
+    _group_by_time_regex = re.compile(r'GROUP BY time\(\w+\)\s?', flags=re.IGNORECASE)
+    _time_regex = re.compile(r'time\(\w+\)\s?', flags=re.IGNORECASE)
+    _time_comma_regex = re.compile(r'time\(\w+\),\s?', flags=re.IGNORECASE)
 
     def _group_by(self, query, time, chart_type, group_map, strip=False):
         if not self.validate_query(query):
@@ -343,7 +365,27 @@ def _group_by(self, query, time, chart_type, group_map, strip=False):
         if 'GROUP BY' not in query.upper():
             query = f'{query} {group_by}'
         else:
-            query = re.sub(self._group_by_regex, group_by, query)
+            # The query could have GROUP BY clause for a TAG
+            if group_by:
+                # The query already contains "GROUP BY", therefore
+                # we remove it from the "group_by" to avoid duplicating
+                # "GROUP BY"
+                group_by = group_by.replace('GROUP BY ', '')
+                # We only need to substitute the time function.
+                # The resulting query would be "GROUP BY time(<group_by>), <tag>"
+                query = re.sub(self._time_regex, group_by, query)
+            else:
+                # The query should not include the "GROUP by time()"
+                matches = re.search(self._group_by_time_tag_regex, query)
+                group_by_fields = matches.group(1)
+                if len(group_by_fields.split(',')) > 1:
+                    # If the query has "GROUP BY time(), tag",
+                    # then return "GROUP BY tag"
+                    query = re.sub(self._time_comma_regex, '', query)
+                else:
+                    # If the query has only has "GROUP BY time()",
+                    # then remove the "GROUP BY" clause
+                    query = re.sub(self._group_by_time_regex, '', query)
         return query
 
     _fields_regex = re.compile(

openwisp_monitoring/db/backends/influxdb/tests.py

@@ -190,6 +190,48 @@ def test_get_query_30d(self):
         self.assertIn(str(last30d)[0:10], q)
         self.assertIn('group by time(24h)', q.lower())
 
+    def test_group_by_tags(self):
+        self.assertEqual(
+            timeseries_db._group_by(
+                'SELECT COUNT(item) FROM measurement GROUP BY time(1d)',
+                time='30d',
+                chart_type='stackedbar+lines',
+                group_map={'30d': '30d'},
+                strip=False,
+            ),
+            'SELECT COUNT(item) FROM measurement GROUP BY time(30d)',
+        )
+        self.assertEqual(
+            timeseries_db._group_by(
+                'SELECT COUNT(item) FROM measurement GROUP BY time(1d)',
+                time='30d',
+                chart_type='stackedbar+lines',
+                group_map={'30d': '30d'},
+                strip=True,
+            ),
+            'SELECT COUNT(item) FROM measurement ',
+        )
+        self.assertEqual(
+            timeseries_db._group_by(
+                'SELECT COUNT(item) FROM measurement GROUP BY time(1d), tag',
+                time='30d',
+                chart_type='stackedbar+lines',
+                group_map={'30d': '30d'},
+                strip=False,
+            ),
+            'SELECT COUNT(item) FROM measurement GROUP BY time(30d), tag',
+        )
+        self.assertEqual(
+            timeseries_db._group_by(
+                'SELECT COUNT(item) FROM measurement GROUP BY time(1d), tag',
+                time='30d',
+                chart_type='stackedbar+lines',
+                group_map={'30d': '30d'},
+                strip=True,
+            ),
+            'SELECT COUNT(item) FROM measurement GROUP BY tag',
+        )
+
     def test_retention_policy(self):
         manage_short_retention_policy()
         manage_default_retention_policy()

openwisp_monitoring/device/apps.py

@@ -415,8 +415,11 @@ def register_dashboard_items(self):
                     'monitoring/css/percircle.min.css',
                     'monitoring/css/chart.css',
                     'monitoring/css/dashboard-chart.css',
+                    'admin/css/vendor/select2/select2.min.css',
+                    'admin/css/autocomplete.css',
                 ),
                 'js': (
+                    'admin/js/vendor/select2/select2.full.min.js',
                     'monitoring/js/lib/moment.min.js',
                     'monitoring/js/lib/daterangepicker.min.js',
                     'monitoring/js/lib/percircle.min.js',

openwisp_monitoring/device/static/monitoring/css/wifi-sessions.css

@@ -1,10 +1,6 @@
 td.original p {
   display: none;
 }
-#inline-wifisession-quick-link-container {
-  text-align: center;
-  margin: 20px 0 40px;
-}
 #wifisession_set-group {
   margin-bottom: 0px;
 }

openwisp_monitoring/device/static/monitoring/js/wifi-session-inline.js

@@ -27,7 +27,7 @@ if (typeof gettext === 'undefined') {
             wifiSessionLinkElement;
         wifiSessionUrl = `${wifiSessionUrl}?device=${getObjectIdFromUrl()}`;
         wifiSessionLinkElement = `
-            <div id="inline-wifisession-quick-link-container">
+            <div class="inline-quick-link-container">
                 <a href="${wifiSessionUrl}"
                     class="button"
                     id="inline-wifisession-quick-link"

openwisp_monitoring/monitoring/api/views.py

@@ -166,5 +166,33 @@ def invalidate_cache(cls, instance, *args, **kwargs):
                 return
         cls._get_charts.invalidate()
 
+    def _get_user_managed_orgs(self, request):
+        """
+        Return list of dictionary containing organization name and slug
+        in select2 compatible format.
+        """
+        orgs = []
+        qs = Organization.objects.only('slug', 'name')
+        if not request.user.is_superuser:
+            if len(request.user.organizations_managed) > 1:
+                qs = qs.filter(pk__in=request.user.organizations_managed)
+            else:
+                return orgs
+        for org in qs.iterator():
+            orgs.append({'id': org.slug, 'text': org.name})
+        if len(orgs) < 2:
+            # Handles scenarios for superuser when the project has only
+            # one organization.
+            return []
+        return orgs
+
+    def get(self, request, *args, **kwargs):
+        response = super().get(request, *args, **kwargs)
+        if not request.GET.get('csv'):
+            user_managed_orgs = self._get_user_managed_orgs(request)
+            if user_managed_orgs:
+                response.data['organizations'] = user_managed_orgs
+        return response
+
 
 dashboard_timeseries = DashboardTimeseriesView.as_view()

openwisp_monitoring/monitoring/base/models.py

@@ -25,7 +25,7 @@
 from openwisp_utils.base import TimeStampedEditableModel
 
 from ...db import default_chart_query, timeseries_db
-from ...settings import CACHE_TIMEOUT
+from ...settings import CACHE_TIMEOUT, DEFAULT_CHART_TIME
 from ..configuration import (
     CHART_CONFIGURATION_CHOICES,
     DEFAULT_COLORS,
@@ -495,7 +495,7 @@ class AbstractChart(TimeStampedEditableModel):
         max_length=16, null=True, choices=CHART_CONFIGURATION_CHOICES
     )
     GROUP_MAP = {'1d': '10m', '3d': '20m', '7d': '1h', '30d': '24h', '365d': '7d'}
-    DEFAULT_TIME = '7d'
+    DEFAULT_TIME = DEFAULT_CHART_TIME
 
     class Meta:
         abstract = True
@@ -552,6 +552,10 @@ def trace_type(self):
     def trace_order(self):
         return self.config_dict.get('trace_order', [])
 
+    @property
+    def trace_labels(self):
+        return self.config_dict.get('trace_labels', {})
+
     @property
     def calculate_total(self):
         return self.config_dict.get('calculate_total', False)
@@ -601,6 +605,12 @@ def query(self):
             return query[timeseries_db.backend_name]
         return self._default_query
 
+    @property
+    def summary_query(self):
+        query = self.config_dict.get('summary_query', None)
+        if query:
+            return query[timeseries_db.backend_name]
+
     @property
     def top_fields(self):
         return self.config_dict.get('top_fields', None)
@@ -659,10 +669,14 @@ def get_query(
         additional_params=None,
     ):
         query = query or self.query
+        if summary and self.summary_query:
+            query = self.summary_query
         additional_params = additional_params or {}
         params = self._get_query_params(time, start_date, end_date)
         params.update(additional_params)
         params.update({'start_date': start_date, 'end_date': end_date})
+        if not params.get('organization_id') and self.config_dict.get('__all__', False):
+            params['organization_id'] = ['__all__']
         return timeseries_db.get_query(
             self.type,
             params,

openwisp_monitoring/monitoring/configuration.py

@@ -1,4 +1,5 @@
 from collections import OrderedDict
+from copy import deepcopy
 
 from django.core.exceptions import ImproperlyConfigured
 from django.utils.translation import gettext_lazy as _
@@ -713,7 +714,19 @@ def unregister_metric_notifications(metric_name):
 
 
 def get_metric_configuration():
-    metrics = deep_merge_dicts(DEFAULT_METRICS, app_settings.ADDITIONAL_METRICS)
+    additional_metrics = deepcopy(app_settings.ADDITIONAL_METRICS)
+    for metric_name in list(additional_metrics.keys()):
+        if additional_metrics[metric_name].get('partial', False):
+            # A partial configuration can be defined in the settings.py
+            # with OPENWISP_MONITORING_METRICS setting to override
+            # metrics that are added with register_metric method in
+            # other django apps.
+            # Since, the partial configuration could be defined to
+            # override limited fields in the configuration, hence
+            # we don't validate the configuration here. Instead.
+            # configuration is validated in the register_metric method.
+            del additional_metrics[metric_name]
+    metrics = deep_merge_dicts(DEFAULT_METRICS, additional_metrics)
     # ensure configuration is not broken
     for metric_config in metrics.values():
         _validate_metric_configuration(metric_config)
@@ -741,6 +754,17 @@ def register_metric(metric_name, metric_config):
         raise ImproperlyConfigured(
             f'{metric_name} is an already registered Metric Configuration.'
         )
+    if metric_name in app_settings.ADDITIONAL_METRICS:
+        # There is partial configuration present for this "metric_name" in
+        # ADDITIONAL_METRICS. We need to merge the partial configuration with
+        # the registered metric before validating. Otherwise, users won't be
+        # able to override registered metrics using OPENWISP_MONITORING_METRICS
+        # setting.
+        metric_config = deep_merge_dicts(
+            metric_config,
+            app_settings.ADDITIONAL_METRICS[metric_name],
+        )
+        metric_config.pop('partial', None)
     _validate_metric_configuration(metric_config)
     for chart in metric_config.get('charts', {}).values():
         _validate_chart_configuration(chart_config=chart)

openwisp_monitoring/monitoring/static/monitoring/css/chart.css

@@ -1,40 +1,38 @@
 #ow-chart-fallback {
   text-align: center;
 }
-#ow-chart-time {
-  margin-top: 15px;
-  text-align: center;
-  padding: 4px;
-}
-#ow-chart-time a {
-  display: inline-block;
-  margin: 0 2px;
-  color: #fff;
-  background-color: #333;
-  padding: 5px 15px;
-  text-decoration: none !important;
-  border-radius: 3px;
-  cursor: pointer;
-}
-#ow-chart-time a:hover {
-  background-color: #777 !important;
-}
-#ow-chart-time a.active {
-  background: transparent;
-  color: #666;
-  cursor: default;
-  border: 1px solid #666;
-}
-#ow-chart-time a.export {
-  float: right;
-  background: #333;
+#ow-chart-utils {
+  display: flex;
+  justify-content: space-between;
+}
+#ow-chart-utils .select2-container {
+  min-width: 150px !important;
+  width: unset !important;
+}
+#ow-chart-utils span.select2-selection {
+  min-height: 27px;
+  min-width: 200px;
+  height: 27px;
+}
+#ow-chart-utils .select2-selection__rendered {
+  line-height: 27px;
+  height: 27px;
+}
+#ow-chart-utils .select2-selection__arrow {
+  top: 0;
+  height: 27px;
+}
+#ow-chart-utils>span {
+  display: flex;
+  justify-content: space-between;
+  width: 100%;
 }
-#ow-chart-time a#daterangepicker-widget {
-  float: left;
-  margin-right: 100px;
+#ow-chart-utils .button {
+  font-size: 14px;
+  padding: 5px 10px;
 }
-#ow-chart-time a.export:hover {
-  background: #777;
+#ow-chart-export {
+  margin-left: 10px;
 }
 #ow-chart-fallback {
   display: none;
@@ -147,3 +145,12 @@
   display: none;
   font-size: 1.2em;
 }
+@media (max-width: 768px) {
+  #ow-chart-utils {
+    display: block;
+  }
+  #ow-chart-utils > span + span {
+    margin: 5px 0px;
+    justify-content: end;
+  }
+}