[feature] Added organization filter to timeseries charts #532

pandafy · pandafy · commit 8045d133e11b · 2023-11-13T06:59:48.000+05:30
- Added option to define "summary_query" for metrics - Added functionality to handle "GROUP by tags" clause to InfluxDB client Related to #532
diff --git a/openwisp_monitoring/db/backends/influxdb/client.py b/openwisp_monitoring/db/backends/influxdb/client.py
@@ -256,7 +256,24 @@ def read(self, key, fields, tags, **kwargs):
         return list(self.query(q, precision='s').get_points())
 
     def get_list_query(self, query, precision='s'):
-        return list(self.query(query, precision=precision).get_points())
+        result = self.query(query, precision=precision)
+        if not len(result.keys()) or result.keys()[0][1] is None:
+            return list(result.get_points())
+        # Handles query which contains "GROUP BY TAG" clause
+        result_points = {}
+        for (measurement, tag), group_points in result.items():
+            tag_suffix = '_'.join(tag.values())
+            for point in group_points:
+                values = {}
+                for key, value in point.items():
+                    if key != 'time':
+                        values[f'{tag_suffix}'] = value
+                values['time'] = point['time']
+                try:
+                    result_points[values['time']].update(values)
+                except KeyError:
+                    result_points[values['time']] = values
+        return list(result_points.values())
 
     @retry
     def get_list_retention_policies(self):
@@ -329,7 +346,12 @@ def get_query(
             query = f'{query} LIMIT 1'
         return f"{query} tz('{timezone}')"
 
-    _group_by_regex = re.compile(r'GROUP BY time\(\w+\)', flags=re.IGNORECASE)
+    _group_by_time_tag_regex = re.compile(
+        r'GROUP BY ((time\(\w+\))(?:,\s+\w+)?)', flags=re.IGNORECASE
+    )
+    _group_by_time_regex = re.compile(r'GROUP BY time\(\w+\)\s?', flags=re.IGNORECASE)
+    _time_regex = re.compile(r'time\(\w+\)\s?', flags=re.IGNORECASE)
+    _time_comma_regex = re.compile(r'time\(\w+\),\s?', flags=re.IGNORECASE)
 
     def _group_by(self, query, time, chart_type, group_map, strip=False):
         if not self.validate_query(query):
@@ -343,7 +365,27 @@ def _group_by(self, query, time, chart_type, group_map, strip=False):
         if 'GROUP BY' not in query.upper():
             query = f'{query} {group_by}'
         else:
-            query = re.sub(self._group_by_regex, group_by, query)
+            # The query could have GROUP BY clause for a TAG
+            if group_by:
+                # The query already contains "GROUP BY", therefore
+                # we remove it from the "group_by" to avoid duplicating
+                # "GROUP BY"
+                group_by = group_by.replace('GROUP BY ', '')
+                # We only need to substitute the time function.
+                # The resulting query would be "GROUP BY time(<group_by>), <tag>"
+                query = re.sub(self._time_regex, group_by, query)
+            else:
+                # The query should not include the "GROUP by time()"
+                matches = re.search(self._group_by_time_tag_regex, query)
+                group_by_fields = matches.group(1)
+                if len(group_by_fields.split(',')) > 1:
+                    # If the query has "GROUP BY time(), tag",
+                    # then return "GROUP BY tag"
+                    query = re.sub(self._time_comma_regex, '', query)
+                else:
+                    # If the query has only has "GROUP BY time()",
+                    # then remove the "GROUP BY" clause
+                    query = re.sub(self._group_by_time_regex, '', query)
         return query
 
     _fields_regex = re.compile(
diff --git a/openwisp_monitoring/db/backends/influxdb/tests.py b/openwisp_monitoring/db/backends/influxdb/tests.py
@@ -190,6 +190,48 @@ def test_get_query_30d(self):
         self.assertIn(str(last30d)[0:10], q)
         self.assertIn('group by time(24h)', q.lower())
 
+    def test_group_by_tags(self):
+        self.assertEqual(
+            timeseries_db._group_by(
+                'SELECT COUNT(item) FROM measurement GROUP BY time(1d)',
+                time='30d',
+                chart_type='stackedbar+lines',
+                group_map={'30d': '30d'},
+                strip=False,
+            ),
+            'SELECT COUNT(item) FROM measurement GROUP BY time(30d)',
+        )
+        self.assertEqual(
+            timeseries_db._group_by(
+                'SELECT COUNT(item) FROM measurement GROUP BY time(1d)',
+                time='30d',
+                chart_type='stackedbar+lines',
+                group_map={'30d': '30d'},
+                strip=True,
+            ),
+            'SELECT COUNT(item) FROM measurement ',
+        )
+        self.assertEqual(
+            timeseries_db._group_by(
+                'SELECT COUNT(item) FROM measurement GROUP BY time(1d), tag',
+                time='30d',
+                chart_type='stackedbar+lines',
+                group_map={'30d': '30d'},
+                strip=False,
+            ),
+            'SELECT COUNT(item) FROM measurement GROUP BY time(30d), tag',
+        )
+        self.assertEqual(
+            timeseries_db._group_by(
+                'SELECT COUNT(item) FROM measurement GROUP BY time(1d), tag',
+                time='30d',
+                chart_type='stackedbar+lines',
+                group_map={'30d': '30d'},
+                strip=True,
+            ),
+            'SELECT COUNT(item) FROM measurement GROUP BY tag',
+        )
+
     def test_retention_policy(self):
         manage_short_retention_policy()
         manage_default_retention_policy()
diff --git a/openwisp_monitoring/device/apps.py b/openwisp_monitoring/device/apps.py
@@ -402,8 +402,11 @@ def register_dashboard_items(self):
                     'monitoring/css/percircle.min.css',
                     'monitoring/css/chart.css',
                     'monitoring/css/dashboard-chart.css',
+                    'admin/css/vendor/select2/select2.min.css',
+                    'admin/css/autocomplete.css',
                 ),
                 'js': (
+                    'admin/js/vendor/select2/select2.full.min.js',
                     'monitoring/js/lib/moment.min.js',
                     'monitoring/js/lib/daterangepicker.min.js',
                     'monitoring/js/lib/percircle.min.js',
diff --git a/openwisp_monitoring/monitoring/api/views.py b/openwisp_monitoring/monitoring/api/views.py
@@ -166,5 +166,33 @@ def invalidate_cache(cls, instance, *args, **kwargs):
                 return
         cls._get_charts.invalidate()
 
+    def _get_user_managed_orgs(self, request):
+        """
+        Return list of dictionary containing organization name and slug
+        in select2 compatible format.
+        """
+        orgs = []
+        qs = Organization.objects.only('slug', 'name')
+        if not request.user.is_superuser:
+            if len(request.user.organizations_managed) > 1:
+                qs = qs.filter(pk__in=request.user.organizations_managed)
+            else:
+                return orgs
+        for org in qs.iterator():
+            orgs.append({'id': org.slug, 'text': org.name})
+        if len(orgs) < 2:
+            # Handles scenarios for superuser when the project has only
+            # one organization.
+            return []
+        return orgs
+
+    def get(self, request, *args, **kwargs):
+        response = super().get(request, *args, **kwargs)
+        if not request.GET.get('csv'):
+            user_managed_orgs = self._get_user_managed_orgs(request)
+            if user_managed_orgs:
+                response.data['organizations'] = user_managed_orgs
+        return response
+
 
 dashboard_timeseries = DashboardTimeseriesView.as_view()
diff --git a/openwisp_monitoring/monitoring/base/models.py b/openwisp_monitoring/monitoring/base/models.py
@@ -601,6 +601,12 @@ def query(self):
             return query[timeseries_db.backend_name]
         return self._default_query
 
+    @property
+    def summary_query(self):
+        query = self.config_dict.get('summary_query', None)
+        if query:
+            return query[timeseries_db.backend_name]
+
     @property
     def top_fields(self):
         return self.config_dict.get('top_fields', None)
@@ -659,6 +665,8 @@ def get_query(
         additional_params=None,
     ):
         query = query or self.query
+        if summary and self.summary_query:
+            query = self.summary_query
         additional_params = additional_params or {}
         params = self._get_query_params(time, start_date, end_date)
         params.update(additional_params)
diff --git a/openwisp_monitoring/monitoring/static/monitoring/js/chart-utils.js b/openwisp_monitoring/monitoring/static/monitoring/js/chart-utils.js
@@ -197,6 +197,10 @@ django.jQuery(function ($) {
             url = `${url}&timezone=${timezone}`;
           }
         }
+        if ($('#org-selector').val()) {
+          var orgSlug = $('#org-selector').val();
+          url = `${url}&organization_slug=${orgSlug}`;
+        }
         return url;
       },
       createCharts = function (data){
@@ -213,6 +217,22 @@ django.jQuery(function ($) {
           createChart(chart, data.x, htmlId, chart.title, chart.type, chartQuickLink);
         });
       },
+      addOrganizationSelector = function (data) {
+        var orgSelector = $('#org-selector');
+        if (data.organizations === undefined) {
+          orgSelector.hide();
+          return;
+        }
+        if (orgSelector.data('select2-id') === 'org-selector') {
+          return;
+        }
+        orgSelector.select2({
+          data: data.organizations,
+          allowClear: true,
+          placeholder: gettext('Organization Filter')
+        });
+        orgSelector.show();
+      },
       loadCharts = function (time, showLoading) {
         $.ajax(getChartFetchUrl(time), {
           dataType: 'json',
@@ -233,6 +253,7 @@ django.jQuery(function ($) {
               fallback.show();
             }
             createCharts(data);
+            addOrganizationSelector(data);
           },
           error: function () {
             alert('Something went wrong while loading the charts');
@@ -318,24 +339,30 @@ django.jQuery(function ($) {
     });
     // bind export button
     $('#ow-chart-time a.export').click(function () {
-      var time = localStorage.getItem(timeRangeKey);
-      location.href = baseUrl + time + '&csv=1';
+      var queryString,
+        queryParams = {'csv': 1};
+        queryParams.time = localStorage.getItem(timeRangeKey);
       // If custom or pickerChosenLabelKey is 'Custom Range', pass pickerEndDate and pickerStartDate to csv url
       if (localStorage.getItem(isCustomDateRange) === 'true' || localStorage.getItem(pickerChosenLabelKey) === customDateRangeLabel) {
-        var startDate = localStorage.getItem(startDateTimeKey);
-        var endDate = localStorage.getItem(endDateTimeKey);
+        queryParams.start = localStorage.getItem(startDateTimeKey);
+        queryParams.end = localStorage.getItem(endDateTimeKey);
         if (localStorage.getItem(isChartZoomed) === 'true') {
-          time = localStorage.getItem(zoomtimeRangeKey);
-          endDate = localStorage.getItem(zoomEndDateTimeKey);
-          startDate = localStorage.getItem(zoomStartDateTimeKey);
+          queryParams.time = localStorage.getItem(zoomtimeRangeKey);
+          queryParams.end = localStorage.getItem(zoomEndDateTimeKey);
+          queryParams.start = localStorage.getItem(zoomStartDateTimeKey);
         }
-        var url = `${apiUrl}?start=${startDate}&end=${endDate}&csv=1`;
-        var timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
+        timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
         if (timezone) {
-          url = `${url}&timezone=${timezone}`;
+          queryParams.timezone = timezone;
         }
-        location.href = url;
       }
+      if ($('#org-selector').val()) {
+        queryParams.organization_slug = $('#org-selector').val();
+      }
+      queryString = Object.keys(queryParams)
+        .map(key => `${encodeURIComponent(key)}=${encodeURIComponent(queryParams[key])}`)
+        .join('&');
+      location.href = `${apiUrl}?${queryString}`;
     });
     // fetch chart data and replace the old charts with the new ones
     function loadFetchedCharts(time){
@@ -352,5 +379,12 @@ django.jQuery(function ($) {
         },
       });
     }
+
+    $('#org-selector').change(function(){
+      loadCharts(
+        localStorage.getItem(timeRangeKey) || defaultTimeRange,
+        true
+      );
+    });
   });
 }(django.jQuery));
diff --git a/openwisp_monitoring/monitoring/templates/monitoring/paritals/chart.html b/openwisp_monitoring/monitoring/templates/monitoring/paritals/chart.html
@@ -12,6 +12,9 @@
     <a id="daterangepicker-widget">
         <span></span>
     </a>
+    <select name="org-selector" id="org-selector" class="hide">
+        <option></option>
+    </select>
     <div id="chart-loading-overlay"><div class="ow-loading-spinner"></div></div>
     <div id="ow-chart-contents"></div>
     <div id="ow-chart-fallback" class="form-row">
diff --git a/openwisp_monitoring/monitoring/tests/__init__.py b/openwisp_monitoring/monitoring/tests/__init__.py
@@ -136,6 +136,25 @@
             )
         },
     },
+    'group_by_tag': {
+        'type': 'stackedbars',
+        'title': 'Group by tag',
+        'description': 'Query is groupped by tag along with time',
+        'unit': 'n.',
+        'order': 999,
+        'query': {
+            'influxdb': (
+                "SELECT CUMULATIVE_SUM(SUM({field_name})) FROM {key} WHERE time >= '{time}'"
+                " GROUP BY time(1d), metric_num"
+            )
+        },
+        'summary_query': {
+            'influxdb': (
+                "SELECT SUM({field_name}) FROM {key} WHERE time >= '{time}'"
+                " GROUP BY time(30d), metric_num"
+            )
+        },
+    },
     'mean_test': {
         'type': 'line',
         'title': 'Mean test',
diff --git a/openwisp_monitoring/monitoring/tests/test_api.py b/openwisp_monitoring/monitoring/tests/test_api.py
@@ -68,14 +68,12 @@ def _create_org_traffic_metric(self, org, interface_name):
 
     def _create_test_users(self, org):
         admin = self._create_admin()
-        org2_administrator = self._create_user()
-        OrganizationUser.objects.create(
-            user=org2_administrator, organization=org, is_admin=True
-        )
+        org_admin = self._create_user()
+        OrganizationUser.objects.create(user=org_admin, organization=org, is_admin=True)
         groups = Group.objects.filter(name='Administrator')
-        org2_administrator.groups.set(groups)
+        org_admin.groups.set(groups)
         operator = self._create_operator()
-        return admin, org2_administrator, operator
+        return admin, org_admin, operator
 
     def test_wifi_client_chart(self):
         def _test_chart_properties(chart):
@@ -242,7 +240,7 @@ def _test_chart_properties(chart):
 
         self.client.force_login(admin)
         with self.subTest('Test superuser retrieves metric for all organizations'):
-            with self.assertNumQueries(2):
+            with self.assertNumQueries(3):
                 response = self.client.get(path)
             self.assertEqual(response.status_code, 200)
             self._test_response_data(response)
@@ -675,3 +673,49 @@ def test_group_by_time(self):
         with self.subTest('Test with invalid group time'):
             response = self.client.get(path, {'time': '3w'})
             self.assertEqual(response.status_code, 400)
+
+    def test_organizations_list(self):
+        path = reverse('monitoring_general:api_dashboard_timeseries')
+        Organization.objects.all().delete()
+        org1 = self._create_org(name='org1', slug='org1')
+        admin, org_admin, _ = self._create_test_users(org1)
+
+        self.client.force_login(admin)
+        with self.subTest('Superuser: Only one organization is present'):
+            response = self.client.get(path)
+            self.assertEqual(response.status_code, 200)
+            self.assertNotIn('organizations', response.data)
+
+        org2 = self._create_org(name='org2', slug='org2', id=self.org2_id)
+
+        with self.subTest('Superuser: Multiple organizations are present'):
+            response = self.client.get(path)
+            self.assertEqual(response.status_code, 200)
+            self.assertIn('organizations', response.data)
+            self.assertEqual(len(response.data['organizations']), 2)
+            self.assertEqual(
+                response.data['organizations'],
+                [{'id': org.slug, 'text': org.name} for org in [org1, org2]],
+            )
+
+        self.client.logout()
+        self.client.force_login(org_admin)
+
+        with self.subTest('Non-superuser: Administrator of one organization'):
+            response = self.client.get(path)
+            self.assertEqual(response.status_code, 200)
+            self.assertNotIn('organizations', response.data)
+
+        OrganizationUser.objects.create(
+            user=org_admin, organization=org2, is_admin=True
+        )
+
+        with self.subTest('Non-superuser: Administrator of multiple organizations'):
+            response = self.client.get(path)
+            self.assertEqual(response.status_code, 200)
+            self.assertIn('organizations', response.data)
+            self.assertEqual(len(response.data['organizations']), 2)
+            self.assertEqual(
+                response.data['organizations'],
+                [{'id': org.slug, 'text': org.name} for org in [org1, org2]],
+            )
diff --git a/openwisp_monitoring/monitoring/tests/test_charts.py b/openwisp_monitoring/monitoring/tests/test_charts.py
