From 11b31ffea2e32b93085e04b5fcb806682cbecc58 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Thu, 20 Oct 2022 10:07:29 -0300 Subject: [PATCH] [feature] Iperf3 #385 #390 #398 #399 #406 #405 #412 #414 #417 #418 #422 #424 #426 #406 #435 #416 #391 #443 Added Iperf3 check to perform active measurements of the maximum achievable bandwidth Closes #385 Closes #390 Closes #398 Closes #399 Closes #406 Closes #405 Closes #412 Closes #414 Closes #417 Closes #418 Closes #422 Closes #424 Closes #426 Closes #406 Closes #435 Closes #416 Closes #391 Closes #443 Signed-off-by: Aryaman Co-authored-by: Aryaman Co-authored-by: Federico Capoano Co-authored-by: Gagan Deep --- README.rst | 552 ++++++++- openwisp_monitoring/check/apps.py | 8 + openwisp_monitoring/check/base/models.py | 44 +- openwisp_monitoring/check/classes/__init__.py | 1 + openwisp_monitoring/check/classes/iperf3.py | 548 +++++++++ .../migrations/0008_alter_check_options.py | 24 + .../0009_add_check_inline_permissions.py | 19 + .../check/migrations/__init__.py | 34 + openwisp_monitoring/check/settings.py | 8 + openwisp_monitoring/check/tasks.py | 48 +- .../check/tests/iperf3_test_utils.py | 1078 +++++++++++++++++ .../check/tests/test_iperf3.py | 783 ++++++++++++ .../check/tests/test_models.py | 50 +- openwisp_monitoring/check/tests/test_ping.py | 2 +- .../db/backends/influxdb/queries.py | 45 + openwisp_monitoring/device/admin.py | 133 +- .../device/tests/test_admin.py | 290 ++++- openwisp_monitoring/monitoring/base/models.py | 77 +- .../monitoring/configuration.py | 124 +- .../0009_alter_alertsettings_options.py | 26 + ...10_add_alertsettings_inline_permissions.py | 19 + .../0011_alter_metric_field_name.py | 23 + .../monitoring/migrations/__init__.py | 31 + .../monitoring/static/monitoring/js/chart.js | 59 +- .../monitoring/tests/__init__.py | 9 + .../monitoring/tests/test_api.py | 2 +- .../monitoring/tests/test_charts.py | 1 + .../monitoring/tests/test_models.py | 25 + .../tests/test_monitoring_notifications.py | 126 ++ openwisp_monitoring/tests/test_selenium.py | 2 +- openwisp_monitoring/views.py | 2 + requirements.txt | 2 +- .../sample_check/migrations/0001_initial.py | 6 + .../0003_add_check_inline_permissions.py | 21 + .../migrations/0001_initial.py | 6 + ...03_add_alertsettings_inline_permissions.py | 21 + .../0004_alter_metric_field_name.py | 23 + tests/openwisp2/settings.py | 26 +- 38 files changed, 4218 insertions(+), 80 deletions(-) create mode 100644 openwisp_monitoring/check/classes/iperf3.py create mode 100644 openwisp_monitoring/check/migrations/0008_alter_check_options.py create mode 100644 openwisp_monitoring/check/migrations/0009_add_check_inline_permissions.py create mode 100644 openwisp_monitoring/check/tests/iperf3_test_utils.py create mode 100644 openwisp_monitoring/check/tests/test_iperf3.py create mode 100644 openwisp_monitoring/monitoring/migrations/0009_alter_alertsettings_options.py create mode 100644 openwisp_monitoring/monitoring/migrations/0010_add_alertsettings_inline_permissions.py create mode 100644 openwisp_monitoring/monitoring/migrations/0011_alter_metric_field_name.py create mode 100644 tests/openwisp2/sample_check/migrations/0003_add_check_inline_permissions.py create mode 100644 tests/openwisp2/sample_monitoring/migrations/0003_add_alertsettings_inline_permissions.py create mode 100644 tests/openwisp2/sample_monitoring/migrations/0004_alter_metric_field_name.py diff --git a/README.rst b/README.rst index 71ad255dd..e3fabb6e6 100644 --- a/README.rst +++ b/README.rst @@ -89,7 +89,9 @@ Available Features `RAM usage <#memory-usage>`_, `CPU load <#cpu-load>`_, `flash/disk usage <#disk-usage>`_, mobile signal (LTE/UMTS/GSM `signal strength <#mobile-signal-strength>`_, `signal quality <#mobile-signal-quality>`_, - `access technology in use <#mobile-access-technology-in-use>`_) + `access technology in use <#mobile-access-technology-in-use>`_), `bandwidth <#iperf3>`_, + `transferred data <#iperf3>`_, `restransmits <#iperf3>`_, `jitter <#iperf3>`_, + `datagram <#iperf3>`_, `datagram loss <#iperf3>`_ * Maintains a record of `WiFi sessions <#monitoring-wifi-sessions>`_ with clients' MAC address and vendor, session start and stop time and connected device along with other information @@ -107,6 +109,8 @@ Available Features * Extensible metrics and charts: it's possible to define new metrics and new charts * API to retrieve the chart metrics and status information of each device based on `NetJSON DeviceMonitoring `_ +* `Iperf3 check <#iperf3-1>`_ that provides network performance measurements such as maximum + achievable bandwidth, jitter, datagram loss etc of the openwrt device using `iperf3 utility `_ ------------ @@ -378,7 +382,15 @@ Configure celery (you may use a different broker if you want): CELERY_BEAT_SCHEDULE = { 'run_checks': { 'task': 'openwisp_monitoring.check.tasks.run_checks', + # Executes only ping & config check every 5 min 'schedule': timedelta(minutes=5), + 'args': ( + [ # Checks path + 'openwisp_monitoring.check.classes.Ping', + 'openwisp_monitoring.check.classes.ConfigApplied', + ], + ), + 'relative': True, }, # Delete old WifiSession 'delete_wifi_clients_and_sessions': { @@ -805,6 +817,59 @@ Mobile Access Technology in use .. figure:: https://github.com/openwisp/openwisp-monitoring/raw/docs/docs/access-technology.png :align: center +Iperf3 +~~~~~~ + ++--------------------+---------------------------------------------------------------------------------------------------------------------------+ +| **measurement**: | ``iperf3`` | ++--------------------+---------------------------------------------------------------------------------------------------------------------------+ +| **types**: | | ``int`` (iperf3_result, sent_bytes_tcp, received_bytes_tcp, retransmits, sent_bytes_udp, total_packets, lost_packets), | +| | | ``float`` (sent_bps_tcp, received_bps_tcp, sent_bps_udp, jitter, lost_percent) | ++--------------------+---------------------------------------------------------------------------------------------------------------------------+ +| **fields**: | | ``iperf3_result``, ``sent_bps_tcp``, ``received_bps_tcp``, ``sent_bytes_tcp``, ``received_bytes_tcp``, ``retransmits``, | +| | | ``sent_bps_udp``, ``sent_bytes_udp``, ``jitter``, ``total_packets``, ``lost_packets``, ``lost_percent`` | ++--------------------+---------------------------------------------------------------------------------------------------------------------------+ +| **configuration**: | ``iperf3`` | ++--------------------+---------------------------------------------------------------------------------------------------------------------------+ +| **charts**: | ``bandwidth``, ``transfer``, ``retransmits``, ``jitter``, ``datagram``, ``datagram_loss`` | ++--------------------+---------------------------------------------------------------------------------------------------------------------------+ + +**Bandwidth**: + +.. figure:: https://github.com/openwisp/openwisp-monitoring/raw/docs/docs/1.1/bandwidth.png + :align: center + +**Transferred Data**: + +.. figure:: https://github.com/openwisp/openwisp-monitoring/raw/docs/docs/1.1/transferred-data.png + :align: center + +**Retransmits**: + +.. figure:: https://github.com/openwisp/openwisp-monitoring/raw/docs/docs/1.1/retransmits.png + :align: center + +**Jitter**: + +.. figure:: https://github.com/openwisp/openwisp-monitoring/raw/docs/docs/1.1/jitter.png + :align: center + +**Datagram**: + +.. figure:: https://github.com/openwisp/openwisp-monitoring/raw/docs/docs/1.1/datagram.png + :align: center + +**Datagram loss**: + +.. figure:: https://github.com/openwisp/openwisp-monitoring/raw/docs/docs/1.1/datagram-loss.png + :align: center + +For more info on how to configure and use Iperf3, please refer to +`iperf3 check usage instructions <#iperf3-check-usage-instructions>`_. + +**Note:** Iperf3 charts uses ``connect_points=True`` in +`default chart configuration <#openwisp_monitoring_charts>`_ that joins it's individual chart data points. + Dashboard Monitoring Charts --------------------------- @@ -821,15 +886,15 @@ You can configure the interfaces included in the **General traffic chart** using the `"OPENWISP_MONITORING_DASHBOARD_TRAFFIC_CHART" <#openwisp_monitoring_dashboard_traffic_chart>`_ setting. -Adaptive byte charts +Adaptive size charts -------------------- .. figure:: https://github.com/openwisp/openwisp-monitoring/raw/docs/docs/1.1/adaptive-chart.png :align: center When configuring charts, it is possible to flag their unit -as ``adaptive_bytes``, this allows to make the charts more readable because -the units are shown in either `B`, `KB`, `MB`, `GB` and `TB` depending on +as ``adaptive_prefix``, this allows to make the charts more readable because +the units are shown in either `K`, `M`, `G` and `T` depending on the size of each point, the summary values and Y axis are also resized. Example taken from the default configuration of the traffic chart: @@ -838,7 +903,17 @@ Example taken from the default configuration of the traffic chart: 'traffic': { # other configurations for this chart - 'unit': 'adaptive_bytes', + + # traffic measured in 'B' (bytes) + # unit B, KB, MB, GB, TB + 'unit': 'adaptive_prefix+B', + }, + + 'bandwidth': { + # adaptive unit for bandwidth related charts + # bandwidth measured in 'bps'(bits/sec) + # unit bps, Kbps, Mbps, Gbps, Tbps + 'unit': 'adaptive_prefix+bps', }, Monitoring WiFi Sessions @@ -941,6 +1016,384 @@ configuration status of a device changes, this ensures the check reacts quickly to events happening in the network and informs the user promptly if there's anything that is not working as intended. +Iperf3 +~~~~~~ + +This check provides network performance measurements such as maximum achievable bandwidth, +jitter, datagram loss etc of the device using `iperf3 utility `_. + +This check is **disabled by default**. You can enable auto creation of this check by setting the +`OPENWISP_MONITORING_AUTO_IPERF3 <#OPENWISP_MONITORING_AUTO_IPERF3>`_ to ``True``. + +You can also `add the iperf3 check +<#add-checks-and-alert-settings-from-the-device-page>`_ directly from the device page. + +It also supports tuning of various parameters. + +You can also change the parameters used for iperf3 checks (e.g. timing, port, username, +password, rsa_publc_key etc) using the `OPENWISP_MONITORING_IPERF3_CHECK_CONFIG +<#OPENWISP_MONITORING_IPERF3_CHECK_CONFIG>`_ setting. + +**Note:** When setting `OPENWISP_MONITORING_AUTO_IPERF3 <#OPENWISP_MONITORING_AUTO_IPERF3>`_ to ``True``, +you may need to update the `metric configuration <#add-checks-and-alert-settings-from-the-device-page>`_ +to enable alerts for the iperf3 check. + +Iperf3 Check Usage Instructions +------------------------------- + +4. Make sure iperf3 is installed on the device +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Register your device to OpenWISP and make sure the `iperf3 openwrt package +`_ is installed on the device, +eg: + +.. code-block:: shell + + opkg install iperf3 # if using without authentication + opkg install iperf3-ssl # if using with authentication (read below for more info) + +2. Ensure SSH access from OpenWISP is enabled on your devices +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Follow the steps in `"How to configure push updates" section of the +OpenWISP documentation +`_ +to allow SSH access to you device from OpenWISP. + +**Note:** Make sure device connection is enabled +& working with right update strategy i.e. ``OpenWRT SSH``. + +.. image:: https://github.com/openwisp/openwisp-monitoring/raw/docs/docs/1.1/enable-openwrt-ssh.png + :alt: Enable ssh access from openwisp to device + :align: center + +3. Set up and configure Iperf3 server settings +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +After having deployed your Iperf3 servers, you need to +configure the iperf3 settings on the django side of OpenWISP, +see the `test project settings for reference +`_. + +The host can be specified by hostname, IPv4 literal, or IPv6 literal. +Example: + +.. code-block:: python + + OPENWISP_MONITORING_IPERF3_CHECK_CONFIG = { + # 'org_pk' : {'host' : [], 'client_options' : {}} + 'a9734710-db30-46b0-a2fc-01f01046fe4f': { + # Some public iperf3 servers + # https://iperf.fr/iperf-servers.php#public-servers + 'host': ['iperf3.openwisp.io', '2001:db8::1', '192.168.5.2'], + 'client_options': { + 'port': 5209, + 'udp': {'bitrate': '30M'}, + 'tcp': {'bitrate': '0'}, + }, + }, + # another org + 'b9734710-db30-46b0-a2fc-01f01046fe4f': { + # available iperf3 servers + 'host': ['iperf3.openwisp2.io', '192.168.5.3'], + 'client_options': { + 'port': 5207, + 'udp': {'bitrate': '50M'}, + 'tcp': {'bitrate': '20M'}, + }, + }, + } + +**Note:** If an organization has more than one iperf3 server configured, then it enables +the iperf3 checks to run concurrently on different devices. If all of the available servers +are busy, then it will add the check back in the queue. + +The celery-beat configuration for the iperf3 check needs to be added too: + +.. code-block:: python + + from celery.schedules import crontab + + # Celery TIME_ZONE should be equal to django TIME_ZONE + # In order to schedule run_iperf3_checks on the correct time intervals + CELERY_TIMEZONE = TIME_ZONE + CELERY_BEAT_SCHEDULE = { + # Other celery beat configurations + # Celery beat configuration for iperf3 check + 'run_iperf3_checks': { + 'task': 'openwisp_monitoring.check.tasks.run_checks', + # https://docs.celeryq.dev/en/latest/userguide/periodic-tasks.html#crontab-schedules + # Executes check every 5 mins from 00:00 AM to 6:00 AM (night) + 'schedule': crontab(minute='*/5', hour='0-6'), + # Iperf3 check path + 'args': (['openwisp_monitoring.check.classes.Iperf3'],), + 'relative': True, + } + } + +Once the changes are saved, you will need to restart all the processes. + +**Note:** We recommended to configure this check to run in non peak +traffic times to not interfere with standard traffic. + +4. Run the check +~~~~~~~~~~~~~~~~ + +This should happen automatically if you have celery-beat correctly +configured and running in the background. +For testing purposes, you can run this check manually using the +`run_checks <#run_checks>`_ command. + +After that, you should see the iperf3 network measurements charts. + +.. image:: https://github.com/openwisp/openwisp-monitoring/raw/docs/docs/1.1/iperf3-charts.png + :alt: Iperf3 network measurement charts + +Iperf3 check parameters +~~~~~~~~~~~~~~~~~~~~~~~ + +Currently, iperf3 check supports the following parameters: + ++-----------------------+----------+--------------------------------------------------------------------+ +| **Parameter** | **Type** | **Default Value** | ++-----------------------+----------+--------------------------------------------------------------------+ +|``host`` | ``list`` | ``[]`` | ++-----------------------+----------+--------------------------------------------------------------------+ +|``username`` | ``str`` | ``''`` | ++-----------------------+----------+--------------------------------------------------------------------+ +|``password`` | ``str`` | ``''`` | ++-----------------------+----------+--------------------------------------------------------------------+ +|``rsa_public_key`` | ``str`` | ``''`` | ++-----------------------+----------+--------------------------------------------------------------------+ +|``client_options`` | +---------------------+----------+------------------------------------------+ | +| | | **Parameters** | **Type** | **Default Value** | | +| | +---------------------+----------+------------------------------------------+ | +| | | ``port`` | ``int`` | ``5201`` | | +| | +---------------------+----------+------------------------------------------+ | +| | | ``time`` | ``int`` | ``10`` | | +| | +---------------------+----------+------------------------------------------+ | +| | | ``bytes`` | ``str`` | ``''`` | | +| | +---------------------+----------+------------------------------------------+ | +| | | ``blockcount`` | ``str`` | ``''`` | | +| | +---------------------+----------+------------------------------------------+ | +| | | ``window`` | ``str`` | ``0`` | | +| | +---------------------+----------+------------------------------------------+ | +| | | ``parallel`` | ``int`` | ``1`` | | +| | +---------------------+----------+------------------------------------------+ | +| | | ``reverse`` | ``bool`` | ``False`` | | +| | +---------------------+----------+------------------------------------------+ | +| | | ``bidirectional`` | ``bool`` | ``False`` | | +| | +---------------------+----------+------------------------------------------+ | +| | | ``connect_timeout`` | ``int`` | ``1`` | | +| | +---------------------+----------+------------------------------------------+ | +| | | ``tcp`` | +----------------+----------+---------------------+ | | +| | | | | **Parameters** | **Type** | **Default Value** | | | +| | | | +----------------+----------+---------------------+ | | +| | | | |``bitrate`` | ``str`` | ``0`` | | | +| | | | +----------------+----------+---------------------+ | | +| | | | |``length`` | ``str`` | ``128K`` | | | +| | | | +----------------+----------+---------------------+ | | +| | +---------------------+-----------------------------------------------------+ | +| | | ``udp`` | +----------------+----------+---------------------+ | | +| | | | | **Parameters** | **Type** | **Default Value** | | | +| | | | +----------------+----------+---------------------+ | | +| | | | |``bitrate`` | ``str`` | ``30M`` | | | +| | | | +----------------+----------+---------------------+ | | +| | | | |``length`` | ``str`` | ``0`` | | | +| | | | +----------------+----------+---------------------+ | | +| | +---------------------+-----------------------------------------------------+ | ++-----------------------+-------------------------------------------------------------------------------+ + +To learn how to use these parameters, please see the +`iperf3 check configuration example <#OPENWISP_MONITORING_IPERF3_CHECK_CONFIG>`_. + +Visit the `official documentation `_ +to learn more about the iperf3 parameters. + +Iperf3 authentication +~~~~~~~~~~~~~~~~~~~~~ + +By default iperf3 check runs without any kind of **authentication**, +in this section we will explain how to configure **RSA authentication** +between the **client** and the **server** to restrict connections +to authenticated clients. + +Server side +########### + +1. Generate RSA keypair +^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: shell + + openssl genrsa -des3 -out private.pem 2048 + openssl rsa -in private.pem -outform PEM -pubout -out public_key.pem + openssl rsa -in private.pem -out private_key.pem -outform PEM + +After running the commands mentioned above, the public key will be stored in +``public_key.pem`` which will be used in **rsa_public_key** parameter +in `OPENWISP_MONITORING_IPERF3_CHECK_CONFIG +<#OPENWISP_MONITORING_IPERF3_CHECK_CONFIG>`_ +and the private key will be contained in the file ``private_key.pem`` +which will be used with **--rsa-private-key-path** command option when +starting the iperf3 server. + +2. Create user credentials +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: shell + + USER=iperfuser PASSWD=iperfpass + echo -n "{$USER}$PASSWD" | sha256sum | awk '{ print $1 }' + ---- + ee17a7f98cc87a6424fb52682396b2b6c058e9ab70e946188faa0714905771d7 #This is the hash of "iperfuser" + +Add the above hash with username in ``credentials.csv`` + +.. code-block:: shell + + # file format: username,sha256 + iperfuser,ee17a7f98cc87a6424fb52682396b2b6c058e9ab70e946188faa0714905771d7 + +3. Now start the iperf3 server with auth options +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: shell + + iperf3 -s --rsa-private-key-path ./private_key.pem --authorized-users-path ./credentials.csv + +Client side (OpenWrt device) +############################ + +1. Install iperf3-ssl +^^^^^^^^^^^^^^^^^^^^^ + +Install the `iperf3-ssl openwrt package +`_ +instead of the normal +`iperf3 openwrt package `_ +because the latter comes without support for authentication. + +You may also check your installed **iperf3 openwrt package** features: + +.. code-block:: shell + + root@vm-openwrt:~ iperf3 -v + iperf 3.7 (cJSON 1.5.2) + Linux vm-openwrt 4.14.171 #0 SMP Thu Feb 27 21:05:12 2020 x86_64 + Optional features available: CPU affinity setting, IPv6 flow label, TCP congestion algorithm setting, + sendfile / zerocopy, socket pacing, authentication # contains 'authentication' + +2. Configure iperf3 check auth parameters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Now, add the following iperf3 authentication parameters +to `OPENWISP_MONITORING_IPERF3_CHECK_CONFIG +<#OPENWISP_MONITORING_IPERF3_CHECK_CONFIG>`_ +in the settings: + +.. code-block:: python + + OPENWISP_MONITORING_IPERF3_CHECK_CONFIG = { + 'a9734710-db30-46b0-a2fc-01f01046fe4f': { + 'host': ['iperf1.openwisp.io', 'iperf2.openwisp.io', '192.168.5.2'], + # All three parameters (username, password, rsa_publc_key) + # are required for iperf3 authentication + 'username': 'iperfuser', + 'password': 'iperfpass', + # Add RSA public key without any headers + # ie. -----BEGIN PUBLIC KEY-----, -----BEGIN END KEY----- + 'rsa_public_key': ( + """ + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwuEm+iYrfSWJOupy6X3N + dxZvUCxvmoL3uoGAs0O0Y32unUQrwcTIxudy38JSuCccD+k2Rf8S4WuZSiTxaoea + 6Du99YQGVZeY67uJ21SWFqWU+w6ONUj3TrNNWoICN7BXGLE2BbSBz9YaXefE3aqw + GhEjQz364Itwm425vHn2MntSp0weWb4hUCjQUyyooRXPrFUGBOuY+VvAvMyAG4Uk + msapnWnBSxXt7Tbb++A5XbOMdM2mwNYDEtkD5ksC/x3EVBrI9FvENsH9+u/8J9Mf + 2oPl4MnlCMY86MQypkeUn7eVWfDnseNky7TyC0/IgCXve/iaydCCFdkjyo1MTAA4 + BQIDAQAB + """ + ), + 'client_options': { + 'port': 5209, + 'udp': {'bitrate': '20M'}, + 'tcp': {'bitrate': '0'}, + }, + } + } + +Adding Checks and Alert settings from the device page +----------------------------------------------------- + +We can add checks and define alert settings directly from the **device page**. + +To add a check, you just need to select an available **check type** as shown below: + +.. figure:: https://github.com/openwisp/openwisp-monitoring/raw/docs/docs/1.1/device-inline-check.png + :align: center + +The following example shows how to use the +`OPENWISP_MONITORING_METRICS setting <#openwisp_monitoring_metrics>`_ +to reconfigure the system for `iperf3 check <#iperf3-1>`_ to send an alert if +the measured **TCP bandwidth** has been less than **10 Mbit/s** for more than **2 days**. + +1. By default, `Iperf3 checks <#iperf3-1>`_ come with default alert settings, +but it is easy to customize alert settings through the device page as shown below: + +.. figure:: https://github.com/openwisp/openwisp-monitoring/raw/docs/docs/1.1/device-inline-alertsettings.png + :align: center + +2. Now, add the following notification configuration to send an alert for **TCP bandwidth**: + +.. code-block:: python + + # Main project settings.py + from django.utils.translation import gettext_lazy as _ + + OPENWISP_MONITORING_METRICS = { + 'iperf3': { + 'notification': { + 'problem': { + 'verbose_name': 'Iperf3 PROBLEM', + 'verb': _('Iperf3 bandwidth is less than normal value'), + 'level': 'warning', + 'email_subject': _( + '[{site.name}] PROBLEM: {notification.target} {notification.verb}' + ), + 'message': _( + 'The device [{notification.target}]({notification.target_link}) ' + '{notification.verb}.' + ), + }, + 'recovery': { + 'verbose_name': 'Iperf3 RECOVERY', + 'verb': _('Iperf3 bandwidth now back to normal'), + 'level': 'info', + 'email_subject': _( + '[{site.name}] RECOVERY: {notification.target} {notification.verb}' + ), + 'message': _( + 'The device [{notification.target}]({notification.target_link}) ' + '{notification.verb}.' + ), + }, + }, + }, + } + +.. figure:: https://github.com/openwisp/openwisp-monitoring/raw/docs/docs/1.1/alert_field_warn.png + :align: center + +.. figure:: https://github.com/openwisp/openwisp-monitoring/raw/docs/docs/1.1/alert_field_info.png + :align: center + +**Note:** To access the features described above, the user must have permissions for ``Check`` and ``AlertSetting`` inlines, +these permissions are included by default in the "Administrator" and "Operator" groups and are shown in the screenshot below. + +.. figure:: https://github.com/openwisp/openwisp-monitoring/raw/docs/docs/1.1/inline-permissions.png + :align: center + Settings -------- @@ -1035,6 +1488,91 @@ validating custom parameters of a ``Check`` object. This setting allows you to choose whether `config_applied <#configuration-applied>`_ checks should be created automatically for newly registered devices. It's enabled by default. +``OPENWISP_MONITORING_AUTO_IPERF3`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++--------------+-------------+ +| **type**: | ``bool`` | ++--------------+-------------+ +| **default**: | ``False`` | ++--------------+-------------+ + +This setting allows you to choose whether `iperf3 <#iperf3-1>`_ checks should be +created automatically for newly registered devices. It's disabled by default. + +``OPENWISP_MONITORING_IPERF3_CHECK_CONFIG`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++--------------+-------------+ +| **type**: | ``dict`` | ++--------------+-------------+ +| **default**: | ``{}`` | ++--------------+-------------+ + +This setting allows to override the default iperf3 check configuration defined in +``openwisp_monitoring.check.classes.iperf3.DEFAULT_IPERF3_CHECK_CONFIG``. + +For example, you can change the values of `supported iperf3 check parameters <#iperf3-check-parameters>`_. + +.. code-block:: python + + OPENWISP_MONITORING_IPERF3_CHECK_CONFIG = { + # 'org_pk' : {'host' : [], 'client_options' : {}} + 'a9734710-db30-46b0-a2fc-01f01046fe4f': { + # Some public iperf3 servers + # https://iperf.fr/iperf-servers.php#public-servers + 'host': ['iperf3.openwisp.io', '2001:db8::1', '192.168.5.2'], + 'client_options': { + 'port': 6209, + # Number of parallel client streams to run + # note that iperf3 is single threaded + # so if you are CPU bound this will not + # yield higher throughput + 'parallel': 5, + # Set the connect_timeout (in milliseconds) for establishing + # the initial control connection to the server, the lower the value + # the faster the down iperf3 server will be detected + 'connect_timeout': 1, + # Window size / socket buffer size + 'window': '300K', + # Only one reverse condition can be chosen, + # reverse or bidirectional + 'reverse': True, + # Only one test end condition can be chosen, + # time, bytes or blockcount + 'blockcount': '1K', + 'udp': {'bitrate': '50M', 'length': '1460K'}, + 'tcp': {'bitrate': '20M', 'length': '256K'}, + }, + } + } + +``OPENWISP_MONITORING_IPERF3_CHECK_DELETE_RSA_KEY`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++--------------+-------------------------------+ +| **type**: | ``bool`` | ++--------------+-------------------------------+ +| **default**: | ``True`` | ++--------------+-------------------------------+ + +This setting allows you to set whether +`iperf3 check RSA public key <#configure-iperf3-check-for-authentication>`_ +will be deleted after successful completion of the check or not. + +``OPENWISP_MONITORING_IPERF3_CHECK_LOCK_EXPIRE`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++--------------+-------------------------------+ +| **type**: | ``int`` | ++--------------+-------------------------------+ +| **default**: | ``600`` | ++--------------+-------------------------------+ + +This setting allows you to set a cache lock expiration time for the iperf3 check when +running on multiple servers. Make sure it is always greater than the total iperf3 check +time, i.e. greater than the TCP + UDP test time. By default, it is set to **600 seconds (10 mins)**. + ``OPENWISP_MONITORING_AUTO_CHARTS`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1616,6 +2154,10 @@ The ``AlertSettings`` of ``ping`` metric will by default use ``threshold`` and ` defined in the ``alert_settings`` key. You can always override them and define your own custom values via the *admin*. +You can also use the ``alert_field`` key in metric configuration +which allows ``AlertSettings`` to check the ``threshold`` on +``alert_field`` instead of the default ``field_name`` key. + **Note**: It will raise ``ImproperlyConfigured`` exception if a metric configuration is already registered with same name (not to be confused with verbose_name). diff --git a/openwisp_monitoring/check/apps.py b/openwisp_monitoring/check/apps.py index 15c9832f0..e0a45998a 100644 --- a/openwisp_monitoring/check/apps.py +++ b/openwisp_monitoring/check/apps.py @@ -32,3 +32,11 @@ def _connect_signals(self): sender=load_model('config', 'Device'), dispatch_uid='auto_config_check', ) + if app_settings.AUTO_IPERF3: + from .base.models import auto_iperf3_check_receiver + + post_save.connect( + auto_iperf3_check_receiver, + sender=load_model('config', 'Device'), + dispatch_uid='auto_iperf3_check', + ) diff --git a/openwisp_monitoring/check/base/models.py b/openwisp_monitoring/check/base/models.py index a0bab9e66..0d040a7b1 100644 --- a/openwisp_monitoring/check/base/models.py +++ b/openwisp_monitoring/check/base/models.py @@ -9,7 +9,11 @@ from jsonfield import JSONField from openwisp_monitoring.check import settings as app_settings -from openwisp_monitoring.check.tasks import auto_create_config_check, auto_create_ping +from openwisp_monitoring.check.tasks import ( + auto_create_config_check, + auto_create_iperf3_check, + auto_create_ping, +) from openwisp_utils.base import TimeStampedEditableModel from ...utils import transaction_on_commit @@ -50,6 +54,13 @@ class Meta: abstract = True unique_together = ('name', 'object_id', 'content_type') + permissions = ( + ('add_check_inline', 'Can add check inline'), + ('change_check_inline', 'Can change check inline'), + ('delete_check_inline', 'Can delete check inline'), + ('view_check_inline', 'Can view check inline'), + ) + def __str__(self): if not self.object_id or not self.content_type: return self.name @@ -60,6 +71,14 @@ def __str__(self): def clean(self): self.check_instance.validate() + def full_clean(self, *args, **kwargs): + # The name of the check will be the same as the + # 'check_type' chosen by the user when the + # name field is empty (useful for CheckInline) + if not self.name: + self.name = self.get_check_type_display() + return super().full_clean(*args, **kwargs) + @cached_property def check_class(self): """ @@ -81,6 +100,11 @@ def perform_check(self, store=True): """ return self.check_instance.check(store=True) + def perform_check_delayed(self, duration=0): + from ..tasks import perform_check + + perform_check.apply_async(args=[self.id], countdown=duration) + def auto_ping_receiver(sender, instance, created, **kwargs): """ @@ -116,3 +140,21 @@ def auto_config_check_receiver(sender, instance, created, **kwargs): object_id=str(instance.pk), ) ) + + +def auto_iperf3_check_receiver(sender, instance, created, **kwargs): + """ + Implements OPENWISP_MONITORING_AUTO_IPERF3 + The creation step is executed in the background + """ + # we need to skip this otherwise this task will be executed + # every time the configuration is requested via checksum + if not created: + return + transaction_on_commit( + lambda: auto_create_iperf3_check.delay( + model=sender.__name__.lower(), + app_label=sender._meta.app_label, + object_id=str(instance.pk), + ) + ) diff --git a/openwisp_monitoring/check/classes/__init__.py b/openwisp_monitoring/check/classes/__init__.py index 33bf8293c..a7d9fde29 100644 --- a/openwisp_monitoring/check/classes/__init__.py +++ b/openwisp_monitoring/check/classes/__init__.py @@ -1,2 +1,3 @@ from .config_applied import ConfigApplied # noqa +from .iperf3 import Iperf3 # noqa from .ping import Ping # noqa diff --git a/openwisp_monitoring/check/classes/iperf3.py b/openwisp_monitoring/check/classes/iperf3.py new file mode 100644 index 000000000..e3f56748a --- /dev/null +++ b/openwisp_monitoring/check/classes/iperf3.py @@ -0,0 +1,548 @@ +import logging +from functools import reduce +from json import loads +from json.decoder import JSONDecodeError + +from django.core.cache import cache +from django.core.exceptions import ValidationError +from jsonschema import draft7_format_checker, validate +from jsonschema.exceptions import ValidationError as SchemaError +from swapper import load_model + +from openwisp_controller.connection.settings import UPDATE_STRATEGIES + +from .. import settings as app_settings +from .base import BaseCheck + +logger = logging.getLogger(__name__) + +Chart = load_model('monitoring', 'Chart') +Metric = load_model('monitoring', 'Metric') +AlertSettings = load_model('monitoring', 'AlertSettings') +DeviceConnection = load_model('connection', 'DeviceConnection') + +DEFAULT_IPERF3_CHECK_CONFIG = { + 'host': { + 'type': 'array', + 'items': { + 'type': 'string', + }, + 'default': [], + }, + # username, password max_length chosen from iperf3 docs to avoid iperf3 param errors + 'username': {'type': 'string', 'default': '', 'minLength': 1, 'maxLength': 20}, + 'password': {'type': 'string', 'default': '', 'minLength': 1, 'maxLength': 20}, + 'rsa_public_key': { + 'type': 'string', + 'default': '', + }, + 'client_options': { + 'type': 'object', + 'properties': { + 'port': { + 'type': 'integer', + 'default': 5201, + # max, min port chosen from iperf3 docs + 'minimum': 1, + 'maximum': 65535, + }, + 'time': { + # sets the interval time in seconds + # between periodic bandwidth, jitter, and loss reports + 'type': 'integer', + 'default': 10, + 'minimum': 1, + # arbitrary chosen to avoid slowing down the queue (30min) + 'maximum': 1800, + }, + 'bytes': { + # number of bytes to transmit (instead of 'time') + 'type': 'string', + # default to '' since we're using time for + # the test end condition instead of bytes + 'default': '', + }, + 'blockcount': { + # number of blocks (packets) to transmit + # instead of 'time' or 'bytes' + 'type': 'string', + # default to '' since we're using time for + # the test end condition instead of blockcount + 'default': '', + }, + 'window': { + # window size / socket buffer size + # this gets sent to the server and used on that side too + 'type': 'string', + 'default': '0', + }, + 'parallel': { + # number of parallel client streams to run + # note that iperf3 is single threaded + # so if you are CPU bound this will not yield higher throughput + 'type': 'integer', + 'default': 1, + # max, min parallel streams chosen from iperf3 docs + 'minimum': 1, + 'maximum': 128, + }, + 'reverse': { + # reverse the direction of a test + # the server sends data to the client + 'type': 'boolean', + 'default': False, + }, + 'bidirectional': { + # test in both directions (normal and reverse) + # with both the client and server sending + # and receiving data simultaneously + 'type': 'boolean', + 'default': False, + }, + 'connect_timeout': { + # set timeout for establishing the initial + # control connection to the server, in milliseconds (ms) + # providing a shorter value (ex. 1 ms) may + # speed up detection of a down iperf3 server + 'type': 'integer', + 'default': 1, + 'minimum': 1, + # arbitrary chosen to avoid slowing down the queue (1000 sec) + 'maximum': 1000000, + }, + 'tcp': { + 'type': 'object', + 'properties': { + 'bitrate': { + # set target bitrate to n bits/sec + 'type': 'string', + 'default': '0', + }, + 'length': { + # length of buffer to read or write + 'type': 'string', + # for TCP tests, the default value is 128KB + 'default': '128K', + }, + }, + }, + 'udp': { + 'type': 'object', + 'properties': { + 'bitrate': { + 'type': 'string', + # set target bitrate to n bits/sec + # 30 Mbps + 'default': '30M', + }, + 'length': { + # iperf3 tries to dynamically determine a + # reasonable sending size based on the path MTU + # if that cannot be determined it uses 1460 bytes + 'type': 'string', + 'default': '0', + }, + }, + }, + }, + }, +} + + +def get_iperf3_schema(): + schema = { + '$schema': 'http://json-schema.org/draft-07/schema#', + 'type': 'object', + 'additionalProperties': True, + 'dependencies': { + 'username': ['password', 'rsa_public_key'], + 'password': ['username', 'rsa_public_key'], + 'rsa_public_key': ['username', 'password'], + }, + } + schema['properties'] = DEFAULT_IPERF3_CHECK_CONFIG + return schema + + +class Iperf3(BaseCheck): + + schema = get_iperf3_schema() + + def validate_params(self, params=None): + try: + if not params: + params = self.params + validate(params, self.schema, format_checker=draft7_format_checker) + except SchemaError as e: + message = 'Invalid param' + path = '/'.join(e.path) + if path: + message = '{0} in "{1}"'.format(message, path) + message = '{0}: {1}'.format(message, e.message) + raise ValidationError({'params': message}) from e + + def _validate_iperf3_config(self, org): + # if iperf3 config is present and validate it's params + if app_settings.IPERF3_CHECK_CONFIG: + self.validate_params( + params=app_settings.IPERF3_CHECK_CONFIG.get(str(org.id)) + ) + + def check(self, store=True): + lock_acquired = False + org = self.related_object.organization + self._validate_iperf3_config(org) + available_iperf3_servers = self._get_param('host', 'host.default') + if not available_iperf3_servers: + logger.warning( + ( + f'Iperf3 servers for organization "{org}" ' + f'is not configured properly, iperf3 check skipped!' + ) + ) + return + time = self._get_param( + 'client_options.time', 'client_options.properties.time.default' + ) + # Try to acquire a lock, or put task back on queue + for server in available_iperf3_servers: + server_lock_key = f'ow_monitoring_{org}_iperf3_check_{server}' + # Set available_iperf3_server to the org device + lock_acquired = cache.add( + server_lock_key, + str(self.related_object), + timeout=app_settings.IPERF3_CHECK_LOCK_EXPIRE, + ) + if lock_acquired: + break + else: + logger.info( + ( + f'At the moment, all available iperf3 servers of organization "{org}" ' + f'are busy running checks, putting "{self.check_instance}" back in the queue..' + ) + ) + # Return the iperf3_check task to the queue, + # it will executed after 2 * iperf3_check_time (TCP+UDP) + self.check_instance.perform_check_delayed(duration=2 * time) + return + try: + # Execute the iperf3 check with current available server + result = self._run_iperf3_check(store, server, time) + finally: + # Release the lock after completion of the check + cache.delete(server_lock_key) + return result + + def _run_iperf3_check(self, store, server, time): + device_connection = self._get_device_connection() + if not device_connection: + logger.warning( + f'Failed to get a working DeviceConnection for "{self.related_object}", iperf3 check skipped!' + ) + return + # The DeviceConnection could fail if the management tunnel is down. + if not device_connection.connect(): + logger.warning( + f'DeviceConnection for "{self.related_object}" is not working, iperf3 check skipped!' + ) + return + command_tcp, command_udp = self._get_check_commands(server) + + # TCP mode + result, exit_code = device_connection.connector_instance.exec_command( + command_tcp, raise_unexpected_exit=False + ) + # Exit code 127 : command doesn't exist + if exit_code == 127: + logger.warning( + f'Iperf3 is not installed on the "{self.related_object}", error - {result.strip()}' + ) + return + + result_tcp = self._get_iperf3_result(result, exit_code, mode='TCP') + # UDP mode + result, exit_code = device_connection.connector_instance.exec_command( + command_udp, raise_unexpected_exit=False + ) + result_udp = self._get_iperf3_result(result, exit_code, mode='UDP') + result = {} + if store and result_tcp and result_udp: + # Store iperf3_result field 1 if any mode passes, store 0 when both fails + iperf3_result = result_tcp['iperf3_result'] | result_udp['iperf3_result'] + result.update({**result_tcp, **result_udp, 'iperf3_result': iperf3_result}) + self.store_result(result) + device_connection.disconnect() + return result + + def _get_check_commands(self, server): + """ + Returns tcp & udp commands for iperf3 check + """ + username = self._get_param('username', 'username.default') + port = self._get_param( + 'client_options.port', 'client_options.properties.port.default' + ) + window = self._get_param( + 'client_options.window', 'client_options.properties.window.default' + ) + parallel = self._get_param( + 'client_options.parallel', 'client_options.properties.parallel.default' + ) + ct = self._get_param( + 'client_options.connect_timeout', + 'client_options.properties.connect_timeout.default', + ) + tcp_bitrate = self._get_param( + 'client_options.tcp.bitrate', + 'client_options.properties.tcp.properties.bitrate.default', + ) + tcp_length = self._get_param( + 'client_options.tcp.length', + 'client_options.properties.tcp.properties.length.default', + ) + udp_bitrate = self._get_param( + 'client_options.udp.bitrate', + 'client_options.properties.udp.properties.bitrate.default', + ) + udp_length = self._get_param( + 'client_options.udp.length', + 'client_options.properties.udp.properties.length.default', + ) + + rev_or_bidir, test_end_condition = self._get_iperf3_test_conditions() + logger.info(f'«« Iperf3 server : {server}, Device : {self.related_object} »»') + command_tcp = ( + f'iperf3 -c {server} -p {port} {test_end_condition} --connect-timeout {ct} ' + f'-b {tcp_bitrate} -l {tcp_length} -w {window} -P {parallel} {rev_or_bidir} -J' + ) + command_udp = ( + f'iperf3 -c {server} -p {port} {test_end_condition} --connect-timeout {ct} ' + f'-b {udp_bitrate} -l {udp_length} -w {window} -P {parallel} {rev_or_bidir} -u -J' + ) + + # All three parameters ie. username, password and rsa_public_key is required + # for authentication to work, checking only username here + if username: + password = self._get_param('password', 'password.default') + key = self._get_param('rsa_public_key', 'rsa_public_key.default') + rsa_public_key = self._get_compelete_rsa_key(key) + rsa_public_key_path = '/tmp/iperf3-public-key.pem' + + command_tcp = ( + f'echo "{rsa_public_key}" > {rsa_public_key_path} && ' + f'IPERF3_PASSWORD="{password}" iperf3 -c {server} -p {port} {test_end_condition} ' + f'--username "{username}" --rsa-public-key-path {rsa_public_key_path} --connect-timeout {ct} ' + f'-b {tcp_bitrate} -l {tcp_length} -w {window} -P {parallel} {rev_or_bidir} -J' + ) + + command_udp = ( + f'IPERF3_PASSWORD="{password}" iperf3 -c {server} -p {port} {test_end_condition} ' + f'--username "{username}" --rsa-public-key-path {rsa_public_key_path} --connect-timeout {ct} ' + f'-b {udp_bitrate} -l {udp_length} -w {window} -P {parallel} {rev_or_bidir} -u -J' + ) + + # If IPERF3_CHECK_DELETE_RSA_KEY, remove rsa_public_key from the device + if app_settings.IPERF3_CHECK_DELETE_RSA_KEY: + command_udp = f'{command_udp} && rm -f {rsa_public_key_path}' + return command_tcp, command_udp + + def _get_iperf3_test_conditions(self): + """ + Returns iperf3 check test conditions (rev_or_bidir, end_condition) + """ + time = self._get_param( + 'client_options.time', 'client_options.properties.time.default' + ) + bytes = self._get_param( + 'client_options.bytes', 'client_options.properties.bytes.default' + ) + blockcount = self._get_param( + 'client_options.blockcount', 'client_options.properties.blockcount.default' + ) + reverse = self._get_param( + 'client_options.reverse', 'client_options.properties.reverse.default' + ) + bidirectional = self._get_param( + 'client_options.bidirectional', + 'client_options.properties.bidirectional.default', + ) + # by default we use 'time' param + # for the iperf3 test end condition + test_end_condition = f'-t {time}' + # if 'bytes' present in config + # use it instead of 'time' + if bytes: + test_end_condition = f'-n {bytes}' + # if 'blockcount' present in config + # use it instead of 'time' or 'bytes' + if blockcount: + test_end_condition = f'-k {blockcount}' + # only one reverse condition can be use + # reverse or bidirectional not both + rev_or_bidir = '' + if reverse: + rev_or_bidir = '--reverse' + if bidirectional: + rev_or_bidir = '--bidir' + return rev_or_bidir, test_end_condition + + def _get_compelete_rsa_key(self, key): + """ + Returns RSA key with proper format + """ + pem_prefix = '-----BEGIN PUBLIC KEY-----\n' + pem_suffix = '\n-----END PUBLIC KEY-----' + key = key.strip() + return f'{pem_prefix}{key}{pem_suffix}' + + def _get_device_connection(self): + """ + Returns an active SSH DeviceConnection for a device + """ + openwrt_ssh = UPDATE_STRATEGIES[0][0] + device_connection = DeviceConnection.objects.filter( + device_id=self.related_object.id, + update_strategy=openwrt_ssh, + enabled=True, + ).first() + return device_connection + + def _deep_get(self, dictionary, keys, default=None): + """ + Returns dict key value using dict & + it's dot_key string ie. key1.key2_nested.key3_nested + if found otherwise returns default + """ + return reduce( + lambda d, key: d.get(key, default) if isinstance(d, dict) else default, + keys.split("."), + dictionary, + ) + + def _get_param(self, conf_key, default_conf_key): + """ + Returns specified param or its default value according to the schema + """ + org_id = str(self.related_object.organization.id) + iperf3_config = app_settings.IPERF3_CHECK_CONFIG + + if self.params: + check_params = self._deep_get(self.params, conf_key) + if check_params: + return check_params + + if iperf3_config: + iperf3_config = iperf3_config.get(org_id) + iperf3_config_param = self._deep_get(iperf3_config, conf_key) + if iperf3_config_param: + return iperf3_config_param + + return self._deep_get(DEFAULT_IPERF3_CHECK_CONFIG, default_conf_key) + + def _get_iperf3_result(self, result, exit_code, mode): + """ + Returns iperf3 test result + """ + try: + result = loads(result) + except JSONDecodeError: + # Errors other than iperf3 test errors + logger.warning( + f'Iperf3 check failed for "{self.related_object}", error - {result.strip()}' + ) + return + + if mode == 'TCP': + if exit_code != 0: + logger.warning( + f'Iperf3 check failed for "{self.related_object}", {result["error"]}' + ) + return { + 'iperf3_result': 0, + 'sent_bps_tcp': 0.0, + 'received_bps_tcp': 0.0, + 'sent_bytes_tcp': 0, + 'received_bytes_tcp': 0, + 'retransmits': 0, + } + else: + sent = result['end']['sum_sent'] + received = result['end']['sum_received'] + return { + 'iperf3_result': 1, + 'sent_bps_tcp': float(sent['bits_per_second']), + 'received_bps_tcp': float(received['bits_per_second']), + 'sent_bytes_tcp': sent['bytes'], + 'received_bytes_tcp': received['bytes'], + 'retransmits': sent['retransmits'], + } + + elif mode == 'UDP': + if exit_code != 0: + logger.warning( + f'Iperf3 check failed for "{self.related_object}", {result["error"]}' + ) + return { + 'iperf3_result': 0, + 'sent_bps_udp': 0.0, + 'sent_bytes_udp': 0, + 'jitter': 0.0, + 'total_packets': 0, + 'lost_packets': 0, + 'lost_percent': 0.0, + } + else: + return { + 'iperf3_result': 1, + 'sent_bps_udp': float(result['end']['sum']['bits_per_second']), + 'sent_bytes_udp': result['end']['sum']['bytes'], + 'jitter': float(result['end']['sum']['jitter_ms']), + 'total_packets': result['end']['sum']['packets'], + 'lost_packets': result['end']['sum']['lost_packets'], + 'lost_percent': float(result['end']['sum']['lost_percent']), + } + + def store_result(self, result): + """ + Store result in the DB + """ + metric = self._get_metric() + copied = result.copy() + iperf3_result = copied.pop('iperf3_result') + metric.write(iperf3_result, extra_values=copied) + + def _get_metric(self): + """ + Gets or creates metric + """ + metric, created = self._get_or_create_metric() + if created: + self._create_alert_settings(metric) + self._create_charts(metric) + return metric + + def _create_alert_settings(self, metric): + """ + Creates default iperf3 alert settings with is_active=False + """ + alert_settings = AlertSettings(metric=metric, is_active=False) + alert_settings.full_clean() + alert_settings.save() + + def _create_charts(self, metric): + """ + Creates iperf3 related charts + """ + charts = [ + 'bandwidth', + 'transfer', + 'retransmits', + 'jitter', + 'datagram', + 'datagram_loss', + ] + for chart in charts: + chart = Chart(metric=metric, configuration=chart) + chart.full_clean() + chart.save() diff --git a/openwisp_monitoring/check/migrations/0008_alter_check_options.py b/openwisp_monitoring/check/migrations/0008_alter_check_options.py new file mode 100644 index 000000000..021ea044c --- /dev/null +++ b/openwisp_monitoring/check/migrations/0008_alter_check_options.py @@ -0,0 +1,24 @@ +# Generated by Django 3.2.14 on 2022-08-12 07:50 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('check', '0007_create_checks'), + ] + + operations = [ + migrations.AlterModelOptions( + name='check', + options={ + 'permissions': ( + ('add_check_inline', 'Can add check inline'), + ('change_check_inline', 'Can change check inline'), + ('delete_check_inline', 'Can delete check inline'), + ('view_check_inline', 'Can view check inline'), + ) + }, + ), + ] diff --git a/openwisp_monitoring/check/migrations/0009_add_check_inline_permissions.py b/openwisp_monitoring/check/migrations/0009_add_check_inline_permissions.py new file mode 100644 index 000000000..3dfdfe345 --- /dev/null +++ b/openwisp_monitoring/check/migrations/0009_add_check_inline_permissions.py @@ -0,0 +1,19 @@ +# Generated by Django 4.0.4 on 2022-08-19 11:28 + +from django.db import migrations + +from . import assign_check_inline_permissions_to_groups + + +class Migration(migrations.Migration): + + dependencies = [ + ('check', '0008_alter_check_options'), + ] + + operations = [ + migrations.RunPython( + assign_check_inline_permissions_to_groups, + reverse_code=migrations.RunPython.noop, + ), + ] diff --git a/openwisp_monitoring/check/migrations/__init__.py b/openwisp_monitoring/check/migrations/__init__.py index e69de29bb..afe4dbbc8 100644 --- a/openwisp_monitoring/check/migrations/__init__.py +++ b/openwisp_monitoring/check/migrations/__init__.py @@ -0,0 +1,34 @@ +from django.contrib.auth.models import Permission + +from openwisp_controller.migrations import create_default_permissions, get_swapped_model + + +def assign_check_inline_permissions_to_groups(apps, schema_editor): + create_default_permissions(apps, schema_editor) + operators_read_only_admins_manage = [ + 'check', + ] + manage_operations = ['add', 'view', 'change', 'delete'] + Group = get_swapped_model(apps, 'openwisp_users', 'Group') + + try: + admin = Group.objects.get(name='Administrator') + operator = Group.objects.get(name='Operator') + # consider failures custom cases + # that do not have to be dealt with + except Group.DoesNotExist: + return + + for model_name in operators_read_only_admins_manage: + try: + permission = Permission.objects.get( + codename='view_{}_inline'.format(model_name) + ) + operator.permissions.add(permission.pk) + except Permission.DoesNotExist: + pass + for operation in manage_operations: + permission = Permission.objects.get( + codename='{}_{}_inline'.format(operation, model_name) + ) + admin.permissions.add(permission.pk) diff --git a/openwisp_monitoring/check/settings.py b/openwisp_monitoring/check/settings.py index 4575c8eca..54b439bab 100644 --- a/openwisp_monitoring/check/settings.py +++ b/openwisp_monitoring/check/settings.py @@ -5,9 +5,17 @@ ( ('openwisp_monitoring.check.classes.Ping', 'Ping'), ('openwisp_monitoring.check.classes.ConfigApplied', 'Configuration Applied'), + ('openwisp_monitoring.check.classes.Iperf3', 'Iperf3'), ), ) AUTO_PING = get_settings_value('AUTO_PING', True) AUTO_CONFIG_CHECK = get_settings_value('AUTO_DEVICE_CONFIG_CHECK', True) MANAGEMENT_IP_ONLY = get_settings_value('MANAGEMENT_IP_ONLY', True) PING_CHECK_CONFIG = get_settings_value('PING_CHECK_CONFIG', {}) +AUTO_IPERF3 = get_settings_value('AUTO_IPERF3', False) +IPERF3_CHECK_CONFIG = get_settings_value('IPERF3_CHECK_CONFIG', {}) +IPERF3_CHECK_LOCK_EXPIRE = get_settings_value( + 'IPERF3_CHECK_LOCK_EXPIRE', 10 * 60 +) # 10 minutes arbitrarily chosen (must be longer than TCP + UDP test time) +IPERF3_CHECK_DELETE_RSA_KEY = get_settings_value('IPERF3_CHECK_DELETE_RSA_KEY', True) +CHECKS_LIST = get_settings_value('CHECK_LIST', list(dict(CHECK_CLASSES).keys())) diff --git a/openwisp_monitoring/check/tasks.py b/openwisp_monitoring/check/tasks.py index 2ae62bc0d..6a643db48 100644 --- a/openwisp_monitoring/check/tasks.py +++ b/openwisp_monitoring/check/tasks.py @@ -4,9 +4,11 @@ from celery import shared_task from django.conf import settings from django.contrib.contenttypes.models import ContentType -from django.core.exceptions import ObjectDoesNotExist +from django.core.exceptions import ImproperlyConfigured, ObjectDoesNotExist from swapper import load_model +from .settings import CHECKS_LIST + logger = logging.getLogger(__name__) @@ -15,7 +17,7 @@ def get_check_model(): @shared_task -def run_checks(): +def run_checks(checks=None): """ Retrieves the id of all active checks in chunks of 2000 items and calls the ``perform_check`` task (defined below) for each of them. @@ -23,9 +25,22 @@ def run_checks(): This allows to enqueue all the checks that need to be performed and execute them in parallel with multiple workers if needed. """ + # If checks is None, We should execute all the checks + if checks is None: + checks = CHECKS_LIST + + if not isinstance(checks, list): + raise ImproperlyConfigured( + f'Check path {checks} should be of type "list"' + ) # pragma: no cover + if not all(check_path in CHECKS_LIST for check_path in checks): + raise ImproperlyConfigured( + f'Check path {checks} should be in {CHECKS_LIST}' + ) # pragma: no cover + iterator = ( get_check_model() - .objects.filter(is_active=True) + .objects.filter(is_active=True, check_type__in=checks) .only('id') .values('id') .iterator() @@ -100,3 +115,30 @@ def auto_create_config_check( ) check.full_clean() check.save() + + +@shared_task +def auto_create_iperf3_check( + model, app_label, object_id, check_model=None, content_type_model=None +): + """ + Called by openwisp_monitoring.check.models.auto_iperf3_check_receiver + """ + Check = check_model or get_check_model() + iperf3_check_path = 'openwisp_monitoring.check.classes.Iperf3' + has_check = Check.objects.filter( + object_id=object_id, content_type__model='device', check_type=iperf3_check_path + ).exists() + # create new check only if necessary + if has_check: + return + content_type_model = content_type_model or ContentType + ct = content_type_model.objects.get(app_label=app_label, model=model) + check = Check( + name='Iperf3', + check_type=iperf3_check_path, + content_type=ct, + object_id=object_id, + ) + check.full_clean() + check.save() diff --git a/openwisp_monitoring/check/tests/iperf3_test_utils.py b/openwisp_monitoring/check/tests/iperf3_test_utils.py new file mode 100644 index 000000000..37ce9b164 --- /dev/null +++ b/openwisp_monitoring/check/tests/iperf3_test_utils.py @@ -0,0 +1,1078 @@ +# flake8: noqa + +RESULT_TCP = """ +{ + "start": { + "connected": [ + { + "socket": 5, + "local_host": "127.0.0.1", + "local_port": 54966, + "remote_host": "127.0.0.1", + "remote_port": 5201 + } + ], + "version": "iperf 3.9", + "system_info": "Linux openwisp-desktop 5.11.2-51-generic #58~20.04.1-Ubuntu SMP Tue Jun 14 11:29:12 UTC 2022 x86_64", + "timestamp": { + "time": "Thu, 30 Jun 2022 21:39:55 GMT", + "timesecs": 1656625195 + }, + "connecting_to": { + "host": "localhost", + "port": 5201 + }, + "cookie": "npx4ad65t3j4wginxr4a7mqedmkhhspx3sob", + "tcp_mss_default": 32768, + "sock_bufsize": 0, + "sndbuf_actual": 16384, + "rcvbuf_actual": 131072, + "test_start": { + "protocol": "TCP", + "num_streams": 1, + "blksize": 131072, + "omit": 0, + "duration": 10, + "bytes": 0, + "blocks": 0, + "reverse": 0, + "tos": 0 + } + }, + "intervals": [ + { + "streams": [ + { + "socket": 5, + "start": 0, + "end": 1.000048, + "seconds": 1.000048041343689, + "bytes": 5790760960, + "bits_per_second": 46323862219.414116, + "retransmits": 0, + "snd_cwnd": 1506109, + "rtt": 22, + "rttvar": 3, + "pmtu": 65535, + "omitted": false, + "sender": true + } + ], + "sum": { + "start": 0, + "end": 1.000048, + "seconds": 1.000048041343689, + "bytes": 5790760960, + "bits_per_second": 46323862219.414116, + "retransmits": 0, + "omitted": false, + "sender": true + } + }, + { + "streams": [ + { + "socket": 5, + "start": 1.000048, + "end": 2.000185, + "seconds": 1.0001369714736938, + "bytes": 5463080960, + "bits_per_second": 43698662209.83867, + "retransmits": 0, + "snd_cwnd": 2160939, + "rtt": 22, + "rttvar": 3, + "pmtu": 65535, + "omitted": false, + "sender": true + } + ], + "sum": { + "start": 1.000048, + "end": 2.000185, + "seconds": 1.0001369714736938, + "bytes": 5463080960, + "bits_per_second": 43698662209.83867, + "retransmits": 0, + "omitted": false, + "sender": true + } + }, + { + "streams": [ + { + "socket": 5, + "start": 2.000185, + "end": 3.00019, + "seconds": 1.0000050067901611, + "bytes": 5679349760, + "bits_per_second": 45434570598.638954, + "retransmits": 0, + "snd_cwnd": 2553837, + "rtt": 21, + "rttvar": 1, + "pmtu": 65535, + "omitted": false, + "sender": true + } + ], + "sum": { + "start": 2.000185, + "end": 3.00019, + "seconds": 1.0000050067901611, + "bytes": 5679349760, + "bits_per_second": 45434570598.638954, + "retransmits": 0, + "omitted": false, + "sender": true + } + }, + { + "streams": [ + { + "socket": 5, + "start": 3.00019, + "end": 4.000232, + "seconds": 1.0000419616699219, + "bytes": 5710807040, + "bits_per_second": 45684539320.4405, + "retransmits": 0, + "snd_cwnd": 2553837, + "rtt": 24, + "rttvar": 5, + "pmtu": 65535, + "omitted": false, + "sender": true + } + ], + "sum": { + "start": 3.00019, + "end": 4.000232, + "seconds": 1.0000419616699219, + "bytes": 5710807040, + "bits_per_second": 45684539320.4405, + "retransmits": 0, + "omitted": false, + "sender": true + } + }, + { + "streams": [ + { + "socket": 5, + "start": 4.000232, + "end": 5.000158, + "seconds": 0.999925971031189, + "bytes": 5307105280, + "bits_per_second": 42459985508.942955, + "retransmits": 0, + "snd_cwnd": 3208667, + "rtt": 27, + "rttvar": 4, + "pmtu": 65535, + "omitted": false, + "sender": true + } + ], + "sum": { + "start": 4.000232, + "end": 5.000158, + "seconds": 0.999925971031189, + "bytes": 5307105280, + "bits_per_second": 42459985508.942955, + "retransmits": 0, + "omitted": false, + "sender": true + } + }, + { + "streams": [ + { + "socket": 5, + "start": 5.000158, + "end": 6.000229, + "seconds": 1.0000710487365723, + "bytes": 5308416000, + "bits_per_second": 42464310964.35657, + "retransmits": 0, + "snd_cwnd": 3208667, + "rtt": 28, + "rttvar": 1, + "pmtu": 65535, + "omitted": false, + "sender": true + } + ], + "sum": { + "start": 5.000158, + "end": 6.000229, + "seconds": 1.0000710487365723, + "bytes": 5308416000, + "bits_per_second": 42464310964.35657, + "retransmits": 0, + "omitted": false, + "sender": true + } + }, + { + "streams": [ + { + "socket": 5, + "start": 6.000229, + "end": 7.000056, + "seconds": 0.9998270273208618, + "bytes": 5241569280, + "bits_per_second": 41939808681.0701, + "retransmits": 0, + "snd_cwnd": 3208667, + "rtt": 23, + "rttvar": 4, + "pmtu": 65535, + "omitted": false, + "sender": true + } + ], + "sum": { + "start": 6.000229, + "end": 7.000056, + "seconds": 0.9998270273208618, + "bytes": 5241569280, + "bits_per_second": 41939808681.0701, + "retransmits": 0, + "omitted": false, + "sender": true + } + }, + { + "streams": [ + { + "socket": 5, + "start": 7.000056, + "end": 8.000202, + "seconds": 1.0001460313797, + "bytes": 5734400000, + "bits_per_second": 45868501759.40331, + "retransmits": 0, + "snd_cwnd": 3208667, + "rtt": 22, + "rttvar": 1, + "pmtu": 65535, + "omitted": false, + "sender": true + } + ], + "sum": { + "start": 7.000056, + "end": 8.000202, + "seconds": 1.0001460313797, + "bytes": 5734400000, + "bits_per_second": 45868501759.40331, + "retransmits": 0, + "omitted": false, + "sender": true + } + }, + { + "streams": [ + { + "socket": 5, + "start": 8.000202, + "end": 9.0003, + "seconds": 1.0000979900360107, + "bytes": 5415895040, + "bits_per_second": 43322915105.98867, + "retransmits": 0, + "snd_cwnd": 3208667, + "rtt": 35, + "rttvar": 12, + "pmtu": 65535, + "omitted": false, + "sender": true + } + ], + "sum": { + "start": 8.000202, + "end": 9.0003, + "seconds": 1.0000979900360107, + "bytes": 5415895040, + "bits_per_second": 43322915105.98867, + "retransmits": 0, + "omitted": false, + "sender": true + } + }, + { + "streams": [ + { + "socket": 5, + "start": 9.0003, + "end": 10.000218, + "seconds": 0.999917984008789, + "bytes": 5402787840, + "bits_per_second": 43225847930.76398, + "retransmits": 0, + "snd_cwnd": 3208667, + "rtt": 26, + "rttvar": 17, + "pmtu": 65535, + "omitted": false, + "sender": true + } + ], + "sum": { + "start": 9.0003, + "end": 10.000218, + "seconds": 0.999917984008789, + "bytes": 5402787840, + "bits_per_second": 43225847930.76398, + "retransmits": 0, + "omitted": false, + "sender": true + } + } + ], + "end": { + "streams": [ + { + "sender": { + "socket": 5, + "start": 0, + "end": 10.000218, + "seconds": 10.000218, + "bytes": 55054172160, + "bits_per_second": 44042377604.16823, + "retransmits": 0, + "max_snd_cwnd": 3208667, + "max_rtt": 35, + "min_rtt": 21, + "mean_rtt": 25, + "sender": true + }, + "receiver": { + "socket": 5, + "start": 0, + "end": 10.000272, + "seconds": 10.000218, + "bytes": 55054172160, + "bits_per_second": 44042139781.797935, + "sender": true + } + } + ], + "sum_sent": { + "start": 0, + "end": 10.000218, + "seconds": 10.000218, + "bytes": 55054172160, + "bits_per_second": 44042377604.16823, + "retransmits": 0, + "sender": true + }, + "sum_received": { + "start": 0, + "end": 10.000272, + "seconds": 10.000272, + "bytes": 55054172160, + "bits_per_second": 44042139781.797935, + "sender": true + }, + "cpu_utilization_percent": { + "host_total": 99.49882081069975, + "host_user": 0.6620490539150914, + "host_system": 98.83676176238454, + "remote_total": 0.377797593572381, + "remote_user": 0.02174276147834767, + "remote_system": 0.35605477540538377 + }, + "sender_tcp_congestion": "cubic", + "receiver_tcp_congestion": "cubic" + } +} +""" + +RESULT_UDP = """ +{ + "start": { + "connected": [ + { + "socket": 5, + "local_host": "127.0.0.1", + "local_port": 54477, + "remote_host": "127.0.0.1", + "remote_port": 5201 + } + ], + "version": "iperf 3.9", + "system_info": "openwisp-desktop 5.11.2-51-generic #58~20.04.1-Ubuntu SMP Tue Jun 14 11:29:12 UTC 2022 x86_64", + "timestamp": { + "time": "Thu, 30 Jun 2022 21:10:31 GMT", + "timesecs": 1656623431 + }, + "connecting_to": { + "host": "localhost", + "port": 5201 + }, + "cookie": "kvuxkz3ncutquvpl2evufmdkn726molzocot", + "sock_bufsize": 0, + "sndbuf_actual": 212992, + "rcvbuf_actual": 212992, + "test_start": { + "protocol": "UDP", + "num_streams": 1, + "blksize": 32768, + "omit": 0, + "duration": 10, + "bytes": 0, + "blocks": 0, + "reverse": 0, + "tos": 0 + } + }, + "intervals": [ + { + "streams": [ + { + "socket": 5, + "start": 0, + "end": 1.000057, + "seconds": 1.0000569820404053, + "bytes": 131072, + "bits_per_second": 1048516.253404483, + "packets": 4, + "omitted": false, + "sender": true + } + ], + "sum": { + "start": 0, + "end": 1.000057, + "seconds": 1.0000569820404053, + "bytes": 131072, + "bits_per_second": 1048516.253404483, + "packets": 4, + "omitted": false, + "sender": true + } + }, + { + "streams": [ + { + "socket": 5, + "start": 1.000057, + "end": 2.000079, + "seconds": 1.000022053718567, + "bytes": 131072, + "bits_per_second": 1048552.875509981, + "packets": 4, + "omitted": false, + "sender": true + } + ], + "sum": { + "start": 1.000057, + "end": 2.000079, + "seconds": 1.000022053718567, + "bytes": 131072, + "bits_per_second": 1048552.875509981, + "packets": 4, + "omitted": false, + "sender": true + } + }, + { + "streams": [ + { + "socket": 5, + "start": 2.000079, + "end": 3.000079, + "seconds": 1, + "bytes": 131072, + "bits_per_second": 1048576, + "packets": 4, + "omitted": false, + "sender": true + } + ], + "sum": { + "start": 2.000079, + "end": 3.000079, + "seconds": 1, + "bytes": 131072, + "bits_per_second": 1048576, + "packets": 4, + "omitted": false, + "sender": true + } + }, + { + "streams": [ + { + "socket": 5, + "start": 3.000079, + "end": 4.000079, + "seconds": 1, + "bytes": 131072, + "bits_per_second": 1048576, + "packets": 4, + "omitted": false, + "sender": true + } + ], + "sum": { + "start": 3.000079, + "end": 4.000079, + "seconds": 1, + "bytes": 131072, + "bits_per_second": 1048576, + "packets": 4, + "omitted": false, + "sender": true + } + }, + { + "streams": [ + { + "socket": 5, + "start": 4.000079, + "end": 5.000182, + "seconds": 1.0001029968261719, + "bytes": 131072, + "bits_per_second": 1048468.0111225117, + "packets": 4, + "omitted": false, + "sender": true + } + ], + "sum": { + "start": 4.000079, + "end": 5.000182, + "seconds": 1.0001029968261719, + "bytes": 131072, + "bits_per_second": 1048468.0111225117, + "packets": 4, + "omitted": false, + "sender": true + } + }, + { + "streams": [ + { + "socket": 5, + "start": 5.000182, + "end": 6.000056, + "seconds": 0.9998739957809448, + "bytes": 131072, + "bits_per_second": 1048708.1416504055, + "packets": 4, + "omitted": false, + "sender": true + } + ], + "sum": { + "start": 5.000182, + "end": 6.000056, + "seconds": 0.9998739957809448, + "bytes": 131072, + "bits_per_second": 1048708.1416504055, + "packets": 4, + "omitted": false, + "sender": true + } + }, + { + "streams": [ + { + "socket": 5, + "start": 6.000056, + "end": 7.000056, + "seconds": 1, + "bytes": 131072, + "bits_per_second": 1048576, + "packets": 4, + "omitted": false, + "sender": true + } + ], + "sum": { + "start": 6.000056, + "end": 7.000056, + "seconds": 1, + "bytes": 131072, + "bits_per_second": 1048576, + "packets": 4, + "omitted": false, + "sender": true + } + }, + { + "streams": [ + { + "socket": 5, + "start": 7.000056, + "end": 8.000056, + "seconds": 1, + "bytes": 131072, + "bits_per_second": 1048576, + "packets": 4, + "omitted": false, + "sender": true + } + ], + "sum": { + "start": 7.000056, + "end": 8.000056, + "seconds": 1, + "bytes": 131072, + "bits_per_second": 1048576, + "packets": 4, + "omitted": false, + "sender": true + } + }, + { + "streams": [ + { + "socket": 5, + "start": 8.000056, + "end": 9.000057, + "seconds": 1.0000009536743164, + "bytes": 131072, + "bits_per_second": 1048575.0000009537, + "packets": 4, + "omitted": false, + "sender": true + } + ], + "sum": { + "start": 8.000056, + "end": 9.000057, + "seconds": 1.0000009536743164, + "bytes": 131072, + "bits_per_second": 1048575.0000009537, + "packets": 4, + "omitted": false, + "sender": true + } + }, + { + "streams": [ + { + "socket": 5, + "start": 9.000057, + "end": 10.00006, + "seconds": 1.0000029802322388, + "bytes": 131072, + "bits_per_second": 1048572.8750093132, + "packets": 4, + "omitted": false, + "sender": true + } + ], + "sum": { + "start": 9.000057, + "end": 10.00006, + "seconds": 1.0000029802322388, + "bytes": 131072, + "bits_per_second": 1048572.8750093132, + "packets": 4, + "omitted": false, + "sender": true + } + } + ], + "end": { + "streams": [ + { + "udp": { + "socket": 5, + "start": 0, + "end": 10.00006, + "seconds": 10.00006, + "bytes": 1310720, + "bits_per_second": 1048569.7085817485, + "jitter_ms": 0.011259258240784126, + "lost_packets": 0, + "packets": 40, + "lost_percent": 0, + "out_of_order": 0, + "sender": true + } + } + ], + "sum": { + "start": 0, + "end": 10.000115, + "seconds": 10.000115, + "bytes": 1310720, + "bits_per_second": 1048569.7085817485, + "jitter_ms": 0.011259258240784126, + "lost_packets": 0, + "packets": 40, + "lost_percent": 0, + "sender": true + }, + "cpu_utilization_percent": { + "host_total": 0.6057128493969417, + "host_user": 0, + "host_system": 0.6057128493969417, + "remote_total": 0.016163250220207454, + "remote_user": 0.01616789349806445, + "remote_system": 0 + } + } +} +""" + +RESULT_FAIL = """ +{ + "start": { + "connected": [], + "version": "iperf 3.7", + "system_info": "Linux vm-openwrt 4.14.171 #0 SMP Thu Feb 27 21:05:12 2020 x86_64" + }, + "intervals": [], + "end": {}, + "error": "error - unable to connect to server: Connection refused" +} +""" +RESULT_AUTH_FAIL = """ +{ + "start": { + "connected": [], + "version": "iperf 3.7", + "system_info": "Linux vm-openwrt 4.14.171 #0 SMP Thu Feb 27 21:05:12 2020 x86_64", + "timestamp": { + "time": "Tue, 19 Jul 2022 12:23:38 UTC", + "timesecs": 1658233418 + }, + "connecting_to": { + "host": "192.168.5.109", + "port": 5201 + }, + "cookie": "llz5f6akwyonbtcj3fx4phvfaflohdlvxr4z", + "tcp_mss_default": 1460 + }, + "intervals": [], + "end": {}, + "error": "error - test authorization failed" +} +""" +PARAM_ERROR = """Usage: iperf3 [-s|-c host] [options] + iperf3 [-h|--help] [-v|--version] + +Server or Client: + -p, --port # server port to listen on/connect to + -f, --format [kmgtKMGT] format to report: Kbits, Mbits, Gbits, Tbits + -i, --interval # seconds between periodic throughput reports + -F, --file name xmit/recv the specified file + -A, --affinity n/n,m set CPU affinity + -B, --bind bind to the interface associated with the address + -V, --verbose more detailed output + -J, --json output in JSON format + --logfile f send output to a log file + --forceflush force flushing output at every interval + -d, --debug emit debugging output + -v, --version show version information and quit + -h, --help show this message and quit +Server specific: + -s, --server run in server mode + -D, --daemon run the server as a daemon + -I, --pidfile file write PID file + -1, --one-off handle one client connection then exit + --rsa-private-key-path path to the RSA private key used to decrypt + authentication credentials + --authorized-users-path path to the configuration file containing user + credentials +Client specific: + -c, --client run in client mode, connecting to + -u, --udp use UDP rather than TCP + --connect-timeout # timeout for control connection setup (ms) + -b, --bitrate #[KMG][/#] target bitrate in bits/sec (0 for unlimited) + (default 1 Mbit/sec for UDP, unlimited for TCP) + (optional slash and packet count for burst mode) + --pacing-timer #[KMG] set the timing for pacing, in microseconds (default 1000) + --fq-rate #[KMG] enable fair-queuing based socket pacing in + bits/sec (Linux only) + -t, --time # time in seconds to transmit for (default 10 secs) + -n, --bytes #[KMG] number of bytes to transmit (instead of -t) + -k, --blockcount #[KMG] number of blocks (packets) to transmit (instead of -t or -n) + -l, --length #[KMG] length of buffer to read or write + (default 128 KB for TCP, dynamic or 1460 for UDP) + --cport bind to a specific client port (TCP and UDP, default: ephemeral port) + -P, --parallel # number of parallel client streams to run + -R, --reverse run in reverse mode (server sends, client receives) + --bidir run in bidirectional mode. + Client and server send and receive data. + -w, --window #[KMG] set window size / socket buffer size + -C, --congestion set TCP congestion control algorithm (Linux and FreeBSD only) + -M, --set-mss # set TCP/SCTP maximum segment size (MTU - 40 bytes) + -N, --no-delay set TCP/SCTP no delay, disabling Nagle's Algorithm + -4, --version4 only use IPv4 + -6, --version6 only use IPv6 + -S, --tos N set the IP type of service, 0-255. + The usual prefixes for octal and hex can be used, + i.e. 52, 064 and 0x34 all specify the same value. + --dscp N or --dscp val set the IP dscp value, either 0-63 or symbolic. + Numeric values can be specified in decimal, + octal and hex (see --tos above). + -L, --flowlabel N set the IPv6 flow label (only supported on Linux) + -Z, --zerocopy use a 'zero copy' method of sending data + -O, --omit N omit the first n seconds + -T, --title str prefix every output line with this string + --extra-data str data string to include in client and server JSON + --get-server-output get results from server + --udp-counters-64bit use 64-bit counters in UDP test packets + --repeating-payload use repeating pattern in payload, instead of + randomized payload (like in iperf2) + --username username for authentication + --rsa-public-key-path path to the RSA public key used to encrypt + authentication credentials + +[KMG] indicates options that support a K/M/G suffix for kilo-, mega-, or giga- + +iperf3 homepage at: https://software.es.net/iperf/ +Report bugs to: https://github.com/esnet/iperf +iperf3: parameter error - you must specify username (max 20 chars), password (max 20 chars) and a path to a valid public rsa client to be used""" + +TEST_RSA_KEY = """MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwuEm+iYrfSWJOupy6X3N +dxZvUCxvmoL3uoGAs0O0Y32unUQrwcTIxudy38JSuCccD+k2Rf8S4WuZSiTxaoea +6Du99YQGVZeY67uJ21SWFqWU+w6ONUj3TrNNWoICN7BXGLE2BbSBz9YaXefE3aqw +GhEjQz364Itwm425vHn2MntSp0weWb4hUCjQUyyooRXPrFUGBOuY+VvAvMyAG4Uk +msapnWnBSxXt7Tbb++A5XbOMdM2mwNYDEtkD5ksC/x3EVBrI9FvENsH9+u/8J9Mf +2oPl4MnlCMY86MQypkeUn7eVWfDnseNky7TyC0/IgCXve/iaydCCFdkjyo1MTAA4 +BQIDAQAB""" + +INVALID_PARAMS = [ + {'host': ''}, + {'host': 12}, + {'host': 'test.openwisp.io'}, + {'username': 121}, + {'password': -323}, + {'rsa_public_key': 1334}, + {'username': ''}, + {'password': 0}, + {'rsa_public_key': 0}, + { + 'username': 'openwisp-test-user', + 'password': 'open-pass', + 'rsa_public_key': -1, + }, + { + 'username': 1123, + 'password': 'rossi', + 'rsa_public_key': '', + }, + { + 'username': 'openwisp-test-user', + 'password': -214, + }, + { + 'client_options': { + 'port': 'testport', + 'time': 120, + 'tcp': {'bitrate': '10M'}, + 'udp': {'bitrate': '50M'}, + } + }, + { + 'host': ['test.openwisp.io'], + 'client_options': { + 'port': 'testport', + 'time': 120, + 'tcp': {'bitrate': '10M'}, + 'udp': {'bitrate': '50M'}, + }, + }, + { + 'host': ['test.openwisp.io'], + 'client_options': { + 'port': 70000, + 'time': 120, + 'tcp': {'bitrate': '10M'}, + 'udp': {'bitrate': '50M'}, + }, + }, + { + 'host': ['test.openwisp.io'], + 'client_options': { + 'port': -21, + 'time': 120, + 'tcp': {'bitrate': '10M'}, + 'udp': {'bitrate': '50M'}, + }, + }, + { + 'host': ['test.openwisp.io'], + 'client_options': { + 'port': 5201, + 'time': 1200000, + 'tcp': {'bitrate': '10M'}, + 'udp': {'bitrate': '50M'}, + }, + }, + { + 'host': ['test.openwisp.io'], + 'client_options': { + 'port': 5201, + 'time': 20, + 'tcp': {'bitrate': 10}, + 'udp': {'bitrate': '50M'}, + }, + }, + { + 'host': ['test.openwisp.io'], + 'client_options': { + 'port': 5201, + 'time': 120, + 'tcp': {'bitrate': '10M'}, + 'udp': {'bitrate': 50}, + }, + }, + { + 'host': ['test.openwisp.io'], + 'client_options': { + 'port': 5201, + 'bytes': 20, + 'tcp': {'bitrate': '10M'}, + 'udp': {'bitrate': 50}, + }, + }, + { + 'host': ['test.openwisp.io'], + 'client_options': { + 'port': 5201, + 'bytes': '', + 'tcp': {'bitrate': '10M'}, + 'udp': {'bitrate': 50}, + }, + }, + { + 'host': ['test.openwisp.io'], + 'client_options': { + 'port': 5201, + 'bytes': -1, + 'tcp': {'bitrate': '10M'}, + 'udp': {'bitrate': 50}, + }, + }, + { + 'host': ['test.openwisp.io'], + 'client_options': { + 'port': 5201, + 'connect_timeout': -1, + 'tcp': {'bitrate': '10M'}, + 'udp': {'bitrate': 50}, + }, + }, + { + 'host': ['test.openwisp.io'], + 'client_options': { + 'port': 5201, + 'connect_timeout': '11000', + 'tcp': {'bitrate': '10M'}, + 'udp': {'bitrate': 50}, + }, + }, + { + 'host': ['test.openwisp.io'], + 'client_options': { + 'port': 5201, + 'blockcount': -13, + 'tcp': {'bitrate': '10M'}, + 'udp': {'bitrate': 50}, + }, + }, + { + 'host': ['test.openwisp.io'], + 'client_options': { + 'port': 5201, + 'blockcount': '', + 'tcp': {'bitrate': '10M'}, + 'udp': {'bitrate': 50}, + }, + }, + { + 'host': ['test.openwisp.io'], + 'client_options': { + 'port': 5201, + 'tcp': {'bitrate': '10M', 'length': 112}, + 'udp': {'bitrate': 50}, + }, + }, + { + 'host': ['test.openwisp.io'], + 'client_options': { + 'port': 5201, + 'connect_timeout': 2000000, + 'blockcount': '100K', + 'tcp': {'bitrate': '10M'}, + 'udp': {'bitrate': 50, 'length': 9595}, + }, + }, + { + 'host': ['test.openwisp.io'], + 'client_options': { + 'port': 5201, + 'parallel': '12', + 'connect_timeout': 2000000, + 'blockcount': '100K', + 'tcp': {'bitrate': '10M'}, + 'udp': {'bitrate': 50, 'length': 9595}, + }, + }, + { + 'host': ['test.openwisp.io'], + 'client_options': { + 'port': 5201, + 'parallel': 0, + 'connect_timeout': 2000000, + 'blockcount': '100K', + 'tcp': {'bitrate': '10M'}, + 'udp': {'bitrate': 50, 'length': 9595}, + }, + }, + { + 'host': ['test.openwisp.io'], + 'client_options': { + 'port': 5201, + 'parallel': 250, + 'connect_timeout': 2000000, + 'blockcount': '100K', + 'tcp': {'bitrate': '10M'}, + 'udp': {'bitrate': 50, 'length': 9595}, + }, + }, + { + 'host': ['test.openwisp.io'], + 'client_options': { + 'port': 5201, + 'bidirectional': True, + 'connect_timeout': 2000000, + 'blockcount': '100K', + 'tcp': {'bitrate': '10M'}, + 'udp': {'bitrate': 50, 'length': 9595}, + }, + }, + { + 'host': ['test.openwisp.io'], + 'client_options': { + 'port': 5201, + 'reverse': False, + 'connect_timeout': 2000000, + 'blockcount': '100K', + 'tcp': {'bitrate': '10M'}, + 'udp': {'bitrate': 50, 'length': 9595}, + }, + }, + { + 'host': ['test.openwisp.io'], + 'client_options': { + 'port': 5201, + 'window': 0, + 'connect_timeout': 2000000, + 'blockcount': '100K', + 'tcp': {'bitrate': '10M'}, + 'udp': {'bitrate': 50, 'length': 9595}, + }, + }, +] diff --git a/openwisp_monitoring/check/tests/test_iperf3.py b/openwisp_monitoring/check/tests/test_iperf3.py new file mode 100644 index 000000000..dd560aa51 --- /dev/null +++ b/openwisp_monitoring/check/tests/test_iperf3.py @@ -0,0 +1,783 @@ +from json import loads +from unittest.mock import call, patch + +from django.core.cache import cache +from django.core.exceptions import ValidationError +from django.test import TransactionTestCase +from swapper import load_model + +from openwisp_controller.connection.connectors.ssh import Ssh +from openwisp_controller.connection.models import DeviceConnection as device_connection +from openwisp_controller.connection.settings import UPDATE_STRATEGIES +from openwisp_controller.connection.tests.utils import CreateConnectionsMixin, SshServer +from openwisp_monitoring.check.classes.iperf3 import get_iperf3_schema +from openwisp_monitoring.check.classes.iperf3 import logger as iperf3_logger + +from ...device.tests import TestDeviceMonitoringMixin +from .. import settings as app_settings +from ..classes import Iperf3 +from .iperf3_test_utils import ( + INVALID_PARAMS, + PARAM_ERROR, + RESULT_AUTH_FAIL, + RESULT_FAIL, + RESULT_TCP, + RESULT_UDP, + TEST_RSA_KEY, +) + +Chart = load_model('monitoring', 'Chart') +AlertSettings = load_model('monitoring', 'AlertSettings') +Metric = load_model('monitoring', 'Metric') +Check = load_model('check', 'Check') + + +class TestIperf3( + CreateConnectionsMixin, TestDeviceMonitoringMixin, TransactionTestCase +): + + _IPERF3 = app_settings.CHECK_CLASSES[2][0] + _RESULT_KEYS = [ + 'iperf3_result', + 'sent_bps_tcp', + 'received_bps_tcp', + 'sent_bytes_tcp', + 'received_bytes_tcp', + 'retransmits', + 'sent_bps_udp', + 'sent_bytes_udp', + 'jitter', + 'total_packets', + 'lost_packets', + 'lost_percent', + ] + _IPERF3_TEST_SERVER = ['iperf3.openwisptestserver.com'] + _IPERF3_TEST_MULTIPLE_SERVERS = [ + 'iperf3.openwisptestserver1.com', + 'iperf3.openwisptestserver2.com', + ] + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.mock_ssh_server = SshServer( + {'root': cls._TEST_RSA_PRIVATE_KEY_PATH} + ).__enter__() + cls.ssh_server.port = cls.mock_ssh_server.port + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + cls.mock_ssh_server.__exit__() + app_settings.IPERF3_CHECK_CONFIG = {} + + def setUp(self): + ckey = self._create_credentials_with_key(port=self.ssh_server.port) + self.dc = self._create_device_connection(credentials=ckey) + self.device = self.dc.device + self.org_id = str(self.device.organization.id) + self.dc.connect() + app_settings.IPERF3_CHECK_CONFIG = { + self.org_id: {'host': self._IPERF3_TEST_SERVER} + } + self._EXPECTED_COMMAND_CALLS = [ + call( + ( + 'iperf3 -c iperf3.openwisptestserver.com -p 5201 -t 10 --connect-timeout 1 ' + '-b 0 -l 128K -w 0 -P 1 -J' + ), + raise_unexpected_exit=False, + ), + call( + ( + 'iperf3 -c iperf3.openwisptestserver.com -p 5201 -t 10 --connect-timeout 1 ' + '-b 30M -l 0 -w 0 -P 1 -u -J' + ), + raise_unexpected_exit=False, + ), + ] + self._EXPECTED_WARN_CALLS = [ + call( + ( + f'Iperf3 check failed for "{self.device}", ' + 'error - unable to connect to server: Connection refused' + ) + ), + call( + ( + f'Iperf3 check failed for "{self.device}", ' + 'error - unable to connect to server: Connection refused' + ) + ), + ] + + def _perform_iperf3_check(self): + check = Check.objects.get(check_type=self._IPERF3) + return check.perform_check(store=False) + + def _set_auth_expected_calls(self, config): + password = config[self.org_id]['password'] + username = config[self.org_id]['username'] + server = 'iperf3.openwisptestserver.com' + test_prefix = '-----BEGIN PUBLIC KEY-----\n' + test_suffix = '\n-----END PUBLIC KEY-----' + key = config[self.org_id]['rsa_public_key'] + rsa_key_path = '/tmp/iperf3-public-key.pem' + + self._EXPECTED_COMMAND_CALLS = [ + call( + ( + f'echo "{test_prefix}{key}{test_suffix}" > {rsa_key_path} && ' + f'IPERF3_PASSWORD="{password}" iperf3 -c {server} -p 5201 -t 10 ' + f'--username "{username}" --rsa-public-key-path {rsa_key_path} --connect-timeout 1 ' + f'-b 0 -l 128K -w 0 -P 1 -J' + ), + raise_unexpected_exit=False, + ), + call( + ( + f'IPERF3_PASSWORD="{password}" iperf3 -c {server} -p 5201 -t 10 ' + f'--username "{username}" --rsa-public-key-path {rsa_key_path} --connect-timeout 1 ' + f'-b 30M -l 0 -w 0 -P 1 -u -J ' + f'&& rm -f {rsa_key_path}' + ), + raise_unexpected_exit=False, + ), + ] + + def _assert_iperf3_fail_result(self, result): + for key in self._RESULT_KEYS: + self.assertIn(key, result) + self.assertEqual(result['iperf3_result'], 0) + self.assertEqual(result['sent_bps_tcp'], 0.0) + self.assertEqual(result['received_bps_tcp'], 0.0) + self.assertEqual(result['sent_bytes_tcp'], 0) + self.assertEqual(result['received_bytes_tcp'], 0) + self.assertEqual(result['retransmits'], 0) + self.assertEqual(result['sent_bps_udp'], 0.0) + self.assertEqual(result['sent_bytes_udp'], 0) + self.assertEqual(result['jitter'], 0.0) + self.assertEqual(result['total_packets'], 0) + self.assertEqual(result['lost_percent'], 0.0) + + @patch.object(Ssh, 'exec_command') + @patch.object(iperf3_logger, 'warning') + def test_iperf3_check_no_params(self, mock_warn, mock_exec_command): + mock_exec_command.side_effect = [(RESULT_TCP, 0), (RESULT_UDP, 0)] + # By default check params {} + tcp_result = loads(RESULT_TCP)['end'] + udp_result = loads(RESULT_UDP)['end']['sum'] + result = self._perform_iperf3_check() + for key in self._RESULT_KEYS: + self.assertIn(key, result) + self.assertEqual(result['iperf3_result'], 1) + self.assertEqual( + result['sent_bps_tcp'], tcp_result['sum_sent']['bits_per_second'] + ) + self.assertEqual( + result['received_bytes_tcp'], tcp_result['sum_received']['bytes'] + ) + self.assertEqual(result['jitter'], udp_result['jitter_ms']) + self.assertEqual(result['total_packets'], udp_result['packets']) + self.assertEqual(mock_warn.call_count, 0) + self.assertEqual(mock_exec_command.call_count, 2) + mock_exec_command.assert_has_calls(self._EXPECTED_COMMAND_CALLS) + + @patch.object(Ssh, 'exec_command') + @patch.object(iperf3_logger, 'warning') + def test_iperf3_check_params(self, mock_warn, mock_exec_command): + mock_exec_command.side_effect = [(RESULT_TCP, 0), (RESULT_UDP, 0)] + check = Check.objects.get(check_type=self._IPERF3) + tcp_result = loads(RESULT_TCP)['end'] + udp_result = loads(RESULT_UDP)['end']['sum'] + server = self._IPERF3_TEST_SERVER[0] + test_prefix = '-----BEGIN PUBLIC KEY-----\n' + test_suffix = '\n-----END PUBLIC KEY-----' + rsa_key_path = '/tmp/test-rsa.pem' + test_params = { + 'username': 'openwisp-test-user', + 'password': 'openwisp_pass', + 'rsa_public_key': TEST_RSA_KEY, + 'client_options': { + 'port': 6201, + 'time': 20, + 'window': '300K', + 'parallel': 5, + 'reverse': True, + 'connect_timeout': 1000, + 'tcp': {'bitrate': '10M', 'length': '128K'}, + 'udp': {'bitrate': '50M', 'length': '400K'}, + }, + } + time = test_params['client_options']['time'] + port = test_params['client_options']['port'] + window = test_params['client_options']['window'] + parallel = test_params['client_options']['parallel'] + tcp_bitrate = test_params['client_options']['tcp']['bitrate'] + tcp_len = test_params['client_options']['tcp']['length'] + udp_bitrate = test_params['client_options']['udp']['bitrate'] + udp_len = test_params['client_options']['udp']['length'] + username = test_params['username'] + password = test_params['password'] + key = test_params['rsa_public_key'] + rsa_key_path = '/tmp/iperf3-public-key.pem' + check.params = test_params + check.save() + self._EXPECTED_COMMAND_CALLS = [ + call( + ( + f'echo "{test_prefix}{key}{test_suffix}" > {rsa_key_path} && ' + f'IPERF3_PASSWORD="{password}" iperf3 -c {server} -p {port} -t {time} ' + f'--username "{username}" --rsa-public-key-path {rsa_key_path} --connect-timeout 1000 ' + f'-b {tcp_bitrate} -l {tcp_len} -w {window} -P {parallel} --reverse -J' + ), + raise_unexpected_exit=False, + ), + call( + ( + f'IPERF3_PASSWORD="{password}" iperf3 -c {server} -p {port} -t {time} ' + f'--username "{username}" --rsa-public-key-path {rsa_key_path} --connect-timeout 1000 ' + f'-b {udp_bitrate} -l {udp_len} -w {window} -P {parallel} --reverse -u -J ' + f'&& rm -f {rsa_key_path}' + ), + raise_unexpected_exit=False, + ), + ] + result = self._perform_iperf3_check() + for key in self._RESULT_KEYS: + self.assertIn(key, result) + self.assertEqual(result['iperf3_result'], 1) + self.assertEqual( + result['sent_bps_tcp'], tcp_result['sum_sent']['bits_per_second'] + ) + self.assertEqual( + result['received_bytes_tcp'], tcp_result['sum_received']['bytes'] + ) + self.assertEqual(result['jitter'], udp_result['jitter_ms']) + self.assertEqual(result['total_packets'], udp_result['packets']) + self.assertEqual(mock_warn.call_count, 0) + self.assertEqual(mock_exec_command.call_count, 2) + mock_exec_command.assert_has_calls(self._EXPECTED_COMMAND_CALLS) + + @patch.object(Ssh, 'exec_command') + @patch.object(iperf3_logger, 'warning') + def test_iperf3_check_config(self, mock_warn, mock_exec_command): + mock_exec_command.side_effect = [(RESULT_TCP, 0), (RESULT_UDP, 0)] + tcp_result = loads(RESULT_TCP)['end'] + udp_result = loads(RESULT_UDP)['end']['sum'] + self._EXPECTED_COMMAND_CALLS = [ + call( + ( + 'iperf3 -c iperf3.openwisptestserver.com -p 9201 -k 1M --connect-timeout 2000 ' + '-b 10M -l 512K -w 0 -P 1 --bidir -J' + ), + raise_unexpected_exit=False, + ), + call( + ( + 'iperf3 -c iperf3.openwisptestserver.com -p 9201 -k 1M --connect-timeout 2000 ' + '-b 50M -l 256K -w 0 -P 1 --bidir -u -J' + ), + raise_unexpected_exit=False, + ), + ] + iperf3_config = { + self.org_id: { + 'host': ['iperf3.openwisptestserver.com'], + 'client_options': { + 'port': 9201, + 'time': 120, + 'connect_timeout': 2000, + 'bytes': '20M', + 'blockcount': '1M', + 'bidirectional': True, + 'tcp': {'bitrate': '10M', 'length': '512K'}, + 'udp': {'bitrate': '50M', 'length': '256K'}, + }, + } + } + with patch.object(app_settings, 'IPERF3_CHECK_CONFIG', iperf3_config): + with patch.object(Iperf3, 'schema', get_iperf3_schema()): + result = self._perform_iperf3_check() + for key in self._RESULT_KEYS: + self.assertIn(key, result) + self.assertEqual(result['iperf3_result'], 1) + self.assertEqual( + result['sent_bps_tcp'], tcp_result['sum_sent']['bits_per_second'] + ) + self.assertEqual( + result['received_bytes_tcp'], tcp_result['sum_received']['bytes'] + ) + self.assertEqual(result['jitter'], udp_result['jitter_ms']) + self.assertEqual(result['total_packets'], udp_result['packets']) + self.assertEqual(mock_warn.call_count, 0) + self.assertEqual(mock_exec_command.call_count, 2) + mock_exec_command.assert_has_calls(self._EXPECTED_COMMAND_CALLS) + + @patch.object(iperf3_logger, 'warning') + def test_iperf3_device_connection(self, mock_warn): + dc = self.dc + with self.subTest('Test active device connection when management tunnel down'): + with patch.object( + device_connection, 'connect', return_value=False + ) as mocked_connect: + self._perform_iperf3_check() + mock_warn.assert_called_with( + f'DeviceConnection for "{self.device}" is not working, iperf3 check skipped!' + ) + self.assertEqual(mocked_connect.call_count, 1) + + with self.subTest('Test device connection is not enabled'): + dc.enabled = False + dc.save() + self._perform_iperf3_check() + mock_warn.assert_called_with( + f'Failed to get a working DeviceConnection for "{self.device}", iperf3 check skipped!' + ) + + with self.subTest('Test device connection is not with right update strategy'): + dc.update_strategy = UPDATE_STRATEGIES[1][0] + dc.is_working = True + dc.enabled = True + dc.save() + self._perform_iperf3_check() + mock_warn.assert_called_with( + f'Failed to get a working DeviceConnection for "{self.device}", iperf3 check skipped!' + ) + + def test_iperf3_check_content_object_none(self): + check = Check(name='Iperf3 check', check_type=self._IPERF3, params={}) + try: + check.check_instance.validate() + except ValidationError as e: + self.assertIn('device', str(e)) + else: + self.fail('ValidationError not raised') + + def test_iperf3_check_content_object_not_device(self): + check = Check( + name='Iperf3 check', + check_type=self._IPERF3, + content_object=self._create_user(), + params={}, + ) + try: + check.check_instance.validate() + except ValidationError as e: + self.assertIn('device', str(e)) + else: + self.fail('ValidationError not raised') + + def test_iperf3_check_schema_violation(self): + for invalid_param in INVALID_PARAMS: + check = Check( + name='Iperf3 check', + check_type=self._IPERF3, + content_object=self.device, + params=invalid_param, + ) + try: + check.check_instance.validate() + except ValidationError as e: + self.assertIn('Invalid param', str(e)) + else: + self.fail('ValidationError not raised') + + @patch.object(Ssh, 'exec_command') + @patch.object(iperf3_logger, 'warning') + def test_iperf3_check(self, mock_warn, mock_exec_command): + error = "ash: iperf3: not found" + tcp_result = loads(RESULT_TCP)['end'] + udp_result = loads(RESULT_UDP)['end']['sum'] + iperf3_json_error_config = { + self.org_id: { + 'host': ['iperf3.openwisptestserver.com'], + 'username': 'test', + 'password': 'testpass', + 'rsa_public_key': 'INVALID_RSA_KEY', + } + } + with patch.object( + app_settings, 'IPERF3_CHECK_CONFIG', iperf3_json_error_config + ): + with self.subTest('Test iperf3 errors not in json format'): + mock_exec_command.side_effect = [(PARAM_ERROR, 1), (PARAM_ERROR, 1)] + EXPECTED_WARN_CALLS = [ + call( + f'Iperf3 check failed for "{self.device}", error - {PARAM_ERROR}' + ), + call( + f'Iperf3 check failed for "{self.device}", error - {PARAM_ERROR}' + ), + ] + self._perform_iperf3_check() + self.assertEqual(mock_warn.call_count, 2) + self.assertEqual(mock_exec_command.call_count, 2) + mock_warn.assert_has_calls(EXPECTED_WARN_CALLS) + mock_warn.reset_mock() + mock_exec_command.reset_mock() + + with self.subTest('Test iperf3 is not installed on the device'): + mock_exec_command.side_effect = [(error, 127)] + self._perform_iperf3_check() + mock_warn.assert_called_with( + f'Iperf3 is not installed on the "{self.device}", error - {error}' + ) + self.assertEqual(mock_warn.call_count, 1) + self.assertEqual(mock_exec_command.call_count, 1) + mock_warn.reset_mock() + mock_exec_command.reset_mock() + + with self.subTest('Test iperf3 check passes in both TCP & UDP'): + mock_exec_command.side_effect = [(RESULT_TCP, 0), (RESULT_UDP, 0)] + self.assertEqual(Chart.objects.count(), 2) + self.assertEqual(Metric.objects.count(), 2) + result = self._perform_iperf3_check() + for key in self._RESULT_KEYS: + self.assertIn(key, result) + self.assertEqual(result['iperf3_result'], 1) + self.assertEqual( + result['sent_bps_tcp'], tcp_result['sum_sent']['bits_per_second'] + ) + self.assertEqual( + result['received_bps_tcp'], + tcp_result['sum_received']['bits_per_second'], + ) + self.assertEqual(result['sent_bytes_tcp'], tcp_result['sum_sent']['bytes']) + self.assertEqual( + result['received_bytes_tcp'], tcp_result['sum_received']['bytes'] + ) + self.assertEqual( + result['retransmits'], tcp_result['sum_sent']['retransmits'] + ) + self.assertEqual(result['sent_bps_udp'], udp_result['bits_per_second']) + self.assertEqual(result['sent_bytes_udp'], udp_result['bytes']) + self.assertEqual(result['jitter'], udp_result['jitter_ms']) + self.assertEqual(result['total_packets'], udp_result['packets']) + self.assertEqual(result['lost_percent'], udp_result['lost_percent']) + self.assertEqual(Chart.objects.count(), 8) + self.assertEqual(Check.objects.count(), 3) + iperf3_metric = Metric.objects.get(key='iperf3') + self.assertEqual(Metric.objects.count(), 3) + self.assertEqual(iperf3_metric.content_object, self.device) + points = iperf3_metric.read(limit=None, extra_fields=list(result.keys())) + self.assertEqual(len(points), 1) + self.assertEqual(points[0]['iperf3_result'], result['iperf3_result']) + self.assertEqual(points[0]['sent_bps_tcp'], result['sent_bps_tcp']) + self.assertEqual( + points[0]['received_bytes_tcp'], result['received_bytes_tcp'] + ) + self.assertEqual(points[0]['retransmits'], result['retransmits']) + self.assertEqual(points[0]['sent_bps_udp'], result['sent_bps_udp']) + self.assertEqual(points[0]['sent_bytes_udp'], result['sent_bytes_udp']) + self.assertEqual(points[0]['jitter'], result['jitter']) + self.assertEqual(points[0]['total_packets'], result['total_packets']) + self.assertEqual(points[0]['lost_packets'], result['lost_packets']) + self.assertEqual(points[0]['lost_percent'], result['lost_percent']) + self.assertEqual(mock_warn.call_count, 0) + self.assertEqual(mock_exec_command.call_count, 2) + mock_exec_command.assert_has_calls(self._EXPECTED_COMMAND_CALLS) + mock_warn.reset_mock() + mock_exec_command.reset_mock() + + with self.subTest('Test iperf3 check fails in both TCP & UDP'): + mock_exec_command.side_effect = [(RESULT_FAIL, 1), (RESULT_FAIL, 1)] + result = self._perform_iperf3_check() + self._assert_iperf3_fail_result(result) + self.assertEqual(Chart.objects.count(), 8) + self.assertEqual(Metric.objects.count(), 3) + self.assertEqual(mock_warn.call_count, 2) + self.assertEqual(mock_exec_command.call_count, 2) + mock_warn.assert_has_calls(self._EXPECTED_WARN_CALLS) + mock_exec_command.assert_has_calls(self._EXPECTED_COMMAND_CALLS) + mock_warn.reset_mock() + mock_exec_command.reset_mock() + + with self.subTest('Test iperf3 check TCP pass UDP fail'): + mock_exec_command.side_effect = [(RESULT_TCP, 0), (RESULT_FAIL, 1)] + result = self._perform_iperf3_check() + for key in self._RESULT_KEYS: + self.assertIn(key, result) + self.assertEqual(result['iperf3_result'], 1) + self.assertEqual( + result['sent_bps_tcp'], tcp_result['sum_sent']['bits_per_second'] + ) + self.assertEqual( + result['received_bps_tcp'], + tcp_result['sum_received']['bits_per_second'], + ) + self.assertEqual(result['sent_bytes_tcp'], tcp_result['sum_sent']['bytes']) + self.assertEqual( + result['received_bytes_tcp'], tcp_result['sum_received']['bytes'] + ) + self.assertEqual( + result['retransmits'], tcp_result['sum_sent']['retransmits'] + ) + self.assertEqual(result['sent_bps_udp'], 0.0) + self.assertEqual(result['sent_bytes_udp'], 0) + self.assertEqual(result['jitter'], 0.0) + self.assertEqual(result['total_packets'], 0) + self.assertEqual(result['lost_percent'], 0.0) + self.assertEqual(Chart.objects.count(), 8) + self.assertEqual(Metric.objects.count(), 3) + self.assertEqual(mock_warn.call_count, 1) + self.assertEqual(mock_exec_command.call_count, 2) + mock_warn.assert_has_calls(self._EXPECTED_WARN_CALLS[1:]) + mock_exec_command.assert_has_calls(self._EXPECTED_COMMAND_CALLS) + mock_warn.reset_mock() + mock_exec_command.reset_mock() + + with self.subTest('Test iperf3 check TCP fail UDP pass'): + mock_exec_command.side_effect = [(RESULT_FAIL, 1), (RESULT_UDP, 0)] + result = self._perform_iperf3_check() + for key in self._RESULT_KEYS: + self.assertIn(key, result) + self.assertEqual(result['iperf3_result'], 1) + self.assertEqual(result['sent_bps_tcp'], 0.0) + self.assertEqual(result['received_bps_tcp'], 0.0) + self.assertEqual(result['sent_bytes_tcp'], 0) + self.assertEqual(result['received_bytes_tcp'], 0) + self.assertEqual(result['retransmits'], 0) + self.assertEqual(result['sent_bps_udp'], udp_result['bits_per_second']) + self.assertEqual(result['sent_bytes_udp'], udp_result['bytes']) + self.assertEqual(result['jitter'], udp_result['jitter_ms']) + self.assertEqual(result['total_packets'], udp_result['packets']) + self.assertEqual(result['lost_percent'], udp_result['lost_percent']) + self.assertEqual(Chart.objects.count(), 8) + self.assertEqual(Metric.objects.count(), 3) + self.assertEqual(mock_warn.call_count, 1) + self.assertEqual(mock_exec_command.call_count, 2) + mock_warn.assert_has_calls(self._EXPECTED_WARN_CALLS[1:]) + mock_exec_command.assert_has_calls(self._EXPECTED_COMMAND_CALLS) + + @patch.object(Ssh, 'exec_command') + @patch.object(iperf3_logger, 'warning') + def test_iperf3_check_auth_config(self, mock_warn, mock_exec_command): + iperf3_config = { + self.org_id: { + 'host': self._IPERF3_TEST_SERVER, + 'username': 'test', + 'password': 'testpass', + 'rsa_public_key': TEST_RSA_KEY, + } + } + iperf3_conf_wrong_pass = { + self.org_id: { + 'host': self._IPERF3_TEST_SERVER, + 'username': 'test', + 'password': 'wrongpass', + 'rsa_public_key': TEST_RSA_KEY, + } + } + iperf3_conf_wrong_user = { + self.org_id: { + 'host': self._IPERF3_TEST_SERVER, + 'username': 'wronguser', + 'password': 'testpass', + 'rsa_public_key': TEST_RSA_KEY, + } + } + auth_error = "test authorization failed" + tcp_result = loads(RESULT_TCP)['end'] + udp_result = loads(RESULT_UDP)['end']['sum'] + + self._EXPECTED_WARN_CALLS = [ + call(f'Iperf3 check failed for "{self.device}", error - {auth_error}'), + call(f'Iperf3 check failed for "{self.device}", error - {auth_error}'), + ] + with self.subTest('Test iperf3 check with right config'): + with patch.object( + app_settings, + 'IPERF3_CHECK_CONFIG', + iperf3_config + # It is required to mock "Iperf3.schema" here so that it + # uses the updated configuration from "IPERF3_CHECK_CONFIG" setting. + ), patch.object(Iperf3, 'schema', get_iperf3_schema()): + self._set_auth_expected_calls(iperf3_config) + mock_exec_command.side_effect = [(RESULT_TCP, 0), (RESULT_UDP, 0)] + result = self._perform_iperf3_check() + for key in self._RESULT_KEYS: + self.assertIn(key, result) + self.assertEqual(result['iperf3_result'], 1) + self.assertEqual( + result['sent_bps_tcp'], tcp_result['sum_sent']['bits_per_second'] + ) + self.assertEqual( + result['received_bytes_tcp'], tcp_result['sum_received']['bytes'] + ) + self.assertEqual(result['jitter'], udp_result['jitter_ms']) + self.assertEqual(result['total_packets'], udp_result['packets']) + self.assertEqual(mock_warn.call_count, 0) + self.assertEqual(mock_exec_command.call_count, 2) + mock_exec_command.assert_has_calls(self._EXPECTED_COMMAND_CALLS) + mock_warn.reset_mock() + mock_exec_command.reset_mock() + + with self.subTest('Test iperf3 check with wrong password'): + with patch.object( + app_settings, 'IPERF3_CHECK_CONFIG', iperf3_conf_wrong_pass + ), patch.object(Iperf3, 'schema', get_iperf3_schema()): + self._set_auth_expected_calls(iperf3_conf_wrong_pass) + mock_exec_command.side_effect = [ + (RESULT_AUTH_FAIL, 1), + (RESULT_AUTH_FAIL, 1), + ] + + result = self._perform_iperf3_check() + self._assert_iperf3_fail_result(result) + self.assertEqual(mock_warn.call_count, 2) + self.assertEqual(mock_exec_command.call_count, 2) + mock_warn.assert_has_calls(self._EXPECTED_WARN_CALLS) + mock_exec_command.assert_has_calls(self._EXPECTED_COMMAND_CALLS) + mock_warn.reset_mock() + mock_exec_command.reset_mock() + + with self.subTest('Test iperf3 check with wrong username'): + with patch.object( + app_settings, 'IPERF3_CHECK_CONFIG', iperf3_conf_wrong_user + ), patch.object(Iperf3, 'schema', get_iperf3_schema()): + self._set_auth_expected_calls(iperf3_conf_wrong_user) + mock_exec_command.side_effect = [ + (RESULT_AUTH_FAIL, 1), + (RESULT_AUTH_FAIL, 1), + ] + + result = self._perform_iperf3_check() + self._assert_iperf3_fail_result(result) + self.assertEqual(mock_warn.call_count, 2) + self.assertEqual(mock_exec_command.call_count, 2) + mock_warn.assert_has_calls(self._EXPECTED_WARN_CALLS) + mock_exec_command.assert_has_calls(self._EXPECTED_COMMAND_CALLS) + + @patch.object(Ssh, 'exec_command') + @patch.object(iperf3_logger, 'warning') + @patch.object(iperf3_logger, 'info') + @patch.object(cache, 'add') + def test_iperf3_check_task_with_multiple_server_config(self, *args): + mock_add = args[0] + mock_info = args[1] + mock_warn = args[2] + mock_exec_command = args[3] + org = self.device.organization + iperf3_multiple_server_config = { + self.org_id: {'host': self._IPERF3_TEST_MULTIPLE_SERVERS} + } + check = Check.objects.get(check_type=self._IPERF3) + + self._EXPECTED_COMMAND_CALLS_SERVER_1 = [ + call( + ( + f'iperf3 -c {self._IPERF3_TEST_MULTIPLE_SERVERS[0]} -p 5201 -t 10 --connect-timeout 1 ' + '-b 0 -l 128K -w 0 -P 1 -J' + ), + raise_unexpected_exit=False, + ), + call( + ( + f'iperf3 -c {self._IPERF3_TEST_MULTIPLE_SERVERS[0]} -p 5201 -t 10 --connect-timeout 1 ' + '-b 30M -l 0 -w 0 -P 1 -u -J' + ), + raise_unexpected_exit=False, + ), + ] + self._EXPECTED_COMMAND_CALLS_SERVER_2 = [ + call( + ( + f'iperf3 -c {self._IPERF3_TEST_MULTIPLE_SERVERS[1]} -p 5201 -t 10 --connect-timeout 1 ' + '-b 0 -l 128K -w 0 -P 1 -J' + ), + raise_unexpected_exit=False, + ), + call( + ( + f'iperf3 -c {self._IPERF3_TEST_MULTIPLE_SERVERS[1]} -p 5201 -t 10 --connect-timeout 1 ' + '-b 30M -l 0 -w 0 -P 1 -u -J' + ), + raise_unexpected_exit=False, + ), + ] + + with patch.object(app_settings, 'IPERF3_CHECK_CONFIG', {}): + with self.subTest('Test iperf3 check without config'): + self._perform_iperf3_check() + mock_warn.assert_called_with( + ( + f'Iperf3 servers for organization "{org}" ' + f'is not configured properly, iperf3 check skipped!' + ) + ) + self.assertEqual(mock_warn.call_count, 1) + mock_warn.reset_mock() + + with patch.object( + app_settings, + 'IPERF3_CHECK_CONFIG', + {'invalid_org_uuid': {'host': self._IPERF3_TEST_SERVER, 'time': 10}}, + ): + with self.subTest('Test iperf3 check with invalid config'): + self._perform_iperf3_check() + mock_warn.assert_called_with( + ( + f'Iperf3 servers for organization "{org}" ' + f'is not configured properly, iperf3 check skipped!' + ) + ) + self.assertEqual(mock_warn.call_count, 1) + mock_warn.reset_mock() + + with patch.object( + app_settings, 'IPERF3_CHECK_CONFIG', iperf3_multiple_server_config + ): + with self.subTest( + 'Test iperf3 check when all iperf3 servers are available' + ): + mock_add.return_value = True + mock_exec_command.side_effect = [(RESULT_TCP, 0), (RESULT_UDP, 0)] + self._perform_iperf3_check() + self.assertEqual(mock_warn.call_count, 0) + self.assertEqual(mock_add.call_count, 1) + self.assertEqual(mock_exec_command.call_count, 2) + mock_exec_command.assert_has_calls( + self._EXPECTED_COMMAND_CALLS_SERVER_1 + ) + mock_add.reset_mock() + mock_warn.reset_mock() + mock_exec_command.reset_mock() + + with self.subTest( + 'Test iperf3 check when single iperf3 server are available' + ): + mock_add.side_effect = [False, True] + mock_exec_command.side_effect = [(RESULT_TCP, 0), (RESULT_UDP, 0)] + self._perform_iperf3_check() + self.assertEqual(mock_warn.call_count, 0) + self.assertEqual(mock_add.call_count, 2) + self.assertEqual(mock_exec_command.call_count, 2) + mock_exec_command.assert_has_calls( + self._EXPECTED_COMMAND_CALLS_SERVER_2 + ) + mock_add.reset_mock() + mock_warn.reset_mock() + mock_exec_command.reset_mock() + + with self.subTest( + 'Test iperf3 check when all iperf3 servers are occupied initially' + ): + # If all available iperf3 servers are occupied initially, + # then push the task back in the queue and acquire the iperf3 + # server only after completion of previous running checks + mock_add.side_effect = [False, False, True] + mock_exec_command.side_effect = [(RESULT_TCP, 0), (RESULT_UDP, 0)] + self._perform_iperf3_check() + mock_info.has_called_with( + ( + f'At the moment, all available iperf3 servers of organization "{org}" ' + f'are busy running checks, putting "{check}" back in the queue..' + ) + ) + self.assertEqual(mock_info.call_count, 4) + self.assertEqual(mock_add.call_count, 3) + self.assertEqual(mock_exec_command.call_count, 2) + mock_exec_command.assert_has_calls( + self._EXPECTED_COMMAND_CALLS_SERVER_1 + ) diff --git a/openwisp_monitoring/check/tests/test_models.py b/openwisp_monitoring/check/tests/test_models.py index abbf8ed13..93ae7a98a 100644 --- a/openwisp_monitoring/check/tests/test_models.py +++ b/openwisp_monitoring/check/tests/test_models.py @@ -9,8 +9,8 @@ from ...device.tests import TestDeviceMonitoringMixin from .. import settings as app_settings -from ..classes import ConfigApplied, Ping -from ..tasks import auto_create_config_check, auto_create_ping +from ..classes import ConfigApplied, Iperf3, Ping +from ..tasks import auto_create_config_check, auto_create_iperf3_check, auto_create_ping Check = load_model('check', 'Check') Metric = load_model('monitoring', 'Metric') @@ -22,6 +22,7 @@ class TestModels(TestDeviceMonitoringMixin, TransactionTestCase): _PING = app_settings.CHECK_CLASSES[0][0] _CONFIG_APPLIED = app_settings.CHECK_CLASSES[1][0] + _IPERF3 = app_settings.CHECK_CLASSES[2][0] def test_check_str(self): c = Check(name='Test check') @@ -48,6 +49,12 @@ def test_check_class(self): check_type=self._CONFIG_APPLIED, ) self.assertEqual(c.check_class, ConfigApplied) + with self.subTest('Test Iperf3 check Class'): + c = Check( + name='Iperf3 class check', + check_type=self._IPERF3, + ) + self.assertEqual(c.check_class, Iperf3) def test_base_check_class(self): path = 'openwisp_monitoring.check.classes.base.BaseCheck' @@ -82,6 +89,18 @@ def test_check_instance(self): self.assertEqual(i.related_object, obj) self.assertEqual(i.params, c.params) + with self.subTest('Test Iperf3 check instance'): + c = Check( + name='Iperf3 class check', + check_type=self._IPERF3, + content_object=obj, + params={}, + ) + i = c.check_instance + self.assertIsInstance(i, Iperf3) + self.assertEqual(i.related_object, obj) + self.assertEqual(i.params, c.params) + def test_validation(self): with self.subTest('Test Ping check validation'): check = Check(name='Ping check', check_type=self._PING, params={}) @@ -105,7 +124,7 @@ def test_validation(self): def test_auto_check_creation(self): self.assertEqual(Check.objects.count(), 0) d = self._create_device(organization=self._create_org()) - self.assertEqual(Check.objects.count(), 2) + self.assertEqual(Check.objects.count(), 3) with self.subTest('Test AUTO_PING'): c1 = Check.objects.filter(check_type=self._PING).first() self.assertEqual(c1.content_object, d) @@ -114,11 +133,15 @@ def test_auto_check_creation(self): c2 = Check.objects.filter(check_type=self._CONFIG_APPLIED).first() self.assertEqual(c2.content_object, d) self.assertEqual(self._CONFIG_APPLIED, c2.check_type) + with self.subTest('Test AUTO_IPERF3'): + c3 = Check.objects.filter(check_type=self._IPERF3).first() + self.assertEqual(c3.content_object, d) + self.assertEqual(self._IPERF3, c3.check_type) def test_device_deleted(self): self.assertEqual(Check.objects.count(), 0) d = self._create_device(organization=self._create_org()) - self.assertEqual(Check.objects.count(), 2) + self.assertEqual(Check.objects.count(), 3) d.delete() self.assertEqual(Check.objects.count(), 0) @@ -129,7 +152,7 @@ def test_config_modified_device_problem(self): self._create_config(status='modified', organization=self._create_org()) d = Device.objects.first() d.monitoring.update_status('ok') - self.assertEqual(Check.objects.count(), 2) + self.assertEqual(Check.objects.count(), 3) self.assertEqual(Metric.objects.count(), 0) self.assertEqual(AlertSettings.objects.count(), 0) check = Check.objects.filter(check_type=self._CONFIG_APPLIED).first() @@ -159,7 +182,7 @@ def test_config_error(self): self._create_config(status='error', organization=self._create_org()) dm = Device.objects.first().monitoring dm.update_status('ok') - self.assertEqual(Check.objects.count(), 2) + self.assertEqual(Check.objects.count(), 3) self.assertEqual(Metric.objects.count(), 0) self.assertEqual(AlertSettings.objects.count(), 0) check = Check.objects.filter(check_type=self._CONFIG_APPLIED).first() @@ -192,7 +215,7 @@ def test_config_error(self): @patch('openwisp_monitoring.check.settings.AUTO_PING', False) def test_config_check_critical_metric(self): self._create_config(status='modified', organization=self._create_org()) - self.assertEqual(Check.objects.count(), 2) + self.assertEqual(Check.objects.count(), 3) d = Device.objects.first() dm = d.monitoring dm.update_status('ok') @@ -211,7 +234,7 @@ def test_config_check_critical_metric(self): def test_no_duplicate_check_created(self): self._create_config(organization=self._create_org()) - self.assertEqual(Check.objects.count(), 2) + self.assertEqual(Check.objects.count(), 3) d = Device.objects.first() auto_create_config_check.delay( model=Device.__name__.lower(), @@ -223,13 +246,18 @@ def test_no_duplicate_check_created(self): app_label=Device._meta.app_label, object_id=str(d.pk), ) - self.assertEqual(Check.objects.count(), 2) + auto_create_iperf3_check.delay( + model=Device.__name__.lower(), + app_label=Device._meta.app_label, + object_id=str(d.pk), + ) + self.assertEqual(Check.objects.count(), 3) def test_device_unreachable_no_config_check(self): self._create_config(status='modified', organization=self._create_org()) d = self.device_model.objects.first() d.monitoring.update_status('critical') - self.assertEqual(Check.objects.count(), 2) + self.assertEqual(Check.objects.count(), 3) c2 = Check.objects.filter(check_type=self._CONFIG_APPLIED).first() c2.perform_check() self.assertEqual(Metric.objects.count(), 0) @@ -240,7 +268,7 @@ def test_device_unknown_no_config_check(self): self._create_config(status='modified', organization=self._create_org()) d = self.device_model.objects.first() d.monitoring.update_status('unknown') - self.assertEqual(Check.objects.count(), 2) + self.assertEqual(Check.objects.count(), 3) c2 = Check.objects.filter(check_type=self._CONFIG_APPLIED).first() c2.perform_check() self.assertEqual(Metric.objects.count(), 0) diff --git a/openwisp_monitoring/check/tests/test_ping.py b/openwisp_monitoring/check/tests/test_ping.py index 11b9ee47b..23b94652d 100644 --- a/openwisp_monitoring/check/tests/test_ping.py +++ b/openwisp_monitoring/check/tests/test_ping.py @@ -239,7 +239,7 @@ def test_store_result(self, mocked_method): device.management_ip = '10.40.0.1' device.save() # check created automatically by autoping - self.assertEqual(Check.objects.count(), 2) + self.assertEqual(Check.objects.count(), 3) self.assertEqual(Metric.objects.count(), 0) self.assertEqual(Chart.objects.count(), 0) self.assertEqual(AlertSettings.objects.count(), 0) diff --git a/openwisp_monitoring/db/backends/influxdb/queries.py b/openwisp_monitoring/db/backends/influxdb/queries.py index 23a4b8ee9..965200c76 100644 --- a/openwisp_monitoring/db/backends/influxdb/queries.py +++ b/openwisp_monitoring/db/backends/influxdb/queries.py @@ -98,6 +98,51 @@ "object_id = '{object_id}' GROUP BY time(1d)" ) }, + 'bandwidth': { + 'influxdb': ( + "SELECT MEAN(sent_bps_tcp) / 1000000000 AS TCP, " + "MEAN(sent_bps_udp) / 1000000000 AS UDP FROM {key} WHERE " + "time >= '{time}' AND content_type = '{content_type}' AND " + "object_id = '{object_id}' GROUP BY time(1d)" + ) + }, + 'transfer': { + 'influxdb': ( + "SELECT SUM(sent_bytes_tcp) / 1000000000 AS TCP," + "SUM(sent_bytes_udp) / 1000000000 AS UDP FROM {key} WHERE " + "time >= '{time}' AND content_type = '{content_type}' AND " + "object_id = '{object_id}' GROUP BY time(1d)" + ) + }, + 'retransmits': { + 'influxdb': ( + "SELECT MEAN(retransmits) AS retransmits FROM {key} " + "WHERE time >= '{time}' AND content_type = '{content_type}' " + "AND object_id = '{object_id}' GROUP BY time(1d)" + ) + }, + 'jitter': { + 'influxdb': ( + "SELECT MEAN(jitter) AS jitter FROM {key} " + "WHERE time >= '{time}' AND content_type = '{content_type}' " + "AND object_id = '{object_id}' GROUP BY time(1d)" + ) + }, + 'datagram': { + 'influxdb': ( + "SELECT MEAN(lost_packets) AS lost_datagram," + "MEAN(total_packets) AS total_datagram FROM {key} WHERE " + "time >= '{time}' AND content_type = '{content_type}' " + "AND object_id = '{object_id}' GROUP BY time(1d)" + ) + }, + 'datagram_loss': { + 'influxdb': ( + "SELECT MEAN(lost_percent) AS datagram_loss FROM {key} " + "WHERE time >= '{time}' AND content_type = '{content_type}' " + "AND object_id = '{object_id}' GROUP BY time(1d)" + ) + }, } default_chart_query = [ diff --git a/openwisp_monitoring/device/admin.py b/openwisp_monitoring/device/admin.py index e415e4732..cc92eea4c 100644 --- a/openwisp_monitoring/device/admin.py +++ b/openwisp_monitoring/device/admin.py @@ -48,26 +48,61 @@ def full_clean(self): obj = form.instance if not obj.content_type or not obj.object_id: setattr( - form.instance, + obj, self.ct_field.get_attname(), ContentType.objects.get_for_model(self.instance).pk, ) - setattr(form.instance, self.ct_fk_field.get_attname(), self.instance.pk) + setattr(obj, self.ct_fk_field.get_attname(), self.instance.pk) super().full_clean() -class CheckInline(GenericStackedInline): +class InlinePermissionMixin: + def has_add_permission(self, request, obj=None): + # User will be able to add objects from inline even + # if it only has permission to add a model object + return super().has_add_permission(request, obj) or request.user.has_perm( + f'{self.model._meta.app_label}.add_{self.inline_permission_suffix}' + ) + + def has_change_permission(self, request, obj=None): + return super().has_change_permission(request, obj) or request.user.has_perm( + f'{self.model._meta.app_label}.change_{self.inline_permission_suffix}' + ) + + def has_view_permission(self, request, obj=None): + return super().has_view_permission(request, obj) or request.user.has_perm( + f'{self.model._meta.app_label}.view_{self.inline_permission_suffix}' + ) + + def has_delete_permission(self, request, obj=None): + return super().has_delete_permission(request, obj) or request.user.has_perm( + f'{self.model._meta.app_label}.delete_{self.inline_permission_suffix}' + ) + + +class CheckInline(InlinePermissionMixin, GenericStackedInline): model = Check extra = 0 formset = CheckInlineFormSet - fields = ['check_type', 'is_active'] - readonly_fields = ['check_type'] + fields = [ + 'is_active', + 'check_type', + ] + inline_permission_suffix = 'check_inline' - def has_add_permission(self, request, obj=None): - return False + def get_fields(self, request, obj=None): + if not self.has_change_permission(request, obj) or not self.has_view_permission( + request, obj + ): + return ['check_type', 'is_active'] + return super().get_fields(request, obj) - def has_delete_permission(self, request, obj=None): - return False + def get_readonly_fields(self, request, obj=None): + if not self.has_change_permission(request, obj) or not self.has_view_permission( + request, obj + ): + return ['check_type'] + return super().get_readonly_fields(request, obj) class AlertSettingsForm(ModelForm): @@ -81,43 +116,92 @@ def __init__(self, *args, **kwargs): } super().__init__(*args, **kwargs) - -class AlertSettingsInline(NestedStackedInline): + def _post_clean(self): + self.instance._delete_instance = False + if all( + self.cleaned_data[field] is None + for field in [ + 'custom_operator', + 'custom_threshold', + 'custom_tolerance', + ] + ): + # "_delete_instance" flag signifies that + # the fields have been set to None by the + # user. Hence, the object should be deleted. + self.instance._delete_instance = True + super()._post_clean() + + def save(self, commit=True): + if self.instance._delete_instance: + self.instance.delete() + return self.instance + return super().save(commit) + + +class AlertSettingsInline(InlinePermissionMixin, NestedStackedInline): model = AlertSettings - extra = 0 - max_num = 0 + extra = 1 + max_num = 1 exclude = ['created', 'modified'] form = AlertSettingsForm + inline_permission_suffix = 'alertsettings_inline' def get_queryset(self, request): return super().get_queryset(request).order_by('created') - def has_add_permission(self, request, obj=None): - return False - - def has_delete_permission(self, request, obj=None): - return False - -class MetricInline(NestedGenericStackedInline): +class MetricInline(InlinePermissionMixin, NestedGenericStackedInline): model = Metric extra = 0 inlines = [AlertSettingsInline] + fieldsets = [ + ( + None, + { + 'fields': ( + 'name', + 'is_healthy', + ) + }, + ), + ( + _('Advanced options'), + {'classes': ('collapse',), 'fields': ('field_name',)}, + ), + ] + readonly_fields = ['name', 'is_healthy'] - fields = ['name', 'is_healthy'] # Explicitly changed name from Metrics to Alert Settings verbose_name = _('Alert Settings') verbose_name_plural = verbose_name + inline_permission_suffix = 'alertsettings_inline' + # Ordering queryset by metric name + ordering = ('name',) + + def get_fieldsets(self, request, obj=None): + if not self.has_change_permission(request, obj) or not self.has_view_permission( + request, obj + ): + return [ + (None, {'fields': ('is_healthy',)}), + ] + return super().get_fieldsets(request, obj) + + def get_queryset(self, request): + # Only show 'Metrics' that have 'AlertSettings' objects + return super().get_queryset(request).filter(alertsettings__isnull=False) def has_add_permission(self, request, obj=None): + # We need to restrict the users from adding the 'metrics' since + # they're created by the system automatically with default 'alertsettings' return False def has_delete_permission(self, request, obj=None): + # We need to restrict the users from deleting the 'metrics' since + # they're created by the system automatically with default 'alertsettings' return False - def get_queryset(self, request): - return super().get_queryset(request).filter(alertsettings__isnull=False) - class DeviceAdmin(BaseDeviceAdmin, NestedModelAdmin): change_form_template = 'admin/config/device/change_form.html' @@ -220,6 +304,7 @@ def get_inlines(self, request, obj=None): if not hasattr(inline, 'sortable_options'): inline.sortable_options = {'disabled': True} if not obj or obj._state.adding: + inlines.remove(CheckInline) inlines.remove(MetricInline) return inlines diff --git a/openwisp_monitoring/device/tests/test_admin.py b/openwisp_monitoring/device/tests/test_admin.py index c2bdf1b9a..586df4610 100644 --- a/openwisp_monitoring/device/tests/test_admin.py +++ b/openwisp_monitoring/device/tests/test_admin.py @@ -1,6 +1,7 @@ from copy import deepcopy from django.contrib.auth import get_user_model +from django.contrib.auth.models import Permission from django.contrib.contenttypes.forms import generic_inlineformset_factory from django.test import TestCase from django.urls import reverse @@ -19,6 +20,7 @@ Chart = load_model('monitoring', 'Chart') Metric = load_model('monitoring', 'Metric') +AlertSettings = load_model('monitoring', 'AlertSettings') DeviceData = load_model('device_monitoring', 'DeviceData') WifiClient = load_model('device_monitoring', 'WifiClient') WifiSession = load_model('device_monitoring', 'WifiSession') @@ -28,6 +30,9 @@ Device = load_model('config', 'Device') DeviceLocation = load_model('geo', 'DeviceLocation') Location = load_model('geo', 'Location') +# model_name changes if swapped +check_model_name = get_model_name('check', 'Check').lower().replace('.', '-') +metric_model_name = get_model_name('monitoring', 'Metric').lower().replace('.', '-') class TestAdmin( @@ -40,6 +45,38 @@ class TestAdmin( resources_fields = TestImportExportMixin.resource_fields resources_fields.append('monitoring__status') app_label = 'config' + _device_params = { + 'group': '', + 'management_ip': '', + 'model': '', + 'os': '', + 'system': '', + 'notes': '', + 'config-TOTAL_FORMS': '0', + 'config-INITIAL_FORMS': '0', + 'config-MIN_NUM_FORMS': '0', + 'config-MAX_NUM_FORMS': '1', + # devicelocation + 'devicelocation-TOTAL_FORMS': '0', + 'devicelocation-INITIAL_FORMS': '0', + 'devicelocation-MIN_NUM_FORMS': '0', + 'devicelocation-MAX_NUM_FORMS': '1', + # deviceconnection + 'deviceconnection_set-TOTAL_FORMS': '0', + 'deviceconnection_set-INITIAL_FORMS': '0', + 'deviceconnection_set-MIN_NUM_FORMS': '0', + 'deviceconnection_set-MAX_NUM_FORMS': '1000', + # command + 'command_set-TOTAL_FORMS': '0', + 'command_set-INITIAL_FORMS': '0', + 'command_set-MIN_NUM_FORMS': '0', + 'command_set-MAX_NUM_FORMS': '1000', + # check + f'{check_model_name}-content_type-object_id-TOTAL_FORMS': '0', + f'{check_model_name}-content_type-object_id-INITIAL_FORMS': '0', + f'{check_model_name}-content_type-object_id-MIN_NUM_FORMS': '0', + f'{check_model_name}-content_type-object_id-MAX_NUM_FORMS': '1000', + } def setUp(self): self._login_admin() @@ -126,7 +163,6 @@ def test_device_add_view(self): self.assertContains(r, '

Configuration

') self.assertContains(r, '

Map

') self.assertContains(r, '

Credentials

') - self.assertContains(r, '

Checks

') def test_remove_invalid_interface(self): d = self._create_device(organization=self._create_org()) @@ -265,9 +301,7 @@ def test_check_inline_formset(self): check_inline_formset = generic_inlineformset_factory( model=Check, form=CheckInline.form, formset=CheckInlineFormSet ) - # model_name changes if swapped - model_name = get_model_name('check', 'Check').lower().replace('.', '-') - ct = f'{model_name}-content_type-object_id' + ct = f'{check_model_name}-content_type-object_id' data = { f'{ct}-TOTAL_FORMS': '1', f'{ct}-INITIAL_FORMS': '0', @@ -345,6 +379,254 @@ def test_wifisession_inline(self): self.assertNotContains(response, '

WiFi Sessions

') self.assertNotContains(response, 'monitoring-wifisession-changelist-url') + def test_check_alertsetting_inline(self): + test_user = self._create_user( + username='test', email='test@inline.com', is_staff=True + ) + self._create_org_user(is_admin=True, user=test_user) + device = self._create_device() + ping_check = Check( + check_type=CHECK_CLASSES[0][0], content_object=device, params={} + ) + ping_check.full_clean() + ping_check.save() + url = reverse('admin:config_device_change', args=[device.pk]) + metric = self._create_general_metric( + name='', content_object=device, configuration='ping' + ) + self._create_alert_settings(metric=metric) + self.client.force_login(test_user) + + def _add_device_permissions(user): + test_user.user_permissions.clear() + self.assertEqual(user.user_permissions.count(), 0) + device_permissions = Permission.objects.filter(codename__endswith='device') + # Permissions required to access device page + test_user.user_permissions.add(*device_permissions), + self.assertEqual(user.user_permissions.count(), 4) + + def _add_user_permissions(user, permission_query, expected_perm_count): + user.user_permissions.add(*Permission.objects.filter(**permission_query)) + self.assertEqual(user.user_permissions.count(), expected_perm_count) + + def _assert_check_inline_in_response(response): + self.assertContains(response, '

Checks

', html=True) + self.assertContains(response, 'check-content_type-object_id-0-is_active') + self.assertContains(response, 'check-content_type-object_id-0-check_type') + self.assertContains(response, 'check-content_type-object_id-0-DELETE') + + def _assert_alertsettings_inline_in_response(response): + self.assertContains(response, '

Alert Settings

', html=True) + self.assertContains(response, 'form-row field-name') + self.assertContains( + response, + 'True', + html=True, + ) + self.assertContains(response, '

Advanced options

', html=True) + self.assertContains( + response, + 'metric-content_type-object_id-0-alertsettings-0-is_active', + ) + self.assertContains( + response, '' + ) + self.assertContains( + response, + 'metric-content_type-object_id-0-alertsettings-0-custom_threshold" value="1"', + ) + self.assertContains( + response, + 'metric-content_type-object_id-0-alertsettings-0-custom_tolerance" value="0"', + ) + + with self.subTest( + 'Test when a user does not have permission to access models or inline' + ): + _add_device_permissions(test_user) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertNotContains(response, '

Checks

', html=True) + self.assertNotContains(response, '

Alert Settings

', html=True) + + with self.subTest('Test check & alert settings with model permissions'): + _add_device_permissions(test_user) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + _add_user_permissions(test_user, {'codename__endswith': 'check'}, 8) + _add_user_permissions(test_user, {'codename__endswith': 'metric'}, 12) + _add_user_permissions( + test_user, {'codename__endswith': 'alertsettings'}, 16 + ) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + _assert_check_inline_in_response(response) + _assert_alertsettings_inline_in_response(response) + + with self.subTest('Test all inline permissions'): + _add_device_permissions(test_user) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + _add_user_permissions(test_user, {'codename__endswith': 'inline'}, 12) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + _assert_check_inline_in_response(response) + _assert_alertsettings_inline_in_response(response) + + with self.subTest('Test view inline permissions'): + _add_device_permissions(test_user) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + _add_user_permissions( + test_user, {'codename__endswith': 'view_check_inline'}, 5 + ) + _add_user_permissions( + test_user, {'codename__endswith': 'view_alertsettings_inline'}, 6 + ) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertContains(response, '

Checks

', html=True) + self.assertContains(response, 'form-row field-check_type') + self.assertContains(response, 'form-row field-is_active') + self.assertContains(response, '

Alert Settings

', html=True) + self.assertContains(response, 'form-row field-is_healthy djn-form-row-last') + self.assertContains( + response, + 'True', + html=True, + ) + self.assertContains( + response, + 'form-row field-is_active', + ) + self.assertContains(response, 'form-row field-custom_operator') + self.assertContains( + response, + 'form-row field-custom_threshold', + ) + self.assertContains( + response, + 'form-row field-custom_tolerance', + ) + + def test_alert_settings_inline_post(self): + device = self._create_device() + metric = self._create_general_metric( + name='', content_object=device, configuration='iperf3' + ) + url = reverse('admin:config_device_change', args=[device.pk]) + alertsettings = self._create_alert_settings(metric=metric) + test_inline_params = { + 'name': device.name, + 'organization': str(device.organization.id), + 'mac_address': device.mac_address, + 'key': device.key, + # metric & alertsettings + f'{metric_model_name}-content_type-object_id-TOTAL_FORMS': '1', + f'{metric_model_name}-content_type-object_id-INITIAL_FORMS': '1', + f'{metric_model_name}-content_type-object_id-MIN_NUM_FORMS': '0', + f'{metric_model_name}-content_type-object_id-MAX_NUM_FORMS': '1000', + f'{metric_model_name}-content_type-object_id-0-field_name': 'iperf3_result', + f'{metric_model_name}-content_type-object_id-0-id': str(metric.id), + f'{metric_model_name}-content_type-object_id-0-alertsettings-TOTAL_FORMS': '1', + f'{metric_model_name}-content_type-object_id-0-alertsettings-INITIAL_FORMS': '0', + f'{metric_model_name}-content_type-object_id-0-alertsettings-MIN_NUM_FORMS': '0', + f'{metric_model_name}-content_type-object_id-0-alertsettings-MAX_NUM_FORMS': '1', + f'{metric_model_name}-content_type-object_id-0-alertsettings-0-is_active': 'on', + f'{metric_model_name}-content_type-object_id-0-alertsettings-0-custom_operator': '<', + f'{metric_model_name}-content_type-object_id-0-alertsettings-0-custom_threshold': '9', + f'{metric_model_name}-content_type-object_id-0-alertsettings-0-custom_tolerance': '1800', + f'{metric_model_name}-content_type-object_id-0-alertsettings-0-id': '', + f'{metric_model_name}-content_type-object_id-0-alertsettings-0-metric': '', + } + # General metrics (clients & traffic) & Iperf3 are present + self.assertEqual(Metric.objects.count(), 3) + self.assertEqual(AlertSettings.objects.count(), 1) + + def _reset_alertsettings_inline(): + AlertSettings.objects.all().delete() + + # Delete AlertSettings objects before any subTests + _reset_alertsettings_inline() + # Delete all Metrics other than 'iperf3' before any subTests + Metric.objects.exclude(configuration='iperf3').delete() + + def _assert_alertsettings_inline(response, operator, threshold, tolerance): + self.assertEqual(response.status_code, 302) + self.assertEqual(Metric.objects.count(), 1) + self.assertEqual(AlertSettings.objects.count(), 1) + alertsettings = AlertSettings.objects.first() + self.assertEqual(alertsettings.operator, operator) + self.assertEqual(alertsettings.threshold, threshold) + self.assertEqual(alertsettings.tolerance, tolerance) + + with self.subTest('Test alert settings inline when all fields are provided'): + self._device_params.update(test_inline_params) + response = self.client.post(url, self._device_params) + _assert_alertsettings_inline(response, '<', 9, 1800) + _reset_alertsettings_inline() + + with self.subTest( + 'Test alert settings inline when partial fields are provided' + ): + test_inline_default_1 = { + f'{metric_model_name}-content_type-object_id-0-alertsettings-0-custom_operator': '>', + f'{metric_model_name}-content_type-object_id-0-alertsettings-0-custom_threshold': '', + f'{metric_model_name}-content_type-object_id-0-alertsettings-0-custom_tolerance': '', + } + self._device_params.update(test_inline_default_1) + response = self.client.post(url, self._device_params) + # 'threshold' and 'tolerance' are set to their default values + _assert_alertsettings_inline(response, '>', 1, 0) + _reset_alertsettings_inline() + + test_inline_default_2 = { + f'{metric_model_name}-content_type-object_id-0-alertsettings-0-custom_operator': '', + f'{metric_model_name}-content_type-object_id-0-alertsettings-0-custom_threshold': '18', + f'{metric_model_name}-content_type-object_id-0-alertsettings-0-custom_tolerance': '99', + } + self._device_params.update(test_inline_default_2) + response = self.client.post(url, self._device_params) + # 'operator' are set to their default values + _assert_alertsettings_inline(response, '<', 18, 99) + _reset_alertsettings_inline() + + with self.subTest('Test alert settings inline when all fields are absent'): + test_inline_params_present = { + f'{metric_model_name}-content_type-object_id-0-alertsettings-0-custom_operator': '<', + f'{metric_model_name}-content_type-object_id-0-alertsettings-0-custom_threshold': '99', + f'{metric_model_name}-content_type-object_id-0-alertsettings-0-custom_tolerance': '1880', + } + self._device_params.update(test_inline_params_present) + response = self.client.post(url, self._device_params) + _assert_alertsettings_inline(response, '<', 99, 1880) + + alertsettings = AlertSettings.objects.first() + metric = Metric.objects.first() + + test_inline_params_absent = { + f'{metric_model_name}-content_type-object_id-INITIAL_FORMS': '1', + f'{metric_model_name}-content_type-object_id-0-id': str(metric.id), + f'{metric_model_name}-content_type-object_id-0-field_name': 'iperf3_result', + f'{metric_model_name}-content_type-object_id-0-alertsettings-INITIAL_FORMS': '1', + f'{metric_model_name}-content_type-object_id-0-alertsettings-0-id': str( + alertsettings.id + ), + f'{metric_model_name}-content_type-object_id-0-alertsettings-0-metric': str( + metric.id + ), + f'{metric_model_name}-content_type-object_id-0-alertsettings-0-custom_operator': '', + f'{metric_model_name}-content_type-object_id-0-alertsettings-0-custom_threshold': '', + f'{metric_model_name}-content_type-object_id-0-alertsettings-0-custom_tolerance': '', + } + self._device_params.update(test_inline_params_absent) + response = self.client.post(url, self._device_params) + # If all the fields are empty, then it deletes the AlertSettings object + # to prevent the default value from being used as a fallback + self.assertEqual(response.status_code, 302) + self.assertEqual(Metric.objects.count(), 1) + self.assertEqual(AlertSettings.objects.count(), 0) + class TestAdminDashboard(TestGeoMixin, DeviceMonitoringTestCase): location_model = Location diff --git a/openwisp_monitoring/monitoring/base/models.py b/openwisp_monitoring/monitoring/base/models.py index ab1df2074..bde1c8957 100644 --- a/openwisp_monitoring/monitoring/base/models.py +++ b/openwisp_monitoring/monitoring/base/models.py @@ -44,7 +44,12 @@ class AbstractMetric(TimeStampedEditableModel): key = models.SlugField( max_length=64, blank=True, help_text=_('leave blank to determine automatically') ) - field_name = models.CharField(max_length=16, default='value') + field_name = models.CharField( + max_length=16, + default='value', + blank=True, + help_text=_('leave blank to determine automatically'), + ) configuration = models.CharField( max_length=16, null=True, choices=METRIC_CONFIGURATION_CHOICES ) @@ -96,11 +101,6 @@ def __setattr__(self, attrname, value): return super().__setattr__(attrname, value) def clean(self): - if ( - self.field_name == 'value' - and self.config_dict['field_name'] != '{field_name}' - ): - self.field_name = self.config_dict['field_name'] if self.key: return elif self.config_dict['key'] != '{key}': @@ -108,11 +108,30 @@ def clean(self): else: self.key = self.codename + def validate_alert_fields(self): + # When field_name is not provided while creating a metric + # then use config_dict['field_name] as metric field_name + if self.config_dict['field_name'] != '{field_name}': + if self.field_name in ['', 'value']: + self.field_name = self.config_dict['field_name'] + return + # field_name must be one of the metric fields + alert_fields = [self.config_dict['field_name']] + self.related_fields + if self.field_name not in alert_fields: + raise ValidationError( + f'"{self.field_name}" must be one of the following metric fields ie. {alert_fields}' + ) + def full_clean(self, *args, **kwargs): + # The name of the metric will be the same as the + # configuration chosen by the user only when the + # name field is empty (useful for AlertSettingsInline) if not self.name: - self.name = self.config_dict['name'] + self.name = self.get_configuration_display() # clean up key before field validation self.key = self._makekey(self.key) + # validate metric field_name for alerts + self.validate_alert_fields() return super().full_clean(*args, **kwargs) @classmethod @@ -202,6 +221,16 @@ def content_type_key(self): except AttributeError: return None + @property + def alert_field(self): + if self.field_name != self.config_dict['field_name']: + return self.field_name + return self.config_dict.get('alert_field', self.field_name) + + @property + def alert_on_related_field(self): + return self.alert_field in self.related_fields + def _get_time(self, time): """ If time is a string, convert it to a datetime @@ -359,6 +388,21 @@ def write( 'send_alert': send_alert, } options['metric_pk'] = self.pk + + # if alert_on_related_field then check threshold + # on the related_field instead of field_name + if self.alert_on_related_field: + if not extra_values: + raise ValueError( + 'write() missing keyword argument: "extra_values" required for alert on related field' + ) + if self.alert_field not in extra_values.keys(): + raise ValueError( + f'"{key}" is not defined for alert_field in metric configuration' + ) + options['check_threshold_kwargs'].update( + {'value': extra_values[self.alert_field]} + ) timeseries_write.delay(name=self.key, values=values, **options) def read(self, **kwargs): @@ -444,6 +488,10 @@ def trace_order(self): def calculate_total(self): return self.config_dict.get('calculate_total', False) + @property + def connect_points(self): + return self.config_dict.get('connect_points', False) + @property def description(self): return self.config_dict['description'].format( @@ -630,6 +678,7 @@ def json(self, time=DEFAULT_TIME, **kwargs): 'trace_type': self.trace_type, 'trace_order': self.trace_order, 'calculate_total': self.calculate_total, + 'connect_points': self.connect_points, 'colors': self.colors, } ) @@ -688,6 +737,12 @@ class Meta: abstract = True verbose_name = _('Alert settings') verbose_name_plural = verbose_name + permissions = ( + ('add_alertsettings_inline', 'Can add Alert settings inline'), + ('change_alertsettings_inline', 'Can change Alert settings inline'), + ('delete_alertsettings_inline', 'Can delete Alert settings inline'), + ('view_alertsettings_inline', 'Can view Alert settings inline'), + ) def full_clean(self, *args, **kwargs): if self.custom_threshold == self.config_dict['threshold']: @@ -757,6 +812,11 @@ def _is_crossed_by(self, current_value, time=None, retention_policy=None): return value_crossed # tolerance is set, we must go back in time # to ensure the threshold is trepassed for enough time + # if alert field is supplied, retrieve such field when reading + # so that we can let the system calculate the threshold on it + extra_fields = [] + if self.metric.alert_on_related_field: + extra_fields = [self.metric.alert_field] if time is None: # retrieves latest measurements, ordered by most recent first points = self.metric.read( @@ -764,6 +824,7 @@ def _is_crossed_by(self, current_value, time=None, retention_policy=None): limit=None, order='-time', retention_policy=retention_policy, + extra_fields=extra_fields, ) # store a list with the results results = [value_crossed] @@ -775,7 +836,7 @@ def _is_crossed_by(self, current_value, time=None, retention_policy=None): continue utc_time = utc.localize(datetime.utcfromtimestamp(point['time'])) # did this point cross the threshold? Append to result list - results.append(self._value_crossed(point[self.metric.field_name])) + results.append(self._value_crossed(point[self.metric.alert_field])) # tolerance is trepassed if self._time_crossed(utc_time): # if the latest results are consistent, the metric being diff --git a/openwisp_monitoring/monitoring/configuration.py b/openwisp_monitoring/monitoring/configuration.py index 36ab9d9dc..448771222 100644 --- a/openwisp_monitoring/monitoring/configuration.py +++ b/openwisp_monitoring/monitoring/configuration.py @@ -207,7 +207,7 @@ def _get_access_tech(): _('Total download traffic'), _('Total upload traffic'), ], - 'unit': 'adaptive_bytes', + 'unit': 'adaptive_prefix+B', 'order': 240, 'query': chart_query['traffic'], 'colors': [ @@ -228,6 +228,7 @@ def _get_access_tech(): 'general_traffic': { 'type': 'stackedbar+lines', 'calculate_total': True, + 'fill': 'none', 'trace_type': { 'download': 'stackedbar', 'upload': 'stackedbar', @@ -244,7 +245,7 @@ def _get_access_tech(): _('Total download traffic'), _('Total upload traffic'), ], - 'unit': 'adaptive_bytes', + 'unit': 'adaptive_prefix+B', 'order': 240, 'query': chart_query['general_traffic'], 'query_default_param': { @@ -544,6 +545,121 @@ def _get_access_tech(): } }, }, + 'iperf3': { + 'label': _('Iperf3'), + 'name': 'Iperf3', + 'key': 'iperf3', + 'field_name': 'iperf3_result', + 'related_fields': [ + 'sent_bps_tcp', + 'received_bps_tcp', + 'sent_bytes_tcp', + 'received_bytes_tcp', + 'retransmits', + 'sent_bytes_udp', + 'sent_bps_udp', + 'jitter', + 'total_packets', + 'lost_packets', + 'lost_percent', + ], + 'charts': { + 'bandwidth': { + 'type': 'scatter', + 'connect_points': True, + 'title': _('Bandwidth'), + 'fill': 'none', + 'description': _('Bitrate during Iperf3 test.'), + 'summary_labels': [ + _('TCP bitrate'), + _('UDP bitrate'), + ], + 'unit': 'adaptive_prefix+bps', + 'order': 280, + 'query': chart_query['bandwidth'], + 'colors': [ + DEFAULT_COLORS[0], + DEFAULT_COLORS[3], + ], + }, + 'transfer': { + 'type': 'scatter', + 'connect_points': True, + 'fill': 'none', + 'title': _('Transferred Data'), + 'description': _('Transferred Data during Iperf3 test.'), + 'summary_labels': [ + _('TCP transferred data'), + _('UDP transferred data'), + ], + 'unit': 'adaptive_prefix+B', + 'order': 290, + 'query': chart_query['transfer'], + 'colors': [ + DEFAULT_COLORS[0], + DEFAULT_COLORS[3], + ], + }, + 'retransmits': { + 'type': 'scatter', + 'connect_points': True, + 'title': _('Retransmits'), + 'description': _('Retransmits during Iperf3 test in TCP mode.'), + 'summary_labels': [_('Restransmits')], + 'unit': '', + 'order': 300, + 'query': chart_query['retransmits'], + 'colors': [DEFAULT_COLORS[-3]], + }, + 'jitter': { + 'type': 'scatter', + 'connect_points': True, + 'title': _('Jitter'), + 'description': _( + 'Jitter is a variance in latency measured using Iperf3 utility in UDP mode.' + ), + 'summary_labels': [ + _('Jitter'), + ], + 'unit': _(' ms'), + 'order': 330, + 'query': chart_query['jitter'], + 'colors': [DEFAULT_COLORS[4]], + }, + 'datagram': { + 'type': 'scatter', + 'fill': 'none', + 'connect_points': True, + 'title': _('Datagram'), + 'description': _( + '(Lost / Total) datagrams measured by Iperf3 test in UDP mode.' + ), + 'summary_labels': [ + _('Lost datagram'), + _('Total datagram'), + ], + 'unit': '', + 'order': 340, + 'query': chart_query['datagram'], + 'colors': [DEFAULT_COLORS[3], DEFAULT_COLORS[2]], + }, + 'datagram_loss': { + 'type': 'scatter', + 'connect_points': True, + 'title': _('Datagram Loss'), + 'description': _( + 'Indicates datagram loss % during Iperf3 test in UDP mode.' + ), + 'summary_labels': [ + _('Datagram loss'), + ], + 'unit': '%', + 'order': 350, + 'query': chart_query['datagram_loss'], + 'colors': [DEFAULT_COLORS[3]], + }, + }, + }, } DEFAULT_CHARTS = {} @@ -556,6 +672,10 @@ def _validate_metric_configuration(metric_config): assert 'name' in metric_config assert 'key' in metric_config assert 'field_name' in metric_config + if 'alert_field' in metric_config: + # ensure only valid alert_field is present + alert_fields = [metric_config['field_name']] + metric_config['related_fields'] + assert metric_config['alert_field'] in alert_fields def _validate_chart_configuration(chart_config): diff --git a/openwisp_monitoring/monitoring/migrations/0009_alter_alertsettings_options.py b/openwisp_monitoring/monitoring/migrations/0009_alter_alertsettings_options.py new file mode 100644 index 000000000..02be22f99 --- /dev/null +++ b/openwisp_monitoring/monitoring/migrations/0009_alter_alertsettings_options.py @@ -0,0 +1,26 @@ +# Generated by Django 3.2.14 on 2022-08-12 11:04 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('monitoring', '0008_create_general_metrics'), + ] + + operations = [ + migrations.AlterModelOptions( + name='alertsettings', + options={ + 'permissions': ( + ('add_alertsettings_inline', 'Can add Alert settings inline'), + ('change_alertsettings_inline', 'Can change Alert settings inline'), + ('delete_alertsettings_inline', 'Can delete Alert settings inline'), + ('view_alertsettings_inline', 'Can view Alert settings inline'), + ), + 'verbose_name': 'Alert settings', + 'verbose_name_plural': 'Alert settings', + }, + ), + ] diff --git a/openwisp_monitoring/monitoring/migrations/0010_add_alertsettings_inline_permissions.py b/openwisp_monitoring/monitoring/migrations/0010_add_alertsettings_inline_permissions.py new file mode 100644 index 000000000..9485f47a7 --- /dev/null +++ b/openwisp_monitoring/monitoring/migrations/0010_add_alertsettings_inline_permissions.py @@ -0,0 +1,19 @@ +# Generated by Django 4.0.4 on 2022-08-19 11:28 + +from django.db import migrations + +from . import assign_alertsettings_inline_permissions_to_groups + + +class Migration(migrations.Migration): + + dependencies = [ + ('monitoring', '0009_alter_alertsettings_options'), + ] + + operations = [ + migrations.RunPython( + assign_alertsettings_inline_permissions_to_groups, + reverse_code=migrations.RunPython.noop, + ), + ] diff --git a/openwisp_monitoring/monitoring/migrations/0011_alter_metric_field_name.py b/openwisp_monitoring/monitoring/migrations/0011_alter_metric_field_name.py new file mode 100644 index 000000000..1c072d3b5 --- /dev/null +++ b/openwisp_monitoring/monitoring/migrations/0011_alter_metric_field_name.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.15 on 2022-09-08 11:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('monitoring', '0010_add_alertsettings_inline_permissions'), + ] + + operations = [ + migrations.AlterField( + model_name='metric', + name='field_name', + field=models.CharField( + blank=True, + default='value', + help_text='leave blank to determine automatically', + max_length=16, + ), + ), + ] diff --git a/openwisp_monitoring/monitoring/migrations/__init__.py b/openwisp_monitoring/monitoring/migrations/__init__.py index 29f5158c1..f519cb6c5 100644 --- a/openwisp_monitoring/monitoring/migrations/__init__.py +++ b/openwisp_monitoring/monitoring/migrations/__init__.py @@ -37,6 +37,37 @@ def assign_permissions_to_groups(apps, schema_editor): ) +def assign_alertsettings_inline_permissions_to_groups(apps, schema_editor): + create_default_permissions(apps, schema_editor) + operators_read_only_admins_manage = [ + 'alertsettings', + ] + manage_operations = ['add', 'view', 'change', 'delete'] + Group = get_swapped_model(apps, 'openwisp_users', 'Group') + + try: + admin = Group.objects.get(name='Administrator') + operator = Group.objects.get(name='Operator') + # consider failures custom cases + # that do not have to be dealt with + except Group.DoesNotExist: + return + + for model_name in operators_read_only_admins_manage: + try: + permission = Permission.objects.get( + codename='view_{}_inline'.format(model_name) + ) + operator.permissions.add(permission.pk) + except Permission.DoesNotExist: + pass + for operation in manage_operations: + permission = Permission.objects.get( + codename='{}_{}_inline'.format(operation, model_name) + ) + admin.permissions.add(permission.pk) + + def create_general_metrics(apps, schema_editor): Chart = swapper.load_model('monitoring', 'Chart') Metric = swapper.load_model('monitoring', 'Metric') diff --git a/openwisp_monitoring/monitoring/static/monitoring/js/chart.js b/openwisp_monitoring/monitoring/static/monitoring/js/chart.js index 8a11ab9e8..bfad3e564 100644 --- a/openwisp_monitoring/monitoring/static/monitoring/js/chart.js +++ b/openwisp_monitoring/monitoring/static/monitoring/js/chart.js @@ -20,19 +20,19 @@ function getAdaptiveScale(value, multiplier, unit) { if (value == 0) { multiplier = 1; - unit = 'B'; + unit = unit; } else if (value < 0.001) { multiplier = 1000000; - unit = 'KB'; + unit = 'K' + unit; } else if (value < 1) { multiplier = 1000; - unit = 'MB'; + unit = 'M' + unit; } else if (value < 1000) { multiplier = 1; - unit = 'GB'; + unit = 'G' + unit; } else if (value >= 1000) { multiplier = 0.001; - unit = 'TB'; + unit = 'T' + unit; } return { multiplier: multiplier, @@ -44,7 +44,7 @@ return Math.round((value * multiplier) * 100) / 100; } - function adaptiveFilterPoints(charts, layout, yRawVal) { + function adaptiveFilterPoints(charts, layout, yRawVal, chartUnit = '') { var y = charts[0].y, sum = 0, count = 0, shownVal, average; for (var i=0; i < y.length; i++) { sum += y[i]; @@ -53,7 +53,7 @@ } } average = sum / count; - var scales = getAdaptiveScale(average, 1, ''); + var scales = getAdaptiveScale(average, 1, chartUnit); var multiplier = scales.multiplier, unit = scales.unit; for (i=0; i < y.length; i++) { @@ -64,7 +64,7 @@ } shownVal = charts[j].y[i]; charts[j].y[i] = getAdaptiveBytes(charts[j].y[i], multiplier); - var hoverScales = getAdaptiveScale(shownVal, 1, ''); + var hoverScales = getAdaptiveScale(shownVal, 1, chartUnit); var hoverMultiplier = hoverScales.multiplier, hoverUnit = hoverScales.unit; shownVal = getAdaptiveBytes(shownVal, hoverMultiplier); @@ -74,8 +74,8 @@ layout.yaxis.title = unit; } - function adaptiveFilterSummary(i, percircles, value) { - var scales = getAdaptiveScale(value, 1, ''), + function adaptiveFilterSummary(i, percircles, value, chartUnit = '') { + var scales = getAdaptiveScale(value, 1, chartUnit), multiplier = scales.multiplier, unit = scales.unit; value = getAdaptiveBytes(value, multiplier); @@ -138,7 +138,7 @@ if (type === 'histogram') { layout.hovermode = 'closest'; } - var map, mapped, label, fixedValue, key; + var map, mapped, label, fixedValue, key, chartUnit, yValues; // given a value, returns its color and description // according to the color map configuration of this chart function findInColorMap(value) { @@ -190,6 +190,7 @@ // We use the "_key" field to sort the charts // according to the order defined in "data.trace_order" _key: key, + _connectPoints : data.connect_points || false, }, yValuesRaw = data.traces[i][1]; if (type !== 'histogram') { @@ -208,7 +209,10 @@ options.type = 'scatter'; options.mode = 'lines+markers'; options.line = {shape: 'hvh'}; - options.fill = "none"; + options.fill = data.fill; + } + if (options._connectPoints) { + options.mode = 'lines'; } } } @@ -228,6 +232,11 @@ layout.margin.b = 45; } } + + var xValuesRaw = options.x; + if (options._connectPoints) { + options.x = []; + } // adjust text to be displayed in Y values // differentiate between values with zero and no values at all (N/A) for (var c=0; c