[deps] Added support for Django 4.1.0 and Django 4.2.0

- Dropped support for Python 3.7
- Dropped support for Django 4.0.0
- Bumped django-reversion~=5.0.4
- Bumped django-taggit~=4.0.0
- Bumped paramiko[ed25519]~=3.2.0
- Bumped django-import-export~=3.2.0
- [tests] Fixed selenium tests
- [fix] Fixed CSS for jsonschema UI
- [fix] Fixed flaky test_create_new_device test

.github/workflows/ci.yml

@@ -22,7 +22,7 @@ jobs:
         ports:
           - 6379:6379
       postgres:
-        image: mdillon/postgis:11-alpine
+        image: postgis/postgis:13-3.3-alpine
         env:
           POSTGRES_PASSWORD: openwisp2
           POSTGRES_USER: openwisp2
@@ -39,10 +39,8 @@ jobs:
           - "3.10"
         django-version:
           - django~=3.2.0
-          - django~=4.0.0
-        include:
-          - django-version: django~=3.2.0
-            python-version: 3.7
+          - django~=4.1.0
+          - django~=4.2.0
 
     steps:
     - uses: actions/checkout@v2
@@ -67,13 +65,13 @@ jobs:
     - name: Upgrade python system packages
       run: pip install -U pip wheel setuptools
 
-    - name: Install openwisp-controller
-      run: |
-        pip install -e .
-
     - name: Install test dependencies
       run: |
         pip install -U -r requirements-test.txt
+
+    - name: Install openwisp-controller
+      run: |
+        pip install -U -e .
         pip install ${{ matrix.django-version }}
 
     - name: QA checks
@@ -82,9 +80,11 @@ jobs:
     - name: Tests
       run: |
         coverage run runtests.py --parallel
+        # the following command runs tests with Postgres/PostGIS but
+        # only for specific test cases which are tagged with "db_tests"
         POSTGRESQL=1 coverage run runtests.py --parallel --keepdb
         coverage combine
-        # SAMPLE tests
+        # tests the extension capability
         SAMPLE_APP=1 ./runtests.py --keepdb
       env:
         SELENIUM_HEADLESS: 1

docker-compose.yml

@@ -21,7 +21,7 @@ services:
     entrypoint: redis-server --appendonly yes
 
   postgres:
-    image: mdillon/postgis:11-alpine
+    image: postgis/postgis:13-3.3-alpine
     environment:
       POSTGRES_PASSWORD: openwisp2
       POSTGRES_USER: openwisp2

openwisp_controller/config/__init__.py

@@ -1 +0,0 @@
-default_app_config = 'openwisp_controller.config.apps.ConfigConfig'

openwisp_controller/config/migrations/0030_django_taggit_update.py

@@ -35,6 +35,8 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='templatetag',
             name='slug',
-            field=models.SlugField(max_length=100, unique=True, verbose_name='slug'),
+            field=models.SlugField(
+                allow_unicode=True, max_length=100, unique=True, verbose_name='slug'
+            ),
         ),
     ]

openwisp_controller/config/static/config/css/lib/advanced-mode.css

@@ -857,6 +857,9 @@ table.jsoneditor-search button.jsoneditor-previous:hover {
 #main .property-selector .form-row {
   padding: 5px 0 6px 10px !important;
 }
+#main .property-selector {
+  text-align: left;
+}
 @media (max-width: 768px){
   .jsoneditor-menu #netjsonconfig-hint-advancedmode{
     display: none;

openwisp_controller/config/static/config/js/widget.js

@@ -543,6 +543,7 @@ JSONEditor.defaults.themes.django = JSONEditor.AbstractTheme.extend({
     },
     getCheckbox: function () {
         var el = this.getFormInputField('checkbox');
+        el.className = null;
         el.style.display = 'inline-block';
         el.style.width = 'auto';
         return el;

openwisp_controller/config/tests/test_selenium.py

@@ -1,4 +1,5 @@
 from django.contrib.staticfiles.testing import StaticLiveServerTestCase
+from django.test import tag
 from django.urls.base import reverse
 from selenium.common.exceptions import (
     StaleElementReferenceException,
@@ -16,6 +17,7 @@
 from .utils import CreateConfigTemplateMixin
 
 
+@tag('selenium_tests')
 class TestDeviceAdmin(
     TestOrganizationMixin,
     CreateConfigTemplateMixin,
@@ -84,7 +86,7 @@ def test_create_new_device(self):
 
         try:
             WebDriverWait(self.web_driver, 2).until(
-                EC.element_to_be_clickable(
+                EC.presence_of_element_located(
                     (By.XPATH, f'//*[@value="{default_template.id}"]')
                 )
             )

openwisp_controller/config/tests/test_template.py

@@ -12,6 +12,7 @@
 from openwisp_users.tests.utils import TestOrganizationMixin
 from openwisp_utils.tests import catch_signal
 
+from ...tests.utils import TransactionTestMixin
 from .. import settings as app_settings
 from ..signals import config_modified, config_status_changed
 from ..tasks import logger as task_logger
@@ -504,6 +505,7 @@ def test_required_vpn_template_corner_case(self):
 
 
 class TestTemplateTransaction(
+    TransactionTestMixin,
     TestOrganizationMixin,
     CreateConfigTemplateMixin,
     TestVpnX509Mixin,

openwisp_controller/connection/__init__.py

@@ -1 +0,0 @@
-default_app_config = 'openwisp_controller.connection.apps.ConnectionConfig'

openwisp_controller/connection/tests/test_models.py

@@ -11,6 +11,7 @@
 
 from openwisp_utils.tests import capture_any_output, catch_signal
 
+from ...tests.utils import TransactionTestMixin
 from .. import settings as app_settings
 from ..apps import _TASK_NAME
 from ..commands import (
@@ -835,7 +836,7 @@ def test_command_multiple_connections(self, connect_mocked):
             self.assertIn(command.connection, [dc1, dc2])
 
 
-class TestModelsTransaction(BaseTestModels, TransactionTestCase):
+class TestModelsTransaction(TransactionTestMixin, BaseTestModels, TransactionTestCase):
     def _prepare_conf_object(self, organization=None):
         if not organization:
             organization = self._create_org(name='org1')

openwisp_controller/geo/__init__.py

@@ -1 +0,0 @@
-default_app_config = 'openwisp_controller.geo.apps.GeoConfig'

openwisp_controller/geo/apps.py

@@ -114,3 +114,6 @@ def register_menu_groups(self):
                 'icon': 'ow-geo',
             },
         )
+
+
+del LociConfig

openwisp_controller/pki/__init__.py

@@ -1 +0,0 @@
-default_app_config = 'openwisp_controller.pki.apps.PkiConfig'

openwisp_controller/pki/apps.py

@@ -41,3 +41,6 @@ def register_menu_groups(self):
                 'icon': 'ow-cer-group',
             },
         )
+
+
+del DjangoX509Config

openwisp_controller/subnet_division/__init__.py

@@ -1 +0,0 @@
-default_app_config = 'openwisp_controller.subnet_division.apps.SubnetDivisionConfig'

openwisp_controller/subnet_division/admin.py

@@ -58,7 +58,7 @@ class SubnetAdmin(BaseSubnetAdmin):
         VpnFilter,
         DeviceFilter,
     ]
-    inlines = [SubnetDivisionRuleInlineAdmin] + BaseSubnetAdmin.inlines
+    inlines = [SubnetDivisionRuleInlineAdmin] + list(BaseSubnetAdmin.inlines)
     list_display = BaseSubnetAdmin.list_display
     list_display.insert(list_display.index('created'), 'related_device')
 

openwisp_controller/subnet_division/tests/test_models.py

@@ -330,7 +330,7 @@ def test_rule_deleted(self):
 
         self.assertEqual(subnet_query.count(), 0)
         self.assertEqual(ip_query.count(), 0)
-        self.assertEqual(index_query.count(), 0)
+        self.assertEqual(SubnetDivisionIndex.objects.count(), 0)
 
     def test_vpnclient_deleted(self):
         rule = self._get_vpn_subdivision_rule()

openwisp_controller/tests/test_selenium.py

@@ -1,9 +1,11 @@
+from unittest.mock import patch
+
 from django.contrib.staticfiles.testing import StaticLiveServerTestCase
 from django.core.management import call_command
+from django.test import tag
 from django.urls.base import reverse
 from reversion.models import Version
-from selenium.common.exceptions import TimeoutException, UnexpectedAlertPresentException
-from selenium.webdriver.common.alert import Alert
+from selenium.common.exceptions import TimeoutException
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
@@ -19,6 +21,7 @@
 DeviceLocation = load_model('geo', 'DeviceLocation')
 
 
+@tag('selenium_tests')
 class TestDeviceConnectionInlineAdmin(
     CreateConnectionsMixin, TestGeoMixin, SeleniumTestMixin, StaticLiveServerTestCase
 ):
@@ -31,27 +34,8 @@ def setUp(self):
             username=self.admin_username, password=self.admin_password
         )
 
-    def tearDown(self):
-        # Accept unsaved changes alert to allow other tests to run
-        try:
-            self.web_driver.refresh()
-        except UnexpectedAlertPresentException:
-            alert = Alert(self.web_driver)
-            alert.accept()
-        else:
-            try:
-                WebDriverWait(self.web_driver, 1).until(EC.alert_is_present())
-            except TimeoutException:
-                pass
-            else:
-                alert = Alert(self.web_driver)
-                alert.accept()
-        self.web_driver.refresh()
-        WebDriverWait(self.web_driver, 2).until(
-            EC.visibility_of_element_located((By.XPATH, '//*[@id="site-name"]'))
-        )
-
-    def test_restoring_deleted_device(self):
+    @patch('reversion.models.logger.warning')
+    def test_restoring_deleted_device(self, *args):
         org = self._get_org()
         self._create_credentials(auto_add=True, organization=org)
         device = self._create_config(organization=org).device

openwisp_controller/tests/utils.py

@@ -1,4 +1,8 @@
+import django
 from django.contrib.auth import get_user_model
+from django.db import connections
+from django.db.utils import DEFAULT_DB_ALIAS
+from django.test.testcases import _AssertNumQueriesContext
 from django.urls import reverse
 
 from openwisp_users.tests.utils import TestMultitenantAdminMixin
@@ -16,3 +20,32 @@ def _test_changelist_recover_deleted(self, app_label, model_label):
 
     def _login(self, username='admin', password='tester'):
         self.client.force_login(user_model.objects.get(username=username))
+
+
+class _ManagementTransactionNumQueriesContext(_AssertNumQueriesContext):
+    def __exit__(self, exc_type, exc_value, traceback):
+        """
+        Django 4.2 introduced support for logging transaction
+        management queries (BEGIN, COMMIT, and ROLLBACK).
+        This method increases the number of expected database
+        queries if COMMIT/ROLLBACK queries are found when
+        using Django 4.2
+        """
+        if exc_type is not None:
+            return
+        for query in self.captured_queries:
+            if django.VERSION > (4, 2) and 'COMMIT' in query['sql']:
+                self.num += 1
+        super().__exit__(exc_type, exc_value, traceback)
+
+
+class TransactionTestMixin(object):
+    def assertNumQueries(self, num, func=None, *args, using=DEFAULT_DB_ALIAS, **kwargs):
+        conn = connections[using]
+
+        context = _ManagementTransactionNumQueriesContext(self, num, conn)
+        if func is None:
+            return context
+
+        with context:
+            func(*args, **kwargs)

requirements-test.txt

@@ -1,11 +1,9 @@
-pytest~=6.0
 pytest-django~=4.5.2
-pytest-asyncio~=0.14.0
-pytest-cov~=2.10.0
+pytest-asyncio~=0.21.0
+pytest-cov~=4.1.0
 openwisp-utils[qa,selenium] @ https://github.com/openwisp/openwisp-utils/tarball/master
-redis~=4.5.4
 channels_redis~=4.1.0
 django_redis~=5.2.0
 mock-ssh-server~=0.9.1
-responses~=0.12.1
-psycopg2-binary~=2.8.0
+responses~=0.23.1
+psycopg2-binary~=2.9.6

requirements.txt

@@ -1,18 +1,18 @@
 django-sortedm2m~=3.1.1
-django-reversion~=4.0.1
-django-taggit~=2.1.0
+django-reversion~=5.0.4
+django-taggit~=4.0.0
 netjsonconfig~=1.0.1
 django-x509 @ https://github.com/openwisp/django-x509/tarball/master
 django-loci @ https://github.com/openwisp/django-loci/tarball/master
 django-flat-json-widget @ https://github.com/openwisp/django-flat-json-widget/tarball/master
 openwisp-users @ https://github.com/openwisp/openwisp-users/tarball/master
+openwisp-utils[celery] @ https://github.com/openwisp/openwisp-utils/tarball/master
 openwisp-notifications @ https://github.com/openwisp/openwisp-notifications/tarball/master
 openwisp-ipam @ https://github.com/openwisp/openwisp-ipam/tarball/master
 djangorestframework-gis~=0.18.0
-paramiko[ed25519]~=2.10.3
+paramiko[ed25519]~=3.2.0
 scp~=0.14.2
-celery~=5.2.3
-django-cache-memoize~=0.1
+django-cache-memoize~=0.1.0
 shortuuid~=1.0.1
 netaddr~=0.8.0
-django-import-export~=2.8.0
+django-import-export~=3.2.0

runtests.py

@@ -24,6 +24,9 @@
 
     if os.environ.get('POSTGRESQL', False):
         args.extend(['--tag', 'db_tests'])
+        args.extend(['--tag', 'selenium_tests'])
+    else:
+        args.extend(['--exclude-tag', 'selenium_tests'])
 
     execute_from_command_line(args)
 

tests/openwisp2/sample_config/__init__.py

@@ -1 +0,0 @@
-default_app_config = 'openwisp2.sample_config.apps.SampleConfigConfig'

tests/openwisp2/sample_config/apps.py

@@ -4,3 +4,6 @@
 class SampleConfigConfig(ConfigConfig):
     name = 'openwisp2.sample_config'
     label = 'sample_config'
+
+
+del ConfigConfig

tests/openwisp2/sample_config/migrations/0001_initial.py

@@ -191,7 +191,12 @@ class Migration(migrations.Migration):
                 ),
                 (
                     'slug',
-                    models.SlugField(max_length=100, unique=True, verbose_name='slug'),
+                    models.SlugField(
+                        allow_unicode=True,
+                        max_length=100,
+                        unique=True,
+                        verbose_name='slug',
+                    ),
                 ),
                 ('details', models.CharField(blank=True, max_length=64, null=True)),
             ],

tests/openwisp2/sample_connection/__init__.py

@@ -1 +0,0 @@
-default_app_config = 'openwisp2.sample_connection.apps.SampleConnectionConfig'

tests/openwisp2/sample_connection/apps.py

@@ -4,3 +4,6 @@
 class SampleConnectionConfig(ConnectionConfig):
     name = 'openwisp2.sample_connection'
     label = 'sample_connection'
+
+
+del ConnectionConfig

tests/openwisp2/sample_geo/__init__.py

@@ -1 +0,0 @@
-default_app_config = 'openwisp2.sample_geo.apps.SampleGeoConfig'

tests/openwisp2/sample_geo/apps.py

@@ -4,3 +4,6 @@
 class SampleGeoConfig(GeoConfig):
     name = 'openwisp2.sample_geo'
     label = 'sample_geo'
+
+
+del GeoConfig

tests/openwisp2/sample_pki/__init__.py

@@ -1 +0,0 @@
-default_app_config = 'openwisp2.sample_pki.apps.SamplePkiConfig'

tests/openwisp2/sample_pki/apps.py

@@ -4,3 +4,6 @@
 class SamplePkiConfig(PkiConfig):
     name = 'openwisp2.sample_pki'
     label = 'sample_pki'
+
+
+del PkiConfig

tests/openwisp2/sample_subnet_division/__init__.py

@@ -1 +0,0 @@
-default_app_config = 'openwisp2.sample_subnet_division.apps.SampleSubnetDivisionConfig'

tests/openwisp2/sample_subnet_division/apps.py

@@ -4,3 +4,6 @@
 class SampleSubnetDivisionConfig(SubnetDivisionConfig):
     name = 'openwisp2.sample_subnet_division'
     label = 'sample_subnet_division'
+
+
+del SubnetDivisionConfig

tests/openwisp2/sample_users/__init__.py

@@ -1 +0,0 @@
-default_app_config = 'openwisp2.sample_users.apps.SampleUsersConfig'

tests/openwisp2/sample_users/apps.py

@@ -4,3 +4,6 @@
 class SampleUsersConfig(OpenwispUsersConfig):
     name = 'openwisp2.sample_users'
     label = 'sample_users'
+
+
+del OpenwispUsersConfig

tests/openwisp2/settings.py

@@ -135,17 +135,6 @@
     }
 ]
 
-if not TESTING:
-    CACHES = {
-        'default': {
-            'BACKEND': 'django_redis.cache.RedisCache',
-            'LOCATION': 'redis://localhost/5',
-            'OPTIONS': {
-                'CLIENT_CLASS': 'django_redis.client.DefaultClient',
-            },
-        }
-    }
-
 FORM_RENDERER = 'django.forms.renderers.TemplatesSetting'
 
 EMAIL_PORT = '1025'  # for testing purposes