Merge up release-v0.6.x to release-v0.7.x

.github/workflows/python-publish.yml

@@ -14,15 +14,13 @@ on:
 
 jobs:
   deploy:
-
-    runs-on: ubuntu-20.04
-
+    runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: 2.7
+        python-version: 3.9
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.github/workflows/python2-lint.yml

@@ -19,13 +19,11 @@ jobs:
         name: Python 2 syntax checking
         needs: pre_job
         if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-20.04
+        container:
+            image: python:2.7.18-buster
         steps:
             - uses: actions/checkout@v2
-            - name: Set up Python 2.7
-              uses: actions/setup-python@v2
-              with:
-                  python-version: 2.7
             - name: Install dependencies
               run: |
                   python -m pip install --upgrade pip

.github/workflows/tox.yml

@@ -23,8 +23,7 @@ jobs:
         strategy:
             max-parallel: 5
             matrix:
-                python-version: [2.7, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
-
+                python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
         steps:
             - uses: actions/checkout@v2
             - name: Set up Python ${{ matrix.python-version }}
@@ -42,6 +41,26 @@ jobs:
                   key: ${{ runner.os }}-tox-py${{ matrix.python-version }}-${{ hashFiles('setup.py', 'requirements/*.txt') }}
             - name: Test with tox
               run: tox -e py${{ matrix.python-version }}
+    unit_test27:
+        name: Python unit tests (2.7)
+        needs: pre_job
+        if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
+        runs-on: ubuntu-20.04
+        container:
+            image: python:2.7.18-buster
+        steps:
+            - uses: actions/checkout@v2
+            - name: Install tox
+              run: |
+                python -m pip install --upgrade pip
+                pip install tox
+            - name: tox env cache
+              uses: actions/cache@v2
+              with:
+                  path: ${{ github.workspace }}/.tox/py2.7
+                  key: ${{ runner.os }}-tox-py2.7-${{ hashFiles('setup.py', 'requirements/*.txt') }}
+            - name: Test with tox
+              run: tox -e py2.7
     cryptography:
         name: Python unit tests + cryptography
         needs: pre_job
@@ -51,7 +70,7 @@ jobs:
             max-parallel: 5
             matrix:
                 # only crypto 3.3 seems to work
-                python-version: [ 2.7, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11' ]
+                python-version: [ 3.6, 3.7, 3.8, 3.9, '3.10', '3.11' ]
                 crypto-version: [ 3.3 ]
 
         steps:
@@ -75,6 +94,30 @@ jobs:
                     key: ${{ runner.os }}-tox-py${{ matrix.python-version }}-crypto${{ matrix.crypto-version }}-${{ hashFiles('setup.py', 'requirements/*.txt') }}
             -   name: Test with tox
                 run: tox -e py${{ matrix.python-version }}-cryptography${{ matrix.crypto-version }}
+    cryptography27:
+        name: Python unit tests (2.7) + cryptography
+        needs: pre_job
+        if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
+        runs-on: ubuntu-20.04
+        container:
+            image: python:2.7.18-buster
+        steps:
+            -   uses: actions/checkout@v2
+            -   name: Install system dependencies
+                run: |
+                    apt-get -y -qq update
+                    apt-get install -y openssl libssl-dev
+            -   name: Install tox
+                run: |
+                    python -m pip install --upgrade pip
+                    pip install tox
+            -   name: tox env cache
+                uses: actions/cache@v2
+                with:
+                    path: ${{ github.workspace }}/.tox/py2.7
+                    key: ${{ runner.os }}-tox-py2.7-crypto3.3-${{ hashFiles('setup.py', 'requirements/*.txt') }}
+            -   name: Test with tox
+                run: tox -e py2.7-cryptography3.3
     postgres:
         name: Python postgres unit tests
         needs: pre_job

.gitignore

@@ -85,6 +85,7 @@ venv/
 ENV/
 .venv
 Pipfile
+.envrc
 
 # Spyder project settings
 .spyderproject
@@ -97,4 +98,4 @@ Pipfile
 
 *.db
 
-.vscode
+.vscode

CHANGELOG.md

@@ -1,6 +1,16 @@
 # Release Notes
 
 List of the most important changes for each release.
+## 0.7.1
+- Ported changes from 0.6.17
+
+## 0.6.17
+- Added `client-instance-id`, `server-instance-id`, `sync-filter`, `push` and `pull` arguments to `cleanupsyncs` management command
+- Added option for resuming a sync to ignore the `SyncSession`'s existing `process_id`
+
+## 0.7.0
+- Dropped support for Python 3.4 and 3.5
+- Upgraded cryptography to latest version that still supports Python 2.7 - 3.3.2
 
 ## 0.6.16
 - Added dedicated `client_instance_id` and `server_instance_id` fields to `SyncSession`

morango/management/commands/cleanupsyncs.py

@@ -1,5 +1,6 @@
 import datetime
 import logging
+import uuid
 
 from django.core.management.base import BaseCommand
 from django.db import transaction
@@ -29,6 +30,36 @@ def add_arguments(self, parser):
             default=6,
             help="Number of hours of inactivity after which a session should be considered stale",
         )
+        parser.add_argument(
+            "--client-instance-id",
+            type=uuid.UUID,
+            default=None,
+            help="Filters the SyncSession models to those with matching 'client_instance_id'",
+        )
+        parser.add_argument(
+            "--server-instance-id",
+            type=uuid.UUID,
+            default=None,
+            help="Filters the SyncSession models to those with matching 'server_instance_id'",
+        )
+        parser.add_argument(
+            "--sync-filter",
+            type=str,
+            default=None,
+            help="Filters the TransferSession models to those with 'filters' starting with 'sync_filter'",
+        )
+        parser.add_argument(
+            "--push",
+            type=bool,
+            default=None,
+            help="Filters the TransferSession models to those with 'push' set to True",
+        )
+        parser.add_argument(
+            "--pull",
+            type=bool,
+            default=None,
+            help="Filters the TransferSession models to those with 'push' set to False",
+        )
 
     def handle(self, *args, **options):
 
@@ -41,13 +72,28 @@ def handle(self, *args, **options):
         if options["ids"]:
             sync_sessions = sync_sessions.filter(id__in=options["ids"])
 
+        if options["client_instance_id"]:
+            sync_sessions = sync_sessions.filter(client_instance_id=options["client_instance_id"])
+
+        if options["server_instance_id"]:
+            sync_sessions = sync_sessions.filter(server_instance_id=options["server_instance_id"])
+
         # retrieve all sessions still marked as active but with no activity since the cutoff
         transfer_sessions = TransferSession.objects.filter(
             sync_session_id__in=sync_sessions.values("id"),
             active=True,
             last_activity_timestamp__lt=cutoff,
         )
 
+        if options["sync_filter"]:
+            transfer_sessions = transfer_sessions.filter(filter__startswith=options["sync_filter"])
+
+        if options["push"] and not options["pull"]:
+            transfer_sessions = transfer_sessions.filter(push=True)
+
+        if options["pull"] and not options["push"]:
+            transfer_sessions = transfer_sessions.filter(push=False)
+
         transfer_count = transfer_sessions.count()
 
         # loop over the stale sessions one by one to close them out

morango/sync/syncsession.py

@@ -260,12 +260,16 @@ def create_sync_session(self, client_cert, server_cert, chunk_size=None):
         sync_session = SyncSession.objects.create(**data)
         return SyncSessionClient(self, sync_session)
 
-    def resume_sync_session(self, sync_session_id, chunk_size=None):
+    def resume_sync_session(self, sync_session_id, chunk_size=None, ignore_existing_process=False):
         """
         Resumes an existing sync session given an ID
 
         :param sync_session_id: The UUID of the `SyncSession` to resume
         :param chunk_size: An optional parameter specifying the size for each transferred chunk
+        :type chunk_size: int
+        :param ignore_existing_process:An optional parameter specifying whether to ignore an
+            existing active process ID
+        :type ignore_existing_process: bool
         :return: A SyncSessionClient instance
         :rtype: SyncSessionClient
         """
@@ -281,7 +285,8 @@ def resume_sync_session(self, sync_session_id, chunk_size=None):
 
         # check that process of existing session isn't still running
         if (
-            sync_session.process_id
+            not ignore_existing_process
+            and sync_session.process_id
             and sync_session.process_id != os.getpid()
             and pid_exists(sync_session.process_id)
         ):

tests/testapp/tests/compat.py

@@ -1,5 +1,10 @@
 try:
-    from test.support import EnvironmentVarGuard # noqa F401
+    # In the Python 2.7 GH workflows, we have to install backported version
+    from backports.test.support import EnvironmentVarGuard # noqa F401
 except ImportError:
-    # In Python 3.10, this has been moved to test.support.os_helper
-    from test.support.os_helper import EnvironmentVarGuard # noqa F401
+    try:
+        # For python >2.7 and <3.10
+        from test.support import EnvironmentVarGuard # noqa F401
+    except ImportError:
+        # In Python 3.10, this has been moved to test.support.os_helper
+        from test.support.os_helper import EnvironmentVarGuard # noqa F401

tests/testapp/tests/sync/test_syncsession.py

@@ -275,6 +275,30 @@ def create(**data):
                 with self.assertRaises(MorangoResumeSyncError):
                     self.network_connection.resume_sync_session(sync_session.id)
 
+    @mock.patch.object(SyncSession.objects, "create")
+    def test_resume_sync_session__still_running_but_ignore(self, mock_create):
+        def create(**data):
+            """Trickery to get around same DB being used for both client and server"""
+            return SyncSession.objects.get(pk=data.get("id"))
+
+        mock_create.side_effect = create
+
+        # first create a session
+        client = self.network_connection.create_sync_session(self.subset_cert, self.root_cert)
+        # reset process ID
+        sync_session = client.sync_session
+        sync_session.process_id = 123000111
+        sync_session.save()
+
+        with mock.patch("morango.sync.syncsession.pid_exists") as mock_pid_exists:
+            mock_pid_exists.return_value = True
+            with mock.patch("morango.sync.syncsession.os.getpid") as mock_getpid:
+                mock_getpid.return_value = 245111222
+                resume_client = self.network_connection.resume_sync_session(
+                    sync_session.id, ignore_existing_process=True
+                )
+                self.assertEqual(sync_session.id, resume_client.sync_session.id)
+
 
 class SyncSessionClientTestCase(BaseClientTestCase):
     def test_get_pull_client(self):

tests/testapp/tests/test_management_commands.py

@@ -10,7 +10,7 @@
 from morango.models.core import TransferSession
 
 
-def _create_sessions(last_activity_offset=0, sync_session=None):
+def _create_sessions(last_activity_offset=0, sync_session=None, push=True):
 
     last_activity_timestamp = timezone.now() - datetime.timedelta(
         hours=last_activity_offset
@@ -21,13 +21,16 @@ def _create_sessions(last_activity_offset=0, sync_session=None):
             id=uuid.uuid4().hex,
             profile="facilitydata",
             last_activity_timestamp=last_activity_timestamp,
+            client_instance_id=uuid.uuid4().hex,
+            server_instance_id=uuid.uuid4().hex,
         )
 
     transfer_session = TransferSession.objects.create(
         id=uuid.uuid4().hex,
         sync_session=sync_session,
-        push=True,
+        push=push,
         last_activity_timestamp=last_activity_timestamp,
+        filter="1:2\n"
     )
 
     return sync_session, transfer_session
@@ -100,6 +103,43 @@ def test_filtering_sessions_cleared(self):
         self.assertTransferSessionIsNotCleared(self.transfersession_new)
         self.assertSyncSessionIsActive(self.syncsession_new)
 
+    def test_filtering_sessions_by_client_instance_id_cleared(self):
+        call_command("cleanupsyncs", client_instance_id=self.syncsession_old.client_instance_id, expiration=0)
+        self.assertTransferSessionIsCleared(self.transfersession_old)
+        self.assertSyncSessionIsNotActive(self.syncsession_old)
+        self.assertTransferSessionIsNotCleared(self.transfersession_new)
+        self.assertSyncSessionIsActive(self.syncsession_new)
+
+    def test_filtering_sessions_by_server_instance_id_cleared(self):
+        call_command("cleanupsyncs", server_instance_id=self.syncsession_old.server_instance_id, expiration=0)
+        self.assertTransferSessionIsCleared(self.transfersession_old)
+        self.assertSyncSessionIsNotActive(self.syncsession_old)
+        self.assertTransferSessionIsNotCleared(self.transfersession_new)
+        self.assertSyncSessionIsActive(self.syncsession_new)
+
+    def test_filtering_sessions_by_sync_filter_cleared(self):
+        call_command("cleanupsyncs", sync_filter=self.transfersession_old.filter, expiration=0)
+        self.assertTransferSessionIsCleared(self.transfersession_old)
+        self.assertSyncSessionIsNotActive(self.syncsession_old)
+        self.assertTransferSessionIsCleared(self.transfersession_new)
+        self.assertSyncSessionIsNotActive(self.syncsession_new)
+
+    def test_filtering_sessions_by_push_cleared(self):
+        call_command("cleanupsyncs", push=self.transfersession_old.push, expiration=0)
+        self.assertTransferSessionIsCleared(self.transfersession_old)
+        self.assertSyncSessionIsNotActive(self.syncsession_old)
+        self.assertTransferSessionIsCleared(self.transfersession_new)
+        self.assertSyncSessionIsNotActive(self.syncsession_new)
+
+    def test_filtering_sessions_by_pull_cleared(self):
+        syncsession_old, transfersession_old = _create_sessions(push=False)
+        syncsession_new, transfersession_new = _create_sessions(push=False)
+        call_command("cleanupsyncs", pull=not transfersession_old.push, expiration=0)
+        self.assertTransferSessionIsCleared(transfersession_old)
+        self.assertSyncSessionIsActive(syncsession_old)
+        self.assertTransferSessionIsCleared(transfersession_new)
+        self.assertSyncSessionIsActive(syncsession_new)
+
     def test_multiple_ids_as_list(self):
         ids = [self.syncsession_old.id, self.syncsession_new.id]
         call_command("cleanupsyncs", ids=ids, expiration=0)

tox.ini

@@ -26,6 +26,7 @@ basepython =
 deps =
   -r{toxinidir}/requirements/test.txt
   cryptography3.3: cryptography==3.3.2
+  py2.7: backports.test.support==0.1.1
 
 commands =
   sh -c '! tests/testapp/manage.py makemigrations --dry-run --exit --noinput'