diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 81619468..5ebde347 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -14,15 +14,13 @@ on: jobs: deploy: - - runs-on: ubuntu-20.04 - + runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: - python-version: 2.7 + python-version: 3.9 - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/python2-lint.yml b/.github/workflows/python2-lint.yml index 668305c0..81ae042b 100644 --- a/.github/workflows/python2-lint.yml +++ b/.github/workflows/python2-lint.yml @@ -19,13 +19,11 @@ jobs: name: Python 2 syntax checking needs: pre_job if: ${{ needs.pre_job.outputs.should_skip != 'true' }} - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 + container: + image: python:2.7.18-buster steps: - uses: actions/checkout@v2 - - name: Set up Python 2.7 - uses: actions/setup-python@v2 - with: - python-version: 2.7 - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index c5c93eeb..80f00f8a 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -18,67 +18,134 @@ jobs: unit_test: name: Python unit tests needs: pre_job - if: ${{ needs.pre_job.outputs.should_skip != 'true' }} runs-on: ubuntu-20.04 strategy: max-parallel: 5 matrix: - python-version: [2.7, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11'] + python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11'] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Install tox + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} run: | python -m pip install --upgrade pip pip install tox - name: tox env cache + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + uses: actions/cache@v2 + with: + path: ${{ github.workspace }}/.tox/py${{ matrix.python-version }} + key: ${{ runner.os }}-tox-py${{ matrix.python-version }}-${{ hashFiles('setup.py', 'requirements/*.txt') }} + - name: Test with tox + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + run: tox -e py${{ matrix.python-version }} + unit_test_deprecated: + name: Python unit tests (deprecated python versions) + needs: pre_job + runs-on: ubuntu-20.04 + strategy: + max-parallel: 3 + matrix: + python-version: [2.7] + container: + image: python:${{ matrix.python-version }} + steps: + - uses: actions/checkout@v2 + - name: Install tox + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + run: | + python -m pip install --upgrade pip + pip install tox + - name: tox env cache + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} uses: actions/cache@v2 with: path: ${{ github.workspace }}/.tox/py${{ matrix.python-version }} key: ${{ runner.os }}-tox-py${{ matrix.python-version }}-${{ hashFiles('setup.py', 'requirements/*.txt') }} - name: Test with tox + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} run: tox -e py${{ matrix.python-version }} cryptography: name: Python unit tests + cryptography needs: pre_job - if: ${{ needs.pre_job.outputs.should_skip != 'true' }} runs-on: ubuntu-20.04 strategy: max-parallel: 5 matrix: # only crypto 3.3 seems to work - python-version: [ 2.7, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11' ] + python-version: [ 3.6, 3.7, 3.8, 3.9, '3.10', '3.11' ] crypto-version: [ 3.3 ] steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install system dependencies - run: | - sudo apt-get -y -qq update - sudo apt-get install -y openssl libssl-dev - - name: Install tox - run: | - python -m pip install --upgrade pip - pip install tox - - name: tox env cache - uses: actions/cache@v2 - with: - path: ${{ github.workspace }}/.tox/py${{ matrix.python-version }} - key: ${{ runner.os }}-tox-py${{ matrix.python-version }}-crypto${{ matrix.crypto-version }}-${{ hashFiles('setup.py', 'requirements/*.txt') }} - - name: Test with tox - run: tox -e py${{ matrix.python-version }}-cryptography${{ matrix.crypto-version }} + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install system dependencies + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + run: | + sudo apt-get -y -qq update + sudo apt-get install -y openssl libssl-dev + - name: Install tox + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + run: | + python -m pip install --upgrade pip + pip install tox + - name: tox env cache + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + uses: actions/cache@v2 + with: + path: ${{ github.workspace }}/.tox/py${{ matrix.python-version }} + key: ${{ runner.os }}-tox-py${{ matrix.python-version }}-crypto${{ matrix.crypto-version }}-${{ hashFiles('setup.py', 'requirements/*.txt') }} + - name: Test with tox + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + run: tox -e py${{ matrix.python-version }}-cryptography${{ matrix.crypto-version }} + cryptography_deprecated: + name: Python unit tests (deprecated python versions) + cryptography + needs: pre_job + runs-on: ubuntu-20.04 + strategy: + max-parallel: 3 + matrix: + # Packages are no longer available for the distribution that comes with the + # python:3.4 image, so we can't install the necessary dependencies. Eventually, + # this will happen to the others as well. + # EOL: [3.4] + python-version: [2.7] + container: + image: python:${{ matrix.python-version }} + steps: + - uses: actions/checkout@v2 + - name: Install system dependencies + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + run: | + apt-get -y -qq update + apt-get install -y openssl libssl-dev + - name: Install tox + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + run: | + python -m pip install --upgrade pip + pip install tox + - name: tox env cache + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + uses: actions/cache@v2 + with: + path: ${{ github.workspace }}/.tox/py${{ matrix.python-version }} + key: ${{ runner.os }}-tox-py${{ matrix.python-version }}-crypto3.3-${{ hashFiles('setup.py', 'requirements/*.txt') }} + - name: Test with tox + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + run: tox -e py${{ matrix.python-version }}-cryptography3.3 postgres: name: Python postgres unit tests needs: pre_job - if: ${{ needs.pre_job.outputs.should_skip != 'true' }} runs-on: ubuntu-20.04 services: # Label used to access the service container @@ -102,19 +169,23 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up Python 3.9 for Postgres + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} uses: actions/setup-python@v2 with: python-version: 3.9 - name: Install tox + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} run: | python -m pip install --upgrade pip pip install tox - name: tox env cache + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} uses: actions/cache@v2 with: - path: ${{ github.workspace }}/.tox/py${{ matrix.python-version }} - key: ${{ runner.os }}-tox-py${{ matrix.python-version }}-${{ hashFiles('setup.py', 'requirements/*.txt') }} + path: ${{ github.workspace }}/.tox/py3.6 + key: ${{ runner.os }}-tox-py3.6-${{ hashFiles('setup.py', 'requirements/*.txt') }} - name: Test with tox + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} run: tox -e postgres windows: name: Python unit tests on Windows Server @@ -123,7 +194,7 @@ jobs: strategy: max-parallel: 5 matrix: - python-version: [3.9] + python-version: [3.8] steps: - uses: actions/checkout@v2 diff --git a/.gitignore b/.gitignore index d40ef0ca..1d20f34b 100644 --- a/.gitignore +++ b/.gitignore @@ -77,14 +77,16 @@ target/ # celery beat schedule file celerybeat-schedule -# dotenv +# dotenv, envrc .env +.envrc # virtualenv venv/ ENV/ .venv Pipfile +.envrc # Spyder project settings .spyderproject @@ -97,4 +99,4 @@ Pipfile *.db -.vscode \ No newline at end of file +.vscode diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..c0da963b --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,12 @@ +version: 2 +formats: all +build: + os: ubuntu-22.04 + tools: + python: "3.9" +sphinx: + configuration: docs/conf.py +python: + install: + - requirements: requirements/docs.txt + - path: . diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a035a2b..092e2bde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ List of the most important changes for each release. +## 0.6.19 +- The `cleanupsyncs` management command now only cleans up sync sessions if also inactive for `expiration` amount of time +- Fixes issue accessing index on queryset in `cleanupsyncs` management command + +## 0.6.18 +- Prevent creation of Deleted and HardDeleted models during deserialization to allow propagation of syncable objects that are recreated after a previous deletion without causing a merge conflict. + +## 0.6.17 +- Added `client-instance-id`, `server-instance-id`, `sync-filter`, `push` and `pull` arguments to `cleanupsyncs` management command +- Added option for resuming a sync to ignore the `SyncSession`'s existing `process_id` +- Added custom user agent to sync HTTP requests +- Fixed documentation build issues +- Makefile no longer defines shell with explicit path + ## 0.6.16 - Added dedicated `client_instance_id` and `server_instance_id` fields to `SyncSession` - Renamed `client_instance` and `server_instance` fields on `SyncSession` to `client_instance_json` and `server_instance_json` respectively diff --git a/docs/conf.py b/docs/conf.py index f15ff743..dbf8cda1 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,6 +20,8 @@ from django.utils.encoding import force_text from django.utils.html import strip_tags +import sphinx_rtd_theme + # If extensions (or modules to document with autodoc) are in another # directory, add these directories to sys.path here. If the directory is # relative to the documentation root, use os.path.abspath to make it @@ -164,14 +166,10 @@ def process_docstring(app, what, name, obj, options, lines): # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' - -if not False: # only import and set the theme if we're building docs locally - import sphinx_rtd_theme - - html_theme = "sphinx_rtd_theme" - html_theme_path = [".", sphinx_rtd_theme.get_html_theme_path()] +# html_theme = 'default' +html_theme = "sphinx_rtd_theme" +html_theme_path = [".", sphinx_rtd_theme.get_html_theme_path()] # Approach 2 for custom stylesheet: # adapted from: http://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html diff --git a/morango/management/commands/cleanupsyncs.py b/morango/management/commands/cleanupsyncs.py index 199fd5fe..4664589f 100644 --- a/morango/management/commands/cleanupsyncs.py +++ b/morango/management/commands/cleanupsyncs.py @@ -1,5 +1,6 @@ import datetime import logging +import uuid from django.core.management.base import BaseCommand from django.db import transaction @@ -29,6 +30,36 @@ def add_arguments(self, parser): default=6, help="Number of hours of inactivity after which a session should be considered stale", ) + parser.add_argument( + "--client-instance-id", + type=uuid.UUID, + default=None, + help="Filters the SyncSession models to those with matching 'client_instance_id'", + ) + parser.add_argument( + "--server-instance-id", + type=uuid.UUID, + default=None, + help="Filters the SyncSession models to those with matching 'server_instance_id'", + ) + parser.add_argument( + "--sync-filter", + type=str, + default=None, + help="Filters the TransferSession models to those with 'filters' starting with 'sync_filter'", + ) + parser.add_argument( + "--push", + type=bool, + default=None, + help="Filters the TransferSession models to those with 'push' set to True", + ) + parser.add_argument( + "--pull", + type=bool, + default=None, + help="Filters the TransferSession models to those with 'push' set to False", + ) def handle(self, *args, **options): @@ -37,10 +68,17 @@ def handle(self, *args, **options): sync_sessions = SyncSession.objects.filter(active=True) - # if ids arg was passed, filter down sessions to only those IDs if included by expiration filter + # if ids arg was passed, filter down sessions to only those IDs + # if included by expiration filter if options["ids"]: sync_sessions = sync_sessions.filter(id__in=options["ids"]) + if options["client_instance_id"]: + sync_sessions = sync_sessions.filter(client_instance_id=options["client_instance_id"]) + + if options["server_instance_id"]: + sync_sessions = sync_sessions.filter(server_instance_id=options["server_instance_id"]) + # retrieve all sessions still marked as active but with no activity since the cutoff transfer_sessions = TransferSession.objects.filter( sync_session_id__in=sync_sessions.values("id"), @@ -48,11 +86,19 @@ def handle(self, *args, **options): last_activity_timestamp__lt=cutoff, ) + if options["sync_filter"]: + transfer_sessions = transfer_sessions.filter(filter__startswith=options["sync_filter"]) + + if options["push"] and not options["pull"]: + transfer_sessions = transfer_sessions.filter(push=True) + + if options["pull"] and not options["push"]: + transfer_sessions = transfer_sessions.filter(push=False) + transfer_count = transfer_sessions.count() # loop over the stale sessions one by one to close them out - for i in range(transfer_count): - transfer_session = transfer_sessions[0] + for i, transfer_session in enumerate(transfer_sessions): logger.info( "TransferSession {} of {}: deleting {} Buffers and {} RMC Buffers...".format( i + 1, @@ -68,17 +114,21 @@ def handle(self, *args, **options): transfer_session.active = False transfer_session.save() + # in order to close a sync session, it must have no active transfer sessions + # and must have no activity since the cutoff + sync_sessions = sync_sessions.filter( + last_activity_timestamp__lt=cutoff, + ).exclude( + transfersession__active=True, + ) sync_count = sync_sessions.count() - # finally loop over sync sessions and close out if there are no other active transfer sessions - for i in range(sync_count): - sync_session = sync_sessions[0] - if not sync_session.transfersession_set.filter(active=True).exists(): - logger.info( - "Closing SyncSession {} of {}".format( - i + 1, - sync_count, - ) + for i, sync_session in enumerate(sync_sessions): + logger.info( + "Closing SyncSession {} of {}".format( + i + 1, + sync_count, ) - sync_session.active = False - sync_session.save() + ) + sync_session.active = False + sync_session.save() diff --git a/morango/models/core.py b/morango/models/core.py index 40377329..ad801031 100644 --- a/morango/models/core.py +++ b/morango/models/core.py @@ -17,6 +17,7 @@ from django.db.models import Func from django.db.models import Max from django.db.models import Q +from django.db.models import signals from django.db.models import TextField from django.db.models import Value from django.db.models.deletion import Collector @@ -468,13 +469,13 @@ def _deserialize_store_model(self, fk_cache, defer_fks=False): # noqa: C901 klass_model = syncable_models.get_model(self.profile, self.model_name) # if store model marked as deleted, attempt to delete in app layer if self.deleted: - # if hard deleted, propagate to related models - if self.hard_deleted: - try: - klass_model.objects.get(id=self.id).delete(hard_delete=True) - except klass_model.DoesNotExist: - pass - else: + # Don't differentiate between deletion and hard deletion here, + # as we don't want to add additional tracking for models in either case, + # just to actually delete them. + # Import here to avoid circular import, as the utils module + # imports core models. + from morango.sync.utils import mute_signals + with mute_signals(signals.post_delete): klass_model.objects.filter(id=self.id).delete() return None, deferred_fks else: diff --git a/morango/sync/session.py b/morango/sync/session.py index 09a04fb5..bbe03161 100644 --- a/morango/sync/session.py +++ b/morango/sync/session.py @@ -1,11 +1,13 @@ import logging from requests import exceptions +from morango import __version__ from requests.sessions import Session from requests.utils import super_len from requests.packages.urllib3.util.url import parse_url from morango.utils import serialize_capabilities_to_client_request +from morango.utils import SETTINGS logger = logging.getLogger(__name__) @@ -35,6 +37,15 @@ class SessionWrapper(Session): bytes_sent = 0 bytes_received = 0 + def __init__(self): + super(SessionWrapper, self).__init__() + user_agent_header = "morango/{}".format(__version__) + if SETTINGS.CUSTOM_INSTANCE_INFO is not None: + instances = list(SETTINGS.CUSTOM_INSTANCE_INFO) + if instances: + user_agent_header += " " + "{}/{}".format(instances[0], SETTINGS.CUSTOM_INSTANCE_INFO.get(instances[0])) + self.headers["User-Agent"] = "{} {}".format(user_agent_header, self.headers["User-Agent"]) + def request(self, method, url, **kwargs): response = None try: diff --git a/morango/sync/syncsession.py b/morango/sync/syncsession.py index ed455af2..1b9d1463 100644 --- a/morango/sync/syncsession.py +++ b/morango/sync/syncsession.py @@ -260,12 +260,16 @@ def create_sync_session(self, client_cert, server_cert, chunk_size=None): sync_session = SyncSession.objects.create(**data) return SyncSessionClient(self, sync_session) - def resume_sync_session(self, sync_session_id, chunk_size=None): + def resume_sync_session(self, sync_session_id, chunk_size=None, ignore_existing_process=False): """ Resumes an existing sync session given an ID :param sync_session_id: The UUID of the `SyncSession` to resume :param chunk_size: An optional parameter specifying the size for each transferred chunk + :type chunk_size: int + :param ignore_existing_process:An optional parameter specifying whether to ignore an + existing active process ID + :type ignore_existing_process: bool :return: A SyncSessionClient instance :rtype: SyncSessionClient """ @@ -281,7 +285,8 @@ def resume_sync_session(self, sync_session_id, chunk_size=None): # check that process of existing session isn't still running if ( - sync_session.process_id + not ignore_existing_process + and sync_session.process_id and sync_session.process_id != os.getpid() and pid_exists(sync_session.process_id) ): diff --git a/requirements/docs.txt b/requirements/docs.txt index 40f8132c..3b2850d7 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -15,5 +15,3 @@ djangorestframework==3.9.1 django-mptt==0.9.1 rsa==3.4.2 ifcfg==0.21 - --r ./accelerated.txt diff --git a/tests/testapp/tests/compat.py b/tests/testapp/tests/compat.py index 508a1844..16b1c8a1 100644 --- a/tests/testapp/tests/compat.py +++ b/tests/testapp/tests/compat.py @@ -1,5 +1,10 @@ try: - from test.support import EnvironmentVarGuard # noqa F401 + # In the Python 2.7 GH workflows, we have to install backported version + from backports.test.support import EnvironmentVarGuard # noqa F401 except ImportError: - # In Python 3.10, this has been moved to test.support.os_helper - from test.support.os_helper import EnvironmentVarGuard # noqa F401 + try: + # For python >2.7 and <3.10 + from test.support import EnvironmentVarGuard # noqa F401 + except ImportError: + # In Python 3.10, this has been moved to test.support.os_helper + from test.support.os_helper import EnvironmentVarGuard # noqa F401 diff --git a/tests/testapp/tests/integration/test_syncsession.py b/tests/testapp/tests/integration/test_syncsession.py index afd88ad1..8dda23c0 100644 --- a/tests/testapp/tests/integration/test_syncsession.py +++ b/tests/testapp/tests/integration/test_syncsession.py @@ -350,3 +350,46 @@ def test_resume(self): with second_environment(): self.assertEqual(5, SummaryLog.objects.filter(user=self.remote_user).count()) self.assertEqual(5, InteractionLog.objects.filter(user=self.remote_user).count()) + + def test_create_sync_delete_sync_recreate_sync(self): + with second_environment(): + SummaryLog.objects.create(user=self.remote_user) + summ_log = SummaryLog.objects.first() + summ_log_id = summ_log.id + content_id = summ_log.content_id + + self.assertEqual(0, SummaryLog.objects.filter(id=summ_log_id).count()) + + # first pull + pull_client = self.client.get_pull_client() + pull_client.initialize(self.filter) + transfer_session = pull_client.context.transfer_session + self.assertEqual(1, transfer_session.records_total) + self.assertEqual(0, transfer_session.records_transferred) + pull_client.run() + self.assertEqual(1, transfer_session.records_transferred) + pull_client.finalize() + + # sanity check pull worked + self.assertEqual(1, SummaryLog.objects.filter(id=summ_log_id).count()) + + with second_environment(): + SummaryLog.objects.get(id=summ_log_id).delete() + + # now do another pull, which should pull in the deletion + second_pull_client = self.client.get_pull_client() + second_pull_client.initialize(self.filter) + second_pull_client.run() + second_pull_client.finalize() + self.assertEqual(0, SummaryLog.objects.filter(id=summ_log_id).count()) + + with second_environment(): + sum_log = SummaryLog.objects.create(user=self.remote_user, content_id=content_id) + self.assertEqual(sum_log.id, summ_log_id) + + # now do another pull, which should pull in the recreation + third_pull_client = self.client.get_pull_client() + third_pull_client.initialize(self.filter) + third_pull_client.run() + third_pull_client.finalize() + self.assertEqual(1, SummaryLog.objects.filter(id=summ_log_id).count()) diff --git a/tests/testapp/tests/sync/test_controller.py b/tests/testapp/tests/sync/test_controller.py index b0c16dd3..f009e216 100644 --- a/tests/testapp/tests/sync/test_controller.py +++ b/tests/testapp/tests/sync/test_controller.py @@ -18,7 +18,6 @@ from morango.constants import transfer_statuses from morango.models.certificates import Filter from morango.models.core import DeletedModels -from morango.models.core import HardDeletedModels from morango.models.core import InstanceIDModel from morango.models.core import RecordMaxCounter from morango.models.core import Store @@ -287,7 +286,7 @@ def test_store_hard_delete_propagates(self): ) # make sure hard_deleted propagates to related models even if they are not hard_deleted self.mc.deserialize_from_store() - self.assertTrue(HardDeletedModels.objects.filter(id=log.id).exists()) + self.assertFalse(SummaryLog.objects.filter(id=log.id).exists()) class RecordMaxCounterUpdatesDuringSerialization(TestCase): diff --git a/tests/testapp/tests/sync/test_session.py b/tests/testapp/tests/sync/test_session.py index 9e96dfcf..25c5baf7 100644 --- a/tests/testapp/tests/sync/test_session.py +++ b/tests/testapp/tests/sync/test_session.py @@ -24,6 +24,19 @@ def test_request(self, mocked_super_request): head_length = len("HTTP/1.1 200 OK") + _length_of_headers(headers) self.assertEqual(wrapper.bytes_received, 1024 + head_length) + def test_request_user_agent(self): + from morango import __version__ as morango_version + from requests import __version__ as requests_version + + wrapper = SessionWrapper() + expected_user_agent = "morango/{} python-requests/{}".format(morango_version, requests_version) + self.assertEqual(wrapper.headers["User-Agent"], expected_user_agent) + + with self.settings(CUSTOM_INSTANCE_INFO={"kolibri": "0.16.0"}): + wrapper = SessionWrapper() + expected_user_agent = "morango/{} kolibri/0.16.0 python-requests/{}".format(morango_version, requests_version) + self.assertEqual(wrapper.headers["User-Agent"], expected_user_agent) + @mock.patch("morango.sync.session.logger") @mock.patch("morango.sync.session.Session.request") def test_request__not_ok(self, mocked_super_request, mocked_logger): diff --git a/tests/testapp/tests/sync/test_syncsession.py b/tests/testapp/tests/sync/test_syncsession.py index f71c5e21..93263193 100644 --- a/tests/testapp/tests/sync/test_syncsession.py +++ b/tests/testapp/tests/sync/test_syncsession.py @@ -275,6 +275,30 @@ def create(**data): with self.assertRaises(MorangoResumeSyncError): self.network_connection.resume_sync_session(sync_session.id) + @mock.patch.object(SyncSession.objects, "create") + def test_resume_sync_session__still_running_but_ignore(self, mock_create): + def create(**data): + """Trickery to get around same DB being used for both client and server""" + return SyncSession.objects.get(pk=data.get("id")) + + mock_create.side_effect = create + + # first create a session + client = self.network_connection.create_sync_session(self.subset_cert, self.root_cert) + # reset process ID + sync_session = client.sync_session + sync_session.process_id = 123000111 + sync_session.save() + + with mock.patch("morango.sync.syncsession.pid_exists") as mock_pid_exists: + mock_pid_exists.return_value = True + with mock.patch("morango.sync.syncsession.os.getpid") as mock_getpid: + mock_getpid.return_value = 245111222 + resume_client = self.network_connection.resume_sync_session( + sync_session.id, ignore_existing_process=True + ) + self.assertEqual(sync_session.id, resume_client.sync_session.id) + class SyncSessionClientTestCase(BaseClientTestCase): def test_get_pull_client(self): diff --git a/tests/testapp/tests/test_management_commands.py b/tests/testapp/tests/test_management_commands.py index d7ece572..9428c94e 100644 --- a/tests/testapp/tests/test_management_commands.py +++ b/tests/testapp/tests/test_management_commands.py @@ -10,7 +10,7 @@ from morango.models.core import TransferSession -def _create_sessions(last_activity_offset=0, sync_session=None): +def _create_sessions(last_activity_offset=0, sync_session=None, push=True): last_activity_timestamp = timezone.now() - datetime.timedelta( hours=last_activity_offset @@ -21,13 +21,16 @@ def _create_sessions(last_activity_offset=0, sync_session=None): id=uuid.uuid4().hex, profile="facilitydata", last_activity_timestamp=last_activity_timestamp, + client_instance_id=uuid.uuid4().hex, + server_instance_id=uuid.uuid4().hex, ) transfer_session = TransferSession.objects.create( id=uuid.uuid4().hex, sync_session=sync_session, - push=True, + push=push, last_activity_timestamp=last_activity_timestamp, + filter="1:2\n" ) return sync_session, transfer_session @@ -100,6 +103,45 @@ def test_filtering_sessions_cleared(self): self.assertTransferSessionIsNotCleared(self.transfersession_new) self.assertSyncSessionIsActive(self.syncsession_new) + def test_filtering_sessions_by_client_instance_id_cleared(self): + call_command("cleanupsyncs", client_instance_id=self.syncsession_old.client_instance_id, expiration=0) + self.assertTransferSessionIsCleared(self.transfersession_old) + self.assertSyncSessionIsNotActive(self.syncsession_old) + self.assertTransferSessionIsNotCleared(self.transfersession_new) + self.assertSyncSessionIsActive(self.syncsession_new) + + def test_filtering_sessions_by_server_instance_id_cleared(self): + call_command("cleanupsyncs", server_instance_id=self.syncsession_old.server_instance_id, expiration=0) + self.assertTransferSessionIsCleared(self.transfersession_old) + self.assertSyncSessionIsNotActive(self.syncsession_old) + self.assertTransferSessionIsNotCleared(self.transfersession_new) + self.assertSyncSessionIsActive(self.syncsession_new) + + def test_filtering_sessions_by_sync_filter_cleared(self): + call_command("cleanupsyncs", sync_filter=self.transfersession_old.filter, expiration=0) + self.assertTransferSessionIsCleared(self.transfersession_old) + self.assertSyncSessionIsNotActive(self.syncsession_old) + self.assertTransferSessionIsCleared(self.transfersession_new) + self.assertSyncSessionIsNotActive(self.syncsession_new) + + def test_filtering_sessions_by_push_cleared(self): + call_command("cleanupsyncs", push=self.transfersession_old.push, expiration=0) + self.assertTransferSessionIsCleared(self.transfersession_old) + self.assertSyncSessionIsNotActive(self.syncsession_old) + self.assertTransferSessionIsCleared(self.transfersession_new) + self.assertSyncSessionIsNotActive(self.syncsession_new) + + def test_filtering_sessions_by_pull_cleared(self): + syncsession_old, transfersession_old = _create_sessions(push=False) + syncsession_new, transfersession_new = _create_sessions(push=False) + call_command("cleanupsyncs", pull=not transfersession_old.push, expiration=0) + self.assertTransferSessionIsCleared(transfersession_old) + self.assertSyncSessionIsNotActive(syncsession_old) + self.assertTransferSessionIsCleared(transfersession_new) + self.assertSyncSessionIsNotActive(syncsession_new) + self.assertTransferSessionIsNotCleared(self.transfersession_old) + self.assertTransferSessionIsNotCleared(self.transfersession_new) + def test_multiple_ids_as_list(self): ids = [self.syncsession_old.id, self.syncsession_new.id] call_command("cleanupsyncs", ids=ids, expiration=0) @@ -107,3 +149,29 @@ def test_multiple_ids_as_list(self): self.assertSyncSessionIsNotActive(self.syncsession_old) self.assertTransferSessionIsCleared(self.transfersession_new) self.assertSyncSessionIsNotActive(self.syncsession_new) + + def test_sync_session_cutoff(self): + """ + Test that sync sessions are not cleared even if they have no active transfer sessions, + if they are still within the cutoff window. + """ + sync_session, transfer_session = _create_sessions(34) + # recent successful transfer session + transfer_session.active = False + transfer_session.save() + # create old incomplete transfer session for same session + _, old_transfer_session = _create_sessions(38, sync_session=sync_session) + + call_command("cleanupsyncs", expiration=36) + self.assertSyncSessionIsActive(sync_session) + + def test_sync_session_cleanup_with_active_xfer(self): + sync_session, transfer_session = _create_sessions(38) + # recent successful transfer session + transfer_session.active = False + transfer_session.save() + # create old incomplete transfer session for same session + _, new_transfer_session = _create_sessions(34, sync_session=sync_session) + + call_command("cleanupsyncs", expiration=36) + self.assertSyncSessionIsActive(sync_session) diff --git a/tests/testapp/tests/test_uuid_utilities.py b/tests/testapp/tests/test_uuid_utilities.py index 2c4eb600..74c7c1c4 100644 --- a/tests/testapp/tests/test_uuid_utilities.py +++ b/tests/testapp/tests/test_uuid_utilities.py @@ -159,7 +159,7 @@ def test_consistent_0_5_instance_id(self, *args): DatabaseIDModel.objects.all().update(current=False) DatabaseIDModel.objects.create( id="7fe445b75cea11858c00fb97bdee8878", current=True - ).id + ) self.assertEqual(get_0_5_system_id(), "54940f560a55bbf7d86b") self.assertEqual(get_0_5_mac_address(), "804f4c20d3b2b5a29b95") diff --git a/tox.ini b/tox.ini index 62ce4e50..3ddfee3a 100644 --- a/tox.ini +++ b/tox.ini @@ -21,11 +21,12 @@ basepython = py3.10: python3.10 py3.11: python3.11 postgres: python3.9 - windows: python3.9 + windows: python3.8 deps = -r{toxinidir}/requirements/test.txt cryptography3.3: cryptography==3.3.2 + py2.7: backports.test.support==0.1.1 commands = sh -c '! tests/testapp/manage.py makemigrations --dry-run --exit --noinput'