Refactor sync (#3312)

docs/user/localizing-your-projects.rst

@@ -68,11 +68,7 @@ following required fields:
 #. **Locales**: select at least one Localizable locale by clicking on it.
 #. **Repository URL**: enter your repository's SSH URL of the form
    ``[email protected]:user/repo.git``.
-#. **Download prefix or path to TOML file**: a URL prefix for downloading localized files. For
-   GitHub repositories, select any localized file on GitHub, click ``Raw`` and
-   replace locale code and the following bits in the URL with ``{locale_code}``.
-   If you use one, you need to select the `project configuration file`_ instead
-   of a localized file.
+#. **Download prefix or path to TOML file**: a URL prefix for downloading localized files.
 #. Click **SAVE PROJECT** at the bottom of the page.
 #. After the page reloads, click **SYNC** and wait for Pontoon to import
    strings. You can monitor the progress in the Sync log (``/sync/log/``).

pontoon/administration/views.py

@@ -30,9 +30,9 @@
     Translation,
 )
 from pontoon.base.utils import require_AJAX
-from pontoon.pretranslation.tasks import pretranslate
+from pontoon.pretranslation.tasks import pretranslate_task
 from pontoon.sync.models import SyncLog
-from pontoon.sync.tasks import sync_project
+from pontoon.sync.tasks import sync_project_task
 
 
 log = logging.getLogger(__name__)
@@ -431,7 +431,7 @@ def _create_or_update_translated_resources(
         resource = _get_resource_for_database_project(project)
 
     for locale in locales:
-        tr, created = TranslatedResource.objects.get_or_create(
+        tr, _ = TranslatedResource.objects.get_or_create(
             locale_id=locale.pk,
             resource=resource,
         )
@@ -542,9 +542,9 @@ def manually_sync_project(request, slug):
             "Forbidden: You don't have permission for syncing projects"
         )
 
-    sync_log = SyncLog.objects.create(start_time=timezone.now())
     project = Project.objects.get(slug=slug)
-    sync_project.delay(project.pk, sync_log.pk)
+    sync_log = SyncLog.objects.create(start_time=timezone.now())
+    sync_project_task.delay(project.pk, sync_log.pk)
 
     return HttpResponse("ok")
 
@@ -558,6 +558,6 @@ def manually_pretranslate_project(request, slug):
         )
 
     project = Project.objects.get(slug=slug)
-    pretranslate.delay(project.pk)
+    pretranslate_task.delay(project.pk)
 
     return HttpResponse("ok")

pontoon/base/__init__.py

@@ -1,13 +0,0 @@
-MOZILLA_REPOS = (
-    "ssh://hg.mozilla.org/users/m_owca.info/firefox-beta/",
-    "ssh://hg.mozilla.org/users/m_owca.info/firefox-for-android-beta/",
-    "ssh://hg.mozilla.org/users/m_owca.info/thunderbird-beta/",
-    "ssh://hg.mozilla.org/users/m_owca.info/lightning-beta/",
-    "ssh://hg.mozilla.org/users/m_owca.info/seamonkey-beta/",
-    "ssh://hg.mozilla.org/users/m_owca.info/firefox-central/",
-    "ssh://hg.mozilla.org/users/m_owca.info/firefox-for-android-central/",
-    "ssh://hg.mozilla.org/users/m_owca.info/thunderbird-central/",
-    "ssh://hg.mozilla.org/users/m_owca.info/lightning-central/",
-    "ssh://hg.mozilla.org/users/m_owca.info/seamonkey-central/",
-    "[email protected]:seamonkey-project/seamonkey-central-l10n.git",
-)

pontoon/base/management/commands/calculate_stats.py

@@ -3,10 +3,8 @@
 from django.core.management.base import BaseCommand
 from django.db.models import Count
 
-from pontoon.base.models import (
-    Project,
-    TranslatedResource,
-)
+from pontoon.base.models import Project
+from pontoon.sync.core.stats import update_locale_stats, update_stats
 
 
 log = logging.getLogger(__name__)
@@ -34,20 +32,9 @@ def handle(self, *args, **options):
             "disabled", "resource_count"
         )
 
-        for index, project in enumerate(projects):
-            log.info(
-                'Calculating stats for project "{project}" ({index}/{total})'.format(
-                    index=index + 1,
-                    total=len(projects),
-                    project=project.name,
-                )
-            )
-
-            translated_resources = TranslatedResource.objects.filter(
-                resource__project=project
-            )
-
-            for translated_resource in translated_resources:
-                translated_resource.calculate_stats()
+        log.info(f"Calculating stats for {len(projects)} projects...")
+        for project in projects:
+            update_stats(project, update_locales=False)
+        update_locale_stats()
 
         log.info("Calculating stats complete for all projects.")

pontoon/base/migrations/0018_populate_entity_context.py

@@ -3,22 +3,20 @@
 from django.db import migrations
 from django.db.models import F, Func, TextField, Value
 
-from pontoon.sync import KEY_SEPARATOR
-
 
 def add_entity_context(apps, schema_editor):
     Entity = apps.get_model("base", "Entity")
 
     split_key_po = Func(
         F("key"),
-        Value(KEY_SEPARATOR),
+        Value("\x04"),
         Value(1),
         function="split_part",
         output_field=TextField(),
     )
     split_key_xliff = Func(
         F("key"),
-        Value(KEY_SEPARATOR),
+        Value("\x04"),
         Value(2),
         function="split_part",
         output_field=TextField(),

pontoon/base/models/changed_entity_locale.py

@@ -1,15 +1,21 @@
+from typing import TYPE_CHECKING
+
 from django.db import models
 from django.utils import timezone
 
 
+if TYPE_CHECKING:
+    from pontoon.base.models import Entity, Locale
+
+
 class ChangedEntityLocale(models.Model):
     """
     ManyToMany model for storing what locales have changed translations for a
     specific entity since the last sync.
     """
 
-    entity = models.ForeignKey("Entity", models.CASCADE)
-    locale = models.ForeignKey("Locale", models.CASCADE)
+    entity: models.ForeignKey["Entity"] = models.ForeignKey("Entity", models.CASCADE)
+    locale: models.ForeignKey["Locale"] = models.ForeignKey("Locale", models.CASCADE)
     when = models.DateTimeField(default=timezone.now)
 
     class Meta:

pontoon/base/models/entity.py

@@ -1,3 +1,4 @@
+from collections.abc import Iterable
 from functools import reduce
 from operator import ior
 from re import escape, findall, match
@@ -14,7 +15,6 @@
 from pontoon.base.models.project import Project
 from pontoon.base.models.project_locale import ProjectLocale
 from pontoon.base.models.resource import Resource
-from pontoon.sync import KEY_SEPARATOR
 
 
 def get_word_count(string):
@@ -443,50 +443,6 @@ def prefetch_entities_data(self, locale, preferred_source_locale):
 
         return entities
 
-    def reset_active_translations(self, locale):
-        """
-        Reset active translation for given set of entities and locale.
-        """
-        from pontoon.base.models.translation import Translation
-
-        translations = Translation.objects.filter(
-            entity__in=self,
-            locale=locale,
-        )
-
-        # First, deactivate all translations
-        translations.update(active=False)
-
-        # Mark all approved, pretranslated and fuzzy translations as active.
-        translations.filter(
-            Q(approved=True) | Q(pretranslated=True) | Q(fuzzy=True)
-        ).update(active=True)
-
-        # Mark most recent unreviewed suggestions without active siblings
-        # for any given combination of (locale, entity, plural_form) as active.
-        unreviewed_pks = set()
-        unreviewed = translations.filter(
-            approved=False,
-            pretranslated=False,
-            fuzzy=False,
-            rejected=False,
-        ).values_list("entity", "plural_form")
-
-        for entity, plural_form in unreviewed:
-            siblings = (
-                Translation.objects.filter(
-                    entity=entity,
-                    locale=locale,
-                    plural_form=plural_form,
-                )
-                .exclude(rejected=True)
-                .order_by("-active", "-date")
-            )
-            if siblings and not siblings[0].active:
-                unreviewed_pks.add(siblings[0].pk)
-
-        translations.filter(pk__in=unreviewed_pks).update(active=True)
-
     def get_or_create(self, defaults=None, **kwargs):
         kwargs["word_count"] = get_word_count(kwargs["string"])
         return super().get_or_create(defaults=defaults, **kwargs)
@@ -532,18 +488,6 @@ class Meta:
             models.Index(fields=["resource", "obsolete", "string_plural"]),
         ]
 
-    @property
-    def cleaned_key(self):
-        """
-        Get cleaned key, without the source string and Translate Toolkit
-        separator.
-        """
-        key = self.key.split(KEY_SEPARATOR)[0]
-        if key == self.string:
-            key = ""
-
-        return key
-
     def __str__(self):
         return self.string
 
@@ -559,90 +503,35 @@ def get_stats(self, locale):
         :return: a dictionary with stats for an Entity, all keys are suffixed with `_diff` to
             make them easier to pass into adjust_all_stats.
         """
-        translations = list(
-            self.translation_set.filter(locale=locale).prefetch_related(
-                "errors",
-                "warnings",
-            )
-        )
-
-        approved_strings_count = len(
-            [
-                t
-                for t in translations
-                if t.approved and not (t.errors.exists() or t.warnings.exists())
-            ]
-        )
-
-        pretranslated_strings_count = len(
-            [
-                t
-                for t in translations
-                if t.pretranslated and not (t.errors.exists() or t.warnings.exists())
-            ]
-        )
-
-        if self.string_plural:
-            approved = int(approved_strings_count == locale.nplurals)
-            pretranslated = int(pretranslated_strings_count == locale.nplurals)
-
-        else:
-            approved = int(approved_strings_count > 0)
-            pretranslated = int(pretranslated_strings_count > 0)
-
-        if not (approved or pretranslated):
-            has_errors = bool(
-                [
-                    t
-                    for t in translations
-                    if (t.approved or t.pretranslated or t.fuzzy) and t.errors.exists()
-                ]
-            )
-            has_warnings = bool(
-                [
-                    t
-                    for t in translations
-                    if (t.approved or t.pretranslated or t.fuzzy)
-                    and t.warnings.exists()
-                ]
-            )
-
-            errors = int(has_errors)
-            warnings = int(has_warnings)
-
-        else:
-            errors = 0
-            warnings = 0
-
-        unreviewed_count = len(
-            [
-                t
-                for t in translations
-                if not (t.approved or t.pretranslated or t.fuzzy or t.rejected)
-            ]
-        )
+        approved = 0
+        pretranslated = 0
+        errors = 0
+        warnings = 0
+        unreviewed = 0
+
+        for t in self.translation_set.filter(locale=locale).prefetch_related(
+            "errors", "warnings"
+        ):
+            if t.errors.exists():
+                if t.approved or t.pretranslated or t.fuzzy:
+                    errors += 1
+            elif t.warnings.exists():
+                if t.approved or t.pretranslated or t.fuzzy:
+                    warnings += 1
+            elif t.approved:
+                approved += 1
+            elif t.pretranslated:
+                pretranslated += 1
+            if not (t.approved or t.pretranslated or t.fuzzy or t.rejected):
+                unreviewed += 1
 
         return {
             "total_strings_diff": 0,
             "approved_strings_diff": approved,
             "pretranslated_strings_diff": pretranslated,
             "strings_with_errors_diff": errors,
             "strings_with_warnings_diff": warnings,
-            "unreviewed_strings_diff": unreviewed_count,
-        }
-
-    @classmethod
-    def get_stats_diff(cls, stats_before, stats_after):
-        """
-        Return stat difference between the two states of the entity.
-
-        :arg dict stats_before: dict returned by get_stats() for the initial state.
-        :arg dict stats_after: dict returned by get_stats() for the current state.
-        :return: dictionary with differences between provided stats.
-        """
-        return {
-            stat_name: stats_after[stat_name] - stats_before[stat_name]
-            for stat_name in stats_before
+            "unreviewed_strings_diff": unreviewed,
         }
 
     def has_changed(self, locale):
@@ -942,7 +831,9 @@ def map_entities(
     ):
         entities_array = []
 
-        entities = entities.prefetch_entities_data(locale, preferred_source_locale)
+        entities: Iterable[Entity] = entities.prefetch_entities_data(
+            locale, preferred_source_locale
+        )
 
         # If requested entity not in the current page
         if requested_entity and requested_entity not in [e.pk for e in entities]:
@@ -981,13 +872,18 @@ def map_entities(
                 if original_plural != "":
                     original_plural = entity.alternative_originals[-1].string
 
+            key_separator = "\x04"
+            cleaned_key = entity.key.split(key_separator)[0]
+            if cleaned_key == entity.string:
+                cleaned_key = ""
+
             entities_array.append(
                 {
                     "pk": entity.pk,
                     "original": original,
                     "original_plural": original_plural,
                     "machinery_original": entity.string,
-                    "key": entity.cleaned_key,
+                    "key": cleaned_key,
                     "context": entity.context,
                     "path": entity.resource.path,
                     "project": entity.resource.project.serialize(),