From 8fca67a2959a0ff5ecc9accd7af57b43e87b1cdc Mon Sep 17 00:00:00 2001 From: rafalp Date: Thu, 5 Sep 2024 23:34:07 +0200 Subject: [PATCH] Add heal category trees utils --- misago/categories/delete.py | 5 + .../management/commands/fixcategoriestree.py | 31 ---- .../management/commands/healcategorytrees.py | 30 ++++ .../migrations/0002_default_categories.py | 8 +- misago/categories/mptt.py | 72 +++++++++ .../tests/test_fixcategoriestree.py | 107 ------------- .../tests/test_heal_category_trees.py | 142 ++++++++++++++++++ .../tests/test_healcategorytrees.py | 39 +++++ 8 files changed, 292 insertions(+), 142 deletions(-) create mode 100644 misago/categories/delete.py delete mode 100644 misago/categories/management/commands/fixcategoriestree.py create mode 100644 misago/categories/management/commands/healcategorytrees.py create mode 100644 misago/categories/mptt.py delete mode 100644 misago/categories/tests/test_fixcategoriestree.py create mode 100644 misago/categories/tests/test_heal_category_trees.py create mode 100644 misago/categories/tests/test_healcategorytrees.py diff --git a/misago/categories/delete.py b/misago/categories/delete.py new file mode 100644 index 0000000000..3a4d22ecde --- /dev/null +++ b/misago/categories/delete.py @@ -0,0 +1,5 @@ +from .models import Category + + +def delete_category(category: Category): + category.delete() diff --git a/misago/categories/management/commands/fixcategoriestree.py b/misago/categories/management/commands/fixcategoriestree.py deleted file mode 100644 index 356d95e4b0..0000000000 --- a/misago/categories/management/commands/fixcategoriestree.py +++ /dev/null @@ -1,31 +0,0 @@ -from django.core.management.base import BaseCommand - -from ....acl.cache import clear_acl_cache -from ....cache.enums import CacheName -from ....cache.versions import invalidate_cache -from ...models import Category - - -class Command(BaseCommand): - """ - This command rebuilds the thread category tree. - It can be useful when the category hierarchy is corrupt due to modifying directly - in the database causing MPTT's nested sets to not align correctly. - A typical case is when injecting default data into the database from outside misago. - """ - - help = "Rebuilds the thread category tree" - - def handle(self, *args, **options): - root = Category.objects.root_category() - Category.objects.partial_rebuild(root.tree_id) - self.stdout.write("Categories tree has been rebuild.") - - invalidate_cache( - CacheName.CATEGORIES, - CacheName.MODERATORS, - CacheName.PERMISSIONS, - ) - - clear_acl_cache() - self.stdout.write("Caches have been cleared.") diff --git a/misago/categories/management/commands/healcategorytrees.py b/misago/categories/management/commands/healcategorytrees.py new file mode 100644 index 0000000000..9aa671eea1 --- /dev/null +++ b/misago/categories/management/commands/healcategorytrees.py @@ -0,0 +1,30 @@ +from django.core.management.base import BaseCommand + +from ....acl.cache import clear_acl_cache +from ....cache.enums import CacheName +from ....cache.versions import invalidate_cache +from ...mptt import heal_category_trees + + +class Command(BaseCommand): + """ + This command rebuilds the category trees in the database. + + It's useful when the MPTT data of one or more categories becomes invalid, + either due to a bug or manual database manipulation. + """ + + help = "Heals the category trees in the database" + + def handle(self, *args, **options): + heal_category_trees() + self.stdout.write("Rebuild category trees in the database.") + + invalidate_cache( + CacheName.CATEGORIES, + CacheName.MODERATORS, + CacheName.PERMISSIONS, + ) + + clear_acl_cache() + self.stdout.write("Cleared caches associated with category trees.") diff --git a/misago/categories/migrations/0002_default_categories.py b/misago/categories/migrations/0002_default_categories.py index 3fdf3172de..be913feae9 100644 --- a/misago/categories/migrations/0002_default_categories.py +++ b/misago/categories/migrations/0002_default_categories.py @@ -23,8 +23,8 @@ def create_default_categories_tree(apps, schema_editor): special_role="root_category", name="Root", slug="root", - lft=3, - rght=6, + lft=1, + rght=4, tree_id=CategoryTreeDeprecated.THREADS, level=0, ) @@ -33,8 +33,8 @@ def create_default_categories_tree(apps, schema_editor): Category.objects.create( parent=root, - lft=4, - rght=5, + lft=2, + rght=3, tree_id=CategoryTreeDeprecated.THREADS, level=1, name=category_name, diff --git a/misago/categories/mptt.py b/misago/categories/mptt.py new file mode 100644 index 0000000000..f537bc68c6 --- /dev/null +++ b/misago/categories/mptt.py @@ -0,0 +1,72 @@ +from .models import Category + +MPTTData = tuple[int, int, int] + + +def heal_category_trees() -> int: + trees: set[int] = set() + categories: dict[int, dict] = {} + + queryset = Category.objects.values( + "id", + "parent_id", + "tree_id", + "level", + "lft", + "rght", + ).order_by("tree_id", "lft") + + for category in queryset: + trees.add(category["tree_id"]) + categories[category["id"]] = category + + healthy_categories: list[dict] = [] + for tree_id in trees: + healthy_categories += heal_tree(tree_id, categories) + + updates = 0 + for healed_category in healthy_categories: + org_category = categories[healed_category["id"]] + if org_category != healed_category: + Category.objects.filter(id=healed_category["id"]).update( + level=healed_category["level"], + lft=healed_category["lft"], + rght=healed_category["rght"], + ) + updates += 1 + + return updates + + +def heal_tree(tree_id: int, categories: dict[int, dict]) -> list[dict]: + tree_categories = { + c["id"]: c.copy() for c in categories.values() if c["tree_id"] == tree_id + } + tree_categories_list = list(tree_categories.values()) + + cursor = 0 + for category in tree_categories_list: + if category["parent_id"]: + continue + + category["level"] = 0 + cursor += 1 + category["lft"] = cursor + category["rght"] = cursor = heal_category(category, tree_categories_list) + 1 + + return sorted(tree_categories_list, key=lambda i: i["lft"]) + + +def heal_category(category: dict, tree_categories_list: list[dict]) -> int: + cursor = category["lft"] + for child in tree_categories_list: + if child["parent_id"] != category["id"]: + continue + + child["level"] = category["level"] + 1 + + cursor += 1 + child["lft"] = cursor + child["rght"] = cursor = heal_category(child, tree_categories_list) + 1 + + return cursor diff --git a/misago/categories/tests/test_fixcategoriestree.py b/misago/categories/tests/test_fixcategoriestree.py deleted file mode 100644 index defc080dd6..0000000000 --- a/misago/categories/tests/test_fixcategoriestree.py +++ /dev/null @@ -1,107 +0,0 @@ -from io import StringIO - -from django.core.management import call_command -from django.test import TestCase - -from ...acl import ACL_CACHE -from ...cache.test import assert_invalidates_cache -from ..management.commands import fixcategoriestree -from ..models import Category - - -def run_command(): - """Run the management command""" - command = fixcategoriestree.Command() - out = StringIO() - call_command(command, stdout=out) - - -class FixCategoriesTreeTests(TestCase): - """ - The purpose is the verify that the management command - fixes the lft/rght values of the thread category tree. - """ - - def setUp(self): - Category.objects.create( - name="Test", slug="test", parent=Category.objects.root_category() - ) - self.fetch_categories() - - def assertValidTree(self, expected_tree): - root = Category.objects.root_category() - queryset = Category.objects.filter(tree_id=root.tree_id).order_by("lft") - - current_tree = [] - for category in queryset: - current_tree.append( - (category, category.get_level(), category.lft, category.rght) - ) - - for i, category in enumerate(expected_tree): - _category = current_tree[i] - if category[0] != _category[0]: - self.fail( - "expected category at index #%s to be %s, found %s instead" - % (i, category[0], _category[0]) - ) - if category[1] != _category[1]: - self.fail( - "expected level at index #%s to be %s, found %s instead" - % (i, category[1], _category[1]) - ) - if category[2] != _category[2]: - self.fail( - "expected lft at index #%s to be %s, found %s instead" - % (i, category[2], _category[2]) - ) - if category[3] != _category[3]: - self.fail( - "expected lft at index #%s to be %s, found %s instead" - % (i, category[3], _category[3]) - ) - - def fetch_categories(self): - """gets a fresh version from the database""" - self.root = Category.objects.root_category() - self.first_category = Category.objects.get(slug="first-category") - self.test_category = Category.objects.get(slug="test") - - def test_fix_categories_tree_unaffected(self): - """Command should not affect a healthy three""" - tree_id = self.root.tree_id - run_command() - - self.fetch_categories() - - self.assertValidTree( - [ - (self.root, 0, 1, 6), - (self.first_category, 1, 2, 3), - (self.test_category, 1, 4, 5), - ] - ) - - self.assertEqual(self.root.tree_id, tree_id, msg="tree_id changed by command") - - def test_fix_categories_tree_affected(self): - """Command should fix a broken tree""" - # Root node with too narrow lft/rght range - Category.objects.filter(id=self.root.id).update(lft=1, rght=4) - # Make conflicting/identical lft/rght range - Category.objects.filter(id=self.test_category.id).update(lft=2, rght=3) - - run_command() - self.fetch_categories() - - self.assertValidTree( - [ - (self.root, 0, 1, 6), - (self.test_category, 1, 2, 3), - (self.first_category, 1, 4, 5), - ] - ) - - def test_fixing_categories_tree_invalidates_acl_cache(self): - with assert_invalidates_cache(ACL_CACHE): - run_command() diff --git a/misago/categories/tests/test_heal_category_trees.py b/misago/categories/tests/test_heal_category_trees.py new file mode 100644 index 0000000000..a1950da091 --- /dev/null +++ b/misago/categories/tests/test_heal_category_trees.py @@ -0,0 +1,142 @@ +import pytest + +from ..delete import delete_category +from ..models import Category +from ..mptt import heal_category_trees + + +@pytest.fixture +def clear_categories(db): + for category in Category.objects.order_by("-lft"): + delete_category(category) + + +def create_category( + tree_id: int, + level: int, + lft: int, + rght: int, + parent_id: int | None = None, +) -> Category: + category = Category.objects.create( + name="Test", + slug="test", + tree_id=0, + level=0, + lft=1, + rght=2, + ) + + Category.objects.filter(id=category.id).update( + parent_id=parent_id, + tree_id=tree_id, + level=level, + lft=lft, + rght=rght, + ) + + category.parent_id = parent_id + category.tree_id = tree_id + category.level = level + category.lft = lft + category.rght = rght + + return category + + +def repr_mptt(category: Category): + return (category.tree_id, category.level, category.lft, category.rght) + + +def test_heal_category_trees_fixes_top_leaf_category(clear_categories): + category = create_category(0, 10, 12, 0) + + heal_category_trees() + + category.refresh_from_db() + assert repr_mptt(category) == (0, 0, 1, 2) + + +def test_heal_category_trees_fixes_separate_trees(clear_categories): + category = create_category(0, 10, 12, 0) + other_category = create_category(1, 15, 4, 10) + + heal_category_trees() + + category.refresh_from_db() + assert repr_mptt(category) == (0, 0, 1, 2) + + other_category.refresh_from_db() + assert repr_mptt(other_category) == (1, 0, 1, 2) + + +def test_heal_category_trees_fixes_overlapping_categories(clear_categories): + category = create_category(0, 10, 5, 10) + other_category = create_category(0, 15, 10, 15) + + heal_category_trees() + + category.refresh_from_db() + assert repr_mptt(category) == (0, 0, 1, 2) + + other_category.refresh_from_db() + assert repr_mptt(other_category) == (0, 0, 3, 4) + + +def test_heal_category_trees_fixes_parents_children(clear_categories): + category = create_category(0, 10, 5, 10) + child_category = create_category(0, 15, 10, 15, category.id) + + heal_category_trees() + + category.refresh_from_db() + assert repr_mptt(category) == (0, 0, 1, 4) + + child_category.refresh_from_db() + assert repr_mptt(child_category) == (0, 1, 2, 3) + + +def test_heal_category_trees_fixes_parents_children_siblings(clear_categories): + category = create_category(0, 10, 5, 10) + sibling_category = create_category(0, 15, 10, 15) + child_category = create_category(0, 15, 10, 15, category.id) + + heal_category_trees() + + category.refresh_from_db() + assert repr_mptt(category) == (0, 0, 1, 4) + + child_category.refresh_from_db() + assert repr_mptt(child_category) == (0, 1, 2, 3) + + sibling_category.refresh_from_db() + assert repr_mptt(sibling_category) == (0, 0, 5, 6) + + +def test_heal_category_trees_fixes_complex_tree(clear_categories): + category = create_category(0, 10, 5, 10) + sibling_category = create_category(0, 15, 10, 15) + child_category = create_category(0, 15, 10, 15, category.id) + deep_child_category = create_category(0, 15, 10, 15, child_category.id) + other_child_category = create_category(0, 15, 11, 18, category.id) + sibling_child_category = create_category(0, 15, 10, 15, sibling_category.id) + + heal_category_trees() + + category.refresh_from_db() + assert repr_mptt(category) == (0, 0, 1, 8) + + child_category.refresh_from_db() + assert repr_mptt(child_category) == (0, 1, 2, 5) + + deep_child_category.refresh_from_db() + assert repr_mptt(deep_child_category) == (0, 2, 3, 4) + + other_child_category.refresh_from_db() + assert repr_mptt(other_child_category) == (0, 1, 6, 7) + + sibling_category.refresh_from_db() + assert repr_mptt(sibling_category) == (0, 0, 9, 12) + + sibling_child_category.refresh_from_db() + assert repr_mptt(sibling_child_category) == (0, 1, 10, 11) diff --git a/misago/categories/tests/test_healcategorytrees.py b/misago/categories/tests/test_healcategorytrees.py new file mode 100644 index 0000000000..72be1a4c7d --- /dev/null +++ b/misago/categories/tests/test_healcategorytrees.py @@ -0,0 +1,39 @@ +from io import StringIO + +from django.core.management import call_command + +from ...cache.enums import CacheName +from ...cache.test import assert_invalidates_cache +from ..management.commands import healcategorytrees +from ..models import Category + + +def run_command(): + """Run the management command""" + command = healcategorytrees.Command() + out = StringIO() + call_command(command, stdout=out) + + +def test_heal_category_trees_command_rebuilds_trees(default_category): + category = Category.objects.create( + name="Test", slug="test", parent=default_category + ) + + Category.objects.filter(id=default_category.id).update(lft=2, rght=2) + Category.objects.filter(id=category.id).update(lft=10, rght=3) + + run_command() + + default_category.refresh_from_db() + assert default_category.lft == 2 + assert default_category.rght == 5 + + category.refresh_from_db() + assert category.lft == 3 + assert category.rght == 4 + + +def test_heal_category_trees_command_invalidates_caches(db): + with assert_invalidates_cache(CacheName.CATEGORIES): + run_command()