From 3b8a65b36f4ed4c1829a9c5e8cd3fd4b95dcf696 Mon Sep 17 00:00:00 2001 From: David Erni Date: Wed, 29 May 2019 18:08:26 +0200 Subject: [PATCH] Add surgery to remove rid from UnIndex. --- ftw/catalogdoctor/surgery.py | 53 ++++++++- ftw/catalogdoctor/tests/test_surgery.py | 146 ++++++++++++++++++++++++ 2 files changed, 195 insertions(+), 4 deletions(-) create mode 100644 ftw/catalogdoctor/tests/test_surgery.py diff --git a/ftw/catalogdoctor/surgery.py b/ftw/catalogdoctor/surgery.py index 561a78a..076bb34 100644 --- a/ftw/catalogdoctor/surgery.py +++ b/ftw/catalogdoctor/surgery.py @@ -12,9 +12,54 @@ from Products.ZCTextIndex.ZCTextIndex import ZCTextIndex +class NullSurgery(object): + """Don't do anything.""" + + def __init__(self, index, rid): + pass + + def perform(self): + pass + + +class RemoveFromUnIndex(object): + """Remove a rid from a simple forward and reverse index.""" + + def __init__(self, index, rid): + self.index = index + self.rid = rid + + def perform(self): + entries_pointing_to_rid = [ + val for val, rids_in_index in self.index._index.items() + if self.rid in rids_in_index] + if entries_pointing_to_rid: + # Could happen in case of an index which has `indexed_attrs` set in + # extra arguments. + for entry in entries_pointing_to_rid: + del self.index._index[entry] + # The method removeForwardIndexEntry from UnIndex updates the + # index length. We assume we only have to update the index + # length when we remove the entry from the forward index, + # assuming somehow removeForwardIndexEntry has not been called + # or raised an exception + self.index._length.change(-1) + + if self.rid in self.index._unindex: + del self.index._unindex[self.rid] + + class Surgery(object): """Surgery can fix a concrete set of symptoms.""" + removal = { + DateIndex: RemoveFromUnIndex, + DateRecurringIndex: RemoveFromUnIndex, + FieldIndex: RemoveFromUnIndex, + GopipIndex: NullSurgery, # not a real index + KeywordIndex: RemoveFromUnIndex, + } + def __init__(self, catalog, unhealthy_rid): self.catalog = catalog self.unhealthy_rid = unhealthy_rid @@ -25,8 +70,9 @@ def perform(self): def unindex_rid_from_all_catalog_indexes(self, rid): for idx in self.catalog.indexes.values(): - if isinstance(idx, GopipIndex): - # Not a real index + surgery = self.removal.get(type(idx)) + if surgery: + surgery(idx, rid).perform() continue if isinstance(idx, (ZCTextIndex, DateRangeIndex, @@ -37,8 +83,7 @@ def unindex_rid_from_all_catalog_indexes(self, rid): idx.unindex_object(rid) continue - if not isinstance(idx, (DateIndex, FieldIndex, KeywordIndex, - ExtendedPathIndex, UUIDIndex)): + if not isinstance(idx, (ExtendedPathIndex, UUIDIndex)): raise CantPerformSurgery( 'Unhandled index type: {0!r}'.format(idx)) diff --git a/ftw/catalogdoctor/tests/test_surgery.py b/ftw/catalogdoctor/tests/test_surgery.py new file mode 100644 index 0000000..505ba76 --- /dev/null +++ b/ftw/catalogdoctor/tests/test_surgery.py @@ -0,0 +1,146 @@ +from datetime import date +from ftw.builder import Builder +from ftw.builder import create +from ftw.catalogdoctor.compat import DateRecurringIndex +from ftw.catalogdoctor.compat import IS_PLONE_5 +from ftw.catalogdoctor.surgery import RemoveFromUnIndex +from ftw.catalogdoctor.tests import FunctionalTestCase +from Products.PluginIndexes.DateIndex.DateIndex import DateIndex +from Products.PluginIndexes.FieldIndex.FieldIndex import FieldIndex +from Products.PluginIndexes.KeywordIndex.KeywordIndex import KeywordIndex +from unittest import skipIf + + +def unindex_entries_pointing_to_rid(index, rid): + """Return all entries in an UnIndex forward index pointing to rid.""" + + entries_pointing_to_rid = [ + val for val, rids_in_index in index._index.items() + if rid in rids_in_index] + return entries_pointing_to_rid + + +class TestRemoveFromUnIndex(FunctionalTestCase): + + def setUp(self): + super(TestRemoveFromUnIndex, self).setUp() + + self.grant('Contributor') + self.folder = create(Builder('folder').titled(u'Foo')) + + # pretend we are something that supports recurring dates + self.folder.start = date(2010, 1, 1) + self.folder.recurrence = 'FREQ=DAILY;INTERVAL=1;COUNT=5' + self.folder.reindexObject() + + def test_remove_object_from_reverse_index_only(self): + rid = self.get_rid(self.folder) + index = self.catalog.indexes['Type'] + + entries_pointing_to_rid = unindex_entries_pointing_to_rid(index, rid) + self.assertEqual(1, len(entries_pointing_to_rid)) + self.assertIn(rid, index._unindex) + self.assertEqual(1, len(index)) + + del index._index[entries_pointing_to_rid[0]] + index._length.change(-1) + + surgery = RemoveFromUnIndex(index, rid) + surgery.perform() + + entries_pointing_to_rid = unindex_entries_pointing_to_rid(index, rid) + self.assertEqual(0, len(entries_pointing_to_rid)) + self.assertNotIn(rid, index._unindex) + self.assertEqual(0, len(index)) + + def test_remove_object_from_forward_index_only(self): + rid = self.get_rid(self.folder) + index = self.catalog.indexes['Type'] + + entries_pointing_to_rid = unindex_entries_pointing_to_rid(index, rid) + self.assertEqual(1, len(entries_pointing_to_rid)) + self.assertIn(rid, index._unindex) + self.assertEqual(1, len(index)) + + del index._unindex[rid] + + surgery = RemoveFromUnIndex(index, rid) + surgery.perform() + + entries_pointing_to_rid = unindex_entries_pointing_to_rid(index, rid) + self.assertEqual(0, len(entries_pointing_to_rid)) + self.assertNotIn(rid, index._unindex) + self.assertEqual(0, len(index)) + + def test_remove_healthy_object_from_fieldindex(self): + rid = self.get_rid(self.folder) + index = self.catalog.indexes['Type'] + self.assertIs(FieldIndex, type(index)) + + entries_pointing_to_rid = unindex_entries_pointing_to_rid(index, rid) + self.assertEqual(1, len(entries_pointing_to_rid)) + self.assertIn(rid, index._unindex) + self.assertEqual(1, len(index)) + + surgery = RemoveFromUnIndex(index, rid) + surgery.perform() + + entries_pointing_to_rid = unindex_entries_pointing_to_rid(index, rid) + self.assertEqual(0, len(entries_pointing_to_rid)) + self.assertNotIn(rid, index._unindex) + self.assertEqual(0, len(index)) + + def test_remove_healthy_object_from_dateindex(self): + rid = self.get_rid(self.folder) + index = self.catalog.indexes['modified'] + self.assertIs(DateIndex, type(index)) + + entries_pointing_to_rid = unindex_entries_pointing_to_rid(index, rid) + self.assertEqual(1, len(entries_pointing_to_rid)) + self.assertIn(rid, index._unindex) + self.assertEqual(1, len(index)) + + surgery = RemoveFromUnIndex(index, rid) + surgery.perform() + + entries_pointing_to_rid = unindex_entries_pointing_to_rid(index, rid) + self.assertEqual(0, len(entries_pointing_to_rid)) + self.assertNotIn(rid, index._unindex) + self.assertEqual(0, len(index)) + + @skipIf(IS_PLONE_5, "Start attribute trickery does not work on Plone >= 5") + def test_remove_healthy_object_from_daterecurringindex(self): + rid = self.get_rid(self.folder) + index = self.catalog.indexes['start'] + self.assertIs(DateRecurringIndex, type(index)) + + entries_pointing_to_rid = unindex_entries_pointing_to_rid(index, rid) + self.assertEqual(5, len(entries_pointing_to_rid)) + self.assertIn(rid, index._unindex) + self.assertEqual(5, len(index)) + + surgery = RemoveFromUnIndex(index, rid) + surgery.perform() + + entries_pointing_to_rid = unindex_entries_pointing_to_rid(index, rid) + self.assertEqual(0, len(entries_pointing_to_rid)) + self.assertNotIn(rid, index._unindex) + self.assertEqual(0, len(index)) + + def test_remove_healthy_object_from_keywordindex(self): + rid = self.get_rid(self.folder) + index = self.catalog.indexes['object_provides'] + self.assertIs(KeywordIndex, type(index)) + + entries_pointing_to_rid = unindex_entries_pointing_to_rid(index, rid) + self.assertGreater(len(entries_pointing_to_rid), 0) + self.assertIn(rid, index._unindex) + self.assertGreater(len(index), 0) + + surgery = RemoveFromUnIndex(index, rid) + surgery.perform() + + entries_pointing_to_rid = unindex_entries_pointing_to_rid(index, rid) + self.assertEqual(0, len(entries_pointing_to_rid)) + self.assertNotIn(rid, index._unindex) + self.assertEqual(0, len(index))