Add surgery to remove rid from UnIndex.

ftw/catalogdoctor/surgery.py

@@ -12,9 +12,54 @@
 from Products.ZCTextIndex.ZCTextIndex import ZCTextIndex
 
 
+class NullSurgery(object):
+    """Don't do anything."""
+
+    def __init__(self, index, rid):
+        pass
+
+    def perform(self):
+        pass
+
+
+class RemoveFromUnIndex(object):
+    """Remove a rid from a simple forward and reverse index."""
+
+    def __init__(self, index, rid):
+        self.index = index
+        self.rid = rid
+
+    def perform(self):
+        entries_pointing_to_rid = [
+            val for val, rids_in_index in self.index._index.items()
+            if self.rid in rids_in_index]
+        if entries_pointing_to_rid:
+            # Could happen in case of an index which has `indexed_attrs` set in
+            # extra arguments.
+            for entry in entries_pointing_to_rid:
+                del self.index._index[entry]
+                # The method removeForwardIndexEntry from UnIndex updates the
+                # index length. We assume we only have to update the index
+                # length when we remove the entry from the forward index,
+                # assuming somehow removeForwardIndexEntry has not been called
+                # or raised an exception
+                self.index._length.change(-1)
+
+        if self.rid in self.index._unindex:
+            del self.index._unindex[self.rid]
+
+
 class Surgery(object):
     """Surgery can fix a concrete set of symptoms."""
 
+    removal = {
+        DateIndex: RemoveFromUnIndex,
+        DateRecurringIndex: RemoveFromUnIndex,
+        FieldIndex: RemoveFromUnIndex,
+        GopipIndex: NullSurgery,  # not a real index
+        KeywordIndex: RemoveFromUnIndex,
+    }
+
     def __init__(self, catalog, unhealthy_rid):
         self.catalog = catalog
         self.unhealthy_rid = unhealthy_rid
@@ -25,8 +70,9 @@ def perform(self):
 
     def unindex_rid_from_all_catalog_indexes(self, rid):
         for idx in self.catalog.indexes.values():
-            if isinstance(idx, GopipIndex):
-                # Not a real index
+            surgery = self.removal.get(type(idx))
+            if surgery:
+                surgery(idx, rid).perform()
                 continue
 
             if isinstance(idx, (ZCTextIndex, DateRangeIndex,
@@ -37,8 +83,7 @@ def unindex_rid_from_all_catalog_indexes(self, rid):
                 idx.unindex_object(rid)
                 continue
 
-            if not isinstance(idx, (DateIndex, FieldIndex, KeywordIndex,
-                                    ExtendedPathIndex, UUIDIndex)):
+            if not isinstance(idx, (ExtendedPathIndex, UUIDIndex)):
                 raise CantPerformSurgery(
                     'Unhandled index type: {0!r}'.format(idx))
 

ftw/catalogdoctor/tests/test_surgery.py

@@ -0,0 +1,146 @@
+from datetime import date
+from ftw.builder import Builder
+from ftw.builder import create
+from ftw.catalogdoctor.compat import DateRecurringIndex
+from ftw.catalogdoctor.compat import IS_PLONE_5
+from ftw.catalogdoctor.surgery import RemoveFromUnIndex
+from ftw.catalogdoctor.tests import FunctionalTestCase
+from Products.PluginIndexes.DateIndex.DateIndex import DateIndex
+from Products.PluginIndexes.FieldIndex.FieldIndex import FieldIndex
+from Products.PluginIndexes.KeywordIndex.KeywordIndex import KeywordIndex
+from unittest import skipIf
+
+
+def unindex_entries_pointing_to_rid(index, rid):
+    """Return all entries in an UnIndex forward index pointing to rid."""
+
+    entries_pointing_to_rid = [
+        val for val, rids_in_index in index._index.items()
+        if rid in rids_in_index]
+    return entries_pointing_to_rid
+
+
+class TestRemoveFromUnIndex(FunctionalTestCase):
+
+    def setUp(self):
+        super(TestRemoveFromUnIndex, self).setUp()
+
+        self.grant('Contributor')
+        self.folder = create(Builder('folder').titled(u'Foo'))
+
+        # pretend we are something that supports recurring dates
+        self.folder.start = date(2010, 1, 1)
+        self.folder.recurrence = 'FREQ=DAILY;INTERVAL=1;COUNT=5'
+        self.folder.reindexObject()
+
+    def test_remove_object_from_reverse_index_only(self):
+        rid = self.get_rid(self.folder)
+        index = self.catalog.indexes['Type']
+
+        entries_pointing_to_rid = unindex_entries_pointing_to_rid(index, rid)
+        self.assertEqual(1, len(entries_pointing_to_rid))
+        self.assertIn(rid, index._unindex)
+        self.assertEqual(1, len(index))
+
+        del index._index[entries_pointing_to_rid[0]]
+        index._length.change(-1)
+
+        surgery = RemoveFromUnIndex(index, rid)
+        surgery.perform()
+
+        entries_pointing_to_rid = unindex_entries_pointing_to_rid(index, rid)
+        self.assertEqual(0, len(entries_pointing_to_rid))
+        self.assertNotIn(rid, index._unindex)
+        self.assertEqual(0, len(index))
+
+    def test_remove_object_from_forward_index_only(self):
+        rid = self.get_rid(self.folder)
+        index = self.catalog.indexes['Type']
+
+        entries_pointing_to_rid = unindex_entries_pointing_to_rid(index, rid)
+        self.assertEqual(1, len(entries_pointing_to_rid))
+        self.assertIn(rid, index._unindex)
+        self.assertEqual(1, len(index))
+
+        del index._unindex[rid]
+
+        surgery = RemoveFromUnIndex(index, rid)
+        surgery.perform()
+
+        entries_pointing_to_rid = unindex_entries_pointing_to_rid(index, rid)
+        self.assertEqual(0, len(entries_pointing_to_rid))
+        self.assertNotIn(rid, index._unindex)
+        self.assertEqual(0, len(index))
+
+    def test_remove_healthy_object_from_fieldindex(self):
+        rid = self.get_rid(self.folder)
+        index = self.catalog.indexes['Type']
+        self.assertIs(FieldIndex, type(index))
+
+        entries_pointing_to_rid = unindex_entries_pointing_to_rid(index, rid)
+        self.assertEqual(1, len(entries_pointing_to_rid))
+        self.assertIn(rid, index._unindex)
+        self.assertEqual(1, len(index))
+
+        surgery = RemoveFromUnIndex(index, rid)
+        surgery.perform()
+
+        entries_pointing_to_rid = unindex_entries_pointing_to_rid(index, rid)
+        self.assertEqual(0, len(entries_pointing_to_rid))
+        self.assertNotIn(rid, index._unindex)
+        self.assertEqual(0, len(index))
+
+    def test_remove_healthy_object_from_dateindex(self):
+        rid = self.get_rid(self.folder)
+        index = self.catalog.indexes['modified']
+        self.assertIs(DateIndex, type(index))
+
+        entries_pointing_to_rid = unindex_entries_pointing_to_rid(index, rid)
+        self.assertEqual(1, len(entries_pointing_to_rid))
+        self.assertIn(rid, index._unindex)
+        self.assertEqual(1, len(index))
+
+        surgery = RemoveFromUnIndex(index, rid)
+        surgery.perform()
+
+        entries_pointing_to_rid = unindex_entries_pointing_to_rid(index, rid)
+        self.assertEqual(0, len(entries_pointing_to_rid))
+        self.assertNotIn(rid, index._unindex)
+        self.assertEqual(0, len(index))
+
+    @skipIf(IS_PLONE_5, "Start attribute trickery does not work on Plone >= 5")
+    def test_remove_healthy_object_from_daterecurringindex(self):
+        rid = self.get_rid(self.folder)
+        index = self.catalog.indexes['start']
+        self.assertIs(DateRecurringIndex, type(index))
+
+        entries_pointing_to_rid = unindex_entries_pointing_to_rid(index, rid)
+        self.assertEqual(5, len(entries_pointing_to_rid))
+        self.assertIn(rid, index._unindex)
+        self.assertEqual(5, len(index))
+
+        surgery = RemoveFromUnIndex(index, rid)
+        surgery.perform()
+
+        entries_pointing_to_rid = unindex_entries_pointing_to_rid(index, rid)
+        self.assertEqual(0, len(entries_pointing_to_rid))
+        self.assertNotIn(rid, index._unindex)
+        self.assertEqual(0, len(index))
+
+    def test_remove_healthy_object_from_keywordindex(self):
+        rid = self.get_rid(self.folder)
+        index = self.catalog.indexes['object_provides']
+        self.assertIs(KeywordIndex, type(index))
+
+        entries_pointing_to_rid = unindex_entries_pointing_to_rid(index, rid)
+        self.assertGreater(len(entries_pointing_to_rid), 0)
+        self.assertIn(rid, index._unindex)
+        self.assertGreater(len(index), 0)
+
+        surgery = RemoveFromUnIndex(index, rid)
+        surgery.perform()
+
+        entries_pointing_to_rid = unindex_entries_pointing_to_rid(index, rid)
+        self.assertEqual(0, len(entries_pointing_to_rid))
+        self.assertNotIn(rid, index._unindex)
+        self.assertEqual(0, len(index))