Initial implementation. Provide catalog health check and surgery for four existing issues.

README.rst

@@ -4,13 +4,50 @@
 Introduction
 ============
 
-The package `ftw.catalogdoctor` provides checkup and surgery to remove inconsistencies in portal_catalog.
+The package ``ftw.catalogdoctor`` provides healthcheck to find
+inconsistencies in ``portal_catalog`` and surgery to remove some of them. It
+can be run via a ``zopectl.command``.
 
 
-Compatibility
--------------
+Healthcheck
+===========
 
-Plone 4.3.x
+Lists inconsistencies detected in ``portal_catalog``. Finds inconsistencies by
+inspecting the catalog's internal data structures. It currently uses ``paths``
+(the rid-path mapping), ``uids`` (the path-rid mapping), the ``UID`` index and
+catalog metadata to determine if the catalog is healthy or if there are
+problems. Healtcheck is a read-only operation and won't modify the catalog.
+
+It can be run as follows:
+
+.. code:: sh
+
+    $ bin/instance doctor healtcheck
+
+
+Surgery
+=======
+
+Attempts to fix issues found by ``healthcheck``. Will do a healtchcheck before
+surgery, then attempt surgery and finally do a post-surgery healthcheck.
+Surgery is a write operation but changes are only committed to the database if
+the post-surgery healtcheck yields no more health problems.
+Currently the set of available surgery is limited to problems we have observed
+in production.
+
+
+It can be run as follows:
+
+.. code:: sh
+
+    $ bin/instance doctor surgery
+
+
+There is also a `--dry-run` parameter that prevents committing changes.
+
+.. code:: sh
+
+    $ bin/instance doctor --dry-run surgery
 
 
 Installation
@@ -26,6 +63,13 @@ Installation
         ftw.catalogdoctor
 
 
+Compatibility
+-------------
+
+Plone 4.3.x
+Plone 5.1.x
+
+
 Development
 ===========
 

ftw/catalogdoctor/command.py

@@ -0,0 +1,154 @@
+from __future__ import print_function
+from ftw.catalogdoctor.compat import processQueue
+from ftw.catalogdoctor.healthcheck import CatalogHealthCheck
+from ftw.catalogdoctor.surgery import CatalogDoctor
+from Products.CMFCore.utils import getToolByName
+from Products.CMFPlone.interfaces import IPloneSiteRoot
+from Testing.makerequest import makerequest
+from zope.component.hooks import setSite
+import argparse
+import sys
+import transaction
+
+
+def discover_plone_site(app):
+    for item_id, item in app.items():
+        if IPloneSiteRoot.providedBy(item):
+            return item_id
+    return None
+
+
+def load_site(app, path):
+    if not path:
+        print('ERROR: No Plone site found. Use --site or create a Plone site '
+              'in the Zope app root.',
+              file=sys.stderr)
+        sys.exit(1)
+
+    app = makerequest(app)
+    site = app.unrestrictedTraverse(path)
+    app.REQUEST.PARENTS = [site, app]
+    setSite(site)
+
+    return site
+
+
+class ConsoleOutput(object):
+
+    def info(self, msg):
+        print(msg)
+
+    def warning(self, msg):
+        print(msg)
+
+    def error(self, msg):
+        print(msg, file=sys.stderr)
+
+
+def healthcheck_command(portal_catalog, args, formatter):
+    transaction.doom()  # extra paranoia, prevent erroneous commit
+
+    return _run_healthcheck(portal_catalog, formatter)
+
+
+def _run_healthcheck(portal_catalog, formatter):
+    result = CatalogHealthCheck(catalog=portal_catalog).run()
+    result.write_result(formatter)
+    return result
+
+
+def surgery_command(portal_catalog, args, formatter):
+    if args.dryrun:
+        formatter.info('Performing dryrun!')
+        formatter.info('')
+        transaction.doom()
+
+    result = _run_healthcheck(portal_catalog, formatter)
+    if result.is_healthy():
+        transaction.doom()  # extra paranoia, prevent erroneous commit
+        formatter.info('Catalog is healthy, no surgery is needed.')
+        return
+
+    there_is_nothing_we_can_do = []
+    formatter.info('Performing surgery:')
+    for unhealthy_rid in result.get_unhealthy_rids():
+        doctor = CatalogDoctor(result.catalog, unhealthy_rid)
+        if doctor.can_perform_surgery():
+            surgery = doctor.perform_surgery()
+            surgery.write_result(formatter)
+            formatter.info('')
+        else:
+            there_is_nothing_we_can_do.append(unhealthy_rid)
+
+    if there_is_nothing_we_can_do:
+        formatter.info('The following unhealthy rids could not be fixed:')
+        for unhealthy_rid in there_is_nothing_we_can_do:
+            unhealthy_rid.write_result(formatter)
+            formatter.info('')
+
+        formatter.info('Not all health problems could be fixed, aborting.')
+        return
+
+    processQueue()
+
+    formatter.info('Performing post-surgery healthcheck:')
+    post_result = _run_healthcheck(portal_catalog, formatter)
+    if not post_result.is_healthy():
+        transaction.doom()   # extra paranoia, prevent erroneous commit
+        formatter.info('Not all health problems could be fixed, aborting.')
+        return
+
+    if args.dryrun:
+        formatter.info('Surgery would have been successful, but was aborted '
+                       'due to dryrun!')
+    else:
+        transaction.commit()
+        formatter.info('Surgery was successful, known health problems could '
+                       'be fixed!')
+
+
+def _setup_parser(app):
+    parser = argparse.ArgumentParser(
+        description='Provide health check and fixes for portal_catalog.',
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    # swallows instance command
+    parser.add_argument('-c', help=argparse.SUPPRESS)
+
+    parser.add_argument(
+        '-s', '--site', dest='site',
+        default=discover_plone_site(app),
+        help='Path to the Plone site from which portal_catalog is used.')
+    parser.add_argument(
+        '-n', '--dry-run', dest='dryrun',
+        default=False, action="store_true",
+        help='Dryrun, do not commit changes. Only relevant for surgery.')
+
+    commands = parser.add_subparsers(dest='command')
+    healthcheck = commands.add_parser(
+        'healthcheck',
+        help='Run a health check for portal_catalog.')
+    healthcheck.set_defaults(func=healthcheck_command)
+
+    surgery = commands.add_parser(
+        'surgery',
+        help='Run a healthcheck and perform surgery for unhealthy rids in '
+             'portal_catalog.')
+    surgery.set_defaults(func=surgery_command)
+    return parser
+
+
+def _parse(parser, args):
+    return parser.parse_args(args)
+
+
+def _run(parsed_args, app, formatter):
+    site = load_site(app, parsed_args.site)
+    portal_catalog = getToolByName(site, 'portal_catalog')
+
+    return parsed_args.func(portal_catalog, parsed_args, formatter=formatter)
+
+
+def doctor_cmd(app, args, formatter=None):
+    parser = _setup_parser(app)
+    parsed_args = _parse(parser, args)
+    _run(parsed_args, app, formatter or ConsoleOutput())

ftw/catalogdoctor/compat.py

@@ -0,0 +1,22 @@
+import pkg_resources
+
+
+IS_PLONE_5 = pkg_resources.get_distribution('Products.CMFPlone').version >= '5'
+
+
+if IS_PLONE_5:
+    from Products.CMFCore.indexing import processQueue
+else:
+    # optional collective.indexing support
+    try:
+        from collective.indexing.queue import processQueue
+    except ImportError:
+        def processQueue():
+            pass
+
+# optional Products.DateRecurringIndex support
+try:
+    from Products.DateRecurringIndex.index import DateRecurringIndex
+except ImportError:
+    class DateRecurringIndex(object):
+        pass

ftw/catalogdoctor/debug.py

@@ -0,0 +1,58 @@
+from pprint import pprint
+
+
+def btrees_to_python_collections(maybe_btrees):
+    """Convert collections from btrees to python collections for debugging.
+
+    WARNING: naive implementation:
+    - converts sets to lists
+    - should not be used on large data structures
+    - should only be used to debug
+
+    Only use it to display things on the command line. Better not
+    programmatically work with the result otherwise. Stick to BTrees if you
+    can.
+
+    This method is intended to help displaying catalog data structures on the
+    command line for debugging. It can be uses in combination with pprint to
+    quickly analize the state of the catalog's internal data structures.
+
+    """
+    if isinstance(maybe_btrees, (int, basestring)):
+        return maybe_btrees
+    elif hasattr(maybe_btrees, 'items'):
+        return dict((key, btrees_to_python_collections(val))
+                     for key, val in maybe_btrees.items())
+    else:
+        return list(maybe_btrees)
+
+
+def pprint_btrees(btrees):
+    """pretty print a collection from btrees.
+
+    Sample output looks like:
+
+    >>> index = plone.portal_catalog._catalog.indexes['path']
+    >>> pprint_btrees(index._index)
+    {None: {1: [97], 2: [98, 99]},
+     'child': {2: [98]},
+     'otherchild': {2: [99]},
+     'parent': {1: [97, 98, 99]},
+     'plone': {0: [97, 98, 99]}}
+
+    >>> pprint_btrees(index._unindex)
+    {97: '/plone/parent',
+     98: '/plone/parent/child',
+     99: '/plone/parent/otherchild'}
+
+    >>> pprint_btrees(index._index_items)
+    {'/plone/parent': 97,
+     '/plone/parent/child': 98,
+     '/plone/parent/otherchild': 99}
+
+    >>> pprint_btrees(index._index_parents)
+    {'/plone': [97],
+     '/plone/parent': [98, 99]}
+
+    """
+    pprint(btrees_to_python_collections(btrees))

ftw/catalogdoctor/exceptions.py

@@ -0,0 +1,2 @@
+class CantPerformSurgery(Exception):
+    """Raised when a procedure cannot be performed."""