Add breaks if we send None in date, datetime

.gitignore

@@ -5,3 +5,5 @@ MANIFEST
 build
 dist
 docs/_build
+*.egg-info/
+env/

Changelog

@@ -1,30 +1,72 @@
-* 0.7 : unreleased
- - Provide default string-based field for schema field types unknown to
-   Sunburnt. (@davidjb)
- - Escape forward slash characters for compatibility with Solr 4.0 (@davidjb)
- - Fix handling of queries with ``boost_relevancy`` applied - boost was 
-   previously lost in some cases. (@davidjb)
- - Ensure 'more like this' results are transformed using a query's
-   execute() `constructor`, as are normal query results. (@davidjb)
+0.8.1lu (unreleased)
+--------------------
 
-* 0.6 : 2012-01-01
- - Change license to MIT/X11
- - Integrate with Django Paginator (thanks @rlskoeser)
- - Add UUIDField support
- - Add MLT Handler support (thanks @ogrisel)
+- Nothing changed yet.
 
 
-* 0.5 : 2011-05-18
- - Add proper documentation
- - Add field_limit() option (thanks, @danaspiegel)
- - Add basic support for new Solr-3.1 datatypes
- - Add support for __any queries (thanks, @skarab)
- - Add experimental JSON output module
+0.8lu (2013-12-17)
+------------------
 
- - Rationalize data normalization and quoting codepaths, fixing several bugs along the way
- - Fixes for correct serialization of complex nested query objects
+- Added boundaryScanner options to HighlightOptions.
+  [jod]
 
+- Fixed tests.
+  [jod]
 
-* 0.4 : 2011-01-04
+- Use setuptools in setup.py.
+  [jod]
 
- - First release on PyPi
+- Added https://wiki.apache.org/solr/PostingsHighlighter
+  [jod]
+
+0.7.1lu (2013-11-04)
+--------------------
+
+- Add support for hierarchical facet queryies, i.e. facet.pivot. (@florianpilz)
+
+0.7lu (2013-07-02)
+------------------
+
+- Provide default string-based field for schema field types unknown to
+
+- Escape forward slash characters for compatibility with Solr 4.0 (@davidjb)
+  Sunburnt. (@davidjb)
+
+- Fix handling of queries with ``boost_relevancy`` applied - boost was 
+  previously lost in some cases. (@davidjb)
+
+- Ensure 'more like this' results are transformed using a query's
+  execute() `constructor`, as are normal query results. (@davidjb)
+
+0.6 (2012-01-01)
+----------------
+
+- Change license to MIT/X11
+
+- Integrate with Django Paginator (thanks @rlskoeser)
+
+- Add UUIDField support
+
+- Add MLT Handler support (thanks @ogrisel)
+
+0.5 (2011-05-18)
+----------------
+
+- Add proper documentation
+
+- Add field_limit() option (thanks, @danaspiegel)
+
+- Add basic support for new Solr-3.1 datatypes
+
+- Add support for __any queries (thanks, @skarab)
+
+- Add experimental JSON output module
+
+- Rationalize data normalization and quoting codepaths, fixing several bugs along the way
+
+- Fixes for correct serialization of complex nested query objects
+
+0.4 (2011-01-04)
+----------------
+
+- First release on PyPi

setup.cfg

@@ -0,0 +1,6 @@
+[nosetests]                                                                                                                                            
+match=^test
+nocapture=1
+cover-package=sunburnt
+with-coverage=1
+cover-erase=1

setup.py

@@ -1,19 +1,17 @@
 #!/usr/bin/env python
 
-import distutils.core, os, re
+from setuptools import setup
 
-version_number_re = "\s*__version__\s*=\s*((\"([^\"]|\\\\\")*\"|'([^']|\\\\')*'))"
-version_file = os.path.join(os.path.dirname(__file__), 'sunburnt', '__init__.py')
-version_number = re.search(version_number_re, open(version_file).read()).groups()[0][1:-1]
+version = '0.8.1lu.dev0'
 
-distutils.core.setup(
+setup(
     name='sunburnt',
-    version=version_number,
+    version=version,
     description='Python interface to Solr',
     author='Toby White',
     author_email='[email protected]',
     packages=['sunburnt'],
-    requires=['lxml', 'pytz'],
+    install_requires=['httplib2', 'lxml', 'pytz', 'setuptools'],
     classifiers=[
         'Development Status :: 4 - Beta',
         'Intended Audience :: Developers',

sunburnt/__init__.py

@@ -3,6 +3,4 @@
 from .strings import RawString
 from .sunburnt import SolrError, SolrInterface
 
-__version__ = '0.7'
-
 __all__ = ['RawString', 'SolrError', 'SolrInterface']

sunburnt/dates.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 
 import datetime, math, re, warnings
+import pytz
 
 try:
     import mx.DateTime
@@ -55,6 +56,7 @@ def datetime_from_w3_datestring(s):
     del d['tzd_sign']
     del d['tzd_hour']
     del d['tzd_minute']
+    d['tzinfo'] = pytz.utc
     try:
         dt = datetime_factory(**d) + tz_delta
     except DateTimeRangeError:

sunburnt/schema.py

@@ -595,9 +595,14 @@ def doc(self, doc):
         if not doc:
             return self.DOC()
         else:
-            return self.DOC(*reduce(operator.add,
-                                    [self.fields(name, values)
-                                     for name, values in doc.items()]))
+            # XXX remove all None fields this is needed for adding date fields 
+            fields = []
+            for name, values in doc.items():
+                if values is None:
+                    doc.pop(name)
+                    continue
+                fields.append(self.fields(name, values))
+            return self.DOC(*reduce(operator.add, fields))
 
     def add(self, docs):
         if hasattr(docs, "items") or not hasattr(docs, "__iter__"):
@@ -669,7 +674,7 @@ def __str__(self):
 
 
 class SolrFacetCounts(object):
-    members= ["facet_dates", "facet_fields", "facet_queries"]
+    members= ["facet_dates", "facet_fields", "facet_queries", "facet_pivot"]
     def __init__(self, **kwargs):
         for member in self.members:
             setattr(self, member, kwargs.get(member, ()))

sunburnt/search.py

@@ -376,8 +376,9 @@ def add_boost(self, kwargs, boost_score):
 class BaseSearch(object):
     """Base class for common search options management"""
     option_modules = ('query_obj', 'filter_obj', 'paginator',
-                      'more_like_this', 'highlighter', 'faceter',
-                      'sorter', 'facet_querier', 'field_limiter',)
+                      'more_like_this', 'highlighter', 'postings_highlighter',
+                      'faceter', 'sorter', 'facet_querier', 'field_limiter',
+                      'pivoter')
 
     result_constructor = dict
 
@@ -386,7 +387,9 @@ def _init_common_modules(self):
         self.filter_obj = LuceneQuery(self.schema, u'fq')
         self.paginator = PaginateOptions(self.schema)
         self.highlighter = HighlightOptions(self.schema)
+        self.postings_highlighter = PostingsHighlightOptions(self.schema)
         self.faceter = FacetOptions(self.schema)
+        self.pivoter = FacetPivotOptions(self.schema)
         self.sorter = SortOptions(self.schema)
         self.field_limiter = FieldLimitOptions(self.schema)
         self.facet_querier = FacetQueryOptions(self.schema)
@@ -446,6 +449,11 @@ def facet_by(self, field, **kwargs):
         newself.faceter.update(field, **kwargs)
         return newself
 
+    def pivot_by(self, fields, **kwargs):
+        newself = self.clone()
+        newself.pivoter.update(fields, **kwargs)
+        return newself
+
     def facet_query(self, *args, **kwargs):
         newself = self.clone()
         newself.facet_querier.update(self.Q(*args, **kwargs))
@@ -456,6 +464,11 @@ def highlight(self, fields=None, **kwargs):
         newself.highlighter.update(fields, **kwargs)
         return newself
 
+    def postings_highlight(self, fields=None, **kwargs):
+        newself = self.clone()
+        newself.postings_highlighter.update(fields, **kwargs)
+        return newself
+
     def mlt(self, fields, query_fields=None, **kwargs):
         newself = self.clone()
         newself.more_like_this.update(fields, query_fields, **kwargs)
@@ -776,6 +789,31 @@ def field_names_in_opts(self, opts, fields):
             opts["facet.field"] = sorted(fields)
 
 
+class FacetPivotOptions(Options):
+    option_name = "facet.pivot"
+    opts = {
+        "mincount":lambda self, x: int(x) >= 0 and int(x) or self.invalid_value(),
+    }
+
+    def __init__(self, schema, original=None):
+        self.schema = schema
+        if original is None:
+            self.fields = collections.defaultdict(dict)
+        else:
+            self.fields = copy.copy(original.fields)
+
+    def field_names_in_opts(self, opts, fields):
+        opts["facet"] = True
+        if fields:
+            field_opts = {}
+            for field in fields:
+                field_opts = dict(field_opts.items() + self.fields[field].items())
+                del(self.fields[field])
+            self.fields[None] = field_opts
+            opts["facet.pivot"] = ','.join(sorted(fields))
+
+
+
 class HighlightOptions(Options):
     option_name = "hl"
     opts = {"snippets":int,
@@ -794,8 +832,46 @@ class HighlightOptions(Options):
             "highlightMultiTerm":bool,
             "regex.slop":float,
             "regex.pattern":unicode,
-            "regex.maxAnalyzedChars":int
+            "regex.maxAnalyzedChars":int,
+            "boundaryScanner": unicode,
+            "bs.maxScan": unicode,
+            "bs.chars": unicode,
+            "bs.type": unicode,
+            "bs.language": unicode,
+            "bs.country": unicode,
+            }
+    def __init__(self, schema, original=None):
+        self.schema = schema
+        if original is None:
+            self.fields = collections.defaultdict(dict)
+        else:
+            self.fields = copy.copy(original.fields)
+
+    def field_names_in_opts(self, opts, fields):
+        if fields:
+            opts["hl.fl"] = ",".join(sorted(fields))
+
+
+class PostingsHighlightOptions(Options):
+
+    option_name = "hl"
+    opts = {"snippets": int,
+            "tag.pre": unicode,
+            "tag.post": unicode,
+            "tag.ellipsis": unicode,
+            "defaultSummary": bool,
+            "encoder": unicode,
+            "score.k1": float,
+            "score.b": float,
+            "score.pivot": float,
+            "bs.type": unicode,
+            "bs.language": unicode,
+            "bs.country": unicode,
+            "bs.variant": unicode,
+            "maxAnalyzedChars": unicode,
+            "multiValuedSeperatorChar": unicode
             }
+
     def __init__(self, schema, original=None):
         self.schema = schema
         if original is None:

sunburnt/test_schema.py

@@ -51,15 +51,15 @@
     "2009-07-23T03:24:34.1Z":
         datetime.datetime(2009, 07, 23, 3, 24, 34, 100000, pytz.utc),
     "2009-07-23T03:24:34.123Z":
-        datetime.datetime(2009, 07, 23, 3, 24, 34, 123000, pytz.utc)
+        datetime.datetime(2009, 07, 23, 3, 24, 34, 122999, pytz.utc)
     }
 
 def check_solr_date_from_date(s, date, canonical_date):
     assert unicode(solr_date(date)) == s, "Unequal representations of %r: %r and %r" % (date, unicode(solr_date(date)), s)
     check_solr_date_from_string(s, canonical_date)
 
 def check_solr_date_from_string(s, date):
-    assert solr_date(s)._dt_obj == date
+    assert solr_date(s)._dt_obj == date, "Unequal representations of %r: %r" % (solr_date(s)._dt_obj, date)
 
 def test_solr_date_from_pydatetimes():
     for k, v in samples_from_pydatetimes.items():