[fixes #2] fix the case when _source is missing and store all meta-fields

Author: onesuper

pandasticsearch/dataframe.py

@@ -35,7 +35,6 @@ class DataFrame(object):
     def __init__(self, **kwargs):
         self._client = kwargs.get('client', None)
         self._mapping = kwargs.get('mapping', None)
-
         self._index = list(self._mapping.keys())[0] if self._mapping else None
         self._doc_type = DataFrame._get_doc_type(self._mapping) if self._mapping else None
         self._columns = sorted(DataFrame._get_cols(self._mapping)) if self._mapping else None
@@ -281,7 +280,8 @@ def _execute(self):
 
         res_dict = self._client.post(data=self._build_query())
         if self._aggregation is None and self._groupby is None:
-            query = Select.from_dict(res_dict)
+            query = Select()
+            query.explain_result(res_dict)
         else:
             query = Agg.from_dict(res_dict)
         return query
@@ -319,6 +319,7 @@ def count(self):
         [2, 1]
         """
         df = DataFrame(client=self._client,
+                       include_meta_fields=self._include_meta_fields,
                        mapping=self._mapping,
                        filter=self._filter,
                        groupby=self._groupby,
@@ -348,14 +349,18 @@ def show(self, n=10000, truncate=15):
         assert n > 0
 
         if self._aggregation:
-            raise TypeError('show() is not allowed for aggregation. use collect() instead')
+            raise DataFrameException('show() is not allowed for aggregation. use collect() instead')
 
         query = self._execute()
 
         if self._projection:
             cols = [col.field_name() for col in self._projection]
         else:
             cols = self.columns
+
+        if cols is None:
+            raise _unbound_index_err
+
         sys.stdout.write(query.result_as_tabular(cols, n, truncate))
         sys.stdout.write('time: {0}ms\n'.format(query.millis_taken))
 

pandasticsearch/queries.py

@@ -76,7 +76,16 @@ def __init__(self):
 
     def explain_result(self, result=None):
         super(Select, self).explain_result(result)
-        self._values = [hit['_source'] for hit in self._result_dict['hits']['hits']]
+        rows = []
+        for hit in self._result_dict['hits']['hits']:
+            row = {}
+            for k in hit.keys():
+                if k == '_source':
+                    row.update(hit['_source'])
+                elif k.startswith('_'):
+                    row[k] = hit[k]
+            rows.append(row)
+        self._values = rows
 
     def to_pandas(self):
         try:

setup.py

@@ -9,7 +9,7 @@
 
 setup(
     name='pandasticsearch',
-    version='0.2.0',
+    version='0.2.1',
     author='onesuper',
     author_email='[email protected]',
     packages=['pandasticsearch'],