diff --git a/zschema/__main__.py b/zschema/__main__.py index 54c9c9f..cc24d08 100644 --- a/zschema/__main__.py +++ b/zschema/__main__.py @@ -11,7 +11,7 @@ def usage(): sys.stderr.write("USAGE: %s command schema [file].\n" % sys.argv[0].split("/")[-1]) - sys.stderr.write("Valid commands: bigquery, elasticsearch, es-annotated, bq-annotated, json, flat, validate.\n") + sys.stderr.write("Valid commands: bigquery, elasticsearch, docs-es, docs-bq, json, flat, validate.\n") sys.stderr.write("Schema should be passed as file.py:record\n") sys.stderr.write("The optional 'file' argument is used only as the test file for the 'validate' command.\n") sys.stderr.write("VERSION: %s\n" % zschema.__version__) @@ -28,10 +28,10 @@ def main(): print json.dumps(record.to_bigquery()) elif command == "elasticsearch": print json.dumps(record.to_es(recname)) - elif command == "es-annotated": - print json.dumps(record.to_es(recname, annotated=True)) elif command == "bq-annotated": print json.dumps(record.to_bigquery(annotated=True)) + elif command == "docs-es": + print json.dumps(record.docs_es(recname)) elif command == "json": print record.to_json() elif command == "flat": diff --git a/zschema/compounds.py b/zschema/compounds.py index 7caae2c..574beed 100644 --- a/zschema/compounds.py +++ b/zschema/compounds.py @@ -37,11 +37,13 @@ def to_bigquery(self, name, annotated=False, parent_category=None): retv["category"] = category return retv - def to_es(self, annotated=False, parent_category=None): - retv = self.object_.to_es(annotated=annotated) - if annotated: - category = self.category if self.category else parent_category - retv["category"] = category + def to_es(self): + return self.object_.to_es() + + def docs_es(self, parent_category=None): + retv = self.object_.docs_es() + category = self.category if self.category else parent_category + retv["category"] = category return retv def validate(self, name, value): @@ -144,14 +146,27 @@ def print_indent_string(self, name, indent): for name, value in sorted(self.definition.iteritems()): value.print_indent_string(name, indent+1) - def to_es(self, annotated=False, parent_category=None): - category = self.category if self.category else parent_category - p = {self.key_to_es(k): v.to_es(annotated=annotated, parent_category=category) \ + def to_es(self): + p = {self.key_to_es(k): v.to_es() \ for k, v in sorted(self.definition.iteritems()) \ if not v.exclude_elasticsearch} - retv = {"properties": p} - if annotated and self.doc: - retv["doc"] = self.doc + return {"properties": p} + + def _docs_common(self, parent_category): + category = self.category if self.category else parent_category + retv = { + "category": category, + "doc": self.doc, + "type": self.__class__.__name__, + "required": self.required, + } + return retv + + def docs_es(self, parent_category=None): + retv = self._docs_common(parent_category=parent_category) + retv["fields"] = { self.key_to_es(k): v.docs_es() \ + for k, v in sorted(self.definition.iteritems()) \ + if not v.exclude_elasticsearch } return retv def to_dict(self): @@ -191,9 +206,11 @@ def to_bigquery(self, name, annotated=False, parent_category=None): class Record(SubRecord): - def to_es(self, name, annotated=False, parent_category=None): - category = self.category if self.category else parent_category - return {name:SubRecord.to_es(self, annotated=annotated, parent_category=category)} + def to_es(self, name): + return {name:SubRecord.to_es(self)} + + def docs_es(self, name, parent_category=None): + return {name: SubRecord.docs_es(self, parent_category=parent_category)} def to_bigquery(self, annotated=False, parent_category=None): category = self.category if self.category else parent_category diff --git a/zschema/leaves.py b/zschema/leaves.py index e811470..df9716d 100644 --- a/zschema/leaves.py +++ b/zschema/leaves.py @@ -65,32 +65,35 @@ def to_dict(self): "ES_SEARCH_ANALYZER") return retv - def to_es(self, annotated=False, parent_category=None): + def to_es(self): retv = {"type":self.ES_TYPE} self.add_es_var(retv, "index", "es_index", "ES_INDEX") self.add_es_var(retv, "analyzer", "es_analyzer", "ES_ANALYZER") self.add_es_var(retv, "search_analyzer", "es_search_analyzer", "ES_SEARCH_ANALYZER") - if self.es_include_raw: retv["fields"] = { "raw":{"type":"keyword"} } - if annotated: - retv["detail_type"] = self.__class__.__name__ - category = self.category if self.category else parent_category - retv["category"] = category - if self.doc: - retv["doc"] = self.doc - if self.min_value: - retv["min_value"] = self.min_value - if self.max_value: - retv["max_value"] = self.max_value - if hasattr(self, "values_s") and len(self.values_s): - # gotta clean this up but for now... - retv["values"] = list(self.values_s) - else: - retv["examples"] = self.examples + return retv + + def _docs_common(self, parent_category): + retv = { + "detail_type": self.__class__.__name__, + "category": self.category if self.category else parent_category, + "doc": self.doc, + "required": self.required, + } + if hasattr(self, "values_s") and len(self.values_s): + retv["values"] = list(self.values_s) + else: + retv["examples"] = self.examples + return retv + + def docs_es(self, parent_category=None): + retv = self._docs_common(parent_category) + self.add_es_var(retv, "analyzer", "es_analyzer", "ES_ANALYZER") + retv["type"] = self.ES_TYPE return retv def to_bigquery(self, name, annotated=False, parent_category=None):