Merge pull request #1031 from koordinates/faster-diff-4

diff: Add --no-sort-keys option

Author: craigds

CHANGELOG.md

@@ -9,6 +9,7 @@ _When adding new entries to the changelog, please include issue/PR numbers where
 
 - Much faster access to tabular/vector datasets (about 75% more features processed per second) by switching to [msgspec](https://jcristharif.com/msgspec/) - [#1025](https://github.com/koordinates/kart/pull/1025)
 - diff: Faster JSON-Lines output (also using msgspec)
+- diff: Added `--no-sort-keys` option to disable sorting of features by name/PK. This is a minor speed improvement.
 - Linux builds now require glibc 2.28+ [#1027](https://github.com/koordinates/kart/pull/1027) - This means minimum distro versions are:
     - Debian 10+
     - Ubuntu 18.10+

kart/base_diff_writer.py

@@ -91,6 +91,7 @@ def __init__(
         diff_estimate_accuracy=None,
         # used by html diff only
         html_template=None,
+        sort_keys=True,
     ):
         self.repo = repo
         self.commit_spec = commit_spec
@@ -134,6 +135,7 @@ def __init__(
         self.commit = None
         self.do_convert_to_dataset_format = None
         self.do_full_file_diffs = False
+        self.sort_keys = sort_keys
 
     def include_target_commit_as_header(self):
         """
@@ -451,6 +453,11 @@ def get_file_diff(self):
             repo_key_filter=self.repo_key_filter,
         )
 
+    def iter_deltadiff_items(self, deltas):
+        if self.sort_keys:
+            return deltas.sorted_items()
+        return deltas.items()
+
     def filtered_dataset_deltas(self, ds_path, ds_diff):
         """
         Yields the key, delta for only those deltas from the given dataset diff that match
@@ -462,7 +469,7 @@ def filtered_dataset_deltas(self, ds_path, ds_diff):
         if not item_type or item_type not in ds_diff:
             return
 
-        unfiltered_deltas = ds_diff[item_type].sorted_items()
+        unfiltered_deltas = self.iter_deltadiff_items(ds_diff[item_type])
 
         if self.spatial_filter.match_all:
             yield from unfiltered_deltas

kart/diff.py

@@ -154,6 +154,12 @@ def feature_count_diff(
     help="Provide a user defined/specific html template for diff representation",
     type=click.Path(exists=True),
 )
+@click.option(
+    "--sort-keys/--no-sort-keys",
+    is_flag=True,
+    default=True,
+    help="Sort keys in the output. This is the default behaviour, but it can be disabled for a slight speed improvement.",
+)
 @click.argument(
     "args",
     metavar="[REVISIONS] [--] [FILTERS]",
@@ -175,6 +181,7 @@ def diff(
     diff_format,
     delta_filter,
     html_template,
+    sort_keys,
     args,
 ):
     """
@@ -235,6 +242,7 @@ def diff(
         target_crs=crs,
         diff_estimate_accuracy=add_feature_count_estimate,
         html_template=html_template,
+        sort_keys=sort_keys,
     )
     diff_writer.convert_to_dataset_format(convert_to_dataset_format)
     diff_writer.full_file_diffs(diff_files)

kart/json_diff_writers.py

@@ -101,7 +101,7 @@ def default(self, obj):
             if ds_path == FILES_KEY:
                 return {
                     key: self._postprocess_attachment_delta(value)
-                    for key, value in ds_diff[FILES_KEY].sorted_items()
+                    for key, value in self.iter_deltadiff_items(ds_diff[FILES_KEY])
                 }
 
             result = {}
@@ -348,7 +348,7 @@ def write_meta_deltas(self, ds_path, ds_diff):
             return
 
         obj = {"type": "meta", "dataset": ds_path, "key": None, "change": None}
-        for key, delta in ds_diff["meta"].sorted_items():
+        for key, delta in self.iter_deltadiff_items(ds_diff["meta"]):
             obj["key"] = key
             obj["change"] = delta.to_plus_minus_dict()
             self.dump(obj)
@@ -382,7 +382,7 @@ def write_file_diff(self, file_diff):
         if not self.do_full_file_diffs:
             obj.pop("binary")
 
-        for key, delta in file_diff.sorted_items():
+        for key, delta in self.iter_deltadiff_items(file_diff):
             obj["path"] = key
             if self.do_full_file_diffs:
                 delta = self._full_file_delta(delta)

kart/text_diff_writer.py

@@ -61,7 +61,7 @@ def write_header(self):
 
     def write_ds_diff(self, ds_path, ds_diff, diff_format=DiffFormat.FULL):
         if "meta" in ds_diff:
-            for key, delta in ds_diff["meta"].sorted_items():
+            for key, delta in self.iter_deltadiff_items(ds_diff["meta"]):
                 self.write_meta_delta(ds_path, key, delta)
         if diff_format != DiffFormat.NO_DATA_CHANGES:
             item_type = self._get_old_or_new_dataset(ds_path).ITEM_TYPE
@@ -313,7 +313,7 @@ def _pair_items(cls, old_list, new_list):
                 old_index += 1
 
     def write_file_diff(self, file_diff):
-        for key, delta in file_diff.sorted_items():
+        for key, delta in self.iter_deltadiff_items(file_diff):
             self.write_file_delta(key, delta)
         return bool(file_diff)