Deduplicate columns to select in add_keep_cols (#88)

rapidsai · Sep 26, 2024 · 7d4bccd · 7d4bccd
1 parent 2a0272a
commit 7d4bccd
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/crossfit/op/base.py b/crossfit/op/base.py
@@ -70,7 +70,9 @@ def add_keep_cols(self, data, output):
                 output[col] = data[col]
 
         columns = list(output.columns)
-        output = output[self.keep_cols + columns]
+        # we use dict.fromkeys to remove duplicates and preserve order
+        # (to match _build_dask_meta behavior)
+        output = output[list(dict.fromkeys(self.keep_cols + columns))]
 
         return output