Merge pull request #784 from hubmapconsortium/sunset666/bugfixes

Extras directory support on Upload data folder

src/ingest-pipeline/airflow/dags/reorganize_upload.py

@@ -30,9 +30,6 @@
 
 from misc.tools.split_and_create import reorganize
 
-sys.path.append(airflow_conf.as_dict()['connections']['SRC_PATH']
-                .strip("'").strip('"'))
-sys.path.pop()
 
 # Following are defaults which can be overridden later on
 default_args = {

src/ingest-pipeline/misc/tools/split_and_create.py

@@ -213,10 +213,19 @@ def populate(row, source_entity, entity_factory, dryrun=False):
             extras_path.mkdir(0o770)
     source_data_path = source_entity.full_path / old_data_path
     for elt in source_data_path.glob('*'):
+        dst_file = kid_path / elt.name
         if dryrun:
-            print(f'rename {elt} to {kid_path / elt.name}')
+            if dst_file.exists() and dst_file.is_dir():
+                for sub_elt in elt.glob('*'):
+                    print(f'rename {sub_elt} to {kid_path / elt.name / sub_elt.name}')
+                continue
+            print(f'rename {elt} to {dst_file}')
         else:
-            elt.rename(kid_path / elt.name)
+            if dst_file.exists() and dst_file.is_dir():
+                for sub_elt in elt.glob('*'):
+                    sub_elt.rename(kid_path / elt.name / sub_elt.name)
+                continue
+            elt.rename(dst_file)
     if dryrun:
         print(f'copy {old_contrib_path} to {extras_path}')
     else: