Cherry pick from devel HEAD: submodule updates and ingest-validation-…

…tools things

src/ingest-pipeline/airflow/dags/validation_test.py

@@ -17,7 +17,7 @@
 
 sys.path.append(airflow_conf.as_dict()['connections']['SRC_PATH']
                 .strip("'").strip('"'))
-from submodules import (ingest_validation_tools_submission,  # noqa E402
+from submodules import (ingest_validation_tools_upload,  # noqa E402
                         ingest_validation_tools_error_report,
                         ingest_validation_tests)
 sys.path.pop()
@@ -122,17 +122,17 @@ def run_validation(**kwargs):
         #
         # Uncomment offline=True below to avoid validating orcid_id URLs &etc
         #
-        submission = ingest_validation_tools_submission.Submission(
+        upload = ingest_validation_tools_upload.Upload(
             directory_path=Path(lz_path),
             dataset_ignore_globs=ignore_globs,
-            submission_ignore_globs='*',
+            upload_ignore_globs='*',
             plugin_directory=plugin_path,
             #offline=True,  # noqa E265
             add_notes=False
         )
         # Scan reports an error result
         report = ingest_validation_tools_error_report.ErrorReport(
-            submission.get_errors()
+            upload.get_errors()
         )
         with open(os.path.join(lz_path, 'validation_report.txt'), 'w') as f:
             f.write(report.as_text())

src/ingest-pipeline/md/data_file_types/metadatatsv_metadata_file.py

@@ -5,13 +5,13 @@
 from pathlib import Path
 from .tsv_metadata_file import TSVMetadataFile
 from type_base import MetadataError
-from submodules import (ingest_validation_tools_submission,
+from submodules import (ingest_validation_tools_upload,
                         ingest_validation_tools_error_report,
                         ingest_validation_tests)
 
 class MetadataTSVMetadataFile(TSVMetadataFile):
     """
-    A metadata file type for the specialized metadata.tsv files used to store submission info
+    A metadata file type for the specialized metadata.tsv files used to store upload and submission info
     """
     category_name = 'METADATATSV';
 
@@ -23,16 +23,16 @@ def collect_metadata(self):
         #
         # Uncomment offline=True below to avoid validating orcid_id URLs &etc
         #
-        submission = ingest_validation_tools_submission.Submission(directory_path=dirpath,
-                                                                   dataset_ignore_globs=ignore_globs,
-                                                                   submission_ignore_globs='*',
-                                                                   plugin_directory=plugin_path,
-                                                                   #offline=True,
-                                                                   add_notes=False
-                                                                   )
-        if submission.get_errors():
+        upload = ingest_validation_tools_upload.Upload(directory_path=dirpath,
+                                                       dataset_ignore_globs=ignore_globs,
+                                                       upload_ignore_globs='*',
+                                                       plugin_directory=plugin_path,
+                                                       #offline=True,
+                                                       add_notes=False
+        )
+        if upload.get_errors():
             # Scan reports an error result
-            report = ingest_validation_tools_error_report.ErrorReport(submission.get_errors())
+            report = ingest_validation_tools_error_report.ErrorReport(upload.get_errors())
             with open('ingest_validation_tools_report.txt', 'w') as f:
                 f.write(report.as_text())
             raise MetadataError('{} failed ingest validation test'.format(self.path))

src/ingest-pipeline/submodules/__init__.py

@@ -6,12 +6,12 @@
                              'ingest-validation-tools', 'src'))
 sys.path.append(os.path.join(os.path.dirname(__file__),
                              'ingest-validation-tests', 'src'))
-ingest_validation_tools_submission = import_module('ingest_validation_tools.submission')
+ingest_validation_tools_upload = import_module('ingest_validation_tools.upload')
 ingest_validation_tools_error_report = import_module('ingest_validation_tools.error_report')
 ingest_validation_tools_validation_utils = import_module('ingest_validation_tools.validation_utils')
 ingest_validation_tests = import_module('ingest_validation_tests')
 __all__ = ["ingest_validation_tools_validation_utils",
-           "ingest_validation_tools_submission",
+           "ingest_validation_tools_upload",
            "ingest_validation_tools_error_report",
            "ingest_validation_tests"]