Updates for DSpace@MIT uploads

* Refactor parse_value_from_text method
* Refactor get_files_from_s3 method to sort bitstreams
* Update timeouts for POST requests to account for slow responses
* Add dspacemit.json config
* Remove outdated aspace_mapping.json

Author: ehanson8

config/aspace_mapping.json

@@ -0,0 +1,83 @@
+{
+    "settings": {
+        "bitstream_folders": [],
+        "id_regex": "^.*$"
+    },
+    "mapping": {
+        "item_identifier": {
+            "csv_field_name": "item_identifier",
+            "language": null,
+            "delimiter": ""
+        },
+        "dc.publisher": {
+            "csv_field_name": "dc.publisher",
+            "language": "en_US",
+            "delimiter": ""
+        },
+        "dc.identifier.mitlicense": {
+            "csv_field_name": "dc.identifier.mitlicense",
+            "language": "en_US",
+            "delimiter": ""
+        },
+        "dc.eprint.version": {
+            "csv_field_name": "dc.eprint.version",
+            "language": "en_US",
+            "delimiter": ""
+        },
+        "dc.type": {
+            "csv_field_name": "dc.type",
+            "language": "en_US",
+            "delimiter": ""
+        },
+        "dc.source": {
+            "csv_field_name": "dc.source",
+            "language": "en_US",
+            "delimiter": ""
+        },
+        "dc.contributor.author": {
+            "csv_field_name": "dc.contributor.author",
+            "language": "en_US",
+            "delimiter": "|"
+        },
+        "dc.relation.isversionof": {
+            "csv_field_name": "dc.relation.isversionof",
+            "language": "",
+            "delimiter": ""
+        },
+        "dc.title": {
+            "csv_field_name": "dc.title",
+            "language": "en_US",
+            "delimiter": ""
+        },
+        "dc.relation.journal": {
+            "csv_field_name": "dc.relation.journal",
+            "language": "",
+            "delimiter": ""
+        },
+        "dc.identifier.issn": {
+            "csv_field_name": "dc.identifier.issn",
+            "language": "",
+            "delimiter": ""
+        },
+        "dc.date.issued": {
+            "csv_field_name": "dc.date.issued",
+            "language": "",
+            "delimiter": ""
+        },
+        "dc.rights": {
+            "csv_field_name": "dc.rights",
+            "language": "en_US",
+            "delimiter": ""
+        },
+        "dc.rights.uri": {
+            "csv_field_name": "dc.rights.uri",
+            "language": "",
+            "delimiter": ""
+        },
+        "dc.description.sponsorship": {
+            "csv_field_name": "dc.description.sponsorship",
+            "language": "en_US",
+            "delimiter": ""
+        }
+    }
+}

config/dspacemit.json

@@ -28,7 +28,7 @@ def authenticate(self, email, password):
         header = self.header
         data = {"email": email, "password": password}
         session = requests.post(
-            f"{self.url}/login", headers=header, params=data, timeout=30
+            f"{self.url}/login", headers=header, params=data, timeout=120
         ).cookies["JSESSIONID"]
         cookies = {"JSESSIONID": session}
         status = requests.get(
@@ -107,7 +107,7 @@ def post_bitstream(self, item_uuid, bitstream):
                 headers=header_upload,
                 cookies=self.cookies,
                 data=data,
-                timeout=30,
+                timeout=120,
             )
             logger.info(f"Bitstream POST status: {post_response}")
             response = post_response.json()
@@ -128,7 +128,7 @@ def post_collection_to_community(self, comm_handle, coll_name):
             headers=self.header,
             cookies=self.cookies,
             json={"name": coll_name},
-            timeout=30,
+            timeout=120,
         ).json()
         coll_uuid = coll_uuid["uuid"]
         logger.info(f"Collection posted: {coll_uuid}")
@@ -143,7 +143,7 @@ def post_item_to_collection(self, collection_uuid, item):
             headers=self.header,
             cookies=self.cookies,
             json={"metadata": attr.asdict(item)["metadata"]},
-            timeout=30,
+            timeout=120,
         )
         logger.info(f"Item POST status: {post_resp}")
         post_response = post_resp.json()

dsaps/dspace.py

@@ -66,6 +66,10 @@ def get_files_from_s3(
                     continue
             item_identifier = parse_value_from_text(file_name, id_regex)
             files.setdefault(item_identifier, []).append(file_path)
+    for key, value in files.items():
+        files[key] = sorted(
+            value, key=lambda x: x.split(parse_value_from_text(x, id_regex))[1]
+        )
     return dict(sorted(files.items()))
 
 
@@ -74,8 +78,8 @@ def parse_value_from_text(
     regex: str,
 ):
     pattern = re.compile(regex)
-    if match := pattern.search(text):
-        return match.group(1)
+    if matches := pattern.findall(text):
+        return matches[0]
 
 
 def create_ingest_report(items, file_name):