Merge pull request #274 from jhu-idc/ingest_filename_change_447

Migration changes to allow filename changes during ingest AND fix ownership issue on ingested files.
jhu-idc · Nov 19, 2021 · 209c319 · 209c319
2 parents ddf6032 + a796b9e
commit 209c319
Show file tree

Hide file tree

Showing 30 changed files with 364 additions and 77 deletions.
diff --git a/.env b/.env
@@ -135,6 +135,6 @@ ACTIVEMQ_WEB_HOST=0.0.0.0
 
 # Testing
 TEST_OPERATION_TIMEOUT_MS=180000
-MIGRATION_ASSETS_IMAGE_TAG=37f19ad.1636118605
+MIGRATION_ASSETS_IMAGE_TAG=9519543.1637091427
 ALPACA_HOMERUS_HTTP_SOCKET_TIMEOUT_MS=7200000
 DRUPAL_JWT_EXPIRY_INTERVAL='+4 hour'
diff --git a/codebase/composer.lock b/codebase/composer.lock
diff --git a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_audio.yml b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_audio.yml
@@ -17,7 +17,7 @@ source:
   constants:
     STATUS: true
     DISPLAY: true
-    ADMIN: 1
+    ADMIN: -1
     DRUPAL_FS: 'private://'
     TMP_FS: /tmp/
     DATE_FORMAT: Y-m-d
@@ -51,6 +51,10 @@ process:
     plugin: callback
     callable: date
     source: constants/DATE_FORMAT
+  _the_filename:
+      plugin: default_value
+      source: new_filename
+      default: '@_url_filename'
   _destination_dir:
     plugin: concat
     source:
@@ -60,9 +64,8 @@ process:
     plugin: concat
     source:
       - constants/DRUPAL_FS
-      - '@_destination_filepath'
       - '@_destination_dir'
-      - '@_url_filename'
+      - '@_the_filename'
   field_access_terms:
     -
       plugin: skip_on_empty
@@ -91,7 +94,7 @@ process:
     id_only: true
     destination: '@_destination_drupalpath'
     mimetype: mime_type
-    filename: '@_url_filename'
+    filename: '@_the_filename'
   field_mime_type: mime_type
   name: name
   field_media_of:

diff --git a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_document.yml b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_document.yml
@@ -17,7 +17,7 @@ source:
   constants:
     STATUS: true
     DISPLAY: true
-    ADMIN: 1
+    ADMIN: -1
     DRUPAL_FS: 'private://'
     TMP_FS: /tmp/
     DATE_FORMAT: Y-m-d
@@ -51,6 +51,10 @@ process:
     plugin: callback
     callable: date
     source: constants/DATE_FORMAT
+  _the_filename:
+      plugin: default_value
+      source: new_filename
+      default: '@_url_filename'
   _destination_dir:
     plugin: concat
     source:
@@ -61,7 +65,7 @@ process:
     source:
       - constants/DRUPAL_FS
       - '@_destination_dir'
-      - '@_url_filename'
+      - '@_the_filename'
   field_access_terms:
     -
       plugin: skip_on_empty
@@ -90,7 +94,7 @@ process:
     id_only: true
     destination: '@_destination_drupalpath'
     mimetype: mime_type
-    filename: '@_url_filename'
+    filename: '@_the_filename'
   field_mime_type: mime_type
   name/value: name
   field_media_of:

diff --git a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_extracted_text.yml b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_extracted_text.yml
@@ -17,7 +17,7 @@ source:
   constants:
     STATUS: true
     DISPLAY: true
-    ADMIN: 1
+    ADMIN: -1
     DRUPAL_FS: 'private://'
     TMP_FS: /tmp/
     FORMAT: basic_html
@@ -52,6 +52,10 @@ process:
     plugin: callback
     callable: date
     source: constants/DATE_FORMAT
+  _the_filename:
+      plugin: default_value
+      source: new_filename
+      default: '@_url_filename'
   _destination_dir:
     plugin: concat
     source:
@@ -62,7 +66,7 @@ process:
     source:
       - constants/DRUPAL_FS
       - '@_destination_dir'
-      - '@_url_filename'
+      - '@_the_filename'
   field_access_terms:
     -
       plugin: skip_on_empty
@@ -90,7 +94,7 @@ process:
     id_only: true
     destination: '@_destination_drupalpath'
     mimetype: mime_type
-    filename: '@_url_filename'
+    filename: '@_the_filename'
   field_mime_type: mime_type
   name/value: name
   field_media_of:

diff --git a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_file.yml b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_file.yml
@@ -17,7 +17,7 @@ source:
   constants:
     STATUS: true
     DISPLAY: true
-    ADMIN: 1
+    ADMIN: -1
     DRUPAL_FS: 'private://'
     TMP_FS: /tmp/
     DATE_FORMAT: Y-m-d
@@ -51,6 +51,10 @@ process:
     plugin: callback
     callable: date
     source: constants/DATE_FORMAT
+  _the_filename:
+      plugin: default_value
+      source: new_filename
+      default: '@_url_filename'
   _destination_dir:
     plugin: concat
     source:
@@ -61,7 +65,7 @@ process:
     source:
       - constants/DRUPAL_FS
       - '@_destination_dir'
-      - '@_url_filename'
+      - '@_the_filename'
   field_access_terms:
     -
       plugin: skip_on_empty
@@ -90,7 +94,7 @@ process:
     id_only: true
     destination: '@_destination_drupalpath'
     mimetype: mime_type
-    filename: '@_url_filename'
+    filename: '@_the_filename'
   field_mime_type: mime_type
   name: name
   field_media_of:

diff --git a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_image.yml b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_image.yml
@@ -17,7 +17,7 @@ source:
   constants:
     STATUS: true
     DISPLAY: true
-    ADMIN: 1
+    ADMIN: -1
     DRUPAL_FS: 'private://'
     TMP_FS: /tmp/
     DATE_FORMAT: Y-m-d
@@ -51,6 +51,10 @@ process:
     plugin: callback
     callable: date
     source: constants/DATE_FORMAT
+  _the_filename:
+      plugin: default_value
+      source: new_filename
+      default: '@_url_filename'
   _destination_dir:
     plugin: concat
     source:
@@ -61,7 +65,7 @@ process:
     source:
       - constants/DRUPAL_FS
       - '@_destination_dir'
-      - '@_url_filename'
+      - '@_the_filename'
   field_access_terms:
     -
       plugin: skip_on_empty
@@ -90,7 +94,7 @@ process:
     id_only: true
     destination: '@_destination_drupalpath'
     mimetype: mime_type
-    filename: '@_url_filename'
+    filename: '@_the_filename'
   field_media_image/alt: alt_text
   field_mime_type: mime_type
   name: name

diff --git a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_remote_video.yml b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_remote_video.yml
@@ -17,7 +17,7 @@ source:
   constants:
     STATUS: true
     DISPLAY: true
-    ADMIN: 1
+    ADMIN: -1
     DRUPAL_FS: 'private://'
     TMP_FS: /tmp/
 process:

diff --git a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_video.yml b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_video.yml
@@ -17,7 +17,7 @@ source:
   constants:
     STATUS: true
     DISPLAY: true
-    ADMIN: 1
+    ADMIN: -1
     DRUPAL_FS: 'private://'
     TMP_FS: /tmp/
     DATE_FORMAT: Y-m-d
@@ -51,6 +51,10 @@ process:
     plugin: callback
     callable: date
     source: constants/DATE_FORMAT
+  _the_filename:
+      plugin: default_value
+      source: new_filename
+      default: '@_url_filename'
   _destination_dir:
     plugin: concat
     source:
@@ -61,7 +65,7 @@ process:
     source:
       - constants/DRUPAL_FS
       - '@_destination_dir'
-      - '@_url_filename'
+      - '@_the_filename'
   field_access_terms:
     -
       plugin: skip_on_empty
@@ -90,7 +94,7 @@ process:
     id_only: true
     destination: '@_destination_drupalpath'
     mimetype: mime_type
-    filename: '@_url_filename'
+    filename: '@_the_filename'
   field_mime_type: mime_type
   name: name
   field_media_of:

diff --git a/tests/12-media-tests/testcafe/formats.init.js b/tests/12-media-tests/testcafe/formats.init.js
@@ -1,13 +1,14 @@
-import { adminUser } from "./roles.js";
+import { adminUser, globalAdminUser, createGlobalAdmin, createCLA } from "./roles.js";
 import { doMigration } from "./util.js";
 import { migrationType } from "./util.js";
 
-fixture`Ingest Media Formats`
+fixture`Ingest Media Formats and create users`
   .page`https://islandora-idc.traefik.me/migrate_source_ui`.beforeEach(
   async (t) => {
     await t.useRole(adminUser);
-  }
-);
+    await createGlobalAdmin(t);
+    await t.useRole(globalAdminUser);
+  });
 
 test("Migrate Images for testing format support", async (t) => {
   await doMigration(
@@ -36,3 +37,66 @@ test("Migrate Images for testing format support", async (t) => {
     migrationType.NEW_ACCESS_TERMS,
     "./migrations/media-format-accessterms.csv")
 });
+
+test("Migrate Images for testing filename support", async (t) => {
+  await doMigration(
+    t,
+    migrationType.NEW_ACCESS_TERMS,
+    "./migrations/media-fn-accessterms.csv"
+  );
+  await doMigration(
+    t,
+    migrationType.NEW_ACCESS_RIGHTS,
+    "./migrations/media-fn-accessrights.csv"
+  );
+  await doMigration(
+    t,
+    migrationType.NEW_SUBJECTS,
+    "./migrations/media-fn-subjects.csv"
+  );
+  await doMigration(
+    t,
+    migrationType.NEW_COLLECTION,
+    "./migrations/media-fn-collection.csv"
+  );
+  await doMigration(
+    t,
+    migrationType.NEW_ITEM,
+    "./migrations/media-fn-islandora-object.csv"
+  );
+  await doMigration(
+    t,
+    migrationType.NEW_MEDIA_AUDIO,
+    "./migrations/media-fn-audio.csv"
+  );
+  await doMigration(
+    t,
+    migrationType.NEW_MEDIA_DOCUMENT,
+    "./migrations/media-fn-document.csv"
+  );
+  await doMigration(
+    t,
+    migrationType.NEW_MEDIA_EXTRACTED_TEXT,
+    "./migrations/media-fn-extracted-text.csv"
+  );
+  await doMigration(
+    t,
+    migrationType.NEW_MEDIA_FILE,
+    "./migrations/media-fn-file.csv"
+  );
+  await doMigration(
+    t,
+    migrationType.NEW_MEDIA_IMAGE,
+    "./migrations/media-fn-image.csv"
+  );
+  await doMigration(
+    t,
+    migrationType.NEW_MEDIA_VIDEO,
+    "./migrations/media-fn-video.csv"
+  );
+});
+
+test("Create test users", async (t) => {
+    // could creat globalAdmin, but it's created in the fixture already
+    await createCLA(t, 'Islandora Access');
+});