From c43b9e37e126590a0c968c0739367ab353d50390 Mon Sep 17 00:00:00 2001 From: "B. Seeger" Date: Mon, 13 Dec 2021 15:23:26 -0500 Subject: [PATCH] Disable fits file creation and remove checks for fits from tests (#282) * This disables the Context that causes the FITS file to get generated. Also this removes any tests for FITS files. --- .env | 2 +- ...t.context.technical_metadata_on_ingest.yml | 2 +- ..._plus.migration.idc_ingest_media_audio.yml | 6 +- ...us.migration.idc_ingest_media_document.yml | 6 +- ...ration.idc_ingest_media_extracted_text.yml | 6 +- ...e_plus.migration.idc_ingest_media_file.yml | 6 +- ..._plus.migration.idc_ingest_media_image.yml | 6 +- ..._plus.migration.idc_ingest_media_video.yml | 6 +- docker-compose.env.yml | 21 ------ docker-compose.sample.env.yml | 44 ++++++------ end-to-end/tests/admin/s3.spec.js | 5 +- end-to-end/tests/ui/item-detail-image.spec.js | 4 +- .../testcafe/media_formats.spec.js | 57 ++++++++------- .../testcafe/migrate_derivative.spec.js | 72 +++++-------------- .../expected/media-documents-fits.json | 9 --- .../expected/media-image-fits-02.json | 9 --- .../expected/media-image-fits-03.json | 9 --- .../expected/media-image-fits-04.json | 9 --- .../expected/media-image-fits-05.json | 9 --- .../expected/media-largevideo-fits.json | 9 --- .../expected/media-smallvideo-fits.json | 9 --- .../verification/verify_migrations_test.go | 58 --------------- 22 files changed, 89 insertions(+), 275 deletions(-) delete mode 100644 tests/21-large-file-derivatives-nightly/verification/expected/media-documents-fits.json delete mode 100644 tests/21-large-file-derivatives-nightly/verification/expected/media-image-fits-02.json delete mode 100644 tests/21-large-file-derivatives-nightly/verification/expected/media-image-fits-03.json delete mode 100644 tests/21-large-file-derivatives-nightly/verification/expected/media-image-fits-04.json delete mode 100644 tests/21-large-file-derivatives-nightly/verification/expected/media-image-fits-05.json delete mode 100644 tests/21-large-file-derivatives-nightly/verification/expected/media-largevideo-fits.json delete mode 100644 tests/21-large-file-derivatives-nightly/verification/expected/media-smallvideo-fits.json diff --git a/.env b/.env index 3f972948f..ce3adc2a4 100644 --- a/.env +++ b/.env @@ -14,7 +14,7 @@ # to take effect; e.g. 'make dev-up' ENVIRONMENT=local -REQUIRED_SERIVCES=activemq alpaca cantaloupe idc-crayfish crayfits drupal mariadb solr idc-snapshot testcafe minio +REQUIRED_SERIVCES=activemq alpaca cantaloupe idc-crayfish drupal mariadb solr idc-snapshot testcafe minio ############################################################################### # Environment variables specific to composer. ############################################################################### diff --git a/codebase/config/sync/context.context.technical_metadata_on_ingest.yml b/codebase/config/sync/context.context.technical_metadata_on_ingest.yml index 966d68827..8b9ddbea3 100644 --- a/codebase/config/sync/context.context.technical_metadata_on_ingest.yml +++ b/codebase/config/sync/context.context.technical_metadata_on_ingest.yml @@ -14,7 +14,7 @@ label: 'Technical Metadata on Ingest' group: Islandora description: 'Defines FITS harvesting behavior' requireAllConditions: false -disabled: false +disabled: true conditions: media_has_term: id: media_has_term diff --git a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_audio.yml b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_audio.yml index aaf645ada..6225d6207 100644 --- a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_audio.yml +++ b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_audio.yml @@ -52,9 +52,9 @@ process: callable: date source: constants/DATE_FORMAT _the_filename: - plugin: default_value - source: new_filename - default: '@_url_filename' + plugin: default_value + source: new_filename + default: '@_url_filename' _destination_dir: plugin: concat source: diff --git a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_document.yml b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_document.yml index c1573c7e2..65b978e72 100644 --- a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_document.yml +++ b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_document.yml @@ -52,9 +52,9 @@ process: callable: date source: constants/DATE_FORMAT _the_filename: - plugin: default_value - source: new_filename - default: '@_url_filename' + plugin: default_value + source: new_filename + default: '@_url_filename' _destination_dir: plugin: concat source: diff --git a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_extracted_text.yml b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_extracted_text.yml index bb0ec75ef..df88b33a7 100644 --- a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_extracted_text.yml +++ b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_extracted_text.yml @@ -53,9 +53,9 @@ process: callable: date source: constants/DATE_FORMAT _the_filename: - plugin: default_value - source: new_filename - default: '@_url_filename' + plugin: default_value + source: new_filename + default: '@_url_filename' _destination_dir: plugin: concat source: diff --git a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_file.yml b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_file.yml index 5c3efcbd0..bc14635c7 100644 --- a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_file.yml +++ b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_file.yml @@ -52,9 +52,9 @@ process: callable: date source: constants/DATE_FORMAT _the_filename: - plugin: default_value - source: new_filename - default: '@_url_filename' + plugin: default_value + source: new_filename + default: '@_url_filename' _destination_dir: plugin: concat source: diff --git a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_image.yml b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_image.yml index 11a663fe7..6be0b339c 100644 --- a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_image.yml +++ b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_image.yml @@ -52,9 +52,9 @@ process: callable: date source: constants/DATE_FORMAT _the_filename: - plugin: default_value - source: new_filename - default: '@_url_filename' + plugin: default_value + source: new_filename + default: '@_url_filename' _destination_dir: plugin: concat source: diff --git a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_video.yml b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_video.yml index eb6c63aba..255e1863b 100644 --- a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_video.yml +++ b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_video.yml @@ -52,9 +52,9 @@ process: callable: date source: constants/DATE_FORMAT _the_filename: - plugin: default_value - source: new_filename - default: '@_url_filename' + plugin: default_value + source: new_filename + default: '@_url_filename' _destination_dir: plugin: concat source: diff --git a/docker-compose.env.yml b/docker-compose.env.yml index 8bfc57fb6..d78330bff 100644 --- a/docker-compose.env.yml +++ b/docker-compose.env.yml @@ -61,27 +61,6 @@ services: ALPACA_HTTP_SOCKET_TIMEOUT_MS: ${ALPACA_HTTP_SOCKET_TIMEOUT_MS:-7200000} cantaloupe: # No environment variables require overriding. - crayfits: - environment: - NGINX_CLIENT_BODY_TIMEOUT: ${NGINX_CLIENT_BODY_TIMEOUT:-3600} - NGINX_FASTCGI_READ_TIMEOUT: ${NGINX_FASTCGI_READ_TIMEOUT:-3600} - NGINX_FASTCGI_SEND_TIMEOUT: ${NGINX_FASTCGI_SEND_TIMEOUT:-3600} - NGINX_FASTCGI_CONNECT_TIMEOUT: ${NGINX_FASTCGI_CONNECT_TIMEOUT:-3600} - NGINX_KEEPALIVE_TIMEOUT: ${NGINX_KEEPALIVE_TIMEOUT:-3600} - NGINX_LINGERING_TIMEOUT: ${NGINX_LINGERING_TIMEOUT:-3600} - NGINX_PROXY_READ_TIMEOUT: ${NGINX_PROXY_READ_TIMEOUT:-3600} - NGINX_PROXY_SEND_TIMEOUT: ${NGINX_PROXY_SEND_TIMEOUT:-3600} - NGINX_PROXY_CONNECT_TIMEOUT: ${NGINX_PROXY_CONNECT_TIMEOUT:-3600} - NGINX_SEND_TIMEOUT: ${NGINX_SEND_TIMEOUT:-3600} - PHP_DEFAULT_SOCKET_TIMEOUT: ${PHP_DEFAULT_SOCKET_TIMEOUT:-3600} - PHP_MAX_EXECUTION_TIME: ${PHP_MAX_EXECUTION_TIME:-3600} - PHP_MAX_FILE_UPLOADS: ${PHP_MAX_FILE_UPLOADS:-20} - PHP_MAX_INPUT_TIME: ${PHP_MAX_INPUT_TIME:-3600} - PHP_MEMORY_LIMIT: ${PHP_MEMORY_LIMIT:-256M} - PHP_POST_MAX_SIZE: ${PHP_POST_MAX_SIZE:-128M} - PHP_PROCESS_CONTROL_TIMEOUT: ${PHP_PROCESS_CONTROL_TIMEOUT:-3600} - PHP_REQUEST_TERMINATE_TIMEOUT: ${PHP_REQUEST_TERMINATE_TIMEOUT:-3600} - PHP_UPLOAD_MAX_FILESIZE: ${PHP_UPLOAD_MAX_FILESIZE:-128M} mariadb: environment: # diff --git a/docker-compose.sample.env.yml b/docker-compose.sample.env.yml index a1cb1a383..5a186f84a 100644 --- a/docker-compose.sample.env.yml +++ b/docker-compose.sample.env.yml @@ -45,23 +45,21 @@ services: environment: # # Must be exactly the same as ACTIVEMQ_PASSWORD. - # + # ALPACA_ACTIVEMQ_PASSWORD: password # - # Password for admin user: https://karaf.apache.org/manual/latest/security + # Password for admin user: https://karaf.apache.org/manual/latest/security # ALPACA_KARAF_ADMIN_PASSWORD: password blazegraph: # No environment variables require overriding. cantaloupe: # No environment variables require overriding. - crayfits: - # No environment variables require overriding. mariadb: environment: # # Password for the root user. - # + # MYSQL_ROOT_PASSWORD: password # If you use PostgreSQL for any services uncomment this section. # postgresql: @@ -93,14 +91,14 @@ services: DRUPAL_DEFAULT_SALT: 9PPaL0CxZAIcq0l9wxgDGlCZrp7JdT_x7v9gVzpdbUjMt1PqDz3uD0Zy-i16DuJ1-Htuq5hqeg # # The administrator email account for the default site. - # + # DRUPAL_DEFAULT_ACCOUNT_EMAIL: admin@example.org # # The password of the admin account for the default site. - # + # DRUPAL_DEFAULT_ACCOUNT_PASSWORD: password # - # The password Drupal uses to connect to the database for the default site. + # The password Drupal uses to connect to the database for the default site. # DRUPAL_DEFAULT_DB_PASSWORD: password # @@ -108,7 +106,7 @@ services: # DRUPAL_DEFAULT_DB_ROOT_PASSWORD: password # - # JWT Key Pair + # JWT Key Pair # DRUPAL_JWT_PRIVATE_KEY: |- -----BEGIN RSA PRIVATE KEY----- @@ -140,7 +138,7 @@ services: -----END RSA PRIVATE KEY----- # # All other *_JWT_PUBLIC keys should match this value exactly. - # + # DRUPAL_JWT_PUBLIC_KEY: |- -----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6uK3nozywVaRCAB3FHdR @@ -167,7 +165,7 @@ services: FCREPO_JWT_ADMIN_TOKEN: islandora # # Must be exactly the same as DRUPAL_JWT_PUBLIC_KEY. - # + # FCREPO_JWT_PUBLIC_KEY: |- -----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6uK3nozywVaRCAB3FHdR @@ -191,10 +189,10 @@ services: # # Must be exactly the same as FCREPO_JWT_ADMIN_TOKEN. # - GEMINI_JWT_ADMIN_TOKEN: islandora + GEMINI_JWT_ADMIN_TOKEN: islandora # # Must be exactly the same as DRUPAL_JWT_PUBLIC_KEY. - # + # GEMINI_JWT_PUBLIC_KEY: |- -----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6uK3nozywVaRCAB3FHdR @@ -210,10 +208,10 @@ services: # # Must be exactly the same as FCREPO_JWT_ADMIN_TOKEN. # - HOMARUS_JWT_ADMIN_TOKEN: islandora + HOMARUS_JWT_ADMIN_TOKEN: islandora # # Must be exactly the same as DRUPAL_JWT_PUBLIC_KEY. - # + # HOMARUS_JWT_PUBLIC_KEY: |- -----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6uK3nozywVaRCAB3FHdR @@ -229,10 +227,10 @@ services: # # Must be exactly the same as FCREPO_JWT_ADMIN_TOKEN. # - HOUDINI_JWT_ADMIN_TOKEN: islandora + HOUDINI_JWT_ADMIN_TOKEN: islandora # # Must be exactly the same as DRUPAL_JWT_PUBLIC_KEY. - # + # HOUDINI_JWT_PUBLIC_KEY: |- -----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6uK3nozywVaRCAB3FHdR @@ -248,10 +246,10 @@ services: # # Must be exactly the same as FCREPO_JWT_ADMIN_TOKEN. # - HYPERCUBE_JWT_ADMIN_TOKEN: islandora + HYPERCUBE_JWT_ADMIN_TOKEN: islandora # # Must be exactly the same as DRUPAL_JWT_PUBLIC_KEY. - # + # HYPERCUBE_JWT_PUBLIC_KEY: |- -----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6uK3nozywVaRCAB3FHdR @@ -281,12 +279,12 @@ services: # # php -r 'echo password_hash(md5("password"), PASSWORD_DEFAULT) . "\n";' # - # N.B. You must double up '$' characters to prevent them from being + # N.B. You must double up '$' characters to prevent them from being # interpolated by docker-compose.yml MATOMO_USER_PASS: $$2y$$10$$S38e7HPM9LI3aOIvcnRsfuMCm4ipNP572QsvbCK60upoHVJ61hMrS # # Must be exactly the same as DRUPAL_DEFAULT_SITE_URL without the protocol. - # + # # If you are using an IP address with traefik.me, change the dot between COMPOSE_PROJECT_NAME and # DRUPAL_SITE_HOST to a hyphen (i.e. ${COMPOSE_PROJECT_NAME-isle-dc}.${DRUPAL_SITE_HOST-traefik.me} # becomes ${COMPOSE_PROJECT_NAME-isle-dc}-${DRUPAL_SITE_HOST-traefik.me} @@ -303,10 +301,10 @@ services: # # Must be exactly the same as FCREPO_JWT_ADMIN_TOKEN. # - RECAST_JWT_ADMIN_TOKEN: islandora + RECAST_JWT_ADMIN_TOKEN: islandora # # Must be exactly the same as DRUPAL_JWT_PUBLIC_KEY. - # + # RECAST_JWT_PUBLIC_KEY: |- -----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6uK3nozywVaRCAB3FHdR diff --git a/end-to-end/tests/admin/s3.spec.js b/end-to-end/tests/admin/s3.spec.js index d4f816401..0c2c9296d 100644 --- a/end-to-end/tests/admin/s3.spec.js +++ b/end-to-end/tests/admin/s3.spec.js @@ -55,15 +55,13 @@ test('Verify original file and derivatives are in S3', async t => { // (increase timeout in case derivatives haven't been created yet?) const service_derivative = Selector('div.view-content').find('a').withText('Service File.jpg'); const thumb_derivative = Selector('div.view-content').find('a').withText('Thumbnail Image.jpg'); - const fits_derivative = Selector('div.view-content').find('a').withText('FITS File.xml'); const service_count = await service_derivative.count const thumb_count = await thumb_derivative.count - const fits_count = await fits_derivative.count // if a derivative isn't present yet, it may be because it hasn't been generated yet. // in that case, wait 30 seconds and refresh the page, and see if it appears. - if (service_count < 1 || thumb_count < 1 || fits_count < 1) { + if (service_count < 1 || thumb_count < 1) { console.log("Derivatives haven't appeared. Sleeping for 30 seconds, then trying again ...") // sleep 30 seconds, refresh the page await t.wait(30000); @@ -72,7 +70,6 @@ test('Verify original file and derivatives are in S3', async t => { await t.expect(service_derivative.count).eql(1); await t.expect(thumb_derivative.count).eql(1); - await t.expect(fits_derivative.count).eql(1); const original_uri = await media.getAttribute('href'); const thumb_media_uri = await thumb_derivative.getAttribute('href'); diff --git a/end-to-end/tests/ui/item-detail-image.spec.js b/end-to-end/tests/ui/item-detail-image.spec.js index 72e920612..d2375d343 100644 --- a/end-to-end/tests/ui/item-detail-image.spec.js +++ b/end-to-end/tests/ui/item-detail-image.spec.js @@ -60,8 +60,8 @@ test('Download', async (t) => { .expect(ImagePage.downloadModal.visibility().exists).notOk() .click(ImagePage.downloadBtn) .expect(ImagePage.downloadModal.visibility().exists).ok() - .expect(ImagePage.downloadModal.content.count).eql(3) - .expect(ImagePage.downloadModal.content.find('a[download]').count).eql(3); + .expect(ImagePage.downloadModal.content.count).eql(2) + .expect(ImagePage.downloadModal.content.find('a[download]').count).eql(2); }); test('Export metadata', async (t) => { diff --git a/tests/12-media-tests/testcafe/media_formats.spec.js b/tests/12-media-tests/testcafe/media_formats.spec.js index b01b056c5..fdada74b0 100644 --- a/tests/12-media-tests/testcafe/media_formats.spec.js +++ b/tests/12-media-tests/testcafe/media_formats.spec.js @@ -16,8 +16,8 @@ test("Migrate tiff", async (t) => { .expect( await tryUntilTrue(async () => { const media = await findMediaOf(t, "Migrated Tiff Object"); - // 4 for Tif, service jpg, thumbnail jpg, FITS file - return media.length == 4; + // 3 for Tif, service jpg, thumbnail jpg + return media.length == 3; }) ) .eql(true, "Did not find TIFF derivatives"); @@ -33,8 +33,8 @@ test("Upload tiff in UI", async (t) => { .expect( await tryUntilTrue(async () => { const media = await findMediaOf(t, "Uploaded Tiff Object"); - // 4 for Tif, service jpg, thumbnail jpg, FITS file - return media.length == 4; + // 3 for Tif, service jpg, thumbnail jpg + return media.length == 3; }) ) .eql(true, "Did not find TIFF derivatives"); @@ -45,8 +45,8 @@ test("Migrate jpeg 2000", async (t) => { .expect( await tryUntilTrue(async () => { const media = await findMediaOf(t, "Migrated JPEG2000 Object"); - // 4 for jp2000, service jpg, thumbnail jpg, FITS file - return media.length == 4; + // 3 for jp2000, service jpg, thumbnail jpg + return media.length == 3; }) ) .eql(true, "Did not find JPEG 2000 derivatives"); @@ -62,8 +62,8 @@ test("Upload jpeg 2000 in UI", async (t) => { .expect( await tryUntilTrue(async () => { const media = await findMediaOf(t, "Uploaded JPEG2000 Object"); - // 4 for jp2000, service jpg, thumbnail jpg, FITS file - return media.length == 4; + // 3 for jp2000, service jpg, thumbnail jpg + return media.length == 3; }) ) .eql(true, "Did not find jpg 2000 derivatives"); @@ -74,8 +74,8 @@ test("Migrate Geo Tiff", async (t) => { .expect( await tryUntilTrue(async () => { const media = await findMediaOf(t, "Migrated Geo Tiff Object"); - // the geo tiff and just a FITS deriv - return media.length == 2; + // the geo tiff + return media.length == 1; }) ) .eql(true, "Did not find GEO TIF derivatives"); @@ -92,8 +92,8 @@ test("Upload Geo Tiff in UI", async (t) => { .expect( await tryUntilTrue(async () => { const media = await findMediaOf(t, "Uploaded Geo Tiff Object"); - // the geo tiff and just a FITS deriv - return media.length == 2; + // the geo tiff + return media.length == 1; }) ) .eql(true, "Did not find Geo Tif derivatives"); @@ -104,8 +104,8 @@ test("Migrate Zip File", async (t) => { .expect( await tryUntilTrue(async () => { const media = await findMediaOf(t, "Migrated Zip Object"); - // the zip and just a FITS deriv - return media.length == 2; + // the zip + return media.length == 1; }) ) .eql(true, "Did not find Zip derivatives"); @@ -121,8 +121,8 @@ test("Upload Zip in UI", async (t) => { .expect( await tryUntilTrue(async () => { const media = await findMediaOf(t, "Uploaded Zip Object"); - // the zip and just a FITS deriv - return media.length == 2; + // the zip + return media.length == 1; }) ) .eql(true, "Did not find Zip derivatives"); @@ -133,8 +133,8 @@ test("Migrate Tar File", async (t) => { .expect( await tryUntilTrue(async () => { const media = await findMediaOf(t, "Migrated Tar Object"); - // the tar and just a FITS deriv - return media.length == 2; + // the tar + return media.length == 1; }) ) .eql(true, "Did not find Tar derivatives"); @@ -150,8 +150,8 @@ test("Upload Tar in UI", async (t) => { .expect( await tryUntilTrue(async () => { const media = await findMediaOf(t, "Uploaded Tar Object"); - // the tar and just a FITS deriv - return media.length == 2; + // the tar + return media.length == 1; }) ) .eql(true, "Did not find Tar derivatives"); @@ -162,8 +162,8 @@ test("Migrate GZip File", async (t) => { .expect( await tryUntilTrue(async () => { const media = await findMediaOf(t, "Migrated GZip Object"); - // the gzip and just a FITS deriv - return media.length == 2; + // the gzip + return media.length == 1; }) ) .eql(true, "Did not find GZip derivatives"); @@ -179,8 +179,8 @@ test("Upload GZip in UI", async (t) => { .expect( await tryUntilTrue(async () => { const media = await findMediaOf(t, "Uploaded GZip Object"); - // the gzip and just a FITS deriv - return media.length == 2; + // the gzip + return media.length == 1; }) ) .eql(true, "Did not find GZip derivatives"); @@ -191,9 +191,8 @@ test("Migrate TGZ File", async (t) => { .expect( await tryUntilTrue(async () => { const media = await findMediaOf(t, "Migrated TGZ Object"); - // the tgz and just a FITS deriv - console.log("tgz count: " + media.length); - return media.length == 2; + // the tgz + return media.length == 1; }) ) .eql(true, "Did not find TGZ derivatives"); @@ -209,8 +208,8 @@ test("Upload TGZ in UI", async (t) => { .expect( await tryUntilTrue(async () => { const media = await findMediaOf(t, "Uploaded TGZ Object"); - // the tgz and just a FITS deriv - return media.length == 2; + // the tgz + return media.length == 1; }) ) .eql(true, "Did not find TGZ derivatives"); diff --git a/tests/12-media-tests/testcafe/migrate_derivative.spec.js b/tests/12-media-tests/testcafe/migrate_derivative.spec.js index bc1e423cd..0c53b8c0c 100644 --- a/tests/12-media-tests/testcafe/migrate_derivative.spec.js +++ b/tests/12-media-tests/testcafe/migrate_derivative.spec.js @@ -45,16 +45,14 @@ test('Migrate Images for Derivative Generation', async t => { // (increase timeout in case derivatives haven't been created yet?) const service_derivative = Selector('div.view-content').find('a').withText('Service File.jpg'); const thumb_derivative = Selector('div.view-content').find('a').withText('Thumbnail Image.jpg'); - const fits_derivative = Selector('div.view-content').find('a').withText('FITS File.xml'); console.log("Checking for derivatives ...") await t.expect(await tryUntilTrue(async () => { const service_count = await service_derivative.count const thumb_count = await thumb_derivative.count - const fits_count = await fits_derivative.count - console.log("Service_count: ", service_count, ", thumb count: ", thumb_count, ", fits count:", fits_count); - if (service_count < 1 || thumb_count < 1 || fits_count < 1) { + console.log("Service_count: ", service_count, ", thumb count: ", thumb_count); + if (service_count < 1 || thumb_count < 1) { await t.eval(() => location.reload(true)); return false; } @@ -63,12 +61,10 @@ test('Migrate Images for Derivative Generation', async t => { await t.expect(service_derivative.count).eql(1); await t.expect(thumb_derivative.count).eql(1); - await t.expect(fits_derivative.count).eql(1); }); test('Test Derivatives Unique Id Field', async t => { - // note: FITS media do not have this field. const media_list = ['Service File.jpg', 'Thumbnail Image.jpg']; const io_name = "Derivative Repository Item One" @@ -127,16 +123,14 @@ test('Test AudioVideo Derivative Generation Conditions', async t => { // (increase timeout in case derivatives haven't been created yet?) let service_derivative = Selector('div.view-content').find('a').withText('Service File.mp4'); let thumb_derivative = Selector('div.view-content').find('a').withText('Thumbnail Image.jpg'); - let fits_derivative = Selector('div.view-content').find('a').withText('FITS File.xml'); console.log("Checking for derivatives ...") await t.expect(await tryUntilTrue(async () => { const service_count = await service_derivative.count const thumb_count = await thumb_derivative.count - const fits_count = await fits_derivative.count - console.log("Service_count: ", service_count, ", thumb count: ", thumb_count, ", fits count:", fits_count); - if (service_count < 1 || thumb_count < 1 || fits_count < 1) { + console.log("Service_count: ", service_count, ", thumb count: ", thumb_count); + if (service_count < 1 || thumb_count < 1) { await t.eval(() => location.reload(true)); return false; } @@ -145,7 +139,6 @@ test('Test AudioVideo Derivative Generation Conditions', async t => { await t.expect(service_derivative.count).eql(1); await t.expect(thumb_derivative.count).eql(1); - await t.expect(fits_derivative.count).eql(1); // // 2) there should be one video w/o a deriv @@ -173,16 +166,14 @@ test('Test AudioVideo Derivative Generation Conditions', async t => { // (increase timeout in case derivatives haven't been created yet?) service_derivative = Selector('div.view-content').find('a').withText('Service File.mp4'); thumb_derivative = Selector('div.view-content').find('a').withText('Thumbnail Image.jpg'); - fits_derivative = Selector('div.view-content').find('a').withText('FITS File.xml'); console.log("Checking for derivatives ...") await t.expect(await tryUntilTrue(async () => { const service_count = await service_derivative.count const thumb_count = await thumb_derivative.count - const fits_count = await fits_derivative.count - console.log("Service_count: ", service_count, ", thumb count: ", thumb_count, ", fits count:", fits_count); - if (thumb_count < 1 || fits_count < 1) { + console.log("Service_count: ", service_count, ", thumb count: ", thumb_count); + if (thumb_count < 1) { await t.eval(() => location.reload(true)); return false; } @@ -192,7 +183,6 @@ test('Test AudioVideo Derivative Generation Conditions', async t => { // there should be no service file. It could show up later, so this might be a weak test. await t.expect(service_derivative.count).eql(0); await t.expect(thumb_derivative.count).eql(1); - await t.expect(fits_derivative.count).eql(1); // // 3) there should be one audio file with a deriv @@ -216,18 +206,16 @@ test('Test AudioVideo Derivative Generation Conditions', async t => { await t.expect(media.parent('tr').child('td').nth(4).innerText).contains('Original File') await t.expect(media.parent('tr').child('td').nth(4).innerText).notContains('Service File'); - // assert the presence of a derivative service image and FITS file + // assert the presence of a derivative service image // (increase timeout in case derivatives haven't been created yet?) service_derivative = Selector('div.view-content').find('a').withText('Service File.mp3'); - fits_derivative = Selector('div.view-content').find('a').withText('FITS File.xml'); console.log("Checking for derivatives ...") await t.expect(await tryUntilTrue(async () => { const service_count = await service_derivative.count - const fits_count = await fits_derivative.count - console.log("Service_count: ", service_count, ", fits count:", fits_count); - if (service_count < 1 || fits_count < 1) { + console.log("Service_count: ", service_count); + if (service_count < 1) { await t.eval(() => location.reload(true)); return false; } @@ -235,7 +223,6 @@ test('Test AudioVideo Derivative Generation Conditions', async t => { })).eql(true, "Derivatives have not appeared"); await t.expect(service_derivative.count).eql(1); - await t.expect(fits_derivative.count).eql(1); // // 4) there should be one audio file w/o a deriv @@ -259,27 +246,12 @@ test('Test AudioVideo Derivative Generation Conditions', async t => { await t.expect(media.parent('tr').child('td').nth(4).innerText).contains('Original File') await t.expect(media.parent('tr').child('td').nth(4).innerText).contains('Service File'); - // check for the presence of a Service and FITs file. Though there should not be + // check for the presence of a Service. Though there should not be // a separate Service File service_derivative = Selector('div.view-content').find('a').withText('Service File.mp3'); - fits_derivative = Selector('div.view-content').find('a').withText('FITS File.xml'); - - console.log("Checking for derivatives ...") - - await t.expect(await tryUntilTrue(async () => { - const service_count = await service_derivative.count - const fits_count = await fits_derivative.count - console.log("Service_count: ", service_count, ", fits count:", fits_count); - if (fits_count < 1) { - await t.eval(() => location.reload(true)); - return false; - } - return true; - })).eql(true, "Derivatives have not appeared"); // there should be no service file. It could show up later, so this might be a weak test. await t.expect(service_derivative.count).eql(0); - await t.expect(fits_derivative.count).eql(1); }); test('Migrate PDF for Derivative Generation', async t => { @@ -310,23 +282,20 @@ test('Migrate PDF for Derivative Generation', async t => { await t.expect(media.parent('tr').child('td').nth(3).innerText).eql('application/pdf') await t.expect(media.parent('tr').child('td').nth(4).innerText).contains('Original File') - // assert the presence of a derivative thumbnail and fits file + // assert the presence of a derivative thumbnail // (increase timeout in case derivatives haven't been created yet?) // no service files are generated for PDFs const thumb_derivative = Selector('div.view-content').find('a').withText('Thumbnail Image.png'); - const fits_derivative = Selector('div.view-content').find('a').withText('FITS File.xml'); const ocr_derivative = Selector('div.view-content').find('a').withText('Extracted Text.txt'); console.log("Checking for derivatives ...") await t.expect(await tryUntilTrue(async () => { const thumb_count = await thumb_derivative.count - const fits_count = await fits_derivative.count const ocr_count = await ocr_derivative.count - console.log('thumb count: ', thumb_count, ', fits count: ', fits_count, - ', extracted text count: ', ocr_count); - if (thumb_count < 1 || fits_count < 1 || ocr_count < 1) { + console.log('thumb count: ', thumb_count, ', extracted text count: ', ocr_count); + if (thumb_count < 1 || ocr_count < 1) { await t.eval(() => location.reload(true)); return false; } @@ -334,7 +303,6 @@ test('Migrate PDF for Derivative Generation', async t => { })).eql(true, "Derivatives have not appeared"); await t.expect(thumb_derivative.count).eql(1); - await t.expect(fits_derivative.count).eql(1); await t.expect(ocr_derivative.count).eql(1); }); @@ -364,17 +332,15 @@ test('Test Collection Derivative Generation Condition: Image', async t => { // assert the presence of a derivative thumbnail and service image const service_derivative = Selector('div.view-content').find('a').withText('Service File.jpg'); const thumb_derivative = Selector('div.view-content').find('a').withText('Thumbnail Image.jpg'); - const fits_derivative = Selector('div.view-content').find('a').withText('FITS File.xml'); console.log("Checking for derivatives ...") await t.expect(await tryUntilTrue(async () => { const service_count = await service_derivative.count const thumb_count = await thumb_derivative.count - const fits_count = await fits_derivative.count - console.log("Service_count: ", service_count, ", thumb count: ", thumb_count, ", fits count:", fits_count); - if (service_count < 1 || thumb_count < 1 || fits_count < 1) { + console.log("Service_count: ", service_count, ", thumb count: ", thumb_count); + if (service_count < 1 || thumb_count < 1) { await t.eval(() => location.reload(true)); return false; } @@ -383,7 +349,6 @@ test('Test Collection Derivative Generation Condition: Image', async t => { await t.expect(service_derivative.count).eql(1); await t.expect(thumb_derivative.count).eql(1); - await t.expect(fits_derivative.count).eql(1); }); test('Test Paged Content Derivative Generation Condition: Paged Content', async t => { @@ -465,7 +430,6 @@ const checkChildPage = async (t, pageItemName, mediaName, mediaType, mimeType) = // assert the presence of a derivative thumbnail and service image const service_derivative = Selector('div.view-content').find('a').withText('Service File.jpg'); const thumb_derivative = Selector('div.view-content').find('a').withText('Thumbnail Image.jpg'); - const fits_derivative = Selector('div.view-content').find('a').withText('FITS File.xml'); const extracted_text = Selector('div.view-content').find('a').withText('Extracted Text.txt'); console.log('Checking for derivatives ...') @@ -473,12 +437,11 @@ const checkChildPage = async (t, pageItemName, mediaName, mediaType, mimeType) = await t.expect(await tryUntilTrue(async () => { const service_count = await service_derivative.count const thumb_count = await thumb_derivative.count - const fits_count = await fits_derivative.count const extracted_count = await extracted_text.count console.log('Service_count: ', service_count, ', thumb count: ', thumb_count, - ', fits count: ', fits_count, ', extracted text: ', extracted_count) - if (service_count < 1 || thumb_count < 1 || fits_count < 1 || extracted_count < 1) { + ', extracted text: ', extracted_count) + if (service_count < 1 || thumb_count < 1 || extracted_count < 1) { await t.eval(() => location.reload(true)); return false; } @@ -487,6 +450,5 @@ const checkChildPage = async (t, pageItemName, mediaName, mediaType, mimeType) = await t.expect(service_derivative.count).eql(1); await t.expect(thumb_derivative.count).eql(1); - await t.expect(fits_derivative.count).eql(1); await t.expect(extracted_text.count).eql(1); } diff --git a/tests/21-large-file-derivatives-nightly/verification/expected/media-documents-fits.json b/tests/21-large-file-derivatives-nightly/verification/expected/media-documents-fits.json deleted file mode 100644 index 896c988cd..000000000 --- a/tests/21-large-file-derivatives-nightly/verification/expected/media-documents-fits.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "type": "media", - "bundle": "fits_technical_metadata", - "name": "41-FITS File.xml", - "use": [ - "FITS File" - ], - "media_of": "Derivative Documents" -} diff --git a/tests/21-large-file-derivatives-nightly/verification/expected/media-image-fits-02.json b/tests/21-large-file-derivatives-nightly/verification/expected/media-image-fits-02.json deleted file mode 100644 index 7aca851b9..000000000 --- a/tests/21-large-file-derivatives-nightly/verification/expected/media-image-fits-02.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "type": "media", - "bundle": "fits_technical_metadata", - "name": "35-FITS File.xml", - "use": [ - "FITS File" - ], - "media_of": "Derivative Image 02" -} diff --git a/tests/21-large-file-derivatives-nightly/verification/expected/media-image-fits-03.json b/tests/21-large-file-derivatives-nightly/verification/expected/media-image-fits-03.json deleted file mode 100644 index ce99dc671..000000000 --- a/tests/21-large-file-derivatives-nightly/verification/expected/media-image-fits-03.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "type": "media", - "bundle": "fits_technical_metadata", - "name": "36-FITS File.xml", - "use": [ - "FITS File" - ], - "media_of": "Derivative Image 03" -} diff --git a/tests/21-large-file-derivatives-nightly/verification/expected/media-image-fits-04.json b/tests/21-large-file-derivatives-nightly/verification/expected/media-image-fits-04.json deleted file mode 100644 index 190e5ad9e..000000000 --- a/tests/21-large-file-derivatives-nightly/verification/expected/media-image-fits-04.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "type": "media", - "bundle": "fits_technical_metadata", - "name": "37-FITS File.xml", - "use": [ - "FITS File" - ], - "media_of": "Derivative Image 04" -} diff --git a/tests/21-large-file-derivatives-nightly/verification/expected/media-image-fits-05.json b/tests/21-large-file-derivatives-nightly/verification/expected/media-image-fits-05.json deleted file mode 100644 index 71eb05b80..000000000 --- a/tests/21-large-file-derivatives-nightly/verification/expected/media-image-fits-05.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "type": "media", - "bundle": "fits_technical_metadata", - "name": "38-FITS File.xml", - "use": [ - "FITS File" - ], - "media_of": "Derivative Image 05" -} diff --git a/tests/21-large-file-derivatives-nightly/verification/expected/media-largevideo-fits.json b/tests/21-large-file-derivatives-nightly/verification/expected/media-largevideo-fits.json deleted file mode 100644 index 05de69d5a..000000000 --- a/tests/21-large-file-derivatives-nightly/verification/expected/media-largevideo-fits.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "type": "media", - "bundle": "fits_technical_metadata", - "name": "40-FITS File.xml", - "use": [ - "FITS File" - ], - "media_of": "Derivative Large Video" -} diff --git a/tests/21-large-file-derivatives-nightly/verification/expected/media-smallvideo-fits.json b/tests/21-large-file-derivatives-nightly/verification/expected/media-smallvideo-fits.json deleted file mode 100644 index 7a570c466..000000000 --- a/tests/21-large-file-derivatives-nightly/verification/expected/media-smallvideo-fits.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "type": "media", - "bundle": "fits_technical_metadata", - "name": "39-FITS File.xml", - "use": [ - "FITS File" - ], - "media_of": "Derivative Small Video" -} diff --git a/tests/21-large-file-derivatives-nightly/verification/verify_migrations_test.go b/tests/21-large-file-derivatives-nightly/verification/verify_migrations_test.go index e22fcd2d4..465610204 100644 --- a/tests/21-large-file-derivatives-nightly/verification/verify_migrations_test.go +++ b/tests/21-large-file-derivatives-nightly/verification/verify_migrations_test.go @@ -35,7 +35,6 @@ const ( expectedOriginalVideoCount = 2 expectedOriginalImageCount = 5 expectedDerivativeThumbCount = expectedRepoObjectCount - expectedDerivativeFitsCount = expectedRepoObjectCount - 1 // TODO: the first image doesn't have a FITS file expectedDerivativeExtractedTxtCount = expectedOriginalDocumentsMediaCount // Only the PDF has extracted text expectedDerivativeServiceCount = expectedRepoObjectCount - expectedOriginalDocumentsMediaCount // The PDF doesn't have a service file; TODO: Missing service files for the large and small video ) @@ -53,8 +52,6 @@ var expectedCount = struct { originalImages int // The expected number of derivative thumbnails generated after migration derivativeThumbnails int - // The expected number of FITS technical metadata generated after migration - derivativeFits int // The expected number of extracted text media generated after migration derivativeExtractedText int // The expected number of service files generated after migration @@ -66,7 +63,6 @@ var expectedCount = struct { expectedOriginalVideoCount, expectedOriginalImageCount, expectedDerivativeThumbCount, - expectedDerivativeFitsCount, expectedDerivativeExtractedTxtCount, expectedDerivativeServiceCount, } @@ -282,60 +278,6 @@ func Test_Derivative_Thumbnail_Media(t *testing.T) { wg.Wait() } -func Test_Derivative_Fits_Media(t *testing.T) { - // Run derivative tests in parallel to avoid long wait times - t.Parallel() - - // Collect the filenames of the expected fits media - // TODO: media-image-fits-01 won't have one - filenames := filenamesMatching(t, "fits") - assert.Len(t, filenames, expectedCount.derivativeFits, - "Expected %d files but got %d", expectedCount.derivativeFits, len(filenames)) - - // Execute each request for fits media in its own goroutine - - wg := sync.WaitGroup{} - wg.Add(len(filenames)) - for _, filename := range filenames { - expectedMedia := &model.ExpectedMediaGeneric{} - unmarshalErr := unmarshal(filename, expectedMedia) - assert.Nil(t, unmarshalErr, "error unmarshaling '%s': %s", filename, unmarshalErr) - - var filteredMedia filterable - filteredMedia = applyDerivedMediaFilter(expectedMedia, expectedMedia.MediaOf) - - actual := &model.JsonApiFitsMedia{} - - go func() { - err := doUntil(func() error { - var err error - assertExists(t, filteredMedia, actual, func(expected, actual interface{}) { - jsonApiData := actual.(*model.JsonApiFitsMedia).JsonApiData - if len(jsonApiData) == 1 { - assert.NotEmpty(t, expectedMedia.NameOrTitle()) - assert.Contains(t, expectedMedia.NameOrTitle(), stripPrefix("-", jsonApiData[0].JsonApiAttributes.Name)) - assertParentObject(t, expectedMedia) - err = done - } - }) - - if len(actual.JsonApiData) > 1 { - err = errors.New(fmt.Sprintf("too many results retrieving JSONAPI entity %s, bundle %s, %s %s", - expectedMedia.EntityType(), expectedMedia.EntityBundle(), expectedMedia.Field(), expectedMedia.NameOrTitle())) - } - - return err - }, time.Now().Add(defaultTimeout*time.Millisecond), 1000, 2.0) - - assert.Equal(t, done, err, "Failed retrieving '%s': %s", expectedMedia.NameOrTitle(), err) - - wg.Done() - }() - } - - wg.Wait() -} - func Test_Derivative_ExtractedText_Media(t *testing.T) { // Run derivative tests in parallel to avoid long wait times t.Parallel()