From 556198616998e1456cf0a1ae6f01f92f720af7f5 Mon Sep 17 00:00:00 2001 From: mhuffnagle Date: Tue, 23 Oct 2018 09:09:14 -0400 Subject: [PATCH 1/2] WIP Running only parse-pdr tests --- .travis.yml | 153 ++++++++++++----------- tasks/parse-pdr/tests/parse_pdrs_test.js | 23 ++-- 2 files changed, 85 insertions(+), 91 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5b5253479f0..897b6fb8721 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,13 +22,13 @@ jobs: script: ./travis-ci/build-cache.sh - - name: "eslint" - stage: unit test - install: - - ./travis-ci/fetch-cache.sh - - npm install - - npm run bootstrap-no-build - script: ./node_modules/.bin/eslint packages/api packages/cmrjs packages/common + # - name: "eslint" + # stage: unit test + # install: + # - ./travis-ci/fetch-cache.sh + # - npm install + # - npm run bootstrap-no-build + # script: ./node_modules/.bin/eslint packages/api packages/cmrjs packages/common - name: "Unit tests except API" stage: unit test services: @@ -41,76 +41,79 @@ jobs: - npm install - npm run bootstrap-no-build - ./travis-ci/start-local-services.sh - script: ./node_modules/.bin/nyc ./node_modules/.bin/lerna run test --ignore @cumulus/api - - name: "API unit tests" - stage: unit test - services: - - docker - env: - - LOCALSTACK_HOST=127.0.0.1 - - LOCAL_ES_HOST=127.0.0.1 - install: - - ./travis-ci/fetch-cache.sh - - npm install - - npm run bootstrap-no-build - - ./travis-ci/start-local-services.sh - script: ./travis-ci/run-api-unit-tests.sh - - name: "e2e tests" - stage: unit test - services: - - docker - env: - - LOCALSTACK_HOST=127.0.0.1 - - LOCAL_ES_HOST=127.0.0.1 - install: - - ./travis-ci/fetch-cache.sh - - npm install - - npm run bootstrap-no-build - - ./travis-ci/start-local-services.sh - script: npm run e2e + # script: ./node_modules/.bin/nyc ./node_modules/.bin/lerna run test --ignore @cumulus/api + script: + - cd tasks/parse-pdr + - ava tests/parse_pdrs_test.js + # - name: "API unit tests" + # stage: unit test + # services: + # - docker + # env: + # - LOCALSTACK_HOST=127.0.0.1 + # - LOCAL_ES_HOST=127.0.0.1 + # install: + # - ./travis-ci/fetch-cache.sh + # - npm install + # - npm run bootstrap-no-build + # - ./travis-ci/start-local-services.sh + # script: ./travis-ci/run-api-unit-tests.sh + # - name: "e2e tests" + # stage: unit test + # services: + # - docker + # env: + # - LOCALSTACK_HOST=127.0.0.1 + # - LOCAL_ES_HOST=127.0.0.1 + # install: + # - ./travis-ci/fetch-cache.sh + # - npm install + # - npm run bootstrap-no-build + # - ./travis-ci/start-local-services.sh + # script: npm run e2e - - name: "Deploy Integration Test Stack" - stage: deploy integration test stack - if: commit_message !~ \[skip-integration-tests\] OR branch = master - install: - - ./travis-ci/fetch-cache.sh - - npm install - - npm run bootstrap-no-build - script: ./travis-ci/travis_wait_new 50 ./travis-ci/init-integration-tests.sh - after_failure: ./travis-ci/travis_wait_new 50 ./travis-ci/cleanup-integration-tests.sh + # - name: "Deploy Integration Test Stack" + # stage: deploy integration test stack + # if: commit_message !~ \[skip-integration-tests\] OR branch = master + # install: + # - ./travis-ci/fetch-cache.sh + # - npm install + # - npm run bootstrap-no-build + # script: ./travis-ci/travis_wait_new 50 ./travis-ci/init-integration-tests.sh + # after_failure: ./travis-ci/travis_wait_new 50 ./travis-ci/cleanup-integration-tests.sh - - name: "Run integration tests" - stage: integration tests - if: commit_message !~ \[skip-integration-tests\] OR branch = master - install: - - ./travis-ci/fetch-cache.sh - - npm install - - npm run bootstrap-no-build - script: ./travis-ci/travis_wait_new 50 ./travis-ci/run-integration-tests.sh - after_failure: ./travis-ci/travis_wait_new 50 ./travis-ci/cleanup-integration-tests.sh + # - name: "Run integration tests" + # stage: integration tests + # if: commit_message !~ \[skip-integration-tests\] OR branch = master + # install: + # - ./travis-ci/fetch-cache.sh + # - npm install + # - npm run bootstrap-no-build + # script: ./travis-ci/travis_wait_new 50 ./travis-ci/run-integration-tests.sh + # after_failure: ./travis-ci/travis_wait_new 50 ./travis-ci/cleanup-integration-tests.sh - - name: "Cleanup integration tests" - stage: cleanup integration tests - if: commit_message !~ \[skip-integration-tests\] OR branch = master - script: ./travis-ci/travis_wait_new 50 ./travis-ci/cleanup-integration-tests.sh + # - name: "Cleanup integration tests" + # stage: cleanup integration tests + # if: commit_message !~ \[skip-integration-tests\] OR branch = master + # script: ./travis-ci/travis_wait_new 50 ./travis-ci/cleanup-integration-tests.sh - - name: "Deploy to NPM" - stage: deploy - install: - - ./travis-ci/fetch-cache.sh - - npm install - - npm run bootstrap-no-build - script: ./travis-ci/deploy-to-npm.sh - - name: "Deploy to Github Pages" - stage: deploy - install: npm install - script: ./travis-ci/deploy-to-github-pages.sh + # - name: "Deploy to NPM" + # stage: deploy + # install: + # - ./travis-ci/fetch-cache.sh + # - npm install + # - npm run bootstrap-no-build + # script: ./travis-ci/deploy-to-npm.sh + # - name: "Deploy to Github Pages" + # stage: deploy + # install: npm install + # script: ./travis-ci/deploy-to-github-pages.sh - - name: "Run integration tests against deployed packages" - stage: deployment test - if: commit_message !~ \[skip-integration-tests\] OR branch = master - install: true - script: ./travis-ci/run-integration-tests.sh - env: - - DEPLOYMENT=cumulus-from-npm - - USE_NPM_PACKAGES=true + # - name: "Run integration tests against deployed packages" + # stage: deployment test + # if: commit_message !~ \[skip-integration-tests\] OR branch = master + # install: true + # script: ./travis-ci/run-integration-tests.sh + # env: + # - DEPLOYMENT=cumulus-from-npm + # - USE_NPM_PACKAGES=true diff --git a/tasks/parse-pdr/tests/parse_pdrs_test.js b/tasks/parse-pdr/tests/parse_pdrs_test.js index cdf77e870d4..f5f156f32aa 100644 --- a/tasks/parse-pdr/tests/parse_pdrs_test.js +++ b/tasks/parse-pdr/tests/parse_pdrs_test.js @@ -403,22 +403,13 @@ test.serial('Parse a PDR with a granuleIdFilter in the config', async (t) => { await validateInput(t, t.context.payload.input); await validateConfig(t, t.context.payload.config); - let output; - try { - output = await parsePdr(t.context.payload); + const output = await parsePdr(t.context.payload); - await validateOutput(t, output); + await validateOutput(t, output); - t.deepEqual(output.pdr, t.context.payload.input.pdr); - t.is(output.granules.length, 1); - t.is(output.granulesCount, 1); - t.is(output.filesCount, 1); - t.is(output.totalSize, 1503297); - } - catch (err) { - if (err instanceof errors.RemoteResourceError || err.code === 'AllAccessDisabled') { - t.pass('ignoring this test. Test server seems to be down'); - } - else t.fail(err); - } + t.deepEqual(output.pdr, t.context.payload.input.pdr); + t.is(output.granules.length, 1); + t.is(output.granulesCount, 1); + t.is(output.filesCount, 1); + t.is(output.totalSize, 1503297); }); From da820b5a0a03fce978d916caa02120df82d4931f Mon Sep 17 00:00:00 2001 From: mhuffnagle Date: Tue, 23 Oct 2018 11:19:59 -0400 Subject: [PATCH 2/2] Use patched version of jsftp package --- packages/ingest/package.json | 2 +- tasks/parse-pdr/tests/parse_pdrs_test.js | 24 ++++++++---------------- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/packages/ingest/package.json b/packages/ingest/package.json index 02a9c8de62e..3e5520fb0fa 100644 --- a/packages/ingest/package.json +++ b/packages/ingest/package.json @@ -43,7 +43,7 @@ "encodeurl": "^1.0.2", "fs-extra": "^5.0.0", "got": "^9.2.1", - "jsftp": "^2.0.0", + "jsftp": "https://github.com/yjpa7145/jsftp.git#Fix-partial-file-issue", "json-loader": "~0.5.7", "lodash.clonedeep": "^4.5.0", "lodash.get": "^4.4.2", diff --git a/tasks/parse-pdr/tests/parse_pdrs_test.js b/tasks/parse-pdr/tests/parse_pdrs_test.js index f5f156f32aa..4ac0f0aa2e4 100644 --- a/tasks/parse-pdr/tests/parse_pdrs_test.js +++ b/tasks/parse-pdr/tests/parse_pdrs_test.js @@ -291,24 +291,16 @@ test.serial('Parse a PDR without a granuleIdFilter in the config', async (t) => await validateInput(t, t.context.payload.input); await validateConfig(t, t.context.payload.config); - let output; - try { - output = await parsePdr(t.context.payload); + const output = await parsePdr(t.context.payload); - await validateOutput(t, output); + await validateOutput(t, output); + + t.deepEqual(output.pdr, t.context.payload.input.pdr); + t.is(output.granules.length, 2); + t.is(output.granulesCount, 2); + t.is(output.filesCount, 2); + t.is(output.totalSize, 3952643); - t.deepEqual(output.pdr, t.context.payload.input.pdr); - t.is(output.granules.length, 2); - t.is(output.granulesCount, 2); - t.is(output.filesCount, 2); - t.is(output.totalSize, 3952643); - } - catch (err) { - if (err instanceof errors.RemoteResourceError || err.code === 'AllAccessDisabled') { - t.pass('ignoring this test. Test server seems to be down'); - } - else t.fail(err); - } }); test.serial('Empty FILE_ID valule in PDR, parse-pdr throws error', async (t) => {