Workflow v2

.circleci/config.yml

@@ -3,83 +3,155 @@ orbs:
   python: circleci/[email protected]
 
 commands:
-  build_and_test:
+  build_docker:
     parameters:
+      build_name:
+        type: string
       python_version:
         type: string
+      tag:
+        type: string
+      dockerfile_name:
+        type: string
     steps:
-      - checkout
       - run:
-          name: Build docker image
+          name: Build << parameters.build_name >> docker image
           command: |
-            docker build --build-arg OPHYS_ETL_TAG=${CIRCLE_BRANCH} --build-arg OPHYS_ETL_COMMIT_SHA=${CIRCLE_SHA1} --build-arg TEST_PYTHON_VERSION=<< parameters.python_version >> -t alleninstitutepika/ophys_etl_pipelines:${CIRCLE_SHA1} .circleci/docker
+            docker build \
+              --build-arg OPHYS_ETL_TAG=${CIRCLE_BRANCH} \
+              --build-arg OPHYS_ETL_COMMIT_SHA=${CIRCLE_SHA1} \
+              --build-arg PYTHON_VERSION=<< parameters.python_version >> \
+              -t alleninstitutepika/ophys_etl_pipelines:<< parameters.tag >> \
+              -f docker/<<parameters.dockerfile_name>> docker/
+  test:
+    parameters:
+      test_name:
+        type: string
+      python_version:
+        type: string
+      docker_tag:
+        type: string
+      pytest_mark:
+        type: string
+      conda_env:
+        type: string
+      test_directory:
+        type: string
+        default: tests
+      coverage_executable_path:
+        type: string
+    steps:
       - run:
-          name: Run docker image pytest
+          name: Run docker image << parameters.test_name >> tests
           # docker runs should succeed with --read-only flag if expected to be
           # run by singularity
           # https://sylabs.io/guides/3.6/user-guide/singularity_and_docker.html#best-practices
           command: |
-                set -e
-                mkdir /tmp/<< parameters.python_version >>
-                mkdir $PWD/coverage_outputs_<< parameters.python_version >>
-                # set entrypoint like this so we can handle quotes
-                docker run --entrypoint /bin/bash --read-only --tmpfs /tmp --mount type=bind,source=$PWD/coverage_outputs_<< parameters.python_version >>,target=/coverage_outputs_<< parameters.python_version >> alleninstitutepika/ophys_etl_pipelines:${CIRCLE_SHA1} /repos/ophys_etl/.circleci/scripts/run_and_test.sh "not event_detect_only" << parameters.python_version >> ophys_etl general_cov.xml
-                bash <(curl -s https://codecov.io/bash) -t ${CODECOV_TOKEN} -f $PWD/coverage_outputs_<< parameters.python_version >>/general_cov.xml -cF general_tests
+            set -e
+            mkdir -p $PWD/coverage_outputs_<< parameters.python_version >>
+
+            # set entrypoint like this so we can handle quotes
+            docker run \
+              --entrypoint /bin/bash \
+              --read-only \
+              --tmpfs /tmp \
+              -v $PWD/coverage_outputs_<<parameters.python_version>>:/coverage_outputs_<<parameters.python_version>> \
+              alleninstitutepika/ophys_etl_pipelines:<<parameters.docker_tag>> \
+              /repos/ophys_etl/.circleci/scripts/run_and_test.sh "<<parameters.pytest_mark>>" <<parameters.python_version>> <<parameters.conda_env>> <<parameters.test_name>>.xml <<parameters.test_directory>> <<parameters.coverage_executable_path>>
 
-jobs:
-  docker:
-    machine:
-      image: ubuntu-2004:current
+
+  build_and_test:
+    parameters:
+      python_version:
+        type: string
     steps:
       - checkout
-      - run:
-          name: Build docker image
-          command: |
-            docker build --build-arg OPHYS_ETL_TAG=${CIRCLE_BRANCH} --build-arg OPHYS_ETL_COMMIT_SHA=${CIRCLE_SHA1} -t alleninstitutepika/ophys_etl_pipelines:${CIRCLE_SHA1} docker/
-      - run:
-          name: Run docker image pytest
-          # docker runs should succeed with --read-only flag if expected to be
-          # run by singularity
-          # https://sylabs.io/guides/3.6/user-guide/singularity_and_docker.html#best-practices
-          command: |
-                  set -e
-                  mkdir $PWD/coverage_outputs_3.8
-                  # set entrypoint like this so we can handle quotes
-                  docker run --entrypoint /bin/bash --read-only --tmpfs /tmp -v $PWD/coverage_outputs_3.8:/coverage_outputs_3.8 alleninstitutepika/ophys_etl_pipelines:${CIRCLE_SHA1} /repos/ophys_etl/.circleci/scripts/run_and_test.sh "event_detect_only" 3.8 event_detection event_cov.xml
-                  bash <(curl -s https://codecov.io/bash) -t ${CODECOV_TOKEN} -f $PWD/coverage_outputs_3.8/event_cov.xml -cF event_detection_tests
-                  docker run --entrypoint /bin/bash --read-only --tmpfs /tmp -v $PWD/coverage_outputs_3.8:/coverage_outputs_3.8 alleninstitutepika/ophys_etl_pipelines:${CIRCLE_SHA1} /repos/ophys_etl/.circleci/scripts/run_and_test.sh "not event_detect_only" 3.8 ophys_etl general_cov.xml
-                  bash <(curl -s https://codecov.io/bash) -t ${CODECOV_TOKEN} -f $PWD/coverage_outputs_3.8/general_cov.xml -cF general_tests
+      - build_docker:
+          build_name: "General+suite2p"
+          python_version: <<parameters.python_version>>
+          tag: ${CIRCLE_SHA1}
+          dockerfile_name: "Dockerfile"
+      - build_docker:
+          build_name: "DeepInterpolation only"
+          python_version: <<parameters.python_version>>
+          tag: deepinterpolation_${CIRCLE_SHA1}
+          dockerfile_name: "Dockerfile_deepinterpolation"
+      - test:
+          test_name: "event_detection"
+          python_version: <<parameters.python_version>>
+          docker_tag: ${CIRCLE_SHA1}
+          pytest_mark: "event_detect_only"
+          conda_env: event_detection
+          test_directory: tests/modules/event_detection/
+          coverage_executable_path: "/envs/event_detection/bin/coverage"
+      - test:
+          test_name: "deepinterpolation"
+          python_version: <<parameters.python_version>>
+          docker_tag: deepinterpolation_${CIRCLE_SHA1}
+          pytest_mark: deepinterpolation_only
+          conda_env: ophys_etl
+          test_directory: tests/modules/denoising/
+          coverage_executable_path: "/usr/local/bin/coverage"
+      - test:
+          test_name: "general"
+          python_version: <<parameters.python_version>>
+          docker_tag: ${CIRCLE_SHA1}
+          pytest_mark: "not event_detect_only and not deepinterpolation_only"
+          conda_env: ophys_etl
+          coverage_executable_path: "/envs/ophys_etl/bin/coverage"
+  smoke_test_and_push:
+    steps:
       - run:
           name: Run docker image smoke test
           command: |
-                  set -e
-                  docker run --read-only --tmpfs /tmp alleninstitutepika/ophys_etl_pipelines:${CIRCLE_SHA1} /envs/ophys_etl/bin/python -m ophys_etl.modules.suite2p_wrapper --h5py /repos/ophys_etl/tests/modules/suite2p_wrapper/resources/movie_100x100x100.h5 --output_dir /tmp --movie_frame_rate 1.0 --log_level INFO --output_json /tmp/output.json
+            set -e
+            docker run  \
+            --read-only \
+            --tmpfs /tmp \
+            alleninstitutepika/ophys_etl_pipelines:${CIRCLE_SHA1} \
+            /envs/ophys_etl/bin/python -m ophys_etl.modules.suite2p_wrapper \
+              --h5py \
+              /repos/ophys_etl/tests/modules/suite2p_wrapper/resources/movie_100x100x100.h5 \
+              --output_dir /tmp \
+              --movie_frame_rate 1.0 \
+              --log_level INFO \
+              --output_json /tmp/output.json
       - run:
           name: Upload docker image
           command: |
             echo "$DOCKER_PASS" | docker login --username $DOCKER_USER --password-stdin
             if [ "$CIRCLE_BRANCH" = "main" ]
             then
-                docker tag alleninstitutepika/ophys_etl_pipelines:${CIRCLE_SHA1} alleninstitutepika/ophys_etl_pipelines:main
-                docker push alleninstitutepika/ophys_etl_pipelines:main
-            else
-                docker tag alleninstitutepika/ophys_etl_pipelines:${CIRCLE_SHA1} alleninstitutepika/ophys_etl_pipelines:develop
-                docker push alleninstitutepika/ophys_etl_pipelines:develop
-            fi
+                docker tag alleninstitutepika/ophys_etl_pipelines:${CIRCLE_SHA1} alleninstitutepika/ophys_etl_pipelines:main_${CIRCLE_SHA1}
+                docker push alleninstitutepika/ophys_etl_pipelines:main_${CIRCLE_SHA1}
 
-  build377:
-    machine:
-      image: ubuntu-2004:current
-    steps:
-      - build_and_test:
-          python_version: "3.7.7"
+                docker tag alleninstitutepika/ophys_etl_pipelines:deepinterpolation_${CIRCLE_SHA1} alleninstitutepika/ophys_etl_pipelines:deepinterpolation_main_${CIRCLE_SHA1}
+                docker push alleninstitutepika/ophys_etl_pipelines:deepinterpolation_main_${CIRCLE_SHA1}
+            else
+                docker tag alleninstitutepika/ophys_etl_pipelines:${CIRCLE_SHA1} alleninstitutepika/ophys_etl_pipelines:develop_${CIRCLE_SHA1}
+                docker push alleninstitutepika/ophys_etl_pipelines:develop_${CIRCLE_SHA1}
 
-  build38:
+                docker tag alleninstitutepika/ophys_etl_pipelines:deepinterpolation_${CIRCLE_SHA1} alleninstitutepika/ophys_etl_pipelines:deepinterpolation_develop_${CIRCLE_SHA1}
+                docker push alleninstitutepika/ophys_etl_pipelines:deepinterpolation_develop_${CIRCLE_SHA1}
+            fi
+
+jobs:
+  build_and_push:
     machine:
       image: ubuntu-2004:current
     steps:
       - build_and_test:
           python_version: "3.8"
+      - when:
+          condition:
+            or:
+              - equal: [ main, << pipeline.git.branch >> ]
+              - matches:
+                  pattern: "^/*.*dev.*/*$"
+                  value: << pipeline.git.branch >>
+          steps:
+              - smoke_test_and_push
+
 
   lint:
     executor: python/default
@@ -98,16 +170,7 @@ jobs:
 workflows:
   main:
     jobs:
-      - build377
-      - build38
       - lint
-      - docker:
+      - build_and_push:
           requires:
-            - build377
-            - build38
             - lint
-          filters:
-            branches:
-              only:
-                - main
-                - /.*dev.*/

.circleci/scripts/run_and_test.sh

@@ -1,30 +1,30 @@
 PYTEST_MARK=${1}
 PYTHON_VERSION=${2}
-ENV_SPECIFICATION=${3}  # which conda environment to run in
+ENV_SPECIFICATION=${3}  # a name to use for coverage output file
 OUTPUT_XML=${4}    # the name of the coverage xml file (not its full path)
+TEST_DIRECTORY=${5}
+COVERAGE_EXE_PATH=${6}  # path to coverage executable
 
 echo "mark: "${PYTEST_MARK}
 echo "version: "${PYTHON_VERSION}
 echo "env: "${ENV_SPECIFICATION}
 echo "xml: "${OUTPUT_XML}
+echo "test directory": ${TEST_DIRECTORY}
 
 set -e
 export COVERAGE_FILE=/tmp/.coverage_${ENV_SPECIFICATION}_${PYTHON_VERSION}
 echo "COVERAGE_FILE set to "${COVERAGE_FILE}
 cd /repos/ophys_etl/
-/envs/${ENV_SPECIFICATION}/bin/coverage run --rcfile .circleci/coveragerc_file --concurrency=multiprocessing -m \
-    pytest --verbose -s -m "${PYTEST_MARK}" \
-    tests
+${COVERAGE_EXE_PATH} run --rcfile .circleci/coveragerc_file --concurrency=multiprocessing -m \
+    pytest --verbose \
+    -s -m "${PYTEST_MARK}" \
+    ${TEST_DIRECTORY}
 
-/envs/${ENV_SPECIFICATION}/bin/coverage combine --data-file=${COVERAGE_FILE} --rcfile .circleci/coveragerc_file
+${COVERAGE_EXE_PATH} combine --data-file=${COVERAGE_FILE} --rcfile .circleci/coveragerc_file
 
 # coverage automatically adds random salt to the end of the
 # coverage file name; need to detect the name of the file
 # that was actually produced
 coverage_file_list=($(ls /tmp/.coverage_${ENV_SPECIFICATION}_${PYTHON_VERSION}*))
 echo "after combining "
-echo ${coverage_file_list[@]}
-
-/envs/${ENV_SPECIFICATION}/bin/coverage xml --data-file=${COVERAGE_FILE} \
--o /coverage_outputs_${PYTHON_VERSION}/${OUTPUT_XML} \
---rcfile .circleci/coveragerc_file
+echo ${coverage_file_list[@]}

.flake8

@@ -1,4 +1,4 @@
 [flake8]
-extend-ignore = E203
+extend-ignore = E203, E402
 per-file-ignores =
     src/ophys_etl/modules/event_detection/resources/event_decay_time_lookup.py:E501