QE-16040 cleanup CI and processes (#496)

- chore - refactor circle config (config-drive executor, embed commands, move workflows up) - chore - configure renovate - chore - remove dependabot config (duplicates renovate) - chore - remove safety (duplicates sonarqube)
dominodatalab · Jun 13, 2024 · 2e54a7d · 2e54a7d
1 parent cace240
commit 2e54a7d
Show file tree

Hide file tree

Showing 8 changed files with 124 additions and 590 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -3,217 +3,164 @@
 ---
 version: 2.1
 
-commands:
-  install_cucu_dependencies: &install_cucu_dependencies
+parameters:
+  python_version:
+    type: string
+    default: '3.11.7'
+  selenium_version:
+    type: string
+    default: '124.0'
+  se_node_max_sessions:
+    type: integer
+    default: 12
+  se_node_session_timeout:
+    type: integer
+    default: 300
+  screen_width:
+    type: integer
+    default: 1366
+  screen_height:
+    type: integer
+    default: 768
+
+workflows:
+  build-test-publish:
+    jobs:
+      - pre-commit
+      - test:
+          name: test-chrome
+          browser: chrome
+          requires:
+            - pre-commit
+      - test:
+          name: test-edge
+          browser: edge
+          requires:
+            - test-chrome
+      - test:
+          name: test-firefox
+          browser: firefox
+          requires:
+            - test-chrome
+      - publish:
+          filters:
+            branches:
+              ignore: /.*/
+            tags:
+              only: /^\d+\.\d+\.\d+/
+
+jobs:
+  pre-commit:
+    docker:
+      - image: cimg/python:<<pipeline.parameters.python_version>>
+    resource_class: medium
     steps:
+      - checkout
       - run:
-          name: install cucu dependencies
+          name: install dependencies
           command: |
             set +eo pipefail
-            pip install poetry
-            python -m virtualenv --upgrade-embed-wheels
-            poetry install --no-ansi
-
-  install_test_dependencies: &install_test_dependencies
-    steps:
+            pip install --upgrade pip
+            pip install pre-commit
+            pre-commit install-hooks
       - run:
-          name: install test dependencies
+          name: install cucu dependencies
           command: |
             set +eo pipefail
-            apt-get update
-            apt-get install -y expect
-
-  build_cucu_python_package: &build_cucu_python_package
-    steps:
+            python -m virtualenv --upgrade-embed-wheels
+            poetry install --no-ansi
       - run:
           name: build the cucu python package
           command: poetry build
-
-  install_cucu_globally: &install_cucu_globally
-    steps:
       - run:
           name: install cucu globally
           command: pip install dist/cucu-*.tar.gz
-
-  setup_remote_docker_env: &setup_remote_docker_env
-    steps:
-      - setup_remote_docker
-
-  wait_for_selenium: &wait_for_selenium
-    steps:
       - run:
-          # retry connection every 5s x 60 which is a total of 5 minutes
-          command: curl --retry 60 --retry-delay 5 --retry-connrefused http://localhost:4444
+          name: run pre-commit validations
+          command: pre-commit run --show-diff-on-failure --from-ref origin/HEAD --to-ref HEAD
 
-  run_functional_tests: &run_functional_tests
+  test:
     parameters:
       browser:
         type: string
         default: "chrome"
+    docker:
+      - image: cimg/python:<<pipeline.parameters.python_version>>
+      - image: selenium/standalone-<<parameters.browser>>:<<pipeline.parameters.selenium_version>>
+        environment:
+          SE_NODE_MAX_SESSIONS: <<pipeline.parameters.se_node_max_sessions>>
+          SE_NODE_SESSION_TIMEOUT: <<pipeline.parameters.se_node_session_timeout>>
+          SCREEN_WIDTH: <<pipeline.parameters.screen_width>>
+          SCREEN_HEIGHT: <<pipeline.parameters.screen_height>>
+    resource_class: xlarge
     steps:
+      - setup_remote_docker
+      - checkout
+      - run:
+          name: install cucu dependencies
+          command: |
+            set +eo pipefail
+            python -m virtualenv --upgrade-embed-wheels
+            poetry install --no-ansi
+      - run:
+          name: install test dependencies
+          command: |
+            set +eo pipefail
+            sudo apt-get update
+            sudo apt-get install -y expect
       - run:
+          name: wait_for_selenium
+          # retry connection every 5s x 60 which is a total of 5 minutes
+          command: curl --retry 60 --retry-delay 5 --retry-connrefused http://localhost:4444
+      - run:
+          name: run_functional_tests
           command: poetry run cucu run features --workers 6 --selenium-remote-url http://localhost:4444 --generate-report --junit junit_results --browser "<< parameters.browser >>"
           environment:
             COVERAGE_PROCESS_START: pyproject.toml  # set to config file
-
-  run_unit_tests: &run_unit_tests
-    steps:
       - run:
           name: run unit tests
           command: poetry run coverage run -m pytest --junit-xml=results/unit-tests.xml
-
-  code_coverage_check: &code_coverage_check
-    steps:
       - run:
+          name: code_coverage_check
           command: |
             set +eo pipefail
             poetry run coverage combine .coverage.*
             poetry run coverage html
             poetry run coverage report --fail-under=70
-
-executors:
-  unit-test-executor:
-    docker:
-      - image: python:3.11.7
-
-  chrome-test-executor:
-    docker:
-      - image: python:3.11.7
-      - image: selenium/standalone-chrome:124.0
-        environment:
-          CUCU_MONITOR_PNG: ""
-          SE_NODE_MAX_SESSIONS: 12
-          SE_NODE_SESSION_TIMEOUT: 300
-          SCREEN_WIDTH: 1366
-          SCREEN_HEIGHT: 768
-    resource_class: xlarge
-
-  firefox-test-executor:
-    docker:
-      - image: python:3.11.7
-      - image: selenium/standalone-firefox:125.0
-        environment:
-          CUCU_MONITOR_PNG: ""
-          SE_NODE_MAX_SESSIONS: 12
-          SE_NODE_SESSION_TIMEOUT: 300
-          SCREEN_WIDTH: 1366
-          SCREEN_HEIGHT: 768
-    resource_class: xlarge
-
-  edge-test-executor:
-    docker:
-      - image: python:3.11.7
-      - image: selenium/standalone-edge:124.0
-        environment:
-          CUCU_MONITOR_PNG: ""
-          SE_NODE_MAX_SESSIONS: 12
-          SE_NODE_SESSION_TIMEOUT: 300
-          SCREEN_WIDTH: 1366
-          SCREEN_HEIGHT: 768
-    resource_class: xlarge
-
-jobs:
-  pre-commit:
-    docker:
-      - image: cimg/python:3.12.3
-    resource_class: medium
-
-    steps:
-      - checkout
-      - run:
-          name: install dependencies
-          command: |
-            set +eo pipefail
-            pip install --upgrade pip
-            pip install pre-commit
-            pre-commit install-hooks
-
-      - install_cucu_dependencies
-      - build_cucu_python_package
-      - install_cucu_globally
-
-      - run:
-          name: run pre-commit validations
-          command: pre-commit run --show-diff-on-failure --from-ref origin/HEAD --to-ref HEAD
-
-  test:
-    parameters:
-      browser:
-        type: string
-        default: "chrome"
-
-    executor: << parameters.browser >>-test-executor
-
-    steps:
-      - checkout
-      - install_cucu_dependencies
-      - install_test_dependencies
-      - build_cucu_python_package
-      - install_cucu_globally
-      - setup_remote_docker_env
-      - wait_for_selenium
-      - run_functional_tests:
-          browser: "<< parameters.browser >>"
-      - run_unit_tests
-      - code_coverage_check
-
       - run:
           name: tar up results and reporting
           command: |
             tar cvfz results.tgz results
             tar cvfz report.tgz report
             tar cvfz junit_results.tgz junit_results
           when: always
-
       - store_artifacts:
           path: results.tgz
-
       - store_artifacts:
           path: junit_results.tgz
-
       - store_artifacts:
           path: report.tgz
-
       - store_test_results:
           path: junit_results
 
   publish:
     docker:
-      - image: cimg/python:3.12.3
+      - image: cimg/python:<<pipeline.parameters.python_version>>
     steps:
       - checkout
-      - install_cucu_dependencies
-      - build_cucu_python_package
+      - run:
+          name: install cucu dependencies
+          command: |
+            set +eo pipefail
+            python -m virtualenv --upgrade-embed-wheels
+            poetry install --no-ansi
+      - run:
+          name: build the cucu python package
+          command: poetry build
       - run:
           name: Publish to private PyPI repository
           command: |
             set +eo pipefail
             poetry config repositories.private ${PUBLISH_REPOSITORY}
             poetry config http-basic.private ${PUBLISH_USERNAME} ${PUBLISH_PASSWORD}
             poetry publish -r private
-
-workflows:
-  build-test-publish:
-    jobs:
-      - pre-commit
-      - test:
-          name: test-chrome
-          browser: chrome
-          requires:
-            - pre-commit
-      - test:
-          name: test-edge
-          browser: edge
-          requires:
-            - test-chrome
-      - test:
-          name: test-firefox
-          browser: firefox
-          requires:
-            - test-chrome
-      - publish:
-          filters:
-            branches:
-              ignore: /.*/
-            tags:
-              only: /^\d+\.\d+\.\d+/
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
diff --git a/.secrets.baseline b/.secrets.baseline
@@ -114,15 +114,15 @@
         "filename": "Makefile",
         "hashed_secret": "0ef3e7e357d64c01f01b9dc045413c135ce64ff7",
         "is_verified": false,
-        "line_number": 38,
+        "line_number": 34,
         "is_secret": false
       },
       {
         "type": "Secret Keyword",
         "filename": "Makefile",
         "hashed_secret": "dbbabcbd340d134fef3e212d83a00846839903f6",
         "is_verified": false,
-        "line_number": 39,
+        "line_number": 35,
         "is_secret": false
       }
     ],
@@ -219,5 +219,5 @@
       }
     ]
   },
-  "generated_at": "2024-05-15T21:14:22Z"
+  "generated_at": "2024-06-13T18:01:40Z"
 }
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project closely adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 0.198.0
+- chore - refactor circle config (config-drive executor, embed commands, move workflows up)
+- chore - configure renovate
+- chore - remove dependabot config (duplicates renovate)
+- chore - remove safety (duplicates sonarqube)
+
 ## 0.197.0
 - Fix - before all hooks are now executed
 

diff --git a/Makefile b/Makefile
@@ -24,10 +24,6 @@ check:
 	poetry run cucu lint features
 	# lint code for security mistakes
 	poetry run bandit src features data -r -c pyproject.toml -q --severity high
-	# check deps against known security issues
-	# 67599 - issue in `pip`, which is disputed as a CVE
-	# 63687 - issue only affects Windows
-	poetry run safety check --ignore=67599,63687
 	# check project config
 	poetry check
 	# prevent new secrets