chore(build): Provide multiple build jobs. Closes #150

build/ci-build.yml

@@ -8,6 +8,7 @@ trigger:
     include:
     - arcus/*
     - docs/*
+    - requirements.txt
 
 pr:
   paths:
@@ -16,6 +17,7 @@ pr:
     - docs/*
     - tests/*
     - build/ci-build.yml
+    - requirements.txt
 
 resources:
   repositories:
@@ -27,73 +29,86 @@ resources:
 pool:
   vmImage: 'ubuntu-latest'
 
-steps:
-- template: 'pypi/determine-pr-version.yml@templates'
-  parameters:
-    manualTriggerVersion: preview
-
-- task: UsePythonVersion@0
-  inputs:
-    versionSpec: '3.7'
-    addToPath: true
-    architecture: 'x64'
-
-- task: Bash@3
-  displayName: 'Update version number'
-  inputs:
-    targetType: 'inline'
-    script: 'sed -i ''s/1.0.0/$(Package.Version).$(Build.BuildNumber)/g'' arcus/azureml/__init__.py'
-    failOnStderr: true
-
-- script: python -m pip install --upgrade pip setuptools wheel
-  displayName: 'Install tools'
-
-- script: pip install -r requirements.txt
-  displayName: 'Install requirements'
-
-- script: |
-    pip install pytest
-    pip install pytest-cov
-    pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=. --cov-report=xml --cov-report=html
-  displayName: 'Unit tests (pytest)'
-
-- task: PublishTestResults@2
-  condition: succeededOrFailed()
-  inputs:
-    testResultsFiles: '**/test-*.xml'
-    testRunTitle: 'Publish test results for Python $(python.version)'
-
-- task: PublishCodeCoverageResults@1
-  inputs:
-    codeCoverageTool: Cobertura
-    summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
-    reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov'
-
-- script: |    
-    python setup.py sdist bdist_wheel 
-  displayName: 'Build wheels'
-
-- script: python -m pip install --upgrade twine
-  displayName: 'Install Twine'
-
-- task: TwineAuthenticate@1
-  inputs:
-    pythonUploadServiceConnection: 'Arcus AzureML PyPi feed'
-
-- script: |    
-    cat $(PYPIRC_PATH)
-  displayName: 'Print pypirc file'
-
-- script: |
-   python -m twine upload --skip-existing --verbose -r 'arcus-azureml' --config-file $(PYPIRC_PATH) dist/*
-  displayName: 'Publish to PyPi dev'
-
-- task: CopyFiles@2  
-  inputs:    
-    targetFolder: $(Build.ArtifactStagingDirectory)
-
-- task: PublishBuildArtifacts@1  
-  inputs:    
-    PathtoPublish: '$(Build.ArtifactStagingDirectory)'                  
-    ArtifactName: 'dist'    
-    publishLocation: 'Container'
+
+stages:
+  - stage: BuildAndTest
+    jobs: 
+      - job: DefineVersion
+        displayName: 'Increase and define version number'
+        steps:
+          - template: 'pypi/determine-pr-version.yml@templates'
+            parameters:
+              manualTriggerVersion: preview
+
+          - task: UsePythonVersion@0
+            inputs:
+              versionSpec: '3.7'
+              addToPath: true
+              architecture: 'x64'
+
+          - task: Bash@3
+            displayName: 'Update version number'
+            inputs:
+              targetType: 'inline'
+              script: 'sed -i ''s/1.0.0/$(Package.Version).$(Build.BuildNumber)/g'' arcus/azureml/__init__.py'
+              failOnStderr: true
+
+      - job: UnitTests
+        displayName: 'Run unit tests'
+        steps:
+          - script: pip install -r requirements.txt
+            displayName: 'Install requirements.txt'
+
+          - script: |
+              pip install pytest
+              pip install pytest-cov
+              pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=. --cov-report=xml --cov-report=html
+            displayName: 'Unit tests (pytest)'
+
+          - task: PublishTestResults@2
+            condition: succeededOrFailed()
+            inputs:
+              testResultsFiles: '**/test-*.xml'
+              testRunTitle: 'Publish test results for Python $(python.version)'
+
+          - task: PublishCodeCoverageResults@1
+            inputs:
+              codeCoverageTool: Cobertura
+              summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
+              reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov'
+
+  - stage: PackageAndRelease
+    displayName: 'Package and release to PyPi'
+    dependsOn: BuildAndTest
+    condition: succeeded()
+    jobs:
+      - job: PackageAndRelease
+        displayName: 'Package and release'
+        steps:
+          - script: python -m pip install --upgrade pip setuptools wheel lazydocs
+            displayName: 'Install tools'
+
+          - script: python -m pip install --upgrade twine
+            displayName: 'Install Twine'
+
+          - script: |    
+              python setup.py sdist bdist_wheel 
+            displayName: 'Build wheels'
+
+          - task: TwineAuthenticate@1
+            inputs:
+              pythonUploadServiceConnection: 'Arcus AzureML PyPi feed'
+
+          - script: |
+              python -m twine upload --skip-existing --verbose -r 'arcus-azureml' --config-file $(PYPIRC_PATH) dist/*
+            displayName: 'Publish to PyPi dev'
+
+          - task: CopyFiles@2  
+            inputs:    
+              targetFolder: $(Build.ArtifactStagingDirectory)
+
+          - task: PublishBuildArtifacts@1  
+            inputs:    
+              PathtoPublish: '$(Build.ArtifactStagingDirectory)'                  
+              ArtifactName: 'dist'    
+              publishLocation: 'Container'

requirements.txt

@@ -10,9 +10,9 @@ scikit-learn
 scikit-image
 matplotlib
 tensorflow>=2.1.0
-keras
+keras>=2.6.0rc3 # not directly required, pinned by Snyk to avoid a vulnerability
 arcus-ml>=1.0.11
-opencv-python
+opencv-python>=4.4.0.40
 pillow>=8.1.0 # not directly required, pinned by Snyk to avoid a vulnerabilitypillow>=8.1.0 # not directly required, pinned by Snyk to avoid a vulnerability
 kaggle
-
+networkx>=2.6 # not directly required, pinned by Snyk to avoid a vulnerability

samples/Arcus-Kaggle-Download/requirements.txt

@@ -18,4 +18,5 @@ opencv-python
 seaborn
 scikit-image
 opencv-python
-kaggle
+kaggle
+pillow>=8.3.2 # not directly required, pinned by Snyk to avoid a vulnerability