From b6dff40257f0de79449b3011a5d8af95c6bf2fcf Mon Sep 17 00:00:00 2001 From: samvanhoutte Date: Thu, 9 Dec 2021 11:12:29 +0100 Subject: [PATCH 01/12] Bump dependencies to avoid vulnerabilities --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9e4ec3b..50cb7fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,9 +10,9 @@ scikit-learn scikit-image matplotlib tensorflow>=2.1.0 -keras +keras>=2.6.0rc3 # not directly required, pinned by Snyk to avoid a vulnerability arcus-ml>=1.0.11 -opencv-python +opencv-python==4.2.0.32.9 pillow>=8.1.0 # not directly required, pinned by Snyk to avoid a vulnerabilitypillow>=8.1.0 # not directly required, pinned by Snyk to avoid a vulnerability kaggle - +networkx>=2.6 # not directly required, pinned by Snyk to avoid a vulnerability From db8806966658acd0a0c28e0b07ae1ddf46566141 Mon Sep 17 00:00:00 2001 From: samvanhoutte Date: Thu, 9 Dec 2021 11:18:20 +0100 Subject: [PATCH 02/12] trigger ci build on requirements.txt change --- build/ci-build.yml | 2 ++ samples/Arcus-Kaggle-Download/requirements.txt | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/build/ci-build.yml b/build/ci-build.yml index 09383b3..60de41a 100644 --- a/build/ci-build.yml +++ b/build/ci-build.yml @@ -8,6 +8,7 @@ trigger: include: - arcus/* - docs/* + - requirements.txt pr: paths: @@ -16,6 +17,7 @@ pr: - docs/* - tests/* - build/ci-build.yml + - requirements.txt resources: repositories: diff --git a/samples/Arcus-Kaggle-Download/requirements.txt b/samples/Arcus-Kaggle-Download/requirements.txt index db13a12..fd7f997 100644 --- a/samples/Arcus-Kaggle-Download/requirements.txt +++ b/samples/Arcus-Kaggle-Download/requirements.txt @@ -18,4 +18,5 @@ opencv-python seaborn scikit-image opencv-python -kaggle \ No newline at end of file +kaggle +pillow>=8.3.2 # not directly required, pinned by Snyk to avoid a vulnerability \ No newline at end of file From f9be065660503dbc8c3a3d29a9dcb5ecdedb20ed Mon Sep 17 00:00:00 2001 From: samvanhoutte Date: Thu, 9 Dec 2021 11:37:08 +0100 Subject: [PATCH 03/12] Fix: opencv version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 50cb7fd..d99a96d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ matplotlib tensorflow>=2.1.0 keras>=2.6.0rc3 # not directly required, pinned by Snyk to avoid a vulnerability arcus-ml>=1.0.11 -opencv-python==4.2.0.32.9 +opencv-python>=4.4.0.40 pillow>=8.1.0 # not directly required, pinned by Snyk to avoid a vulnerabilitypillow>=8.1.0 # not directly required, pinned by Snyk to avoid a vulnerability kaggle networkx>=2.6 # not directly required, pinned by Snyk to avoid a vulnerability From 8f69a62f75ae53ad3fffd5b2980baa002773d07c Mon Sep 17 00:00:00 2001 From: samvanhoutte Date: Thu, 9 Dec 2021 11:38:11 +0100 Subject: [PATCH 04/12] Add multi stages to ci-build job --- build/ci-build.yml | 169 ++++++++++++++++++++++++++------------------- 1 file changed, 99 insertions(+), 70 deletions(-) diff --git a/build/ci-build.yml b/build/ci-build.yml index 60de41a..7d79ce7 100644 --- a/build/ci-build.yml +++ b/build/ci-build.yml @@ -29,73 +29,102 @@ resources: pool: vmImage: 'ubuntu-latest' -steps: -- template: 'pypi/determine-pr-version.yml@templates' - parameters: - manualTriggerVersion: preview - -- task: UsePythonVersion@0 - inputs: - versionSpec: '3.7' - addToPath: true - architecture: 'x64' - -- task: Bash@3 - displayName: 'Update version number' - inputs: - targetType: 'inline' - script: 'sed -i ''s/1.0.0/$(Package.Version).$(Build.BuildNumber)/g'' arcus/azureml/__init__.py' - failOnStderr: true - -- script: python -m pip install --upgrade pip setuptools wheel - displayName: 'Install tools' - -- script: pip install -r requirements.txt - displayName: 'Install requirements' - -- script: | - pip install pytest - pip install pytest-cov - pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=. --cov-report=xml --cov-report=html - displayName: 'Unit tests (pytest)' - -- task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-*.xml' - testRunTitle: 'Publish test results for Python $(python.version)' - -- task: PublishCodeCoverageResults@1 - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml' - reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov' - -- script: | - python setup.py sdist bdist_wheel - displayName: 'Build wheels' - -- script: python -m pip install --upgrade twine - displayName: 'Install Twine' - -- task: TwineAuthenticate@1 - inputs: - pythonUploadServiceConnection: 'Arcus AzureML PyPi feed' - -- script: | - cat $(PYPIRC_PATH) - displayName: 'Print pypirc file' - -- script: | - python -m twine upload --skip-existing --verbose -r 'arcus-azureml' --config-file $(PYPIRC_PATH) dist/* - displayName: 'Publish to PyPi dev' - -- task: CopyFiles@2 - inputs: - targetFolder: $(Build.ArtifactStagingDirectory) - -- task: PublishBuildArtifacts@1 - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)' - ArtifactName: 'dist' - publishLocation: 'Container' + +stages: + - stage: Prerequisites + jobs: + - job: DefineVersion + displayName: 'Increase and define version number' + pool: + vmImage: '(Vm.Image)' + steps: + - template: 'pypi/determine-pr-version.yml@templates' + parameters: + manualTriggerVersion: preview + + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + addToPath: true + architecture: 'x64' + + - task: Bash@3 + displayName: 'Update version number' + inputs: + targetType: 'inline' + script: 'sed -i ''s/1.0.0/$(Package.Version).$(Build.BuildNumber)/g'' arcus/azureml/__init__.py' + failOnStderr: true + + - job: Dependencies + displayName: 'Install dependencies' + pool: + vmImage: '(Vm.Image)' + steps: + - script: python -m pip install --upgrade pip setuptools wheel lazydocs + displayName: 'Install tools' + + - script: pip install -r requirements.txt + displayName: 'Install requirements.txt' + + - script: python -m pip install --upgrade twine + displayName: 'Install Twine' + + - stage: UnitTests + displayName: Unit Tests + dependsOn: Prerequisites + condition: succeeded() + jobs: + - job: UnitTests + displayName: 'Run unit tests' + pool: + vmImage: '(Vm.Image)' + steps: + - script: | + pip install pytest + pip install pytest-cov + pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=. --cov-report=xml --cov-report=html + displayName: 'Unit tests (pytest)' + + - task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + testResultsFiles: '**/test-*.xml' + testRunTitle: 'Publish test results for Python $(python.version)' + + - task: PublishCodeCoverageResults@1 + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml' + reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov' + + - stage: PackageAndRelease + displayName: 'Package and release to PyPi' + dependsOn: Build + condition: succeeded() + jobs: + - job: PackageAndRelease + displayName: 'Package and release' + pool: + vmImage: '(Vm.Image)' + steps: + - script: | + python setup.py sdist bdist_wheel + displayName: 'Build wheels' + + - task: TwineAuthenticate@1 + inputs: + pythonUploadServiceConnection: 'Arcus AzureML PyPi feed' + + - script: | + python -m twine upload --skip-existing --verbose -r 'arcus-azureml' --config-file $(PYPIRC_PATH) dist/* + displayName: 'Publish to PyPi dev' + + - task: CopyFiles@2 + inputs: + targetFolder: $(Build.ArtifactStagingDirectory) + + - task: PublishBuildArtifacts@1 + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)' + ArtifactName: 'dist' + publishLocation: 'Container' From 9a9add117f644b44c99dfc59e5f91a474c31b289 Mon Sep 17 00:00:00 2001 From: samvanhoutte Date: Thu, 9 Dec 2021 11:40:46 +0100 Subject: [PATCH 05/12] Fix stage dependency --- build/ci-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/ci-build.yml b/build/ci-build.yml index 7d79ce7..6e10313 100644 --- a/build/ci-build.yml +++ b/build/ci-build.yml @@ -99,7 +99,7 @@ stages: - stage: PackageAndRelease displayName: 'Package and release to PyPi' - dependsOn: Build + dependsOn: UnitTests condition: succeeded() jobs: - job: PackageAndRelease From 907c5e84f9a6a7651c752299a9c22138dd5a917b Mon Sep 17 00:00:00 2001 From: samvanhoutte Date: Thu, 9 Dec 2021 11:44:53 +0100 Subject: [PATCH 06/12] Use one VM pool for all jobs --- build/ci-build.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/build/ci-build.yml b/build/ci-build.yml index 6e10313..35e6e40 100644 --- a/build/ci-build.yml +++ b/build/ci-build.yml @@ -35,8 +35,6 @@ stages: jobs: - job: DefineVersion displayName: 'Increase and define version number' - pool: - vmImage: '(Vm.Image)' steps: - template: 'pypi/determine-pr-version.yml@templates' parameters: @@ -57,8 +55,6 @@ stages: - job: Dependencies displayName: 'Install dependencies' - pool: - vmImage: '(Vm.Image)' steps: - script: python -m pip install --upgrade pip setuptools wheel lazydocs displayName: 'Install tools' @@ -76,8 +72,6 @@ stages: jobs: - job: UnitTests displayName: 'Run unit tests' - pool: - vmImage: '(Vm.Image)' steps: - script: | pip install pytest @@ -104,8 +98,6 @@ stages: jobs: - job: PackageAndRelease displayName: 'Package and release' - pool: - vmImage: '(Vm.Image)' steps: - script: | python setup.py sdist bdist_wheel From 4d0cd153055272969798fae172f95308ba488bc0 Mon Sep 17 00:00:00 2001 From: samvanhoutte Date: Thu, 9 Dec 2021 12:18:49 +0100 Subject: [PATCH 07/12] merge stage of test & install --- build/ci-build.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/build/ci-build.yml b/build/ci-build.yml index 35e6e40..f7c635d 100644 --- a/build/ci-build.yml +++ b/build/ci-build.yml @@ -31,7 +31,7 @@ pool: stages: - - stage: Prerequisites + - stage: BuildAndTest jobs: - job: DefineVersion displayName: 'Increase and define version number' @@ -65,11 +65,6 @@ stages: - script: python -m pip install --upgrade twine displayName: 'Install Twine' - - stage: UnitTests - displayName: Unit Tests - dependsOn: Prerequisites - condition: succeeded() - jobs: - job: UnitTests displayName: 'Run unit tests' steps: @@ -99,6 +94,9 @@ stages: - job: PackageAndRelease displayName: 'Package and release' steps: + - script: python -m pip install --upgrade pip setuptools wheel lazydocs + displayName: 'Install tools' + - script: | python setup.py sdist bdist_wheel displayName: 'Build wheels' From 85ac891fae7a8076ac49530f7262e5c288854f00 Mon Sep 17 00:00:00 2001 From: samvanhoutte Date: Thu, 9 Dec 2021 12:47:00 +0100 Subject: [PATCH 08/12] Fix stage dependency --- build/ci-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/ci-build.yml b/build/ci-build.yml index f7c635d..2207674 100644 --- a/build/ci-build.yml +++ b/build/ci-build.yml @@ -88,7 +88,7 @@ stages: - stage: PackageAndRelease displayName: 'Package and release to PyPi' - dependsOn: UnitTests + dependsOn: BuildAndTest condition: succeeded() jobs: - job: PackageAndRelease From 78ec518a03b856f38f8c5f264ef1a424795345c1 Mon Sep 17 00:00:00 2001 From: samvanhoutte Date: Thu, 9 Dec 2021 12:50:04 +0100 Subject: [PATCH 09/12] make tests dependent on requirements --- build/ci-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/build/ci-build.yml b/build/ci-build.yml index 2207674..92bb4c7 100644 --- a/build/ci-build.yml +++ b/build/ci-build.yml @@ -67,6 +67,7 @@ stages: - job: UnitTests displayName: 'Run unit tests' + dependsOn: Dependencies steps: - script: | pip install pytest From 569f4b0f26c809e38e87f2914995885fa58557d7 Mon Sep 17 00:00:00 2001 From: samvanhoutte Date: Thu, 9 Dec 2021 12:54:12 +0100 Subject: [PATCH 10/12] move requirements to test phase --- build/ci-build.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/build/ci-build.yml b/build/ci-build.yml index 92bb4c7..74c2e75 100644 --- a/build/ci-build.yml +++ b/build/ci-build.yml @@ -59,9 +59,6 @@ stages: - script: python -m pip install --upgrade pip setuptools wheel lazydocs displayName: 'Install tools' - - script: pip install -r requirements.txt - displayName: 'Install requirements.txt' - - script: python -m pip install --upgrade twine displayName: 'Install Twine' @@ -69,6 +66,10 @@ stages: displayName: 'Run unit tests' dependsOn: Dependencies steps: + - script: pip install -r requirements.txt + displayName: 'Install requirements.txt' + + - script: | pip install pytest pip install pytest-cov From 2757f611a759becde43d1998955993778b99ac92 Mon Sep 17 00:00:00 2001 From: samvanhoutte Date: Thu, 9 Dec 2021 13:00:02 +0100 Subject: [PATCH 11/12] move twine dependencies to package stage --- build/ci-build.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/build/ci-build.yml b/build/ci-build.yml index 74c2e75..3a5c4e8 100644 --- a/build/ci-build.yml +++ b/build/ci-build.yml @@ -53,23 +53,12 @@ stages: script: 'sed -i ''s/1.0.0/$(Package.Version).$(Build.BuildNumber)/g'' arcus/azureml/__init__.py' failOnStderr: true - - job: Dependencies - displayName: 'Install dependencies' - steps: - - script: python -m pip install --upgrade pip setuptools wheel lazydocs - displayName: 'Install tools' - - - script: python -m pip install --upgrade twine - displayName: 'Install Twine' - - job: UnitTests displayName: 'Run unit tests' - dependsOn: Dependencies steps: - script: pip install -r requirements.txt displayName: 'Install requirements.txt' - - script: | pip install pytest pip install pytest-cov @@ -99,6 +88,9 @@ stages: - script: python -m pip install --upgrade pip setuptools wheel lazydocs displayName: 'Install tools' + - script: python -m pip install --upgrade twine + displayName: 'Install Twine' + - script: | python setup.py sdist bdist_wheel displayName: 'Build wheels' From 4261488c34151ca88f6b2946dc62cfec4905e516 Mon Sep 17 00:00:00 2001 From: samvanhoutte Date: Fri, 17 Dec 2021 12:10:40 +0100 Subject: [PATCH 12/12] Add jobs to release build --- build/cd-build.yml | 101 +++++++++++++++++++++++++-------------------- 1 file changed, 56 insertions(+), 45 deletions(-) diff --git a/build/cd-build.yml b/build/cd-build.yml index f13733e..628b62a 100644 --- a/build/cd-build.yml +++ b/build/cd-build.yml @@ -17,58 +17,69 @@ resources: name: arcus-azure/azure-devops-templates endpoint: arcus-azure -steps: -- task: UsePythonVersion@0 - inputs: - versionSpec: '3.7' - addToPath: true - architecture: 'x64' +stages: + - stage: Prepare + jobs: + - job: InstallPrerequisites + displayName: 'Install pre requisites & tools' + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + addToPath: true + architecture: 'x64' -- script: python -m pip install --upgrade pip setuptools wheel - displayName: 'Install tools' + - script: python -m pip install --upgrade pip setuptools wheel + displayName: 'Install tools' -- script: pip install -r requirements.txt - displayName: 'Install requirements' + - script: pip install -r requirements.txt + displayName: 'Install requirements' -- task: Bash@3 - displayName: 'Update version number' - inputs: - targetType: 'inline' - script: 'sed -i ''s/1.0.0/$(Package.Version)/g'' arcus/azureml/__init__.py' - failOnStderr: true + - job: DefineVersion + displayName: 'Increase and define version number' + steps: + - task: Bash@3 + displayName: 'Update version number' + inputs: + targetType: 'inline' + script: 'sed -i ''s/1.0.0/$(Package.Version)/g'' arcus/azureml/__init__.py' + failOnStderr: true -- script: | - python setup.py sdist bdist_wheel - displayName: 'Artifact creation' + - job: PackageAndRelease + displayName: 'Package and release' + steps: + - script: | + python setup.py sdist bdist_wheel + displayName: 'Artifact creation' -- script: python -m pip install --upgrade twine - displayName: 'Install Twine' + - script: python -m pip install --upgrade twine + displayName: 'Install Twine' -- task: TwineAuthenticate@1 - inputs: - pythonUploadServiceConnection: 'Arcus AzureML PyPi feed' + - task: TwineAuthenticate@1 + inputs: + pythonUploadServiceConnection: 'Arcus AzureML PyPi feed' -- template: github/create-release.yml@templates - parameters: - tag: '$(Package.Version)' - releaseTitle: '$(Package.Version)' - repositoryName: 'arcus-azure/arcus.azureml' - releaseNotes: | - Install new version via [PyPi](https://pypi.org/project/arcus-azureml/$(Package.Version)) - ```shell - PM > pip install arcus-azureml==$(Package.Version) - ``` + - template: github/create-release.yml@templates + parameters: + tag: '$(Package.Version)' + releaseTitle: '$(Package.Version)' + repositoryName: 'arcus-azure/arcus.azureml' + releaseNotes: | + Install new version via [PyPi](https://pypi.org/project/arcus-azureml/$(Package.Version)) + ```shell + PM > pip install arcus-azureml==$(Package.Version) + ``` -- script: | - python -m twine upload --skip-existing --verbose -r 'arcus-azureml' --config-file $(PYPIRC_PATH) dist/* - displayName: 'Publish to PyPi' + - script: | + python -m twine upload --skip-existing --verbose -r 'arcus-azureml' --config-file $(PYPIRC_PATH) dist/* + displayName: 'Publish to PyPi' -- task: CopyFiles@2 - inputs: - targetFolder: $(Build.ArtifactStagingDirectory) + - task: CopyFiles@2 + inputs: + targetFolder: $(Build.ArtifactStagingDirectory) -- task: PublishBuildArtifacts@1 - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)' - ArtifactName: 'dist' - publishLocation: 'Container' \ No newline at end of file + - task: PublishBuildArtifacts@1 + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)' + ArtifactName: 'dist' + publishLocation: 'Container' \ No newline at end of file