diff --git a/build/cd-build.yml b/build/cd-build.yml index f13733e..628b62a 100644 --- a/build/cd-build.yml +++ b/build/cd-build.yml @@ -17,58 +17,69 @@ resources: name: arcus-azure/azure-devops-templates endpoint: arcus-azure -steps: -- task: UsePythonVersion@0 - inputs: - versionSpec: '3.7' - addToPath: true - architecture: 'x64' +stages: + - stage: Prepare + jobs: + - job: InstallPrerequisites + displayName: 'Install pre requisites & tools' + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + addToPath: true + architecture: 'x64' -- script: python -m pip install --upgrade pip setuptools wheel - displayName: 'Install tools' + - script: python -m pip install --upgrade pip setuptools wheel + displayName: 'Install tools' -- script: pip install -r requirements.txt - displayName: 'Install requirements' + - script: pip install -r requirements.txt + displayName: 'Install requirements' -- task: Bash@3 - displayName: 'Update version number' - inputs: - targetType: 'inline' - script: 'sed -i ''s/1.0.0/$(Package.Version)/g'' arcus/azureml/__init__.py' - failOnStderr: true + - job: DefineVersion + displayName: 'Increase and define version number' + steps: + - task: Bash@3 + displayName: 'Update version number' + inputs: + targetType: 'inline' + script: 'sed -i ''s/1.0.0/$(Package.Version)/g'' arcus/azureml/__init__.py' + failOnStderr: true -- script: | - python setup.py sdist bdist_wheel - displayName: 'Artifact creation' + - job: PackageAndRelease + displayName: 'Package and release' + steps: + - script: | + python setup.py sdist bdist_wheel + displayName: 'Artifact creation' -- script: python -m pip install --upgrade twine - displayName: 'Install Twine' + - script: python -m pip install --upgrade twine + displayName: 'Install Twine' -- task: TwineAuthenticate@1 - inputs: - pythonUploadServiceConnection: 'Arcus AzureML PyPi feed' + - task: TwineAuthenticate@1 + inputs: + pythonUploadServiceConnection: 'Arcus AzureML PyPi feed' -- template: github/create-release.yml@templates - parameters: - tag: '$(Package.Version)' - releaseTitle: '$(Package.Version)' - repositoryName: 'arcus-azure/arcus.azureml' - releaseNotes: | - Install new version via [PyPi](https://pypi.org/project/arcus-azureml/$(Package.Version)) - ```shell - PM > pip install arcus-azureml==$(Package.Version) - ``` + - template: github/create-release.yml@templates + parameters: + tag: '$(Package.Version)' + releaseTitle: '$(Package.Version)' + repositoryName: 'arcus-azure/arcus.azureml' + releaseNotes: | + Install new version via [PyPi](https://pypi.org/project/arcus-azureml/$(Package.Version)) + ```shell + PM > pip install arcus-azureml==$(Package.Version) + ``` -- script: | - python -m twine upload --skip-existing --verbose -r 'arcus-azureml' --config-file $(PYPIRC_PATH) dist/* - displayName: 'Publish to PyPi' + - script: | + python -m twine upload --skip-existing --verbose -r 'arcus-azureml' --config-file $(PYPIRC_PATH) dist/* + displayName: 'Publish to PyPi' -- task: CopyFiles@2 - inputs: - targetFolder: $(Build.ArtifactStagingDirectory) + - task: CopyFiles@2 + inputs: + targetFolder: $(Build.ArtifactStagingDirectory) -- task: PublishBuildArtifacts@1 - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)' - ArtifactName: 'dist' - publishLocation: 'Container' \ No newline at end of file + - task: PublishBuildArtifacts@1 + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)' + ArtifactName: 'dist' + publishLocation: 'Container' \ No newline at end of file diff --git a/build/ci-build.yml b/build/ci-build.yml index 09383b3..3a5c4e8 100644 --- a/build/ci-build.yml +++ b/build/ci-build.yml @@ -8,6 +8,7 @@ trigger: include: - arcus/* - docs/* + - requirements.txt pr: paths: @@ -16,6 +17,7 @@ pr: - docs/* - tests/* - build/ci-build.yml + - requirements.txt resources: repositories: @@ -27,73 +29,86 @@ resources: pool: vmImage: 'ubuntu-latest' -steps: -- template: 'pypi/determine-pr-version.yml@templates' - parameters: - manualTriggerVersion: preview - -- task: UsePythonVersion@0 - inputs: - versionSpec: '3.7' - addToPath: true - architecture: 'x64' - -- task: Bash@3 - displayName: 'Update version number' - inputs: - targetType: 'inline' - script: 'sed -i ''s/1.0.0/$(Package.Version).$(Build.BuildNumber)/g'' arcus/azureml/__init__.py' - failOnStderr: true - -- script: python -m pip install --upgrade pip setuptools wheel - displayName: 'Install tools' - -- script: pip install -r requirements.txt - displayName: 'Install requirements' - -- script: | - pip install pytest - pip install pytest-cov - pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=. --cov-report=xml --cov-report=html - displayName: 'Unit tests (pytest)' - -- task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-*.xml' - testRunTitle: 'Publish test results for Python $(python.version)' - -- task: PublishCodeCoverageResults@1 - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml' - reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov' - -- script: | - python setup.py sdist bdist_wheel - displayName: 'Build wheels' - -- script: python -m pip install --upgrade twine - displayName: 'Install Twine' - -- task: TwineAuthenticate@1 - inputs: - pythonUploadServiceConnection: 'Arcus AzureML PyPi feed' - -- script: | - cat $(PYPIRC_PATH) - displayName: 'Print pypirc file' - -- script: | - python -m twine upload --skip-existing --verbose -r 'arcus-azureml' --config-file $(PYPIRC_PATH) dist/* - displayName: 'Publish to PyPi dev' - -- task: CopyFiles@2 - inputs: - targetFolder: $(Build.ArtifactStagingDirectory) - -- task: PublishBuildArtifacts@1 - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)' - ArtifactName: 'dist' - publishLocation: 'Container' + +stages: + - stage: BuildAndTest + jobs: + - job: DefineVersion + displayName: 'Increase and define version number' + steps: + - template: 'pypi/determine-pr-version.yml@templates' + parameters: + manualTriggerVersion: preview + + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + addToPath: true + architecture: 'x64' + + - task: Bash@3 + displayName: 'Update version number' + inputs: + targetType: 'inline' + script: 'sed -i ''s/1.0.0/$(Package.Version).$(Build.BuildNumber)/g'' arcus/azureml/__init__.py' + failOnStderr: true + + - job: UnitTests + displayName: 'Run unit tests' + steps: + - script: pip install -r requirements.txt + displayName: 'Install requirements.txt' + + - script: | + pip install pytest + pip install pytest-cov + pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=. --cov-report=xml --cov-report=html + displayName: 'Unit tests (pytest)' + + - task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + testResultsFiles: '**/test-*.xml' + testRunTitle: 'Publish test results for Python $(python.version)' + + - task: PublishCodeCoverageResults@1 + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml' + reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov' + + - stage: PackageAndRelease + displayName: 'Package and release to PyPi' + dependsOn: BuildAndTest + condition: succeeded() + jobs: + - job: PackageAndRelease + displayName: 'Package and release' + steps: + - script: python -m pip install --upgrade pip setuptools wheel lazydocs + displayName: 'Install tools' + + - script: python -m pip install --upgrade twine + displayName: 'Install Twine' + + - script: | + python setup.py sdist bdist_wheel + displayName: 'Build wheels' + + - task: TwineAuthenticate@1 + inputs: + pythonUploadServiceConnection: 'Arcus AzureML PyPi feed' + + - script: | + python -m twine upload --skip-existing --verbose -r 'arcus-azureml' --config-file $(PYPIRC_PATH) dist/* + displayName: 'Publish to PyPi dev' + + - task: CopyFiles@2 + inputs: + targetFolder: $(Build.ArtifactStagingDirectory) + + - task: PublishBuildArtifacts@1 + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)' + ArtifactName: 'dist' + publishLocation: 'Container' diff --git a/requirements.txt b/requirements.txt index 9e4ec3b..d99a96d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,9 +10,9 @@ scikit-learn scikit-image matplotlib tensorflow>=2.1.0 -keras +keras>=2.6.0rc3 # not directly required, pinned by Snyk to avoid a vulnerability arcus-ml>=1.0.11 -opencv-python +opencv-python>=4.4.0.40 pillow>=8.1.0 # not directly required, pinned by Snyk to avoid a vulnerabilitypillow>=8.1.0 # not directly required, pinned by Snyk to avoid a vulnerability kaggle - +networkx>=2.6 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/samples/Arcus-Kaggle-Download/requirements.txt b/samples/Arcus-Kaggle-Download/requirements.txt index db13a12..fd7f997 100644 --- a/samples/Arcus-Kaggle-Download/requirements.txt +++ b/samples/Arcus-Kaggle-Download/requirements.txt @@ -18,4 +18,5 @@ opencv-python seaborn scikit-image opencv-python -kaggle \ No newline at end of file +kaggle +pillow>=8.3.2 # not directly required, pinned by Snyk to avoid a vulnerability \ No newline at end of file