From e90834a8d6dd0608fb819801161729713fc7c95c Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 15 Sep 2022 11:20:43 +0100 Subject: [PATCH 1/6] ENH: Simplify analysis finalisation --- app.py | 12 +++++------- setup.py | 4 ++-- tests/test_app.py | 2 +- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/app.py b/app.py index fa4b0ce..8ed78d5 100644 --- a/app.py +++ b/app.py @@ -3,10 +3,8 @@ import os import tempfile -import coolname -from octue.cloud import storage from octue.resources import Datafile, Dataset -from octue.utils.processes import run_subprocess_and_log_stdout_and_stderr +from octue.utils.threads import run_logged_subprocess logger = logging.getLogger(__name__) @@ -30,7 +28,7 @@ def run(analysis): input_file = input_dataset.files.one() logger.info("Starting turbsim analysis.") - run_subprocess_and_log_stdout_and_stderr(command=["turbsim", input_file.local_path], logger=logger) + run_logged_subprocess(command=["turbsim", input_file.local_path], logger=logger) old_output_filename = os.path.splitext(input_file.local_path)[0] + OUTPUT_EXTENSION @@ -45,8 +43,8 @@ def run(analysis): analysis.output_manifest.datasets["turbsim"] = Dataset(name="turbsim", path=new_temporary_directory) - analysis.finalise( - upload_output_datasets_to=storage.path.join(analysis.output_location, coolname.generate_slug()) - ) + # Explicitly call `finalise` here instead of relying on implicit finalisation so the temporary directory + # still when it's called. + analysis.finalise() logger.info("Finished turbsim analysis.") diff --git a/setup.py b/setup.py index 24e94cc..c92e115 100644 --- a/setup.py +++ b/setup.py @@ -3,11 +3,11 @@ setup( name="turbsim-service", - version="0.1.0", + version="0.1.1", author="cortadocodes ", py_modules=["app"], install_requires=[ "coolname>=1.1,<2", - "octue==0.26.0", + "octue @ https://github.com/octue/octue-sdk-python/archive/enhancement/automatically-upload-output-datasets-to-output-location.zip", ], ) diff --git a/tests/test_app.py b/tests/test_app.py index 3c6fa6d..c16f8c0 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -27,7 +27,7 @@ def test_app(self): input_manifest = Manifest(datasets={"turbsim": "gs://openfast-data/testing/turbsim"}) # Mock running an OpenFAST analysis by creating an empty output file. - with patch("app.run_subprocess_and_log_stdout_and_stderr", self._create_mock_output_file): + with patch("app.run_logged_subprocess", self._create_mock_output_file): analysis = runner.run(input_manifest=input_manifest.serialise()) self.assertIsNone(analysis.output_values) From 57bf95b25a7502fa3099b6da1db35e3212bbc0a8 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 15 Sep 2022 11:26:43 +0100 Subject: [PATCH 2/6] TST: Update `Child` usage in test --- tests/test_deployment.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_deployment.py b/tests/test_deployment.py index 5c607f3..ef24743 100644 --- a/tests/test_deployment.py +++ b/tests/test_deployment.py @@ -23,7 +23,6 @@ def test_cloud_run_deployment(self): input_manifest = Manifest(datasets={"turbsim": "gs://openfast-data/testing/turbsim"}) child = Child( - name="turbsim-service", id="aerosense/turbsim-service", backend={"name": "GCPPubSubBackend", "project_name": os.environ["TEST_PROJECT_NAME"]}, ) From 009627d0ec0e4b3e6ff5f85d5314b1a801222079 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 15 Sep 2022 11:28:46 +0100 Subject: [PATCH 3/6] OPS: Use octue==0.35.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c92e115..b4dcf8d 100644 --- a/setup.py +++ b/setup.py @@ -8,6 +8,6 @@ py_modules=["app"], install_requires=[ "coolname>=1.1,<2", - "octue @ https://github.com/octue/octue-sdk-python/archive/enhancement/automatically-upload-output-datasets-to-output-location.zip", + "octue==0.35.1", ], ) From fa24cb76a3c983584fbef554007d120875d5b810 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 15 Sep 2022 11:33:59 +0100 Subject: [PATCH 4/6] OPS: Add GitHub workflows --- .github/workflows/python-ci.yml | 26 +++++++++++++++++ .github/workflows/release.yml | 30 +++++++++++++++++++ .github/workflows/update-pull-request.yml | 35 +++++++++++++++++++++++ 3 files changed, 91 insertions(+) create mode 100644 .github/workflows/python-ci.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/update-pull-request.yml diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml new file mode 100644 index 0000000..a254a30 --- /dev/null +++ b/.github/workflows/python-ci.yml @@ -0,0 +1,26 @@ +name: python-ci + +on: + push: + branches-ignore: + - main + +jobs: + check-semantic-version: + if: "!contains(github.event.head_commit.message, 'skipci')" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + # Set fetch-depth to 0 to fetch all tags (necessary for git-mkver to determine the correct semantic version). + fetch-depth: 0 + - uses: actions/setup-python@v2 + - name: Install git-mkver + run: | + curl -L https://github.com/idc101/git-mkver/releases/download/v1.2.1/git-mkver-linux-amd64-1.2.1.tar.gz \ + | tar xvz \ + && sudo mv git-mkver /usr/local/bin + - name: Install semantic version checker + run: pip install git+https://github.com/octue/conventional-commits + - name: Check version + run: check-semantic-version setup.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..5a5d177 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,30 @@ +# This workflow releases a new version of the package. + +name: Release + +# Only trigger when a pull request into main branch is merged. +on: + pull_request: + types: [closed] + branches: + - main + +jobs: + release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Get package version + run: echo "PACKAGE_VERSION=$(python setup.py --version)" >> $GITHUB_ENV + + - name: Create Release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, no need to create your own. + with: + tag_name: ${{ env.PACKAGE_VERSION }} + release_name: ${{ github.event.pull_request.title }} + body: ${{ github.event.pull_request.body }} + draft: false + prerelease: false diff --git a/.github/workflows/update-pull-request.yml b/.github/workflows/update-pull-request.yml new file mode 100644 index 0000000..3586423 --- /dev/null +++ b/.github/workflows/update-pull-request.yml @@ -0,0 +1,35 @@ +# This workflow updates the pull request description with an auto-generated section containing the categorised commit +# message headers of the commits since the last pull request merged into main. The auto generated section is enveloped +# between two comments: "" and "". Anything +# outside these in the description is left untouched. Auto-generated updates can be skipped for a commit if +# "" is added to the pull request description. + +name: update-pull-request + +# Only trigger for pull requests into main branch. +on: + pull_request: + branches: + - main + +jobs: + description: + if: "!contains(github.event.pull_request.body, '')" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + # Set fetch-depth to 0 to fetch all commit history (necessary for compiling pull request description). + fetch-depth: 0 + - name: Install release note compiler + run: pip install git+https://github.com/octue/conventional-commits + - name: Compile new pull request description + run: | + echo 'PULL_REQUEST_NOTES<> $GITHUB_ENV + echo "$(compile-release-notes PULL_REQUEST_START --pull-request-url=${{ github.event.pull_request.url }} --api-token=${{ secrets.GITHUB_TOKEN }})" >> $GITHUB_ENV + echo EOF >> $GITHUB_ENV + - name: Update pull request body + uses: riskledger/update-pr-description@v2 + with: + body: ${{ env.PULL_REQUEST_NOTES }} + token: ${{ secrets.GITHUB_TOKEN }} From 23838eccd899fcea281af4c4fd6d5fdc53ae90bf Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 15 Sep 2022 11:36:04 +0100 Subject: [PATCH 5/6] OPS: Add mkver configuration --- mkver.conf | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 mkver.conf diff --git a/mkver.conf b/mkver.conf new file mode 100644 index 0000000..0370c28 --- /dev/null +++ b/mkver.conf @@ -0,0 +1,41 @@ +tagPrefix: "" +defaults { + tag: false + tagMessageFormat: "Release/{Tag}" + preReleaseFormat: "RC{PreReleaseNumber}" + buildMetaDataFormat: "{Branch}.{ShortHash}" + includeBuildMetaData: false + whenNoValidCommitMessages: IncrementPatch + patches: [setup.py] +} + +patches: [ + { + name: setup.py + filePatterns: ["setup.py"] + replacements: [ + { + find: "version=\"{VersionRegex}\"" + replace: "version=\"{Version}\"" + } + ] + } +] + +commitMessageActions: [ + # Disable major version increments while package is still in beta (i.e. keep the version below 1.0.0). + { + pattern: "BREAKING CHANGE" + action: IncrementMinor + } + { + pattern: "BREAKING-CHANGE" + action: IncrementMinor + } + + # All new features require a minor version increase. + { + pattern: "FEA:" + action: IncrementMinor + } +] From 90bbad02b967a20efbe25e94a9716a06e7530ce1 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 15 Sep 2022 12:47:26 +0100 Subject: [PATCH 6/6] CHO: Add missing word in comment skipci --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index 8ed78d5..1d51599 100644 --- a/app.py +++ b/app.py @@ -44,7 +44,7 @@ def run(analysis): analysis.output_manifest.datasets["turbsim"] = Dataset(name="turbsim", path=new_temporary_directory) # Explicitly call `finalise` here instead of relying on implicit finalisation so the temporary directory - # still when it's called. + # still exists when it's called. analysis.finalise() logger.info("Finished turbsim analysis.")