diff --git a/.github/actions/bump-version-tag/action.yml b/.github/actions/bump-version-tag/action.yml new file mode 100644 index 0000000..b799405 --- /dev/null +++ b/.github/actions/bump-version-tag/action.yml @@ -0,0 +1,117 @@ +name: Bump Version and Tag Reusable Workflow + +description: | + Get the would-be bumped version and version tag for a commit based on the + changes since last version. +inputs: + bump_type: + description: 'Version Bump Type' + type: choice + options: ['major', 'minor', 'patch', 'none'] + required: false + default: 'none' + dry_run: + description: 'Dry Run' + type: boolean + required: false + default: false + token: + description: 'Token for AllenInstitute GitHub' + required: true +outputs: + major: + description: 'The major version number' + value: ${{ steps.set-outputs.outputs.major }} + minor: + description: 'The minor version number' + value: ${{ steps.set-outputs.outputs.minor }} + patch: + description: 'The patch version number' + value: ${{ steps.set-outputs.outputs.patch }} + increment: + description: 'The increment. This is the number of commits since the last tag.' + value: ${{ steps.set-outputs.outputs.increment }} + version: + description: 'The version number (e.g. 1.2.3)' + value: ${{ steps.set-outputs.outputs.version }} + version_tag: + description: 'The version tag (e.g. v1.2.3)' + value: ${{ steps.set-outputs.outputs.version_tag }} + version_type: + description: 'The version type (e.g. major, minor, patch, none)' + value: ${{ steps.set-outputs.outputs.version_type }} + previous_version: + description: 'The previous version number (e.g. 1.2.2)' + value: ${{ steps.set-outputs.outputs.previous_version }} + +runs: + using: "composite" + steps: + - name: Bump patch version and tag + id: bump-version-tag + uses: anothrNick/github-tag-action@1.64.0 + env: + GITHUB_TOKEN: ${{ inputs.token }} + DEFAULT_BUMP: ${{ inputs.bump_type || 'none' }} + MAJOR_STRING_TOKEN: '(MAJOR)' + MINOR_STRING_TOKEN: '(MINOR)' + PATCH_STRING_TOKEN: '(PATCH)' + NONE_STRING_TOKEN: '(NONE)' + WITH_V: "true" + RELEASE_BRANCHES: main + DRY_RUN: true + - name: Set Outputs + id: set-outputs + run: | + new_tag=${{ steps.bump-version-tag.outputs.new_tag }} + new_version=$(echo $new_tag | sed 's/^v//') + major=$(echo $new_version | cut -d. -f1) + minor=$(echo $new_version | cut -d. -f2) + patch=$(echo $new_version | cut -d. -f3) + increment=0 + version_type=${{ steps.bump-version-tag.outputs.part }} + previous_version=$(git describe --tags --abbrev=0 | sed 's/^v//') + + echo "major=$major" >> $GITHUB_OUTPUT + echo "minor=$minor" >> $GITHUB_OUTPUT + echo "patch=$patch" >> $GITHUB_OUTPUT + echo "increment=$increment" >> $GITHUB_OUTPUT + echo "version=$new_version" >> $GITHUB_OUTPUT + echo "version_tag=$new_tag" >> $GITHUB_OUTPUT + echo "version_type=$version_type" >> $GITHUB_OUTPUT + echo "previous_version=$previous_version" >> $GITHUB_OUTPUT + shell: bash + + # Currently not using this Version bumping tool, but considering it for the future. + # The main limitation is being able to override the default bump type even + # if there are no commits. + - name: Bump Version Alternate + uses: PaulHatch/semantic-version@v5.4.0 + id: bump-version-tag-alt + with: + major_pattern: "/\\((MAJOR|BREAKING)\\)/" + minor_pattern: "/\\((MINOR|FEATURE)\\)/" + bump_each_commit: true + bump_each_commit_patch_pattern: "/\\((PATCH|BUG)\\)/" + - name: Set Outputs Alt + id: set-outputs-alt + shell: bash + run: | + echo 'changed: ${{ steps.bump-version-tag-alt.outputs.changed }}' + echo 'major: ${{ steps.bump-version-tag-alt.outputs.major }}' + echo 'minor: ${{ steps.bump-version-tag-alt.outputs.minor }}' + echo 'patch: ${{ steps.bump-version-tag-alt.outputs.patch }}' + echo 'increment: ${{ steps.bump-version-tag-alt.outputs.increment }}' + echo 'version: ${{ steps.bump-version-tag-alt.outputs.version }}' + echo 'version_tag: ${{ steps.bump-version-tag-alt.outputs.version_tag }}' + echo 'version_type: ${{ steps.bump-version-tag-alt.outputs.version_type }}' + echo 'previous_version: ${{ steps.bump-version-tag-alt.outputs.previous_version }}' + + # echo "major=${{ steps.bump-version-tag-alt.outputs.major }}" >> $GITHUB_OUTPUT + # echo "minor=${{ steps.bump-version-tag-alt.outputs.minor }}" >> $GITHUB_OUTPUT + # echo "patch=${{ steps.bump-version-tag-alt.outputs.patch }}" >> $GITHUB_OUTPUT + # echo "increment=${{ steps.bump-version-tag-alt.outputs.increment }}" >> $GITHUB_OUTPUT + # echo "version=${{ steps.bump-version-tag-alt.outputs.version }}" >> $GITHUB_OUTPUT + # echo "version_tag=${{ steps.bump-version-tag-alt.outputs.version_tag }}" >> $GITHUB_OUTPUT + # echo "version_type=${{ steps.bump-version-tag-alt.outputs.version_type }}" >> $GITHUB_OUTPUT + # echo "previous_version=${{ steps.bump-version-tag-alt.outputs.previous_version }}" >> $GITHUB_OUTPUT diff --git a/.github/actions/setup-ai-github-urls/action.yml b/.github/actions/configure-org-repo-authorization/action.yml similarity index 94% rename from .github/actions/setup-ai-github-urls/action.yml rename to .github/actions/configure-org-repo-authorization/action.yml index 14aaae6..0aada1d 100644 --- a/.github/actions/setup-ai-github-urls/action.yml +++ b/.github/actions/configure-org-repo-authorization/action.yml @@ -1,4 +1,4 @@ -name: AllenInstitute Repo Setup +name: AllenInstitute Repo Permissions Setup description: | Configures all credentials to use AllenInstitute Repos in GitHub. diff --git a/.github/actions/source-code-version-get/action.yml b/.github/actions/source-code-version-get/action.yml new file mode 100644 index 0000000..6102631 --- /dev/null +++ b/.github/actions/source-code-version-get/action.yml @@ -0,0 +1,37 @@ +# +name: Get Version from Source Code + +description: | + Get the version from the source code and output the version and tag. + +inputs: + version_file: + description: 'The file containing the version number' + default: '_version.py' + required: false + version_regex: + description: 'The regex to extract the version number' + default: "__version__ = (?:\"|')([0-9]+\.[0-9]+\.[0-9]+)(?:\"|')" + required: false + version_tag_prefix: + description: 'The prefix for the version tag' + required: false + default: 'v' +outputs: + version: + description: 'The version number (e.g. 1.2.3)' + version_tag: + description: 'The version tag (e.g. v1.2.3)' + +runs: + using: "composite" + steps: + - name: Get Version + id: get-version + run: | + version=$(find . -name ${{ inputs.version_file }} -exec cat {} \; | grep -oP -m 1 "${{ inputs.version_regex }}") + echo "Version: $version" + echo "version=$version" >> $GITHUB_OUTPUT + echo "version_tag=${{ inputs.version_tag_prefix }}$version" >> $GITHUB_OUTPUT + shell: bash +``` \ No newline at end of file diff --git a/.github/actions/source-code-version-update/action.yml b/.github/actions/source-code-version-update/action.yml new file mode 100644 index 0000000..3a710f0 --- /dev/null +++ b/.github/actions/source-code-version-update/action.yml @@ -0,0 +1,52 @@ +# +name: Update Version in Source Code + +description: | + Get the version from the source code and output the version and tag. + +inputs: + version: + description: 'The version number (e.g. 1.2.3)' + required: true + version_tag: + description: 'Optionally, can The version tag (e.g. v1.2.3)' + version_file: + description: 'The file containing the version number' + default: '_version.py' + required: false + version_regex: + description:| + The regex to extract everything BUT the version number. It is very important to + capture BEFORE and AFTER the version number. This is going to be used with `sed -E` + default: "(__version__ = ['\"])[0-9]+\.[0-9]+\.[0-9]+(['\"])" + required: false + +runs: + using: "composite" + steps: + - name: Update Version in Source Code + run: | + echo "Updating version to ${{ inputs.version }}" + find . -name ${{ inputs.version_file }} -exec sed -i -E "s/${{ inputs.version_regex }}/\1${new_version}\2/" {} \; + echo git diff following update: + git diff + - name: Create Git commit and tag + if ! git diff --name-only -- **/${{ inputs.version_file }} | grep -q '${{ inputs.version_file }}'; then + echo "No changes detected. Version already ${{ inputs.version }}." + echo "Skipping commit and tag." + else + echo "Changes detected." + git add **/${{ inputs.version_file }} + git commit -m "Bump version to ${{ inputs.version }}" + git push + + if [ -z "${{ inputs.version_tag }}" ]; then + echo "No version tag provided. Skipping tag." + echo "Skipping tag." + else + echo "Creating tag ${{ inputs.version_tag }}" + git tag -a ${{ inputs.version_tag }} -m "Release ${{ inputs.version_tag }}" + git push --tags + fi + fi +``` \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bd052b4..417f5ac 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,8 +3,12 @@ name: Build and Test on: pull_request: branches: [ main ] + paths-ignore: + - '**/_version.py' push: branches: [ main ] + paths-ignore: + - '**/_version.py' jobs: test: @@ -14,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -23,11 +27,11 @@ jobs: python-version: ${{ matrix.python-version }} cache: 'pip' - name: Set up AllenInstitute Repo Authorization - uses: ./.github/actions/setup-ai-github-urls + uses: ./.github/actions/configure-org-repo-authorization with: token: ${{ secrets.AI_PACKAGES_TOKEN }} ssh_private_key: ${{ secrets.AIBSGITHUB_PRIVATE_KEY }} - name: Run Release - shell: bash run: | make release + shell: bash diff --git a/.github/workflows/bump_version.yml b/.github/workflows/bump_version.yml new file mode 100644 index 0000000..1e59ffa --- /dev/null +++ b/.github/workflows/bump_version.yml @@ -0,0 +1,115 @@ +name: Bump Version and Tag + +on: + push: + branches: [ main ] + paths-ignore: + - '**/_version.py' + + workflow_dispatch: + inputs: + bump_type: + description: 'Version bump type to use. If target_version is set, this will be ignored.' + type: choice + options: ['major', 'minor', 'patch', 'none'] + required: false + default: 'none' + target_version: + description: | + Optional target version (e.g. 1.2.3) to bump to. If not set, bump type will be used. + (leave empty for default) + required: false + default: '' + dry_run: + description: 'Dry Run' + type: boolean + required: false + default: false + +jobs: + get-version-info: + name: Get New Version Tag + runs-on: ubuntu-latest + if: github.actor != 'github-actions[bot]' + outputs: + version: ${{ steps.set-target-version.outputs.version || steps.version-tag.outputs.version }} + version_tag: ${{ steps.set-target-version.outputs.version_tag || steps.version-tag.outputs.version_tag }} + version_type: ${{ steps.set-target-version.outputs.version_type || steps.version-tag.outputs.version_type }} + previous_version: ${{ steps.get-current-version.outputs.previous_version || steps.version-tag.outputs.previous_version }} + update_required: ${{ steps.set-update-required.outputs.update_required }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Get Bumped Version Tag + uses: ./.github/actions/bump-version-tag + id: version-tag + with: + bump_type: ${{ github.event.inputs.bump_type }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Print Version Info + # We only want to print if we are not a workflow call or if the target version is 'N/A' + if: github.event_name != 'workflow_dispatch' || !inputs.target_version + run: | + echo "Version: ${{ steps.version-tag.outputs.version }}" + echo "Version Tag: ${{ steps.version-tag.outputs.version_tag }}" + echo "Version Type: ${{ steps.version-tag.outputs.version_type }}" + echo "Previous Version: ${{ steps.version-tag.outputs.previous_version }}" + - name: Set Target Version + id: set-target-version + # We only want to set the target version if we are a workflow call and the target version is set + if: github.event_name == 'workflow_dispatch' && inputs.target_version + run: | + echo "Setting target version to ${{ inputs.target_version }}" + echo "version=${{ inputs.target_version }}" >> $GITHUB_OUTPUT + echo "version_tag=v${{ inputs.target_version }}" >> $GITHUB_OUTPUT + - name: Get Current Version + id: get-current-version + # We only want to get current version if we are a workflow call and the target version is set + if: github.event_name == 'workflow_dispatch' && inputs.target_version + uses: ./.github/actions/source-code-version-get + with: + version_file: _version.py + - name: Set Update Required + id: set-update-required + run: | + if [ "${{ steps.set-target-version.outputs.version || steps.version-tag.outputs.version }}" != "${{ steps.version-tag.outputs.previous_version }}" ]; then + echo "Update required" + echo "update_required=true" >> $GITHUB_OUTPUT + else + echo "No update required" + echo "update_required=false" >> $GITHUB_OUTPUT + fi + + update-version-and-tag: + name: Update Repo Tag and Version + runs-on: ubuntu-latest + needs: get-version-info + # We only want to run if: + # 1. We are not the GitHub bot + # 2. We are not a workflow call or we are not in dry run mode + # 3. The update is required (i.e. the version has changed) + if: | + github.actor != 'github-actions[bot]' && + (github.event_name != 'workflow_dispatch' || !inputs.dry_run) && + needs.get-version-info.outputs.update_required == 'true' + + steps: + - uses: actions/checkout@v4 + # This sets up the git user for the GitHub bot + - name: Configure Git User + run: | + git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --local user.name "github-actions[bot]" + # This sets up ssh keys for the AllenInstitute GitHub + - name: Configure AllenInstitute Repo Authorization + uses: ./.github/actions/configure-org-repo-authorization + with: + token: ${{ secrets.AI_PACKAGES_TOKEN }} + ssh_private_key: ${{ secrets.AIBSGITHUB_PRIVATE_KEY }} + - name: Update Version + uses: ./.github/actions/source-code-version-update + with: + version: ${{ needs.get-version-info.outputs.version }} + version_tag: ${{ needs.get-version-info.outputs.version_tag }} + version_file: _version.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..82e4b7a --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,37 @@ +name: Create Release + +on: + # push: + # tags: + # - 'v*' + + workflow_dispatch: + inputs: + tag: + description: 'Target Tag for Release' + type: string + required: true + force: + description: 'Force Release?' + type: boolean + required: false + default: false + +jobs: + create-release: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.tag || github.ref }} + - name: Create Release with Changelog + id: create_release + uses: softprops/action-gh-release@v1 + with: + tag_name: ${{ github.event.inputs.tag || github.ref_name }} + name: Release ${{ github.event.inputs.tag || github.ref_name }} + draft: false + prerelease: false + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/pyproject.toml b/pyproject.toml index 6f4475f..92c6596 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,10 +11,9 @@ build-backend = "setuptools.build_meta" # ----------------------------------------------------------------------------- [project] name = "aibs-informatics-aws-utils" -version = "0.0.1" description = "Library of AWS utility code for informatics projects at the Allen Institute for Brain Science" readme = "README.md" -# dynamic = [ "version"] +dynamic = [ "version"] requires-python = ">=3.9" dependencies = [ @@ -35,6 +34,9 @@ where = ["src"] [tool.setuptools.package-data] "*" = ['py.typed'] +[tool.setuptools.dynamic] +version = {attr = "aibs_informatics_aws_utils._version.__version__"} + # ----------------------------------------------------------------------------- ## Pyright Configurations # https://github.com/microsoft/pyright/blob/main/docs/configuration.md diff --git a/src/aibs_informatics_aws_utils/__init__.py b/src/aibs_informatics_aws_utils/__init__.py index dff2249..e4b0853 100644 --- a/src/aibs_informatics_aws_utils/__init__.py +++ b/src/aibs_informatics_aws_utils/__init__.py @@ -1,4 +1 @@ -__version_info__ = (0, 0, 1) -__version__ = ".".join(map(str, __version_info__)) - from aibs_informatics_aws_utils.core import * # type: ignore diff --git a/src/aibs_informatics_aws_utils/_version.py b/src/aibs_informatics_aws_utils/_version.py new file mode 100644 index 0000000..f102a9c --- /dev/null +++ b/src/aibs_informatics_aws_utils/_version.py @@ -0,0 +1 @@ +__version__ = "0.0.1" diff --git a/src/aibs_informatics_aws_utils/auth.py b/src/aibs_informatics_aws_utils/auth.py index 498bddd..f2ca95c 100644 --- a/src/aibs_informatics_aws_utils/auth.py +++ b/src/aibs_informatics_aws_utils/auth.py @@ -5,11 +5,12 @@ from botocore.awsrequest import AWSRequest from botocore.compat import parse_qsl, urlparse from botocore.session import Session +from requests.auth import AuthBase from aibs_informatics_aws_utils.core import get_session -class IamAWSRequestsAuth(requests.auth.AuthBase): +class IamAWSRequestsAuth(AuthBase): """ IAM authorizer. @@ -24,8 +25,12 @@ class IamAWSRequestsAuth(requests.auth.AuthBase): def __init__(self, session: Optional[Session] = None, service_name: str = "execute-api"): self.boto3_session = get_session(session) + credentials = self.boto3_session.get_credentials() + if not credentials: + raise ValueError("No AWS credentials found") + self.sigv4 = SigV4Auth( - credentials=self.boto3_session.get_credentials().get_frozen_credentials(), + credentials=credentials.get_frozen_credentials(), service_name=service_name, region_name=self.boto3_session.region_name, ) diff --git a/src/aibs_informatics_aws_utils/core.py b/src/aibs_informatics_aws_utils/core.py index e457f53..1728f27 100644 --- a/src/aibs_informatics_aws_utils/core.py +++ b/src/aibs_informatics_aws_utils/core.py @@ -13,7 +13,17 @@ import os import re from dataclasses import dataclass -from typing import TYPE_CHECKING, ClassVar, Generic, Literal, Optional, Pattern, TypeVar, cast +from typing import ( + TYPE_CHECKING, + ClassVar, + Generic, + Literal, + Optional, + Pattern, + TypeVar, + Union, + cast, +) import boto3 from aibs_informatics_core.models.aws.core import AWSRegion @@ -22,6 +32,7 @@ from boto3 import Session from boto3.resources.base import ServiceResource from botocore.client import BaseClient, ClientError +from botocore.session import Session as BotocoreSession if TYPE_CHECKING: # pragma: no cover from mypy_boto3_apigateway import APIGatewayClient @@ -78,8 +89,13 @@ AWS_REGION_VAR = "AWS_REGION" -def get_session(session: Optional[Session] = None) -> Session: - return session or Session() +def get_session(session: Optional[Union[Session, BotocoreSession]] = None) -> Session: + if not session: + return Session() + elif isinstance(session, BotocoreSession): + return Session(botocore_session=session) + else: + return session def get_region(region: Optional[str] = None) -> str: diff --git a/src/aibs_informatics_aws_utils/lambda_.py b/src/aibs_informatics_aws_utils/lambda_.py index 979eca6..2cf1c6b 100644 --- a/src/aibs_informatics_aws_utils/lambda_.py +++ b/src/aibs_informatics_aws_utils/lambda_.py @@ -1,9 +1,15 @@ -from typing import TYPE_CHECKING, List, Optional, Union +import json +from typing import TYPE_CHECKING, List, Literal, Optional, Union +from urllib.parse import parse_qs +import requests from aibs_informatics_core.models.aws.core import AWSAccountId, AWSRegion -from aibs_informatics_core.models.aws.lambda_ import LambdaFunctionName +from aibs_informatics_core.models.aws.lambda_ import LambdaFunctionName, LambdaFunctionUrl +from aibs_informatics_core.models.base import ModelProtocol from botocore.exceptions import ClientError +from requests.auth import AuthBase +from aibs_informatics_aws_utils.auth import IamAWSRequestsAuth from aibs_informatics_aws_utils.core import AWSService, get_client_error_code if TYPE_CHECKING: # pragma: no cover @@ -16,8 +22,8 @@ def get_lambda_function_url( - function_name: Union[LambdaFunctionName, str], region: AWSRegion = None -) -> Optional[str]: + function_name: Union[LambdaFunctionName, str], region: Optional[AWSRegion] = None +) -> Optional[LambdaFunctionUrl]: function_name = LambdaFunctionName(function_name) lambda_client = get_lambda_client(region=region) @@ -29,11 +35,11 @@ def get_lambda_function_url( return None else: raise e - return response["FunctionUrl"] + return LambdaFunctionUrl(response["FunctionUrl"]) def get_lambda_function_file_systems( - function_name: Union[LambdaFunctionName, str], region: AWSRegion = None + function_name: Union[LambdaFunctionName, str], region: Optional[AWSRegion] = None ) -> List[FileSystemConfigTypeDef]: function_name = LambdaFunctionName(function_name) @@ -44,3 +50,60 @@ def get_lambda_function_file_systems( fs_configs = response.get("FileSystemConfigs") return fs_configs + + +def call_lambda_function_url( + function_name: Union[LambdaFunctionName, LambdaFunctionUrl, str], + payload: Optional[Union[ModelProtocol, dict, str, bytes]] = None, + region: Optional[AWSRegion] = None, + headers: Optional[dict] = None, + auth: Optional[AuthBase] = None, + **request_kwargs, +) -> Union[dict, str, None]: + if LambdaFunctionName.is_valid(function_name): + function_url = get_lambda_function_url(LambdaFunctionName(function_name), region=region) + if function_url is None: + raise ValueError(f"Function {function_name} not found") + function_url = LambdaFunctionUrl(function_url) + elif LambdaFunctionUrl.is_valid(function_name): + function_url = LambdaFunctionUrl(function_name) + else: + raise ValueError(f"Invalid function name or url: {function_name}") + + json_payload: Optional[str] = None + if isinstance(payload, (dict, list)): + json_payload = json.dumps(payload) + elif isinstance(payload, ModelProtocol): + json_payload = json.dumps(payload.to_dict()) + elif isinstance(payload, str): + json_payload = payload + elif isinstance(payload, bytes): + json_payload = payload.decode("utf-8") + elif payload is None: + pass + else: + raise ValueError(f"Invalid payload type: {type(payload)}") + + if headers is None: + headers = {} + + if auth is None: + auth = IamAWSRequestsAuth(service_name="lambda") + + response = requests.request( + method="POST" if json_payload else "GET", + url=function_url.base_url + function_url.path, + json=json_payload, + params=function_url.query, + headers=headers, + auth=auth, + **request_kwargs, + ) + if response.ok: + if response.headers.get("Content-Type") == "application/json": + return response.json() + else: + return response.text + else: + response.raise_for_status() + return None