-
Notifications
You must be signed in to change notification settings - Fork 80
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[oidc] Clang tidy binary upload to use OIDC instead of secrets (#5980)
The role is defined in pytorch-labs/pytorch-gha-infra#551 This action has only been run 6 times, and has failed 4 of those 6 runs. The most recent run failed due to lacking credentials, however it doesn't appear that those changes did any functionality change for the files related to these workflows. The last successful runs where last year Also * add a python script for checking if its safe to upload the new binary (overwriting with a new file that has a different sha256 will break lintrunner init if the config is not updated. It will also require people to pull the new config) * linux build - only start the container and copy the file, do not upload the image * upload the artifact to GHA artifacts as well * Move sanity check s3 python script + all uploads into reusable action to reduce code duplication --- Testing: Merge the role definition, add my branch in the console, run the workflow (clean up: remove my branch in the console)
- Loading branch information
Showing
4 changed files
with
175 additions
and
69 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| name: Sanity check and upload clang-tidy and clang-format binaries | ||
|
|
||
| description: Sanity check and upload clang-tidy and clang-format binaries to s3 and GHA | ||
|
|
||
| # Used by clang-tidy-macos and clang-tidy-linux workflows since the sanity | ||
| # checks and uploads are pretty much the same for them, so this helps reduce | ||
| # code duplication | ||
|
|
||
| inputs: | ||
| platform: | ||
| description: 'The platform to upload the binaries for' | ||
| required: true | ||
| version: | ||
| description: 'The version of the binaries' | ||
| required: true | ||
| upload-to-s3: | ||
| description: 'Whether to upload the binaries to s3' | ||
| required: false | ||
| default: false | ||
|
|
||
| runs: | ||
| using: composite | ||
| steps: | ||
| - name: Check if binaries have changed | ||
| shell: bash | ||
| run: | | ||
| set -ex | ||
| python3 tools/clang-tidy-checks/check_s3.py \ | ||
| --s3-key "${{ inputs.platform }}/${{ inputs.version }}/clang-tidy" \ | ||
| --local-file clang-tidy | ||
| python3 tools/clang-tidy-checks/check_s3.py \ | ||
| --s3-key "${{ inputs.platform }}/${{ inputs.version }}/clang-format" \ | ||
| --local-file clang-format | ||
| - name: configure aws credentials | ||
| id: aws-credentials | ||
| uses: aws-actions/configure-aws-credentials@v4 | ||
| if: ${{ fromJSON(inputs.upload-to-s3) }} | ||
| with: | ||
| role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_clang_tidy_upload | ||
| aws-region: us-east-1 | ||
| output-credentials: true | ||
|
|
||
| - uses: seemethere/upload-artifact-s3@v5 | ||
| name: Publish clang-tidy binary | ||
| if: ${{ fromJSON(inputs.upload-to-s3) }} | ||
| with: | ||
| if-no-files-found: error | ||
| s3-prefix: ${{ inputs.platform }}/${{ inputs.version }} | ||
| s3-bucket: oss-clang-format | ||
| path: clang-tidy | ||
|
|
||
| - uses: seemethere/upload-artifact-s3@v5 | ||
| name: Publish clang-format binary | ||
| if: ${{ fromJSON(inputs.upload-to-s3) }} | ||
| with: | ||
| if-no-files-found: error | ||
| s3-prefix: ${{ inputs.platform }}/${{ inputs.version }} | ||
| s3-bucket: oss-clang-format | ||
| path: clang-format | ||
|
|
||
| - name: Upload clang-tidy to GHA | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: ${{ inputs.platform }}-${{ inputs.version }}-clang-tidy | ||
| retention-days: 14 | ||
| if-no-files-found: warn | ||
| path: clang-tidy | ||
|
|
||
| - name: Upload clang-format to GHA | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: ${{ inputs.platform }}-${{ inputs.version }}-clang-format | ||
| retention-days: 14 | ||
| if-no-files-found: warn | ||
| path: clang-format |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| """ | ||
| This script checks if a file on s3 doesn't exist or matches the local file. If | ||
| this script returns a non zero exit code, it means that the file you are trying | ||
| to upload will overwrite an existing file on s3. If the s3 path is used in | ||
| lintrunner on pytorch/pytorch, this will break `lintrunner init`. If this | ||
| returns with exit code 0, then it is safe to upload to s3 for usage with | ||
| lintrunner in pytorch/pytorch. If you upload a new file, remember to add the | ||
| s3 path and hash to the lintrunner s3 init config: | ||
| https://github.com/pytorch/pytorch/blob/915625307eeda338fef00c984e223c5774c00a2b/tools/linter/adapters/s3_init_config.json#L1 | ||
| """ | ||
| import hashlib | ||
| from urllib.request import Request, urlopen | ||
| import argparse | ||
| from urllib.error import HTTPError | ||
|
|
||
|
|
||
| def download_s3_file(s3_key): | ||
| url = f"https://oss-clang-format.s3.us-east-2.amazonaws.com/{s3_key}" | ||
| req = Request(url) | ||
| try: | ||
| with urlopen(req) as response: | ||
| # File exists, return the contents | ||
| return response.read() | ||
| except HTTPError as e: | ||
| if "The specified key does not exist" in e.read().decode(): | ||
| # Acceptable error, file can be uploaded safely without overwriting | ||
| print(f"Cannot find the file on s3") | ||
| return | ||
| raise | ||
|
|
||
|
|
||
| def hash_file(file): | ||
| with open(file, "rb") as f: | ||
| return hashlib.sha256(f.read()).hexdigest() | ||
|
|
||
|
|
||
| def check_s3_file(s3_key, local_file): | ||
| s3_file = download_s3_file(s3_key) | ||
| local_hash = hash_file(local_file) | ||
| if not s3_file: | ||
| print(f"Hash of local file: {local_hash}") | ||
| return | ||
| s3_hash = hashlib.sha256(s3_file).hexdigest() | ||
| if local_hash != s3_hash: | ||
| raise RuntimeError(f"Hash mismatch for {local_file}: {local_hash} != {s3_hash}") | ||
| print(f"Hashes for local file and remote file match: {local_hash}") | ||
|
|
||
|
|
||
| def parse_args(): | ||
| parser = argparse.ArgumentParser() | ||
| parser.add_argument("--s3-key", required=True) | ||
| parser.add_argument("--local-file", required=True) | ||
| return parser.parse_args() | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| args = parse_args() | ||
| check_s3_file(args.s3_key, args.local_file) |