Upload model to S3 #5
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Upload model to S3 | |
| on: | |
| workflow_dispatch: | |
| workflow_run: | |
| workflows: ["Model test on push"] | |
| types: | |
| - completed | |
| jobs: | |
| upload_model_to_s3: | |
| if: ${{ github.repository != 'ersilia-os/eos-template' && github.event.workflow_run.conclusion == 'success' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout persist credentials | |
| uses: actions/checkout@master | |
| with: | |
| persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token | |
| fetch-depth: 0 # otherwise, you will failed to push refs to dest repo | |
| lfs: 'true' | |
| - name: Add conda to system path | |
| run: echo $CONDA/bin >> $GITHUB_PATH | |
| - name: Source conda | |
| run: source $CONDA/etc/profile.d/conda.sh | |
| - name: Set Python to 3.10.10 | |
| run: | |
| conda install -y python=3.10.10 | |
| - name: Install dependencies | |
| run: | | |
| source activate | |
| conda init | |
| conda install git-lfs -c conda-forge | |
| git-lfs install | |
| conda install gh -c conda-forge | |
| - name: Install ersilia | |
| run: | | |
| source activate | |
| python --version | |
| echo "After conda init" | |
| conda init | |
| python -m pip install git+https://github.com/ersilia-os/ersilia.git | |
| - name: Check metadata file | |
| id: checkMetadata | |
| continue-on-error: true | |
| run: | | |
| if [[ ! -f metadata.yml ]]; then | |
| echo "metadata.yml file not found" | |
| exit 1 | |
| fi | |
| - name: Update Metadata JSON file with DockerHub info | |
| if: steps.checkMetadata.outcome == 'failure' | |
| id: UpdateMetadataJSON | |
| run: | | |
| python3 -c " | |
| import json | |
| with open('metadata.json', 'r') as f: | |
| data = json.load(f) | |
| data['S3'] = 'https://ersilia-models-zipped.s3.eu-central-1.amazonaws.com/{0}.zip'.format(data['Identifier']) | |
| with open('metadata.json', 'w') as f: | |
| json.dump(data, f, indent=4) | |
| " | |
| - name: Update Metadata YAML file with DockerHub info | |
| if: steps.checkMetadata.outcome == 'success' | |
| id: UpdateMetadataYAML | |
| run: | | |
| python3 -c " | |
| import yaml | |
| with open('metadata.yml', 'r') as f: | |
| data = yaml.safe_load(f) | |
| data['S3'] = 'https://ersilia-models-zipped.s3.eu-central-1.amazonaws.com/{0}.zip'.format(data['Identifier']) | |
| with open('metadata.yml', 'w') as f: | |
| yaml.dump(data, f, default_flow_style=False, sort_keys=False) | |
| " | |
| - name: Commit and push changes done to the Metadata JSON file | |
| uses: actions-js/push@156f2b10c3aa000c44dbe75ea7018f32ae999772 # [email protected] | |
| with: | |
| author_name: "ersilia-bot" | |
| author_email: "[email protected]" | |
| message: "updating metadata [skip ci]" | |
| repository: "ersilia-os/${{ github.event.repository.name }}" | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| amend: true | |
| force: true | |
| - name: Update metadata to AirTable | |
| id: update-metadata-to-airtable | |
| env: | |
| USER_NAME: ${{ github.repository_owner }} | |
| BRANCH: "main" | |
| REPO_NAME: ${{ github.event.repository.name }} | |
| AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }} | |
| run: | | |
| source activate | |
| echo "Updating metadata to AirTable looking at owner: $USER_NAME" | |
| wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_metadata_to_airtable.py | |
| python3 update_metadata_to_airtable.py $USER_NAME $REPO_NAME $BRANCH $AIRTABLE_API_KEY | |
| rm update_metadata_to_airtable.py | |
| - name: Update README file | |
| id: update-readme-file | |
| env: | |
| MODEL_ID: ${{ github.event.repository.name }} | |
| run: | | |
| source activate | |
| echo "Updating README file with AirTable metadata for model: $MODEL_ID" | |
| wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_readme_from_airtable.py | |
| python3 update_readme_from_airtable.py $MODEL_ID . | |
| rm update_readme_from_airtable.py | |
| less README.md | |
| - name: Commit and push changes done to the README file | |
| uses: actions-js/push@156f2b10c3aa000c44dbe75ea7018f32ae999772 # [email protected] | |
| with: | |
| author_name: "ersilia-bot" | |
| author_email: "[email protected]" | |
| message: "updating readme [skip ci]" | |
| repository: "ersilia-os/${{ github.event.repository.name }}" | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| amend: true | |
| force: true | |
| - name: Upload model to S3 | |
| id: upload-model-to-s3 | |
| env: | |
| REPO_NAME: ${{ github.event.repository.name }} | |
| AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| run: | | |
| source activate | |
| echo "Uploading model to S3 bucket" | |
| wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/upload_model_to_s3.py | |
| python3 upload_model_to_s3.py $REPO_NAME $AWS_ACCESS_KEY $AWS_SECRET_ACCESS_KEY . | |
| rm upload_model_to_s3.py |