Release #2038
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Release | |
| on: | |
| push: | |
| branches: | |
| - "release/*" | |
| schedule: | |
| # beijing: 2 am. | |
| # utc: 6 pm. | |
| - cron: "0 18 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| placeholder: | |
| description: "update .github/workflows/release.yml to config your build" | |
| required: false | |
| concurrency: | |
| group: release-${{ github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} | |
| env: | |
| ONEFLOW_SRC: . | |
| jobs: | |
| generate-build-matrix: | |
| name: "Generate build matrix" | |
| runs-on: ubuntu-latest | |
| env: | |
| ONEFLOW_SRC: . | |
| outputs: | |
| matrix: ${{ steps.find-cache.outputs.matrix }} | |
| formatted_date: ${{ steps.date.outputs.formatted_date }} | |
| steps: | |
| - name: Checkout Oneflow-Inc/oneflow | |
| uses: actions/checkout@v2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| repository: ${{github.event.pull_request.head.repo.full_name}} | |
| - uses: Oneflow-Inc/get-oneflow/cache-complete/matrix/build@add-nightly-date | |
| name: find cache | |
| id: find-cache | |
| timeout-minutes: 5 | |
| with: | |
| delete-cache: ${{ contains(github.event.pull_request.labels.*.name, 'need-clean-ccache') }} | |
| runner-labels: | | |
| self-hosted | |
| linux | |
| release | |
| oneflow-src: ${{ env.ONEFLOW_SRC }} | |
| entries: | | |
| cu118 | |
| cu117 | |
| cpu | |
| - name: Get current date | |
| id: date | |
| run: echo "formatted_date=$(date +'%Y%m%d')" >> $GITHUB_OUTPUT | |
| staging_release: | |
| env: | |
| MANYLINUX_CACHE_DIR: ~/manylinux-cache-dir/release/${{ matrix.entry }} | |
| WHEELHOUSE_DIR: manylinux_wheelhouse | |
| OSS_DIR: branch/${{ github.ref_name }}/${{ matrix.entry }}/${{ github.sha }} | |
| needs: [generate-build-matrix] | |
| name: Staging Release | |
| timeout-minutes: 180 | |
| runs-on: [self-hosted, linux, release] | |
| if: github.repository == 'Oneflow-Inc/oneflow' | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 5 | |
| matrix: ${{ fromJson(needs.generate-build-matrix.outputs.matrix) }} | |
| steps: | |
| - name: Fix permissions | |
| run: | | |
| docker run --rm -v $PWD:/p -w /p busybox rm -rf * | |
| - name: Install dependencies | |
| run: | | |
| python3 -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple | |
| python3 -m pip install -U setuptools wheel --user | |
| python3 -m pip install oss2 --user | |
| - uses: actions/checkout@v2 | |
| - uses: Oneflow-Inc/get-oneflow@add-nightly-date | |
| name: Build ${{ matrix.entry }} | |
| if: ${{ matrix.entry =='cu118' }} | |
| with: | |
| cmake-init-cache: ${{ env.ONEFLOW_SRC }}/cmake/caches/ci/release/${{ matrix.entry }}.cmake | |
| build-script: ${{ env.ONEFLOW_SRC }}/ci/manylinux/build-gcc9.sh | |
| oneflow-src: ${{ env.ONEFLOW_SRC }} | |
| oneflow-build-env: manylinux | |
| wheelhouse-dir: ${{ env.WHEELHOUSE_DIR }} | |
| clear-wheelhouse-dir: true | |
| self-hosted: true | |
| compute-platform: ${{ matrix.entry }} | |
| manylinux-cache-dir: ${{ env.MANYLINUX_CACHE_DIR }} | |
| docker-run-use-system-http-proxy: false | |
| docker-run-use-lld: false | |
| retry-failed-build: true | |
| clean-ccache: true | |
| nightly: ${{ github.event_name == 'schedule' || github.ref == 'refs/heads/release/add_nightly_date_index'}} | |
| nightly-date: ${{ needs.generate-build-matrix.outputs.formatted_date }} | |
| use-nvidia-wheels: ${{ matrix.entry !='cu112' }} | |
| python-versions: | | |
| 3.7 | |
| 3.8 | |
| 3.9 | |
| 3.10 | |
| - uses: Oneflow-Inc/get-oneflow@add-nightly-date | |
| name: Build ${{ matrix.entry }} | |
| if: ${{ matrix.entry !='cpu' && matrix.entry !='cu118' }} | |
| with: | |
| cmake-init-cache: ${{ env.ONEFLOW_SRC }}/cmake/caches/ci/release/cuda.cmake | |
| build-script: ${{ env.ONEFLOW_SRC }}/ci/manylinux/build-gcc9.sh | |
| oneflow-src: ${{ env.ONEFLOW_SRC }} | |
| oneflow-build-env: manylinux | |
| wheelhouse-dir: ${{ env.WHEELHOUSE_DIR }} | |
| clear-wheelhouse-dir: true | |
| self-hosted: true | |
| compute-platform: ${{ matrix.entry }} | |
| manylinux-cache-dir: ${{ env.MANYLINUX_CACHE_DIR }} | |
| docker-run-use-system-http-proxy: false | |
| docker-run-use-lld: false | |
| retry-failed-build: true | |
| clean-ccache: true | |
| nightly: ${{ github.event_name == 'schedule' || github.ref == 'refs/heads/release/add_nightly_date_index'}} | |
| nightly-date: ${{ needs.generate-build-matrix.outputs.formatted_date }} | |
| use-nvidia-wheels: ${{ matrix.entry !='cu112' }} | |
| python-versions: | | |
| 3.7 | |
| 3.8 | |
| 3.9 | |
| 3.10 | |
| - uses: Oneflow-Inc/get-oneflow@add-nightly-date | |
| name: Build ${{ matrix.entry }} | |
| if: ${{ matrix.entry =='cpu' }} | |
| with: | |
| cmake-init-cache: ${{ env.ONEFLOW_SRC }}/cmake/caches/ci/release/cpu.cmake | |
| build-script: ${{ env.ONEFLOW_SRC }}/ci/manylinux/build.sh | |
| oneflow-src: ${{ env.ONEFLOW_SRC }} | |
| oneflow-build-env: manylinux | |
| wheelhouse-dir: ${{ env.WHEELHOUSE_DIR }} | |
| clear-wheelhouse-dir: true | |
| self-hosted: true | |
| compute-platform: ${{ matrix.entry }} | |
| manylinux-cache-dir: ${{ env.MANYLINUX_CACHE_DIR }} | |
| docker-run-use-system-http-proxy: false | |
| docker-run-use-lld: false | |
| retry-failed-build: true | |
| clean-ccache: false | |
| nightly: ${{ github.event_name == 'schedule' || github.ref == 'refs/heads/release/add_nightly_date_index'}} | |
| nightly-date: ${{ needs.generate-build-matrix.outputs.formatted_date }} | |
| python-versions: | | |
| 3.7 | |
| 3.8 | |
| 3.9 | |
| 3.10 | |
| - name: Upload wheel | |
| uses: ./.github/actions/upload_oss | |
| with: | |
| src_path: ${{ env.WHEELHOUSE_DIR }} | |
| oss_dst_path: oss://oneflow-staging/${{ env.OSS_DIR }} | |
| oss_access_key_id: ${{ secrets.OSS_ACCESS_KEY_ID }} | |
| oss_access_key_secret: ${{ secrets.OSS_ACCESS_KEY_SECRET }} | |
| - name: Update pip index | |
| env: | |
| OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }} | |
| OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }} | |
| run: | | |
| python3 -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple | |
| python3 -m pip install oss2 beautifulsoup4 --user | |
| python3 tools/create_pip_index.py --dir_key ${{ env.OSS_DIR }} -b oneflow-staging \ | |
| --index_key=branch/${{ github.ref_name }}/${{ matrix.entry }}/index.html \ | |
| --index_key=branch/${{ github.ref_name }}/date/${{ needs.generate-build-matrix.outputs.formatted_date }}/${{ matrix.entry }}/index.html \ | |
| --index_key=${{ env.OSS_DIR }}/index.html \ | |
| --index_key=commit/${{ github.sha }}/${{ matrix.entry }}/index.html | |
| - name: Update API docs | |
| if: github.ref == 'refs/heads/master' && matrix.entry == 'cpu' | |
| env: | |
| READTHEDOCS_TOKEN: ${{ secrets.READTHEDOCS_TOKEN }} | |
| run: | | |
| curl -X POST -d "branches=master" -d "token=${READTHEDOCS_TOKEN}" https://readthedocs.org/api/v2/webhook/oneflow/135376/ | |
| pack_src: | |
| name: Pack source code | |
| if: github.ref == 'refs/heads/master' && github.repository == 'Oneflow-Inc/oneflow' | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Pack OneFlow source code | |
| env: | |
| OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }} | |
| OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }} | |
| run: | | |
| git reset --hard | |
| git clean -f | |
| git archive --prefix oneflow/ --format zip HEAD > oneflow-src.zip | |
| curl http://gosspublic.alicdn.com/ossutil/1.6.19/ossutil64 -o $HOME/ossutil64 | |
| chmod 755 $HOME/ossutil64 | |
| $HOME/ossutil64 config -e oss-cn-beijing.aliyuncs.com -i ${OSS_ACCESS_KEY_ID} -k ${OSS_ACCESS_KEY_SECRET} -L EN -c $HOME/.ossutilconfig | |
| $HOME/ossutil64 cp --update oneflow-src.zip oss://oneflow-public/oneflow-src.zip |