From 0ac729f670345d1f8c100c648d45ca1b1f95cde6 Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 11:09:41 +0800 Subject: [PATCH 01/68] refine --- .github/workflows/build.yml | 47 +++++++++++++++++++++++++++---------- build.sh | 5 ++-- 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ce2925b74..e4aafbcc3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,6 +1,7 @@ name: Build on: + workflow_dispatch: push: branches-ignore: - "update-dependencies-pr" @@ -18,6 +19,11 @@ concurrency: group: test-${{ github.ref }} cancel-in-progress: true +env: + REGION_ID: cn-beijing + REGISTRY: registry.cn-beijing.aliyuncs.com + NAMESPACE: oneflow + jobs: build_manylinux: name: ${{ matrix.policy }}_${{ matrix.platform }} @@ -25,22 +31,35 @@ jobs: strategy: fail-fast: false matrix: - policy: ["manylinux2014", "manylinux_2_24", "musllinux_1_1"] - platform: ["i686", "x86_64", "ppc64le"] + tag-prefix: ["cuda11.2"] include: - - policy: "manylinux2010" - platform: "i686" - - policy: "manylinux2010" - platform: "x86_64" + - tag-prefix: "cuda11.2" + - policy: "manylinux2014" + - platform: "x86_64" + - CUDA_BASE_IMAGE: "nvidia/cuda:11.2.2-cudnn8-devel-centos7" env: POLICY: ${{ matrix.policy }} PLATFORM: ${{ matrix.platform }} COMMIT_SHA: ${{ github.sha }} + DOCKERHUB_TAG: oneflowinc/${{ matrix.policy }}_${{ matrix.platform }}:${{ github.sha }} + ACR_TAG: registry.cn-beijing.aliyuncs.com/oneflow/${{ matrix.policy }}_${{ matrix.platform }}:${{ github.sha }} steps: - name: Checkout uses: actions/checkout@v2 + # 1.1 Login to ACR + - name: Login to ACR with the AccessKey pair + uses: aliyun/acr-login@v1 + with: + login-server: https://registry.${{env.REGION_ID}}.aliyuncs.com + username: "${{ secrets.ACR_USERNAME }}" + password: "${{ secrets.ACR_PASSWORD }}" + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up emulation if: matrix.platform == 'ppc64le' @@ -50,7 +69,7 @@ jobs: # - name: Set BuildKit for ppc64le # if: matrix.platform == 'ppc64le' - # run: echo 'MANYLINUX_BUILD_FRONTEND=buildkit' >> $GITHUB_ENV + # run: echo 'MANYLINUX_BUILD_FRONTEND=buildkit' >> $GITHUB_ENV - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 @@ -63,11 +82,13 @@ jobs: restore-keys: buildx-cache-${{ matrix.policy }}-${{ matrix.platform }}- - name: Build + env: + CUDA_BASE_IMAGE: ${{ matrix.CUDA_BASE_IMAGE }} run: ./build.sh - - name: Deploy - if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'pypa/manylinux' - run: ./deploy.sh - env: - QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} - QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }} + - name: Push image to DockerHub + run: | + docker push "${{ env.DOCKERHUB_TAG }}" + - name: Push image to ACR + run: | + docker push "${{ env.ACR_TAG }}" diff --git a/build.sh b/build.sh index 6d65ec345..9f6083d40 100755 --- a/build.sh +++ b/build.sh @@ -48,7 +48,7 @@ elif [ "${POLICY}" == "manylinux2014" ]; then if [ "${PLATFORM}" == "s390x" ]; then BASEIMAGE="s390x/clefos:7" else - BASEIMAGE="${MULTIARCH_PREFIX}centos:7" + BASEIMAGE="${CUDA_BASE_IMAGE}" fi DEVTOOLSET_ROOTPATH="/opt/rh/devtoolset-10/root" PREPEND_PATH="${DEVTOOLSET_ROOTPATH}/usr/bin:" @@ -79,7 +79,8 @@ export LD_LIBRARY_PATH_ARG BUILD_ARGS_COMMON=" --build-arg POLICY --build-arg PLATFORM --build-arg BASEIMAGE --build-arg DEVTOOLSET_ROOTPATH --build-arg PREPEND_PATH --build-arg LD_LIBRARY_PATH_ARG - --rm -t quay.io/pypa/${POLICY}_${PLATFORM}:${COMMIT_SHA} + --rm -t ${DOCKERHUB_TAG} + --rm -t ${ACR_TAG} -f docker/Dockerfile docker/ " From 2ce06006c0b8bfea464e7b354b61a0778f2cce0a Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 11:45:15 +0800 Subject: [PATCH 02/68] refine --- build.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 9f6083d40..6970de58d 100755 --- a/build.sh +++ b/build.sh @@ -79,8 +79,9 @@ export LD_LIBRARY_PATH_ARG BUILD_ARGS_COMMON=" --build-arg POLICY --build-arg PLATFORM --build-arg BASEIMAGE --build-arg DEVTOOLSET_ROOTPATH --build-arg PREPEND_PATH --build-arg LD_LIBRARY_PATH_ARG - --rm -t ${DOCKERHUB_TAG} - --rm -t ${ACR_TAG} + --rm + -t ${DOCKERHUB_TAG} + -t ${ACR_TAG} -f docker/Dockerfile docker/ " From d9858f4107787081d8056fc9505c3e931eec5ae0 Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 11:50:14 +0800 Subject: [PATCH 03/68] refine --- .github/workflows/build.yml | 8 ++++---- build.sh | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e4aafbcc3..addad6b1a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,9 +31,9 @@ jobs: strategy: fail-fast: false matrix: - tag-prefix: ["cuda11.2"] + tag-suffix: ["cuda11.2"] include: - - tag-prefix: "cuda11.2" + - tag-suffix: "cuda11.2" - policy: "manylinux2014" - platform: "x86_64" - CUDA_BASE_IMAGE: "nvidia/cuda:11.2.2-cudnn8-devel-centos7" @@ -42,8 +42,8 @@ jobs: POLICY: ${{ matrix.policy }} PLATFORM: ${{ matrix.platform }} COMMIT_SHA: ${{ github.sha }} - DOCKERHUB_TAG: oneflowinc/${{ matrix.policy }}_${{ matrix.platform }}:${{ github.sha }} - ACR_TAG: registry.cn-beijing.aliyuncs.com/oneflow/${{ matrix.policy }}_${{ matrix.platform }}:${{ github.sha }} + DOCKERHUB_TAG: oneflowinc/${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}:${{ github.sha }} + ACR_TAG: registry.cn-beijing.aliyuncs.com/oneflow/${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}:${{ github.sha }} steps: - name: Checkout diff --git a/build.sh b/build.sh index 6970de58d..cc0a7d2aa 100755 --- a/build.sh +++ b/build.sh @@ -113,7 +113,7 @@ else exit 1 fi -docker run --rm -v $(pwd)/tests:/tests:ro quay.io/pypa/${POLICY}_${PLATFORM}:${COMMIT_SHA} /tests/run_tests.sh +docker run --rm -v $(pwd)/tests:/tests:ro ${DOCKERHUB_TAG} /tests/run_tests.sh if [ "${MANYLINUX_BUILD_FRONTEND}" != "docker" ]; then if [ -d $(pwd)/.buildx-cache-${POLICY}_${PLATFORM} ]; then From 67daa2a74fd6757b0455234bb45c25ca1e04ce9f Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 11:51:17 +0800 Subject: [PATCH 04/68] refine --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index addad6b1a..2fe59a447 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -78,8 +78,8 @@ jobs: uses: actions/cache@v2 with: path: .buildx-cache-${{ matrix.policy }}_${{ matrix.platform }}/* - key: buildx-cache-${{ matrix.policy }}-${{ matrix.platform }}-${{ hashFiles('docker/**') }} - restore-keys: buildx-cache-${{ matrix.policy }}-${{ matrix.platform }}- + key: buildx-cache-${{ matrix.policy }}-${{ matrix.platform }}-${{ matrix.tag-suffix }}-${{ hashFiles('docker/**') }} + restore-keys: buildx-cache-${{ matrix.policy }}-${{ matrix.platform }}-${{ matrix.tag-suffix }} - name: Build env: From c8509b29575331f9306dd1a3387a7d1dfea0b9f3 Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 16:02:42 +0800 Subject: [PATCH 05/68] refine --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2fe59a447..8825564a4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,8 +42,8 @@ jobs: POLICY: ${{ matrix.policy }} PLATFORM: ${{ matrix.platform }} COMMIT_SHA: ${{ github.sha }} - DOCKERHUB_TAG: oneflowinc/${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}:${{ github.sha }} - ACR_TAG: registry.cn-beijing.aliyuncs.com/oneflow/${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}:${{ github.sha }} + DOCKERHUB_TAG: "oneflowinc/${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}:${{ github.sha }}" + ACR_TAG: "registry.cn-beijing.aliyuncs.com/oneflow/${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}:${{ github.sha }}" steps: - name: Checkout From 342da77c5171438063a9d1a451299d8520065802 Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 16:10:11 +0800 Subject: [PATCH 06/68] refine --- .github/workflows/build.yml | 12 ++++++------ build.sh | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8825564a4..233f50108 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,8 +21,9 @@ concurrency: env: REGION_ID: cn-beijing - REGISTRY: registry.cn-beijing.aliyuncs.com - NAMESPACE: oneflow + ACR_REGISTRY: registry.cn-beijing.aliyuncs.com + ACR_NAMESPACE: oneflow + DOCKER_HUB_NAMESPACE: oneflowinc jobs: build_manylinux: @@ -42,8 +43,7 @@ jobs: POLICY: ${{ matrix.policy }} PLATFORM: ${{ matrix.platform }} COMMIT_SHA: ${{ github.sha }} - DOCKERHUB_TAG: "oneflowinc/${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}:${{ github.sha }}" - ACR_TAG: "registry.cn-beijing.aliyuncs.com/oneflow/${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}:${{ github.sha }}" + TAG: "${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}:${{ github.sha }}" steps: - name: Checkout @@ -88,7 +88,7 @@ jobs: - name: Push image to DockerHub run: | - docker push "${{ env.DOCKERHUB_TAG }}" + docker push ${{ env.DOCKER_HUB_NAMESPACE }}/${{ env.TAG }} - name: Push image to ACR run: | - docker push "${{ env.ACR_TAG }}" + docker push ${{ env.ACR_REGISTRY }}/${{ env.ACR_NAMESPACE }}/${{ env.TAG }} diff --git a/build.sh b/build.sh index cc0a7d2aa..126f0b208 100755 --- a/build.sh +++ b/build.sh @@ -80,8 +80,8 @@ BUILD_ARGS_COMMON=" --build-arg POLICY --build-arg PLATFORM --build-arg BASEIMAGE --build-arg DEVTOOLSET_ROOTPATH --build-arg PREPEND_PATH --build-arg LD_LIBRARY_PATH_ARG --rm - -t ${DOCKERHUB_TAG} - -t ${ACR_TAG} + -t ${DOCKER_HUB_NAMESPACE}/${TAG} + -t ${ACR_REGISTRY}/${ACR_NAMESPACE}/${TAG} -f docker/Dockerfile docker/ " @@ -113,7 +113,7 @@ else exit 1 fi -docker run --rm -v $(pwd)/tests:/tests:ro ${DOCKERHUB_TAG} /tests/run_tests.sh +docker run --rm -v $(pwd)/tests:/tests:ro ${DOCKER_HUB_NAMESPACE}/${TAG} /tests/run_tests.sh if [ "${MANYLINUX_BUILD_FRONTEND}" != "docker" ]; then if [ -d $(pwd)/.buildx-cache-${POLICY}_${PLATFORM} ]; then From f37d281acf1fc32f865fd7b9634ef203d4942e85 Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 16:12:55 +0800 Subject: [PATCH 07/68] refine --- build.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 126f0b208..7c87bf656 100755 --- a/build.sh +++ b/build.sh @@ -76,11 +76,13 @@ export DEVTOOLSET_ROOTPATH export PREPEND_PATH export LD_LIBRARY_PATH_ARG +# For strange reasons, 'oneflowinc' will be redacted in github action +# for instance, ***/manylinux2014_x86_64_cuda11.2:342da77c5171438063a9d1a451299d8520065802 BUILD_ARGS_COMMON=" --build-arg POLICY --build-arg PLATFORM --build-arg BASEIMAGE --build-arg DEVTOOLSET_ROOTPATH --build-arg PREPEND_PATH --build-arg LD_LIBRARY_PATH_ARG --rm - -t ${DOCKER_HUB_NAMESPACE}/${TAG} + -t oneflowinc/${TAG} -t ${ACR_REGISTRY}/${ACR_NAMESPACE}/${TAG} -f docker/Dockerfile docker/ " @@ -113,7 +115,7 @@ else exit 1 fi -docker run --rm -v $(pwd)/tests:/tests:ro ${DOCKER_HUB_NAMESPACE}/${TAG} /tests/run_tests.sh +docker run --rm -v $(pwd)/tests:/tests:ro oneflowinc/${TAG} /tests/run_tests.sh if [ "${MANYLINUX_BUILD_FRONTEND}" != "docker" ]; then if [ -d $(pwd)/.buildx-cache-${POLICY}_${PLATFORM} ]; then From 655317f5699b76573d48192b13e6d94b2fd1a0e2 Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 16:21:57 +0800 Subject: [PATCH 08/68] refine --- build.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/build.sh b/build.sh index 7c87bf656..d67fb2a3d 100755 --- a/build.sh +++ b/build.sh @@ -81,9 +81,7 @@ export LD_LIBRARY_PATH_ARG BUILD_ARGS_COMMON=" --build-arg POLICY --build-arg PLATFORM --build-arg BASEIMAGE --build-arg DEVTOOLSET_ROOTPATH --build-arg PREPEND_PATH --build-arg LD_LIBRARY_PATH_ARG - --rm - -t oneflowinc/${TAG} - -t ${ACR_REGISTRY}/${ACR_NAMESPACE}/${TAG} + --rm -t oneflowinc/${TAG} -t ${ACR_REGISTRY}/${ACR_NAMESPACE}/${TAG} -f docker/Dockerfile docker/ " From d7f9d687025cb7ab617c83bdcf523b18576db979 Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 16:25:52 +0800 Subject: [PATCH 09/68] refine --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 233f50108..a1e9333c0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,7 +27,7 @@ env: jobs: build_manylinux: - name: ${{ matrix.policy }}_${{ matrix.platform }} + name: ${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }} runs-on: ubuntu-20.04 strategy: fail-fast: false From 585ce28cd41aa977f84c16b7b730096ae96bc9b4 Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 16:32:15 +0800 Subject: [PATCH 10/68] fix --- build.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build.sh b/build.sh index d67fb2a3d..6a31e98e4 100755 --- a/build.sh +++ b/build.sh @@ -76,12 +76,12 @@ export DEVTOOLSET_ROOTPATH export PREPEND_PATH export LD_LIBRARY_PATH_ARG -# For strange reasons, 'oneflowinc' will be redacted in github action -# for instance, ***/manylinux2014_x86_64_cuda11.2:342da77c5171438063a9d1a451299d8520065802 BUILD_ARGS_COMMON=" --build-arg POLICY --build-arg PLATFORM --build-arg BASEIMAGE --build-arg DEVTOOLSET_ROOTPATH --build-arg PREPEND_PATH --build-arg LD_LIBRARY_PATH_ARG - --rm -t oneflowinc/${TAG} -t ${ACR_REGISTRY}/${ACR_NAMESPACE}/${TAG} + --rm + -t ${DOCKER_HUB_NAMESPACE}/${TAG} + -t ${ACR_REGISTRY}/${ACR_NAMESPACE}/${TAG} -f docker/Dockerfile docker/ " @@ -107,13 +107,13 @@ elif [ "${MANYLINUX_BUILD_FRONTEND}" == "buildkit" ]; then --export-cache type=local,dest=$(pwd)/.buildx-cache-staging-${POLICY}_${PLATFORM} \ --opt build-arg:POLICY=${POLICY} --opt build-arg:PLATFORM=${PLATFORM} --opt build-arg:BASEIMAGE=${BASEIMAGE} \ --opt "build-arg:DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH}" --opt "build-arg:PREPEND_PATH=${PREPEND_PATH}" --opt "build-arg:LD_LIBRARY_PATH_ARG=${LD_LIBRARY_PATH_ARG}" \ - --output type=docker,name=quay.io/pypa/${POLICY}_${PLATFORM}:${COMMIT_SHA} | docker load + --output type=docker,name=${DOCKER_HUB_NAMESPACE}/${TAG} | docker load else echo "Unsupported build frontend: '${MANYLINUX_BUILD_FRONTEND}'" exit 1 fi -docker run --rm -v $(pwd)/tests:/tests:ro oneflowinc/${TAG} /tests/run_tests.sh +docker run --rm -v $(pwd)/tests:/tests:ro ${DOCKER_HUB_NAMESPACE}/${TAG} /tests/run_tests.sh if [ "${MANYLINUX_BUILD_FRONTEND}" != "docker" ]; then if [ -d $(pwd)/.buildx-cache-${POLICY}_${PLATFORM} ]; then From a131ab76de0702a3bf111c012e9c75ca46c4d5db Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 16:52:18 +0800 Subject: [PATCH 11/68] refine --- .github/workflows/build.yml | 5 +++-- build.sh | 16 ++++++++++------ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a1e9333c0..fdd5bbd33 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,7 +43,7 @@ jobs: POLICY: ${{ matrix.policy }} PLATFORM: ${{ matrix.platform }} COMMIT_SHA: ${{ github.sha }} - TAG: "${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}:${{ github.sha }}" + DOCKER_REPO: "${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }} steps: - name: Checkout @@ -88,7 +88,8 @@ jobs: - name: Push image to DockerHub run: | - docker push ${{ env.DOCKER_HUB_NAMESPACE }}/${{ env.TAG }} + docker push ${{ env.DOCKER_HUB_NAMESPACE }}/${{ env.DOCKER_REPO }}:${{ github.sha }} + docker push ${{ env.DOCKER_HUB_NAMESPACE }}/${{ env.DOCKER_REPO }}:latest - name: Push image to ACR run: | docker push ${{ env.ACR_REGISTRY }}/${{ env.ACR_NAMESPACE }}/${{ env.TAG }} diff --git a/build.sh b/build.sh index 6a31e98e4..93df76c7f 100755 --- a/build.sh +++ b/build.sh @@ -80,8 +80,10 @@ BUILD_ARGS_COMMON=" --build-arg POLICY --build-arg PLATFORM --build-arg BASEIMAGE --build-arg DEVTOOLSET_ROOTPATH --build-arg PREPEND_PATH --build-arg LD_LIBRARY_PATH_ARG --rm - -t ${DOCKER_HUB_NAMESPACE}/${TAG} - -t ${ACR_REGISTRY}/${ACR_NAMESPACE}/${TAG} + -t ${DOCKER_HUB_NAMESPACE}/${DOCKER_REPO}:${COMMIT_SHA} + -t ${DOCKER_HUB_NAMESPACE}/${DOCKER_REPO}:latest + -t ${ACR_REGISTRY}/${ACR_NAMESPACE}/${DOCKER_REPO}:${COMMIT_SHA} + -t ${ACR_REGISTRY}/${ACR_NAMESPACE}/${DOCKER_REPO}:latest -f docker/Dockerfile docker/ " @@ -95,10 +97,12 @@ if [ "${MANYLINUX_BUILD_FRONTEND}" == "docker" ]; then elif [ "${MANYLINUX_BUILD_FRONTEND}" == "docker-buildx" ]; then docker buildx build \ --load \ - --cache-from=type=local,src=$(pwd)/.buildx-cache-${POLICY}_${PLATFORM} \ - --cache-to=type=local,dest=$(pwd)/.buildx-cache-staging-${POLICY}_${PLATFORM} \ + --cache-from=type=type=registry,ref=${DOCKER_HUB_NAMESPACE}/${DOCKER_REPO}:latest \ + --cache-to=type=inline \ ${BUILD_ARGS_COMMON} elif [ "${MANYLINUX_BUILD_FRONTEND}" == "buildkit" ]; then + echo "Unsupported build frontend: buildkit" + exit 1 buildctl build \ --frontend=dockerfile.v0 \ --local context=./docker/ \ @@ -107,13 +111,13 @@ elif [ "${MANYLINUX_BUILD_FRONTEND}" == "buildkit" ]; then --export-cache type=local,dest=$(pwd)/.buildx-cache-staging-${POLICY}_${PLATFORM} \ --opt build-arg:POLICY=${POLICY} --opt build-arg:PLATFORM=${PLATFORM} --opt build-arg:BASEIMAGE=${BASEIMAGE} \ --opt "build-arg:DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH}" --opt "build-arg:PREPEND_PATH=${PREPEND_PATH}" --opt "build-arg:LD_LIBRARY_PATH_ARG=${LD_LIBRARY_PATH_ARG}" \ - --output type=docker,name=${DOCKER_HUB_NAMESPACE}/${TAG} | docker load + --output type=docker,name=${DOCKER_HUB_NAMESPACE}/${DOCKER_REPO}:latest | docker load else echo "Unsupported build frontend: '${MANYLINUX_BUILD_FRONTEND}'" exit 1 fi -docker run --rm -v $(pwd)/tests:/tests:ro ${DOCKER_HUB_NAMESPACE}/${TAG} /tests/run_tests.sh +docker run --rm -v $(pwd)/tests:/tests:ro ${DOCKER_HUB_NAMESPACE}/${DOCKER_REPO}:latest /tests/run_tests.sh if [ "${MANYLINUX_BUILD_FRONTEND}" != "docker" ]; then if [ -d $(pwd)/.buildx-cache-${POLICY}_${PLATFORM} ]; then From 0634ea7ae178e31286e1b653d62eaaf62cef74de Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 16:52:56 +0800 Subject: [PATCH 12/68] refine --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fdd5bbd33..5f4f2c384 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,7 +43,7 @@ jobs: POLICY: ${{ matrix.policy }} PLATFORM: ${{ matrix.platform }} COMMIT_SHA: ${{ github.sha }} - DOCKER_REPO: "${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }} + DOCKER_REPO: "${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}" steps: - name: Checkout From e01504a78e890792bb3e409a59d3bd5e15c41e92 Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 17:32:40 +0800 Subject: [PATCH 13/68] refine --- build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 93df76c7f..19d53be55 100755 --- a/build.sh +++ b/build.sh @@ -111,13 +111,13 @@ elif [ "${MANYLINUX_BUILD_FRONTEND}" == "buildkit" ]; then --export-cache type=local,dest=$(pwd)/.buildx-cache-staging-${POLICY}_${PLATFORM} \ --opt build-arg:POLICY=${POLICY} --opt build-arg:PLATFORM=${PLATFORM} --opt build-arg:BASEIMAGE=${BASEIMAGE} \ --opt "build-arg:DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH}" --opt "build-arg:PREPEND_PATH=${PREPEND_PATH}" --opt "build-arg:LD_LIBRARY_PATH_ARG=${LD_LIBRARY_PATH_ARG}" \ - --output type=docker,name=${DOCKER_HUB_NAMESPACE}/${DOCKER_REPO}:latest | docker load + --output type=docker,name=quay.io/pypa/${POLICY}_${PLATFORM}:${COMMIT_SHA} | docker load else echo "Unsupported build frontend: '${MANYLINUX_BUILD_FRONTEND}'" exit 1 fi -docker run --rm -v $(pwd)/tests:/tests:ro ${DOCKER_HUB_NAMESPACE}/${DOCKER_REPO}:latest /tests/run_tests.sh +docker run --rm -v $(pwd)/tests:/tests:ro ${DOCKER_HUB_NAMESPACE}/${DOCKER_REPO}:${COMMIT_SHA} /tests/run_tests.sh if [ "${MANYLINUX_BUILD_FRONTEND}" != "docker" ]; then if [ -d $(pwd)/.buildx-cache-${POLICY}_${PLATFORM} ]; then From a603ce68d3be23e0a4294efc904ef9ad28653b02 Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 17:42:50 +0800 Subject: [PATCH 14/68] refine --- build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 19d53be55..e2d6103fd 100755 --- a/build.sh +++ b/build.sh @@ -80,10 +80,10 @@ BUILD_ARGS_COMMON=" --build-arg POLICY --build-arg PLATFORM --build-arg BASEIMAGE --build-arg DEVTOOLSET_ROOTPATH --build-arg PREPEND_PATH --build-arg LD_LIBRARY_PATH_ARG --rm - -t ${DOCKER_HUB_NAMESPACE}/${DOCKER_REPO}:${COMMIT_SHA} - -t ${DOCKER_HUB_NAMESPACE}/${DOCKER_REPO}:latest -t ${ACR_REGISTRY}/${ACR_NAMESPACE}/${DOCKER_REPO}:${COMMIT_SHA} -t ${ACR_REGISTRY}/${ACR_NAMESPACE}/${DOCKER_REPO}:latest + -t ${DOCKER_HUB_NAMESPACE}/${DOCKER_REPO}:${COMMIT_SHA} + -t ${DOCKER_HUB_NAMESPACE}/${DOCKER_REPO}:latest -f docker/Dockerfile docker/ " From 44a139c26a0c167f8777a2e690c0a374aee35fed Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 17:43:12 +0800 Subject: [PATCH 15/68] debug --- docker/Dockerfile | 168 ---------------------------------------------- 1 file changed, 168 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index c92ea10fa..d2e770100 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,172 +1,4 @@ # default to latest supported policy, x86_64 ARG BASEIMAGE=amd64/debian:9 -ARG POLICY=manylinux_2_24 -ARG PLATFORM=x86_64 -ARG DEVTOOLSET_ROOTPATH= -ARG LD_LIBRARY_PATH_ARG= -ARG PREPEND_PATH= - -FROM $BASEIMAGE AS runtime_base -ARG POLICY -ARG PLATFORM -ARG DEVTOOLSET_ROOTPATH -ARG LD_LIBRARY_PATH_ARG -ARG PREPEND_PATH -LABEL maintainer="The ManyLinux project" - -ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM} -ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 -ENV DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH} -ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH_ARG} -ENV PATH=${PREPEND_PATH}${PATH} -ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig - -# first copy the fixup mirrors script, keep the script around -COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors - -# setup entrypoint, this will wrap commands with `linux32` with i686 images -COPY build_scripts/install-entrypoint.sh \ - build_scripts/update-system-packages.sh \ - build_scripts/build_utils.sh \ - /build_scripts/ - -RUN /build_scripts/install-entrypoint.sh && rm -rf /build_scripts -COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint -ENTRYPOINT ["manylinux-entrypoint"] - -COPY build_scripts/install-runtime-packages.sh \ - build_scripts/update-system-packages.sh \ - build_scripts/build_utils.sh \ - /build_scripts/ -RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/ - -COPY build_scripts/build_utils.sh /build_scripts/ - -COPY build_scripts/install-autoconf.sh /build_scripts/ -RUN export AUTOCONF_ROOT=autoconf-2.71 && \ - export AUTOCONF_HASH=431075ad0bf529ef13cb41e9042c542381103e80015686222b8a9d4abef42a1c && \ - export AUTOCONF_DOWNLOAD_URL=http://ftp.gnu.org/gnu/autoconf && \ - manylinux-entrypoint /build_scripts/install-autoconf.sh - -COPY build_scripts/install-automake.sh /build_scripts/ -RUN export AUTOMAKE_ROOT=automake-1.16.5 && \ - export AUTOMAKE_HASH=07bd24ad08a64bc17250ce09ec56e921d6343903943e99ccf63bbf0705e34605 && \ - export AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake && \ - manylinux-entrypoint /build_scripts/install-automake.sh - -COPY build_scripts/install-libtool.sh /build_scripts/ -RUN export LIBTOOL_ROOT=libtool-2.4.6 && \ - export LIBTOOL_HASH=e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3 && \ - export LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool && \ - manylinux-entrypoint /build_scripts/install-libtool.sh - -COPY build_scripts/install-libxcrypt.sh /build_scripts/ -RUN export LIBXCRYPT_VERSION=4.4.27 && \ - export LIBXCRYPT_HASH=e4ff47b7fa5370fa08e27258f8a0de5cb7e9b09c297ec7ab56a8a82e229a1c84 && \ - export LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive && \ - export PERL_ROOT=perl-5.34.0 && \ - export PERL_HASH=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a && \ - export PERL_DOWNLOAD_URL=https://www.cpan.org/src/5.0 && \ - manylinux-entrypoint /build_scripts/install-libxcrypt.sh - -FROM runtime_base AS build_base -COPY build_scripts/install-build-packages.sh /build_scripts/ -RUN manylinux-entrypoint /build_scripts/install-build-packages.sh - - -FROM build_base AS build_git -COPY build_scripts/build-git.sh /build_scripts/ -RUN export GIT_ROOT=git-2.34.1 && \ - export GIT_HASH=fc4eb5ecb9299db91cdd156c06cdeb41833f53adc5631ddf8c0cb13eaa2911c1 && \ - export GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git && \ - manylinux-entrypoint /build_scripts/build-git.sh - - -FROM build_base AS build_swig -COPY build_scripts/build-swig.sh /build_scripts/ -RUN export SWIG_ROOT=swig-4.0.2 && \ - export SWIG_HASH=d53be9730d8d58a16bf0cbd1f8ac0c0c3e1090573168bfa151b01eb47fa906fc && \ - export SWIG_DOWNLOAD_URL=https://sourceforge.net/projects/swig/files/swig/${SWIG_ROOT} && \ - export PCRE_ROOT=pcre-8.45 && \ - export PCRE_HASH=4e6ce03e0336e8b4a3d6c2b70b1c5e18590a5673a98186da90d4f33c23defc09 && \ - export PCRE_DOWNLOAD_URL=https://sourceforge.net/projects/pcre/files/pcre/8.45 && \ - manylinux-entrypoint /build_scripts/build-swig.sh - - -FROM build_base AS build_cpython -COPY build_scripts/build-sqlite3.sh /build_scripts/ -RUN export SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3370200 && \ - export SQLITE_AUTOCONF_HASH=4089a8d9b467537b3f246f217b84cd76e00b1d1a971fe5aca1e30e230e46b2d8 && \ - export SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2022 && \ - manylinux-entrypoint /build_scripts/build-sqlite3.sh - -COPY build_scripts/build-openssl.sh /build_scripts/ -RUN export OPENSSL_ROOT=openssl-1.1.1m && \ - export OPENSSL_HASH=f89199be8b23ca45fc7cb9f1d8d3ee67312318286ad030f5316aca6462db6c96 && \ - export OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source && \ - manylinux-entrypoint /build_scripts/build-openssl.sh - -COPY build_scripts/build-cpython.sh /build_scripts/ - - -FROM build_cpython AS build_cpython36 -COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.6.15 - - -FROM build_cpython AS build_cpython37 -COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.7.12 - - -FROM build_cpython AS build_cpython38 -COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.8.12 - - -FROM build_cpython AS build_cpython39 -COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.9.10 - - -FROM build_cpython AS build_cpython310 -COPY build_scripts/cpython-pubkey-310-311.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.10.2 - - -FROM build_cpython AS all_python -COPY build_scripts/install-pypy.sh \ - build_scripts/pypy.sha256 \ - build_scripts/finalize-python.sh \ - /build_scripts/ -RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.7 7.3.7 -RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.8 7.3.7 -COPY --from=build_cpython36 /opt/_internal /opt/_internal/ -COPY --from=build_cpython37 /opt/_internal /opt/_internal/ -COPY --from=build_cpython38 /opt/_internal /opt/_internal/ -COPY --from=build_cpython39 /opt/_internal /opt/_internal/ -COPY --from=build_cpython310 /opt/_internal /opt/_internal/ -RUN manylinux-entrypoint /build_scripts/finalize-python.sh - - -FROM runtime_base -COPY --from=build_git /manylinux-rootfs / -COPY --from=build_swig /manylinux-rootfs / -COPY --from=build_cpython /manylinux-rootfs / -COPY --from=all_python /opt/_internal /opt/_internal/ -COPY build_scripts/finalize.sh \ - build_scripts/update-system-packages.sh \ - build_scripts/python-tag-abi-tag.py \ - build_scripts/requirements3.6.txt \ - build_scripts/requirements3.7.txt \ - build_scripts/requirements3.8.txt \ - build_scripts/requirements3.9.txt \ - build_scripts/requirements3.10.txt \ - build_scripts/requirements-base-tools.txt \ - /build_scripts/ -COPY build_scripts/requirements-tools/* /build_scripts/requirements-tools/ -RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts - -ENV SSL_CERT_FILE=/opt/_internal/certs.pem CMD ["/bin/bash"] From 5c7a46685f8910c5524d127e5497d4a55fe5b518 Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 17:45:05 +0800 Subject: [PATCH 16/68] debug --- docker/Dockerfile | 153 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index d2e770100..de38919dd 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,157 @@ # default to latest supported policy, x86_64 ARG BASEIMAGE=amd64/debian:9 +ARG POLICY=manylinux_2_24 +ARG PLATFORM=x86_64 +ARG DEVTOOLSET_ROOTPATH= +ARG LD_LIBRARY_PATH_ARG= +ARG PREPEND_PATH= + +FROM $BASEIMAGE AS runtime_base +ARG POLICY +ARG PLATFORM +ARG DEVTOOLSET_ROOTPATH +ARG LD_LIBRARY_PATH_ARG +ARG PREPEND_PATH +LABEL maintainer="The ManyLinux project" + +ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM} +ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 +ENV DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH} +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH_ARG} +ENV PATH=${PREPEND_PATH}${PATH} +ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig + +# first copy the fixup mirrors script, keep the script around +COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors + +# setup entrypoint, this will wrap commands with `linux32` with i686 images +COPY build_scripts/install-entrypoint.sh \ + build_scripts/update-system-packages.sh \ + build_scripts/build_utils.sh \ + /build_scripts/ + +RUN /build_scripts/install-entrypoint.sh && rm -rf /build_scripts +COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint +ENTRYPOINT ["manylinux-entrypoint"] + +COPY build_scripts/install-runtime-packages.sh \ + build_scripts/update-system-packages.sh \ + build_scripts/build_utils.sh \ + /build_scripts/ +RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/ + +COPY build_scripts/build_utils.sh /build_scripts/ + +COPY build_scripts/install-autoconf.sh /build_scripts/ +RUN export AUTOCONF_ROOT=autoconf-2.71 && \ + export AUTOCONF_HASH=431075ad0bf529ef13cb41e9042c542381103e80015686222b8a9d4abef42a1c && \ + export AUTOCONF_DOWNLOAD_URL=http://ftp.gnu.org/gnu/autoconf && \ + manylinux-entrypoint /build_scripts/install-autoconf.sh + +COPY build_scripts/install-automake.sh /build_scripts/ +RUN export AUTOMAKE_ROOT=automake-1.16.5 && \ + export AUTOMAKE_HASH=07bd24ad08a64bc17250ce09ec56e921d6343903943e99ccf63bbf0705e34605 && \ + export AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake && \ + manylinux-entrypoint /build_scripts/install-automake.sh + +COPY build_scripts/install-libtool.sh /build_scripts/ +RUN export LIBTOOL_ROOT=libtool-2.4.6 && \ + export LIBTOOL_HASH=e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3 && \ + export LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool && \ + manylinux-entrypoint /build_scripts/install-libtool.sh + +COPY build_scripts/install-libxcrypt.sh /build_scripts/ +RUN export LIBXCRYPT_VERSION=4.4.27 && \ + export LIBXCRYPT_HASH=e4ff47b7fa5370fa08e27258f8a0de5cb7e9b09c297ec7ab56a8a82e229a1c84 && \ + export LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive && \ + export PERL_ROOT=perl-5.34.0 && \ + export PERL_HASH=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a && \ + export PERL_DOWNLOAD_URL=https://www.cpan.org/src/5.0 && \ + manylinux-entrypoint /build_scripts/install-libxcrypt.sh + +FROM runtime_base AS build_base +COPY build_scripts/install-build-packages.sh /build_scripts/ +RUN manylinux-entrypoint /build_scripts/install-build-packages.sh + + +FROM build_base AS build_git +COPY build_scripts/build-git.sh /build_scripts/ +RUN export GIT_ROOT=git-2.34.1 && \ + export GIT_HASH=fc4eb5ecb9299db91cdd156c06cdeb41833f53adc5631ddf8c0cb13eaa2911c1 && \ + export GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git && \ + manylinux-entrypoint /build_scripts/build-git.sh + + +FROM build_base AS build_swig +COPY build_scripts/build-swig.sh /build_scripts/ +RUN export SWIG_ROOT=swig-4.0.2 && \ + export SWIG_HASH=d53be9730d8d58a16bf0cbd1f8ac0c0c3e1090573168bfa151b01eb47fa906fc && \ + export SWIG_DOWNLOAD_URL=https://sourceforge.net/projects/swig/files/swig/${SWIG_ROOT} && \ + export PCRE_ROOT=pcre-8.45 && \ + export PCRE_HASH=4e6ce03e0336e8b4a3d6c2b70b1c5e18590a5673a98186da90d4f33c23defc09 && \ + export PCRE_DOWNLOAD_URL=https://sourceforge.net/projects/pcre/files/pcre/8.45 && \ + manylinux-entrypoint /build_scripts/build-swig.sh + + +FROM build_base AS build_cpython +COPY build_scripts/build-sqlite3.sh /build_scripts/ +RUN export SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3370200 && \ + export SQLITE_AUTOCONF_HASH=4089a8d9b467537b3f246f217b84cd76e00b1d1a971fe5aca1e30e230e46b2d8 && \ + export SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2022 && \ + manylinux-entrypoint /build_scripts/build-sqlite3.sh + +COPY build_scripts/build-openssl.sh /build_scripts/ +RUN export OPENSSL_ROOT=openssl-1.1.1m && \ + export OPENSSL_HASH=f89199be8b23ca45fc7cb9f1d8d3ee67312318286ad030f5316aca6462db6c96 && \ + export OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source && \ + manylinux-entrypoint /build_scripts/build-openssl.sh + +COPY build_scripts/build-cpython.sh /build_scripts/ + + +FROM build_cpython AS build_cpython36 +COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt +RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.6.15 + + +FROM build_cpython AS build_cpython37 +COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt +RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.7.12 + + +FROM build_cpython AS build_cpython38 +COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt +RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.8.12 + + +FROM build_cpython AS build_cpython39 +COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt +RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.9.10 + + +FROM build_cpython AS build_cpython310 +COPY build_scripts/cpython-pubkey-310-311.txt /build_scripts/cpython-pubkeys.txt +RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.10.2 + + +FROM build_cpython AS all_python +COPY build_scripts/install-pypy.sh \ + build_scripts/pypy.sha256 \ + build_scripts/finalize-python.sh \ + /build_scripts/ +RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.7 7.3.7 +RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.8 7.3.7 +COPY --from=build_cpython36 /opt/_internal /opt/_internal/ +COPY --from=build_cpython37 /opt/_internal /opt/_internal/ +COPY --from=build_cpython38 /opt/_internal /opt/_internal/ +COPY --from=build_cpython39 /opt/_internal /opt/_internal/ +COPY --from=build_cpython310 /opt/_internal /opt/_internal/ +RUN manylinux-entrypoint /build_scripts/finalize-python.sh + + +FROM runtime_base +RUN ls + +ENV SSL_CERT_FILE=/opt/_internal/certs.pem CMD ["/bin/bash"] From 5a49b913e7c66af6187168a72960b13648b53a6e Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 18:12:16 +0800 Subject: [PATCH 17/68] refine --- build.sh | 4 ++-- docker/Dockerfile | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/build.sh b/build.sh index e2d6103fd..e8f114de6 100755 --- a/build.sh +++ b/build.sh @@ -80,10 +80,10 @@ BUILD_ARGS_COMMON=" --build-arg POLICY --build-arg PLATFORM --build-arg BASEIMAGE --build-arg DEVTOOLSET_ROOTPATH --build-arg PREPEND_PATH --build-arg LD_LIBRARY_PATH_ARG --rm - -t ${ACR_REGISTRY}/${ACR_NAMESPACE}/${DOCKER_REPO}:${COMMIT_SHA} -t ${ACR_REGISTRY}/${ACR_NAMESPACE}/${DOCKER_REPO}:latest - -t ${DOCKER_HUB_NAMESPACE}/${DOCKER_REPO}:${COMMIT_SHA} + -t ${ACR_REGISTRY}/${ACR_NAMESPACE}/${DOCKER_REPO}:${COMMIT_SHA} -t ${DOCKER_HUB_NAMESPACE}/${DOCKER_REPO}:latest + -t ${DOCKER_HUB_NAMESPACE}/${DOCKER_REPO}:${COMMIT_SHA} -f docker/Dockerfile docker/ " diff --git a/docker/Dockerfile b/docker/Dockerfile index de38919dd..6b914894c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -150,7 +150,22 @@ RUN manylinux-entrypoint /build_scripts/finalize-python.sh FROM runtime_base -RUN ls +COPY --from=build_git /manylinux-rootfs / +COPY --from=build_swig /manylinux-rootfs / +COPY --from=build_cpython /manylinux-rootfs / +COPY --from=all_python /opt/_internal /opt/_internal/ +COPY build_scripts/finalize.sh \ + build_scripts/update-system-packages.sh \ + build_scripts/python-tag-abi-tag.py \ + build_scripts/requirements3.6.txt \ + build_scripts/requirements3.7.txt \ + build_scripts/requirements3.8.txt \ + build_scripts/requirements3.9.txt \ + build_scripts/requirements3.10.txt \ + build_scripts/requirements-base-tools.txt \ + /build_scripts/ +COPY build_scripts/requirements-tools/* /build_scripts/requirements-tools/ +RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts ENV SSL_CERT_FILE=/opt/_internal/certs.pem From a7ad540dedb1760962b2fb07dfb4cd8f1c00f9b9 Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 19:09:12 +0800 Subject: [PATCH 18/68] refine --- .github/workflows/build.yml | 16 +++++++++++++--- build.sh | 7 ++----- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5f4f2c384..51d2db78b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -44,6 +44,7 @@ jobs: PLATFORM: ${{ matrix.platform }} COMMIT_SHA: ${{ github.sha }} DOCKER_REPO: "${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}" + TEST_TAG: ${{ matrix.DOCKER_HUB_NAMESPACE }}/${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}:${COMMIT_SHA} steps: - name: Checkout @@ -88,8 +89,17 @@ jobs: - name: Push image to DockerHub run: | - docker push ${{ env.DOCKER_HUB_NAMESPACE }}/${{ env.DOCKER_REPO }}:${{ github.sha }} - docker push ${{ env.DOCKER_HUB_NAMESPACE }}/${{ env.DOCKER_REPO }}:latest + LATEST_TAG="${{ env.DOCKER_HUB_NAMESPACE }}/${{ env.DOCKER_REPO }}:latest" + COMMIT_TAG="${{ env.DOCKER_HUB_NAMESPACE }}/${{ env.DOCKER_REPO }}:${{ env.COMMIT_SHA }}" + docker tag ${{ env.TEST_TAG}} ${LATEST_TAG} + docker tag ${{ env.TEST_TAG}} ${COMMIT_TAG} + docker push ${LATEST_TAG} + docker push ${COMMIT_TAG} - name: Push image to ACR run: | - docker push ${{ env.ACR_REGISTRY }}/${{ env.ACR_NAMESPACE }}/${{ env.TAG }} + LATEST_TAG="${{ env.ACR_REGISTRY }}/${{ env.ACR_NAMESPACE }}/${DOCKER_REPO}:latest" + COMMIT_TAG="${{ env.ACR_REGISTRY }}/${{ env.ACR_NAMESPACE }}/${DOCKER_REPO}:${{ env.COMMIT_SHA }}" + docker tag ${{ env.TEST_TAG}} ${LATEST_TAG} + docker tag ${{ env.TEST_TAG}} ${COMMIT_TAG} + docker push ${LATEST_TAG} + docker push ${COMMIT_TAG} diff --git a/build.sh b/build.sh index e8f114de6..c5edb3b2b 100755 --- a/build.sh +++ b/build.sh @@ -80,10 +80,7 @@ BUILD_ARGS_COMMON=" --build-arg POLICY --build-arg PLATFORM --build-arg BASEIMAGE --build-arg DEVTOOLSET_ROOTPATH --build-arg PREPEND_PATH --build-arg LD_LIBRARY_PATH_ARG --rm - -t ${ACR_REGISTRY}/${ACR_NAMESPACE}/${DOCKER_REPO}:latest - -t ${ACR_REGISTRY}/${ACR_NAMESPACE}/${DOCKER_REPO}:${COMMIT_SHA} - -t ${DOCKER_HUB_NAMESPACE}/${DOCKER_REPO}:latest - -t ${DOCKER_HUB_NAMESPACE}/${DOCKER_REPO}:${COMMIT_SHA} + -t ${TEST_TAG} -f docker/Dockerfile docker/ " @@ -117,7 +114,7 @@ else exit 1 fi -docker run --rm -v $(pwd)/tests:/tests:ro ${DOCKER_HUB_NAMESPACE}/${DOCKER_REPO}:${COMMIT_SHA} /tests/run_tests.sh +docker run --rm -v $(pwd)/tests:/tests:ro ${TEST_TAG} /tests/run_tests.sh if [ "${MANYLINUX_BUILD_FRONTEND}" != "docker" ]; then if [ -d $(pwd)/.buildx-cache-${POLICY}_${PLATFORM} ]; then From a6f552b23bcbfbf57e9f98f137aac4ca0449e29b Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 19:09:35 +0800 Subject: [PATCH 19/68] refine --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 51d2db78b..b59a89064 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -97,8 +97,8 @@ jobs: docker push ${COMMIT_TAG} - name: Push image to ACR run: | - LATEST_TAG="${{ env.ACR_REGISTRY }}/${{ env.ACR_NAMESPACE }}/${DOCKER_REPO}:latest" - COMMIT_TAG="${{ env.ACR_REGISTRY }}/${{ env.ACR_NAMESPACE }}/${DOCKER_REPO}:${{ env.COMMIT_SHA }}" + LATEST_TAG="${{ env.ACR_REGISTRY }}/${{ env.ACR_NAMESPACE }}/${{ env.DOCKER_REPO }}:latest" + COMMIT_TAG="${{ env.ACR_REGISTRY }}/${{ env.ACR_NAMESPACE }}/${{ env.DOCKER_REPO }}:${{ env.COMMIT_SHA }}" docker tag ${{ env.TEST_TAG}} ${LATEST_TAG} docker tag ${{ env.TEST_TAG}} ${COMMIT_TAG} docker push ${LATEST_TAG} From 4ae598a8cd0c85e71cb690e437eb7d86ed187208 Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 19:10:50 +0800 Subject: [PATCH 20/68] refine --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b59a89064..9847a3c25 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -44,7 +44,7 @@ jobs: PLATFORM: ${{ matrix.platform }} COMMIT_SHA: ${{ github.sha }} DOCKER_REPO: "${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}" - TEST_TAG: ${{ matrix.DOCKER_HUB_NAMESPACE }}/${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}:${COMMIT_SHA} + TEST_TAG: ${{ matrix.DOCKER_HUB_NAMESPACE }}/${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}:${{ github.sha }} steps: - name: Checkout From c46c7670e0ae07cfa61890b55b1a109323f7ca9c Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 19:11:48 +0800 Subject: [PATCH 21/68] refine --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9847a3c25..efc92399f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -44,7 +44,7 @@ jobs: PLATFORM: ${{ matrix.platform }} COMMIT_SHA: ${{ github.sha }} DOCKER_REPO: "${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}" - TEST_TAG: ${{ matrix.DOCKER_HUB_NAMESPACE }}/${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}:${{ github.sha }} + TEST_TAG: ${{ env.DOCKER_HUB_NAMESPACE }}/${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}:${{ github.sha }} steps: - name: Checkout From 8aea2c73244f2e7192582bd5c3db53af511d5212 Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 19:12:54 +0800 Subject: [PATCH 22/68] refine --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index efc92399f..b7c82bf52 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -44,7 +44,7 @@ jobs: PLATFORM: ${{ matrix.platform }} COMMIT_SHA: ${{ github.sha }} DOCKER_REPO: "${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}" - TEST_TAG: ${{ env.DOCKER_HUB_NAMESPACE }}/${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}:${{ github.sha }} + TEST_TAG: ${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}:${{ github.sha }} steps: - name: Checkout From b656f5a6335248bb5aab10ceba29748d52a69f09 Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 19:50:04 +0800 Subject: [PATCH 23/68] debug --- docker/Dockerfile | 127 +++++++++++++++++++++++----------------------- 1 file changed, 64 insertions(+), 63 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 6b914894c..6ab55bd73 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,53 +22,53 @@ ENV PATH=${PREPEND_PATH}${PATH} ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig # first copy the fixup mirrors script, keep the script around -COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors - -# setup entrypoint, this will wrap commands with `linux32` with i686 images -COPY build_scripts/install-entrypoint.sh \ - build_scripts/update-system-packages.sh \ - build_scripts/build_utils.sh \ - /build_scripts/ - -RUN /build_scripts/install-entrypoint.sh && rm -rf /build_scripts -COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint -ENTRYPOINT ["manylinux-entrypoint"] - -COPY build_scripts/install-runtime-packages.sh \ - build_scripts/update-system-packages.sh \ - build_scripts/build_utils.sh \ - /build_scripts/ -RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/ - -COPY build_scripts/build_utils.sh /build_scripts/ - -COPY build_scripts/install-autoconf.sh /build_scripts/ -RUN export AUTOCONF_ROOT=autoconf-2.71 && \ - export AUTOCONF_HASH=431075ad0bf529ef13cb41e9042c542381103e80015686222b8a9d4abef42a1c && \ - export AUTOCONF_DOWNLOAD_URL=http://ftp.gnu.org/gnu/autoconf && \ - manylinux-entrypoint /build_scripts/install-autoconf.sh - -COPY build_scripts/install-automake.sh /build_scripts/ -RUN export AUTOMAKE_ROOT=automake-1.16.5 && \ - export AUTOMAKE_HASH=07bd24ad08a64bc17250ce09ec56e921d6343903943e99ccf63bbf0705e34605 && \ - export AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake && \ - manylinux-entrypoint /build_scripts/install-automake.sh - -COPY build_scripts/install-libtool.sh /build_scripts/ -RUN export LIBTOOL_ROOT=libtool-2.4.6 && \ - export LIBTOOL_HASH=e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3 && \ - export LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool && \ - manylinux-entrypoint /build_scripts/install-libtool.sh - -COPY build_scripts/install-libxcrypt.sh /build_scripts/ -RUN export LIBXCRYPT_VERSION=4.4.27 && \ - export LIBXCRYPT_HASH=e4ff47b7fa5370fa08e27258f8a0de5cb7e9b09c297ec7ab56a8a82e229a1c84 && \ - export LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive && \ - export PERL_ROOT=perl-5.34.0 && \ - export PERL_HASH=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a && \ - export PERL_DOWNLOAD_URL=https://www.cpan.org/src/5.0 && \ - manylinux-entrypoint /build_scripts/install-libxcrypt.sh - +# COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors + +# # setup entrypoint, this will wrap commands with `linux32` with i686 images +# COPY build_scripts/install-entrypoint.sh \ +# build_scripts/update-system-packages.sh \ +# build_scripts/build_utils.sh \ +# /build_scripts/ + +# RUN /build_scripts/install-entrypoint.sh && rm -rf /build_scripts +# COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint +# ENTRYPOINT ["manylinux-entrypoint"] + +# COPY build_scripts/install-runtime-packages.sh \ +# build_scripts/update-system-packages.sh \ +# build_scripts/build_utils.sh \ +# /build_scripts/ +# RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/ + +# COPY build_scripts/build_utils.sh /build_scripts/ + +# COPY build_scripts/install-autoconf.sh /build_scripts/ +# RUN export AUTOCONF_ROOT=autoconf-2.71 && \ +# export AUTOCONF_HASH=431075ad0bf529ef13cb41e9042c542381103e80015686222b8a9d4abef42a1c && \ +# export AUTOCONF_DOWNLOAD_URL=http://ftp.gnu.org/gnu/autoconf && \ +# manylinux-entrypoint /build_scripts/install-autoconf.sh + +# COPY build_scripts/install-automake.sh /build_scripts/ +# RUN export AUTOMAKE_ROOT=automake-1.16.5 && \ +# export AUTOMAKE_HASH=07bd24ad08a64bc17250ce09ec56e921d6343903943e99ccf63bbf0705e34605 && \ +# export AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake && \ +# manylinux-entrypoint /build_scripts/install-automake.sh + +# COPY build_scripts/install-libtool.sh /build_scripts/ +# RUN export LIBTOOL_ROOT=libtool-2.4.6 && \ +# export LIBTOOL_HASH=e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3 && \ +# export LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool && \ +# manylinux-entrypoint /build_scripts/install-libtool.sh + +# COPY build_scripts/install-libxcrypt.sh /build_scripts/ +# RUN export LIBXCRYPT_VERSION=4.4.27 && \ +# export LIBXCRYPT_HASH=e4ff47b7fa5370fa08e27258f8a0de5cb7e9b09c297ec7ab56a8a82e229a1c84 && \ +# export LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive && \ +# export PERL_ROOT=perl-5.34.0 && \ +# export PERL_HASH=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a && \ +# export PERL_DOWNLOAD_URL=https://www.cpan.org/src/5.0 && \ +# manylinux-entrypoint /build_scripts/install-libxcrypt.sh +RUN ls FROM runtime_base AS build_base COPY build_scripts/install-build-packages.sh /build_scripts/ RUN manylinux-entrypoint /build_scripts/install-build-packages.sh @@ -150,22 +150,23 @@ RUN manylinux-entrypoint /build_scripts/finalize-python.sh FROM runtime_base -COPY --from=build_git /manylinux-rootfs / -COPY --from=build_swig /manylinux-rootfs / -COPY --from=build_cpython /manylinux-rootfs / -COPY --from=all_python /opt/_internal /opt/_internal/ -COPY build_scripts/finalize.sh \ - build_scripts/update-system-packages.sh \ - build_scripts/python-tag-abi-tag.py \ - build_scripts/requirements3.6.txt \ - build_scripts/requirements3.7.txt \ - build_scripts/requirements3.8.txt \ - build_scripts/requirements3.9.txt \ - build_scripts/requirements3.10.txt \ - build_scripts/requirements-base-tools.txt \ - /build_scripts/ -COPY build_scripts/requirements-tools/* /build_scripts/requirements-tools/ -RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts +# COPY --from=build_git /manylinux-rootfs / +# COPY --from=build_swig /manylinux-rootfs / +# COPY --from=build_cpython /manylinux-rootfs / +# COPY --from=all_python /opt/_internal /opt/_internal/ +# COPY build_scripts/finalize.sh \ +# build_scripts/update-system-packages.sh \ +# build_scripts/python-tag-abi-tag.py \ +# build_scripts/requirements3.6.txt \ +# build_scripts/requirements3.7.txt \ +# build_scripts/requirements3.8.txt \ +# build_scripts/requirements3.9.txt \ +# build_scripts/requirements3.10.txt \ +# build_scripts/requirements-base-tools.txt \ +# /build_scripts/ +# COPY build_scripts/requirements-tools/* /build_scripts/requirements-tools/ +# RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts +RUN ls ENV SSL_CERT_FILE=/opt/_internal/certs.pem From c273c1752a85dcb5aed452e4255fab3a3e06012b Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 20:06:05 +0800 Subject: [PATCH 24/68] revert --- docker/Dockerfile | 127 +++++++++++++++++++++++----------------------- 1 file changed, 63 insertions(+), 64 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 6ab55bd73..6b914894c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,53 +22,53 @@ ENV PATH=${PREPEND_PATH}${PATH} ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig # first copy the fixup mirrors script, keep the script around -# COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors - -# # setup entrypoint, this will wrap commands with `linux32` with i686 images -# COPY build_scripts/install-entrypoint.sh \ -# build_scripts/update-system-packages.sh \ -# build_scripts/build_utils.sh \ -# /build_scripts/ - -# RUN /build_scripts/install-entrypoint.sh && rm -rf /build_scripts -# COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint -# ENTRYPOINT ["manylinux-entrypoint"] - -# COPY build_scripts/install-runtime-packages.sh \ -# build_scripts/update-system-packages.sh \ -# build_scripts/build_utils.sh \ -# /build_scripts/ -# RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/ - -# COPY build_scripts/build_utils.sh /build_scripts/ - -# COPY build_scripts/install-autoconf.sh /build_scripts/ -# RUN export AUTOCONF_ROOT=autoconf-2.71 && \ -# export AUTOCONF_HASH=431075ad0bf529ef13cb41e9042c542381103e80015686222b8a9d4abef42a1c && \ -# export AUTOCONF_DOWNLOAD_URL=http://ftp.gnu.org/gnu/autoconf && \ -# manylinux-entrypoint /build_scripts/install-autoconf.sh - -# COPY build_scripts/install-automake.sh /build_scripts/ -# RUN export AUTOMAKE_ROOT=automake-1.16.5 && \ -# export AUTOMAKE_HASH=07bd24ad08a64bc17250ce09ec56e921d6343903943e99ccf63bbf0705e34605 && \ -# export AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake && \ -# manylinux-entrypoint /build_scripts/install-automake.sh - -# COPY build_scripts/install-libtool.sh /build_scripts/ -# RUN export LIBTOOL_ROOT=libtool-2.4.6 && \ -# export LIBTOOL_HASH=e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3 && \ -# export LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool && \ -# manylinux-entrypoint /build_scripts/install-libtool.sh - -# COPY build_scripts/install-libxcrypt.sh /build_scripts/ -# RUN export LIBXCRYPT_VERSION=4.4.27 && \ -# export LIBXCRYPT_HASH=e4ff47b7fa5370fa08e27258f8a0de5cb7e9b09c297ec7ab56a8a82e229a1c84 && \ -# export LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive && \ -# export PERL_ROOT=perl-5.34.0 && \ -# export PERL_HASH=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a && \ -# export PERL_DOWNLOAD_URL=https://www.cpan.org/src/5.0 && \ -# manylinux-entrypoint /build_scripts/install-libxcrypt.sh -RUN ls +COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors + +# setup entrypoint, this will wrap commands with `linux32` with i686 images +COPY build_scripts/install-entrypoint.sh \ + build_scripts/update-system-packages.sh \ + build_scripts/build_utils.sh \ + /build_scripts/ + +RUN /build_scripts/install-entrypoint.sh && rm -rf /build_scripts +COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint +ENTRYPOINT ["manylinux-entrypoint"] + +COPY build_scripts/install-runtime-packages.sh \ + build_scripts/update-system-packages.sh \ + build_scripts/build_utils.sh \ + /build_scripts/ +RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/ + +COPY build_scripts/build_utils.sh /build_scripts/ + +COPY build_scripts/install-autoconf.sh /build_scripts/ +RUN export AUTOCONF_ROOT=autoconf-2.71 && \ + export AUTOCONF_HASH=431075ad0bf529ef13cb41e9042c542381103e80015686222b8a9d4abef42a1c && \ + export AUTOCONF_DOWNLOAD_URL=http://ftp.gnu.org/gnu/autoconf && \ + manylinux-entrypoint /build_scripts/install-autoconf.sh + +COPY build_scripts/install-automake.sh /build_scripts/ +RUN export AUTOMAKE_ROOT=automake-1.16.5 && \ + export AUTOMAKE_HASH=07bd24ad08a64bc17250ce09ec56e921d6343903943e99ccf63bbf0705e34605 && \ + export AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake && \ + manylinux-entrypoint /build_scripts/install-automake.sh + +COPY build_scripts/install-libtool.sh /build_scripts/ +RUN export LIBTOOL_ROOT=libtool-2.4.6 && \ + export LIBTOOL_HASH=e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3 && \ + export LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool && \ + manylinux-entrypoint /build_scripts/install-libtool.sh + +COPY build_scripts/install-libxcrypt.sh /build_scripts/ +RUN export LIBXCRYPT_VERSION=4.4.27 && \ + export LIBXCRYPT_HASH=e4ff47b7fa5370fa08e27258f8a0de5cb7e9b09c297ec7ab56a8a82e229a1c84 && \ + export LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive && \ + export PERL_ROOT=perl-5.34.0 && \ + export PERL_HASH=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a && \ + export PERL_DOWNLOAD_URL=https://www.cpan.org/src/5.0 && \ + manylinux-entrypoint /build_scripts/install-libxcrypt.sh + FROM runtime_base AS build_base COPY build_scripts/install-build-packages.sh /build_scripts/ RUN manylinux-entrypoint /build_scripts/install-build-packages.sh @@ -150,23 +150,22 @@ RUN manylinux-entrypoint /build_scripts/finalize-python.sh FROM runtime_base -# COPY --from=build_git /manylinux-rootfs / -# COPY --from=build_swig /manylinux-rootfs / -# COPY --from=build_cpython /manylinux-rootfs / -# COPY --from=all_python /opt/_internal /opt/_internal/ -# COPY build_scripts/finalize.sh \ -# build_scripts/update-system-packages.sh \ -# build_scripts/python-tag-abi-tag.py \ -# build_scripts/requirements3.6.txt \ -# build_scripts/requirements3.7.txt \ -# build_scripts/requirements3.8.txt \ -# build_scripts/requirements3.9.txt \ -# build_scripts/requirements3.10.txt \ -# build_scripts/requirements-base-tools.txt \ -# /build_scripts/ -# COPY build_scripts/requirements-tools/* /build_scripts/requirements-tools/ -# RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts -RUN ls +COPY --from=build_git /manylinux-rootfs / +COPY --from=build_swig /manylinux-rootfs / +COPY --from=build_cpython /manylinux-rootfs / +COPY --from=all_python /opt/_internal /opt/_internal/ +COPY build_scripts/finalize.sh \ + build_scripts/update-system-packages.sh \ + build_scripts/python-tag-abi-tag.py \ + build_scripts/requirements3.6.txt \ + build_scripts/requirements3.7.txt \ + build_scripts/requirements3.8.txt \ + build_scripts/requirements3.9.txt \ + build_scripts/requirements3.10.txt \ + build_scripts/requirements-base-tools.txt \ + /build_scripts/ +COPY build_scripts/requirements-tools/* /build_scripts/requirements-tools/ +RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts ENV SSL_CERT_FILE=/opt/_internal/certs.pem From 9123b4cda584392863a4113f68da13ce037015d4 Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 20:14:08 +0800 Subject: [PATCH 25/68] refactor --- .github/workflows/build.yml | 54 ++++++++++++++++++++++++------------- build.sh | 24 +---------------- 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b7c82bf52..4f4002581 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -83,23 +83,39 @@ jobs: restore-keys: buildx-cache-${{ matrix.policy }}-${{ matrix.platform }}-${{ matrix.tag-suffix }} - name: Build - env: - CUDA_BASE_IMAGE: ${{ matrix.CUDA_BASE_IMAGE }} run: ./build.sh - - - name: Push image to DockerHub - run: | - LATEST_TAG="${{ env.DOCKER_HUB_NAMESPACE }}/${{ env.DOCKER_REPO }}:latest" - COMMIT_TAG="${{ env.DOCKER_HUB_NAMESPACE }}/${{ env.DOCKER_REPO }}:${{ env.COMMIT_SHA }}" - docker tag ${{ env.TEST_TAG}} ${LATEST_TAG} - docker tag ${{ env.TEST_TAG}} ${COMMIT_TAG} - docker push ${LATEST_TAG} - docker push ${COMMIT_TAG} - - name: Push image to ACR - run: | - LATEST_TAG="${{ env.ACR_REGISTRY }}/${{ env.ACR_NAMESPACE }}/${{ env.DOCKER_REPO }}:latest" - COMMIT_TAG="${{ env.ACR_REGISTRY }}/${{ env.ACR_NAMESPACE }}/${{ env.DOCKER_REPO }}:${{ env.COMMIT_SHA }}" - docker tag ${{ env.TEST_TAG}} ${LATEST_TAG} - docker tag ${{ env.TEST_TAG}} ${COMMIT_TAG} - docker push ${LATEST_TAG} - docker push ${COMMIT_TAG} + - name: Build and push + uses: docker/build-push-action@v2 + with: + push: true + tags: | + ${{ env.DOCKER_HUB_NAMESPACE }}/${{ env.DOCKER_REPO }}:latest + ${{ env.DOCKER_HUB_NAMESPACE }}/${{ env.DOCKER_REPO }}:${{ env.COMMIT_SHA }} + ${{ env.ACR_REGISTRY }}/${{ env.ACR_NAMESPACE }}/${{ env.DOCKER_REPO }}:latest + ${{ env.ACR_REGISTRY }}/${{ env.ACR_NAMESPACE }}/${{ env.DOCKER_REPO }}:${{ env.COMMIT_SHA }} + cache-from: type=registry,ref=${{ env.DOCKER_HUB_NAMESPACE }}/${{ env.DOCKER_REPO }}:latest + cache-to: type=inline + context: docker + build-args: | + POLICY + PLATFORM + BASEIMAGE + DEVTOOLSET_ROOTPATH + PREPEND_PATH + LD_LIBRARY_PATH_ARG + # - name: Push image to DockerHub + # run: | + # LATEST_TAG="${{ env.DOCKER_HUB_NAMESPACE }}/${{ env.DOCKER_REPO }}:latest" + # COMMIT_TAG="${{ env.DOCKER_HUB_NAMESPACE }}/${{ env.DOCKER_REPO }}:${{ env.COMMIT_SHA }}" + # docker tag ${{ env.TEST_TAG}} ${LATEST_TAG} + # docker tag ${{ env.TEST_TAG}} ${COMMIT_TAG} + # docker push ${LATEST_TAG} + # docker push ${COMMIT_TAG} + # - name: Push image to ACR + # run: | + # LATEST_TAG="${{ env.ACR_REGISTRY }}/${{ env.ACR_NAMESPACE }}/${{ env.DOCKER_REPO }}:latest" + # COMMIT_TAG="${{ env.ACR_REGISTRY }}/${{ env.ACR_NAMESPACE }}/${{ env.DOCKER_REPO }}:${{ env.COMMIT_SHA }}" + # docker tag ${{ env.TEST_TAG}} ${LATEST_TAG} + # docker tag ${{ env.TEST_TAG}} ${COMMIT_TAG} + # docker push ${LATEST_TAG} + # docker push ${COMMIT_TAG} diff --git a/build.sh b/build.sh index c5edb3b2b..9dc880831 100755 --- a/build.sh +++ b/build.sh @@ -92,33 +92,11 @@ fi if [ "${MANYLINUX_BUILD_FRONTEND}" == "docker" ]; then docker build ${BUILD_ARGS_COMMON} elif [ "${MANYLINUX_BUILD_FRONTEND}" == "docker-buildx" ]; then - docker buildx build \ - --load \ - --cache-from=type=type=registry,ref=${DOCKER_HUB_NAMESPACE}/${DOCKER_REPO}:latest \ - --cache-to=type=inline \ - ${BUILD_ARGS_COMMON} + env elif [ "${MANYLINUX_BUILD_FRONTEND}" == "buildkit" ]; then echo "Unsupported build frontend: buildkit" exit 1 - buildctl build \ - --frontend=dockerfile.v0 \ - --local context=./docker/ \ - --local dockerfile=./docker/ \ - --import-cache type=local,src=$(pwd)/.buildx-cache-${POLICY}_${PLATFORM} \ - --export-cache type=local,dest=$(pwd)/.buildx-cache-staging-${POLICY}_${PLATFORM} \ - --opt build-arg:POLICY=${POLICY} --opt build-arg:PLATFORM=${PLATFORM} --opt build-arg:BASEIMAGE=${BASEIMAGE} \ - --opt "build-arg:DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH}" --opt "build-arg:PREPEND_PATH=${PREPEND_PATH}" --opt "build-arg:LD_LIBRARY_PATH_ARG=${LD_LIBRARY_PATH_ARG}" \ - --output type=docker,name=quay.io/pypa/${POLICY}_${PLATFORM}:${COMMIT_SHA} | docker load else echo "Unsupported build frontend: '${MANYLINUX_BUILD_FRONTEND}'" exit 1 fi - -docker run --rm -v $(pwd)/tests:/tests:ro ${TEST_TAG} /tests/run_tests.sh - -if [ "${MANYLINUX_BUILD_FRONTEND}" != "docker" ]; then - if [ -d $(pwd)/.buildx-cache-${POLICY}_${PLATFORM} ]; then - rm -rf $(pwd)/.buildx-cache-${POLICY}_${PLATFORM} - fi - mv $(pwd)/.buildx-cache-staging-${POLICY}_${PLATFORM} $(pwd)/.buildx-cache-${POLICY}_${PLATFORM} -fi From af86d4ce70d50f67178646825935c8b3e702124a Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 20:15:07 +0800 Subject: [PATCH 26/68] refine --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4f4002581..62ebe0e05 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,6 +45,7 @@ jobs: COMMIT_SHA: ${{ github.sha }} DOCKER_REPO: "${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}" TEST_TAG: ${{ matrix.policy }}_${{ matrix.platform }}_${{ matrix.tag-suffix }}:${{ github.sha }} + CUDA_BASE_IMAGE: ${{ matrix.CUDA_BASE_IMAGE }} steps: - name: Checkout From 867ad76f24d4aa5aaf4595a8cebad63abffd1d1e Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 20:24:22 +0800 Subject: [PATCH 27/68] refine --- .github/workflows/build.yml | 2 +- build.sh | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 62ebe0e05..a4f7ee9d9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -96,7 +96,7 @@ jobs: ${{ env.ACR_REGISTRY }}/${{ env.ACR_NAMESPACE }}/${{ env.DOCKER_REPO }}:${{ env.COMMIT_SHA }} cache-from: type=registry,ref=${{ env.DOCKER_HUB_NAMESPACE }}/${{ env.DOCKER_REPO }}:latest cache-to: type=inline - context: docker + context: ./docker/ build-args: | POLICY PLATFORM diff --git a/build.sh b/build.sh index 9dc880831..524de393e 100755 --- a/build.sh +++ b/build.sh @@ -79,8 +79,7 @@ export LD_LIBRARY_PATH_ARG BUILD_ARGS_COMMON=" --build-arg POLICY --build-arg PLATFORM --build-arg BASEIMAGE --build-arg DEVTOOLSET_ROOTPATH --build-arg PREPEND_PATH --build-arg LD_LIBRARY_PATH_ARG - --rm - -t ${TEST_TAG} + --rm -t quay.io/pypa/${POLICY}_${PLATFORM}:${COMMIT_SHA} -f docker/Dockerfile docker/ " From 25d0ac3ce7b473bfdb44454de9d982938272e04d Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 20:30:55 +0800 Subject: [PATCH 28/68] refine --- .github/workflows/build.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a4f7ee9d9..365bb74dc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -84,7 +84,14 @@ jobs: restore-keys: buildx-cache-${{ matrix.policy }}-${{ matrix.platform }}-${{ matrix.tag-suffix }} - name: Build - run: ./build.sh + run: | + ./build.sh + echo "{POLICY}={POLICY}" >> $GITHUB_ENV + echo "{PLATFORM}={PLATFORM}" >> $GITHUB_ENV + echo "{BASEIMAGE}={BASEIMAGE}" >> $GITHUB_ENV + echo "{DEVTOOLSET_ROOTPATH}={DEVTOOLSET_ROOTPATH}" >> $GITHUB_ENV + echo "{PREPEND_PATH}={PREPEND_PATH}" >> $GITHUB_ENV + echo "{LD_LIBRARY_PATH_ARG}={LD_LIBRARY_PATH_ARG}" >> $GITHUB_ENV - name: Build and push uses: docker/build-push-action@v2 with: From f9a2eb147db96233a5c866c1f90567c7f5495874 Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 20:32:06 +0800 Subject: [PATCH 29/68] refine --- .github/workflows/build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 365bb74dc..06f5f5cf5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -86,12 +86,12 @@ jobs: - name: Build run: | ./build.sh - echo "{POLICY}={POLICY}" >> $GITHUB_ENV - echo "{PLATFORM}={PLATFORM}" >> $GITHUB_ENV - echo "{BASEIMAGE}={BASEIMAGE}" >> $GITHUB_ENV - echo "{DEVTOOLSET_ROOTPATH}={DEVTOOLSET_ROOTPATH}" >> $GITHUB_ENV - echo "{PREPEND_PATH}={PREPEND_PATH}" >> $GITHUB_ENV - echo "{LD_LIBRARY_PATH_ARG}={LD_LIBRARY_PATH_ARG}" >> $GITHUB_ENV + echo "POLICY=${POLICY}" >> $GITHUB_ENV + echo "PLATFORM=${PLATFORM}" >> $GITHUB_ENV + echo "BASEIMAGE=${BASEIMAGE}" >> $GITHUB_ENV + echo "DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH}" >> $GITHUB_ENV + echo "PREPEND_PATH=${PREPEND_PATH}" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH_ARG=${LD_LIBRARY_PATH_ARG}" >> $GITHUB_ENV - name: Build and push uses: docker/build-push-action@v2 with: From 1cd9bbbe5c06ef54979105c1f1e02f435b2f8ea4 Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 20:34:26 +0800 Subject: [PATCH 30/68] refine --- .github/workflows/build.yml | 9 +-------- build.sh | 7 +++++++ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 06f5f5cf5..a4f7ee9d9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -84,14 +84,7 @@ jobs: restore-keys: buildx-cache-${{ matrix.policy }}-${{ matrix.platform }}-${{ matrix.tag-suffix }} - name: Build - run: | - ./build.sh - echo "POLICY=${POLICY}" >> $GITHUB_ENV - echo "PLATFORM=${PLATFORM}" >> $GITHUB_ENV - echo "BASEIMAGE=${BASEIMAGE}" >> $GITHUB_ENV - echo "DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH}" >> $GITHUB_ENV - echo "PREPEND_PATH=${PREPEND_PATH}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH_ARG=${LD_LIBRARY_PATH_ARG}" >> $GITHUB_ENV + run: ./build.sh - name: Build and push uses: docker/build-push-action@v2 with: diff --git a/build.sh b/build.sh index 524de393e..ef621f9b3 100755 --- a/build.sh +++ b/build.sh @@ -99,3 +99,10 @@ else echo "Unsupported build frontend: '${MANYLINUX_BUILD_FRONTEND}'" exit 1 fi + +echo "POLICY=${POLICY}" >> $GITHUB_ENV +echo "PLATFORM=${PLATFORM}" >> $GITHUB_ENV +echo "BASEIMAGE=${BASEIMAGE}" >> $GITHUB_ENV +echo "DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH}" >> $GITHUB_ENV +echo "PREPEND_PATH=${PREPEND_PATH}" >> $GITHUB_ENV +echo "LD_LIBRARY_PATH_ARG=${LD_LIBRARY_PATH_ARG}" >> $GITHUB_ENV From 22cab7bb1e9b6d5c270f477be435a1a96677902c Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 20:50:16 +0800 Subject: [PATCH 31/68] refine --- .github/workflows/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a4f7ee9d9..e0cdf8ce8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,11 +9,11 @@ on: - ".github/workflows/build.yml" - "docker/**" - "*.sh" - pull_request: - paths: - - ".github/workflows/build.yml" - - "docker/**" - - "build.sh" + # pull_request: + # paths: + # - ".github/workflows/build.yml" + # - "docker/**" + # - "build.sh" concurrency: group: test-${{ github.ref }} From 178281fc28b36aa021d2b51374e071f6b8d24b39 Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 21:30:32 +0800 Subject: [PATCH 32/68] refine --- docker/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 6b914894c..8c340dc9f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -169,4 +169,9 @@ RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts ENV SSL_CERT_FILE=/opt/_internal/certs.pem +ARG BAZEL_URL="https://github.com/bazelbuild/bazel/releases/download/3.4.1/bazel-3.4.1-linux-x86_64" +RUN curl -L $BAZEL_URL -o /usr/local/bin/bazel \ + && chmod +x /usr/local/bin/bazel +RUN yum install -y wget nasm rdma-core-devel rsync gdb ninja-build openblas-static autoconf libtool wget devtoolset-7-gcc* vim ccache + CMD ["/bin/bash"] From 8622f720d54748e5b7e1ca925f3352bfb7f6d2d0 Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 21:43:59 +0800 Subject: [PATCH 33/68] refine --- .github/workflows/build.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e0cdf8ce8..de1bb7a64 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,12 +32,16 @@ jobs: strategy: fail-fast: false matrix: - tag-suffix: ["cuda11.2"] + tag-suffix: ["cuda11.2", "cuda10.2"] include: - tag-suffix: "cuda11.2" - policy: "manylinux2014" - platform: "x86_64" - CUDA_BASE_IMAGE: "nvidia/cuda:11.2.2-cudnn8-devel-centos7" + - tag-suffix: "cuda10.2" + - policy: "manylinux2014" + - platform: "x86_64" + - CUDA_BASE_IMAGE: "nvidia/cuda:10.2-cudnn8-devel-centos7" env: POLICY: ${{ matrix.policy }} From 98a1548e170b56066051b2a3635605a262357da2 Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 21:50:28 +0800 Subject: [PATCH 34/68] refine --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 8c340dc9f..1eb6bbae1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -172,6 +172,6 @@ ENV SSL_CERT_FILE=/opt/_internal/certs.pem ARG BAZEL_URL="https://github.com/bazelbuild/bazel/releases/download/3.4.1/bazel-3.4.1-linux-x86_64" RUN curl -L $BAZEL_URL -o /usr/local/bin/bazel \ && chmod +x /usr/local/bin/bazel -RUN yum install -y wget nasm rdma-core-devel rsync gdb ninja-build openblas-static autoconf libtool wget devtoolset-7-gcc* vim ccache +RUN yum install -y wget nasm rdma-core-devel rsync gdb ninja-build openblas-static devtoolset-7-gcc* vim ccache CMD ["/bin/bash"] From 8d00c7cd756387331ecc57c2e31ddb217245dd17 Mon Sep 17 00:00:00 2001 From: tsai Date: Sun, 23 Jan 2022 22:10:44 +0800 Subject: [PATCH 35/68] refine --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 1eb6bbae1..e2f0bc3ee 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -172,6 +172,6 @@ ENV SSL_CERT_FILE=/opt/_internal/certs.pem ARG BAZEL_URL="https://github.com/bazelbuild/bazel/releases/download/3.4.1/bazel-3.4.1-linux-x86_64" RUN curl -L $BAZEL_URL -o /usr/local/bin/bazel \ && chmod +x /usr/local/bin/bazel -RUN yum install -y wget nasm rdma-core-devel rsync gdb ninja-build openblas-static devtoolset-7-gcc* vim ccache +RUN yum install -y wget nasm rdma-core-devel rsync gdb ninja-build openblas-static devtoolset-7-gcc* vim ccache htop CMD ["/bin/bash"] From c2b77f67c1be0e7ed1b4ef36f69fca99910809ec Mon Sep 17 00:00:00 2001 From: tsai Date: Mon, 24 Jan 2022 01:26:15 +0800 Subject: [PATCH 36/68] refine --- .github/workflows/build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index de1bb7a64..01b5fcdfc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,13 +35,13 @@ jobs: tag-suffix: ["cuda11.2", "cuda10.2"] include: - tag-suffix: "cuda11.2" - - policy: "manylinux2014" - - platform: "x86_64" - - CUDA_BASE_IMAGE: "nvidia/cuda:11.2.2-cudnn8-devel-centos7" + policy: "manylinux2014" + platform: "x86_64" + CUDA_BASE_IMAGE: "nvidia/cuda:11.2.2-cudnn8-devel-centos7" - tag-suffix: "cuda10.2" - - policy: "manylinux2014" - - platform: "x86_64" - - CUDA_BASE_IMAGE: "nvidia/cuda:10.2-cudnn8-devel-centos7" + policy: "manylinux2014" + platform: "x86_64" + CUDA_BASE_IMAGE: "nvidia/cuda:10.2-cudnn8-devel-centos7" env: POLICY: ${{ matrix.policy }} From 7341b7586b43260e55914b2fc20130657479efb2 Mon Sep 17 00:00:00 2001 From: tsai Date: Mon, 24 Jan 2022 10:49:50 +0800 Subject: [PATCH 37/68] refine --- .github/workflows/build.yml | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 01b5fcdfc..6467da738 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -73,20 +73,9 @@ jobs: with: platforms: ${{ matrix.platform }} - # - name: Set BuildKit for ppc64le - # if: matrix.platform == 'ppc64le' - # run: echo 'MANYLINUX_BUILD_FRONTEND=buildkit' >> $GITHUB_ENV - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - - name: Setup cache - uses: actions/cache@v2 - with: - path: .buildx-cache-${{ matrix.policy }}_${{ matrix.platform }}/* - key: buildx-cache-${{ matrix.policy }}-${{ matrix.platform }}-${{ matrix.tag-suffix }}-${{ hashFiles('docker/**') }} - restore-keys: buildx-cache-${{ matrix.policy }}-${{ matrix.platform }}-${{ matrix.tag-suffix }} - - name: Build run: ./build.sh - name: Build and push @@ -108,19 +97,3 @@ jobs: DEVTOOLSET_ROOTPATH PREPEND_PATH LD_LIBRARY_PATH_ARG - # - name: Push image to DockerHub - # run: | - # LATEST_TAG="${{ env.DOCKER_HUB_NAMESPACE }}/${{ env.DOCKER_REPO }}:latest" - # COMMIT_TAG="${{ env.DOCKER_HUB_NAMESPACE }}/${{ env.DOCKER_REPO }}:${{ env.COMMIT_SHA }}" - # docker tag ${{ env.TEST_TAG}} ${LATEST_TAG} - # docker tag ${{ env.TEST_TAG}} ${COMMIT_TAG} - # docker push ${LATEST_TAG} - # docker push ${COMMIT_TAG} - # - name: Push image to ACR - # run: | - # LATEST_TAG="${{ env.ACR_REGISTRY }}/${{ env.ACR_NAMESPACE }}/${{ env.DOCKER_REPO }}:latest" - # COMMIT_TAG="${{ env.ACR_REGISTRY }}/${{ env.ACR_NAMESPACE }}/${{ env.DOCKER_REPO }}:${{ env.COMMIT_SHA }}" - # docker tag ${{ env.TEST_TAG}} ${LATEST_TAG} - # docker tag ${{ env.TEST_TAG}} ${COMMIT_TAG} - # docker push ${LATEST_TAG} - # docker push ${COMMIT_TAG} From f7296eabdc91ee7b0cb6e470c4c53ef652f5e918 Mon Sep 17 00:00:00 2001 From: tsai Date: Mon, 24 Jan 2022 10:51:26 +0800 Subject: [PATCH 38/68] use cudnn7 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6467da738..96adedfdb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,7 +41,7 @@ jobs: - tag-suffix: "cuda10.2" policy: "manylinux2014" platform: "x86_64" - CUDA_BASE_IMAGE: "nvidia/cuda:10.2-cudnn8-devel-centos7" + CUDA_BASE_IMAGE: "nvidia/cuda:10.2-cudnn7-devel-centos7" env: POLICY: ${{ matrix.policy }} From c4bec039af509e9a6852ef09656190e2013f3b27 Mon Sep 17 00:00:00 2001 From: tsai Date: Mon, 24 Jan 2022 12:53:51 +0800 Subject: [PATCH 39/68] add 10.1 and cpu --- .github/workflows/build.yml | 8 ++++++++ build.sh | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 96adedfdb..0fe5a2ba3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,6 +42,14 @@ jobs: policy: "manylinux2014" platform: "x86_64" CUDA_BASE_IMAGE: "nvidia/cuda:10.2-cudnn7-devel-centos7" + - tag-suffix: "cuda10.1" + policy: "manylinux2014" + platform: "x86_64" + CUDA_BASE_IMAGE: "nvidia/cuda:10.1-cudnn7-devel-centos7" + - tag-suffix: "cpu" + policy: "manylinux2014" + platform: "x86_64" + CUDA_BASE_IMAGE: "" env: POLICY: ${{ matrix.policy }} diff --git a/build.sh b/build.sh index ef621f9b3..255c3d2da 100755 --- a/build.sh +++ b/build.sh @@ -48,7 +48,8 @@ elif [ "${POLICY}" == "manylinux2014" ]; then if [ "${PLATFORM}" == "s390x" ]; then BASEIMAGE="s390x/clefos:7" else - BASEIMAGE="${CUDA_BASE_IMAGE}" + DEFAULT_BASEIMAGE="${MULTIARCH_PREFIX}centos:7" + BASEIMAGE="${CUDA_BASE_IMAGE:-${DEFAULT_BASEIMAGE}}" fi DEVTOOLSET_ROOTPATH="/opt/rh/devtoolset-10/root" PREPEND_PATH="${DEVTOOLSET_ROOTPATH}/usr/bin:" From bd7bd63020a2d8924e5b1928346e4f3456bf2d48 Mon Sep 17 00:00:00 2001 From: tsai Date: Mon, 24 Jan 2022 20:00:05 +0800 Subject: [PATCH 40/68] refine --- .github/workflows/build.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0fe5a2ba3..911a642f5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,10 +42,6 @@ jobs: policy: "manylinux2014" platform: "x86_64" CUDA_BASE_IMAGE: "nvidia/cuda:10.2-cudnn7-devel-centos7" - - tag-suffix: "cuda10.1" - policy: "manylinux2014" - platform: "x86_64" - CUDA_BASE_IMAGE: "nvidia/cuda:10.1-cudnn7-devel-centos7" - tag-suffix: "cpu" policy: "manylinux2014" platform: "x86_64" From afff8e39e28b8a852209111566377e90631d1d5f Mon Sep 17 00:00:00 2001 From: tsai Date: Mon, 24 Jan 2022 20:26:37 +0800 Subject: [PATCH 41/68] add 113 --- .github/workflows/build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 911a642f5..dceec330a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,6 +34,10 @@ jobs: matrix: tag-suffix: ["cuda11.2", "cuda10.2"] include: + - tag-suffix: "cuda11.3" + policy: "manylinux2014" + platform: "x86_64" + CUDA_BASE_IMAGE: "nvidia/cuda:11.3.1-cudnn8-devel-centos7" - tag-suffix: "cuda11.2" policy: "manylinux2014" platform: "x86_64" From 617d3245410d4d35d9a0637269d0aab14c996029 Mon Sep 17 00:00:00 2001 From: tsai Date: Tue, 25 Jan 2022 11:08:35 +0800 Subject: [PATCH 42/68] add 114 --- .github/workflows/build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dceec330a..895f7362b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,6 +34,10 @@ jobs: matrix: tag-suffix: ["cuda11.2", "cuda10.2"] include: + - tag-suffix: "cuda11.4" + policy: "manylinux2014" + platform: "x86_64" + CUDA_BASE_IMAGE: "nvidia/cuda:11.4.3-cudnn8-devel-centos7" - tag-suffix: "cuda11.3" policy: "manylinux2014" platform: "x86_64" From 78e5485bc983684cf6b1234d4d0175d361c8f66a Mon Sep 17 00:00:00 2001 From: tsai Date: Fri, 25 Feb 2022 22:25:52 +0800 Subject: [PATCH 43/68] add 11.0 --- .github/workflows/build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 895f7362b..619ba1020 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -46,6 +46,11 @@ jobs: policy: "manylinux2014" platform: "x86_64" CUDA_BASE_IMAGE: "nvidia/cuda:11.2.2-cudnn8-devel-centos7" + - tag-suffix: "cuda11.0" + policy: "manylinux2014" + platform: "x86_64" + CUDA_BASE_IMAGE: "nvidia/cuda:11.0.3-cudnn8-devel-centos7" + target: "manylinux" - tag-suffix: "cuda10.2" policy: "manylinux2014" platform: "x86_64" From 4fd9cc268bbe59c6245ca3941b8264fd256a8670 Mon Sep 17 00:00:00 2001 From: tsai Date: Mon, 21 Mar 2022 18:54:20 +0800 Subject: [PATCH 44/68] fix --- .github/workflows/build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 619ba1020..4464b0389 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,6 +34,10 @@ jobs: matrix: tag-suffix: ["cuda11.2", "cuda10.2"] include: + - tag-suffix: "cuda11.5" + policy: "manylinux2014" + platform: "x86_64" + CUDA_BASE_IMAGE: "nvidia/cuda:11.5.1-cudnn8-devel-centos7" - tag-suffix: "cuda11.4" policy: "manylinux2014" platform: "x86_64" From 1155dac6446fe32c069b160c5560cb7838a81e15 Mon Sep 17 00:00:00 2001 From: tsai Date: Wed, 23 Mar 2022 11:34:03 +0800 Subject: [PATCH 45/68] don't update cudnn --- .gitignore | 1 + docker/build_scripts/update-system-packages.sh | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index a6a86cf99..dc2ec39a5 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,4 @@ docker/sources # buildx cache .buildx-cache-*/ +.vscode/ diff --git a/docker/build_scripts/update-system-packages.sh b/docker/build_scripts/update-system-packages.sh index 32bb294ea..2a75c8ab2 100755 --- a/docker/build_scripts/update-system-packages.sh +++ b/docker/build_scripts/update-system-packages.sh @@ -10,6 +10,8 @@ MY_DIR=$(dirname "${BASH_SOURCE[0]}") # Get build utilities source $MY_DIR/build_utils.sh +yum -x libcudnn8 -x libcudnn8-devel update + fixup-mirrors if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then yum -y update From 9e32badc23eb66faa1f9511d044330906bb1e034 Mon Sep 17 00:00:00 2001 From: tsai Date: Wed, 23 Mar 2022 11:39:01 +0800 Subject: [PATCH 46/68] Revert "don't update cudnn" This reverts commit 1155dac6446fe32c069b160c5560cb7838a81e15. --- docker/build_scripts/update-system-packages.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/build_scripts/update-system-packages.sh b/docker/build_scripts/update-system-packages.sh index 2a75c8ab2..32bb294ea 100755 --- a/docker/build_scripts/update-system-packages.sh +++ b/docker/build_scripts/update-system-packages.sh @@ -10,8 +10,6 @@ MY_DIR=$(dirname "${BASH_SOURCE[0]}") # Get build utilities source $MY_DIR/build_utils.sh -yum -x libcudnn8 -x libcudnn8-devel update - fixup-mirrors if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then yum -y update From ddf4ee997b7048c8a036cc00f53366dec127e950 Mon Sep 17 00:00:00 2001 From: tsai Date: Wed, 23 Mar 2022 11:42:10 +0800 Subject: [PATCH 47/68] Update update-system-packages.sh --- docker/build_scripts/update-system-packages.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/build_scripts/update-system-packages.sh b/docker/build_scripts/update-system-packages.sh index 32bb294ea..bfd4ad39c 100755 --- a/docker/build_scripts/update-system-packages.sh +++ b/docker/build_scripts/update-system-packages.sh @@ -12,6 +12,11 @@ source $MY_DIR/build_utils.sh fixup-mirrors if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then + yum -y install yum-versionlock + yum versionlock add libcudnn8-devel + yum versionlock add libcudnn8 + yum versionlock list libcudnn8-devel + yum versionlock list libcudnn8 yum -y update if ! localedef -V &> /dev/null; then # somebody messed up glibc-common package to squeeze image size, reinstall the package From 888d9481dcb2c02c6ebf9b3906e9d475cb16efa0 Mon Sep 17 00:00:00 2001 From: tsai Date: Wed, 23 Mar 2022 11:46:24 +0800 Subject: [PATCH 48/68] Revert "Update update-system-packages.sh" This reverts commit ddf4ee997b7048c8a036cc00f53366dec127e950. --- docker/build_scripts/update-system-packages.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docker/build_scripts/update-system-packages.sh b/docker/build_scripts/update-system-packages.sh index bfd4ad39c..32bb294ea 100755 --- a/docker/build_scripts/update-system-packages.sh +++ b/docker/build_scripts/update-system-packages.sh @@ -12,11 +12,6 @@ source $MY_DIR/build_utils.sh fixup-mirrors if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then - yum -y install yum-versionlock - yum versionlock add libcudnn8-devel - yum versionlock add libcudnn8 - yum versionlock list libcudnn8-devel - yum versionlock list libcudnn8 yum -y update if ! localedef -V &> /dev/null; then # somebody messed up glibc-common package to squeeze image size, reinstall the package From 6aa1da7b70bb3383cb0aa07c613e0bc783fc7418 Mon Sep 17 00:00:00 2001 From: tsai Date: Wed, 23 Mar 2022 11:48:57 +0800 Subject: [PATCH 49/68] fix --- docker/build_scripts/install-runtime-packages.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/build_scripts/install-runtime-packages.sh b/docker/build_scripts/install-runtime-packages.sh index 3ddd44377..f6c894c57 100755 --- a/docker/build_scripts/install-runtime-packages.sh +++ b/docker/build_scripts/install-runtime-packages.sh @@ -34,6 +34,11 @@ source $MY_DIR/build_utils.sh # MANYLINUX_DEPS: Install development packages (except for libgcc which is provided by gcc install) if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then MANYLINUX_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel zlib-devel expat-devel" + yum -y install yum-versionlock + yum versionlock add libcudnn8-devel + yum versionlock add libcudnn8 + yum versionlock list libcudnn8-devel + yum versionlock list libcudnn8 elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then MANYLINUX_DEPS="libc6-dev libstdc++-6-dev libglib2.0-dev libx11-dev libxext-dev libxrender-dev libgl1-mesa-dev libice-dev libsm-dev libz-dev libexpat1-dev" elif [ "${AUDITWHEEL_POLICY}" == "musllinux_1_1" ]; then From 84296bddcd2f8684ed0bff545ef0f064a01a6fe5 Mon Sep 17 00:00:00 2001 From: tsai Date: Fri, 29 Apr 2022 19:24:36 +0800 Subject: [PATCH 50/68] add clangd --- docker/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index e2f0bc3ee..eea37c530 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -173,5 +173,10 @@ ARG BAZEL_URL="https://github.com/bazelbuild/bazel/releases/download/3.4.1/bazel RUN curl -L $BAZEL_URL -o /usr/local/bin/bazel \ && chmod +x /usr/local/bin/bazel RUN yum install -y wget nasm rdma-core-devel rsync gdb ninja-build openblas-static devtoolset-7-gcc* vim ccache htop +RUN wget https://github.com/clangd/clangd/releases/download/14.0.0/clangd-linux-14.0.0.zip && \ + unzip clangd-linux-14.0.0.zip && \ + mv clangd_14.0.0/bin/clangd /usr/local/bin/clangd && \ + chmod +x /usr/local/bin/clangd && \ + rm clangd-linux-14.0.0.zip CMD ["/bin/bash"] From 3cdf3997c6bae0f3c5090d28196906c71296f7ba Mon Sep 17 00:00:00 2001 From: tsai Date: Fri, 29 Apr 2022 19:32:04 +0800 Subject: [PATCH 51/68] refine --- docker/build_scripts/install-runtime-packages.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/build_scripts/install-runtime-packages.sh b/docker/build_scripts/install-runtime-packages.sh index f6c894c57..ca2d4df6f 100755 --- a/docker/build_scripts/install-runtime-packages.sh +++ b/docker/build_scripts/install-runtime-packages.sh @@ -37,6 +37,9 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" == yum -y install yum-versionlock yum versionlock add libcudnn8-devel yum versionlock add libcudnn8 + yum versionlock add cuda-* + yum versionlock add libnccl + yum versionlock add libnccl-devel yum versionlock list libcudnn8-devel yum versionlock list libcudnn8 elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then From 4994c3e0be99c4cdc0e190c0e95c8ba88de5f7d3 Mon Sep 17 00:00:00 2001 From: tsai Date: Fri, 29 Apr 2022 19:34:37 +0800 Subject: [PATCH 52/68] refine --- docker/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index eea37c530..be74bd33d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -173,6 +173,8 @@ ARG BAZEL_URL="https://github.com/bazelbuild/bazel/releases/download/3.4.1/bazel RUN curl -L $BAZEL_URL -o /usr/local/bin/bazel \ && chmod +x /usr/local/bin/bazel RUN yum install -y wget nasm rdma-core-devel rsync gdb ninja-build openblas-static devtoolset-7-gcc* vim ccache htop + +# clangd: /usr/lib64/libc.so.6: version `GLIBC_2.18' not found (required by clangd) RUN wget https://github.com/clangd/clangd/releases/download/14.0.0/clangd-linux-14.0.0.zip && \ unzip clangd-linux-14.0.0.zip && \ mv clangd_14.0.0/bin/clangd /usr/local/bin/clangd && \ From 726170b3871da782d034d60b7100606fe89bad01 Mon Sep 17 00:00:00 2001 From: tsai Date: Fri, 29 Apr 2022 20:14:54 +0800 Subject: [PATCH 53/68] fix --- docker/Dockerfile | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index be74bd33d..d7ad635d0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -174,11 +174,8 @@ RUN curl -L $BAZEL_URL -o /usr/local/bin/bazel \ && chmod +x /usr/local/bin/bazel RUN yum install -y wget nasm rdma-core-devel rsync gdb ninja-build openblas-static devtoolset-7-gcc* vim ccache htop -# clangd: /usr/lib64/libc.so.6: version `GLIBC_2.18' not found (required by clangd) -RUN wget https://github.com/clangd/clangd/releases/download/14.0.0/clangd-linux-14.0.0.zip && \ - unzip clangd-linux-14.0.0.zip && \ - mv clangd_14.0.0/bin/clangd /usr/local/bin/clangd && \ - chmod +x /usr/local/bin/clangd && \ - rm clangd-linux-14.0.0.zip +RUN wget https://github.com/Oneflow-Inc/llvm-project/releases/download/llvmorg-13.0.0-maybe/clangd-13.AppImage && \ + mv clangd-13.AppImage /usr/local/bin/clangd && \ + chmod +x /usr/local/bin/clangd CMD ["/bin/bash"] From d8808a7992ef58476ff44a2baa8d9a8f4fd1bc33 Mon Sep 17 00:00:00 2001 From: tsai Date: Fri, 29 Apr 2022 20:15:15 +0800 Subject: [PATCH 54/68] refine --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d7ad635d0..e76a9bba5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -172,7 +172,7 @@ ENV SSL_CERT_FILE=/opt/_internal/certs.pem ARG BAZEL_URL="https://github.com/bazelbuild/bazel/releases/download/3.4.1/bazel-3.4.1-linux-x86_64" RUN curl -L $BAZEL_URL -o /usr/local/bin/bazel \ && chmod +x /usr/local/bin/bazel -RUN yum install -y wget nasm rdma-core-devel rsync gdb ninja-build openblas-static devtoolset-7-gcc* vim ccache htop +RUN yum install -y wget nasm rdma-core-devel rsync gdb ninja-build openblas-static devtoolset-7-gcc* vim ccache htop fuse-devel RUN wget https://github.com/Oneflow-Inc/llvm-project/releases/download/llvmorg-13.0.0-maybe/clangd-13.AppImage && \ mv clangd-13.AppImage /usr/local/bin/clangd && \ From fc1744fe9e0564cbc10547ecafd73577a70ddaf9 Mon Sep 17 00:00:00 2001 From: tsai Date: Fri, 29 Apr 2022 20:16:08 +0800 Subject: [PATCH 55/68] fix --- docker/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e76a9bba5..b5da4e032 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -174,8 +174,7 @@ RUN curl -L $BAZEL_URL -o /usr/local/bin/bazel \ && chmod +x /usr/local/bin/bazel RUN yum install -y wget nasm rdma-core-devel rsync gdb ninja-build openblas-static devtoolset-7-gcc* vim ccache htop fuse-devel -RUN wget https://github.com/Oneflow-Inc/llvm-project/releases/download/llvmorg-13.0.0-maybe/clangd-13.AppImage && \ - mv clangd-13.AppImage /usr/local/bin/clangd && \ +RUN wget https://github.com/Oneflow-Inc/llvm-project/releases/download/llvmorg-13.0.0-maybe/clangd-13.AppImage -O /usr/local/bin/clangd && \ chmod +x /usr/local/bin/clangd CMD ["/bin/bash"] From 40c8cb8ece7bdb5baae38194745be373f248b472 Mon Sep 17 00:00:00 2001 From: tsai Date: Mon, 13 Jun 2022 11:20:01 +0800 Subject: [PATCH 56/68] update clangd --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b5da4e032..e25cbeea5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -174,7 +174,7 @@ RUN curl -L $BAZEL_URL -o /usr/local/bin/bazel \ && chmod +x /usr/local/bin/bazel RUN yum install -y wget nasm rdma-core-devel rsync gdb ninja-build openblas-static devtoolset-7-gcc* vim ccache htop fuse-devel -RUN wget https://github.com/Oneflow-Inc/llvm-project/releases/download/llvmorg-13.0.0-maybe/clangd-13.AppImage -O /usr/local/bin/clangd && \ +RUN wget https://github.com/Oneflow-Inc/llvm-project/releases/download/maybe-14.0.4/clang-tidy-14.AppImage -O /usr/local/bin/clangd && \ chmod +x /usr/local/bin/clangd CMD ["/bin/bash"] From 228b5f4289deb68562481f32f683a72f6e272c8a Mon Sep 17 00:00:00 2001 From: tsai Date: Tue, 14 Jun 2022 16:58:43 +0800 Subject: [PATCH 57/68] add 11.6 --- .github/workflows/build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4464b0389..455f2533e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,6 +34,10 @@ jobs: matrix: tag-suffix: ["cuda11.2", "cuda10.2"] include: + - tag-suffix: "cuda11.6" + policy: "manylinux2014" + platform: "x86_64" + CUDA_BASE_IMAGE: "nvidia/cuda:11.6.1-cudnn8-devel-centos7" - tag-suffix: "cuda11.5" policy: "manylinux2014" platform: "x86_64" From 4a90cc5d561a1f167b0ea87530b3c425fd1085af Mon Sep 17 00:00:00 2001 From: tsai Date: Tue, 14 Jun 2022 16:59:43 +0800 Subject: [PATCH 58/68] use 11.6.2 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 455f2533e..916099758 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,7 +37,7 @@ jobs: - tag-suffix: "cuda11.6" policy: "manylinux2014" platform: "x86_64" - CUDA_BASE_IMAGE: "nvidia/cuda:11.6.1-cudnn8-devel-centos7" + CUDA_BASE_IMAGE: "nvidia/cuda:11.6.2-cudnn8-devel-centos7" - tag-suffix: "cuda11.5" policy: "manylinux2014" platform: "x86_64" From 80d25aa11aead20570b0a8dbb5b0e93038900f5b Mon Sep 17 00:00:00 2001 From: daquexian Date: Mon, 18 Jul 2022 15:47:20 +0800 Subject: [PATCH 59/68] install asan, ubsan and tsan --- docker/build_scripts/install-runtime-packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/build_scripts/install-runtime-packages.sh b/docker/build_scripts/install-runtime-packages.sh index ca2d4df6f..cbf328af2 100755 --- a/docker/build_scripts/install-runtime-packages.sh +++ b/docker/build_scripts/install-runtime-packages.sh @@ -104,7 +104,7 @@ elif [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then yum -y update yum -y install yum-utils curl yum-config-manager --enable extras - TOOLCHAIN_DEPS="devtoolset-10-binutils devtoolset-10-gcc devtoolset-10-gcc-c++ devtoolset-10-gcc-gfortran" + TOOLCHAIN_DEPS="devtoolset-10-binutils devtoolset-10-gcc devtoolset-10-gcc-c++ devtoolset-10-gcc-gfortran devtoolset-10-libasan-devel devtoolset-10-libubsan-devel devtoolset-10-libtsan-devel" if [ "${AUDITWHEEL_ARCH}" == "x86_64" ]; then # Software collection (for devtoolset-10) yum -y install centos-release-scl-rh From 190c92408855fe17ae664f2de1a9d6f484b2da2b Mon Sep 17 00:00:00 2001 From: daquexian Date: Mon, 18 Jul 2022 15:59:09 +0800 Subject: [PATCH 60/68] install in dockerfile Signed-off-by: daquexian --- docker/Dockerfile | 1 + docker/build_scripts/install-runtime-packages.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e25cbeea5..44e185f2e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -173,6 +173,7 @@ ARG BAZEL_URL="https://github.com/bazelbuild/bazel/releases/download/3.4.1/bazel RUN curl -L $BAZEL_URL -o /usr/local/bin/bazel \ && chmod +x /usr/local/bin/bazel RUN yum install -y wget nasm rdma-core-devel rsync gdb ninja-build openblas-static devtoolset-7-gcc* vim ccache htop fuse-devel +RUN yum install -y devtoolset-10-libasan-devel devtoolset-10-libubsan-devel devtoolset-10-libtsan-devel RUN wget https://github.com/Oneflow-Inc/llvm-project/releases/download/maybe-14.0.4/clang-tidy-14.AppImage -O /usr/local/bin/clangd && \ chmod +x /usr/local/bin/clangd diff --git a/docker/build_scripts/install-runtime-packages.sh b/docker/build_scripts/install-runtime-packages.sh index cbf328af2..ca2d4df6f 100755 --- a/docker/build_scripts/install-runtime-packages.sh +++ b/docker/build_scripts/install-runtime-packages.sh @@ -104,7 +104,7 @@ elif [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then yum -y update yum -y install yum-utils curl yum-config-manager --enable extras - TOOLCHAIN_DEPS="devtoolset-10-binutils devtoolset-10-gcc devtoolset-10-gcc-c++ devtoolset-10-gcc-gfortran devtoolset-10-libasan-devel devtoolset-10-libubsan-devel devtoolset-10-libtsan-devel" + TOOLCHAIN_DEPS="devtoolset-10-binutils devtoolset-10-gcc devtoolset-10-gcc-c++ devtoolset-10-gcc-gfortran" if [ "${AUDITWHEEL_ARCH}" == "x86_64" ]; then # Software collection (for devtoolset-10) yum -y install centos-release-scl-rh From bf3c074be80c0b03c0cc90c0c23a955d5fd2e894 Mon Sep 17 00:00:00 2001 From: tsai Date: Tue, 25 Oct 2022 13:38:44 +0800 Subject: [PATCH 61/68] add --- .github/workflows/build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 916099758..fd66ac87f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,6 +34,10 @@ jobs: matrix: tag-suffix: ["cuda11.2", "cuda10.2"] include: + - tag-suffix: "cuda11.8" + policy: "manylinux2014" + platform: "x86_64" + CUDA_BASE_IMAGE: "nvidia/cuda:11.8.0-cudnn8-devel-centos7" - tag-suffix: "cuda11.6" policy: "manylinux2014" platform: "x86_64" From 328e477069c80035adb3cd4db9632997e6284edd Mon Sep 17 00:00:00 2001 From: tsai Date: Thu, 22 Dec 2022 21:33:47 +0800 Subject: [PATCH 62/68] add117 --- .github/workflows/build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fd66ac87f..9df801a55 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,6 +38,10 @@ jobs: policy: "manylinux2014" platform: "x86_64" CUDA_BASE_IMAGE: "nvidia/cuda:11.8.0-cudnn8-devel-centos7" + - tag-suffix: "cuda11.7" + policy: "manylinux2014" + platform: "x86_64" + CUDA_BASE_IMAGE: "nvidia/cuda:11.7.0-cudnn8-devel-centos7" - tag-suffix: "cuda11.6" policy: "manylinux2014" platform: "x86_64" From de5063cadb23b8708e86498d731213150ef67f12 Mon Sep 17 00:00:00 2001 From: daquexian Date: Mon, 30 Jan 2023 19:39:49 +0800 Subject: [PATCH 63/68] install gcc9 Signed-off-by: daquexian --- docker/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 44e185f2e..12bfb02fc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -173,6 +173,7 @@ ARG BAZEL_URL="https://github.com/bazelbuild/bazel/releases/download/3.4.1/bazel RUN curl -L $BAZEL_URL -o /usr/local/bin/bazel \ && chmod +x /usr/local/bin/bazel RUN yum install -y wget nasm rdma-core-devel rsync gdb ninja-build openblas-static devtoolset-7-gcc* vim ccache htop fuse-devel +RUN yum install -y devtoolset-9 RUN yum install -y devtoolset-10-libasan-devel devtoolset-10-libubsan-devel devtoolset-10-libtsan-devel RUN wget https://github.com/Oneflow-Inc/llvm-project/releases/download/maybe-14.0.4/clang-tidy-14.AppImage -O /usr/local/bin/clangd && \ From fb0693807a10ee5a6ac5c3b2802860655158264e Mon Sep 17 00:00:00 2001 From: tsai Date: Wed, 25 Oct 2023 15:52:04 +0800 Subject: [PATCH 64/68] add --- .github/workflows/build.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9df801a55..141ae98e8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,6 +34,18 @@ jobs: matrix: tag-suffix: ["cuda11.2", "cuda10.2"] include: + - tag-suffix: "cuda12.2" + policy: "manylinux2014" + platform: "x86_64" + CUDA_BASE_IMAGE: "nvidia/cuda:12.2.2-cudnn8-devel-centos7" + - tag-suffix: "cuda12.1" + policy: "manylinux2014" + platform: "x86_64" + CUDA_BASE_IMAGE: "nvidia/cuda:12.1.1-cudnn8-devel-centos7" + - tag-suffix: "cuda12.0" + policy: "manylinux2014" + platform: "x86_64" + CUDA_BASE_IMAGE: "nvidia/cuda:12.0.1-cudnn8-devel-centos7" - tag-suffix: "cuda11.8" policy: "manylinux2014" platform: "x86_64" From bba403da5da597c3f7905dfc52b24989ee21ffb1 Mon Sep 17 00:00:00 2001 From: tsai Date: Wed, 25 Oct 2023 16:27:20 +0800 Subject: [PATCH 65/68] fix 11 --- .github/workflows/build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 141ae98e8..7cceba101 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,7 +53,7 @@ jobs: - tag-suffix: "cuda11.7" policy: "manylinux2014" platform: "x86_64" - CUDA_BASE_IMAGE: "nvidia/cuda:11.7.0-cudnn8-devel-centos7" + CUDA_BASE_IMAGE: "nvidia/cuda:11.7.1-cudnn8-devel-centos7" - tag-suffix: "cuda11.6" policy: "manylinux2014" platform: "x86_64" @@ -61,7 +61,7 @@ jobs: - tag-suffix: "cuda11.5" policy: "manylinux2014" platform: "x86_64" - CUDA_BASE_IMAGE: "nvidia/cuda:11.5.1-cudnn8-devel-centos7" + CUDA_BASE_IMAGE: "nvidia/cuda:11.5.2-cudnn8-devel-centos7" - tag-suffix: "cuda11.4" policy: "manylinux2014" platform: "x86_64" @@ -79,10 +79,10 @@ jobs: platform: "x86_64" CUDA_BASE_IMAGE: "nvidia/cuda:11.0.3-cudnn8-devel-centos7" target: "manylinux" - - tag-suffix: "cuda10.2" - policy: "manylinux2014" - platform: "x86_64" - CUDA_BASE_IMAGE: "nvidia/cuda:10.2-cudnn7-devel-centos7" + # - tag-suffix: "cuda10.2" + # policy: "manylinux2014" + # platform: "x86_64" + # CUDA_BASE_IMAGE: "nvidia/cuda:10.2-cudnn7-devel-centos7" - tag-suffix: "cpu" policy: "manylinux2014" platform: "x86_64" From 6aa4e8c03eab1d823acd67953b7e4193424bca3a Mon Sep 17 00:00:00 2001 From: tsai Date: Wed, 25 Oct 2023 17:22:55 +0800 Subject: [PATCH 66/68] fix --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7cceba101..57a9fefa6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -79,10 +79,10 @@ jobs: platform: "x86_64" CUDA_BASE_IMAGE: "nvidia/cuda:11.0.3-cudnn8-devel-centos7" target: "manylinux" - # - tag-suffix: "cuda10.2" - # policy: "manylinux2014" - # platform: "x86_64" - # CUDA_BASE_IMAGE: "nvidia/cuda:10.2-cudnn7-devel-centos7" + - tag-suffix: "cuda10.2" + policy: "manylinux2014" + platform: "x86_64" + CUDA_BASE_IMAGE: "nvidia/cuda:10.2-cudnn7-devel-centos7" - tag-suffix: "cpu" policy: "manylinux2014" platform: "x86_64" From a39b5115db7590833a344f89e56d8e9cfb06ef4e Mon Sep 17 00:00:00 2001 From: tsai Date: Wed, 13 Dec 2023 11:01:44 +0800 Subject: [PATCH 67/68] reduce --- .github/workflows/build.yml | 37 ------------------------------------- 1 file changed, 37 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 57a9fefa6..e60c560cb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,47 +42,10 @@ jobs: policy: "manylinux2014" platform: "x86_64" CUDA_BASE_IMAGE: "nvidia/cuda:12.1.1-cudnn8-devel-centos7" - - tag-suffix: "cuda12.0" - policy: "manylinux2014" - platform: "x86_64" - CUDA_BASE_IMAGE: "nvidia/cuda:12.0.1-cudnn8-devel-centos7" - tag-suffix: "cuda11.8" policy: "manylinux2014" platform: "x86_64" CUDA_BASE_IMAGE: "nvidia/cuda:11.8.0-cudnn8-devel-centos7" - - tag-suffix: "cuda11.7" - policy: "manylinux2014" - platform: "x86_64" - CUDA_BASE_IMAGE: "nvidia/cuda:11.7.1-cudnn8-devel-centos7" - - tag-suffix: "cuda11.6" - policy: "manylinux2014" - platform: "x86_64" - CUDA_BASE_IMAGE: "nvidia/cuda:11.6.2-cudnn8-devel-centos7" - - tag-suffix: "cuda11.5" - policy: "manylinux2014" - platform: "x86_64" - CUDA_BASE_IMAGE: "nvidia/cuda:11.5.2-cudnn8-devel-centos7" - - tag-suffix: "cuda11.4" - policy: "manylinux2014" - platform: "x86_64" - CUDA_BASE_IMAGE: "nvidia/cuda:11.4.3-cudnn8-devel-centos7" - - tag-suffix: "cuda11.3" - policy: "manylinux2014" - platform: "x86_64" - CUDA_BASE_IMAGE: "nvidia/cuda:11.3.1-cudnn8-devel-centos7" - - tag-suffix: "cuda11.2" - policy: "manylinux2014" - platform: "x86_64" - CUDA_BASE_IMAGE: "nvidia/cuda:11.2.2-cudnn8-devel-centos7" - - tag-suffix: "cuda11.0" - policy: "manylinux2014" - platform: "x86_64" - CUDA_BASE_IMAGE: "nvidia/cuda:11.0.3-cudnn8-devel-centos7" - target: "manylinux" - - tag-suffix: "cuda10.2" - policy: "manylinux2014" - platform: "x86_64" - CUDA_BASE_IMAGE: "nvidia/cuda:10.2-cudnn7-devel-centos7" - tag-suffix: "cpu" policy: "manylinux2014" platform: "x86_64" From c0a6a9d8b674c0b0f680211aeb92a1dc3f019f83 Mon Sep 17 00:00:00 2001 From: tsai Date: Wed, 13 Dec 2023 11:02:21 +0800 Subject: [PATCH 68/68] fix --- .github/workflows/build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e60c560cb..f829da083 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,7 +32,6 @@ jobs: strategy: fail-fast: false matrix: - tag-suffix: ["cuda11.2", "cuda10.2"] include: - tag-suffix: "cuda12.2" policy: "manylinux2014"