From ee7febadc6bb2fab648a0f13eb87328075389cff Mon Sep 17 00:00:00 2001 From: gongsu832 Date: Sat, 19 Oct 2024 12:40:55 -0400 Subject: [PATCH] Switch from docker.io to ghcr.io (#2976) * Switch from docker.io to ghcr.io * Increase push retry limit * Increase docker API timeout * Fix python 3.12 SyntaxWarning: invalid escape sequence '\s' * Update default base image to ghcr.io/onnxmlir/ubuntu:jammy Signed-off-by: Gong Su --- .buildbot/Jenkinsfile | 8 ++++---- .buildbot/jenkins-build-llvm-project.py | 4 ++-- .buildbot/jenkins-build-onnx-mlir.py | 2 +- .buildbot/jenkins-watch-llvm-project.py | 2 +- .buildbot/jenkins_common.py | 5 +++-- docker/Dockerfile.llvm-project | 2 +- docker/onnx-mlir.py | 2 +- docs/Docker.md | 14 +++++++------- docs/TestingHighLevel.md | 2 +- docs/docker-example/Dockerfile | 2 +- 10 files changed, 22 insertions(+), 21 deletions(-) diff --git a/.buildbot/Jenkinsfile b/.buildbot/Jenkinsfile index e63fb6de3d..daab23bbce 100644 --- a/.buildbot/Jenkinsfile +++ b/.buildbot/Jenkinsfile @@ -49,25 +49,25 @@ def call() { DOCKER_DAEMON_SOCKET = 'unix://var/run/docker.sock' DOCKER_REGISTRY_TOKEN_ACCESS = 'true' /* Settings for docker.io */ + /* DOCKER_REGISTRY_HOST_NAME = '' DOCKER_REGISTRY_USER_NAME = 'onnxmlir' DOCKER_REGISTRY_LOGIN_NAME = 'onnxmlir' + */ /* Settings for ghcr.io */ - /* DOCKER_REGISTRY_HOST_NAME = 'ghcr.io' DOCKER_REGISTRY_USER_NAME = 'onnxmlir' DOCKER_REGISTRY_LOGIN_NAME = 'onnxmlir' - */ /* Credentials defined in Jenkins */ JENKINS_REST_API_TOKEN = credentials('Jenkins-REST-API-Token') GITHUB_REPO_ACCESS_TOKEN = credentials('jenkins-buildbot-access-token') /* Settings for docker.io */ + /* DOCKER_REGISTRY_LOGIN_TOKEN = credentials('DOCKERHUB-ONNXMLIR-TOKEN') + */ /* Settings for ghcr.io */ - /* DOCKER_REGISTRY_LOGIN_TOKEN = credentials('GITHUB-ONNXMLIR-TOKEN') - */ /* Environment variables that depend on the arch */ JENKINS_REST_API_URL = sh(returnStdout: true, diff --git a/.buildbot/jenkins-build-llvm-project.py b/.buildbot/jenkins-build-llvm-project.py index de7b8f2ba7..06c5bb115e 100755 --- a/.buildbot/jenkins-build-llvm-project.py +++ b/.buildbot/jenkins-build-llvm-project.py @@ -7,7 +7,7 @@ LLVM_PROJECT_DOCKERFILE = "docker/Dockerfile.llvm-project" LLVM_PROJECT_GITHUB_URL = "https://api.github.com/repos/llvm/llvm-project" LLVM_PROJECT_BASE_IMAGE = { - "static": "ubuntu:jammy", + "static": "ghcr.io/onnxmlir/ubuntu:jammy", "shared": "registry.access.redhat.com/ubi8-minimal:latest", } LLVM_PROJECT_IMAGE = { @@ -187,7 +187,7 @@ def setup_per_pr_llvm_project(image_type, exp): ): if "stream" in line: # Keep track of the latest successful image layer - m = re.match("^\s*---> ([0-9a-f]+)$", line["stream"]) + m = re.match(r"^\s*---> ([0-9a-f]+)$", line["stream"]) if m: layer_sha256 = m.group(1) print(line["stream"], end="", flush=True) diff --git a/.buildbot/jenkins-build-onnx-mlir.py b/.buildbot/jenkins-build-onnx-mlir.py index f3c83c9d7a..4ff4d589d7 100755 --- a/.buildbot/jenkins-build-onnx-mlir.py +++ b/.buildbot/jenkins-build-onnx-mlir.py @@ -178,7 +178,7 @@ def build_per_pr_onnx_mlir(image_type, exp): ): if "stream" in line: # Keep track of the latest successful image layer - m = re.match("^\s*---> ([0-9a-f]+)$", line["stream"]) + m = re.match(r"^\s*---> ([0-9a-f]+)$", line["stream"]) if m: layer_sha256 = m.group(1) print(line["stream"], end="", flush=True) diff --git a/.buildbot/jenkins-watch-llvm-project.py b/.buildbot/jenkins-watch-llvm-project.py index abaae7d198..b6c6c396bb 100755 --- a/.buildbot/jenkins-watch-llvm-project.py +++ b/.buildbot/jenkins-watch-llvm-project.py @@ -336,7 +336,7 @@ def build_watch_image(repo, commit, dockerfile, base_image, image_repo, image_ta ): if "stream" in line: # Keep track of the latest successful image layer - m = re.match("^\s*---> ([0-9a-f]+)$", line["stream"]) + m = re.match(r"^\s*---> ([0-9a-f]+)$", line["stream"]) if m: layer_sha256 = m.group(1) print(line["stream"], end="", flush=True) diff --git a/.buildbot/jenkins_common.py b/.buildbot/jenkins_common.py index d2fbb1a1e2..eb36a16ae6 100755 --- a/.buildbot/jenkins_common.py +++ b/.buildbot/jenkins_common.py @@ -31,10 +31,11 @@ MEMORY_IN_GB = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") / (1024.0**3) NPROC = str(math.ceil(min(max(2, MEMORY_IN_GB / 8), os.cpu_count()))) -RETRY_LIMIT = 5 +RETRY_LIMIT = 10 READ_CHUNK_SIZE = 1024 * 1024 BASE_BRANCH = "main" +DOCKER_API_TIMEOUT = 3600 DOCKER_DIST_MANIFEST = "application/vnd.docker.distribution.manifest.v2+json" DOCKER_DIST_MANIFEST_LIST = "application/vnd.docker.distribution.manifest.list.v2+json" @@ -47,7 +48,7 @@ docker_registry_login_token = os.getenv("DOCKER_REGISTRY_LOGIN_TOKEN") docker_registry_token_access = os.getenv("DOCKER_REGISTRY_TOKEN_ACCESS") docker_rwlock = fasteners.InterProcessReaderWriterLock(docker_pushpull_rwlock) -docker_api = docker.APIClient(base_url=docker_daemon_socket) +docker_api = docker.APIClient(base_url=docker_daemon_socket, timeout=DOCKER_API_TIMEOUT) github_repo_access_token = os.getenv("GITHUB_REPO_ACCESS_TOKEN") github_repo_name = os.getenv("GITHUB_REPO_NAME") diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project index c32160915f..ecee3d1ea8 100644 --- a/docker/Dockerfile.llvm-project +++ b/docker/Dockerfile.llvm-project @@ -1,5 +1,5 @@ # By default, use ubuntu:jammy, remember to change Jenkins build script as well -ARG BASE_IMAGE="ubuntu:jammy" +ARG BASE_IMAGE="ghcr.io/onnxmlir/ubuntu:jammy" FROM ${BASE_IMAGE} # Label the image for various checking and cleanup diff --git a/docker/onnx-mlir.py b/docker/onnx-mlir.py index b26f2c49aa..d9d95db65e 100755 --- a/docker/onnx-mlir.py +++ b/docker/onnx-mlir.py @@ -33,7 +33,7 @@ import sys DOCKER_SOCKET = "/var/run/docker.sock" -ONNX_MLIR_IMAGE = "onnxmlir/onnx-mlir" +ONNX_MLIR_IMAGE = "ghcr.io/onnxmlir/onnx-mlir" KNOWN_INPUT_TYPE = (".onnx", ".json", ".mlir") mount_dirs = [] diff --git a/docs/Docker.md b/docs/Docker.md index bcc071a948..40b60c4757 100644 --- a/docs/Docker.md +++ b/docs/Docker.md @@ -12,7 +12,7 @@ There are three ways to use ONNX-MLIR with Docker. An easy way to get started with ONNX-MLIR is to use a prebuilt Docker image. These images are created as a result of a successful merge build on the trunk. This means that the latest image represents the tip of the trunk. -Currently there are both Release and Debug mode images for `amd64`, `ppc64le` and `s390x` saved in Docker Hub as, respectively, [onnxmlir/onnx-mlir](https://hub.docker.com/r/onnxmlir/onnx-mlir) and [onnxmlir/onnx-mlir-dev](https://hub.docker.com/r/onnxmlir/onnx-mlir-dev). +Currently there are both Release and Debug mode images for `amd64`, `ppc64le` and `s390x` saved in Docker Hub as, respectively, [onnxmlir/onnx-mlir](https://github.com/users/onnxmlir/packages/container/onnx-mlir) and [onnxmlir/onnx-mlir-dev](https://github.com/users/onnxmlir/packages/container/onnx-mlir-dev). To use one of these images either pull it directly from Docker Hub, launch a container and run an interactive bash shell in it, or use it as the base image in a Dockerfile. Here are the differences between the two Docker images. @@ -53,7 +53,7 @@ The Dockerfile is shown here, and should be modified according to one's need. Th [same-as-file]: <> (docs/docker-example/Dockerfile) ``` -FROM onnxmlir/onnx-mlir-dev +FROM ghcr.io/onnxmlir/onnx-mlir-dev WORKDIR /workdir ENV HOME=/workdir @@ -122,9 +122,9 @@ cd ~/DockerOnnxMlir # Edit the Dockerfile. vi Dockerfile # Build the Docker image. -docker build --tag onnx-mlir-dev . +docker build --tag ghcr.io/onnxmlir/onnx-mlir-dev . # Start a container using the Docker dashboard or a docker run command. -docker run -it onnx-mlir-dev +docker run -it ghcr.io/onnxmlir/onnx-mlir-dev ``` **NOTE:** If you are using a MacBook with the Apple M1 chip, please follow the steps below for configuration: @@ -135,11 +135,11 @@ cd ~/DockerOnnxMlir # Edit the Dockerfile. vi Dockerfile # Pull the Docker image with the specified platform -docker pull --platform linux/amd64 onnxmlir/onnx-mlir-dev +docker pull --platform linux/amd64 ghcr.io/onnxmlir/onnx-mlir-dev # Build the Docker image. -docker build --platform linux/amd64 --tag onnx-mlir-dev . +docker build --platform linux/amd64 --tag ghcr.io/onnxmlir/onnx-mlir-dev . # Start a container using the Docker dashboard or a docker run command. -docker run --platform linux/amd64 -it onnx-mlir-dev +docker run --platform linux/amd64 -it ghcr.io/onnxmlir/onnx-mlir-dev ``` Tip: Instead of adding the platform flag for every docker pull, build, and run command. You can set the environment variable `DOCKER_DEFAULT_PLATFORM` and use the first set of steps: diff --git a/docs/TestingHighLevel.md b/docs/TestingHighLevel.md index 48fc363447..fa53e66bce 100644 --- a/docs/TestingHighLevel.md +++ b/docs/TestingHighLevel.md @@ -40,7 +40,7 @@ If you run into protobuf related errors during the build, check the following po * llvm-project, onnx, and/or onnx-mlir may detect different versions of python3 (so watch their cmake output) if you have multiple python versions installed * cmake caches stuff and you should never use "make clean" when rebuilding. Instead remove everything under the build tree and start from scratch. -These and many other trickeries for setting up the build env are the reason why we recommend using the `onnxmlir/onnx-mlir-dev` docker image for development. +These and many other trickeries for setting up the build env are the reason why we recommend using the [onnxmlir/onnx-mlir-dev](https://github.com/users/onnxmlir/packages/container/onnx-mlir-dev) docker image for development. ## High level testing of ONNX-MLIR diff --git a/docs/docker-example/Dockerfile b/docs/docker-example/Dockerfile index 3db740090c..f44ff97be7 100644 --- a/docs/docker-example/Dockerfile +++ b/docs/docker-example/Dockerfile @@ -1,4 +1,4 @@ -FROM onnxmlir/onnx-mlir-dev +FROM ghcr.io/onnxmlir/onnx-mlir-dev WORKDIR /workdir ENV HOME=/workdir