From f4cb166a4e1717d09138cadef881d41c07df1a71 Mon Sep 17 00:00:00 2001 From: ZackYoung Date: Thu, 9 Jan 2025 10:31:51 +0800 Subject: [PATCH] [Optimize][Docker]Optimize docker image build (#4126) Co-authored-by: zackyoungh --- .github/workflows/backend.yaml | 1 + .github/workflows/docker_build.yaml | 120 ++++++++++++++++++++++++++-- deploy/docker/Dockerfile | 45 +++-------- 3 files changed, 124 insertions(+), 42 deletions(-) diff --git a/.github/workflows/backend.yaml b/.github/workflows/backend.yaml index c1f23eb91d..4929e13651 100644 --- a/.github/workflows/backend.yaml +++ b/.github/workflows/backend.yaml @@ -23,6 +23,7 @@ on: pull_request: branches: - dev + - 1.2 paths-ignore: - 'docs/**' - '**/*.md' diff --git a/.github/workflows/docker_build.yaml b/.github/workflows/docker_build.yaml index 0333ed3dd1..d1ee09b57e 100644 --- a/.github/workflows/docker_build.yaml +++ b/.github/workflows/docker_build.yaml @@ -23,14 +23,98 @@ on: dinky_version: description: 'dinky version' required: true - docker_space: - description: 'docker space(eg: dinky)' - required: true jobs: - build_releases: - name: build releases + build_front: + name: Build_NPM runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + frontend: + - 'dinky-web/**' + - uses: actions/setup-node@v3 + with: + node-version: 16 + - name: Get npm cache directory + id: npm-cache-dir + run: | + echo "::set-output name=dir::$(npm config get cache)" + - uses: actions/cache@v3 + id: npm-cache # use this to check for `cache-hit` ==> if: steps.npm-cache.outputs.cache-hit != 'true' + with: + path: | + ${{ steps.npm-cache-dir.outputs.dir }} + dinky-web/dist + key: ${{ runner.os }}-node-${{ hashFiles('dinky-web/**/package.json') }} + restore-keys: | + ${{ runner.os }}-node- + - name: Install Dependencies + run: cd dinky-web && npm install --no-audit --progress=false --legacy-peer-deps + - name: Npm Web Build + run: cd dinky-web && npm run build + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: dinky-web + path: ./dinky-web/dist + + + build_release: + name: Build Release + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + flink: [ '1.14', '1.15', '1.16', '1.17', '1.18', '1.19', '1.20' ] + env: + MAVEN_OPTS: -Xmx2G -Xms2G + steps: + - uses: actions/checkout@v3 + # maven编译 + - name: Set up JDK + uses: actions/setup-java@v2 + with: + java-version: 8 + distribution: 'adopt' + - name: Cache local Maven repository + uses: actions/cache@v3 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/flink + key: ${{ runner.os }}-maven-${{ hashFiles('pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: Cache local Flink repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository/org/apache/flink + key: ${{ runner.os }}-${{ matrix.flink }}-maven-${{ hashFiles('pom.xml') }} + restore-keys: | + ${{ runner.os }}-${{ matrix.flink }}-maven- + - name: Build and Package + run: | + ./mvnw -B clean install \ + -Dmaven.test.skip=true \ + -Dspotless.check.skip=true \ + -Denforcer.skip=false \ + -Dmaven.javadoc.skip=true \ + -P prod,flink-single-version,flink-${{ matrix.flink }},maven-central \ + --no-snapshot-updates + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: dinky-realease-${{ matrix.flink }} + path: ./build/dinky-release*.tar.gz + + build_image: + name: build image + runs-on: ubuntu-latest + needs: [build_front,build_release] strategy: fail-fast: true matrix: @@ -39,6 +123,18 @@ jobs: # git checkout 代码 - name: Checkout uses: actions/checkout@v4 + - name: Download backed artifact + uses: actions/download-artifact@v4 + with: + name: dinky-realease-${{ matrix.flink }} + path: ./build + - name: Download front artifact + uses: actions/download-artifact@v4 + with: + name: dinky-web + path: ./build/dist + - run: | + tree ./build # 设置 QEMU, 后面 docker buildx 依赖此. - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -54,16 +150,28 @@ jobs: # DOCKERHUB_TOKEN: docker hub => Account Setting => Security 创建. username: ${{ secrets.DOCKER_IO_USER }} password: ${{ secrets.DOCKER_IO_PASS }} + # 登录 aliyun docker hub + - name: Login to Aliyun Docker + uses: docker/login-action@v3 + with: + registry: registry.cn-hangzhou.aliyuncs.com + username: ${{ secrets.DOCKER_ALIYUN_USER }} + password: ${{ secrets.DOCKER_ALIYUN_PASS }} # 构建 Docker 并推送到 Docker hub - name: Build and push id: docker_build uses: docker/build-push-action@v5 with: + platforms: linux/amd64,linux/arm64 file: ./deploy/docker/Dockerfile # 是否 docker push push: true + context: . build-args: | FLINK_VERSION=${{ matrix.flink }} DINKY_VERSION=${{ inputs.dinky_version }} tags: | - ${{inputs.docker_space}}/dinky-standalone-server:${{ inputs.dinky_version }}-flink${{ matrix.flink }} + dinkydocker/dinky-standalone-server:${{ inputs.dinky_version }}-flink${{ matrix.flink }} + registry.cn-hangzhou.aliyuncs.com/dinky/dinky-standalone-server:${{ inputs.dinky_version }}-flink${{ matrix.flink }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index d51347236c..8f00846d87 100755 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,54 +1,27 @@ ARG FLINK_VERSION ARG DINKY_VERSION -FROM flink:${FLINK_VERSION}-scala_2.12-java8 as flink-base - -FROM node:18.15.0-alpine3.17 AS ui-build -WORKDIR /build/ - -ENV NODE_OPTIONS=--openssl-legacy-provider -ENV UMI_ENV=production - -# 单独分离 package.json，是为了安装依赖可最大限度利用缓存 -ADD ./dinky-web/package.json /build/package.json -RUN npm install --legacy-peer-deps -ADD ./dinky-web . -RUN npm run build - -FROM maven:3.9-eclipse-temurin-8-alpine AS build -WORKDIR /build/ +FROM flink:${FLINK_VERSION}-scala_2.12-java8 as builder ARG FLINK_VERSION -ARG DINKY_VERSION -ENV FLINK_VERSION=${FLINK_VERSION} -ENV DINKY_VERSION=${DINKY_VERSION} - -ADD . . -COPY --from=ui-build /build/dist/ /build/dinky-web/dist/ - -RUN mvn package -Dmaven.test.skip=true -P prod,flink-single-version,flink-${FLINK_VERSION},fast -RUN mkdir release && \ - tar -C release -xvf build/dinky-release-${FLINK_VERSION}-*.tar.gz && \ - mv release/dinky-release-* release/dinky - - -FROM eclipse-temurin:8-jre-jammy +ADD build/dinky-release*.tar.gz /opt +RUN mv /opt/dinky-release* /opt/dinky && mkdir /opt/dinky/config/static +ADD build/dist/ /opt/dinky/config/static +FROM flink:${FLINK_VERSION}-scala_2.12-java8 ARG FLINK_VERSION +COPY --from=builder /opt/dinky /opt/dinky +RUN rm -f /opt/flink/lib/flink-table-planner-loader*.jar && cp /opt/flink/opt/flink-python*.jar /opt/flink/lib/ && cp /opt/flink/opt/flink-table-planner*.jar /opt/flink/lib/ 2>/dev/null || : && ln -s /opt/flink/lib/* /opt/dinky/extends/flink${FLINK_VERSION}/ ENV FLINK_VERSION=${FLINK_VERSION} ENV DINKY_HOME=/opt/dinky/ ENV H2_DB=./tmp/db/h2 -WORKDIR /opt/dinky/ USER root -COPY --from=build /build/release/dinky /opt/dinky/ -COPY --from=flink-base /opt/flink/lib/*.jar /opt/dinky/extends/flink${FLINK_VERSION}/flink/ -RUN rm -f /opt/dinky/extends/flink${FLINK_VERSION}/flink/flink-table-planner-loader*.jar +WORKDIR /opt/dinky/ -COPY --from=flink-base /opt/flink/opt/flink-table-planner*.jar /opt/dinky/extends/flink${FLINK_VERSION}/flink/ -RUN mkdir /opt/dinky/customJar && chmod -R 777 /opt/dinky/ && sed -i 's/-Xms512M -Xmx2048M -XX:PermSize=512M/-XX:+UseContainerSupport -XX:InitialRAMPercentage=70.0 -XX:MaxRAMPercentage=70.0/g' ./bin/auto.sh +RUN mkdir /opt/dinky/customJar && chmod -R 777 /opt/dinky/ && mkdir /dinky && chmod -R 777 /dinky/ && sed -i 's/-Xms512M -Xmx2048M -XX:ermSize=512M/-XX:+UseContainerSupport -XX:InitialRAMPercentage=70.0 -XX:MaxRAMPercentage=70.0/g' ./bin/auto.sh EXPOSE 8888