From 08151df88d9f15a079c8e5c4d3a0ba666b7a4c5b Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 30 Oct 2023 14:29:29 -0400 Subject: [PATCH 1/6] Onboarding jenkins prod docker images to github actions Signed-off-by: Peter Zhu --- .github/workflows/benchmark.yml | 20 +++++-- .../workflows/test_build_multi_platform.yml | 54 +++++++++++++++++-- .github/workflows/test_bwc.yml | 22 ++++++-- 3 files changed, 86 insertions(+), 10 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c5caeedd0..edf7a0bbb 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -8,7 +8,13 @@ on: - "*" jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + Build-ad: + needs: Get-CI-Image-Tag strategy: matrix: java: [17] @@ -16,18 +22,26 @@ jobs: name: Run Anomaly detection model performance benchmark runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root steps: - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v3 with: + distribution: 'temurin' java-version: ${{ matrix.java }} # anomaly-detection - name: Checkout AD - uses: actions/checkout@v4 + uses: actions/checkout@v3 - name: Build and Run Tests run: | - ./gradlew ':test' --tests "org.opensearch.ad.ml.HCADModelPerfTests" -Dtests.seed=2AEBDBBAE75AC5E0 -Dtests.security.manager=false -Dtests.locale=es-CU -Dtests.timezone=Chile/EasterIsland -Dtest.logs=true -Dmodel-benchmark=true - ./gradlew integTest --tests "org.opensearch.ad.e2e.SingleStreamModelPerfIT" -Dtests.seed=60CDDB34427ACD0C -Dtests.security.manager=false -Dtests.locale=kab-DZ -Dtests.timezone=Asia/Hebron -Dtest.logs=true -Dmodel-benchmark=true \ No newline at end of file + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew ':test' --tests 'org.opensearch.ad.ml.HCADModelPerfTests' -Dtests.seed=2AEBDBBAE75AC5E0 -Dtests.security.manager=false -Dtests.locale=es-CU -Dtests.timezone=Chile/EasterIsland -Dtest.logs=true -Dmodel-benchmark=true" + su `id -un 1000` -c "./gradlew integTest --tests 'org.opensearch.ad.e2e.SingleStreamModelPerfIT' -Dtests.seed=60CDDB34427ACD0C -Dtests.security.manager=false -Dtests.locale=kab-DZ -Dtests.timezone=Asia/Hebron -Dtest.logs=true -Dmodel-benchmark=true" diff --git a/.github/workflows/test_build_multi_platform.yml b/.github/workflows/test_build_multi_platform.yml index db0e1dfc0..ac81a3697 100644 --- a/.github/workflows/test_build_multi_platform.yml +++ b/.github/workflows/test_build_multi_platform.yml @@ -8,6 +8,11 @@ on: - "*" jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + Build-ad-windows: strategy: matrix: @@ -20,6 +25,7 @@ jobs: - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v3 with: + distribution: 'temurin' java-version: ${{ matrix.java }} - name: Checkout Anomaly Detection uses: actions/checkout@v4 @@ -38,15 +44,56 @@ jobs: with: file: ./build/reports/jacoco/test/jacocoTestReport.xml flags: plugin - Build-ad: + + Build-ad-linux: + needs: Get-CI-Image-Tag + strategy: + matrix: + java: [11, 17, 20] + fail-fast: false + name: Build and Test Anomaly detection Plugin + runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + + env: + JENKINS_URL: build.ci.opensearch.org + + steps: + - name: Setup Java ${{ matrix.java }} + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + + - name: Checkout AD + uses: actions/checkout@v3 + + - name: Assemble / build / mavenlocal / integTest + run: | + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew assemble && + ./gradlew build && + ./gradlew publishToMavenLocal && + ./gradlew integTest -PnumNodes=3" + - name: Upload Coverage Report + uses: codecov/codecov-action@v3 + with: + file: ./build/reports/jacoco/test/jacocoTestReport.xml + flags: plugin + + Build-ad-macos: strategy: matrix: java: [11,17,20] - os: [ubuntu-latest, macos-latest] fail-fast: false name: Build and Test Anomaly detection Plugin - runs-on: ${{ matrix.os }} + runs-on: macos-latest env: JENKINS_URL: build.ci.opensearch.org @@ -54,6 +101,7 @@ jobs: - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v3 with: + distribution: 'temurin' java-version: ${{ matrix.java }} - name: Checkout AD diff --git a/.github/workflows/test_bwc.yml b/.github/workflows/test_bwc.yml index 3c86f238a..8c9d68133 100644 --- a/.github/workflows/test_bwc.yml +++ b/.github/workflows/test_bwc.yml @@ -8,7 +8,12 @@ on: - "*" jobs: - Build-ad: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + + Build-ad-linux: strategy: matrix: java: [11,17,20] @@ -16,22 +21,30 @@ jobs: name: Test Anomaly detection BWC runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root steps: - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v3 with: + distribution: 'temurin' java-version: ${{ matrix.java }} # anomaly-detection - name: Checkout AD - uses: actions/checkout@v4 + uses: actions/checkout@v3 - name: Assemble anomaly-detection run: | plugin_version=`./gradlew properties -q | grep "opensearch_build:" | awk '{print $2}'` echo plugin_version $plugin_version - ./gradlew assemble + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew assemble" echo "Creating ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..." mkdir -p ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version echo "Copying ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..." @@ -42,4 +55,5 @@ jobs: - name: Run AD Backwards Compatibility Tests run: | echo "Running backwards compatibility tests ..." - ./gradlew bwcTestSuite -Dtests.security.manager=false + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew bwcTestSuite -Dtests.security.manager=false" From fbccdeefaaedf444bf3674b8c3a076e939f9a447 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 30 Oct 2023 19:04:32 -0400 Subject: [PATCH 2/6] Add temurin Signed-off-by: Peter Zhu --- .github/workflows/test_security.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index 74229918d..861da6dc4 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -21,6 +21,7 @@ jobs: - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v3 with: + distribution: 'temurin' java-version: ${{ matrix.java }} # time-series-analytics From 65971d8c7548fa56105bf7f12e8898babd8070f1 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 30 Oct 2023 19:10:29 -0400 Subject: [PATCH 3/6] Add temurin Signed-off-by: Peter Zhu --- .github/workflows/test_bwc.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_bwc.yml b/.github/workflows/test_bwc.yml index 8c9d68133..f0080664e 100644 --- a/.github/workflows/test_bwc.yml +++ b/.github/workflows/test_bwc.yml @@ -41,15 +41,16 @@ jobs: - name: Assemble anomaly-detection run: | - plugin_version=`./gradlew properties -q | grep "opensearch_build:" | awk '{print $2}'` - echo plugin_version $plugin_version chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew properties -q | grep 'opensearch_build:' | awk '{print $2}'" > /tmp/plugin_version.txt + plugin_version=`cat /tmp/plugin_version.txt` + echo plugin_version $plugin_version su `id -un 1000` -c "./gradlew assemble" echo "Creating ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..." - mkdir -p ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version + su `id -un 1000` -c "mkdir -p ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version" echo "Copying ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..." ls ./build/distributions/ - cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version + su `id -un 1000` -c "cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version" echo "Copied ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..." ls ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version - name: Run AD Backwards Compatibility Tests From 4bd6d202d0f8b949f05b3b517888431163523001 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 30 Oct 2023 19:12:14 -0400 Subject: [PATCH 4/6] Add more Signed-off-by: Peter Zhu --- .github/workflows/test_bwc.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_bwc.yml b/.github/workflows/test_bwc.yml index f0080664e..0dc33d264 100644 --- a/.github/workflows/test_bwc.yml +++ b/.github/workflows/test_bwc.yml @@ -14,6 +14,7 @@ jobs: product: opensearch Build-ad-linux: + needs: Get-CI-Image-Tag strategy: matrix: java: [11,17,20] From b4226e9eb7b44860b6a63b6231c8341d395c75c8 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 30 Oct 2023 19:20:57 -0400 Subject: [PATCH 5/6] Add more Signed-off-by: Peter Zhu --- .github/workflows/test_bwc.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_bwc.yml b/.github/workflows/test_bwc.yml index 0dc33d264..e3ed491dc 100644 --- a/.github/workflows/test_bwc.yml +++ b/.github/workflows/test_bwc.yml @@ -42,16 +42,15 @@ jobs: - name: Assemble anomaly-detection run: | + plugin_version=`./gradlew properties -q | grep "opensearch_build:" | awk '{print $2}'` chown -R 1000:1000 `pwd` - su `id -un 1000` -c "./gradlew properties -q | grep 'opensearch_build:' | awk '{print $2}'" > /tmp/plugin_version.txt - plugin_version=`cat /tmp/plugin_version.txt` - echo plugin_version $plugin_version + echo plugin_version ($plugin_version) su `id -un 1000` -c "./gradlew assemble" echo "Creating ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..." - su `id -un 1000` -c "mkdir -p ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version" + mkdir -p ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version echo "Copying ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..." ls ./build/distributions/ - su `id -un 1000` -c "cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version" + cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version echo "Copied ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..." ls ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version - name: Run AD Backwards Compatibility Tests From 04c5aa90d1c761fdeecb3d2ea7c6d174c3faa82b Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 30 Oct 2023 19:28:15 -0400 Subject: [PATCH 6/6] Add more Signed-off-by: Peter Zhu --- .github/workflows/test_bwc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_bwc.yml b/.github/workflows/test_bwc.yml index 355613324..10d41b70c 100644 --- a/.github/workflows/test_bwc.yml +++ b/.github/workflows/test_bwc.yml @@ -44,7 +44,7 @@ jobs: run: | plugin_version=`./gradlew properties -q | grep "opensearch_build:" | awk '{print $2}'` chown -R 1000:1000 `pwd` - echo plugin_version ($plugin_version) + echo plugin_version $plugin_version su `id -un 1000` -c "./gradlew assemble" echo "Creating ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..." su `id -un 1000` -c "mkdir -p ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version"