Skip to content

fix memory CB bugs and upgrade UTs to compatible with core changes #5916

fix memory CB bugs and upgrade UTs to compatible with core changes

fix memory CB bugs and upgrade UTs to compatible with core changes #5916

Workflow file for this run

name: Build and Test ml-commons
# This workflow is triggered on pull requests and push to any branches
on:
push:
branches-ignore:
- 'backport/**'
- 'create-pull-request/**'
- 'dependabot/**'
pull_request_target:
types: [opened, synchronize, reopened]
permissions:
id-token: write
contents: read
jobs:
Get-CI-Image-Tag:
uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
with:
product: opensearch
Build-ml-linux:
needs: Get-CI-Image-Tag
strategy:
matrix:
java: [11, 17, 21]
name: Build and Test MLCommons Plugin on linux
if: github.repository == 'opensearch-project/ml-commons'
environment: ml-commons-cicd-env
outputs:
build-test-linux: ${{ steps.step-build-test-linux.outputs.build-test-linux }}
runs-on: ubuntu-latest
container:
# using the same image which is used by opensearch-build team to build the OpenSearch Distribution
# this image tag is subject to change as more dependencies and updates will arrive over time
image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
# need to switch to root so that github actions can install runner binary on container without permission issues.
options: --user root
steps:
- name: Setup Java ${{ matrix.java }}
uses: actions/setup-java@v1
with:
java-version: ${{ matrix.java }}
- uses: aws-actions/configure-aws-credentials@v2
with:
role-to-assume: ${{ secrets.ML_ROLE }}
aws-region: us-west-2
- name: Checkout MLCommons
uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Build and Run Tests
id: step-build-test-linux
run: |
chown -R 1000:1000 `pwd`
su `id -un 1000` -c 'whoami && java -version &&
export OPENAI_KEY=`aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text` &&
export COHERE_KEY=`aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text` &&
echo "::add-mask::$OPENAI_KEY" &&
echo "::add-mask::$COHERE_KEY" &&
echo "build and run tests" && ./gradlew build &&
echo "Publish to Maven Local" && ./gradlew publishToMavenLocal &&
echo "Multi Nodes Integration Testing" && ./gradlew integTest -PnumNodes=3'
plugin=`basename $(ls plugin/build/distributions/*.zip)`
echo $plugin
mv -v plugin/build/distributions/$plugin ./
echo "build-test-linux=$plugin" >> $GITHUB_OUTPUT
- name: Upload Coverage Report
uses: codecov/codecov-action@v1
with:
flags: ml-commons
token: ${{ secrets.CODECOV_TOKEN }}
- uses: actions/upload-artifact@v3
with:
name: ml-plugin-linux-${{ matrix.java }}
path: ${{ steps.step-build-test-linux.outputs.build-test-linux }}
if-no-files-found: error
Test-ml-linux-docker:
needs: Build-ml-linux
strategy:
matrix:
java: [11, 17, 21]
name: Test MLCommons Plugin on linux docker
if: github.repository == 'opensearch-project/ml-commons'
environment: ml-commons-cicd-env
runs-on: ubuntu-latest
steps:
- uses: aws-actions/configure-aws-credentials@v2
with:
role-to-assume: ${{ secrets.ML_ROLE }}
aws-region: us-west-2
- name: Checkout MLCommons
uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.head.sha }}
- uses: actions/download-artifact@v3
with:
name: ml-plugin-linux-${{ matrix.java }}
- name: Pull and Run Docker
run: |
plugin=${{ needs.Build-ml-linux.outputs.build-test-linux }}
version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-3`
plugin_version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-4`
qualifier=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-1`
if [ -n "$qualifier" ] && [ "$qualifier" != "SNAPSHOT" ]; then
qualifier=-${qualifier}
else
qualifier=""
fi
docker_version=$version$qualifier
echo plugin version plugin_version qualifier docker_version
echo "($plugin) ($version) ($plugin_version) ($qualifier) ($docker_version)"
pwd && ls -l ./$plugin
if docker pull opensearchstaging/opensearch:$docker_version
then
echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile
echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-skills ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-skills; fi" >> Dockerfile
echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-ml ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-ml; fi" >> Dockerfile
echo "COPY $plugin /tmp/" >> Dockerfile
echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile
docker build -t opensearch-ml:test .
echo "imagePresent=true" >> $GITHUB_ENV
else
echo "imagePresent=false" >> $GITHUB_ENV
fi
- name: Generate Password For Admin
id: genpass
run: |
PASSWORD=$(openssl rand -base64 20 | tr -dc 'A-Za-z0-9!@#$%^&*()_+=-')
echo "password={$PASSWORD}" >> $GITHUB_OUTPUT
- name: Run Docker Image
if: env.imagePresent == 'true'
run: |
cd ..
docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" -e OPENSEARCH_INITIAL_ADMIN_PASSWORD=${{ steps.genpass.outputs.password }} opensearch-ml:test
sleep 90
- name: Run MLCommons Test
if: env.imagePresent == 'true'
run: |
security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:${{ steps.genpass.outputs.password }} --insecure |grep opensearch-security|wc -l`
export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text)
export COHERE_KEY=$(aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text)
echo "::add-mask::$OPENAI_KEY"
echo "::add-mask::$COHERE_KEY"
if [ $security -gt 0 ]
then
echo "Security plugin is available"
./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=${{ steps.genpass.outputs.password }}
else
echo "Security plugin is NOT available"
./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster"
fi
- name: Upload Coverage Report
uses: codecov/codecov-action@v3
with:
flags: ml-commons
token: ${{ secrets.CODECOV_TOKEN }}
Build-ml-windows:
strategy:
matrix:
java: [11, 17, 21]
name: Build and Test MLCommons Plugin on Windows
if: github.repository == 'opensearch-project/ml-commons'
environment: ml-commons-cicd-env
runs-on: windows-latest
steps:
- name: Setup Java ${{ matrix.java }}
uses: actions/setup-java@v1
with:
java-version: ${{ matrix.java }}
- uses: aws-actions/configure-aws-credentials@v2
with:
role-to-assume: ${{ secrets.ML_ROLE }}
aws-region: us-west-2
# ml-commons
- name: Checkout MLCommons
uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Build and Run Tests
shell: bash
run: |
export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text)
export COHERE_KEY=$(aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text)
echo "::add-mask::$OPENAI_KEY"
echo "::add-mask::$COHERE_KEY"
./gradlew.bat build
- name: Publish to Maven Local
run: |
./gradlew publishToMavenLocal
# - name: Multi Nodes Integration Testing
# shell: bash
# run: |
# export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text)
# export COHERE_KEY=$(aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text)
# echo "::add-mask::$OPENAI_KEY"
# echo "::add-mask::$COHERE_KEY"
# ./gradlew integTest -PnumNodes=3