updated to handle both arm and arm64 #9
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Reusable build and test | ||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| arch: | ||
| type: string | ||
| description: "Architecture to build the image for (amd64/arm64)" | ||
| required: true | ||
| build_scope: | ||
| type: string | ||
| description: "Mage build scope" | ||
| default: 'all' | ||
| build_target: | ||
| type: string | ||
| description: "Mage build target" | ||
| default: 'prod' | ||
| memgraph_version: | ||
| type: string | ||
| description: "Memgraph version built into this image (format: X.Y.Z)" | ||
| default: '3.0.0' | ||
| memgraph_download_link: | ||
| type: string | ||
| description: "Memgraph package download link. Leave empty to use the official download link." | ||
| required: false | ||
| memgraph_ref: | ||
| type: string | ||
| description: "Memgraph submodule branch to build query modules from" | ||
| default: 'master' | ||
| memgraph_ref_update: | ||
| type: string | ||
| description: "Update Memgraph submodule to the latest commit" | ||
| default: true | ||
| env: | ||
| PY_VERSION: "3.12" | ||
| CORE_COUNT: "8" | ||
| MAGE_CONTAINER: "mage" | ||
| MEMGRAPH_PORT: 7687 | ||
| NEO4J_PORT: 7687 | ||
| NEO4J_CONTAINER: "neo4j_test" | ||
| MEMGRAPH_NETWORK: "memgraph_test_network" | ||
| OS: "ubuntu-24.04" | ||
| s3_region: "eu-west-1" | ||
| MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }} | ||
| MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }} | ||
| jobs: | ||
| BuildAndTest: | ||
| runs-on: ${{ contains('arm,arm64', inputs.arch) ? fromJSON('["self-hosted", "DockerMgBuild", "ARM64"]') : fromJSON('["self-hosted", "DockerMgBuild", "X64"]') }} | ||
| steps: | ||
| - name: Checkout repository and submodules | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| fetch-depth: 0 | ||
| submodules: recursive | ||
| token: ${{ secrets.GITHUB_TOKEN }} | ||
| - name: Checkout specific branch in submodule | ||
| run: | | ||
| cd cpp/memgraph | ||
| git checkout ${{ inputs.memgraph_ref }} | ||
| if [[ "${{ inputs.memgraph_ref_update }}" == "true" ]]; then | ||
| git pull | ||
| fi | ||
| # - name: Set up QEMU | ||
| # uses: docker/setup-qemu-action@v3 | ||
| - name: Set up Docker Buildx | ||
| id: buildx | ||
| uses: docker/setup-buildx-action@v3 | ||
| - name: Set up and check memgraph download link | ||
| run: | | ||
| mg_url=${{ inputs.memgraph_download_link }} | ||
| if [[ -z "${{ inputs.memgraph_download_link }}" ]]; then | ||
| mg_url="s3://download.memgraph.com/memgraph/v${{ inputs.memgraph_version }}/$OS${{ inputs.arch == 'arm64' && '-aarch64' || '' }}/memgraph_${{ inputs.memgraph_version }}-1_${{ inputs.arch }}.deb" | ||
| fi | ||
| echo "Download link: $mg_url" | ||
| url_protocol=${mg_url%%://*} | ||
| if [[ $url_protocol == "s3" ]]; then | ||
| echo "MEMGRAPH_S3_URI=${mg_url}" >> $GITHUB_ENV | ||
| elif [[ $url_protocol == "https" ]]; then | ||
| echo "Checking if the https link is valid" | ||
| if curl --output /dev/null --silent --head --fail $mg_url; then | ||
| echo "Memgraph download link is valid" | ||
| echo "MEMGRAPH_OBJECT_URL=${mg_url}" >> $GITHUB_ENV | ||
| else | ||
| echo "Memgraph download link is not valid" | ||
| exit 1 | ||
| fi | ||
| else | ||
| echo "Invalid download link protocol, only supporting s3:// and https://" | ||
| exit 1 | ||
| fi | ||
| - name: Download memgraph binary (Object URL) | ||
| if: env.MEMGRAPH_OBJECT_URL | ||
| run: curl -L ${{ env.MEMGRAPH_OBJECT_URL }} > memgraph-${{ inputs.arch }}.deb | ||
| - name: Setup AWS credentials | ||
| if: env.MEMGRAPH_S3_URI | ||
| uses: aws-actions/configure-aws-credentials@v4 | ||
| with: | ||
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | ||
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | ||
| aws-region: ${{ env.s3_region }} | ||
| - name: Download memgraph binary (S3 URI) | ||
| if: env.MEMGRAPH_S3_URI | ||
| run: aws s3 cp ${{ env.MEMGRAPH_S3_URI }} memgraph-${{ inputs.arch }}.deb | ||
| - name: Build and run Memgraph MAGE:${{ inputs.build_target }} | ||
| run: | | ||
| if [[ "${{ inputs.arch }}" == "arm64" ]]; then | ||
| MGBUILD_IMAGE=memgraph/mgbuild:v6_ubuntu-24.04-arm | ||
| else | ||
| MGBUILD_IMAGE=memgraph/mgbuild:v6_ubuntu-24.04 | ||
| fi | ||
| DOCKER_BUILDKIT=1 docker buildx build \ | ||
| --tag memgraph-mage:${{ inputs.build_target }} \ | ||
| --target ${{ inputs.build_target }} \ | ||
| --platform linux/${{ inputs.arch }} \ | ||
| --file Dockerfile.${{ inputs.build_scope == 'without ML' && 'no_ML' || 'release' }} \ | ||
| --build-arg MGBUILD_IMAGE=$MGBUILD_IMAGE \ | ||
| --load . | ||
| docker network create ${{ env.MEMGRAPH_NETWORK }} || true | ||
| docker run -d \ | ||
| --rm \ | ||
| --network ${{ env.MEMGRAPH_NETWORK }} \ | ||
| --name ${{ env.MAGE_CONTAINER }} \ | ||
| -e MEMGRAPH_ENTERPRISE_LICENSE=${{ env.MEMGRAPH_ENTERPRISE_LICENSE }} \ | ||
| -e MEMGRAPH_ORGANIZATION_NAME=${{ env.MEMGRAPH_ORGANIZATION_NAME }} \ | ||
| memgraph-mage:${{ inputs.build_target }} \ | ||
| --telemetry-enabled=False | ||
| - name: Rust library tests | ||
| if: inputs.build_target == 'dev' | ||
| run: | | ||
| docker exec -i -u memgraph ${{ env.MAGE_CONTAINER }} bash -c "source ~/.cargo/env && cd /mage/rust/rsmgp-sys && cargo fmt -- --check && cargo test" | ||
| - name: C++ Modules unit tests | ||
| if: inputs.build_target == 'dev' | ||
| run: | | ||
| docker exec -i -u memgraph ${{ env.MAGE_CONTAINER }} bash -c "cd /mage/cpp/build/ && ctest --output-on-failure -j${{ env.CORE_COUNT }}" | ||
| - name: Install Python dependencies | ||
| run: | | ||
| docker exec -i -u memgraph ${{ env.MAGE_CONTAINER }} bash -c "python3 -m pip install -r /mage/python/tests/requirements.txt --break-system-packages" | ||
| - name: Python modules unit tests | ||
| if: inputs.build_target == 'dev' && inputs.build_scope != 'without ML' | ||
| env: | ||
| PYTHONPATH: "$PWD/python" | ||
| run: | | ||
| docker exec -i -u memgraph ${{ env.MAGE_CONTAINER }} bash -c "cd /mage/python/ && python3 -m pytest ." | ||
| - name: Run End-to-end tests | ||
| if: inputs.arch != 'arm64' | ||
| env: | ||
| PYTHONPATH: "$PWD/e2e" | ||
| run: | | ||
| cd e2e | ||
| docker exec -i -u memgraph ${{ env.MAGE_CONTAINER }} bash -c \ | ||
| "cd /mage/e2e/ && python3 -m pytest . -k 'not cugraph ${{ inputs.build_scope == 'without ML' && 'and not tgn and not node_classification and not link_prediction' || '' }}'" | ||
| - name: Run End-to-end correctness tests | ||
| if: inputs.arch != 'arm64' | ||
| env: | ||
| PYTHONPATH: "$PWD/e2e" | ||
| run: | | ||
| ./run_e2e_correctness_tests.sh ${{ env.MEMGRAPH_PORT }} ${{ env.NEO4J_PORT }} ${{ env.NEO4J_CONTAINER }} ${{ env.MAGE_CONTAINER }} ${{ env.MEMGRAPH_NETWORK }} | ||
| - name: Cleanup | ||
| if: always() | ||
| run: | | ||
| docker stop ${{ env.MAGE_CONTAINER }} || true | ||
| docker rm ${{ env.MAGE_CONTAINER }} || true | ||
| docker rmi memgraph-mage:${{ inputs.build_target }} || true | ||
| docker network rm ${{ env.MEMGRAPH_NETWORK }} || true | ||
| docker stop ${{ env.NEO4J_CONTAINER }} || true | ||
