From e19331964c611f080152b193eb9e8d46d517e176 Mon Sep 17 00:00:00 2001 From: Marya Date: Sat, 2 Dec 2023 22:29:31 +0000 Subject: [PATCH 1/3] Add build and run Resnet50 and VGG16 actions to the tartan CI job. --- .../build_vgg_resnet_action/action.yml | 75 +++++++++++++++++++ .github/workflows/run_tartan.yml | 36 ++++++++- 2 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 .github/actions/build_vgg_resnet_action/action.yml diff --git a/.github/actions/build_vgg_resnet_action/action.yml b/.github/actions/build_vgg_resnet_action/action.yml new file mode 100644 index 000000000..16b75066d --- /dev/null +++ b/.github/actions/build_vgg_resnet_action/action.yml @@ -0,0 +1,75 @@ +name: Setup and build networks; VGG16, Resnet50 +description: Build and run VV16 and Resnet50 with oneAPI base toolkit and oneAPI-Construction-Kit + +inputs: + workspace: + description: 'Main workflow workspace' + default: ${{ github.workspace }} + +runs: + # We don't want a new docker just a list of steps, so mark as composite + using: "composite" + steps: + - name: setup python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + + - name: Install Python libraries and system dependencies + shell: bash + run: | + pip install requests numpy h5py Pillow + sudo add-apt-repository universe + sudo apt-get update + sudo apt-get install -y libblas-dev libopenblas64-dev libopenblas-dev libpthread-stubs0-dev libboost-all-dev + + - name: Clone portDNN + shell: bash + run: git clone --recursive --depth 1 --single-branch https://github.com/codeplaysoftware/portDNN.git + + - name: Download Daily Release + shell: bash + run: | + wget "https://github.com/intel/llvm/releases/download/nightly-2023-12-18/sycl_linux.tar.gz" + mkdir linux_nightly_release + tar -xzf sycl_linux.tar.gz -C linux_nightly_release + + - name: Set up environment and build networks + shell: bash + run: | + export LD_LIBRARY_PATH=${{ inputs.workspace }}/build/lib:${{ inputs.workspace }}/linux_nightly_release/lib/libsycl.so:${{ inputs.workspace }}/linux_nightly_release/lib:$LD_LIBRARY_PATH + export CMAKE_CXX_COMPILER=${{ inputs.workspace }}/linux_nightly_release/bin/clang++ + export CMAKE_C_COMPILER=${{ inputs.workspace }}/linux_nightly_release/bin/clang + export CA_HAL_DEBUG=1 + export CA_PROFILE_LEVEL=3 + export ONEAPI_DEVICE_SELECTOR=opencl:acc + export OCL_ICD_FILENAMES=${{ inputs.workspace }}/build/lib/libCL.so + # As the oneAPI basetoolkit release has a whitelist of devices, it filters out RefSi. + # To override it, as a temporary solution we can point SYCL_CONFIG_FILE_NAME to ``. + # This way it doesn't set the default sycl.conf. + export SYCL_CONFIG_FILE_NAME="" + export portDNN_source=$(pwd)/portDNN + export OCL_ICD_VENDORS=/dev/null + + # VGG16 + mkdir vdata + cd vdata + wget --no-verbose https://storage.googleapis.com/tensorflow/keras-applications/vgg16/vgg16_weights_tf_dim_ordering_tf_kernels.h5 + python $portDNN_source/samples/networks/vgg/h5toBin.py vgg16_weights_tf_dim_ordering_tf_kernels.h5 + cd .. + + # Resnet50 + mkdir rdata + cd rdata + wget --no-verbose https://storage.googleapis.com/tensorflow/keras-applications/resnet/resnet50_weights_tf_dim_ordering_tf_kernels.h5 + python $portDNN_source/samples/networks/resnet50/h5toBin.py resnet50_weights_tf_dim_ordering_tf_kernels.h5 + cd .. + + # Preparing an image + python $portDNN_source/samples/networks/img2bin.py ${{ inputs.workspace }}/examples/technical_blogs/tartan_blog/Simple.jpg + + # Testing on image for VGG16 + CA_HAL_DEBUG=1 CA_PROFILE_LEVEL=3 OCL_ICD_FILENAMES=${{ inputs.workspace }}/build/lib/libCL.so ONEAPI_DEVICE_SELECTOR=opencl:acc SYCL_CONFIG_FILE_NAME= ${{ inputs.workspace }}/portDNN_build_dir/samples/networks/vgg/vgg vdata/ ${{ inputs.workspace }}/examples/technical_blogs/tartan_blog/Simple.jpg.bin + + # Testing on image for Resnet50 + CA_HAL_DEBUG=1 CA_PROFILE_LEVEL=3 OCL_ICD_FILENAMES=${{ inputs.workspace }}/build/lib/libCL.so ONEAPI_DEVICE_SELECTOR=opencl:acc SYCL_CONFIG_FILE_NAME= ${{ inputs.workspace }}/portDNN_build_dir/samples/networks/resnet50/resnet50 rdata/ ${{ inputs.workspace }}/examples/technical_blogs/tartan_blog/Simple.jpg.bin diff --git a/.github/workflows/run_tartan.yml b/.github/workflows/run_tartan.yml index 5ebb5baa4..09cfbe9c0 100644 --- a/.github/workflows/run_tartan.yml +++ b/.github/workflows/run_tartan.yml @@ -34,7 +34,7 @@ jobs: - name: Install Python libraries and system dependencies run: | - pip install requests numpy h5py Pillow + pip install --user requests numpy h5py Pillow sudo add-apt-repository universe sudo apt-get update sudo apt-get install -y libblas-dev libopenblas64-dev libopenblas-dev libpthread-stubs0-dev libboost-all-dev @@ -47,7 +47,6 @@ jobs: wget "https://github.com/intel/llvm/releases/download/nightly-2023-11-22/sycl_linux.tar.gz" mkdir linux_nightly_release tar -xzf sycl_linux.tar.gz -C linux_nightly_release - ls - name: Build portBLAS uses: ./.github/actions/build_portBLAS_action @@ -56,5 +55,38 @@ jobs: - name: Build portDNN uses: ./.github/actions/build_portDNN_action + with: + workspace: ${{ github.workspace }} + + - name: Package Tartan Artifacts + run: | + ls + tar -cvzf new_tartan.tar.gz build portDNN_build_dir portBLAS_build_dir + + - name: Upload Artifacts + uses: actions/upload-artifact@v2 + with: + name: tartan-build + path: new_tartan.tar.gz + + build_and_run_networks: + runs-on: ubuntu-22.04 + needs: run_riscv_m1_tartan + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Download artifacts + uses: actions/download-artifact@v2 + with: + name: tartan-build + + - name: Untar artifacts + run: | + tar -xvzf new_tartan.tar.gz + ls + + - name: Build vgg and resnet + uses: ./.github/actions/build_vgg_resnet_action with: workspace: ${{ github.workspace }} \ No newline at end of file From 13d7d219a46986675adcc9434f38b410480af82f Mon Sep 17 00:00:00 2001 From: Marya Date: Tue, 9 Jan 2024 00:31:03 +0000 Subject: [PATCH 2/3] update --- .../build_vgg_resnet_action/action.yml | 11 ++++ .github/workflows/run_tartan.yml | 66 +++++++++++++++++-- 2 files changed, 71 insertions(+), 6 deletions(-) diff --git a/.github/actions/build_vgg_resnet_action/action.yml b/.github/actions/build_vgg_resnet_action/action.yml index 16b75066d..c24013698 100644 --- a/.github/actions/build_vgg_resnet_action/action.yml +++ b/.github/actions/build_vgg_resnet_action/action.yml @@ -73,3 +73,14 @@ runs: # Testing on image for Resnet50 CA_HAL_DEBUG=1 CA_PROFILE_LEVEL=3 OCL_ICD_FILENAMES=${{ inputs.workspace }}/build/lib/libCL.so ONEAPI_DEVICE_SELECTOR=opencl:acc SYCL_CONFIG_FILE_NAME= ${{ inputs.workspace }}/portDNN_build_dir/samples/networks/resnet50/resnet50 rdata/ ${{ inputs.workspace }}/examples/technical_blogs/tartan_blog/Simple.jpg.bin + + - name: Package artifacts + shell: bash + run: | + tar -cvzf network_artifacts.tar.gz vdata rdata + + - name: Upload Artifacts + uses: actions/upload-artifact@v2 + with: + name: network-build + path: network_artifacts.tar.gz diff --git a/.github/workflows/run_tartan.yml b/.github/workflows/run_tartan.yml index 09cfbe9c0..e01c45003 100644 --- a/.github/workflows/run_tartan.yml +++ b/.github/workflows/run_tartan.yml @@ -5,7 +5,9 @@ on: - cron: '30 23 * * *' # Runs the workflow at midnight every day # Allows you to run this workflow manually from the Actions tab workflow_dispatch: - + pull_request: + paths: + - '.github/workflows/run_tartan.yml' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true @@ -58,16 +60,16 @@ jobs: with: workspace: ${{ github.workspace }} - - name: Package Tartan Artifacts + - name: Package Build Artifacts run: | ls - tar -cvzf new_tartan.tar.gz build portDNN_build_dir portBLAS_build_dir + tar -cvzf artifacts.tar.gz build portDNN_build_dir portBLAS_build_dir - name: Upload Artifacts uses: actions/upload-artifact@v2 with: name: tartan-build - path: new_tartan.tar.gz + path: artifacts.tar.gz build_and_run_networks: runs-on: ubuntu-22.04 @@ -83,10 +85,62 @@ jobs: - name: Untar artifacts run: | - tar -xvzf new_tartan.tar.gz + tar -xvzf artifacts.tar.gz ls - name: Build vgg and resnet uses: ./.github/actions/build_vgg_resnet_action with: - workspace: ${{ github.workspace }} \ No newline at end of file + workspace: ${{ github.workspace }} + + publish_OCK_demo_artifacts: + runs-on: ubuntu-22.04 + needs: run_riscv_m1_tartan, build_and_run_networks + env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} + steps: + - name: Download artifacts + uses: actions/download-artifact@v2 + with: + name: tartan-build + + - name: Download network artifacts + uses: actions/download-artifact@v2 + with: + name: network-build + + - name: Untar artifacts and package OCK demo tar + run: | + tar -xvzf artifacts.tar.gz + tar -xvzf network_artifacts.tar.gz + ls + tar -cvzf ock_demo.tar.gz build portDNN_build_dir portBLAS_build_dir vdata rdata + + - name: Upload Artifacts + uses: actions/upload-artifact@v2 + with: + name: ock-demo + path: ock_demo.tar.gz + + - name: Compute tag + id: tag + run: | + if [ "${{ github.event_name == 'schedule' }}" == "true" ]; then + echo "TAG=$(date +'%Y-%m-%d')" >> "$GITHUB_OUTPUT" + else + # TODO: Use date of the commit? + echo "TAG=$(date +'%Y-%m-%d')-${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT" + fi + + - name: Create OCK demo release + uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + with: + files: + ock_demo.tar.gz + tag_name: nightly-${{ steps.tag.outputs.TAG }} + name: OCK daily ${{ steps.tag.outputs.TAG }} + prerelease: true + body: "Daily build ${{ steps.tag.outputs.TAG }}" + target_commitish: ${{ github.sha }} \ No newline at end of file From fe77f0949186eb5b3784d702dc91705aa1045014 Mon Sep 17 00:00:00 2001 From: Marya Date: Tue, 9 Jan 2024 00:36:09 +0000 Subject: [PATCH 3/3] update --- .github/workflows/run_tartan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_tartan.yml b/.github/workflows/run_tartan.yml index e01c45003..e324c58fa 100644 --- a/.github/workflows/run_tartan.yml +++ b/.github/workflows/run_tartan.yml @@ -95,7 +95,7 @@ jobs: publish_OCK_demo_artifacts: runs-on: ubuntu-22.04 - needs: run_riscv_m1_tartan, build_and_run_networks + needs: [run_riscv_m1_tartan, build_and_run_networks] env: GH_TOKEN: ${{ secrets.GH_TOKEN }} steps: