Hostexec: Enable building with cmake4

.github/workflows/PSDB-amd-mainline.yml

@@ -1,5 +1,4 @@
-# This workflow is used to invoke the PSDB jenkins job for ROCm Compiler CI. The python script can be used to invoke any jenkins job but input params needs to be configured properly
-name: Compiler CI test
+name: Compiler CI PSDB trigger on amd-mainline branch
 
 # Controls when the workflow will run
 on:
@@ -10,46 +9,64 @@ on:
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel, below is a single job called invoke jenkins jobs
 jobs:
-  # This workflow contains a single job called "build"
+  # This workflow contains a single job called "invoke_jenkins_PSDB"
   invoke_jenkins_PSDB:
-    # The type of runner that the job will run on. For github hosted runner use (${{ 'ubuntu-latest' }}) or self-hosted for sel-hosted runner.
-    #runs-on: ubuntu-latest
     if: github.event.pull_request.draft == false
-    runs-on: self-hosted
-    container:
-      image: compute-artifactory.amd.com:5000/rocm-base-images/ghemu-action-ubuntu-24.04:2024101101
+    runs-on: 
+      group: compiler-generic-runners
     env:
-        svc_acc_org_secret: ${{'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'}}
+        svc_acc_org_secret: ${{secrets.CI_GITHUB_TOKEN}}
         input_sha: ${{ github.event.pull_request.head.sha != '' && github.event.pull_request.head.sha || github.sha }}
         input_pr_num: ${{ github.event.pull_request.number != '' && github.event.pull_request.number || 0 }}
         input_pr_url: ${{ github.event.pull_request.html_url != '' && github.event.pull_request.html_url || '' }}
         input_pr_title: ${{ github.event.pull_request.title != '' && github.event.pull_request.title || '' }}
         # set the pipeline name here based on branch name
-        pipeline_name: ${{ 'compiler-psdb-amd-mainline' }}
-        JENKINS_URL: ${{'https://compiler-ci.amd.com/'}}
+        pipeline_name:  ${{secrets.CI_JENKINS_MAINLINE_JOB_NAME}}
+        JENKINS_URL: ${{secrets.CI_JENKINS_URL}}
+        CONTAINER_IMAGE: ${{ secrets.JENKINS_TRIGGER_DOCKER_IMAGE }}
+
     # Steps represent a sequence of tasks that will be executed as part of the job
-    steps:
-      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - name: Check out rocm_ci_infra private repo
-        uses: actions/checkout@main
-        with:
-          #ref: ci-utils-dev-siva
-          #fetch-depth: 2
-          repository: AMD-Lightning-Internal/ci-utils
-          token: ${{ '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' }}
-          #token: ${{'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'}}
+    steps:    
 
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - name: Set environment variable for container image
+        run: |
+         echo "CONTAINER_IMAGE=${{ secrets.JENKINS_TRIGGER_DOCKER_IMAGE }}" >> $GITHUB_ENV
+         echo "CONTAINER_NAME=my_container_${{ github.run_id }}" >> $GITHUB_ENV
+
+
+      - name: Pull container image
+        run: docker pull "${{env.CONTAINER_IMAGE}}"      
 
+
+      - name: Run container
+        run: |
+          docker run -d --name "${{env.CONTAINER_NAME}}" $CONTAINER_IMAGE sleep infinity
+          #docker exec "${{env.CONTAINER_NAME}}" /bin/bash -c  "git clone ${{secrets.CI_UTILS_REPO}} ."
+          docker exec "${{env.CONTAINER_NAME}}" /bin/bash -c "echo 'Running commands inside the container'"
+
+      - name: Escape pull request title
+        run: |
+          import json
+          import os
+          import shlex
+          with open('${{ github.event_path }}') as fh:
+              event = json.load(fh)         
+          escaped = event['pull_request']['title']
+          with open(os.environ['GITHUB_ENV'], 'a') as fh:
+              print(f'PR_TITLE={escaped}', file=fh)
+        shell: python3 {0}          
+
       - name: Run Jenkins Cancel Script
         env:
-          JENKINS_URL: ${{ 'https://compiler-ci.amd.com/' }}
-          JENKINS_USER: ${{ 'z1_cciauto' }}
-          JENKINS_API_TOKEN: ${{ '11bdb3dcd61f1a00f3999c8e3a0d6da9a7' }}
-          JENKINS_JOB_NAME: ${{ 'compiler-psdb-amd-mainline' }}
+          JENKINS_URL: ${{secrets.CI_JENKINS_URL}}
+          JENKINS_USER:  ${{secrets.CI_JENKINS_USER}}
+          JENKINS_API_TOKEN: ${{secrets.CI_JENKINS_TOKEN}}
+          JENKINS_JOB_NAME: ${{secrets.CI_JENKINS_JOB_NAME}}
           PR_NUMBER: ${{ github.event.pull_request.number }}
           COMMIT_HASH: ${{ github.event.after }}
         run: |
-          python3 cancel_previous_build.py
+          docker exec -e JENKINS_JOB_NAME=${{secrets.CI_JENKINS_JOB_NAME}}  -e PR_NUMBER=${{ github.event.pull_request.number }}  -e COMMIT_HASH=${{ github.event.after }} -e JENKINS_URL=${{secrets.CI_JENKINS_URL}} -e JENKINS_USER=${{secrets.CI_JENKINS_USER}} -e JENKINS_API_TOKEN=${{secrets.CI_JENKINS_TOKEN}}  "${{env.CONTAINER_NAME}}" /bin/bash -c "PYTHONHTTPSVERIFY=0 python3 cancel_previous_build.py"
 
 
       # Runs a set of commands using the runners shell
@@ -65,21 +82,25 @@ jobs:
           echo "GITHUB_REF_NAME is: $GITHUB_REF_NAME"
           echo "github.event.pull_request.id is: ${{github.event.pull_request.id}}"
           echo "github.event.pull_request.html_url is: ${{github.event.pull_request.html_url}}"
-          echo "github.event.pull_request.number is: ${{github.event.pull_request.number}}"          
+          echo "github.event.pull_request.number is: ${{github.event.pull_request.number}}"
           echo "github.event.pull_request.url is: ${{github.event.pull_request.url}}"
           echo "github.event.pull_request.issue_url is: ${{github.event.pull_request.issue_url}}"
-          echo "github.event.pull_request.comments_url is: ${{github.event.pull_request.comments_url}}"
-          echo "github.event.pull_request.statuses_url is: ${{github.event.pull_request.statuses_url}}"
           echo "github.event.pull_request.head.sha is: ${{github.event.pull_request.head.sha}}"
           echo "github.event.pull_request.base.ref is: ${{github.event.pull_request.base.ref}}"
           echo "github.event.pull_request.merge_commit_sha is: ${{github.event.pull_request.merge_commit_sha}}"
           echo "github.event.pull_request is: ${{github.event.pull_request}}"
-          pip3 show python-jenkins || echo "python-jenkins is not installed"
-          #sudo -H pip3 install --upgrade python-jenkins
-          # pipeline name shuould be unique to the workfow yml for a given repository
-          #curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{'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'}}" ${{github.event.pull_request.comments_url}} -d '{"body":"Github action triggered jenkins job for compute-psdb-staging-smi-libs-ghemu "}'
+
+
       - name: Trigger Jenkins Pipeline
         if: steps.check_changes.outcome != 'failure'
         run: |
-          echo "running jenkins_api.py with input sha - $input_sha for pull request - $input_pr_url" 
-          python3 jenkins_api.py -s $JENKINS_URL -jn $pipeline_name -ghr $GITHUB_REPOSITORY -ghsha $input_sha -ghprn $input_pr_num -ghpru "$input_pr_url" -ghprt "$input_pr_title" -ghpat $svc_acc_org_secret
+          echo "--Running jenkins_api.py with input sha - $input_sha for pull request - $input_pr_url" 
+          docker exec -e GITHUB_REPOSITORY="$GITHUB_REPOSITORY" -e svc_acc_org_secret="$svc_acc_org_secret" -e input_sha="$input_sha" -e input_pr_url="$input_pr_url" -e pipeline_name="$pipeline_name" \
+                     -e input_pr_num="$input_pr_num" -e PR_TITLE="$PR_TITLE" -e JENKINS_URL="$JENKINS_URL" -e GITHUB_PAT="$svc_acc_org_secret" "${{env.CONTAINER_NAME}}"  \
+                     /bin/bash -c 'echo \"PR NUM: "$input_pr_num"\" && PYTHONHTTPSVERIFY=0 python3 jenkins_api.py -s \"${JENKINS_URL}\" -jn "$pipeline_name" -ghr "$GITHUB_REPOSITORY" -ghsha "$input_sha" -ghprn "$input_pr_num" -ghpru "$input_pr_url" -ghprt "$PR_TITLE" -ghpat="$svc_acc_org_secret"'
+
+      - name: Stop and remove container
+        if: always()
+        run: |
+          docker stop "${{env.CONTAINER_NAME}}"
+          docker rm "${{env.CONTAINER_NAME}}"

amd/comgr/CMakeLists.txt

@@ -38,7 +38,6 @@ endif(EXISTS "${CMAKE_SOURCE_DIR}/../../.git")
 
 include(GNUInstallDirs)
 include(CMakePackageConfigHelpers)
-option(FILE_REORG_BACKWARD_COMPATIBILITY "Enable File Reorg with backward compatibility" OFF)
 # Optionally, build Compiler Support with ccache.
 set(ROCM_CCACHE_BUILD OFF CACHE BOOL "Set to ON for a ccache enabled build")
 if (ROCM_CCACHE_BUILD)
@@ -71,6 +70,10 @@ option(COMGR_BUILD_SHARED_LIBS "Build the shared library"
        ${build_shared_libs_default})
 
 set(SOURCES
+  src/comgr-cache.cpp
+  src/comgr-cache-command.cpp
+  src/comgr-cache-bundler-command.cpp
+  src/comgr-clang-command.cpp
   src/comgr-compiler.cpp
   src/comgr.cpp
   src/comgr-device-libs.cpp
@@ -81,6 +84,7 @@ set(SOURCES
   src/comgr-metadata.cpp
   src/comgr-objdump.cpp
   src/comgr-signal.cpp
+  src/comgr-spirv-command.cpp
   src/comgr-symbol.cpp
   src/comgr-symbolizer.cpp
   src/time-stat/time-stat.cpp)
@@ -178,9 +182,12 @@ message("")
 option(COMGR_DISABLE_SPIRV "To disable SPIRV in Comgr" OFF)
 
 if (NOT COMGR_DISABLE_SPIRV)
-  CHECK_INCLUDE_FILE_CXX(LLVMSPIRVLib/LLVMSPIRVLib.h HAVE_LLVMSPIRVLIB_H)
-  if (NOT HAVE_LLVMSPIRVLIB_H)
+  # TODO: Explore switching this to CHECK_INCLUDE_FILE_CXX() macro
+  if (NOT EXISTS "${LLVM_INCLUDE_DIRS}/LLVMSPIRVLib/LLVMSPIRVLib.h")
+    message("-- LLVMSPIRVLib/LLVMSPIRVLib.h not found")
     set(COMGR_DISABLE_SPIRV ON)
+  else()
+    message("-- LLVMSPIRVLib/LLVMSPIRVLib.h found")
   endif()
 endif()
 
@@ -319,26 +326,6 @@ install(FILES
   COMPONENT amd-comgr
   DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${amd_comgr_NAME})
 
-#File reorg Backward compatibility function
-if(FILE_REORG_BACKWARD_COMPATIBILITY)
-# To enable/disable #error in wrapper header files
- if(NOT DEFINED ROCM_HEADER_WRAPPER_WERROR)
-      if(DEFINED ENV{ROCM_HEADER_WRAPPER_WERROR})
-          set(ROCM_HEADER_WRAPPER_WERROR "$ENV{ROCM_HEADER_WRAPPER_WERROR}"
-                  CACHE STRING "Header wrapper warnings as errors.")
-      else()
-          set(ROCM_HEADER_WRAPPER_WERROR "OFF" CACHE STRING "Header wrapper warnings as errors.")
-      endif()
-  endif()
-  if(ROCM_HEADER_WRAPPER_WERROR)
-      set(deprecated_error 1)
-  else()
-      set(deprecated_error 0)
-  endif()
-
-  include(comgr-backward-compat.cmake)
-endif()
-
 if(ENABLE_ASAN_PACKAGING)
   install(FILES
     "LICENSE.txt"

amd/comgr/README.md

@@ -125,7 +125,27 @@ These include:
   certain runtime headers. If this is not set, it has a default value of
   "${ROCM_PATH}/llvm".
 
-Comgr also supports some environment variables to aid in debugging. These
+Comgr utilizes a cache to preserve the results of compilations between executions.
+The cache's status (enabled/disabled), storage location for its results,
+and eviction policy can be manipulated through specific environment variables.
+If an issue arises during cache initialization, the execution will proceed with
+the cache turned off.
+
+By default, the cache is enabled.
+
+* `AMD_COMGR_CACHE`: When unset or set to a value different than "0", the cache is enabled.
+  Disabled when set to "0".
+* `AMD_COMGR_CACHE_DIR`: If assigned a non-empty value, that value is used as
+  the path for cache storage. If the variable is unset or set to an empty string `""`,
+  it is directed to "$XDG_CACHE_HOME/comgr" (which defaults to
+  "$USER/.cache/comgr" on Linux, and "%LOCALAPPDATA%\cache\comgr"
+  on Microsoft Windows).
+* `AMD_COMGR_CACHE_POLICY`: If assigned a value, the string is interpreted and
+  applied to the cache pruning policy. The cache is pruned only upon program
+  termination. The string format aligns with [Clang's ThinLTO cache pruning policy](https://clang.llvm.org/docs/ThinLTO.html#cache-pruning).
+  The default policy is set as: "prune_interval=1h:prune_expiration=0h:cache_size=75%:cache_size_bytes=30g:cache_size_files=0".
+
+Comgr supports some environment variables to aid in debugging. These
 include:
 
 * `AMD_COMGR_SAVE_TEMPS`: If this is set, and is not "0", Comgr does not delete
@@ -143,6 +163,20 @@ include:
 * `AMD_COMGR_TIME_STATISTICS`: If this is set, and is not "0", logs will
   include additional Comgr-specific timing information for compilation actions.
 
+Comgr implements support for an in-memory, virtual filesystem (VFS) for storing
+temporaries generated during intermediate compilation steps. This is aimed at 
+improving performance by reducing on-disk file I/O. Currently, VFS is only supported 
+for the device library link step, but we aim to progressively add support for
+more actions.
+
+By default, VFS is turned on.
+
+* `AMD_COMGR_USE_VFS`: When set to "0", VFS support is turned off.
+* Users may use the API `amd_comgr_action_info_set_vfs` to disable VFS for individual actions
+  without having to modify system-wide environment variables.
+* If `AMD_COMGR_SAVE_TEMPS` is set and not "0", VFS support is turned off irrespective
+  of `AMD_COMGR_USE_VFS` or the use of `amd_comgr_action_info_set_vfs`.
+
 Versioning
 ----------
 

amd/comgr/VERSION.txt

@@ -1,4 +1,4 @@
 #COMGR_VERSION_MAJOR
-2
+3
 #COMGR_VERSION_MINOR
-9
+0

amd/comgr/cmake/DeviceLibs.cmake

@@ -59,6 +59,7 @@ foreach(AMDGCN_LIB_TARGET ${AMD_DEVICE_LIBS_TARGETS})
   add_dependencies(amd_comgr ${AMDGCN_LIB_TARGET}_header)
 
   list(APPEND TARGETS_INCLUDES "#include \"${header}\"")
+  list(APPEND TARGETS_HEADERS "${INC_DIR}/${header}")
 endforeach()
 
 list(JOIN TARGETS_INCLUDES "\n" TARGETS_INCLUDES)
@@ -110,4 +111,17 @@ list(APPEND TARGETS_DEFS "#undef AMD_DEVICE_LIBS_FUNCTION")
 list(JOIN TARGETS_DEFS "\n" TARGETS_DEFS)
 file(GENERATE OUTPUT ${GEN_LIBRARY_DEFS_INC_FILE} CONTENT "${TARGETS_DEFS}")
 
+# compute the sha256 of the device libraries to detect changes and pass them to comgr (used by the cache)
+find_package(Python3 REQUIRED Interpreter)
+set(DEVICE_LIBS_ID_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/cmake/device-libs-id.py")
+set(DEVICE_LIBS_ID_HEADER ${INC_DIR}/libraries_sha.inc)
+add_custom_command(OUTPUT ${DEVICE_LIBS_ID_HEADER}
+  COMMAND ${Python3_EXECUTABLE} ${DEVICE_LIBS_ID_SCRIPT} --varname DEVICE_LIBS_ID --output ${DEVICE_LIBS_ID_HEADER} ${TARGETS_HEADERS}
+  DEPENDS ${DEVICE_LIBS_ID_SCRIPT} ${TARGETS_HEADERS}
+    COMMENT "Generating ${INC_DIR}/libraries_sha.inc"
+)
+set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${INC_DIR}/libraries_sha.inc)
+add_custom_target(libraries_sha_header DEPENDS ${INC_DIR}/libraries_sha.inc)
+add_dependencies(amd_comgr libraries_sha_header)
+
 include_directories(${INC_DIR})