Support CLANGPDB to build ARM SBSA

.azurepipelines/Platform-Build-Job.yml

@@ -91,7 +91,7 @@ jobs:
 
       - template: Steps/BuildPlatform.yml@mu_devops
         parameters:
-          tool_chain_tag: $(tool_chain_tag)
+          tool_chain_tag: ${{ parameters.tool_chain_tag }}
           build_pkg: ${{ item.Value.BuildPackage }}
           build_target: ${{ item.Value.BuildTarget }}
           build_file: ${{ item.Value.BuildFile }}
@@ -100,7 +100,7 @@ jobs:
           run_timeout: ${{ parameters.run_timeout }}
           extra_install_step: ${{ parameters.extra_install_step }}
           install_tools: ${{ and(ne(item.Value.SelfHostAgent, true), eq(parameters.container_image, '')) }}
-          artifacts_identifier: '${{ item.Value.BuildPackage }} $(tool_chain_tag) ${{ item.Value.BuildTarget }} ${{ item.Value.BuildExtraTag }}'
+          artifacts_identifier: '${{ item.Value.BuildPackage }} ${{ parameters.tool_chain_tag }} ${{ item.Value.BuildTarget }} ${{ item.Value.BuildExtraTag }}'
           artifacts_binary: ${{ item.Value.BuildArtifactsBinary }}
           ${{ if or(eq(item.Value.Run, true), eq(parameters.os_type, 'Linux'), EndsWith(item.Key, 'CODE_COVERAGE')) }}:
             artifacts_other: ${{ item.Value.BuildArtifactsOther }}

.azurepipelines/Platform-Build-VS.yml

@@ -122,3 +122,70 @@ jobs:
             displayName: Install Report Generator
         BuildArtifactsOther: "**/Coverage/**"
         Run: false
+
+- template: Platform-Build-Job.yml
+  parameters:
+    extra_install_step:
+    - template: Steps/InstallCoverageTools.yml@mu_devops
+    tool_chain_tag: 'CLANGPDB'
+    vm_image: $(vm_image)
+    os_type: Windows_NT
+    build_matrix:
+      QemuQ35_DEBUG_CLANGPDB:
+        BuildPackage: QemuQ35Pkg
+        BuildFile: "Platforms/QemuQ35Pkg/PlatformBuild.py"
+        BuildFlags: ""
+        BuildTarget: "DEBUG"
+        BuildExtraTag: ""
+        BuildExtraStep:
+          - template: Steps/RustSetupSteps.yml@mu_devops
+        Run: true
+        RunFlags: "SHUTDOWN_AFTER_RUN=TRUE QEMU_HEADLESS=TRUE EMPTY_DRIVE=TRUE BLD_*_QEMU_CORE_NUM=2 TEST_REGEX=*TestApp*.efi RUN_TESTS=TRUE"
+        BuildArtifactsBinary: |
+          **/QEMUQ35_*.fd
+        BuildArtifactsOther: "**/unit_test_results/*"
+
+      QemuQ35_RELEASE_CLANGPDB:
+        BuildPackage: QemuQ35Pkg
+        BuildFile: "Platforms/QemuQ35Pkg/PlatformBuild.py"
+        BuildFlags: ""
+        BuildTarget: "RELEASE"
+        BuildExtraTag: ""
+        BuildExtraStep:
+          - template: Steps/RustSetupSteps.yml@mu_devops
+        Run: true
+        RunFlags: "SHUTDOWN_AFTER_RUN=TRUE QEMU_HEADLESS=TRUE EMPTY_DRIVE=TRUE BLD_*_QEMU_CORE_NUM=2 TEST_REGEX=*TestApp*.efi RUN_TESTS=TRUE"
+        BuildArtifactsBinary: |
+          **/QEMUQ35_*.fd
+        BuildArtifactsOther: "**/unit_test_results/*"
+
+      QemuSbsa_DEBUG_CLANGPDB:
+        BuildPackage: QemuSbsaPkg
+        BuildFile: "Platforms/QemuSbsaPkg/PlatformBuild.py"
+        BuildTarget: "DEBUG"
+        BuildExtraTag: ""
+        BuildExtraStep:
+          - template: Steps/RustSetupSteps.yml@mu_devops
+          - template: Steps/SetupToolChainTagPreReqs.yml@mu_devops
+        Run: true
+        RunFlags: "SHUTDOWN_AFTER_RUN=TRUE QEMU_HEADLESS=TRUE EMPTY_DRIVE=TRUE TEST_REGEX=*TestApp*.efi RUN_TESTS=TRUE"
+        BuildArtifactsBinary: |
+          **/QEMU_EFI.fd
+          **/SECURE_FLASH0.fd
+        BuildArtifactsOther: "**/unit_test_results/*"
+
+      QemuSbsa_RELEASE_CLANGPDB:
+        BuildPackage: QemuSbsaPkg
+        BuildFile: "Platforms/QemuSbsaPkg/PlatformBuild.py"
+        BuildFlags: ""
+        BuildTarget: "RELEASE"
+        BuildExtraTag: ""
+        BuildExtraStep:
+          - template: Steps/RustSetupSteps.yml@mu_devops
+          - template: Steps/SetupToolChainTagPreReqs.yml@mu_devops
+        Run: true
+        RunFlags: "SHUTDOWN_AFTER_RUN=TRUE QEMU_HEADLESS=TRUE EMPTY_DRIVE=TRUE TEST_REGEX=*TestApp*.efi RUN_TESTS=TRUE"
+        BuildArtifactsBinary: |
+          **/QEMU_EFI.fd
+          **/SECURE_FLASH0.fd
+        BuildArtifactsOther: "**/unit_test_results/*"

.gitmodules

@@ -31,6 +31,6 @@
 [submodule "Features/MM_SUPV"]
 	path = Features/MM_SUPV
 	url = https://github.com/microsoft/mu_feature_mm_supv.git
-[submodule "Features/DEBUGGER"]
-	path = Features/DEBUGGER
-	url = https://github.com/microsoft/mu_feature_debugger.git
+[submodule "Features/DEBUGGER"]
+	path = Features/DEBUGGER
+	url = https://github.com/microsoft/mu_feature_debugger.git

Platforms/QemuQ35Pkg/QemuQ35Pkg.dsc

@@ -1533,4 +1533,5 @@ QemuQ35Pkg/Library/ResetSystemLib/StandaloneMmResetSystemLib.inf
 # protection of DXE_SMM_DRIVER/SMM_CORE modules
 [BuildOptions.common.EDKII.DXE_SMM_DRIVER, BuildOptions.common.EDKII.DXE_RUNTIME_DRIVER, BuildOptions.common.EDKII.SMM_CORE, BuildOptions.common.EDKII.DXE_DRIVER, BuildOptions.common.EDKII.DXE_CORE, BuildOptions.common.EDKII.UEFI_DRIVER, BuildOptions.common.EDKII.UEFI_APPLICATION, BuildOptions.common.EDKII.MM_CORE_STANDALONE, BuildOptions.common.EDKII.MM_STANDALONE]
   MSFT:*_*_*_DLINK_FLAGS = /ALIGN:4096
-  GCC:*_*_*_DLINK_FLAGS = -z common-page-size=0x1000
+  GCC:*_GCC5_*_DLINK_FLAGS = -z common-page-size=0x1000
+  GCC:*_CLANGPDB_*_DLINK_FLAGS = /ALIGN:4096

Platforms/QemuSbsaPkg/Library/SbsaConfigDataLib/SbsaConfigDataLib.c

@@ -14,5 +14,6 @@
 #include <Generated/ConfigClientGenerated.h>
 #include <Generated/ConfigDataGenerated.h>
 
-PROFILE  gProfileData = { 0 };
-UINTN    gNumProfiles = 0;
+PROFILE  gProfileData           = { 0 };
+UINTN    gNumProfiles           = 0;
+CHAR8    *gProfileFlavorNames[] = { NULL };

Platforms/QemuSbsaPkg/PlatformBuild.py

@@ -20,6 +20,8 @@
 from edk2toolext.invocables.edk2_update import UpdateSettingsManager
 from edk2toolext.invocables.edk2_parse import ParseSettingsManager
 from edk2toollib.utility_functions import RunCmd
+from edk2toollib.windows.locate_tools import QueryVcVariables
+from edk2toollib.utility_functions import GetHostInfo
 
 # Declare test whose failure will not return a non-zero exit code
 FAILURE_EXEMPT_TESTS = {
@@ -40,7 +42,7 @@ class CommonPlatform():
     PackagesSupported = ("QemuSbsaPkg",)
     ArchSupported = ("AARCH64",)
     TargetsSupported = ("DEBUG", "RELEASE", "NOOPT")
-    Scopes = ('qemu', 'qemusbsa', 'gcc_aarch64_linux', 'edk2-build', 'cibuild', 'configdata', 'rust-ci')
+    Scopes = ('qemu', 'qemusbsa', 'edk2-build', 'cibuild', 'configdata', 'rust-ci')
     WorkspaceRoot = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
     PackagesPath = (
         "Platforms",
@@ -113,7 +115,13 @@ def GetWorkspaceRoot(self):
 
     def GetActiveScopes(self):
         ''' return tuple containing scopes that should be active for this process '''
-        return CommonPlatform.Scopes
+        scopes = CommonPlatform.Scopes
+        actual_tool_chain_tag = shell_environment.GetBuildVars().GetValue(
+                "TOOL_CHAIN_TAG", ""
+            )
+        if actual_tool_chain_tag.upper().startswith("GCC"):
+            scopes += ("gcc_aarch64_linux",)
+        return scopes
 
     def FilterPackagesToTest(self, changedFilesList: list, potentialPackagesList: list) -> list:
         ''' Filter other cases that this package should be built
@@ -157,13 +165,54 @@ class PlatformBuilder(UefiBuilder, BuildSettingsManager):
     def __init__(self):
         UefiBuilder.__init__(self)
 
+    # Helper function to query the VC variables of interest and inject them into the environment
+    def InjectVcVarsOfInterests(self, vcvars: list):
+        HostInfo = GetHostInfo()
+
+        # check to see if host is configured
+        # HostType for VS tools should be (defined in tools_def):
+        # x86   == 32bit Intel
+        # x64   == 64bit Intel
+        # arm   == 32bit Arm
+        # arm64 == 64bit Arm
+        #
+        HostType = shell_environment.GetEnvironment().get_shell_var("CLANG_VS_HOST")
+        if HostType is not None:
+            HostType = HostType.lower()
+            logging.info(
+                f"CLANG_VS_HOST defined by environment.  Value is {HostType}")
+        else:
+            #figure it out based on host info
+            if HostInfo.arch == "x86":
+                if HostInfo.bit == "32":
+                    HostType = "x86"
+                elif HostInfo.bit == "64":
+                    HostType = "x64"
+            else:
+                # anything other than x86 or x64 is not supported
+                raise NotImplementedError()
+
+        # CLANG_VS_HOST options are not exactly the same as QueryVcVariables. This translates.
+        VC_HOST_ARCH_TRANSLATOR = {
+            "x86": "x86", "x64": "AMD64", "arm": "not supported", "arm64": "not supported"}
+
+        # now get the environment variables for the platform
+        shell_env = shell_environment.GetEnvironment()
+        # Use the tools lib to determine the correct values for the vars that interest us.
+        vs_vars = QueryVcVariables(
+            vcvars, VC_HOST_ARCH_TRANSLATOR[HostType])
+        for (k, v) in vs_vars.items():
+            shell_env.set_shell_var(k, v)
+
     def CleanTree(self, RemoveConfTemplateFilesToo=False):
+        # If this is a Windows Clang build, we need to inject the VC variables of interest
+        if self.env.GetValue("TOOL_CHAIN_TAG") == "CLANGPDB" and os.name == 'nt':
+            self.InjectVcVarsOfInterests(["VCToolsInstallDir", "Path", "LIB"])
+
         # Add a step to clean up BL31 as well, if asked
         cmd = "make"
         args = "distclean"
-        ret = RunCmd(cmd, args, workingdir= self.env.GetValue("ARM_TFA_PATH"))
-        if ret != 0:
-            return ret
+        RunCmd(cmd, args, workingdir=self.env.GetValue("ARM_TFA_PATH"))
 
         return super().CleanTree(RemoveConfTemplateFilesToo)
 
@@ -182,7 +231,13 @@ def GetPackagesPath(self):
 
     def GetActiveScopes(self):
         ''' return tuple containing scopes that should be active for this process '''
-        return CommonPlatform.Scopes
+        scopes = CommonPlatform.Scopes
+        actual_tool_chain_tag = shell_environment.GetBuildVars().GetValue(
+                "TOOL_CHAIN_TAG", ""
+            )
+        if actual_tool_chain_tag.upper().startswith("GCC"):
+            scopes += ("gcc_aarch64_linux",)
+        return scopes
 
     def GetName(self):
         ''' Get the name of the repo, platform, or product being build '''
@@ -278,8 +333,39 @@ def PlatformPostBuild(self):
         op_fv = os.path.join(self.env.GetValue("BUILD_OUTPUT_BASE"), "FV")
 
         logging.info("Building TF-A")
+
+        shell_environment.CheckpointBuildVars()  # checkpoint our config before we mess with it
+        if self.env.GetValue("TOOL_CHAIN_TAG") == "CLANGPDB":
+            if os.name == 'nt':
+                # If this is a Windows build, we need to demolish the path and inject the VC variables of interest
+                # otherwise the build could pick up wrong tools
+                shell_environment.GetEnvironment().set_path('')
+                self.InjectVcVarsOfInterests(["LIB", "Path"])
+
+                clang_exe = "clang.exe"
+                choco_path = shell_environment.GetEnvironment().get_shell_var("CHOCOLATEYINSTALL")
+                shell_environment.GetEnvironment().insert_path(os.path.join(choco_path, "bin"))
+                shell_environment.GetEnvironment().insert_path(shell_environment.GetEnvironment().get_shell_var("CLANG_BIN"))
+
+                # Need to build fiptool separately because the build system will override LIB with LIBC for firmware builds
+                cmd = "make"
+                args = " fiptool MAKEFLAGS= LIB=\"" + shell_environment.GetEnvironment().get_shell_var("LIB") + "\""
+                ret = RunCmd(cmd, args, workingdir=self.env.GetValue("ARM_TFA_PATH"))
+                if ret != 0:
+                    return ret
+                # Then we can make the firmware images with the fiptool built above
+            else:
+                clang_exe = "clang"
+
         cmd = "make"
-        args = "CROSS_COMPILE=" + shell_environment.GetEnvironment().get_shell_var("GCC5_AARCH64_PREFIX")
+        if self.env.GetValue("TOOL_CHAIN_TAG") == "CLANGPDB":
+            args = "CC="+clang_exe
+        elif self.env.GetValue("TOOL_CHAIN_TAG") == "GCC5":
+            args = "CROSS_COMPILE=" + shell_environment.GetEnvironment().get_shell_var("GCC5_AARCH64_PREFIX")
+            args += " -j $(nproc)"
+        else:
+            logging.error("Unsupported toolchain")
+            return -1
         args += " PLAT=" + self.env.GetValue("QEMU_PLATFORM").lower()
         args += " ARCH=" + self.env.GetValue("TARGET_ARCH").lower()
         args += " DEBUG=" + str(1 if self.env.GetValue("TARGET").lower() == 'debug' else 0)
@@ -288,11 +374,13 @@ def PlatformPostBuild(self):
         # args += " FEATURE_DETECTION=1" # Enforces support for features enabled.
         args += " BL32=" + os.path.join(op_fv, "BL32_AP_MM.fd")
         args += " all fip"
-        args += " -j $(nproc)"
         ret = RunCmd(cmd, args, workingdir= self.env.GetValue("ARM_TFA_PATH"))
         if ret != 0:
             return ret
 
+        # Revert the build vars to the original state
+        shell_environment.RevertBuildVars()
+
         # Now that BL31 is built with BL32 supplied, patch BL1 and BL31 built fip.bin into the SECURE_FLASH0.fd
         op_tfa = os.path.join (
             self.env.GetValue("ARM_TFA_PATH"), "build",

Platforms/QemuSbsaPkg/QemuSbsaPkg.dsc

@@ -464,9 +464,6 @@
 [LibraryClasses.common.DXE_RUNTIME_DRIVER]
   MemoryAllocationLib|MdePkg/Library/UefiMemoryAllocationLib/UefiMemoryAllocationLib.inf
   CapsuleLib|MdeModulePkg/Library/DxeCapsuleLibNull/DxeCapsuleLibNull.inf
-!if $(TARGET) != RELEASE
-  DebugLib|MdePkg/Library/DxeRuntimeDebugLibSerialPort/DxeRuntimeDebugLibSerialPort.inf
-!endif
   VariablePolicyLib|MdeModulePkg/Library/VariablePolicyLib/VariablePolicyLibRuntimeDxe.inf
   ResetSystemLib|MdeModulePkg/Library/RuntimeResetSystemLib/RuntimeResetSystemLib.inf
 
@@ -626,9 +623,15 @@
   #
   gEmbeddedTokenSpaceGuid.PcdMemoryTypeEfiACPIReclaimMemory|0x40
   gEmbeddedTokenSpaceGuid.PcdMemoryTypeEfiACPIMemoryNVS|0x0
+!if $(TOOL_CHAIN_TAG) == GCC5     # This is really odd on why CLANGPDB has runtime memory consumption differences
   gEmbeddedTokenSpaceGuid.PcdMemoryTypeEfiReservedMemoryType|0x505
   gEmbeddedTokenSpaceGuid.PcdMemoryTypeEfiRuntimeServicesData|0x258
   gEmbeddedTokenSpaceGuid.PcdMemoryTypeEfiRuntimeServicesCode|0x260
+!else
+  gEmbeddedTokenSpaceGuid.PcdMemoryTypeEfiReservedMemoryType|0x30
+  gEmbeddedTokenSpaceGuid.PcdMemoryTypeEfiRuntimeServicesData|0x40
+  gEmbeddedTokenSpaceGuid.PcdMemoryTypeEfiRuntimeServicesCode|0x300
+!endif
   gEmbeddedTokenSpaceGuid.PcdMemoryTypeEfiBootServicesCode|0x5DC
   gEmbeddedTokenSpaceGuid.PcdMemoryTypeEfiBootServicesData|0x2EE0
   gEmbeddedTokenSpaceGuid.PcdMemoryTypeEfiLoaderCode|0x14
@@ -1365,7 +1368,7 @@
   MdeModulePkg/Universal/FaultTolerantWriteDxe/FaultTolerantWriteStandaloneMm.inf
   MdeModulePkg/Universal/Variable/RuntimeDxe/VariableStandaloneMm.inf {
     <LibraryClasses>
-      DevicePathLib|MdePkg/Library/UefiDevicePathLib/UefiDevicePathLib.inf
+      DevicePathLib|MdePkg/Library/UefiDevicePathLib/UefiDevicePathLibStandaloneMm.inf
       NULL|MdeModulePkg/Library/VarCheckUefiLib/VarCheckUefiLib.inf
       NULL|MdeModulePkg/Library/VarCheckPolicyLib/VarCheckPolicyLibStandaloneMm.inf
       BaseMemoryLib|MdePkg/Library/BaseMemoryLib/BaseMemoryLib.inf
@@ -1394,12 +1397,16 @@
   RVCT:*_*_*_CC_FLAGS = -DDISABLE_NEW_DEPRECATED_INTERFACES
   GCC:*_*_*_CC_FLAGS = -DDISABLE_NEW_DEPRECATED_INTERFACES
 
+[BuildOptions.common.EDKII.SEC,BuildOptions.common.EDKII.MM_CORE_STANDALONE]
+  GCC:*_CLANGPDB_*_DLINK_FLAGS = /ALIGN:0x1000 /FILEALIGN:0x1000
+
 [BuildOptions.common.EDKII.DXE_CORE,BuildOptions.common.EDKII.DXE_DRIVER,BuildOptions.common.EDKII.UEFI_DRIVER,BuildOptions.common.EDKII.UEFI_APPLICATION,BuildOptions.common.EDKII.MM_CORE_STANDALONE,BuildOptions.common.EDKII.MM_STANDALONE]
-  GCC:*_*_*_DLINK_FLAGS = -z common-page-size=0x1000
+  GCC:*_GCC5_*_DLINK_FLAGS = -z common-page-size=0x1000
+  GCC:*_CLANGPDB_*_DLINK_FLAGS = /ALIGN:0x1000
 
 [BuildOptions.common.EDKII.DXE_RUNTIME_DRIVER]
-  GCC:*_*_ARM_DLINK_FLAGS = -z common-page-size=0x1000
-  GCC:*_*_AARCH64_DLINK_FLAGS = -z common-page-size=0x10000
+  GCC:*_GCC5_AARCH64_DLINK_FLAGS = -z common-page-size=0x10000
+  GCC:*_CLANGPDB_AARCH64_DLINK_FLAGS = /ALIGN:0x10000
   RVCT:*_*_ARM_DLINK_FLAGS = --scatter $(EDK_TOOLS_PATH)/Scripts/Rvct-Align4K.sct
 
 [BuildOptions.AARCH64.EDKII.MM_CORE_STANDALONE,BuildOptions.AARCH64.EDKII.MM_STANDALONE]