[msbuild] Add support for bundling original resources in libraries. F…

…ixes #19028. (#20822)

If a library references resources, until now we've pre-compile/pre-processed
some of those before embedding them the library. This applies to resources of
the following item groups:

* AtlasTexture
* BundleResource
* Collada
* CoreMLModel
* ImageAsset
* InterfaceDefinition
* SceneKitAsset

However, pre-processing resources as a few problems:

* It requires a native (Xcode) toolchain.
   * This is unfortunate when building from Windows: the current approach is
     that when building a library as a referenced project, the remoting part
     is skipped, so all such resources are just dropped.
   * It also means building on Linux doesn't work.
* It makes it impossible to merge resources with the same name, if we wanted
  to do that.

So I'm adding support for bundling the original resources in library projects.

This is enabled using the MSBuild property `BundleOriginalResources=true`,
which is turned off by default for .NET 9 and turned on by default for .NET 10+.

Additionally I've added `PartialAppManifest` items to the list of bundled
resources from class libraries.

This means that there are numerous task we don't have to remote to the Mac
anymore (when bundling original resources) - in particular libraries can be
fully built on Windows (or any other platform) now.

Fixes #19028.
Fixes #19513.

docs/building-apps/build-properties.md

@@ -125,6 +125,31 @@ Only applicable to iOS and tvOS projects.
 
 See [CreatePackage](#createpackage) for macOS and Mac Catalyst projects.
 
+## BundleOriginalResources
+
+This property determines whether resources are compiled before being embedded
+into library projects, or if the original (uncompiled) version is embedded.
+
+Historically resources have been compiled before being embedded into library
+projects, but this requires having Xcode available, which has a few drawbacks:
+
+* It slows down remote builds on Windows.
+* It won't work when building locally on Windows, and neither on any other
+  platform except macOS.
+* Resources are compiled using the current available Xcode, which may not have
+  the same features as a potentially newer Xcode available when the library in
+  question is consumed.
+* It makes it impossible to have a whole-program view of all the resources
+  when building an app, which is necessary to detect clashing resources.
+
+As such, we've added supported for embedding the original resources into
+libraries. This will be opt-in in .NET 9, but opt-out starting in .NET 10.
+
+Default value: `false` in .NET 9, `true` in .NET 10+.
+
+Note: please file an issue if you find that you need to disable this feature,
+as it's possible we'll remove the option to disable it at some point.
+
 ## CodesignAllocate
 
 The path to the `codesign_allocate` tool.

dotnet/targets/Xamarin.Shared.Sdk.targets

@@ -233,6 +233,7 @@
 		<!-- outer build for multi-rid build -->
 		<BuildDependsOn Condition="'$(RuntimeIdentifiers)' != ''">
 			_ErrorRuntimeIdentifiersClash;
+			BuildOnlySettings;
 			_CollectBundleResources;
 			_RunRidSpecificBuild;
 			_DetectAppManifest;

msbuild/Xamarin.Localization.MSBuild/MSBStrings.resx

@@ -1654,4 +1654,12 @@
             {1}: the exit code of a process
         </comment>
     </data>
+
+    <data name="E7135" xml:space="preserve">
+        <value>Unknown resource type: {1}.</value>
+    </data>
+
+    <data name="E7136" xml:space="preserve">
+        <value>Unknown resource type: {1}.</value>
+    </data>
 </root>

msbuild/Xamarin.MacDev.Tasks/Tasks/CollectBundleResources.cs

@@ -32,6 +32,8 @@ public class CollectBundleResources : XamarinTask, ICancelableTask, IHasProjectD
 		[Output]
 		public ITaskItem [] BundleResourcesWithLogicalNames { get; set; } = Array.Empty<ITaskItem> ();
 
+		public ITaskItem [] UnpackedResources { get; set; } = Array.Empty<ITaskItem> ();
+
 		#endregion
 
 		static bool CanOptimize (string path)
@@ -68,37 +70,8 @@ bool ExecuteImpl ()
 			var bundleResources = new List<ITaskItem> ();
 
 			foreach (var item in BundleResources) {
-				// Skip anything with the PublishFolderType metadata, these are copied directly to the ResolvedFileToPublish item group instead.
-				var publishFolderType = item.GetMetadata ("PublishFolderType");
-				if (!string.IsNullOrEmpty (publishFolderType))
-					continue;
-
-				var logicalName = BundleResource.GetLogicalName (this, item);
-				// We need a physical path here, ignore the Link element
-				var path = item.GetMetadata ("FullPath");
-
-				if (!File.Exists (path)) {
-					Log.LogError (MSBStrings.E0099, logicalName, path);
-					continue;
-				}
-
-				if (logicalName.StartsWith (".." + Path.DirectorySeparatorChar, StringComparison.Ordinal)) {
-					Log.LogError (null, null, null, item.ItemSpec, 0, 0, 0, 0, MSBStrings.E0100, logicalName);
-					continue;
-				}
-
-				if (logicalName == "Info.plist") {
-					Log.LogWarning (null, null, null, item.ItemSpec, 0, 0, 0, 0, MSBStrings.E0101);
+				if (!TryCreateItemWithLogicalName (this, item, out var bundleResource))
 					continue;
-				}
-
-				if (BundleResource.IsIllegalName (logicalName, out var illegal)) {
-					Log.LogError (null, null, null, item.ItemSpec, 0, 0, 0, 0, MSBStrings.E0102, illegal);
-					continue;
-				}
-
-				var bundleResource = new TaskItem (item);
-				bundleResource.SetMetadata ("LogicalName", logicalName);
 
 				bool optimize = false;
 
@@ -122,11 +95,51 @@ bool ExecuteImpl ()
 				bundleResources.Add (bundleResource);
 			}
 
+			bundleResources.AddRange (UnpackedResources);
+
 			BundleResourcesWithLogicalNames = bundleResources.ToArray ();
 
 			return !Log.HasLoggedErrors;
 		}
 
+		public static bool TryCreateItemWithLogicalName<T> (T task, ITaskItem item, [NotNullWhen (true)] out TaskItem? itemWithLogicalName) where T : Task, IHasProjectDir, IHasResourcePrefix, IHasSessionId
+		{
+			itemWithLogicalName = null;
+
+			// Skip anything with the PublishFolderType metadata, these are copied directly to the ResolvedFileToPublish item group instead.
+			var publishFolderType = item.GetMetadata ("PublishFolderType");
+			if (!string.IsNullOrEmpty (publishFolderType))
+				return false;
+
+			var logicalName = BundleResource.GetLogicalName (task, item);
+			// We need a physical path here, ignore the Link element
+			var path = item.GetMetadata ("FullPath");
+
+			if (!File.Exists (path)) {
+				task.Log.LogError (MSBStrings.E0099, logicalName, path);
+				return false;
+			}
+
+			if (logicalName.StartsWith (".." + Path.DirectorySeparatorChar, StringComparison.Ordinal)) {
+				task.Log.LogError (null, null, null, item.ItemSpec, 0, 0, 0, 0, MSBStrings.E0100, logicalName);
+				return false;
+			}
+
+			if (logicalName == "Info.plist") {
+				task.Log.LogWarning (null, null, null, item.ItemSpec, 0, 0, 0, 0, MSBStrings.E0101);
+				return false;
+			}
+
+			if (BundleResource.IsIllegalName (logicalName, out var illegal)) {
+				task.Log.LogError (null, null, null, item.ItemSpec, 0, 0, 0, 0, MSBStrings.E0102, illegal);
+				return false;
+			}
+
+			itemWithLogicalName = new TaskItem (item);
+			itemWithLogicalName.SetMetadata ("LogicalName", logicalName);
+			return true;
+		}
+
 		public void Cancel ()
 		{
 			if (ShouldExecuteRemotely ())

msbuild/Xamarin.MacDev.Tasks/Tasks/CollectPackLibraryResources.cs

@@ -0,0 +1,86 @@
+using System;
+using System.IO;
+using System.Collections.Generic;
+
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using Xamarin.Localization.MSBuild;
+using Xamarin.Messaging.Build.Client;
+
+namespace Xamarin.MacDev.Tasks {
+	// This task will collect several item groups with various types of assets/resources,
+	// add/compute the LogicalName value for each of them, and then add them to the
+	// ItemsWithLogicalNames item group. The items in this item group will have the
+	// 'OriginalItemGroup' metadata set indicating where they came from.
+	public class CollectPackLibraryResources : XamarinTask, IHasProjectDir, IHasResourcePrefix {
+		#region Inputs
+
+		public ITaskItem [] AtlasTextures { get; set; } = Array.Empty<ITaskItem> ();
+
+		public ITaskItem [] BundleResources { get; set; } = Array.Empty<ITaskItem> ();
+
+		public ITaskItem [] ImageAssets { get; set; } = Array.Empty<ITaskItem> ();
+
+		public ITaskItem [] InterfaceDefinitions { get; set; } = Array.Empty<ITaskItem> ();
+
+		public ITaskItem [] ColladaAssets { get; set; } = Array.Empty<ITaskItem> ();
+
+		public ITaskItem [] CoreMLModels { get; set; } = Array.Empty<ITaskItem> ();
+
+		public ITaskItem [] PartialAppManifests { get; set; } = Array.Empty<ITaskItem> ();
+
+		public ITaskItem [] SceneKitAssets { get; set; } = Array.Empty<ITaskItem> ();
+
+		[Required]
+		public string ProjectDir { get; set; } = string.Empty;
+
+		[Required]
+		public string ResourcePrefix { get; set; } = string.Empty;
+
+		#endregion
+
+		#region Outputs
+
+		// These items will have the following metadata set:
+		// * LogicalName
+		// * OriginalItemGroup: the name of the originating item group
+		[Output]
+		public ITaskItem [] ItemsWithLogicalNames { get; set; } = Array.Empty<ITaskItem> ();
+
+		#endregion
+
+		public override bool Execute ()
+		{
+			var prefixes = BundleResource.SplitResourcePrefixes (ResourcePrefix);
+			var rv = new List<ITaskItem> ();
+
+			var resources = new [] {
+				new { Name = "AtlasTexture", Items = AtlasTextures },
+				new { Name = "BundleResource", Items = BundleResources },
+				new { Name = "Collada", Items = ColladaAssets },
+				new { Name = "CoreMLModel", Items = CoreMLModels },
+				new { Name = "ImageAsset", Items = ImageAssets },
+				new { Name = "InterfaceDefinition", Items = InterfaceDefinitions },
+				new { Name = "PartialAppManifest", Items = PartialAppManifests },
+				new { Name = "SceneKitAsset", Items = SceneKitAssets },
+			};
+
+			foreach (var kvp in resources) {
+				var itemName = kvp.Name;
+				var items = kvp.Items;
+
+				foreach (var item in items) {
+					if (!CollectBundleResources.TryCreateItemWithLogicalName (this, item, out var itemWithLogicalName))
+						continue;
+
+					itemWithLogicalName.SetMetadata ("OriginalItemGroup", itemName);
+					rv.Add (itemWithLogicalName);
+				}
+			}
+
+			ItemsWithLogicalNames = rv.ToArray ();
+
+			return !Log.HasLoggedErrors;
+		}
+	}
+}

msbuild/Xamarin.MacDev.Tasks/Tasks/CompileSceneKitAssets.cs

@@ -118,6 +118,14 @@ Task CopySceneKitAssets (string scnassets, string output, string intermediate)
 			return ExecuteAsync (GetFullPathToTool (), args, sdkDevPath: SdkDevPath, environment: environment, showErrorIfFailure: true);
 		}
 
+		static bool TryGetScnAssetsPath (string file, out string scnassets)
+		{
+			scnassets = file;
+			while (scnassets.Length > 0 && Path.GetExtension (scnassets).ToLowerInvariant () != ".scnassets")
+				scnassets = Path.GetDirectoryName (scnassets);
+			return scnassets.Length > 0;
+		}
+
 		public override bool Execute ()
 		{
 			if (ShouldExecuteRemotely ()) {
@@ -140,15 +148,9 @@ public override bool Execute ()
 					continue;
 
 				// get the .scnassets directory path
-				var scnassets = Path.GetDirectoryName (asset.ItemSpec);
-				while (scnassets.Length > 0 && Path.GetExtension (scnassets).ToLowerInvariant () != ".scnassets")
-					scnassets = Path.GetDirectoryName (scnassets);
-
-				if (scnassets.Length == 0)
+				if (!TryGetScnAssetsPath (asset.ItemSpec, out var scnassets))
 					continue;
 
-				asset.RemoveMetadata ("LogicalName");
-
 				var bundleName = BundleResource.GetLogicalName (this, asset);
 				var output = new TaskItem (Path.Combine (intermediate, bundleName));
 
@@ -159,6 +161,13 @@ public override bool Execute ()
 					// .. but we really want it to be for @scnassets, so set ItemSpec accordingly
 					scnassetsItem.ItemSpec = scnassets;
 
+					// .. and set LogicalName, the original one is for @asset
+					if (!TryGetScnAssetsPath (bundleName, out var logicalScnAssetsPath)) {
+						Log.LogError (null, null, null, asset.ItemSpec, MSBStrings.E7136 /* Unable to compute the path of the *.scnassets path from the item's LogicalName '{0}'. */ , bundleName);
+						continue;
+					}
+					scnassetsItem.SetMetadata ("LogicalName", logicalScnAssetsPath);
+
 					// .. and remove the @OriginalItemSpec which is for @asset
 					scnassetsItem.RemoveMetadata ("OriginalItemSpec");
 

msbuild/Xamarin.MacDev.Tasks/Tasks/CreateEmbeddedResources.cs

@@ -19,17 +19,6 @@ public class CreateEmbeddedResources : XamarinTask {
 
 		public override bool Execute ()
 		{
-			if (ShouldExecuteRemotely ()) {
-				foreach (var bundleResource in this.BundleResources) {
-					var logicalName = bundleResource.GetMetadata ("LogicalName");
-
-					if (!string.IsNullOrEmpty (logicalName)) {
-						logicalName = logicalName.Replace ("\\", "/");
-						bundleResource.SetMetadata ("LogicalName", logicalName);
-					}
-				}
-			}
-
 			EmbeddedResources = new ITaskItem [BundleResources.Length];
 
 			for (int i = 0; i < BundleResources.Length; i++) {