Use ProjectReference items for live built dependencies

Directory.Build.props

@@ -2,12 +2,6 @@
 
   <Import Project="Sdk.props" Sdk="Microsoft.DotNet.Arcade.Sdk" />
 
-  <PropertyGroup>
-    <!-- Disable TFM filtering because it filteres old netstandard TFMs.
-         Once all old netstandard TFMs are cleanup, this could be removed. -->
-    <NoTargetFrameworkFiltering>true</NoTargetFrameworkFiltering>
-  </PropertyGroup>
-
   <PropertyGroup>
     <Copyright>$(CopyrightNetFoundation)</Copyright>
     <PackageLicenseExpression>MIT</PackageLicenseExpression>

README.md

@@ -53,11 +53,6 @@ for detailed guidance.
 ./generate.sh --package system.buffers,4.5.1
 ```
 
-After generating new reference packages, all new projects must be referenced as a
-[DependencyPackageProjects](https://github.com/dotnet/source-build-reference-packages/blob/main/eng/Build.props#L9).
-These must be defined in dependency order. There is a
-[tracking issue](https://github.com/dotnet/source-build/issues/1690) to address this manual step.
-
 The tooling does not handle all situations and sometimes the generated code will need manual tweeks to get
 it to compile. If this occurs when generating a newer version of an existing package, it can be helpful to
 regenerate the older version to see what customizations to the generated code were made.
@@ -73,9 +68,6 @@ generated packages show changes when being regenerated.
     the case, the changes to the existing package should be reverted.
     2. The generate tooling has changed since the last time this package was generated. The new changes should
     be considered better/correct and should be committed.
-* Add `DependencyPackageProjects` for all new projects in the
-[eng/Build.props](https://github.com/dotnet/source-build-reference-packages/blob/main/eng/Build.props#L9)
-in the correct dependency order.
 * Run build with the `./build.sh -sb` command.
 * If the compilation produces numerous compilation issue - run the `./build.sh --projects <path to .csproj file>`
   command for each generated reference package separately.

eng/Build.props

@@ -9,72 +9,6 @@
   </ItemGroup>
 
   <ItemGroup Condition="'$(GeneratePackageSource)' != 'true' and '$(Test)' != 'true'">
-    <!--
-      All new projects must be added to DependencyPackageProjects. This will ensure that they get built first
-      and in order of inclusion. The resulting packages from these projects will get added the source-build
-      package cache when building with source-build to prevent prebuilts.
-
-      All newly added packages should be grouped together at the end of the list separated by empty lines.
-      The reason this is important is that when previous source-built artifacts are updated, the entries
-      for the new projects it includes can be removed. New PRs may miss that cut and if they are not grouped
-      at the bottom, this management becomes more difficult.
-
-      Format:
-      <DependencyPackageProjects Include="$(RepoRoot)src\referencePackages\src\**\Microsoft.Extensions.Options.5.0.0.csproj" />
-    -->
-
-    <DependencyPackageProjects Include="$(RepoRoot)src\referencePackages\src\**\System.Formats.Asn1.5.0.0.csproj" />
-    <DependencyPackageProjects Include="$(RepoRoot)src\referencePackages\src\**\System.Formats.Asn1.6.0.0.csproj" />
-    <DependencyPackageProjects Include="$(RepoRoot)src\referencePackages\src\**\System.Formats.Asn1.7.0.0.csproj" />
-    <DependencyPackageProjects Include="$(RepoRoot)src\referencePackages\src\**\System.Formats.Asn1.8.0.0.csproj" />
-    <DependencyPackageProjects Include="$(RepoRoot)src\referencePackages\src\**\System.Security.Cryptography.Pkcs.6.0.0.csproj" />
-    <DependencyPackageProjects Include="$(RepoRoot)src\referencePackages\src\**\System.Security.Cryptography.Pkcs.6.0.1.csproj" />
-    <DependencyPackageProjects Include="$(RepoRoot)src\referencePackages\src\**\System.Security.Cryptography.Pkcs.7.0.0.csproj" />
-    <DependencyPackageProjects Include="$(RepoRoot)src\referencePackages\src\**\System.Security.Cryptography.Xml.6.0.0.csproj" />
-    <DependencyPackageProjects Include="$(RepoRoot)src\referencePackages\src\**\System.Text.Json.6.0.0.csproj" />
-    <DependencyPackageProjects Include="$(RepoRoot)src\referencePackages\src\**\System.Text.Json.8.0.4.csproj" />
-
-    <DependencyPackageProjects Include="$(RepoRoot)src\referencePackages\src\**\System.Runtime.CompilerServices.Unsafe.6.1.1.csproj" />
-    <DependencyPackageProjects Include="$(RepoRoot)src\referencePackages\src\**\System.Reflection.DispatchProxy.4.8.1.csproj" />
-
-    <DependencyPackageProjects Include="$(RepoRoot)src\referencePackages\src\**\System.Reflection.MetadataLoadContext.8.0.0.csproj" />
-    <DependencyPackageProjects Include="$(RepoRoot)src\referencePackages\src\**\Microsoft.Build.17.12.6.csproj" />
-  </ItemGroup>
-
-  <ItemGroup Condition="'$(BuildDependencyPackageProjects)' == 'true'">
-    <!-- Building an empty project triggers building the Arcade Tools.proj which is needed before BuildDependencyPackageProjects -->
-    <ProjectToBuild Include="$(RepoRoot)\eng\NoOp.csproj" />
-  </ItemGroup>
-
-  <Target Name="BuildDependencyPackageProjects"
-          AfterTargets="Execute"
-          Condition="'$(BuildDependencyPackageProjects)' == 'true'"
-          Outputs="%(DependencyPackageProjects.Identity)">
-    <MSBuild Condition="'@(DependencyPackageProjects)' != ''"
-             Projects="@(DependencyPackageProjects)"
-             Targets="Restore;Build;Pack" />
-
-    <Copy Condition="'$(LocalNuGetPackageCacheDirectory)' != ''"
-          SourceFiles="@(DependencyPackageProjects->'$(ArtifactsShippingPackagesDir)%(FileName).nupkg')"
-          DestinationFolder="$(LocalNuGetPackageCacheDirectory)" />
-
-    <!--
-      When building in the VMR, any dependencyPackageProjects that existing in the ReferencePackagesDir
-      must be cleaned up. This can happen when manually updating packages to address vulnerable references.
-      In this case the ReferencePackagesDir contains the unpatched versions. This will note be needed when
-      the need for DependencyPackageProjects is removed as part of https://github.com/dotnet/source-build/issues/1690.
-    -->
-    <ItemGroup>
-      <FilesToDelete Condition="'$(VmrReferencePackagesDir)' != ''"
-                     Include="@(DependencyPackageProjects->'$(VmrReferencePackagesDir)%(FileName).nupkg')"/>
-    </ItemGroup>
-    <Message Condition="'$(VmrReferencePackagesDir)' != ''"
-             Text="Deleting Files @(FilesToDelete)" />
-    <Delete Condition="'$(VmrReferencePackagesDir)' != ''"
-            Files="@(FilesToDelete)" />
-  </Target>
-
-  <ItemGroup Condition="'$(GeneratePackageSource)' != 'true' and '$(BuildDependencyPackageProjects)' != 'true' and '$(Test)' != 'true'">
     <TargetingPackageProject Include="$(RepoRoot)src\targetPacks\ILsrc\**\*.csproj" />
     <ProjectToBuild Include="@(TargetingPackageProject)" />
 

eng/DotNetBuild.props

@@ -7,28 +7,4 @@
     <SourceBuildManagedOnly>true</SourceBuildManagedOnly>
   </PropertyGroup>
 
-  <Target Name="TriggerBuildDependencyPackageProjects"
-          Condition="'$(DotNetBuildInnerRepo)' == 'true'"
-          BeforeTargets="Execute">
-    <PropertyGroup>
-      <LocalNuGetPackageCacheDirectory Condition="'$(LocalNuGetPackageCacheDirectory)' == ''">$(CurrentRepoSourceBuiltNupkgCacheDir)</LocalNuGetPackageCacheDirectory>
-      <_AdditionalDependencyProjectsBuildArgs />
-      <_AdditionalDependencyProjectsBuildArgs Condition="'$(RestoreConfigFile)' != ''" >$(_AdditionalDependencyProjectsBuildArgs) /p:RestoreConfigFile=$(RestoreConfigFile)</_AdditionalDependencyProjectsBuildArgs>
-    </PropertyGroup>
-
-    <MakeDir Condition="'$(LocalNuGetPackageCacheDirectory)' != ''"
-             Directories="$(LocalNuGetPackageCacheDirectory)" />
-
-    <!-- 'SourceBuildOutputDir' must be included as a build parameter in order to correct new SBRPs referencing online feeds when dependent on DependencyPackageProjects. -->
-    <!-- See https://github.com/dotnet/source-build-reference-packages/pull/858 -->
-    <!-- This command purposefully does not pass DotNetBuildRepo or DotNetBuildOrchestrator, even though that would be the typical pattern for
-         inner command args. The issue is that if they are passed, this file will re-imported on the dependency package project invocation,
-         and because this target executes before Execute, the build will infinitely recurse. This probably could be fixed in other ways, but
-         given that SBRP is slated at some point to get proper support for project refs as a replacement for this invocation, this isn't really worth doing. -->
-    <Exec
-      Command="./build.sh --configuration $(Configuration) /bl:$(ArtifactsDir)sourcebuild-dependency-projects.binlog /p:LocalNuGetPackageCacheDirectory=$(LocalNuGetPackageCacheDirectory) /p:VmrReferencePackagesDir=$(VmrReferencePackagesDir) /p:SourceBuildOutputDir=$(SourceBuildOutputDir) /p:BuildDependencyPackageProjects=true /p:SetUpSourceBuildIntermediateNupkgCache=true /p:DotNetBuildOrchestrator=$(DotNetBuildOrchestrator) /p:DotNetBuildSourceOnly=true /p:DotNetBuildInnerRepo=true /p:MicrosoftNetCoreIlasmPackageRuntimeId=$(MicrosoftNetCoreIlasmPackageRuntimeId) $(_AdditionalDependencyProjectsBuildArgs)"
-      WorkingDirectory="$(InnerSourceBuildRepoRoot)"
-      EnvironmentVariables="@(InnerBuildEnv)" />
-  </Target>
-
 </Project>

eng/SourceBuildPrebuiltBaseline.xml

@@ -6,11 +6,6 @@
     <!-- runtime prebuilts.  Update when runtime produces intermediate packages correctly. -->
     <UsagePattern IdentityGlob="runtime.linux-x64.Microsoft.NETCore.ILAsm/*" />
     <UsagePattern IdentityGlob="runtime.linux-x64.Microsoft.NETCore.ILDAsm/*" />
-
-    <!-- Necessary to work around NuGet non-determinism with online builds/feeds. Can be removed once 
-    dependencies on it are removed from the previous-source-built packages. -->
-    <UsagePattern IdentityGlob="Newtonsoft.Json/13.0.1" />
-
   </IgnorePatterns>
   <Usages/>
 </UsageData>

src/packageSourceGenerator/PackageSourceGenerator.proj

@@ -26,6 +26,11 @@
     <PackageTargetDirectory>$([MSBuild]::NormalizeDirectory('$(BasePackageTargetDirectory)', '$(PackageVersion)'))</PackageTargetDirectory>
   </PropertyGroup>
 
+  <!-- The list of allowed external dependencies -->
+  <ItemGroup>
+    <AllowedPackageReference Include="Newtonsoft.Json" />
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="PackageSourceGeneratorTask\PackageSourceGeneratorTask.csproj"
                       ReferenceOutputAssembly="false"
@@ -344,10 +349,12 @@
     <GenerateProject PackageId="$(RealPackageId)"
                      PackageVersion="$(PackageVersion)"
                      ProjectTemplate="$(PackageProjectTemplate)"
+                     ProjectRoot="$(PackagesTargetDirectory)"
                      TargetPath="$(PackageProjectTargetPath)"
                      CompileItems="@(PackageCompileItem)"
                      PackageDependencies="@(PackageDependency)"
-                     FrameworkReferences="@(PackageFrameworkReference)" />
+                     FrameworkReferences="@(PackageFrameworkReference)"
+                     AllowedPackageReference="@(AllowedPackageReference)" />
 
     <Message Text="$(MSBuildProjectName) -> $(PackageProjectTargetPath)"
              Importance="high" />
@@ -401,14 +408,10 @@
     </ItemGroup>
 
     <!-- External packages aren't visited as they shouldn't be added to SBRP. --> 
-    <ItemGroup>
-      <ExternalPackage Include="Newtonsoft.Json" />
-    </ItemGroup>
-
     <ItemGroup>
       <FilteredPackageDependency Include="@(PackageDependency)"
                                  Exclude="@(TargetingPackage);
-                                          @(ExternalPackage)" />
+                                          @(AllowedPackageReference)" />
     </ItemGroup>
 
     <MSBuild Projects="$(MSBuildThisFileFullPath)"

src/packageSourceGenerator/PackageSourceGeneratorTask/GenerateProject.cs

@@ -35,6 +35,12 @@ public class GenerateProject : Task
         [Required]
         public required string TargetPath { get; set; }
 
+        /// <summary>
+        /// The root directory that the projects are written into.
+        /// </summary>
+        [Required]
+        public required string ProjectRoot { get; set; }
+
        /// <summary>
         /// The package's compile items, including target framework metadata.
         /// </summary>
@@ -50,6 +56,11 @@ public class GenerateProject : Task
         /// </summary>
         public ITaskItem[] FrameworkReferences { get; set; } = Array.Empty<ITaskItem>();
 
+        /// <summary>
+        /// The list of dependencies (package id) that should get emitted as PackageReference items.
+        /// </summary>
+        public string[]? AllowedPackageReference { get; set; }
+
         public override bool Execute()
         {
             string referenceIncludes = "";
@@ -77,18 +88,31 @@ public override bool Execute()
 
             foreach (string targetFramework in targetFrameworks)
             {
-                string references = string.Empty;
+                string packageReferences = string.Empty;
+                string projectReferences = string.Empty;
 
                 // Add package dependencies
                 foreach (ITaskItem packageDependency in PackageDependencies.Where(packageDependency => packageDependency.GetMetadata(SharedMetadata.TargetFrameworkMetadataName) == targetFramework))
                 {
-                    references += $"    <PackageReference Include=\"{packageDependency.ItemSpec}\" Version=\"{packageDependency.GetMetadata("Version")}\" />{Environment.NewLine}";
+                    string dependencyVersion = packageDependency.GetMetadata("Version");
+                    string dependencyProjectRelativePath = Path.Combine(packageDependency.ItemSpec.ToLowerInvariant(), dependencyVersion, $"{packageDependency.ItemSpec}.{dependencyVersion}.csproj"); 
+
+                    // If the dependency is on the package reference allowed list (i.e. for source-build-externals packages like Newtonsoft.Json), emit a package reference. Otherwise, emit a project reference.
+                    if (AllowedPackageReference is not null && AllowedPackageReference.Contains(packageDependency.ItemSpec))
+                    {
+                        packageReferences += $"    <PackageReference Include=\"{packageDependency.ItemSpec}\" Version=\"{dependencyVersion}\" />{Environment.NewLine}";
+                    }
+                    else
+                    {
+                        // Make sure that the path always uses forward slashes, even on Windows.
+                        projectReferences += $"    <ProjectReference Include=\"../../{dependencyProjectRelativePath.Replace('\\', '/')}\" />{Environment.NewLine}";
+                    }
                 }
 
-                if (references != string.Empty)
+                if (packageReferences != string.Empty || projectReferences != string.Empty)
                 {
                     referenceIncludes += $"  <ItemGroup Condition=\"'$(TargetFramework)' == '{targetFramework}'\">{Environment.NewLine}";
-                    referenceIncludes += references;
+                    referenceIncludes += packageReferences + projectReferences;
                     referenceIncludes += $"  </ItemGroup>{Environment.NewLine}{Environment.NewLine}";
                 }
 

src/referencePackages/Directory.Build.props

@@ -34,6 +34,9 @@
     <LangVersion>latest</LangVersion>
 
     <CommonSrc>$([MSBuild]::NormalizeDirectory('$(MSBuildThisFileDirectory)', 'common'))</CommonSrc>
+
+    <!-- Make RAR not unify assembly dependencies. This is necessary to make P2Ps with same package ids but different versions work. -->
+    <_FindDependencies>false</_FindDependencies>
   </PropertyGroup>
 
   <PropertyGroup>

src/referencePackages/Directory.Build.targets

@@ -18,7 +18,7 @@
     <Compile Include="$(CustomizationsSourcePath)" Condition="Exists('$(CustomizationsSourcePath)')" />
   </ItemGroup>
 
-    <!-- Customization extension point -->
+  <!-- Customization extension point -->
   <Import Project="$(CustomizationsPropsPath)" Condition="Exists('$(CustomizationsPropsPath)')" />
 
   <!--
@@ -43,14 +43,22 @@
     </KnownFrameworkReference>
   </ItemGroup>
 
+  <!-- Conflict resolution doesn't see P2Ps: https://github.com/dotnet/sdk/issues/2674. -->
+  <Target Name="PassProjectReferencesToConflictResolution"
+          BeforeTargets="_HandlePackageFileConflicts"
+          DependsOnTargets="ResolveProjectReferences">
+    <ItemGroup>
+      <Reference Include="@(_ResolvedProjectReferencePaths)" />
+      <_ResolvedProjectReferencePaths Remove="@(_ResolvedProjectReferencePaths)" />
+    </ItemGroup>
+  </Target>
+
   <!-- Filter out conflicting implicit assembly references. -->
   <Target Name="FilterImplicitAssemblyReferences"
-          Condition="'$(DisableImplicitFrameworkReferences)' != 'true'"
           DependsOnTargets="ResolveProjectReferences"
           AfterTargets="ResolveTargetingPackAssets">
     <ItemGroup>
       <_targetingPackReferenceExclusion Include="$(TargetName)" />
-      <_targetingPackReferenceExclusion Include="@(_ResolvedProjectReferencePaths->Metadata('Filename'))" />
     </ItemGroup>
     <ItemGroup>
       <_targetingPackReferenceWithProjectName Include="@(Reference->WithMetadataValue('ExternallyResolved', 'true')->Metadata('Filename'))"

src/referencePackages/src/microsoft.bcl.asyncinterfaces/6.0.0/Microsoft.Bcl.AsyncInterfaces.6.0.0.csproj

@@ -8,7 +8,7 @@
   </PropertyGroup>
 
   <ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
-    <PackageReference Include="System.Threading.Tasks.Extensions" Version="4.5.4" />
+    <ProjectReference Include="../../system.threading.tasks.extensions/4.5.4/System.Threading.Tasks.Extensions.4.5.4.csproj" />
   </ItemGroup>
 
 </Project>

src/referencePackages/src/microsoft.bcl.asyncinterfaces/7.0.0/Microsoft.Bcl.AsyncInterfaces.7.0.0.csproj

@@ -8,7 +8,7 @@
   </PropertyGroup>
 
   <ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
-    <PackageReference Include="System.Threading.Tasks.Extensions" Version="4.5.4" />
+    <ProjectReference Include="../../system.threading.tasks.extensions/4.5.4/System.Threading.Tasks.Extensions.4.5.4.csproj" />
   </ItemGroup>
 
 </Project>

src/referencePackages/src/microsoft.bcl.asyncinterfaces/8.0.0/Microsoft.Bcl.AsyncInterfaces.8.0.0.csproj

@@ -8,7 +8,7 @@
   </PropertyGroup>
 
   <ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
-    <PackageReference Include="System.Threading.Tasks.Extensions" Version="4.5.4" />
+    <ProjectReference Include="../../system.threading.tasks.extensions/4.5.4/System.Threading.Tasks.Extensions.4.5.4.csproj" />
   </ItemGroup>
 
 </Project>

src/referencePackages/src/microsoft.build.framework/17.12.6/Microsoft.Build.Framework.17.12.6.csproj

@@ -7,10 +7,10 @@
   </PropertyGroup>
 
   <ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
-    <PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0" />
-    <PackageReference Include="System.Memory" Version="4.5.5" />
-    <PackageReference Include="System.Runtime.CompilerServices.Unsafe" Version="6.0.0" />
-    <PackageReference Include="System.Security.Principal.Windows" Version="5.0.0" />
+    <ProjectReference Include="../../microsoft.win32.registry/5.0.0/Microsoft.Win32.Registry.5.0.0.csproj" />
+    <ProjectReference Include="../../system.memory/4.5.5/System.Memory.4.5.5.csproj" />
+    <ProjectReference Include="../../system.runtime.compilerservices.unsafe/6.0.0/System.Runtime.CompilerServices.Unsafe.6.0.0.csproj" />
+    <ProjectReference Include="../../system.security.principal.windows/5.0.0/System.Security.Principal.Windows.5.0.0.csproj" />
   </ItemGroup>
 
 </Project>

src/referencePackages/src/microsoft.build.framework/17.3.4/Microsoft.Build.Framework.17.3.4.csproj

@@ -7,11 +7,11 @@
   </PropertyGroup>
 
   <ItemGroup Condition="'$(TargetFramework)' == 'net6.0'">
-    <PackageReference Include="System.Security.Permissions" Version="6.0.0" />
+    <ProjectReference Include="../../system.security.permissions/6.0.0/System.Security.Permissions.6.0.0.csproj" />
   </ItemGroup>
 
   <ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
-    <PackageReference Include="System.Security.Permissions" Version="6.0.0" />
+    <ProjectReference Include="../../system.security.permissions/6.0.0/System.Security.Permissions.6.0.0.csproj" />
   </ItemGroup>
 
 </Project>