Azure Split - Step 3 - Tie Everything Together

.gitignore

@@ -193,12 +193,10 @@ PublishScripts/
 *.nupkg
 # NuGet Symbol Packages
 *.snupkg
-# The packages folder can be ignored because of Package Restore
+# Most of the packages folder can be ignored.
 **/[Pp]ackages/*
-# except build/, which is used as an MSBuild target.
-!**/[Pp]ackages/build/
-# Uncomment if necessary however generally it will be regenerated when needed
-#!**/[Pp]ackages/repositories.config
+!**/[Pp]ackages/.gitkeep
+
 # NuGet v3's project.json files produces more ignorable files
 *.nuget.props
 *.nuget.targets

BUILDGUIDE.md

@@ -211,27 +211,13 @@ dependencies.
 Alternatively, the `ReferenceType` build property may be specified with a value
 of `Package`.  This will change inter-component dependencies to use
 `<PackageReference>` dependencies, and require that dependent components be
-built and packaged before building the depending component.  In this scenario,
-the root `NuGet.config` file must be updated to include the following entry
-under the `<packageSources>` element:
-
-```xml
-<configuration>
-  <packageSources>
-    ...
-    <add key="local" value="packages/" />
-  </packageSources>
-</configuration>
-```
-
-As a convenience, a `NuGet.config.local` file is supplied with the above
-package source already present.  You may simply copy it over `NuGet.config`
-when using `Package` references.
+built and packaged before building the depending component.  This will generate NuGet
+packages in the root packages/ directory, and will be automatically searched by NuGet
+(see our root `NuGet.config`).
 
 Then, you can specify `Package` references be used, for example:
 
 ```bash
-cp NuGet.config.local NuGet.config
 dotnet build -t:BuildAbstractions
 dotnet build -t:BuildAzure -p:ReferenceType=Package
 dotnet build -t:BuildAll -p:ReferenceType=Package

NuGet.config

@@ -1,18 +1,18 @@
 <?xml version="1.0" encoding="utf-8"?>
 <configuration>
   <packageSources>
-    
+
     <!-- Clear all inherited feeds. -->
     <clear />
 
     <!--
-      We do not use the public nuget.org feed.  Instead, we use a governed
-      feed maintained by our SqlClientDrivers ADO organization.  Packages must
-      be individually vetted and added to this feed manually before they will
-      be available for consumption by our builds.
+      We do not use the public nuget.org feed.  Instead, we use a governed feed maintained by our
+      SqlClientDrivers ADO organization.  Packages must be individually vetted and added to this
+      feed manually before they will be available for consumption by our builds.
 
       https://sqlclientdrivers.visualstudio.com/public/_artifacts/feed/sqlclient
     -->
     <add key="governed" value="https://sqlclientdrivers.pkgs.visualstudio.com/public/_packaging/sqlclient/nuget/v3/index.json" />
+
   </packageSources>
 </configuration>

NuGet.config.local

@@ -1,16 +1,25 @@
 <?xml version="1.0" encoding="utf-8"?>
-<!--
-  This NuGet config is used by developers and our pipelines builds when we need
-  to reference packages that are not available on the public nuget.org feed,
-  for example to test AKV that depends on an un-published version of MDS.
--->
 <configuration>
   <packageSources>
+
+    <!-- Clear all inherited feeds. -->
     <clear />
-    <!-- The normal feed. -->
+
+    <!--
+      We do not use the public nuget.org feed.  Instead, we use a governed feed maintained by our
+      SqlClientDrivers ADO organization.  Packages must be individually vetted and added to this
+      feed manually before they will be available for consumption by our builds.
+
+      https://sqlclientdrivers.visualstudio.com/public/_artifacts/feed/sqlclient
+    -->
     <add key="governed" value="https://sqlclientdrivers.pkgs.visualstudio.com/public/_packaging/sqlclient/nuget/v3/index.json" />
 
-    <!-- Our local packages/ directory. -->
+    <!--
+      This package source is used by developers and our pipelines builds when we need to reference
+      packages that are not available on the public nuget.org feed, for example to test MDS that
+      depends on a version of Abstractions that was just built in the workspace.
+    -->
     <add key="local" value="packages/" />
+
   </packageSources>
 </configuration>

build.proj

@@ -10,7 +10,10 @@
     <!-- SourceLink variable-->
     <DisableSourceLink>false</DisableSourceLink>
 
-    <RestoreConfigFile>src\NuGet.config</RestoreConfigFile>
+    <!-- Use the NuGet.config with a local source for Package reference builds. -->
+    <NuGetConfigFile>$(RepoRoot)/NuGet.config</NuGetConfigFile>
+    <NuGetConfigFile Condition="'$(ReferenceType)' == 'Package'">$(NuGetConfigFile).local</NuGetConfigFile>
+
     <Configuration Condition="'$(Configuration)' == ''">Debug</Configuration>
     <Platform Condition="'$(Platform)' == ''">AnyCPU</Platform>
     <!-- Flag to control whether or not to build Microsoft.DotNet.GenAPI project in tools -->
@@ -29,7 +32,7 @@
     <TargetNetFxVersion Condition="$(TargetGroup) == 'netfx' AND $(TargetNetFxVersion) == ''">$(TF)</TargetNetFxVersion>
     <GenerateNuget Condition="'$(GenerateNuget)' == '' AND '$(IsEnabledWindows)' == 'true'">true</GenerateNuget>
 
-    <CommonProperties>Configuration=$(Configuration);ReferenceType=$(ReferenceType)</CommonProperties>
+    <CommonProperties>Configuration=$(Configuration);ReferenceType=$(ReferenceType);RestoreConfigFile=$(NuGetConfigFile)</CommonProperties>
     <SqlServerLibProperties>$(CommonProperties);AssemblyVersion=$(SqlServerAssemblyVersion);AssemblyFileVersion=$(SqlServerAssemblyFileVersion);Version=$(SqlServerPackageVersion);</SqlServerLibProperties>
     <ProjectProperties>$(CommonProperties);AssemblyFileVersion=$(AssemblyFileVersion);TargetsWindows=$(TargetsWindows);TargetsUnix=$(TargetsUnix);</ProjectProperties>
     <TestProjectProperties>$(ProjectProperties);BuildForRelease=false;TargetNetCoreVersion=$(TargetNetCoreVersion);TargetNetFxVersion=$(TargetNetFxVersion)</TestProjectProperties>
@@ -285,6 +288,10 @@
   <!-- Run all unit tests applicable to Windows. -->
   <Target Name="RunUnitTestsWindows" Condition="'$(IsEnabledWindows)' == 'true' AND $(ReferenceType.Contains('Project'))">
     <PropertyGroup>
+      <!--
+        The UnitTests project exclusively uses Project references, which is the default applied by
+        src/Directory.Build.props, so no need to specify the ReferenceType property here.
+      -->
       <TestCommand>
         $(DotnetPath)dotnet test "@(UnitTestsProj)"
         -f $(TF)
@@ -301,7 +308,7 @@
       <TestCommand>$([System.Text.RegularExpressions.Regex]::Replace($(TestCommand), "\s+", " "))</TestCommand>
     </PropertyGroup>
     <Message Text=">>> Running unit tests for Windows via command: $(TestCommand)"/>
     <Exec ConsoleToMsBuild="true" Command="$(TestCommand)"/>
   </Target>
 
   <!-- Run all unit tests applicable to Unix. -->
@@ -337,6 +344,8 @@
         -f $(TF)
         -p:Configuration=$(Configuration)
         -p:ReferenceType=$(ReferenceType)
+        -p:AbstractionsPackageVersion=$(AbstractionsPackageVersion)
+        -p:MdsPackageVersion=$(MdsPackageVersion)
         $(CollectStatement)
         --results-directory $(ResultsDirectory)
         --filter "$(FilterStatement)"
@@ -360,6 +369,8 @@
         -f $(TF)
         -p:Configuration=$(Configuration)
         -p:ReferenceType=$(ReferenceType)
+        -p:AbstractionsPackageVersion=$(AbstractionsPackageVersion)
+        -p:MdsPackageVersion=$(MdsPackageVersion)
         $(CollectStatement)
         --results-directory $(ResultsDirectory)
         --filter "$(FilterStatement)"
@@ -390,6 +401,8 @@
         -p:ReferenceType=$(ReferenceType)
         -p:TestSet=$(TestSet)
         -p:TestTargetOS=Windows$(TargetGroup)
+        -p:AbstractionsPackageVersion=$(AbstractionsPackageVersion)
+        -p:MdsPackageVersion=$(MdsPackageVersion)
         $(CollectStatement)
         --results-directory $(ResultsDirectory)
         --filter "$(FilterStatement)"
@@ -401,7 +414,7 @@
       <TestCommand>$(TestCommand.Replace($([System.Environment]::NewLine), " "))</TestCommand>
     </PropertyGroup>
     <Message Text=">>> Running Manual test for Windows via command: $(TestCommand)" />
     <Exec ConsoleToMsBuild="true" Command="$(TestCommand)" />
   </Target>
 
   <!-- Run all Manual tests applicable to Unix. -->
@@ -416,6 +429,8 @@
         -p:ReferenceType=$(ReferenceType)
         -p:TestSet=$(TestSet)
         -p:TestTargetOS=Unixnetcoreapp
+        -p:AbstractionsPackageVersion=$(AbstractionsPackageVersion)
+        -p:MdsPackageVersion=$(MdsPackageVersion)
         $(CollectStatement)
         --results-directory $(ResultsDirectory)
         --filter "$(FilterStatement)"
@@ -436,7 +451,12 @@
     <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".", "bin", SearchOption.AllDirectories))' />
     <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".", ".nuget", SearchOption.AllDirectories))' />
     <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".", "obj", SearchOption.AllDirectories))' />
-    <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".","packages", SearchOption.AllDirectories))' />
+    <!-- We keep packages/ but remove all of the generated files. -->
+    <ItemGroup>
+      <PackagesToDelete Include="$(RepoRoot)/packages/*.nupkg" />
+      <PackagesToDelete Include="$(RepoRoot)/packages/*.snupkg" />
+    </ItemGroup>
+    <Delete Files="@(PackagesToDelete)" />
   </Target>
 
   <!-- AKV Targets ========================================================= -->
@@ -445,7 +465,7 @@
     <!-- @TODO: TestTargetOS makes this far more complicated than it needs to be. We should remove it. -->
     <!-- @TODO: RemoveProperties shouldn't be necessary -->
     <Message Text=">>> Restoring AKV project" />
-    <MSBuild Projects="@(AKVProvider)" Targets="Restore" />
+    <MSBuild Projects="@(AKVProvider)" Targets="Restore" Properties="$(CommonProperties)" />
 
     <PropertyGroup>
       <BuildAkvProperties>$(CI);TestTargetOS=$(TestOS)netfx;Platform=AnyCPU;$(ProjectProperties);$(NugetPackProperties)</BuildAkvProperties>
@@ -473,7 +493,7 @@
   </Target>
 
   <Target Name="BuildAKVNetFx" DependsOnTargets="BuildNetFx" Condition="'$(IsEnabledWindows)' == 'true'">
-    <MSBuild Projects="@(AKVProvider)" Targets="restore" Properties="TestTargetOS=$(TestOS)netfx" />
+    <MSBuild Projects="@(AKVProvider)" Targets="restore" Properties="$(CommonProperties);TestTargetOS=$(TestOS)netfx" />
     <Message Text=">>> Building AKVNetFx [$(CI);TestTargetOS=$(TestOS)netfx;Platform=AnyCPU;$(TestProjectProperties)] ..." Condition="!$(ReferenceType.Contains('Package'))" />
     <MSBuild Projects="@(AKVProvider)" Properties="$(CI);TestTargetOS=$(TestOS)netfx;Platform=AnyCPU;$(TestProjectProperties);$(NugetPackProperties);" Condition="!$(ReferenceType.Contains('Package'))" />
 
@@ -483,7 +503,7 @@
   </Target>
 
   <Target Name="BuildAKVNetCore" DependsOnTargets="BuildNetCore">
-    <MSBuild Projects="@(AKVProvider)" Targets="restore" Properties="TestTargetOS=$(TestOS)netcoreapp" />
+    <MSBuild Projects="@(AKVProvider)" Targets="restore" Properties="$(CommonProperties);TestTargetOS=$(TestOS)netcoreapp" />
     <Message Text=">>> Building AKVNetCore [$(CI);TestTargetOS=$(TestOS)netcoreapp;$(ProjectProperties);Platform=AnyCPU;ReferenceType=$(ReferenceType);] ..." Condition="!$(ReferenceType.Contains('Package'))" />
     <MSBuild Projects="@(AKVProvider)" Properties="$(CI);TestTargetOS=$(TestOS)netcoreapp;$(ProjectProperties);Platform=AnyCPU;" Condition="!$(ReferenceType.Contains('Package'))" />
 
@@ -493,7 +513,7 @@
   </Target>
 
   <Target Name="BuildAKVNetCoreAllOS" DependsOnTargets="BuildNetCore">
-    <MSBuild Projects="@(AKVProvider)" Targets="restore" Properties="TestTargetOS=$(TestOS)netcoreapp" />
+    <MSBuild Projects="@(AKVProvider)" Targets="restore" Properties="$(CommonProperties);TestTargetOS=$(TestOS)netcoreapp" />
     <MSBuild Projects="@(AKVProvider)" Properties="$(CI);TestTargetOS=$(TestOS)netcoreapp;$(ProjectProperties);Platform=AnyCPU;OSGroup=Unix;" RemoveProperties="TargetsWindows;TargetsUnix;" />
     <MSBuild Projects="@(AKVProvider)" Properties="$(CI);TestTargetOS=$(TestOS)netcoreapp;$(ProjectProperties);Platform=AnyCPU;OSGroup=Windows_NT;" RemoveProperties="TargetsWindows;TargetsUnix;" Condition="'$(IsEnabledWindows)' == 'true'" />
     <MSBuild Projects="@(AKVProvider)" Properties="$(CI);TestTargetOS=$(TestOS)netcoreapp;$(ProjectProperties);Platform=AnyCPU;OSGroup=AnyOS;" RemoveProperties="TargetsWindows;TargetsUnix;" />

eng/pipelines/common/templates/jobs/build-signed-package-job.yml

@@ -27,6 +27,8 @@ jobs:
   variables:
   - template: /eng/pipelines/libraries/variables.yml@self
   - ${{ if parameters.isPreview }}:
+    - name: abstractionsPackageVersion
+      value: $(abstractionsPackagePreviewVersion)
     - name: mdsPackageVersion
       value: $(previewMdsPackageVersion)
 
@@ -53,14 +55,14 @@ jobs:
   # Perform analysis before building, since this step will clobber build output.
   - template: /eng/pipelines/common/templates/steps/code-analyze-step.yml@self
 
-  # Build MDS in Package mode, producing signed DLLs.
+  # Build MDS, producing signed DLLs.
   - template: /eng/pipelines/common/templates/steps/build-all-configurations-signed-dlls-step.yml@self
     parameters:
-      buildConfiguration: Release
       # These variables are sourced from common-variables.yml.
+      abstractionsAssemblyFileVersion: $(abstractionsAssemblyFileVersion)
+      abstractionsPackageVersion: $(abstractionsPackageVersion)
       mdsAssemblyFileVersion: $(mdsAssemblyFileVersion)
       mdsPackageVersion: $(mdsPackageVersion)
-      referenceType: Package
 
   - template: /eng/pipelines/common/templates/steps/esrp-code-signing-step.yml@self
     parameters:
@@ -74,22 +76,17 @@ jobs:
       nuspecPath: $(nuspecPath)
       outputDirectory: $(artifactDirectory)
       packageVersion: $(mdsPackageVersion)
+      properties: 'AbstractionsPackageVersion=$(abstractionsPackageVersion)'
       referenceType: Package
 
   - template: /eng/pipelines/common/templates/steps/esrp-code-signing-step.yml@self
     parameters:
       artifactType: pkg
 
   - template: /eng/pipelines/common/templates/steps/copy-dlls-for-test-step.yml@self
-    parameters:
-      buildConfiguration: Release
-      referenceType: Package
-      product: MDS
 
   # Publish symbols to servers
-  - template: /eng/pipelines/common/templates/steps/publish-symbols-step.yml@self
-    parameters:
-      buildConfiguration: Release
-      referenceType: Package
-      publishSymbols: ${{ parameters['PublishSymbols'] }}
-      symbolsArtifactName: mds_symbols_$(System.TeamProject)_$(Build.Repository.Name)_$(Build.SourceBranchName)_$(mdsPackageVersion)_$(System.TimelineId)
+  - ${{ if eq(parameters.publishSymbols, true) }}:
+    - template: /eng/pipelines/common/templates/steps/publish-symbols-step.yml@self
+      parameters:
+        symbolsArtifactName: mds_symbols_$(System.TeamProject)_$(Build.Repository.Name)_$(Build.SourceBranchName)_$(mdsPackageVersion)_$(System.TimelineId)

eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml

@@ -23,14 +23,18 @@ parameters:
     type: string
     default: ADO-MMS22-SQL19
 
+  - name: abstractionsArtifactsName
+    type: string
+    default: Abstractions.Artifacts
+
   - name: mdsArtifactsName
     type: string
     default: MDS.Artifacts
-  
+
   - name: platform
     type: string
     default: $(Platform)
-  
+
   - name: buildConfiguration
     type: string
     values:
@@ -41,6 +45,9 @@ parameters:
     type: stepList
     default: []
 
+  - name: abstractionsPackageVersion
+    type: string
+
   - name: mdsPackageVersion
     type: string
 
@@ -68,14 +75,13 @@ jobs:
      # configuration
     - ${{ parameters.prebuildSteps }}
 
-  # If we're testing in Package mode, create the packages/ directory and then
-  # setup the top-level NuGet.config to look in packages/ for local NuGet
-  # dependencies.
+  # If we're testing in Package mode, then we must download the Abstractions package artifacts.
   - ${{ if eq(parameters.referenceType, 'Package') }}:
-    - pwsh: New-Item -Path "$(Build.SourcesDirectory)/packages" -ItemType Directory -Force
-      displayName: Create packages/ directory
-    - pwsh: Copy-Item -Path "$(Build.SourcesDirectory)/NuGet.config.local" -Destination "$(Build.SourcesDirectory)/NuGet.config" -Force
-      displayName: Use local NuGet packages
+    - task: DownloadPipelineArtifact@2
+      displayName: Download Abstractions Package Artifacts
+      inputs:
+        artifactName: ${{ parameters.abstractionsArtifactsName }}
+        targetPath: $(packagePath)
 
   # Install the .NET SDK.
   - template: /eng/pipelines/steps/install-dotnet.yml@self
@@ -99,16 +105,18 @@ jobs:
       referenceType: ${{ parameters.referenceType }}
       operatingSystem: Windows
       build: MDS
+      abstractionsPackageVersion: ${{parameters.abstractionsPackageVersion}}
       mdsPackageVersion: ${{ parameters.mdsPackageVersion }}
 
   - template: /eng/pipelines/common/templates/steps/generate-nuget-package-step.yml@self
     parameters:
       buildConfiguration: ${{ parameters.buildConfiguration }}
       displayName: 'Create MDS NuGet Package'
       generateSymbolsPackage: false
-      packageVersion: ${{ parameters.mdsPackageVersion }}
       nuspecPath: 'tools/specs/Microsoft.Data.SqlClient.nuspec'
       outputDirectory: $(packagePath)
+      packageVersion: ${{ parameters.mdsPackageVersion }}
+      properties: 'AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }}'
       referenceType: ${{ parameters.referenceType }}
 
   - template: /eng/pipelines/common/templates/steps/ci-project-build-step.yml@self