Merge branch 'main' of https://github.com/dotnet/machinelearning at r…

…elease 1.6.0.
darth-vader-lg · Jul 21, 2021 · bd8d68d · bd8d68d
2 parents f11c475 + eb9cee6
commit bd8d68d
Show file tree

Hide file tree

Showing 109 changed files with 14,438 additions and 209 deletions.
diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml
@@ -10,7 +10,57 @@ resources:
   - container: UbuntuContainer
     image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-mlnet-20200515184230-2c829e8
 
+  - container: UbuntuCrossArmContainer
+    image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-mlnet-cross-arm-20210512124625-2e59a5f
+
+  - container: UbuntuCrossArm64Container
+    image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-mlnet-cross-arm64-20210512124625-2e59a5f
+
 jobs:
+- template: /build/ci/job-template.yml
+  parameters:
+    architecture: arm
+    name: Ubuntu_x64_cross_arm_NetCoreApp31
+    buildScript: ./build.sh
+    container: UbuntuCrossArmContainer
+    customMatrixes:
+      Debug_Build:
+        _configuration: Debug-netcoreapp3_1
+        _config_short: DI
+        _includeBenchmarkData: false
+        _targetFramework: netcoreapp3.1
+      Release_Build:
+        _configuration: Release-netcoreapp3_1
+        _config_short: RI
+        _includeBenchmarkData: true
+        _targetFramework: netcoreapp3.1
+    innerLoop: true
+    pool:
+      name: Hosted Ubuntu 1604
+    helixQueue: Ubuntu.1804.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm32v7-20210531091521-97d8652
+
+- template: /build/ci/job-template.yml
+  parameters:
+    architecture: arm64
+    name: Ubuntu_x64_cross_arm64_NetCoreApp31
+    buildScript: ./build.sh
+    container: UbuntuCrossArm64Container
+    customMatrixes:
+      Debug_Build:
+        _configuration: Debug-netcoreapp3_1
+        _config_short: DI
+        _includeBenchmarkData: false
+        _targetFramework: netcoreapp3.1
+      Release_Build:
+        _configuration: Release-netcoreapp3_1
+        _config_short: RI
+        _includeBenchmarkData: true
+        _targetFramework: netcoreapp3.1
+    innerLoop: true
+    pool:
+      name: Hosted Ubuntu 1604
+    helixQueue: Ubuntu.1804.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm64v8-20210531091519-97d8652
+
 - template: /build/ci/job-template.yml
   parameters:
     name: Centos_x64_NetCoreApp31
@@ -51,6 +101,28 @@ jobs:
       name: Hosted macOS
     helixQueue: OSX.1015.Amd64.Open
 
+- template: /build/ci/job-template.yml
+  parameters:
+    architecture: arm64
+    name: MacOS_cross_arm64_NetCoreApp21
+    buildScript: ./build.sh
+    customMatrixes:
+      Debug_Build:
+        _configuration: Debug-netcoreapp3_1
+        _config_short: DI
+        _includeBenchmarkData: false
+        _targetFramework: netcoreapp3.1
+      Release_Build:
+        _configuration: Release-netcoreapp3_1
+        _config_short: RI
+        _includeBenchmarkData: true
+        _targetFramework: netcoreapp3.1
+    innerLoop: true
+    pool:
+      vmImage: macOS-10.15
+    testTargetFramework: net6.0
+    helixQueue: OSX.1100.Arm64.Open
+
 - template: /build/ci/job-template.yml
   parameters:
     name: Windows_x64_NetCoreApp31

diff --git a/build/ci/job-template.yml b/build/ci/job-template.yml
@@ -13,6 +13,7 @@ parameters:
   container: ''
   useVSTestTask: false
   helixQueue: ''
+  testTargetFramework: ''
 
 jobs:
   - job: ${{ parameters.name }}
@@ -36,6 +37,12 @@ jobs:
         spaceValue: ' '
       ${{ if eq(parameters.buildScript, './build.sh') }}:
         spaceValue: '%20'
+      ${{ if and(eq(parameters.pool.name, 'Hosted Ubuntu 1604'), contains(parameters.name, 'cross')) }}:
+        ROOTFS_DIR: '/crossrootfs/${{ parameters.architecture }}'
+      ${{ if eq(parameters.testTargetFramework, '') }}:
+        testTargetFramework: ''
+      ${{ if ne(parameters.testTargetFramework, '') }}:
+        testTargetFramework: '/p:TestTargetFramework=${{ parameters.testTargetFramework }}'
     strategy:
       matrix:
         ${{ if eq(parameters.customMatrixes, '') }}:
@@ -64,13 +71,17 @@ jobs:
         displayName: MacOS Homebrew bug Workaround
         continueOnError: true
     # Extra MacOS step required to install OS-specific dependencies
-    - ${{ if eq(parameters.pool.name, 'Hosted macOS') }}:
+    - ${{ if and(eq(parameters.pool.name, 'Hosted macOS'), not(contains(parameters.name, 'cross')))  }}:
       - script: brew update && brew install mono-libgdiplus && brew install $(Build.SourcesDirectory)/build/libomp.rb --build-from-source --formula && brew link libomp --force
         displayName: Install MacOS build dependencies
+    # Extra Apple MacOS step required to install OS-specific dependencies
+    - ${{ if and(contains(parameters.pool.vmImage, 'macOS'), contains(parameters.name, 'cross'))  }}:
+      - script: brew update && brew install mono-libgdiplus && brew install libomp && brew link libomp --force
+        displayName: Install MacOS ARM build dependencies
     - ${{ if and( eq(parameters.nightlyBuild, 'true'), eq(parameters.pool.name, 'Hosted Ubuntu 1604')) }}:
       - bash: echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(nightlyBuildRunPath):$LD_LIBRARY_PATH"
         displayName: Set LD_LIBRARY_PATH for Ubuntu and CentOS to locate Native shared library in current running path
-    - script: ${{ parameters.buildScript }} -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} /p:Coverage=${{ parameters.codeCoverage }}
+    - script: ${{ parameters.buildScript }} -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} /p:Coverage=${{ parameters.codeCoverage }} $(testTargetFramework)
       displayName: Build
     - ${{ if eq(parameters.pool.name, 'Hosted macOS') }}:
       - task: Bash@3
@@ -144,7 +155,7 @@ jobs:
               publishRunAttachments: true
       - ${{ if eq(parameters.innerLoop, 'true') }}:
         - ${{ if eq(parameters.codeCoverage, True) }}:
-          - script: ${{ parameters.buildScript }} /p:Build=false -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} -test -integrationTest -ci /p:TestRunnerAdditionalArguments='-notrait$(spaceValue)Category=SkipInCI' /p:Coverage=${{ parameters.codeCoverage }}
+          - script: ${{ parameters.buildScript }} /p:Build=false -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} -test -integrationTest -ci /p:TestRunnerAdditionalArguments='-notrait$(spaceValue)Category=SkipInCI' /p:Coverage=${{ parameters.codeCoverage }} $(testTargetFramework)
             displayName: Run CI Tests.
         - ${{ if eq(parameters.codeCoverage, False) }}:
           - template: /build/ci/send-to-helix.yml
@@ -155,9 +166,14 @@ jobs:
               ${{ if eq(parameters.buildScript, 'build.cmd') }}:
                 MsBuildScript: 'powershell $(Build.SourcesDirectory)/eng/common/msbuild.ps1'
                 WarnAsError: '-warnAsError 0'
+                TestTargetFramework: '$(testTargetFramework)'
               ${{ if eq(parameters.buildScript, './build.sh') }}:
                 MsBuildScript: '$(Build.SourcesDirectory)/eng/common/msbuild.sh'
                 WarnAsError: '--warnAsError false'
+                TestTargetFramework: '$(testTargetFramework)'
+              # Don't have M1 failures fail the build for now.
+              ${{ if and(contains(parameters.pool.vmImage, 'macOS'), contains(parameters.name, 'cross'))  }}:
+                continueOnError: true
     - script: $(dotnetPath) msbuild -restore build/Codecoverage.proj
       displayName: Upload coverage to codecov.io
       condition: and(succeeded(), eq(${{ parameters.codeCoverage }}, True))
@@ -221,5 +237,5 @@ jobs:
           script: Get-ChildItem -Path  '.\artifacts\bin' -Recurse | Where-Object {$_.FullName -like "*runtimes*"} | Remove-Item -Recurse -Confirm:$false -Force
         displayName: Clean up runtime folder for package (Unix)
     - ${{ if eq(parameters.nightlyBuild, 'false') }}:
-      - script: ${{ parameters.buildScript }} /p:Build=false -pack -ci -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }}
+      - script: ${{ parameters.buildScript }} /p:Build=false -pack -ci -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} $(testTargetFramework)
         displayName: Build Packages
diff --git a/build/ci/send-to-helix.yml b/build/ci/send-to-helix.yml
@@ -9,6 +9,7 @@ parameters:
   Architecture: 'x64'
   MsBuildScript: ''
   WarnAsError: ''
+  TestTargetFramework: ''
   HelixConfiguration: ''                 # optional -- additional property attached to a job
   IncludeDotNetCli: true                 # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
   EnableXUnitReporter: true              # optional -- true enables XUnit result reporting to Mission Control
@@ -38,9 +39,12 @@ steps:
             /p:WaitForWorkItemCompletion=${{ parameters.WaitForWorkItemCompletion }}
             /p:HelixBaseUri=${{ parameters.HelixBaseUri }}
             ${{ parameters.WarnAsError }}
+            ${{ parameters.TestTargetFramework }}
     displayName: ${{ parameters.DisplayNamePrefix }}
     env:
       SYSTEM_ACCESSTOKEN: $(System.AccessToken)
       Creator: ${{ parameters.Creator }}
       HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
     continueOnError: ${{ parameters.continueOnError }}
+    ${{ if and(contains(parameters.HelixTargetQueues, 'OSX.1100.Arm64.Open'), in(variables['Build.Reason'], 'PullRequest')) }}:
+      enabled: False