[main] Update dependencies from dotnet/arcade (#4442)

microsoft · May 15, 2023 · 565413e · 565413e
1 parent 95b5000
commit 565413e
Show file tree

Hide file tree

Showing 9 changed files with 153 additions and 72 deletions.
diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
@@ -26,9 +26,9 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.23226.4">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.23262.5">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>469dcc0cbcba5221727c8a5b9eec4a478e24a780</Sha>
+      <Sha>1aff4eb33aa7cbf26ccd9fc43c17cb609a14dad4</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
     <Dependency Name="Microsoft.Net.Compilers.Toolset" Version="4.6.0-1.23107.10">
@@ -44,9 +44,9 @@
       <Uri>https://github.com/dotnet/symreader-converter</Uri>
       <Sha>c5ba7c88f92e2dde156c324a8c8edc04d9fa4fe0</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.XliffTasks" Version="1.0.0-beta.23218.1" CoherentParentDependency="Microsoft.DotNet.Arcade.Sdk">
+    <Dependency Name="Microsoft.DotNet.XliffTasks" Version="1.0.0-beta.23261.1" CoherentParentDependency="Microsoft.DotNet.Arcade.Sdk">
       <Uri>https://github.com/dotnet/xliff-tasks</Uri>
-      <Sha>f35dcbb16190810eb88b103fb1f2cd48a3a76cf0</Sha>
+      <Sha>a0391b3beff0696561189b5881f8af3b651d64ac</Sha>
       <SourceBuild RepoName="xliff-tasks" ManagedOnly="true" />
     </Dependency>
     <Dependency Name="Microsoft.SourceLink.GitHub" Version="8.0.0-beta.23218.3" CoherentParentDependency="Microsoft.DotNet.Arcade.Sdk">

diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh
@@ -27,6 +27,7 @@ __AlpineArch=armv7
 __FreeBSDArch=arm
 __FreeBSDMachineArch=armv7
 __IllumosArch=arm7
+__HaikuArch=arm
 __QEMUArch=arm
 __UbuntuArch=armhf
 __UbuntuRepo="http://ports.ubuntu.com/"
@@ -85,8 +86,12 @@ __IllumosPackages+=" mit-krb5"
 __IllumosPackages+=" openssl"
 __IllumosPackages+=" zlib"
 
-__HaikuPackages="gmp"
+__HaikuPackages="gcc_syslibs"
+__HaikuPackages+=" gcc_syslibs_devel"
+__HaikuPackages+=" gmp"
 __HaikuPackages+=" gmp_devel"
+__HaikuPackages+=" icu66"
+__HaikuPackages+=" icu66_devel"
 __HaikuPackages+=" krb5"
 __HaikuPackages+=" krb5_devel"
 __HaikuPackages+=" libiconv"
@@ -95,6 +100,10 @@ __HaikuPackages+=" llvm12_libunwind"
 __HaikuPackages+=" llvm12_libunwind_devel"
 __HaikuPackages+=" mpfr"
 __HaikuPackages+=" mpfr_devel"
+__HaikuPackages+=" openssl"
+__HaikuPackages+=" openssl_devel"
+__HaikuPackages+=" zlib"
+__HaikuPackages+=" zlib_devel"
 
 # ML.NET dependencies
 __UbuntuPackages+=" libomp5"
@@ -210,6 +219,7 @@ while :; do
             __FreeBSDArch=amd64
             __FreeBSDMachineArch=amd64
             __illumosArch=x86_64
+            __HaikuArch=x86_64
             __UbuntuRepo="http://archive.ubuntu.com/ubuntu/"
             ;;
         x86)
@@ -340,7 +350,6 @@ while :; do
             ;;
         haiku)
             __CodeName=haiku
-            __BuildArch=x64
             __SkipUnmount=1
             ;;
         --skipunmount)
@@ -559,67 +568,54 @@ elif [[ "$__CodeName" == "illumos" ]]; then
 elif [[ "$__CodeName" == "haiku" ]]; then
     JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"}
 
-    echo "Building Haiku sysroot for x86_64"
+    echo "Building Haiku sysroot for $__HaikuArch"
     mkdir -p "$__RootfsDir/tmp"
-    cd "$__RootfsDir/tmp"
-    git clone -b hrev56235  https://review.haiku-os.org/haiku
-    git clone -b btrev43195 https://review.haiku-os.org/buildtools
-    cd "$__RootfsDir/tmp/buildtools" && git checkout 7487388f5110021d400b9f3b88e1a7f310dc066d
-
-    # Fetch some unmerged patches
-    cd "$__RootfsDir/tmp/haiku"
-    ## Add development build profile (slimmer than nightly)
-    git fetch origin refs/changes/64/4164/1 && git -c commit.gpgsign=false cherry-pick FETCH_HEAD
-
-    # Build jam
-    cd "$__RootfsDir/tmp/buildtools/jam"
-    make
-
-    # Configure cross tools
-    echo "Building cross-compiler"
-    mkdir -p "$__RootfsDir/generated"
-    cd "$__RootfsDir/generated"
-    "$__RootfsDir/tmp/haiku/configure" -j"$JOBS" --sysroot "$__RootfsDir" --cross-tools-source "$__RootfsDir/tmp/buildtools" --build-cross-tools x86_64
-
-    # Build Haiku packages
-    echo "Building Haiku"
-    echo 'HAIKU_BUILD_PROFILE = "development-raw" ;' > UserProfileConfig
-    "$__RootfsDir/tmp/buildtools/jam/jam0" -j"$JOBS" -q '<build>package' '<repository>Haiku'
-
-    BaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg"
-
-    # Download additional packages
-    echo "Downloading additional required packages"
+    pushd "$__RootfsDir/tmp"
+
+    mkdir "$__RootfsDir/tmp/download"
+
+    echo "Downloading Haiku package tool"
+    git clone https://github.com/haiku/haiku-toolchains-ubuntu --depth 1 $__RootfsDir/tmp/script
+    wget -O "$__RootfsDir/tmp/download/hosttools.zip" $($__RootfsDir/tmp/script/fetch.sh --hosttools)
+    unzip -o "$__RootfsDir/tmp/download/hosttools.zip" -d "$__RootfsDir/tmp/bin"
+
+    DepotBaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg"
+    HpkgBaseUrl="https://eu.hpkg.haiku-os.org/haiku/master/$__HaikuArch/current"
+
+    # Download Haiku packages
+    echo "Downloading Haiku packages"
     read -ra array <<<"$__HaikuPackages"
     for package in "${array[@]}"; do
         echo "Downloading $package..."
         # API documented here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L60
         # The schema here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L598
-        hpkgDownloadUrl="$(wget -qO- --post-data='{"name":"'"$package"'","repositorySourceCode":"haikuports_x86_64","versionType":"LATEST","naturalLanguageCode":"en"}' \
-            --header='Content-Type:application/json' "$BaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')"
-        wget -P "$__RootfsDir/generated/download" "$hpkgDownloadUrl"
+        hpkgDownloadUrl="$(wget -qO- --post-data='{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \
+            --header='Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')"
+        wget -P "$__RootfsDir/tmp/download" "$hpkgDownloadUrl"
+    done
+    for package in haiku haiku_devel; do
+        echo "Downloading $package..."
+        hpkgVersion="$(wget -qO- $HpkgBaseUrl | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
+        wget -P "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
     done
 
-    # Setup the sysroot
-    echo "Setting up sysroot and extracting needed packages"
+    # Set up the sysroot
+    echo "Setting up sysroot and extracting required packages"
     mkdir -p "$__RootfsDir/boot/system"
-    for file in "$__RootfsDir/generated/objects/haiku/x86_64/packaging/packages/"*.hpkg; do
-        "$__RootfsDir/generated/objects/linux/x86_64/release/tools/package/package" extract -C "$__RootfsDir/boot/system" "$file"
-    done
-    for file in "$__RootfsDir/generated/download/"*.hpkg; do
-        "$__RootfsDir/generated/objects/linux/x86_64/release/tools/package/package" extract -C "$__RootfsDir/boot/system" "$file"
+    for file in "$__RootfsDir/tmp/download/"*.hpkg; do
+        echo "Extracting $file..."
+        LD_LIBRARY_PATH="$__RootfsDir/tmp/bin" "$__RootfsDir/tmp/bin/package" extract -C "$__RootfsDir/boot/system" "$file"
     done
 
+    # Download buildtools
+    echo "Downloading Haiku buildtools"
+    wget -O "$__RootfsDir/tmp/download/buildtools.zip" $($__RootfsDir/tmp/script/fetch.sh --buildtools --arch=$__HaikuArch)
+    unzip -o "$__RootfsDir/tmp/download/buildtools.zip" -d "$__RootfsDir"
+
     # Cleaning up temporary files
     echo "Cleaning up temporary files"
+    popd
     rm -rf "$__RootfsDir/tmp"
-    for name in "$__RootfsDir/generated/"*; do
-        if [[ "$name" =~ "cross-tools-" ]]; then
-            : # Keep the cross-compiler
-        else
-            rm -rf "$name"
-        fi
-    done
 elif [[ -n "$__CodeName" ]]; then
 
     if [[ "$__SkipSigCheck" == "0" ]]; then

diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake
@@ -6,6 +6,7 @@ unset(FREEBSD)
 unset(ILLUMOS)
 unset(ANDROID)
 unset(TIZEN)
+unset(HAIKU)
 
 set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH})
 if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version)
@@ -16,6 +17,7 @@ elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc)
   set(ILLUMOS 1)
 elseif(EXISTS ${CROSS_ROOTFS}/boot/system/develop/headers/config/HaikuConfig.h)
   set(CMAKE_SYSTEM_NAME Haiku)
+  set(HAIKU 1)
 else()
   set(CMAKE_SYSTEM_NAME Linux)
   set(LINUX 1)
@@ -67,13 +69,25 @@ elseif(TARGET_ARCH_NAME STREQUAL "armv6")
   endif()
 elseif(TARGET_ARCH_NAME STREQUAL "ppc64le")
   set(CMAKE_SYSTEM_PROCESSOR ppc64le)
-  set(TOOLCHAIN "powerpc64le-linux-gnu")
+  if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/powerpc64le-alpine-linux-musl)
+    set(TOOLCHAIN "powerpc64le-alpine-linux-musl")
+  else()
+    set(TOOLCHAIN "powerpc64le-linux-gnu")
+  endif()
 elseif(TARGET_ARCH_NAME STREQUAL "riscv64")
   set(CMAKE_SYSTEM_PROCESSOR riscv64)
-  set(TOOLCHAIN "riscv64-linux-gnu")
+  if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/riscv64-alpine-linux-musl)
+    set(TOOLCHAIN "riscv64-alpine-linux-musl")
+  else()
+    set(TOOLCHAIN "riscv64-linux-gnu")
+  endif()
 elseif(TARGET_ARCH_NAME STREQUAL "s390x")
   set(CMAKE_SYSTEM_PROCESSOR s390x)
-  set(TOOLCHAIN "s390x-linux-gnu")
+  if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/s390x-alpine-linux-musl)
+    set(TOOLCHAIN "s390x-alpine-linux-musl")
+  else()
+    set(TOOLCHAIN "s390x-linux-gnu")
+  endif()
 elseif(TARGET_ARCH_NAME STREQUAL "x64")
   set(CMAKE_SYSTEM_PROCESSOR x86_64)
   if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/x86_64-alpine-linux-musl)
@@ -88,11 +102,15 @@ elseif(TARGET_ARCH_NAME STREQUAL "x64")
   elseif(ILLUMOS)
     set(TOOLCHAIN "x86_64-illumos")
   elseif(HAIKU)
-    set(TOOLCHAIN "x64_64-unknown-haiku")
+    set(TOOLCHAIN "x86_64-unknown-haiku")
   endif()
 elseif(TARGET_ARCH_NAME STREQUAL "x86")
   set(CMAKE_SYSTEM_PROCESSOR i686)
-  set(TOOLCHAIN "i686-linux-gnu")
+  if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl)
+    set(TOOLCHAIN "i586-alpine-linux-musl")
+  else()
+    set(TOOLCHAIN "i686-linux-gnu")
+  endif()
   if(TIZEN)
     set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu/9.2.0")
   endif()
@@ -198,10 +216,8 @@ elseif(HAIKU)
             return()
         endif()
 
-        set(SEARCH_PATH "${CROSS_ROOTFS}/generated/cross-tools-x86_64/bin")
-
         find_program(EXEC_LOCATION_${exec}
-            PATHS ${SEARCH_PATH}
+            PATHS "${CROSS_ROOTFS}/cross-tools-x86_64/bin"
             NAMES
             "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}"
             "${TOOLSET_PREFIX}${exec}")
@@ -266,8 +282,11 @@ elseif(TARGET_ARCH_NAME MATCHES "^(arm64|x64)$")
     add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
   endif()
 elseif(TARGET_ARCH_NAME STREQUAL "x86")
+  if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl)
+    add_toolchain_linker_flag("--target=${TOOLCHAIN}")
+    add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib/gcc/${TOOLCHAIN}")
+  endif()
   add_toolchain_linker_flag(-m32)
-
   if(TIZEN)
     add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
     add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib")
@@ -277,6 +296,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86")
 elseif(ILLUMOS)
   add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib/amd64")
   add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/amd64/lib")
+elseif(HAIKU)
+  add_toolchain_linker_flag("-lnetwork")
+  add_toolchain_linker_flag("-lroot")
 endif()
 
 # Specify compile options
@@ -307,6 +329,9 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$")
     add_compile_options(-mfloat-abi=softfp)
   endif()
 elseif(TARGET_ARCH_NAME STREQUAL "x86")
+  if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl)
+    add_compile_options(--target=${TOOLCHAIN})
+  endif()
   add_compile_options(-m32)
   add_compile_options(-Wno-error=unused-command-line-argument)
 endif()

diff --git a/eng/common/cross/x64/sources.list.bionic b/eng/common/cross/x64/sources.list.bionic
@@ -0,0 +1,11 @@
+deb http://archive.ubuntu.com/ubuntu/ bionic main restricted universe
+deb-src http://archive.ubuntu.com/ubuntu/ bionic main restricted universe
+
+deb http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe
+deb-src http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe
+
+deb http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted
+deb-src http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted
+
+deb http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse
+deb-src http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse
diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh
@@ -64,7 +64,7 @@ if [ -z "$CLR_CC" ]; then
     if [ -z "$majorVersion" ]; then
         # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero.
         if [ "$compiler" = "clang" ]; then versions="16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5"
-        elif [ "$compiler" = "gcc" ]; then versions="12 11 10 9 8 7 6 5 4.9"; fi
+        elif [ "$compiler" = "gcc" ]; then versions="13 12 11 10 9 8 7 6 5 4.9"; fi
 
         for version in $versions; do
             _major="${version%%.*}"

diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml
@@ -158,7 +158,7 @@ jobs:
   - template: /eng/common/templates/steps/component-governance.yml
     parameters:
       ${{ if eq(parameters.disableComponentGovernance, '') }}:
-        ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/internal/release'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}:
+        ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}:
           disableComponentGovernance: false
         ${{ else }}:
           disableComponentGovernance: true

diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1
@@ -287,6 +287,25 @@ function InstallDotNet([string] $dotnetRoot,
   [string] $runtimeSourceFeedKey = '',
   [switch] $noPath) {
 
+  $dotnetVersionLabel = "'sdk v$version'"
+
+  if ($runtime -ne '' -and $runtime -ne 'sdk') {
+    $runtimePath = $dotnetRoot
+    $runtimePath = $runtimePath + "\shared"
+    if ($runtime -eq "dotnet") { $runtimePath = $runtimePath + "\Microsoft.NETCore.App" }
+    if ($runtime -eq "aspnetcore") { $runtimePath = $runtimePath + "\Microsoft.AspNetCore.App" }
+    if ($runtime -eq "windowsdesktop") { $runtimePath = $runtimePath + "\Microsoft.WindowsDesktop.App" }
+    $runtimePath = $runtimePath + "\" + $version
+
+    $dotnetVersionLabel = "runtime toolset '$runtime/$architecture v$version'"
+
+    if (Test-Path $runtimePath) {
+      Write-Host "  Runtime toolset '$runtime/$architecture v$version' already installed."
+      $installSuccess = $true
+      Exit
+    }
+  }
+
   $installScript = GetDotNetInstallScript $dotnetRoot
   $installParameters = @{
     Version = $version
@@ -323,18 +342,18 @@ function InstallDotNet([string] $dotnetRoot,
     } else {
       $location = "public location";
     }
-    Write-Host "Attempting to install dotnet from $location."
+    Write-Host "  Attempting to install $dotnetVersionLabel from $location."
     try {
       & $installScript @variation
       $installSuccess = $true
       break
     }
     catch {
-      Write-Host "Failed to install dotnet from $location."
+      Write-Host "  Failed to install $dotnetVersionLabel from $location."
     }
   }
   if (-not $installSuccess) {
-    Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from any of the specified locations."
+    Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install $dotnetVersionLabel from any of the specified locations."
     ExitWithExitCode 1
   }
 }
@@ -399,7 +418,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
   # Locate Visual Studio installation or download x-copy msbuild.
   $vsInfo = LocateVisualStudio $vsRequirements
   if ($vsInfo -ne $null) {
-    $vsInstallDir = $vsInfo.installationPath
+    # Ensure vsInstallDir has a trailing slash
+    $vsInstallDir = Join-Path $vsInfo.installationPath "\"
     $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0]
 
     InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion