Merge pull request #6833 from hoisie/add-icu-submodule

Add libicu as a git submodule

.github/workflows/check_aggregateDocs.yml

@@ -17,12 +17,12 @@ jobs:
           fetch-depth: 0
           submodules: recursive
 
-      - name: Set up JDK 
+      - name: Set up JDK
         uses: actions/setup-java@v2
         with:
           distribution: 'zulu' # zulu suports complete JDK list
           java-version: 14
           cache: 'gradle'
 
       - name: Run aggregateDocs
-        run: ./gradlew clean aggregateDocs
+        run: SKIP_NATIVERUNTIME_BUILD=true ./gradlew clean aggregateDocs # building the native runtime is not required for checking javadoc

.github/workflows/tests.yml

@@ -7,6 +7,10 @@ on:
   pull_request:
     branches: [ master ]
 
+env:
+  CXX: clang++
+  CC: clang
+
 jobs:
   build:
     runs-on: ubuntu-18.04
@@ -20,7 +24,6 @@ jobs:
           path: |
             ~/.gradle
             ~/.m2
-            ~/sqlite
           key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
           restore-keys: |
             ${{ runner.os }}-gradle-
@@ -30,8 +33,23 @@ jobs:
         with:
           java-version: 11.0.8
 
+      - name: Cache ICU build output
+        id: cache-icu
+        uses: actions/cache@v2
+        with:
+          path: ~/icu-bin
+          key: ${{ runner.os }}-icu-${{ hashFiles('nativeruntime/external/icu/**') }}
+
+      - name: Build ICU
+        if: steps.cache-icu.outputs.cache-hit != 'true'
+        run: |
+          cd nativeruntime/external/icu/icu4c/source
+          CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./runConfigureICU Linux --enable-static --prefix=$HOME/icu-bin
+          make -j4
+          make install
+
       - name: Build
-        run: SKIP_ERRORPRONE=true SKIP_JAVADOC=true ./gradlew clean assemble testClasses --parallel --stacktrace
+        run: ICU_ROOT_DIR=$HOME/icu-bin SKIP_ICU_BUILD=true SKIP_ERRORPRONE=true SKIP_JAVADOC=true ./gradlew clean assemble testClasses --parallel --stacktrace
 
   unit-tests:
     runs-on: ubuntu-18.04
@@ -60,9 +78,16 @@ jobs:
         with:
           java-version: 11.0.8
 
+      - name: Cache ICU build output
+        id: cache-icu
+        uses: actions/cache@v2
+        with:
+          path: ~/icu-bin
+          key: ${{ runner.os }}-icu-${{ hashFiles('nativeruntime/external/icu/**') }}
+
       - name: Run unit tests
         run: |
-          SKIP_ERRORPRONE=true SKIP_JAVADOC=true ./gradlew test --info --stacktrace --continue \
+          ICU_ROOT_DIR=$HOME/icu-bin SKIP_ICU_BUILD=true SKIP_ERRORPRONE=true SKIP_JAVADOC=true ./gradlew test --info --stacktrace --continue \
           --parallel \
           -Drobolectric.enabledSdks=${{ matrix.api-versions }} \
           -Drobolectric.alwaysIncludeVariantMarkersInTestName=true \

.gitmodules

@@ -2,3 +2,9 @@
 	path = nativeruntime/external/sqlite
 	url = https://android.googlesource.com/platform/external/sqlite
 	branch = android11-release
+
+[submodule "nativeruntime/external/icu"]
+	path = nativeruntime/external/icu
+	url = https://github.com/unicode-org/icu
+	branch = release-69-1
+	shallow = false

nativeruntime/build.gradle

@@ -24,34 +24,92 @@ static def arch() {
   return arch
 }
 
-task cmakeNativeRuntime(type:Exec) {
-  workingDir "$buildDir/cpp"
-  commandLine 'cmake', "$projectDir/cpp/"
-  doFirst {
+task cmakeNativeRuntime {
+  doLast {
     mkdir "$buildDir/cpp"
+    exec {
+      // Building the nativeruntime does not work with GCC due to libstddc++ linker errors.
+      // TODO: figure out which linker args are needed to build with GCC.
+      environment "CC", "clang"
+      environment "CXX", "clang++"
+      workingDir "$buildDir/cpp"
+      commandLine 'cmake', "-B", ".", "-S","$projectDir/cpp/"
+    }
   }
 }
 
-task makeNativeRuntime(type:Exec) {
+task configureICU {
+  onlyIf { !System.getenv('SKIP_ICU_BUILD') }
+  doLast {
+    def os = osName()
+    if (!file("$projectDir/external/icu/icu4c/source").exists()) {
+      throw new GradleException("ICU submodule not detected. Please run `git submodule update --init`")
+    }
+    exec {
+      workingDir "$projectDir/external/icu/icu4c/source"
+      if (os.contains("linux")) {
+        environment "CFLAGS", "-fPIC"
+        environment "CXXFLAGS", "-fPIC"
+        commandLine './runConfigureICU', 'Linux', '--enable-static', '--disable-shared'
+      } else if (os.contains("mac")) {
+        environment "CFLAGS", "-arch x86_64 -arch arm64"
+        environment "CXXFLAGS", "-arch x86_64 -arch arm64"
+        commandLine './runConfigureICU', 'MacOSX', '--enable-static', '--disable-shared'
+      } else {
+        println("Skipping the nativeruntime build for OS '${System.getProperty("os.name")}'")
+      }
+    }
+  }
+}
+
+task buildICU {
+  onlyIf { !System.getenv('SKIP_ICU_BUILD') }
+  dependsOn configureICU
+  doLast {
+    exec {
+      def os = osName()
+      if (os.contains("linux") || os.contains("mac")) {
+        workingDir "$projectDir/external/icu/icu4c/source"
+        commandLine 'make', '-j4'
+      }
+    }
+  }
+}
+
+task makeNativeRuntime {
+  dependsOn buildICU
   dependsOn cmakeNativeRuntime
-  workingDir "$buildDir/cpp"
-  commandLine 'make'
+  doLast {
+    exec {
+      workingDir "$buildDir/cpp"
+      commandLine 'make'
+    }
+  }
 }
 
-task copyNativeRuntime(type: Copy) {
+task copyNativeRuntime {
   dependsOn makeNativeRuntime
-  from ("$buildDir/cpp") {
-    include '*libnativeruntime.*'
-  }
-  rename { String fileName ->
-    fileName.replace("libnativeruntime", "librobolectric-nativeruntime")
+  doLast {
+    copy {
+      from ("$buildDir/cpp") {
+        include '*libnativeruntime.*'
+      }
+      rename { String fileName ->
+        fileName.replace("libnativeruntime", "librobolectric-nativeruntime")
+      }
+      def os = osName()
+      def arch = arch()
+      if (os.contains("mac")) {
+        arch = "universal"
+      }
+      into project.file("$buildDir/resources/main/native/${os}/${arch}/")
+    }
   }
-  into project.file("$buildDir/resources/main/native/${osName()}/${arch()}/")
 }
 
 jar {
   def os = osName()
-  if (os.contains("linux") || os.contains("mac")) {
+  if (!System.getenv('SKIP_NATIVERUNTIME_BUILD') && (os.contains("linux") || os.contains("mac"))) {
     dependsOn copyNativeRuntime
   } else {
     println("Skipping the nativeruntime build for OS '${System.getProperty("os.name")}'")

nativeruntime/cpp/CMakeLists.txt

@@ -1,49 +1,56 @@
 cmake_minimum_required(VERSION 3.10)
 
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+set(CMAKE_OSX_ARCHITECTURES "arm64;x86_64") # Universal libraries for Mac OS
+
 project(nativeruntime)
 
 # Some libutils headers require C++17
 set (CMAKE_CXX_STANDARD 17)
 
 find_package(JNI REQUIRED)
 
-# On Mac OS, search Homebrew for the icu4c distribution. The system version
-# does not include headers and static libraries.
-if (CMAKE_HOST_SYSTEM_NAME MATCHES "Darwin")
-  execute_process(
-    COMMAND brew --prefix icu4c
-    RESULT_VARIABLE BREW_ICU4C
-    OUTPUT_VARIABLE BREW_ICU4C_PREFIX
-    OUTPUT_STRIP_TRAILING_WHITESPACE
-  )
+set(ANDROID_SQLITE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../external/sqlite")
 
-  if (NOT BREW_ICU4C EQUAL 0)
-    message( FATAL_ERROR "'brew --prefix icu4c' failed. Ensure homebrew is installed and run 'brew install icu4c'.")
+if(NOT EXISTS "${ANDROID_SQLITE_DIR}/dist/sqlite3.c")
+  message(FATAL_ERROR "SQLite submodule missing. Please run `git submodule update --init`.")
+endif()
+
+if(DEFINED ENV{ICU_ROOT_DIR})
+
+  if(NOT EXISTS "$ENV{ICU_ROOT_DIR}/lib/libicui18n.a")
+    message(FATAL_ERROR "ICU_ROOT_DIR does not contain 'lib/libicui18n.a'")
   endif()
 
-  if (BREW_ICU4C EQUAL 0 AND EXISTS "${BREW_ICU4C_PREFIX}")
-    message(STATUS "Found icu4c installed by Homebrew at ${BREW_ICU4C_PREFIX}")
-    list(APPEND CMAKE_PREFIX_PATH ${BREW_ICU4C_PREFIX})
-    find_library(BREW_ICUUC_LIBRARY libicuuc.a)
-    find_library(BREW_ICUI18N_LIBRARY libicui18n.a)
-    find_library(BREW_ICUDATA_LIBRARY libicudata.a)
-    include_directories(${BREW_ICU4C_PREFIX}/include)
+  message(NOTICE "Using $ENV{ICU_ROOT_DIR} as the ICU root dir")
+  list(APPEND CMAKE_PREFIX_PATH "$ENV{ICU_ROOT_DIR}")
+  find_library(STATIC_ICUI18N_LIBRARY libicui18n.a)
+  find_library(STATIC_ICUUC_LIBRARY libicuuc.a)
+  find_library(STATIC_ICUDATA_LIBRARY libicudata.a)
+  include_directories($ENV{ICU_ROOT_DIR}/include)
+else()
+  set(ICU_SUBMODULE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../external/icu")
+
+  if(NOT EXISTS "${ICU_SUBMODULE_DIR}/icu4c/source/i18n/ucol.cpp")
+    message(FATAL_ERROR "ICU submodule missing. Please run `git submodule update --init`.")
   endif()
 
-  if (NOT BREW_ICUUC_LIBRARY)
-    message(FATAL_ERROR "libicuuc.a not found. Please run 'brew install icu4c'")
+  message(NOTICE "Using ${ICU_SUBMODULE_DIR} as the ICU root dir")
+
+  if(NOT EXISTS "${ICU_SUBMODULE_DIR}/icu4c/source/lib/libicui18n.a")
+    message(FATAL_ERROR "ICU not built. Please run `./gradlew :nativeruntime:buildICU`.")
   endif()
-endif()
 
-set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+  list(APPEND CMAKE_PREFIX_PATH "${ICU_SUBMODULE_DIR}/icu4c/source/")
+  find_library(STATIC_ICUI18N_LIBRARY libicui18n.a)
+  find_library(STATIC_ICUUC_LIBRARY libicuuc.a)
+  find_library(STATIC_ICUDATA_LIBRARY libicudata.a)
+  include_directories(${ICU_SUBMODULE_DIR}/icu4c/source/i18n)
+  include_directories(${ICU_SUBMODULE_DIR}/icu4c/source/common)
+endif()
 
 # Build flags derived from
 # https://cs.android.com/android/platform/superproject/+/android-11.0.0_r1:external/sqlite/dist/Android.bp
-set(ANDROID_SQLITE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../external/sqlite")
-
-if(NOT EXISTS "${ANDROID_SQLITE_DIR}/dist/sqlite3.c")
-  message(FATAL_ERROR "SQLite submodule missing. Please run `git submodule update --init`.")
-endif()
 
 set(SQLITE_COMPILE_OPTIONS
   -DHAVE_USLEEP=1
@@ -84,13 +91,11 @@ add_library(androidsqlite STATIC
 
 target_compile_options(androidsqlite PRIVATE ${SQLITE_COMPILE_OPTIONS})
 
-if (CMAKE_HOST_SYSTEM_NAME MATCHES "Darwin")
-  target_link_libraries(androidsqlite
-    ${BREW_ICUUC_LIBRARY}
-    ${BREW_ICUI18N_LIBRARY}
-    ${BREW_ICUDATA_LIBRARY}
-  )
-endif()
+target_link_libraries(androidsqlite
+  ${STATIC_ICUI18N_LIBRARY}
+  ${STATIC_ICUUC_LIBRARY}
+  ${STATIC_ICUDATA_LIBRARY}
+)
 
 include_directories(${JNI_INCLUDE_DIRS})
 
@@ -132,5 +137,8 @@ if (CMAKE_HOST_SYSTEM_NAME MATCHES "Linux")
   target_link_libraries(nativeruntime
     -static-libgcc
     -static-libstdc++
+    -ldl
+    -lpthread
+    -Wl,--no-undefined # print an error if there are any undefined symbols
   )
 endif()

nativeruntime/src/main/java/org/robolectric/nativeruntime/NativeRuntimeLoader.java

@@ -50,8 +50,13 @@ private static boolean isSupported() {
   }
 
   private static String nativeLibraryPath() {
+    String os = osName();
+    String arch = arch();
+    if (os.equals("mac")) {
+      arch = "universal"; // use the universal library
+    }
     return String.format(
-        "native/%s/%s/%s", osName(), arch(), System.mapLibraryName("robolectric-nativeruntime"));
+        "native/%s/%s/%s", os, arch, System.mapLibraryName("robolectric-nativeruntime"));
   }
 
   private static String osName() {