Move two tests from GTest to TFLite testing to run on CortexM (#263)

* Move two CortexM tests from GTest to TFLite testing

GTest is replaced with the TFLiteMicro testing framework, making it
possible to run these tests also on the CortexM. The two tests for
which this is done don't used advanced features (parameter sweep)
and therefore work equally well without GTest.

* Add two existing tests for the CorexM target

Added two existing tests for BGEMV and the quantized multiplier to
the build_lcem script to run on CortexM. The tests are slightly
modified since std::cerr and the C++ random number generator are
not available.

* Fix Bazel linter issues

* Fix compilation error with std::round for RISC-V

* Fix issues for RISC-V target

* Improve QEMU check in build script

* Fix error reporting and micro compiler flags

* Fix issue with tflite error reporting

Co-authored-by: Cedric Nugteren <cedric@plumerai.com>

larq_compute_engine/core/tests/BUILD

@@ -1,3 +1,9 @@
+load("@rules_cc//cc:defs.bzl", "cc_test")
+load(
+    "@org_tensorflow//tensorflow/lite/micro:build_def.bzl",
+    "micro_copts",
+)
+
 licenses(["notice"])  # Apache 2.0
 
 package(default_visibility = ["//visibility:public"])
@@ -6,11 +12,13 @@ cc_test(
     name = "bgemv_tests",
     size = "small",
     srcs = ["bgemv_tests.cc"],
-    copts = ["-Iext/gtest/include"],
+    copts = micro_copts(),
     linkopts = ["-lm -lrt"],
     deps = [
         "//larq_compute_engine/core/cortexm:bgemv",
-        "@com_google_googletest//:gtest_main",
+        "//larq_compute_engine/micro/platform",
+        "@org_tensorflow//tensorflow/lite/micro:micro_error_reporter",
+        "@org_tensorflow//tensorflow/lite/micro/testing:micro_test",
     ],
 )
 
@@ -34,15 +42,17 @@ cc_test(
     name = "quantized_multiplier_tests",
     size = "small",
     srcs = ["quantized_multiplier_tests.cc"],
-    copts = ["-Iext/gtest/include"],
+    copts = micro_copts(),
     linkopts = [
         "-lm",
         "-lrt",
     ],
     deps = [
         "//larq_compute_engine/core/int8:quantized_multiplier",
-        "@com_google_googletest//:gtest_main",
+        "//larq_compute_engine/micro/platform",
         "@org_tensorflow//tensorflow/lite/kernels/internal:quantization_util",
+        "@org_tensorflow//tensorflow/lite/micro:micro_error_reporter",
+        "@org_tensorflow//tensorflow/lite/micro/testing:micro_test",
     ],
 )
 

larq_compute_engine/core/tests/bgemv_tests.cc

@@ -1,43 +1,61 @@
-#include <gmock/gmock.h>
-
 #include <array>
 #include <random>
 
 #include "larq_compute_engine/core/cortexm/bgemv.h"
+#include "larq_compute_engine/micro/platform/platform.h"
+#include "tensorflow/lite/micro/micro_error_reporter.h"
+#include "tensorflow/lite/micro/testing/micro_test.h"
 
 using compute_engine::core::bgemv;
 using compute_engine::core::popcount_triple;
 using compute_engine::core::bconv2d::OutputTransform;
 
-TEST(BGemvTests, TriplePopcount) {
+uint32_t naive_popcount(uint32_t bits) {
+  // Slow and naive reference implementation of popcount
+  uint32_t result = 0;
+  for (auto i = 0; i < 32; ++i) {
+    result += (bits & 0x1);
+    bits >>= 1;
+  }
+  return result;
+}
+
+TF_LITE_MICRO_TESTS_BEGIN
+
+// The above macro starts a main() function, so we can put this here
+InitializePlatform();
+
+TF_LITE_MICRO_TEST(BGemvTestsTriplePopcount) {
   // Test overflows using the minimum and maximum possible values
   int32_t acc0 = popcount_triple(0, 0, 0);
   int32_t acc1 = popcount_triple(0xffffffff, 0xffffffff, 0xffffffff);
   int32_t acc2 = popcount_triple(0xffffffff, 0xffffffff, 0x00000000);
   int32_t acc3 = popcount_triple(0xffffffff, 0x00000000, 0xffffffff);
   int32_t acc4 = popcount_triple(0x00000000, 0xffffffff, 0xffffffff);
 
-  EXPECT_EQ(acc0, 0);
-  EXPECT_EQ(acc1, 3 * 32);
-  EXPECT_EQ(acc2, 2 * 32);
-  EXPECT_EQ(acc3, 2 * 32);
-  EXPECT_EQ(acc4, 2 * 32);
+  TF_LITE_MICRO_EXPECT_EQ(acc0, 0);
+  TF_LITE_MICRO_EXPECT_EQ(acc1, 3 * 32);
+  TF_LITE_MICRO_EXPECT_EQ(acc2, 2 * 32);
+  TF_LITE_MICRO_EXPECT_EQ(acc3, 2 * 32);
+  TF_LITE_MICRO_EXPECT_EQ(acc4, 2 * 32);
 
   // Test some random values
+#if !defined(__arm__) and !defined(__riscv)
   std::random_device rd;
-  std::mt19937 gen(rd());
+  std::mt19937 rand(rd());
+#endif  // __arm__
   for (int i = 0; i < 100; ++i) {
-    uint32_t x1 = gen();
-    uint32_t x2 = gen();
-    uint32_t x3 = gen();
-    int32_t expected = __builtin_popcountl(x1) + __builtin_popcountl(x2) +
-                       __builtin_popcountl(x3);
+    uint32_t x1 = rand();
+    uint32_t x2 = rand();
+    uint32_t x3 = rand();
+    int32_t expected =
+        naive_popcount(x1) + naive_popcount(x2) + naive_popcount(x3);
     int32_t actual = popcount_triple(x1, x2, x3);
-    EXPECT_EQ(actual, expected);
+    TF_LITE_MICRO_EXPECT_EQ(actual, expected);
   }
 }
 
-TEST(BGemvTests, Int8Output) {
+TF_LITE_MICRO_TEST(BGemvTestsInt8Output) {
   std::array<float, 3> mult = {1.0f, 1.0f, 1.0f};
   std::array<float, 3> bias = {0.0f, 0.0f, 0.0f};
   OutputTransform<std::int8_t> output_transform;
@@ -54,10 +72,12 @@ TEST(BGemvTests, Int8Output) {
   bgemv<std::int8_t>(lhs.data(), rhs.data(), 3, 2, out.data(), output_transform,
                      nullptr);
 
-  EXPECT_THAT(out, ::testing::ElementsAreArray(expected));
+  for (auto i = 0; i < 3; ++i) {
+    TF_LITE_MICRO_EXPECT_EQ(out[i], expected[i]);
+  }
 }
 
-TEST(BGemvTests, BitpackedOutput) {
+TF_LITE_MICRO_TEST(BGemvTestsBitpackedOutput) {
   std::array<std::int32_t, 3> thresholds = {1, 2, 3};
   OutputTransform<std::int32_t> output_transform;
   output_transform.thresholds = thresholds.data();
@@ -77,5 +97,7 @@ TEST(BGemvTests, BitpackedOutput) {
   bgemv<std::int32_t>(lhs.data(), rhs.data(), 3, 2, out.data(),
                       output_transform, nullptr);
 
-  EXPECT_THAT(out, ::testing::ElementsAreArray(expected));
+  TF_LITE_MICRO_EXPECT_EQ(out[0], expected[0]);
 }
+
+TF_LITE_MICRO_TESTS_END

larq_compute_engine/core/tests/quantized_multiplier_tests.cc

@@ -1,11 +1,17 @@
-#include <gmock/gmock.h>
-
 #include <cmath>
 
 #include "larq_compute_engine/core/int8/quantized_multiplier.h"
+#include "larq_compute_engine/micro/platform/platform.h"
 #include "tensorflow/lite/kernels/internal/quantization_util.h"
+#include "tensorflow/lite/micro/micro_error_reporter.h"
+#include "tensorflow/lite/micro/testing/micro_test.h"
+
+TF_LITE_MICRO_TESTS_BEGIN
+
+// The above macro starts a main() function, so we can put this here
+InitializePlatform();
 
-TEST(QuantizedMultiplier, ErrorCount) {
+TF_LITE_MICRO_TEST(QuantizedMultiplierErrorCount) {
   // We try many multipliers and count the number of errors
   // This has to stay below a threshold.
   // We can't test for an exact match with tflite because the tflite version
@@ -38,7 +44,7 @@ TEST(QuantizedMultiplier, ErrorCount) {
 
     for (int i = -128; i <= 127; ++i) {
       // Try to find `x` such that `multiplier * x` is close to `i + 0.5`
-      int x_base = std::round((0.5 + i) / multiplier);
+      int x_base = std::floor(0.5 + (0.5 + i) / multiplier);
       // Try a few x near this number
       for (int x = x_base - 8; x <= x_base + 8; ++x) {
         int32_t result = compute_engine::core::MultiplyByQuantizedMultiplier(
@@ -48,8 +54,7 @@ TEST(QuantizedMultiplier, ErrorCount) {
         double diff = ((double)result) - result_unrounded;
         double abs_diff = std::fabs(diff);
 
-        EXPECT_LE(abs_diff, 0.51)
-            << "Mismatch of " << diff << " for " << multiplier << " * " << x;
+        TF_LITE_MICRO_EXPECT_LE(abs_diff, 0.51);
 
         total_tests++;
 
@@ -73,19 +78,22 @@ TEST(QuantizedMultiplier, ErrorCount) {
   }
 
   // Expect less than 0.1% of the tests to be off by more than 0.5001
-  EXPECT_LE(error_count_05001, total_tests / 1000);
+  TF_LITE_MICRO_EXPECT_LE(error_count_05001, total_tests / 1000);
 
   // Display error statistics in case of any errors
+
   if (error_count_0500001 > 0) {
-    std::cerr << error_count_051 << "/" << total_tests
-              << " errors larger than 0.51\n";
-    std::cerr << error_count_0501 << "/" << total_tests
-              << " errors larger than 0.501\n";
-    std::cerr << error_count_05001 << "/" << total_tests
-              << " errors larger than 0.5001\n";
-    std::cerr << error_count_050001 << "/" << total_tests
-              << " errors larger than 0.50001\n";
-    std::cerr << error_count_0500001 << "/" << total_tests
-              << " errors larger than 0.500001\n";
+    micro_test::reporter->Report("%d/%d errors larger than 0.51",
+                                 error_count_051, total_tests);
+    micro_test::reporter->Report("%d/%d errors larger than 0.501",
+                                 error_count_0501, total_tests);
+    micro_test::reporter->Report("%d/%d errors larger than 0.5001",
+                                 error_count_05001, total_tests);
+    micro_test::reporter->Report("%d/%d errors larger than 0.50001",
+                                 error_count_050001, total_tests);
+    micro_test::reporter->Report("%d/%d errors larger than 0.500001",
+                                 error_count_0500001, total_tests);
   }
 }
+
+TF_LITE_MICRO_TESTS_END

larq_compute_engine/micro/build_make/build_lcem.sh

@@ -163,6 +163,7 @@ mkdir -p ${TF_DIR}/larq_compute_engine/tflite/kernels
 mkdir -p ${TF_DIR}/larq_compute_engine/core
 mkdir -p ${TF_DIR}/larq_compute_engine/core/bconv2d
 mkdir -p ${TF_DIR}/larq_compute_engine/core/bitpacking
+mkdir -p ${TF_DIR}/larq_compute_engine/core/tests
 
 cp ${ROOT_DIR}/larq_compute_engine/micro/build_make/fix_arduino_includes.py ${TF_DIR}/larq_compute_engine/
 
@@ -179,7 +180,9 @@ COREFILES="bconv2d/reference.h \
     micro_profiling.h \
     profiler.cc \
     profiler.h \
-    types.h"
+    types.h \
+    tests/bgemv_tests.cc \
+    tests/quantized_multiplier_tests.cc"
 
 COREDIRS="cortexm int8 riscv"
 
@@ -234,7 +237,7 @@ trap cleanup EXIT
 NUM_HYPERTHREADS=$(grep -c ^processor /proc/cpuinfo 2>/dev/null || sysctl -n hw.ncpu)
 
 
-TEST_PROJECTS="bitpack single_bconv single_conv single_quantize_dequantize single_bmaxpool single_avgpool toy_model"
+TEST_PROJECTS="bgemv quantized_multiplier bitpack single_bconv single_conv single_quantize_dequantize single_bmaxpool single_avgpool toy_model"
 ALL_PROJECTS="hello_lce lce_benchmark unit_benchmarks"
 for TEST_PROJECT in ${TEST_PROJECTS}; do
     ALL_PROJECTS="${ALL_PROJECTS} lce_test_${TEST_PROJECT}"
@@ -542,7 +545,7 @@ if [ "$unittest_riscv" == "1" ]; then
       -j ${NUM_HYPERTHREADS} \
       ${RISCV_TEST_PROJECTS}
 
-    QEMU_PATH=$(which qemu-system-riscv32)
+    QEMU_PATH=$(which qemu-system-riscv32 || true)
     if [ -x "$QEMU_PATH" ]; then
         BIN_DIR=${TF_GEN_DIR}/${HOST_OS}_riscv32/bin
         for PROJECT_NAME in ${RISCV_TEST_PROJECTS}; do

larq_compute_engine/micro/platform/BUILD

@@ -0,0 +1,9 @@
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//visibility:public"])
+
+cc_library(
+    name = "platform",
+    srcs = ["platform.cc"],
+    hdrs = ["platform.h"],
+)

larq_compute_engine/micro/tests/unittests/Makefile.inc

@@ -1,7 +1,15 @@
+TEST_BGEMV_SRCS := larq_compute_engine/core/tests/bgemv_tests.cc
+
+TEST_QUANTIZED_MULTIPLIER_SRCS := larq_compute_engine/core/tests/quantized_multiplier_tests.cc
+
 TEST_BITPACK_SRCS := larq_compute_engine/micro/tests/unittests/bitpack_test.cc
 
 TEST_PULP_SRCS := larq_compute_engine/micro/tests/unittests/pulp_test.cc
 
+$(eval $(call microlite_test,lce_test_bgemv,$(TEST_BGEMV_SRCS),))
+
+$(eval $(call microlite_test,lce_test_quantized_multiplier,$(TEST_QUANTIZED_MULTIPLIER_SRCS),))
+
 $(eval $(call microlite_test,lce_test_bitpack,$(TEST_BITPACK_SRCS),))
 
 $(eval $(call microlite_test,lce_test_pulp,$(TEST_PULP_SRCS),))