[WIP] Make the JIT'd code completely portable.

CMakeLists.txt

@@ -29,6 +29,7 @@ add_subdirectory("vm")
         STEP_TARGETS build)
 
 if(UBPF_ENABLE_TESTS)
+  add_subdirectory("custom_tests")
   add_subdirectory("ubpf_plugin")
   if (NOT UBPF_SKIP_EXTERNAL)
   endif()

cmake/settings.cmake

@@ -17,7 +17,8 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
       -Wall
       -Werror
       -Iinc
-      -O2
+      -O0
+      -g
       -Wunused-parameter
       -fPIC
     )

custom_tests/CMakeLists.txt

@@ -0,0 +1,111 @@
+# Copyright (c) Microsoft Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+set(CMAKE_CXX_STANDARD 20)
+
+file(GLOB test_descr_files ${CMAKE_SOURCE_DIR}/custom_tests/descrs/*.md)
+
+add_library(ubpf_custom_test_support srcs/ubpf_custom_test_support.cc)
+target_link_libraries(
+    ubpf_custom_test_support
+    ubpf
+)
+
+target_include_directories(ubpf_custom_test_support PUBLIC ".srcs/")
+target_include_directories(ubpf_custom_test_support PRIVATE
+    "${CMAKE_SOURCE_DIR}/vm"
+    "${CMAKE_BINARY_DIR}/vm"
+    "${CMAKE_SOURCE_DIR}/vm/inc"
+    "${CMAKE_BINARY_DIR}/vm/inc"
+)
+
+foreach(test_file ${test_descr_files})
+    get_filename_component(test_name ${test_file} NAME_WE)
+    set(test_source_path "${CMAKE_SOURCE_DIR}/custom_tests/srcs/${test_name}.cc")
+    message(WARNING "test_name: ${test_name}")
+    message(WARNING "test_source_path: ${test_source_path}")
+
+    add_executable(
+        ${test_name}
+        ${test_source_path}
+    )
+    target_include_directories(${test_name} PRIVATE
+        "${CMAKE_SOURCE_DIR}/vm"
+        "${CMAKE_BINARY_DIR}/vm"
+        "${CMAKE_SOURCE_DIR}/vm/inc"
+        "${CMAKE_BINARY_DIR}/vm/inc"
+    )
+    target_link_libraries(
+        ${test_name}
+        ubpf
+        ubpf_custom_test_support
+        ubpf_settings
+    )
+    set(potential_input_file ${CMAKE_SOURCE_DIR}/custom_tests/data/${test_name}.input)
+    if (EXISTS ${potential_input_file})
+        add_test(
+            NAME ${test_name}-Custom
+            COMMAND sh -c "cat ${potential_input_file} | $<TARGET_FILE:${test_name}>"
+        )
+    else()
+        add_test(
+            NAME ${test_name}-Custom
+            COMMAND $<TARGET_FILE:${test_name}>
+        )
+    endif()
+    message(WARNING "test_source: ${test_source}")
+endforeach()
+
+#if(NOT BPF_CONFORMANCE_RUNNER)
+#    set(BPF_CONFORMANCE_RUNNER ${CMAKE_BINARY_DIR}/external/bpf_conformance/bin/bpf_conformance_runner)
+#else()
+#    message(STATUS "Using custom bpf_conformance_runner: ${BPF_CONFORMANCE_RUNNER}")
+#endif()
+
+#if(CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64 AND (NOT CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL aarch64))
+#    set(PLUGIN_JIT --plugin_path ${CMAKE_BINARY_DIR}/bin/run-jit.sh)
+#    set(PLUGIN_INTERPRET --plugin_path ${CMAKE_BINARY_DIR}/bin/run-interpret.sh)
+#else()
+#    if(PLATFORM_WINDOWS)
+#        set(PLATFORM_EXECUTABLE_EXTENSION ".exe")
+#    else()
+#        set(PLATFORM_EXECUTABLE_EXTENSION "")
+#    endif()
+#    set(PLUGIN_JIT --plugin_path ${CMAKE_BINARY_DIR}/bin/ubpf_plugin${PLATFORM_EXECUTABLE_EXTENSION} --plugin_options --jit)
+#    set(PLUGIN_INTERPRET --plugin_path ${CMAKE_BINARY_DIR}/bin/ubpf_plugin${PLATFORM_EXECUTABLE_EXTENSION} --plugin_options --interpret)
+#endif()
+
+## Add all names of tests that are expected to fail to the TESTS_EXPECTED_TO_FAIL list
+#list(APPEND TESTS_EXPECTED_TO_FAIL "duplicate_label")
+## TODO: remove this once we have a proper implementation of interlocked operations
+## and support for calling local functions.
+#list(APPEND TESTS_EXPECTED_TO_FAIL "lock")
+
+#foreach(file ${files})
+#    unset(EXPECT_FAILURE)
+#    foreach(to_fail ${TESTS_EXPECTED_TO_FAIL})
+#        if(NOT EXPECT_FAILURE)
+#            string(REGEX MATCH "${to_fail}" EXPECT_FAILURE "${file}")
+#            if(EXPECT_FAILURE)
+#                message(STATUS "Expecting ${file} test to fail.")
+#            endif()
+#        endif()
+#    endforeach()
+#    add_test(
+#        NAME ${file}-JIT
+#        COMMAND ${BPF_CONFORMANCE_RUNNER} --test_file_path ${file} ${PLUGIN_JIT}
+#    )
+
+#    if(EXPECT_FAILURE)
+#        set_tests_properties(${file}-JIT PROPERTIES WILL_FAIL TRUE)
+#    endif()
+
+#    add_test(
+#        NAME ${file}-Interpreter
+#        COMMAND ${BPF_CONFORMANCE_RUNNER} --test_file_path ${file} ${PLUGIN_INTERPRET}
+#    )
+
+#    if(EXPECT_FAILURE)
+#        set_tests_properties(${file}-Interpreter PROPERTIES WILL_FAIL TRUE)
+#    endif()
+#endforeach()

custom_tests/README.md

@@ -0,0 +1,35 @@
+## Writing a uBPF Custom Tests
+
+Custom tests are enabled by creating two (2) or three (3) different files in the `custom_tests` directory.
+
+### Files Of a uBPF Custom Test
+
+#### Description Files
+
+The first file to create is the Description File. The Description File is a file with a `.md` extension that resides in the `descrs` directory. The purpose of this file is to identify the name of the test (everything before the `.md` extension) and provide a place to document the purpose of the test.
+
+#### Source Files
+
+The second file to create is the Source File. The Source file should reside in the `srcs` directory and have a name that matches its Description File (with the `.cc` extension rather than the `.md` extension).
+
+#### Input Files
+
+The final file is optional. The Input File resides in the `data` directory and should have the same name as the other two (2) files but with an `.input` extension rather than  `.cc` or `.md` for the Source and Description File respectively. If present, the contents of this file will be given to the executed custom test over standard input.
+
+### Building
+
+The Source Files for a custom test are compiled using C++20 and are saved as an executable named according to the name of the test in the CMake build directory.
+
+### Return Values
+
+All successful tests should return `0`. All failing tests should return something other than `0`.
+
+### Supporting Libraries
+
+To reduce the boilerplate needed to write custom tests, there is a custom test library with several helpful functions. These functions are documented in the library's header file (`custom_tests/srcs/ubpf_custom_test_support.h`).
+
+### Putting It Together
+
+After describing the test's purpose in a Markdown syntax in a file named, say, `test_example.md` and stored in the `descrs` directory, you can write the test's Source Code (in C++20) and give it the name `test_example.cc` in the `srcs` directory. If the test needs input, you can save that input in the tests Input File (`test_input.input`) in the `data` directory.
+
+Because all the files are present, this test will be run when the CTest target is invoked. Because there the optional `test_input.input` file is present, the contents of that file will be given to the executable via standard input.

custom_tests/data/ubpf_test_external_dispatcher_context_overwrite.input

@@ -0,0 +1 @@
+b7  01  00  00  01  02  03  04  85  00  00  00  01  00  00  00  95  00  00  00  00  00  00  00

custom_tests/data/ubpf_test_external_dispatcher_simple_context.input

@@ -0,0 +1 @@
+85  00  00  00  01  00  00  00 95  00  00  00  00  00  00  00

custom_tests/data/ubpf_test_update_dispatcher.input

@@ -0,0 +1 @@
+85  00  00  00  01  00  00  00 95  00  00  00  00  00  00  00

custom_tests/data/ubpf_test_update_helpers.input

@@ -0,0 +1 @@
+85  00  00  00  01  00  00  00 95  00  00  00  00  00  00  00

custom_tests/srcs/test_helpers.h

@@ -0,0 +1,128 @@
+// Copyright (c) Microsoft Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+#include "ubpf.h"
+#include <string.h>
+#include <cmath>
+#include <cstdint>
+#include <map>
+
+#if !defined(UNREFERENCED_PARAMETER)
+#define UNREFERENCED_PARAMETER(P) (void)(P)
+#endif
+
+static uint64_t
+gather_bytes(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e)
+{
+    return ((uint64_t)(a & 0xff) << 32) | ((uint64_t)(b & 0xff) << 24) | ((uint64_t)(c & 0xff) << 16) |
+           ((uint64_t)(d & 0xff) << 8) | (e & 0xff);
+};
+
+static uint64_t
+memfrob(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e)
+{
+    UNREFERENCED_PARAMETER(c);
+    UNREFERENCED_PARAMETER(d);
+    UNREFERENCED_PARAMETER(e);
+
+    uint8_t* p = reinterpret_cast<uint8_t*>(a);
+    for (uint64_t i = 0; i < b; i++) {
+        p[i] ^= 42;
+    }
+    return 0;
+};
+
+;
+
+static uint64_t
+no_op(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e)
+{
+    UNREFERENCED_PARAMETER(a);
+    UNREFERENCED_PARAMETER(b);
+    UNREFERENCED_PARAMETER(c);
+    UNREFERENCED_PARAMETER(d);
+    UNREFERENCED_PARAMETER(e);
+
+    return 0;
+}
+
+static uint64_t
+sqrti(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e)
+{
+    UNREFERENCED_PARAMETER(b);
+    UNREFERENCED_PARAMETER(c);
+    UNREFERENCED_PARAMETER(d);
+    UNREFERENCED_PARAMETER(e);
+
+    return static_cast<uint64_t>(std::sqrt(a));
+}
+
+static uint64_t
+strcmp_ext(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e)
+{
+    UNREFERENCED_PARAMETER(c);
+    UNREFERENCED_PARAMETER(d);
+    UNREFERENCED_PARAMETER(e);
+    return strcmp(reinterpret_cast<char*>(a), reinterpret_cast<char*>(b));
+}
+
+static uint64_t
+unwind(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e)
+{
+    UNREFERENCED_PARAMETER(b);
+    UNREFERENCED_PARAMETER(c);
+    UNREFERENCED_PARAMETER(d);
+    UNREFERENCED_PARAMETER(e);
+    return a;
+}
+
+static std::map<uint32_t, external_function_t> helper_functions = {
+    {0, gather_bytes},
+    {1, memfrob},
+    {2, no_op},
+    {3, sqrti},
+    {4, strcmp_ext},
+    {5, unwind},
+};
+
+static uint64_t
+dispatcher_test_memfrob(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e)
+{
+    UNREFERENCED_PARAMETER(a);
+    UNREFERENCED_PARAMETER(b);
+    UNREFERENCED_PARAMETER(c);
+    UNREFERENCED_PARAMETER(d);
+    UNREFERENCED_PARAMETER(e);
+    return 42;
+
+}
+
+static uint64_t
+updated_dispatcher_test_memfrob(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e)
+{
+    UNREFERENCED_PARAMETER(a);
+    UNREFERENCED_PARAMETER(b);
+    UNREFERENCED_PARAMETER(c);
+    UNREFERENCED_PARAMETER(d);
+    UNREFERENCED_PARAMETER(e);
+    return 43;
+}
+
+static std::map<uint32_t, external_function_t> dispatch_test_helper_functions = {
+    {0, gather_bytes},
+    {1, dispatcher_test_memfrob },
+    {2, no_op},
+    {3, sqrti},
+    {4, strcmp_ext},
+    {5, unwind},
+};
+
+static std::map<uint32_t, external_function_t> updated_dispatch_test_helper_functions = {
+    {0, gather_bytes},
+    {1, updated_dispatcher_test_memfrob },
+    {2, no_op},
+    {3, sqrti},
+    {4, strcmp_ext},
+    {5, unwind},
+};