Merge branch 'main' into test_elf

.github/workflows/posix.yml

@@ -66,7 +66,7 @@ jobs:
 
     - name: Initialize CodeQL
       if: inputs.build_codeql == true
-      uses: github/codeql-action/init@407ffafae6a767df3e0230c3df91b6443ae8df75
+      uses: github/codeql-action/init@c7f9125735019aa87cfc361530512d50ea439c71
       with:
         languages: 'cpp'
 
@@ -75,7 +75,7 @@ jobs:
       run: echo "VALUE=platform-${{ inputs.platform }}_arch=${{ inputs.arch }}_type-${{ inputs.build_type }}_sanitizers-${{ inputs.enable_sanitizers }}_coverage-${{ inputs.enable_coverage }}_scan_build-${{ inputs.scan_build }}_retpolines-${{ inputs.disable_retpolines }}" >> $GITHUB_OUTPUT
 
     - name: Update the cache (ccache)
-      uses: actions/cache@v3.3.2
+      uses: actions/cache@v4.0.2
       with:
         path: ccache
         key: ${{ steps.cache_key.outputs.VALUE }}_ccache
@@ -295,4 +295,4 @@ jobs:
 
     - name: Perform CodeQL Analysis
       if: inputs.build_codeql == true
-      uses: github/codeql-action/analyze@407ffafae6a767df3e0230c3df91b6443ae8df75
+      uses: github/codeql-action/analyze@c7f9125735019aa87cfc361530512d50ea439c71

.github/workflows/udpate-docs.yml

@@ -30,7 +30,7 @@ jobs:
     steps:
 
     - name: Harden Runner
-      uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895
+      uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142
       with:
         egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
 

.github/workflows/windows.yml

@@ -42,7 +42,7 @@ jobs:
 
     - name: Initialize CodeQL
       if: inputs.build_codeql == true
-      uses: github/codeql-action/init@407ffafae6a767df3e0230c3df91b6443ae8df75
+      uses: github/codeql-action/init@c7f9125735019aa87cfc361530512d50ea439c71
       with:
         languages: 'cpp'
 
@@ -65,14 +65,14 @@ jobs:
     - name: Run the bpf_conformance tests (Windows)
       run: |
         cd .\build\bin\${{ inputs.build_type }}
-        $BPF_CONFORMANCE_RUNNER = "..\..\external\bin\${{ inputs.build_type }}\bpf_conformance_runner.exe"
+        $BPF_CONFORMANCE_RUNNER = "..\..\external\bpf_conformance\bin\${{ inputs.build_type }}\bpf_conformance_runner.exe"
         & $BPF_CONFORMANCE_RUNNER --test_file_directory ..\tests --plugin_path ubpf_plugin.exe --plugin_options --jit
         if ($LastExitCode -ne 0) { throw "Test failed"; }
         & $BPF_CONFORMANCE_RUNNER --test_file_directory ..\tests --plugin_path ubpf_plugin.exe --plugin_options --interpret
         if ($LastExitCode -ne 0) { throw "Test failed"; }
-        & $BPF_CONFORMANCE_RUNNER --exclude lock_ --test_file_directory ..\..\external\tests --plugin_path ubpf_plugin.exe --plugin_options --jit
+        & $BPF_CONFORMANCE_RUNNER --exclude_regex lock_ --test_file_directory ..\..\external\bpf_conformance\tests --plugin_path ubpf_plugin.exe --plugin_options --jit
         if ($LastExitCode -ne 0) { throw "Test failed"; }
-        & $BPF_CONFORMANCE_RUNNER --exclude lock_ --test_file_directory ..\..\external\tests --plugin_path ubpf_plugin.exe --plugin_options --interpret
+        & $BPF_CONFORMANCE_RUNNER --exclude_regex lock_ --test_file_directory ..\..\external\bpf_conformance\tests --plugin_path ubpf_plugin.exe --plugin_options --interpret
         if ($LastExitCode -ne 0) { throw "Test failed"; }
 
     - name: Generate the TGZ package
@@ -104,4 +104,4 @@ jobs:
 
     - name: Perform CodeQL Analysis
       if: inputs.build_codeql == true
-      uses: github/codeql-action/analyze@407ffafae6a767df3e0230c3df91b6443ae8df75
+      uses: github/codeql-action/analyze@c7f9125735019aa87cfc361530512d50ea439c71

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "vm/compat/libraries/win-c/src"]
 	path = vm/compat/libraries/win-c/src
 	url = https://github.com/takamin/win-c
+[submodule "external/bpf_conformance"]
+	path = external/bpf_conformance
+	url = https://github.com/Alan-Jowett/bpf_conformance.git

CMakeLists.txt

@@ -28,15 +28,16 @@ if(UBPF_ENABLE_TESTS)
 endif()
 
 add_subdirectory("vm")
+    ExternalProject_Add(Conformance
+        INSTALL_COMMAND ""
+        SOURCE_DIR ${CMAKE_SOURCE_DIR}/external/bpf_conformance
+				BINARY_DIR ${CMAKE_BINARY_DIR}/external/bpf_conformance
+        EXCLUDE_FROM_ALL true
+        STEP_TARGETS build)
 
 if(UBPF_ENABLE_TESTS)
   add_subdirectory("ubpf_plugin")
   if (NOT UBPF_SKIP_EXTERNAL)
-    ExternalProject_Add(Conformance GIT_REPOSITORY "https://github.com/Alan-Jowett/bpf_conformance.git"
-        GIT_SUBMODULES_RECURSE true
-        GIT_TAG main
-        INSTALL_COMMAND ""
-        BINARY_DIR ${CMAKE_BINARY_DIR}/external/)
   endif()
   add_subdirectory("bpf")
   add_subdirectory("aarch64_test")

README.md

@@ -52,7 +52,7 @@ produces output about how to execute the program. With `nuget.exe` installed, th
 First, make sure that you have the XCode Command Line Tools installed:
 
 ```console
-$ xcode-select -install
+$ xcode-select --install
 ```
 
 Installing the XCode Command Linux Tools will install Apple's version of the Clang compiler and other developer-support tools.

requirements.txt

@@ -1,4 +1,4 @@
 # pypi version of parcon does not support python3
 git+https://github.com/javawizard/parcon
 nose ~= 1.3.7
-pyelftools ~= 0.30
+pyelftools ~= 0.31

tests/div32-by-zero-reg.data

@@ -0,0 +1,7 @@
+-- asm
+mov32 %r0, 1
+lddw %r1, 0x100000000
+div32 %r0, %r1
+exit
+-- result
+0x0

tests/mul-loop-memory-iterations.data

@@ -0,0 +1,12 @@
+-- asm
+mov %r0, 0x7
+ldxw %r1, [%r1]
+add %r1, 0x2
+mul %r0, 0x7
+add %r1, 0xffffffff
+jne %r1, 0x0, -3
+exit
+-- mem
+00 00 00 00
+-- result
+0x157

ubpf_plugin/CMakeLists.txt

@@ -1,6 +1,11 @@
 # Copyright (c) Microsoft Corporation
 # SPDX-License-Identifier: Apache-2.0
 
+if (UBPF_SKIP_EXTERNAL)
+    message(WARNING "Skipping configuration of tests that require external package support.")
+    return()
+endif()
+
 set(CMAKE_CXX_STANDARD 20)
 
 file(COPY ${CMAKE_SOURCE_DIR}/tests DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
@@ -10,6 +15,8 @@ add_executable(
     ubpf_plugin.cc
 )
 
+add_dependencies(ubpf_plugin Conformance-build)
+
 target_include_directories("ubpf_plugin" PRIVATE
     "${CMAKE_SOURCE_DIR}/vm"
     "${CMAKE_BINARY_DIR}/vm"
@@ -23,13 +30,13 @@ target_link_libraries(
     ubpf_settings
 )
 
-file(GLOB external_files ${CMAKE_SOURCE_DIR}/external/tests/*.data)
+file(GLOB external_files ${CMAKE_SOURCE_DIR}/external/bpf_conformance/tests/*.data)
 file(GLOB local_files ${CMAKE_SOURCE_DIR}/tests/*.data)
 
 set(files ${external_files} ${local_files})
 
 if(NOT BPF_CONFORMANCE_RUNNER)
-    set(BPF_CONFORMANCE_RUNNER ${CMAKE_BINARY_DIR}/external/bin/bpf_conformance_runner)
+    set(BPF_CONFORMANCE_RUNNER ${CMAKE_BINARY_DIR}/external/bpf_conformance/bin/bpf_conformance_runner)
 else()
     message(STATUS "Using custom bpf_conformance_runner: ${BPF_CONFORMANCE_RUNNER}")
 endif()

ubpf_plugin/test_helpers.h

@@ -2,6 +2,8 @@
 // SPDX-License-Identifier: Apache-2.0
 
 #pragma once
+#include "ubpf.h"
+#include <string.h>
 #include <cmath>
 #include <cstdint>
 #include <map>
@@ -73,12 +75,11 @@ unwind(uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e)
     return a;
 }
 
-static std::map<uint32_t, uint64_t (*)(uint64_t r1, uint64_t r2, uint64_t r3, uint64_t r4, uint64_t r5)>
-    helper_functions = {
-        {0, gather_bytes},
-        {1, memfrob},
-        {2, no_op},
-        {3, sqrti},
-        {4, strcmp_ext},
-        {5, unwind},
+static std::map<uint32_t, external_function_t> helper_functions = {
+    {0, gather_bytes},
+    {1, memfrob},
+    {2, no_op},
+    {3, sqrti},
+    {4, strcmp_ext},
+    {5, unwind},
 };

ubpf_plugin/ubpf_plugin.cc

@@ -6,6 +6,7 @@
 // value of %r0 at the end of execution.
 // The program is intended to be used with the bpf conformance test suite.
 
+#include <cstdint>
 #include <cstring>
 #include <iostream>
 #include <memory>
@@ -21,6 +22,16 @@ extern "C"
 
 #include "test_helpers.h"
 
+uint64_t test_helpers_dispatcher(uint64_t p0, uint64_t p1,uint64_t p2,uint64_t p3, uint64_t p4, unsigned int idx, void* cookie) {
+    UNREFERENCED_PARAMETER(cookie);
+    return helper_functions[idx](p0, p1, p2, p3, p4);
+}
+
+bool test_helpers_validater(unsigned int idx, void *cookie) {
+    UNREFERENCED_PARAMETER(cookie);
+    return helper_functions.contains(idx);
+}
+
 /**
  * @brief Read in a string of hex bytes and return a vector of bytes.
  *
@@ -108,6 +119,11 @@ int main(int argc, char **argv)
 
     auto program_byte = base16_decode(program_string);
     std::vector<uint8_t> memory = base16_decode(memory_string);
+    uint8_t *memory_ptr{nullptr};
+
+    if (memory.size() != 0) {
+        memory_ptr = memory.data();
+    }
 
     std::unique_ptr<ubpf_vm, decltype(&ubpf_destroy)> vm(ubpf_create(), ubpf_destroy);
     char* error = nullptr;
@@ -118,14 +134,18 @@ int main(int argc, char **argv)
         return 1;
     }
 
+/*
     for (auto &[key, value] : helper_functions)
     {
-        if (ubpf_register(vm.get(), key, "unnamed", reinterpret_cast<void *>(value)) != 0)
+        if (ubpf_register(vm.get(), key, "unnamed", value) != 0)
         {
             std::cerr << "Failed to register helper function" << std::endl;
             return 1;
         }
     }
+*/
+
+    ubpf_register_external_dispatcher(vm.get(), test_helpers_dispatcher, test_helpers_validater, NULL);
 
     if (ubpf_set_unwind_function_index(vm.get(), 5) != 0)
     {
@@ -168,11 +188,11 @@ int main(int argc, char **argv)
             free(error);
             return 1;
         }
-        actual_result = fn(memory.data(), memory.size());
+        actual_result = fn(memory_ptr, memory.size());
     }
     else
     {
-        if (ubpf_exec(vm.get(), memory.data(), memory.size(), &actual_result) != 0)
+        if (ubpf_exec(vm.get(), memory_ptr, memory.size(), &actual_result) != 0)
         {
             std::cerr << "Failed to execute program" << std::endl;
             return 1;

vm/inc/ubpf.h

@@ -98,6 +98,23 @@ extern "C"
     void
     ubpf_set_error_print(struct ubpf_vm* vm, int (*error_printf)(FILE* stream, const char* format, ...));
 
+    /**
+     * @brief The type of an external function.
+     */
+    typedef uint64_t (*external_function_t)(uint64_t p0, uint64_t p1, uint64_t p2, uint64_t p3, uint64_t p4);
+
+    /**
+     * @brief Cast an external function to external_function_t
+     * Some external functions may not use all the parameters and, therefore,
+     * not match the external_function_t typedef. Use this for a conversion.
+     *
+     * @param[in] f The function to cast to match the signature of an
+     *              external function.
+     * @retval The external function, as external_function_t.
+     */
+    external_function_t
+    as_external_function_t(void* f);
+
     /**
      * @brief Register an external function.
      * The immediate field of a CALL instruction is an index into an array of
@@ -112,7 +129,43 @@ extern "C"
      * @retval -1 Failure.
      */
     int
-    ubpf_register(struct ubpf_vm* vm, unsigned int index, const char* name, void* fn);
+    ubpf_register(struct ubpf_vm* vm, unsigned int index, const char* name, external_function_t fn);
+
+    /**
+     * @brief The type of an external helper dispatcher function.
+     */
+    typedef uint64_t (*external_function_dispatcher_t)(
+        uint64_t, uint64_t, uint64_t, uint64_t, uint64_t, unsigned int index, void* cookie);
+
+    /**
+     * @brief The type of an external helper validation function.
+     */
+    typedef bool (*external_function_validate_t)(unsigned int index, void* cookie);
+
+    /**
+     * @brief Register a function that dispatches to external helpers
+     * The immediate field of a CALL instruction is an index of a helper
+     * function to invoke. This API sets a callback that will choose the
+     * helper function to invoke (based on the index) and then invoke it.
+     * This API also sets a callback that the validator will use to determine
+     * if a given index is a valid external function.
+     *
+     * @param[in] vm The VM to register the function on.
+     * @param[in] dispatcher The callback that will dispatch to the external
+     *                       helper.
+     * @param[in] validater The callback that will validate that a given index
+     *                      is valid for an external helper.
+     * @param[in] cookie A pointer to some user-defined cookie that will be
+     *                   passed to the callbacks.
+     * @retval 0 Success.
+     * @retval -1 Failure.
+     */
+    int
+    ubpf_register_external_dispatcher(
+        struct ubpf_vm* vm,
+        external_function_dispatcher_t dispatcher,
+        external_function_validate_t validater,
+        void* cookie);
 
     /**
      * @brief Load code into a VM.

vm/test.c

@@ -516,14 +516,26 @@ bpf_map_delete_elem_impl(struct bpf_map* map, const void* key)
 static void
 register_functions(struct ubpf_vm* vm)
 {
-    ubpf_register(vm, 0, "gather_bytes", gather_bytes);
-    ubpf_register(vm, 1, "memfrob", memfrob);
-    ubpf_register(vm, 2, "trash_registers", trash_registers);
-    ubpf_register(vm, 3, "sqrti", sqrti);
-    ubpf_register(vm, 4, "strcmp_ext", strcmp);
-    ubpf_register(vm, 5, "unwind", unwind);
+    ubpf_register(vm, 0, "gather_bytes", as_external_function_t(gather_bytes));
+    ubpf_register(vm, 1, "memfrob", as_external_function_t(memfrob));
+    ubpf_register(vm, 2, "trash_registers", as_external_function_t(trash_registers));
+    ubpf_register(vm, 3, "sqrti", as_external_function_t(sqrti));
+    ubpf_register(vm, 4, "strcmp_ext", as_external_function_t(strcmp));
+    ubpf_register(vm, 5, "unwind", as_external_function_t(unwind));
     ubpf_set_unwind_function_index(vm, 5);
-    ubpf_register(vm, (unsigned int)(uintptr_t)bpf_map_lookup_elem, "bpf_map_lookup_elem", bpf_map_lookup_elem_impl);
-    ubpf_register(vm, (unsigned int)(uintptr_t)bpf_map_update_elem, "bpf_map_update_elem", bpf_map_update_elem_impl);
-    ubpf_register(vm, (unsigned int)(uintptr_t)bpf_map_delete_elem, "bpf_map_delete_elem", bpf_map_delete_elem_impl);
+    ubpf_register(
+        vm,
+        (unsigned int)(uintptr_t)bpf_map_lookup_elem,
+        "bpf_map_lookup_elem",
+        as_external_function_t(bpf_map_lookup_elem_impl));
+    ubpf_register(
+        vm,
+        (unsigned int)(uintptr_t)bpf_map_update_elem,
+        "bpf_map_update_elem",
+        as_external_function_t(bpf_map_update_elem_impl));
+    ubpf_register(
+        vm,
+        (unsigned int)(uintptr_t)bpf_map_delete_elem,
+        "bpf_map_delete_elem",
+        as_external_function_t(bpf_map_delete_elem_impl));
 }