Adding a emulator

CMakeLists.txt

@@ -310,6 +310,9 @@ add_subdirectory(${JBPF_SOURCE_CODE}/core)
 ################ jbpf lcm cli ################
 add_subdirectory(${JBPF_TOOLS}/lcm_cli)
 
+################ jbpf emulator ################
+add_subdirectory(${JBPF_SOURCE_CODE}/emulator)
+
 ################ Doxygen ################
 
 find_package(Doxygen)

deploy/mariner.Dockerfile

@@ -8,6 +8,9 @@ RUN tdnf -y install yaml-cpp-devel yaml-cpp-static boost-devel gcovr clang pytho
 RUN tdnf -y install doxygen
 ## clang-format
 RUN tdnf -y install clang-tools-extra
+## emulator
+RUN tdnf -y install python3-devel python3-pip
+RUN pip3 install ctypesgen pyyaml regex 
 
 WORKDIR /jbpf
 COPY . /jbpf

deploy/ubuntu20.04.Dockerfile

@@ -13,6 +13,10 @@ RUN apt update --fix-missing && \
 
 RUN apt install -y cppcheck
 
+## Emulator
+RUN apt install -y python-dev python3-pip
+RUN pip3 install ctypesgen pyyaml regex
+
 # Install specific versions for CVEs
 ## CVE-2023-4016
 RUN apt install -y procps=2:3.3.16-1ubuntu2.4 --allow-downgrades

deploy/ubuntu22.04.Dockerfile

@@ -14,6 +14,10 @@ RUN apt install -y clang-format cppcheck
 RUN apt install -y clang gcc-multilib
 RUN apt install -y libyaml-cpp-dev
 
+## Emulator
+RUN apt install -y python3-dev python3-pip
+RUN pip3 install ctypesgen pyyaml regex
+
 WORKDIR /jbpf
 COPY . /jbpf
 ENTRYPOINT ["./helper_build_files/build_jbpf_lib.sh"]

deploy/ubuntu24.04.Dockerfile

@@ -13,6 +13,10 @@ RUN apt install -y clang-format cppcheck
 RUN apt install -y clang gcc-multilib
 RUN apt install -y libyaml-cpp-dev
 
+## Emulator
+RUN apt install -y python3-dev python3-pip
+RUN pip3 install ctypesgen pyyaml regex --break-system-packages
+
 WORKDIR /jbpf
 COPY . /jbpf
 ENTRYPOINT ["./helper_build_files/build_jbpf_lib.sh"]

helper_build_files/build_jbpf_lib.sh

@@ -73,6 +73,23 @@ if [[ "$RUN_TESTS" == "1" ]]; then
     fi
 fi
 
+## Test the emulator
+if [[ "$RUN_EMULATOR_TESTS" == "1" ]]; then
+    export JBPF_PATH=/jbpf
+    EMULATOR_TEST_CASES="test_0 test_1 test_2 test_report_stats test_periodic_call test_time_event"
+
+    ## TOFIX: Disable ODR violation detection for the emulator tests
+    ## As the emulator tests are not built with the same flags as the rest of the project, they may trigger ODR violations
+    export ASAN_OPTIONS=detect_odr_violation=0
+    export LSAN_OPTIONS="suppressions=/jbpf/lsan_agent.supp"
+    for test in $EMULATOR_TEST_CASES; do
+        if ! /jbpf/out/bin/jbpf_emulator /jbpf/out/emulator/test/ $test; then
+            echo "Error running emulator test $test!"
+            exit 1
+        fi
+    done
+fi
+
 ## Check that all of the libraries we link against are approved, and exit if it fails.
 CHECK_LIB_ARGS="--build-dir /jbpf/build --approved-libraries /jbpf/helper_build_files/approved_libraries.txt"
 if [[ "$JBPF_STATIC" != "1" ]]; then

jbpf_tests/functional/CMakeLists.txt

@@ -31,4 +31,3 @@ add_subdirectory(verifier_extensions)
 add_subdirectory(ctrl_hooks)
 add_subdirectory(helper_functions)
 add_subdirectory(array)
-

jbpf_tests/test_files/codelets/simple_output2/Makefile

@@ -0,0 +1,2 @@
+include ../Makefile.defs
+include ../Makefile.common

jbpf_tests/test_files/codelets/simple_output2/simple_output2.c

@@ -0,0 +1,21 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+
+#include <string.h>
+#include "jbpf_defs.h"
+#include "jbpf_test_def.h"
+#include "jbpf_helper.h"
+
+// Send a single int value out
+jbpf_ringbuf_map(output_map, int, 10);
+
+SEC("jbpf_generic")
+uint64_t
+jbpf_main(void* state)
+{
+    int data = 123;
+    if (jbpf_ringbuf_output(&output_map, &data, sizeof(data)) < 0) {
+        return 1;
+    }
+
+    return 0;
+}

jbpf_tests/verifier/jbpf_basic_verifier_test.cpp

@@ -14,6 +14,7 @@ std::string correct_codelets[] = {
     "/jbpf_tests/test_files/codelets/codelet-per-thread/codelet-per-thread.o",
     "/jbpf_tests/test_files/codelets/simple_input_shared/simple_input_shared.o",
     "/jbpf_tests/test_files/codelets/simple_output/simple_output.o",
+    "/jbpf_tests/test_files/codelets/simple_output2/simple_output2.o",
     "/jbpf_tests/test_files/codelets/time_events/time_events.o",
     "/jbpf_tests/test_files/codelets/simple_output_shared/simple_output_shared.o",
     "/jbpf_tests/test_files/codelets/simple_test_shared_multiple_maps/simple_test1_shared_multiple_maps.o",

lsan_agent.supp

@@ -0,0 +1 @@
+leak:libpython

pipeline/standard-tests.yaml

@@ -62,6 +62,15 @@ steps:
     displayName: Build codelets /tools/stats_report
     continueOnError: false
 
+  - bash: |
+      set -x
+      docker run --init --privileged --network host --cap-add=SYS_ADMIN --shm-size=1g --mount type=tmpfs,destination=/dev/shm -v `pwd`:/jbpf_out_lib -v `pwd`/build:/jbpf/build ${{parameters.testCase['sanitizerBuildParam']}} ${{parameters.testCase['staticBuildParam']}} -e RUN_EMULATOR_TESTS=1 jbpf_lib
+
+    displayName: Test Emulator
+    continueOnError: false
+    ## the emulator doesn't work on dynamic builds: the hook_report_stats and hook_periodic_call tests fail.
+    condition: ne('${{ parameters.testCase.staticBuildParam }}', '')
+
   - bash: |
       set -x
       docker run --init --privileged --network host --cap-add=SYS_ADMIN --shm-size=1g --mount type=tmpfs,destination=/dev/shm -v `pwd`:/jbpf_out_lib -v `pwd`/build:/jbpf/build ${{parameters.testCase['sanitizerBuildParam']}} ${{parameters.testCase['staticBuildParam']}} -e RUN_TESTS=1 jbpf_lib

src/core/jbpf_helper_impl.c

@@ -598,4 +598,16 @@ int
 __jbpf_runtime_limit_exceeded(void)
 {
     return jbpf_runtime_limit_exceeded();
-}
+}
+
+jbpf_helper_func_def_t*
+__get_custom_helper_functions(void)
+{
+    return helper_funcs;
+}
+
+const jbpf_helper_func_def_t*
+__get_default_helper_functions(void)
+{
+    return default_helper_funcs;
+}

src/core/jbpf_helper_impl.h

@@ -2,6 +2,7 @@
 #define JBPF_HELPER_IMPL_H
 
 #include "ubpf.h"
+#include "jbpf.h"
 
 void
 jbpf_register_helper_functions(struct ubpf_vm* vm);
@@ -22,4 +23,20 @@ __jbpf_mark_runtime_init(void);
 int
 __jbpf_runtime_limit_exceeded(void);
 
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+    // emulator
+    jbpf_helper_func_def_t*
+    __get_custom_helper_functions(void);
+
+    const jbpf_helper_func_def_t*
+    __get_default_helper_functions(void);
+
+#ifdef __cplusplus
+}
+#endif
+
 #endif

src/emulator/CMakeLists.txt

@@ -0,0 +1,148 @@
+project(jbpf_emulator)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+SET(CMAKE_CXX_FLAGS "-std=c++17")
+
+## require ENV JBPF_PATH
+if(NOT DEFINED ENV{JBPF_PATH})
+    message(FATAL_ERROR "Please set the environment variable JBPF_PATH to the jbpf directory")
+endif()
+
+set(JBPF_PATH $ENV{JBPF_PATH})
+
+set(JBPF_EMULATOR jbpf_emulator)
+set(JBPF_EMULATOR_SRC_DIR ${PROJECT_SOURCE_DIR})
+set(JBPF_EMULATOR_SOURCES ${JBPF_EMULATOR_SRC_DIR}/emulator.cpp ${JBPF_PATH}/src/core/jbpf_helper_impl.c)
+
+## require ENV JBPF_OUT_DIR
+if(NOT DEFINED ENV{JBPF_OUT_DIR})
+    message(FATAL_ERROR "Please set the environment variable JBPF_OUT_DIR to the output directory")
+endif()
+
+set(JBPF_OUT_DIR $ENV{JBPF_OUT_DIR})
+
+find_package(Python COMPONENTS Interpreter Development REQUIRED)
+
+message(STATUS "Python_EXECUTABLE: ${Python_EXECUTABLE}")
+message(STATUS "Python_VERSION: ${Python_VERSION}")
+message(STATUS "Python_INCLUDE_DIRS: ${Python_INCLUDE_DIRS}")
+message(STATUS "Python_LIBRARIES: ${Python_LIBRARIES}")
+set(Python3_LINK_OPTIONS "-lpython${Python_VERSION_MAJOR}.${Python_VERSION_MINOR}")
+message(STATUS "Python3_LINK_OPTIONS: ${Python3_LINK_OPTIONS}")
+
+include_directories(${Python_INCLUDE_DIRS})
+
+add_executable(${JBPF_EMULATOR} ${JBPF_EMULATOR_SOURCES})
+target_include_directories(${JBPF_EMULATOR} PUBLIC 
+    ${JBPF_EMULATOR_HEADERS} 
+    ${JBPF_LIB_HEADER_FILES}
+    ${JBPF_IO_HEADER_FILES}
+    ${JBPF_MEM_MGMT_HEADER_FILES}
+    ${JBPF_LCM_IPC_HEADER_FILES}
+    ${JBPF_COMMON_HEADERS}
+    ${JBPF_CK_HEADER_FILES}
+    ${JBPF_CK_INCLUDES}
+    ${JBPF_OUT_DIR}/inc
+    ${CMAKE_SOURCE_DIR}/jbpf_tests/common
+    ${UBPF_INCLUDE_DIRS}
+    ${JBPF_LOGGER_HEADERS}
+)
+add_dependencies(${JBPF_EMULATOR} jbpf::core_lib jbpf::logger_lib jbpf::mem_mgmt_lib jbpf_verifier jbpf_io generate_hook_wrappers)
+
+target_link_options(${JBPF_EMULATOR} PUBLIC ${Python3_LINK_OPTIONS} "-lpthread" "-ldl" "-lrt")
+
+# Define the common libraries
+set(COMMON_LIBS
+    ${JBPF_OUT_DIR}/lib/libjbpf_io.a
+    ${JBPF_OUT_DIR}/lib/libjbpf_mem_mgmt.a
+    ${JBPF_OUT_DIR}/lib/libjbpf_verifier.a
+    ${JBPF_OUT_DIR}/lib/libubpf.a
+    ${JBPF_OUT_DIR}/lib/libck.a
+    pthread
+    dl
+    rt    
+    ${Python_LIBRARIES}
+)
+
+if (JBPF_STATIC)
+    MESSAGE(STATUS "Linking the the emulator to libjbpf.a")
+    target_link_libraries(${JBPF_EMULATOR} PUBLIC
+        ${JBPF_OUT_DIR}/lib/libjbpf.a
+        ${COMMON_LIBS}        
+    )
+else()
+    MESSAGE(WARNING "Linking the emulator to libjbpf.so may not work as expected for report_stats and periodic_call hooks.")
+    target_link_libraries(${JBPF_EMULATOR} PUBLIC
+        ${JBPF_OUT_DIR}/lib/libjbpf.so
+        ${COMMON_LIBS}        
+    )
+endif()
+
+add_clang_format_check(${JBPF_EMULATOR} ${JBPF_EMULATOR_SOURCES})
+add_cppcheck(${JBPF_EMULATOR} ${JBPF_EMULATOR_SOURCES})
+
+set_target_properties(${JBPF_EMULATOR}
+    PROPERTIES
+    RUNTIME_OUTPUT_DIRECTORY  "${OUTPUT_DIR}/bin"
+)
+
+add_custom_command(TARGET ${JBPF_EMULATOR} POST_BUILD
+    COMMAND ctypesgen -a ../lcm/jbpf_lcm_api.h --no-macro-warnings -I ../io -I ../mem_mgmt -I ../common -o ${OUTPUT_DIR}/lib/jbpf_lcm_api.py
+    COMMAND python3 ${PROJECT_SOURCE_DIR}/patch.py ${OUTPUT_DIR}/lib/jbpf_lcm_api.py
+    COMMAND ctypesgen -a ../../jbpf_tests/common/jbpf_test_def.h --no-macro-warnings -I ../io -I ../mem_mgmt -I ../common -o ${OUTPUT_DIR}/lib/jbpf_test_def.py
+    COMMAND python3 ${PROJECT_SOURCE_DIR}/patch.py ${OUTPUT_DIR}/lib/jbpf_test_def.py
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMENT "Building python APIs for tests using the C headers (jbpf_lcm_api.h)..."
+)
+
+## copy the source files to the out/emulator directory
+add_custom_command(TARGET ${JBPF_EMULATOR} POST_BUILD
+    ## make out/emulator directory
+    COMMAND ${CMAKE_COMMAND} -E make_directory ${OUTPUT_DIR}/emulator
+    COMMAND ${CMAKE_COMMAND} -E make_directory ${OUTPUT_DIR}/emulator/test
+    ## minimal emulator files
+    COMMAND ${CMAKE_COMMAND} -E copy ${JBPF_EMULATOR_SRC_DIR}/emulator_utils.py ${OUTPUT_DIR}/emulator/
+    COMMAND ${CMAKE_COMMAND} -E copy ${JBPF_EMULATOR_SRC_DIR}/patch.py ${OUTPUT_DIR}/emulator/
+    COMMAND ${CMAKE_COMMAND} -E copy ${JBPF_EMULATOR_SRC_DIR}/emulator.cpp ${OUTPUT_DIR}/emulator/
+    COMMAND ${CMAKE_COMMAND} -E copy ${JBPF_EMULATOR_SRC_DIR}/hook_wrapper.h ${OUTPUT_DIR}/emulator/
+    COMMAND ${CMAKE_COMMAND} -E copy ${JBPF_EMULATOR_SRC_DIR}/autogen-hook-wrappers.py ${OUTPUT_DIR}/emulator/
+    ## example test files
+    COMMAND ${CMAKE_COMMAND} -E copy ${JBPF_EMULATOR_SRC_DIR}/test/helper_functions.hpp ${OUTPUT_DIR}/emulator/test
+    COMMAND ${CMAKE_COMMAND} -E copy ${JBPF_EMULATOR_SRC_DIR}/test/hooks.h ${OUTPUT_DIR}/emulator/test
+    COMMAND ${CMAKE_COMMAND} -E copy ${JBPF_EMULATOR_SRC_DIR}/test/test_0.py ${OUTPUT_DIR}/emulator/test
+    COMMAND ${CMAKE_COMMAND} -E copy ${JBPF_EMULATOR_SRC_DIR}/test/test_1.py ${OUTPUT_DIR}/emulator/test
+    COMMAND ${CMAKE_COMMAND} -E copy ${JBPF_EMULATOR_SRC_DIR}/test/test_2.py ${OUTPUT_DIR}/emulator/test
+    COMMAND ${CMAKE_COMMAND} -E copy ${JBPF_EMULATOR_SRC_DIR}/test/test_report_stats.py ${OUTPUT_DIR}/emulator/test
+    COMMAND ${CMAKE_COMMAND} -E copy ${JBPF_EMULATOR_SRC_DIR}/test/test_periodic_call.py ${OUTPUT_DIR}/emulator/test
+    COMMAND ${CMAKE_COMMAND} -E copy ${JBPF_EMULATOR_SRC_DIR}/test/test_time_event.py ${OUTPUT_DIR}/emulator/test
+    ## hook header files
+    COMMAND ${CMAKE_COMMAND} -E copy ${JBPF_PATH}/jbpf_tests/common/jbpf_test_def.h ${OUTPUT_DIR}/emulator/test
+    COMMAND ${CMAKE_COMMAND} -E copy ${JBPF_PATH}/src/lcm/jbpf_lcm_api.h ${OUTPUT_DIR}/emulator/test
+    ## copy CMakeLists_emulator.txt to out/emulator/CMakeLists.txt
+    COMMAND ${CMAKE_COMMAND} -E copy ${JBPF_EMULATOR_SRC_DIR}/CMakeLists_emulator.txt ${OUTPUT_DIR}/emulator/CMakeLists.txt
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMENT "copy the source files to the out/emulator directory..."
+)
+
+# convert C header to Python structures, require ctypesgen
+add_custom_command(TARGET ${JBPF_EMULATOR} POST_BUILD
+    COMMAND echo "typedef unsigned char uint8_t\;" > /tmp/tmp_jbpf_perf_ext.h
+    COMMAND echo "typedef unsigned int uint32_t\;" >> /tmp/tmp_jbpf_perf_ext.h
+    COMMAND echo "typedef unsigned long long uint64_t\;" >> /tmp/tmp_jbpf_perf_ext.h
+    COMMAND echo "" >> /tmp/tmp_jbpf_perf_ext.h
+    COMMAND cat ${JBPF_PATH}/src/core/jbpf_perf_ext.h >> /tmp/tmp_jbpf_perf_ext.h
+    COMMAND ctypesgen /tmp/tmp_jbpf_perf_ext.h -I ${JBPF_PATH}/src/common/ --no-macro-warnings -o ${OUTPUT_DIR}/lib/jbpf_perf_ext.py
+    COMMENT "Building python APIs for tests using the C headers (jbpf_perf_ext.h)..."
+)
+
+# generate "auto-generated-hook-wrappers.hpp", this should be before the build
+add_custom_target(generate_hook_wrappers ALL
+    COMMAND echo "Generating hook wrappers: auto_generated_hooks_wrapper.hpp..."
+    COMMAND python3 ${JBPF_EMULATOR_SRC_DIR}/autogen-hook-wrappers.py ${JBPF_PATH}/jbpf_tests/common/jbpf_agent_common.h > ${OUTPUT_DIR}/inc/auto_generated_hooks_wrapper.hpp
+    ## remove the lines that start with "DEFINE_JBPF_"
+    COMMAND sed -i '/^DEFINE_JBPF_/d' ${OUTPUT_DIR}/inc/auto_generated_hooks_wrapper.hpp
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMENT "Generating hook wrappers: auto_generated_hooks_wrapper.hpp..."
+)