Fixed skupperproject#129: Configure coredumping in GHA and print and …

…archive backtraces (skupperproject#131)

.github/workflows/build.yaml

@@ -187,6 +187,7 @@ jobs:
       LD_LIBRARY_PATH: ${{github.workspace}}/install/lib
       QPID_SYSTEM_TEST_TIMEOUT: 300
       QPID_SYSTEM_TEST_SKIP_FALLBACK_SWITCHOVER_TEST: True
+
     steps:
 
       - name: Show environment (Linux)
@@ -210,7 +211,7 @@ jobs:
       - name: Install Linux runtime/test dependencies
         if: ${{ runner.os == 'Linux' }}
         run: |
-          sudo apt update; sudo apt install -y libsasl2-2 libsasl2-modules sasl2-bin libjsoncpp1 libwebsockets15 libbenchmark1 pixz bubblewrap curl
+          sudo apt update; sudo apt install -y libsasl2-2 libsasl2-modules sasl2-bin libjsoncpp1 libwebsockets15 libbenchmark1 pixz bubblewrap curl gdb elfutils findutils file python3-dbg
 
       - name: Unpack archive
         run: tar -I pixz -xf archive.tar.xz
@@ -221,9 +222,16 @@ jobs:
       - name: CTest
         working-directory: ${{env.RouterBuildDir}}
         run: |
+          sudo sysctl -w kernel.core_pattern="coredump.%e.%p"
           ulimit -c unlimited
+
           ctest --timeout 1200 -C ${BuildType} -V -T Test --no-compress-output -I ${{matrix.shard}},,${{matrix.shards}} -j100 ${{env.RouterCTestExtraArgs}}
 
+      - name: Report coredump stacktraces (if any tests failed)
+        if: ${{ failure() }}
+        run: |
+          find -name 'coredump*' -exec ${{github.workspace}}/skupper-router/scripts/gha_analyze_coredump.sh {} \;
+
       - name: Upload test results
         uses: actions/upload-artifact@v2
         if: ${{ ! cancelled() }}
@@ -245,7 +253,7 @@ jobs:
         with:
           name: cores_${{matrix.os}}_${{matrix.buildType}}_${{matrix.runtimeCheck}}_${{matrix.protonGitRef}}_${{matrix.shard}}
           path: |
-            **/core
+            **/coredump*
 
   compile_and_test:
     name: "Compile and Test (${{matrix.container}}:${{matrix.containerTag}}, ${{matrix.runtimeCheck}}, proton ${{matrix.protonGitRef}}, shard ${{matrix.shard}} of ${{matrix.shards}})"
@@ -295,6 +303,7 @@ jobs:
       image: 'quay.io/${{ matrix.container }}/${{ matrix.container }}:${{ matrix.containerTag }}'
       volumes:
         - ${{github.workspace}}:${{github.workspace}}
+      options: --privileged --ulimit core=-1 --security-opt apparmor:unconfined --security-opt seccomp=unconfined
 
     env:
       BuildType: ${{matrix.buildType}}
@@ -434,22 +443,25 @@ jobs:
       - name: Install Linux runtime/test dependencies
         if: ${{ runner.os == 'Linux' }}
         run: |
-          dnf install -y curl
-
-      - name: Install Linux runtime/test dependencies (for sanitizers)
-        if: ${{ matrix.runtimeCheck != 'OFF' }}
-        run: |
-          dnf install -y binutils
+          dnf install -y curl gdb binutils elfutils elfutils-debuginfod-client findutils file
+          dnf debuginfo-install -y python3
 
       - name: install qpid-proton python wheel
         run: python3 -m pip install ${ProtonBuildDir}/python/pkgs/python_qpid_proton*.whl
 
       - name: CTest
         working-directory: ${{env.RouterBuildDir}}
         run: |
+          echo "coredump.%e.%p" > /proc/sys/kernel/core_pattern
           ulimit -c unlimited
+
           ctest --timeout 1200 -C ${BuildType} -V -T Test --output-on-failure --no-compress-output -I ${{matrix.shard}},,${{matrix.shards}} -j12 ${{env.RouterCTestExtraArgs}}
 
+      - name: Report coredump stacktraces (if any tests failed)
+        if: ${{ failure() }}
+        run: |
+          find -name 'coredump*' -exec ${{github.workspace}}/skupper-router/scripts/gha_analyze_coredump.sh {} \;
+
       - name: Upload test results
         uses: actions/upload-artifact@v2
         if: ${{ ! cancelled() }}
@@ -471,7 +483,7 @@ jobs:
         with:
           name: cores_${{matrix.container}}_${{matrix.buildType}}_${{matrix.runtimeCheck}}_${{matrix.protonGitRef}}_${{matrix.shard}}
           path: |
-            **/core
+            **/coredump*
 
   python:
     name: 'Python Checker (${{ matrix.os }})'

cmake/RuntimeChecks.cmake

@@ -138,9 +138,9 @@ elseif(RUNTIME_CHECK STREQUAL "asan" OR RUNTIME_CHECK STREQUAL "hwasan")
   # `detect_leaks=1` is set by default where it is available; better not to set it conditionally ourselves
   # https://github.com/openSUSE/systemd/blob/1270e56526cd5a3f485ae2aba975345c38860d37/docs/TESTING_WITH_SANITIZERS.md
   # TODO(DISPATCH-2148) re-enable odr violation detection when Proton linking issue in test-sender is fixed
-  set(RUNTIME_ASAN_ENV_OPTIONS "detect_odr_violation=0 strict_string_checks=1 detect_stack_use_after_return=1 check_initialization_order=1 strict_init_order=1 detect_invalid_pointer_pairs=2 suppressions=${CMAKE_SOURCE_DIR}/tests/asan.supp")
-  set(RUNTIME_LSAN_ENV_OPTIONS "suppressions=${CMAKE_BINARY_DIR}/tests/lsan.supp")
-  set(RUNTIME_UBSAN_ENV_OPTIONS "print_stacktrace=1 print_summary=1")
+  set(RUNTIME_ASAN_ENV_OPTIONS "disable_coredump=0 detect_odr_violation=0 strict_string_checks=1 detect_stack_use_after_return=1 check_initialization_order=1 strict_init_order=1 detect_invalid_pointer_pairs=2 suppressions=${CMAKE_SOURCE_DIR}/tests/asan.supp")
+  set(RUNTIME_LSAN_ENV_OPTIONS "disable_coredump=0 suppressions=${CMAKE_BINARY_DIR}/tests/lsan.supp")
+  set(RUNTIME_UBSAN_ENV_OPTIONS "disable_coredump=0 print_stacktrace=1 print_summary=1")
 
 elseif(RUNTIME_CHECK STREQUAL "tsan")
   assert_has_sanitizers()
@@ -150,7 +150,7 @@ elseif(RUNTIME_CHECK STREQUAL "tsan")
   endif(TSAN_LIBRARY-NOTFOUND)
   message(STATUS "Runtime race checker: gcc/clang thread sanitizer")
   set(SANITIZE_FLAGS "-g -fno-omit-frame-pointer -fsanitize=thread")
-  set(RUNTIME_TSAN_ENV_OPTIONS "history_size=4 second_deadlock_stack=1 suppressions=${CMAKE_SOURCE_DIR}/tests/tsan.supp")
+  set(RUNTIME_TSAN_ENV_OPTIONS "disable_coredump=0 history_size=4 second_deadlock_stack=1 suppressions=${CMAKE_SOURCE_DIR}/tests/tsan.supp")
 
 elseif(RUNTIME_CHECK)
   message(FATAL_ERROR "'RUNTIME_CHECK=${RUNTIME_CHECK}' is invalid, valid values: ${runtime_checks}")

scripts/gha_analyze_coredump.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+set -Exeuo pipefail
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# args
+corefile=$(realpath "${1}")
+
+# EXECFN seems to be most reliable way to find excutable, other ways may give truncated path
+executable=$(file "${corefile}" | python3 -c "import re; print( re.search(\"execfn: '([^']+)'\", input())[1] )")
+
+# new gdb can automatically download debuginfo as needed
+#  see https://www.redhat.com/en/blog/how-debuginfod-project-evolved-2021
+export DEBUGINFOD_URLS=https://debuginfod.elfutils.org
+
+# despite debuginfod, it still makes sense to install python3-debuginfo
+#  because it provides py-bt command, see https://fedoraproject.org/wiki/Features/EasierPythonDebugging
+
+# executable path stored in corefile might be relative; change directory to handle that
+pushd "$(dirname "${corefile}")"
+gdb "${executable}" \
+  --core "${corefile}" \
+  -iex 'set pagination off' -iex 'set debuginfod enabled on' \
+  -ex 'thread apply all bt' -ex 'thread apply all py-bt' \
+  --batch
+popd