Run sanitizer tests on PG12

Change sanitizer test to run on PG12 and make it use the same
infrastructure as the other linux regression tests.

Co-authored-by: Sven Klemm <sven@timescale.com>

.github/workflows/sanitizer-build-and-test.yaml

@@ -0,0 +1,146 @@
+name: Sanitizer test
+on:
+  schedule:
+    # run daily 20:00 on master branch
+    - cron: '0 20 * * *'
+  push:
+    branches:
+      - prerelease_test
+env:
+  name: "Sanitizer"
+  PG_SRC_DIR: "pgbuild"
+  PG_INSTALL_DIR: "postgresql"
+  MAKE_JOBS: 6
+  # llvm-symbolizer crashes so we don't use clang until we resolve that
+  extra_packages: "clang-10 llvm-10 llvm-10-dev llvm-10-tools"
+  llvm_config: "llvm-config-10"
+  CLANG: "clang-10"
+  # gcc-10 is the minimum version that can parse the suppression files barely adequately
+  CC: "gcc-10"
+  CXX: "g++-10"
+  # gcc CFLAGS, disable inlining for function name pattern matching to work for suppressions
+  CFLAGS: "-g -fsanitize=address,undefined -fno-omit-frame-pointer -O1 -fno-inline"
+  CXXFLAGS: "-g -fsanitize=address,undefined -fno-omit-frame-pointer -O1 -fno-inline"
+  # clang CFLAGS
+  #CFLAGS: "-g -fsanitize=address,undefined -fno-omit-frame-pointer -O1 -fno-inline-functions"
+  #CXXFLAGS: "-g -fsanitize=address,undefined -fno-omit-frame-pointer -O1 -fno-inline-functions"
+
+  # We do not link libasan dynamically to avoid problems with libdl and our libraries.
+  # clang does this by default, but we need to explicitly state that for gcc.
+  # static gcc LDFLAGS
+  LDFLAGS: "-fsanitize=address,undefined -static-libasan -static-liblsan -static-libubsan"
+  # static sanitizer clang LDFLAGS or dynamic sanitizer gcc LDFLAGS
+  #LDFLAGS: "-fsanitize=address,undefined"
+  ASAN_OPTIONS: suppressions=${{ github.workspace }}/scripts/suppressions/suppr_asan.txt detect_odr_violation=0 log_path=${{ github.workspace }}/sanitizer.log log_exe_name=true print_suppressions=false exitcode=27 external_symbolizer_path=/usr/lib/llvm-10/bin/llvm-symbolizer
+  LSAN_OPTIONS: suppressions=${{ github.workspace }}/scripts/suppressions/suppr_leak.txt print_suppressions=0 log_path=${{ github.workspace }}/sanitizer.log log_exe_name=true print_suppressions=false exitcode=27 external_symbolizer_path=/usr/lib/llvm-10/bin/llvm-symbolizer
+  UBSAN_OPTIONS: suppressions=${{ github.workspace }}/scripts/suppressions/suppr_ub.txt print_stacktrace=1 halt_on_error=1 log_path=${{ github.workspace }}/sanitizer.log log_exe_name=true print_suppressions=false exitcode=27 external_symbolizer_path=/usr/lib/llvm-10/bin/llvm-symbolizer
+jobs:
+  sanitizer:
+    name: PG${{ matrix.pg }} Sanitizer ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # "os" has to be in the matrix due to a bug in "env": https://github.community/t/how-to-use-env-context/16975
+        os: ["ubuntu-20.04"]
+        pg: ["12.7", "13.3"]
+    steps:
+    - name: Install Linux Dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install flex bison lcov systemd-coredump gdb libipc-run-perl libtest-most-perl ${{ env.extra_packages }}
+
+    # we cache the build directory instead of the install directory here
+    # because extension installation will write files to install directory
+    # leading to a tainted cache
+    - name: Cache PostgreSQL ${{ matrix.pg }}
+      id: cache-postgresql
+      uses: actions/cache@v2
+      with:
+        path: ~/${{ env.PG_SRC_DIR }}
+        key: ${{ matrix.os }}-${{ env.name }}-postgresql-${{ matrix.pg }}-${{ env.CC }}
+
+    - name: Checkout TimescaleDB
+      uses: actions/checkout@v2
+
+    - name: Build PostgreSQL ${{ matrix.pg }} if not in cache
+      if: steps.cache-postgresql.outputs.cache-hit != 'true'
+      run: |
+        wget -q -O postgresql.tar.bz2 https://ftp.postgresql.org/pub/source/v${{ matrix.pg }}/postgresql-${{ matrix.pg }}.tar.bz2
+        mkdir -p ~/$PG_SRC_DIR
+        tar --extract --file postgresql.tar.bz2 --directory ~/$PG_SRC_DIR --strip-components 1
+        cd ~/$PG_SRC_DIR
+        ./configure --prefix=$HOME/$PG_INSTALL_DIR --enable-debug --enable-cassert --with-openssl --without-readline --without-zlib --without-libxml
+        make -j $MAKE_JOBS
+        make -j $MAKE_JOBS -C src/test/isolation
+        make -j $MAKE_JOBS -C contrib/postgres_fdw
+
+    - name: Install PostgreSQL ${{ matrix.pg }}
+      run: |
+        make -C ~/$PG_SRC_DIR install
+        make -C ~/$PG_SRC_DIR/contrib/postgres_fdw install
+
+    - name: Build TimescaleDB
+      run: |
+        ./bootstrap -DCMAKE_BUILD_TYPE=Debug -DPG_SOURCE_DIR=~/$PG_SRC_DIR -DPG_PATH=~/$PG_INSTALL_DIR -DCODECOVERAGE=OFF
+        make -j $MAKE_JOBS -C build
+        make -C build install
+
+    - name: make installcheck
+      run: |
+        set -o pipefail
+        # IGNORE some test since they fail under ASAN. At least the remote_txn
+        # test seems to fail due to a PostgreSQL bug where AbortStartTime in
+        # postmaster.c is not atomic but read/written across signal handlers
+        # and ServerLoop.
+        make -k -C build installcheck SKIPS='remote_txn' IGNORES='bgw_db_scheduler' | tee installcheck.log
+
+    - name: Show regression diffs
+      if: always()
+      id: collectlogs
+      run: |
+        find . -name regression.diffs -exec cat {} + > regression.log
+        find . -name postmaster.log -exec cat {} + > postgres.log
+        if [[ "${{ runner.os }}" == "Linux" ]] ; then
+          # wait in case there are in-progress coredumps
+          sleep 10
+          if coredumpctl -q list >/dev/null; then echo "::set-output name=coredumps::true"; fi
+        fi
+        if [[ -s regression.log ]]; then echo "::set-output name=regression_diff::true"; fi
+        grep -e 'FAILED' -e 'failed (ignored)' installcheck.log || true
+        cat regression.log
+
+    - name: Save regression diffs
+      if: always() && steps.collectlogs.outputs.regression_diff == 'true'
+      uses: actions/upload-artifact@v2
+      with:
+        name: Regression diff ${{ matrix.os }} ${{ env.name }} ${{ matrix.pg }}
+        path: regression.log
+
+    - name: Save postmaster.log
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: PostgreSQL log ${{ matrix.os }} ${{ env.name }} ${{ matrix.pg }}
+        path: postgres.log
+
+    - name: Stack trace
+      if: always() && steps.collectlogs.outputs.coredumps == 'true'
+      run: |
+        echo "bt full" | sudo coredumpctl gdb
+        ./scripts/bundle_coredumps.sh
+        false
+
+    - name: Coredumps
+      if: always() && steps.collectlogs.outputs.coredumps == 'true'
+      uses: actions/upload-artifact@v2
+      with:
+        name: Coredumps ${{ matrix.os }} ${{ env.name }} ${{ matrix.pg }}
+        path: coredumps
+
+    - name: sanitizer logs
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: sanitizer logs ${{ matrix.os }} ${{ env.name }} ${{ matrix.pg }}
+        path: ${{ github.workspace }}/sanitizer.log.*

scripts/export_prefix_check.sh.in

@@ -26,6 +26,13 @@ fi
 #    loader_hello                 used to test that our name collision resistance works
 #    test_symbol_conflict         used to test that our name collision resistance works
 #    tsl_license_update_check     used to check for valid license
+#    __sanitizer.* __sancov.*     llvm sanitizer
+#    _ZN11__sanitizer.*           llvm sanitizer
+#    __asan.* __odr_asan.*        address sanitizer
+#    __ubsan.* _ZN7__ubsan.*      undefined behaviour sanitizer
+#    internal_sigreturn           gcc
+#    OnPrint                      gcc
+#    backtrace_uncompress_zdebug  libbacktrace
 # all of these symbols start with an additional leading '_' on macos
 exports=$(find @CMAKE_BINARY_DIR@ -not -path '*/\.*' -name '*.so' -print0 \
     | xargs -0 @NM@ ${DEFINED_ONLY} ${EXTERN_ONLY} \
@@ -50,6 +57,16 @@ exports=$(find @CMAKE_BINARY_DIR@ -not -path '*/\.*' -name '*.so' -print0 \
         -e '^_\?timescaledb_' \
         -e '^_\?loader_hello$' \
         -e '^_\?test_symbol_conflict$' \
+        -e '^_\?__sanitizer' \
+        -e '^_\?__sancov' \
+        -e '^_\?__asan' \
+        -e '^_\?__odr_asan' \
+        -e '^_\?__ubsan' \
+        -e '^_\?_ZN7__ubsan' \
+        -e '^_\?_ZN11__sanitizer' \
+        -e '^_\?internal_sigreturn$' \
+        -e '^_\?OnPrint$' \
+        -e '^_\?backtrace_uncompress_zdebug$' \
 )
 
 num_exports=$(echo "${exports}" | grep -v '^$' | wc -l)

scripts/suppressions/suppr_leak.txt

@@ -2,6 +2,8 @@ leak:save_ps_display_args
 leak:psql/startup.c
 #don't care about the frontend leaks
 leak:fe_memutils.c
+leak:fe-connect.c
+leak:fe-exec.c
 leak:initdb.c
 #annoingly have to supress strdup because it leaks in PostmasterMain -D option
 leak:strdup
@@ -9,6 +11,7 @@ leak:ProcessConfigFileInternal
 leak:internal_load_library
 
 leak:pg_timezone_abbrev_initialize
+leak:check_timezone_abbreviations
 leak:check_session_authorization
 leak:InitializeGUCOptions
 leak:CheckMyDatabase

scripts/suppressions/suppr_ub.txt

@@ -9,6 +9,18 @@ nonnull-attribute:TransactionIdSetPageStatus
 nonnull-attribute:SerializeTransactionState
 nonnull-attribute:initscan
 nonnull-attribute:SetTransactionSnapshot
-nonnull-attribute:/usr/src/postgresql/src/fe_utils/print.c
+nonnull-attribute:shm_mq_receive
+#care, gcc cannot parse the path, only the filename
+nonnull-attribute:*/src/fe_utils/print.c
+nonnull-attribute:print_aligned_text
+#PG13.3-copyfuncs.c:2374:2: runtime error: null pointer passed as argument 2, which is declared to never be null
+nonnull-attribute:_copyAppendRelInfo
+#PG13.3-copyfuncs.c:1190:2: runtime error: null pointer passed as argument 2, which is declared to never be null
+nonnull-attribute:_copyLimit
+nonnull-attribute:*/src/backend/nodes/copyfuncs.c
+#PG12.7-relcache.c:5960:6: runtime error: null pointer passed as argument 1, which is declared to never be null
+#in write_item postgresql-12.6/src/backend/utils/cache/relcache.c:5960
+#looks like a real postgresql bug
+nonnull-attribute:write_item
 #division by 0, looks like a real postgres ug
 float-divide-by-zero:_bt_vacuum_needs_cleanup