diff --git a/.github/workflows/abi.yaml b/.github/workflows/abi.yaml index fcc56daf8..e47578923 100644 --- a/.github/workflows/abi.yaml +++ b/.github/workflows/abi.yaml @@ -134,8 +134,8 @@ jobs: id: collectlogs run: | sudo chmod a+rw . - sudo find . -name regression.diffs -exec cat {} + > regression.log - sudo find . -name postmaster.log -exec cat {} + > postgres.log + sudo find build_abi -name regression.diffs -exec cat {} + > regression.log + sudo find build_abi -name postmaster.log -exec cat {} + > postmaster.log if [[ -s regression.log ]]; then echo "regression_diff=true" >>$GITHUB_OUTPUT; fi grep -e 'FAILED' -e 'failed (ignored)' -e 'not ok' installcheck.log || true cat regression.log @@ -152,5 +152,5 @@ jobs: uses: actions/upload-artifact@v4 with: name: PostgreSQL log ABI Breakage ${{ matrix.dir }} PG${{ matrix.pg }} - path: postgres.log + path: postmaster.log diff --git a/.github/workflows/libfuzzer.yaml b/.github/workflows/libfuzzer.yaml index d7d8bffbb..d4a164ef3 100644 --- a/.github/workflows/libfuzzer.yaml +++ b/.github/workflows/libfuzzer.yaml @@ -28,7 +28,7 @@ jobs: # If needed, install them before opening the core dump. sudo apt-get update sudo apt-get install 7zip clang lld llvm flex bison libipc-run-perl \ - libtest-most-perl tree + libtest-most-perl tree jq - name: Checkout TimescaleDB uses: actions/checkout@v4 @@ -134,6 +134,7 @@ jobs: env: PG_SRC_DIR: pgbuild PG_INSTALL_DIR: postgresql + JOB_NAME: Fuzz decompression ${{ matrix.case.algo }} ${{ matrix.case.pgtype }} ${{ matrix.case.bulk && 'bulk' || 'rowbyrow' }} steps: - name: Install Linux dependencies @@ -204,7 +205,9 @@ jobs: export PGPORT=5432 export PGDATABASE=postgres export PATH=$HOME/$PG_INSTALL_DIR/bin:$PATH - pg_ctl -l postgres.log start + pg_ctl -o "-clogging_collector=true" -o "-clog_destination=jsonlog,stderr" \ + -o "-clog_directory=$(readlink -f .)" -o "-clog_filename=postmaster.log" \ + -o "-clog_error_verbosity=verbose" start psql -c "create extension timescaledb;" @@ -265,7 +268,7 @@ jobs: [ $errors -eq 0 ] || exit 1 # Shouldn't have any WARNINGS in the log. - ! grep -F "] WARNING: " postgres.log + ! grep -F "] WARNING: " postmaster.log # Check that the server is still alive. psql -c "select 1" @@ -285,7 +288,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: PostgreSQL log for ${{ steps.config.outputs.name }} - path: postgres.log + path: postgres.* - name: Save fuzzer-generated crash cases if: always() @@ -350,7 +353,6 @@ jobs: bt full " 2>&1 | tee stacktrace.log ./scripts/bundle_coredumps.sh - grep -C40 "was terminated by signal" postgres.log > postgres-failure.log ||: exit 1 # Fail the job if we have core dumps. - name: Upload core dumps @@ -359,3 +361,24 @@ jobs: with: name: Coredumps for ${{ steps.config.outputs.name }} path: coredumps + + - name: Upload test results to the database + if: always() + env: + CI_STATS_DB: ${{ secrets.CI_STATS_DB }} + GITHUB_EVENT_NAME: ${{ github.event_name }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + JOB_STATUS: ${{ job.status }} + run: | + if [[ "${{ github.event_name }}" == "pull_request" ]] ; + then + GITHUB_PR_NUMBER="${{ github.event.number }}" + else + GITHUB_PR_NUMBER=0 + fi + export GITHUB_PR_NUMBER + scripts/upload_ci_stats.sh diff --git a/.github/workflows/linux-32bit-build-and-test.yaml b/.github/workflows/linux-32bit-build-and-test.yaml index 7a5d9cab0..ed3cdacc0 100644 --- a/.github/workflows/linux-32bit-build-and-test.yaml +++ b/.github/workflows/linux-32bit-build-and-test.yaml @@ -142,8 +142,8 @@ jobs: id: collectlogs shell: bash run: | - find . -name regression.diffs -exec cat {} + > regression.log - find . -name postmaster.log -exec cat {} + > postgres.log + find build -name regression.diffs -exec cat {} + > regression.log + find build -name postmaster.log -exec cat {} + > postmaster.log if [[ -s regression.log ]]; then echo "regression_diff=true" >>$GITHUB_OUTPUT; fi grep -e 'FAILED' -e 'failed (ignored)' -e 'not ok' installcheck.log || true cat regression.log @@ -205,7 +205,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: PostgreSQL log linux-i386 PG${{ matrix.pg }} - path: postgres.log + path: postmaster.log - name: Save TAP test logs if: always() diff --git a/.github/workflows/linux-build-and-test.yaml b/.github/workflows/linux-build-and-test.yaml index 8b2eac6ac..6fe6726c6 100644 --- a/.github/workflows/linux-build-and-test.yaml +++ b/.github/workflows/linux-build-and-test.yaml @@ -144,7 +144,8 @@ jobs: run: | BUILD_DIR=nossl ./bootstrap -DCMAKE_BUILD_TYPE=Debug \ -DPG_SOURCE_DIR=~/$PG_SRC_DIR -DPG_PATH=~/$PG_INSTALL_DIR \ - ${{ matrix.tsdb_build_args }} -DCODECOVERAGE=${{ matrix.coverage }} -DUSE_OPENSSL=OFF + ${{ matrix.tsdb_build_args }} -DCODECOVERAGE=${{ matrix.coverage }} -DUSE_OPENSSL=OFF \ + -DTEST_PG_LOG_DIRECTORY="$(readlink -f .)" make -j $MAKE_JOBS -C nossl make -C nossl install make -C nossl regresscheck TESTS=telemetry @@ -153,7 +154,8 @@ jobs: run: | ./bootstrap -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ -DPG_SOURCE_DIR=~/$PG_SRC_DIR -DPG_PATH=~/$PG_INSTALL_DIR \ - ${{ matrix.tsdb_build_args }} -DCODECOVERAGE=${{ matrix.coverage }} + ${{ matrix.tsdb_build_args }} -DCODECOVERAGE=${{ matrix.coverage }} \ + -DTEST_PG_LOG_DIRECTORY="$(readlink -f .)" make -j $MAKE_JOBS -C build make -C build install @@ -195,7 +197,7 @@ jobs: id: collectlogs run: | find . -name regression.diffs -exec cat {} + > regression.log - find . -name postmaster.log -exec cat {} + > postgres.log + if [[ "${{ runner.os }}" == "Linux" ]] ; then # wait in case there are in-progress coredumps sleep 10 @@ -223,7 +225,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: PostgreSQL log ${{ matrix.os }} ${{ matrix.name }} ${{ matrix.pg }} - path: postgres.log + path: postmaster.* - name: Stack trace Linux if: always() && steps.collectlogs.outputs.coredumps == 'true' && runner.os == 'Linux' @@ -248,7 +250,6 @@ jobs: info locals " 2>&1 | tee stacktrace.log ./scripts/bundle_coredumps.sh - grep -C40 "was terminated by signal" postgres.log > postgres-failure.log ||: - name: Stack trace macOS if: always() && steps.collectlogs.outputs.coredumps == 'true' && runner.os == 'macOS' @@ -275,7 +276,7 @@ jobs: # Don't upload the results of the flaky check, because the db schema only # supports running one test once per job. Also disable uploading of the # tests on macOS till the next MINOR release of PG - if: always() && (! contains(matrix.name, 'Flaky')) && (! runner.os == 'macOS') + if: always() && (! contains(matrix.name, 'Flaky')) && ( runner.os != 'macOS' ) env: # GitHub Actions allow you neither to use the env context for the job name, # nor to access the job name from the step context, so we have to diff --git a/.github/workflows/sanitizer-build-and-test.yaml b/.github/workflows/sanitizer-build-and-test.yaml index 261455cd4..bc9f20f98 100644 --- a/.github/workflows/sanitizer-build-and-test.yaml +++ b/.github/workflows/sanitizer-build-and-test.yaml @@ -140,7 +140,8 @@ jobs: - name: Build TimescaleDB run: | ./bootstrap -DCMAKE_BUILD_TYPE=Debug -DPG_SOURCE_DIR=~/$PG_SRC_DIR \ - -DPG_PATH=~/$PG_INSTALL_DIR -DCODECOVERAGE=OFF -DREQUIRE_ALL_TESTS=ON -DTEST_GROUP_SIZE=5 + -DPG_PATH=~/$PG_INSTALL_DIR -DCODECOVERAGE=OFF -DREQUIRE_ALL_TESTS=ON \ + -DTEST_GROUP_SIZE=5 -DTEST_PG_LOG_DIRECTORY="$(readlink -f .)" make -j$(nproc) -C build make -C build install @@ -157,7 +158,6 @@ jobs: id: collectlogs run: | find . -name regression.diffs -exec cat {} + > regression.log - find . -name postmaster.log -exec cat {} + > postgres.log if [[ "${{ runner.os }}" == "Linux" ]] ; then # wait in case there are in-progress coredumps sleep 10 @@ -183,7 +183,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: PostgreSQL log ${{ matrix.os }} ${{ env.name }} ${{ matrix.pg }} - path: postgres.log + path: postmaster.* - name: Stack trace if: always() && steps.collectlogs.outputs.coredumps == 'true' @@ -202,7 +202,6 @@ jobs: bt full " 2>&1 | tee stacktrace.log ./scripts/bundle_coredumps.sh - grep -C40 "was terminated by signal" postgres.log > postgres-failure.log ||: - name: Show sanitizer logs if: always() diff --git a/.github/workflows/sqlsmith.yaml b/.github/workflows/sqlsmith.yaml index a211e7f03..8b4d6cb43 100644 --- a/.github/workflows/sqlsmith.yaml +++ b/.github/workflows/sqlsmith.yaml @@ -34,7 +34,7 @@ jobs: sudo apt-get update sudo apt-get install gnupg systemd-coredump gdb postgresql-common \ libkrb5-dev build-essential autoconf autoconf-archive \ - libboost-regex-dev libsqlite3-dev + libboost-regex-dev libsqlite3-dev jq yes | sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh sudo apt-get purge postgresql* @@ -75,7 +75,9 @@ jobs: /usr/lib/postgresql/${{ matrix.pg }}/bin/pg_ctl -D ~/pgdata start \ -o "-cshared_preload_libraries=timescaledb" -o "-cmax_connections=200" \ -o "-cmax_prepared_transactions=100" -o "-cunix_socket_directories=/tmp" \ - -o "-clog_statement=all" -l postgres.log + -o "-clog_statement=all" -o "-clogging_collector=true" \ + -o "-clog_destination=jsonlog,stderr" -o "-clog_directory=$(readlink -f .)" \ + -o "-clog_error_verbosity=verbose" -o "-clog_filename=postmaster.log" psql -h /tmp postgres -c 'CREATE DATABASE smith;' psql -h /tmp smith -c 'CREATE EXTENSION timescaledb;' psql -h /tmp smith -c '\i ${{ github.workspace }}/tsl/test/shared/sql/include/shared_setup.sql' @@ -100,8 +102,6 @@ jobs: 2>&1 | tee -a sqlsmith.log psql "host=/tmp dbname=smith" -c "select 1" - - truncate --size=0 postgres.log done - name: Check for coredumps @@ -146,7 +146,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: PostgreSQL log for PG${{ matrix.pg }} - path: postgres.log + path: postgres.* - name: Upload test results to the database if: always() diff --git a/.github/workflows/windows-build-and-test.yaml b/.github/workflows/windows-build-and-test.yaml index 5a0d4cbdd..db28fabda 100644 --- a/.github/workflows/windows-build-and-test.yaml +++ b/.github/workflows/windows-build-and-test.yaml @@ -149,7 +149,7 @@ jobs: copy build_win/test/postgresql.conf ${{ env.PGDATA }} copy build_win/test/pg_hba.conf ${{ env.PGDATA }} icacls . /grant runneradmin:F /T - ~/PostgreSQL/${{ matrix.pg }}/bin/pg_ctl start -o "${{ matrix.pg_config }}" --log=postgres.log + ~/PostgreSQL/${{ matrix.pg }}/bin/pg_ctl start -o "${{ matrix.pg_config }}" --log=postmaster.log ~/PostgreSQL/${{ matrix.pg }}/bin/pg_isready -U postgres -d postgres --timeout=30 ~/PostgreSQL/${{ matrix.pg }}/bin/psql -U postgres -d postgres -c 'CREATE USER root SUPERUSER LOGIN;' echo "PG version:" @@ -191,7 +191,7 @@ jobs: ~/PostgreSQL/${{ matrix.pg }}/bin/initdb -U postgres -A trust --locale=en_US --encoding=UTF8 copy build_win/tsl/test/postgresql.conf ${{ env.PGDATA }} copy build_win/tsl/test/pg_hba.conf ${{ env.PGDATA }} - ~/PostgreSQL/${{ matrix.pg }}/bin/pg_ctl start -o "${{ matrix.pg_config }}" --log="postgres.log" + ~/PostgreSQL/${{ matrix.pg }}/bin/pg_ctl start -o "${{ matrix.pg_config }}" --log="postmaster.log" ~/PostgreSQL/${{ matrix.pg }}/bin/pg_isready -U postgres -d postgres --timeout=30 ~/PostgreSQL/${{ matrix.pg }}/bin/psql -U postgres -d postgres -c 'CREATE USER root SUPERUSER LOGIN;' @@ -237,7 +237,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: PostgreSQL ${{ matrix.pg }} log ${{ matrix.os }} ${{ matrix.build_type }} Build - path: postgres.log + path: postmaster.log - name: Upload test results to the database if: always() diff --git a/scripts/upload_ci_stats.sh b/scripts/upload_ci_stats.sh index e6b6ba7e8..8bbfc924a 100755 --- a/scripts/upload_ci_stats.sh +++ b/scripts/upload_ci_stats.sh @@ -56,6 +56,15 @@ create table log( create unique index on log(job_date, test_name); select create_hypertable('log', 'job_date'); + +create table ipe( + job_date timestamptz, + error text, + location text, + statement text +); + +select create_hypertable('ipe', 'job_date'); " # Create the job record. @@ -149,6 +158,15 @@ do mv "$x.tmp" "$x" done +# Save a snippet of logs where a backend was terminated by signal. +grep -C40 "was terminated by signal" postmaster.log > postgres-failure.log ||: + +# Find internal program errors in Postgres logs. +jq 'select(.state_code == "XX000" and .error_severity != "LOG") + | [env.JOB_DATE, .message, .func_name, .statement] | @tsv +' -r postmaster.json > ipe.tsv ||: +"${PSQL[@]}" -c "\copy ipe from ipe.tsv" + # Upload the logs. # Note that the sanitizer setting log_path means "write logs to 'log_path.pid'". for x in sanitizer* sanitizer/* {sqlsmith/sqlsmith,sanitizer,stacktrace,postgres-failure}.log *.diff diff --git a/test/postgresql.conf.in b/test/postgresql.conf.in index 85348d0e1..cb03e9c0f 100644 --- a/test/postgresql.conf.in +++ b/test/postgresql.conf.in @@ -1,20 +1,27 @@ # NOTE: any changes here require changes to tsl/test/postgresql.conf. Its prefix # must be the same as this file. -shared_preload_libraries=timescaledb -max_worker_processes=24 autovacuum=false -random_page_cost=1.0 -timezone='US/Pacific' datestyle='Postgres, MDY' -timescaledb.license='apache' -@TELEMETRY_DEFAULT_SETTING@ +hba_file='@TEST_PG_HBA_FILE@' +log_destination='@TEST_PG_LOG_DESTINATION@' +log_directory='@TEST_PG_LOG_DIRECTORY@' +log_filename='postmaster.log' +log_line_prefix='%m: %u [%p] %d ' +logging_collector=true +max_worker_processes=24 +random_page_cost=1.0 +shared_preload_libraries=timescaledb timescaledb.last_tuned='1971-02-03 04:05:06.789012 -0300' timescaledb.last_tuned_version='0.0.1' +timescaledb.passfile='@TEST_PASSFILE@' timescaledb_telemetry.cloud='ci' -log_line_prefix='%m: %u [%p] %d ' +timezone='US/Pacific' + # Set extra_float_digits=0 to retain the pre PG12 rounding behaviour # of floating point numbers, which are needed to make our tests work. extra_float_digits=0 -timescaledb.passfile='@TEST_PASSFILE@' -hba_file='@TEST_PG_HBA_FILE@' + +@TELEMETRY_DEFAULT_SETTING@ + +timescaledb.license='apache' diff --git a/test/test-defs.cmake b/test/test-defs.cmake index cff52d559..6b9ba043e 100644 --- a/test/test-defs.cmake +++ b/test/test-defs.cmake @@ -52,6 +52,19 @@ endif() configure_file(${PRIMARY_TEST_DIR}/pg_hba.conf.in pg_hba.conf) set(TEST_PG_HBA_FILE ${TEST_OUTPUT_DIR}/pg_hba.conf) +# Enable json logs that are supported since PG15, to get additional information +# about errors from them for CI database. +if(PG_VERSION_MAJOR LESS 15) + set(TEST_PG_LOG_DESTINATION stderr) +else() + set(TEST_PG_LOG_DESTINATION jsonlog,stderr) +endif() + +# This variable is set differently in CI. We use it to save the logs outside the +# tmp instance, because it is deleted by pg_regress on successful test +# completion, and we want to run some additional checks on the logs in any case. +option(TEST_PG_LOG_DIRECTORY "Log directory for regression tests" "log") + if(USE_TELEMETRY) set(TELEMETRY_DEFAULT_SETTING "timescaledb.telemetry_level=off") else() diff --git a/tsl/test/postgresql.conf.in b/tsl/test/postgresql.conf.in index fcf879a41..8b043fb44 100644 --- a/tsl/test/postgresql.conf.in +++ b/tsl/test/postgresql.conf.in @@ -1,20 +1,27 @@ # This section has to be equivalent to test/postgresql.conf -shared_preload_libraries=timescaledb -max_worker_processes=24 + autovacuum=false -random_page_cost=1.0 -timezone='US/Pacific' datestyle='Postgres, MDY' -@TELEMETRY_DEFAULT_SETTING@ +hba_file='@TEST_PG_HBA_FILE@' +log_destination='@TEST_PG_LOG_DESTINATION@' +log_directory='@TEST_PG_LOG_DIRECTORY@' +log_filename='postmaster.log' +log_line_prefix='%m: %u [%p] %d ' +logging_collector=true +max_worker_processes=24 +random_page_cost=1.0 +shared_preload_libraries=timescaledb timescaledb.last_tuned='1971-02-03 04:05:06.789012 -0300' timescaledb.last_tuned_version='0.0.1' +timescaledb.passfile='@TEST_PASSFILE@' timescaledb_telemetry.cloud='ci' -log_line_prefix='%m %u [%p] %d ' +timezone='US/Pacific' + # Set extra_float_digits=0 to retain the pre PG12 rounding behaviour # of floating point numbers, which are needed to make our tests work. extra_float_digits=0 -timescaledb.passfile='@TEST_PASSFILE@' -hba_file='@TEST_PG_HBA_FILE@' + +@TELEMETRY_DEFAULT_SETTING@ # This section adds additional options required by TSL. timescaledb.license='timescale'