Add pg_upgrade test to CI

It's long we need to have binary upgrade tests in our CI. One example
is #6935 (and others) that can be prevented if we have such kind of
test in our CI.

To implement the `pg_upgrade` test we used the python Testing Framework
for PostgreSQL (https://github.com/postgrespro/testgres).

Unfortunately the testing framework don't have the ability to retain the
pg_upgrade log files after a successful execution, then we created a PR
to make it possible and we'll use this patched version until we get the
code merged and released on upstream.

https://github.com/postgrespro/testgres/pull/125

Closes #3868 #4428

.github/workflows/code_style.yaml

@@ -107,8 +107,24 @@ jobs:
             psutil pygithub pglast
           pip list
           pip list --user
+
+      # Using e375302a until 1.10.1 get released including the following PR
+      # https://github.com/postgrespro/testgres/pull/125
+      - name: Checkout testgres
+        uses: actions/checkout@v4
+        with:
+          repository: 'postgrespro/testgres'
+          path: 'testgres'
+          ref: e375302a114cd4df3ceed54d6526f250c44c08e7
+
+      - name: Build and install testgres
+        run: |
+          cd testgres
+          python setup.py install --user
+
       - name: Checkout source
         uses: actions/checkout@v4
+
       - name: Run prospector
         run: |
           find . -type f -name "*.py" -print -exec prospector {} + -exec black {} +

.github/workflows/pg_upgrade-test.yaml

@@ -0,0 +1,117 @@
+name: pg_upgrade test
+"on":
+  push:
+    branches:
+      - main
+      - prerelease_test
+  pull_request:
+
+jobs:
+  pg_upgrade_test:
+    name: pg_upgrade test from PG${{ matrix.pg_version_old }} to PG${{ matrix.pg_version_new }}
+    runs-on: 'ubuntu-latest'
+    strategy:
+      matrix:
+        include:
+          - pg_version_old: 14 # 14 to 15
+            pg_version_new: 15
+          - pg_version_old: 14 # 14 to 16
+            pg_version_new: 16
+          - pg_version_old: 15 # 15 to 16
+            pg_version_new: 16
+      fail-fast: false
+    env:
+      OUTPUT_DIR: ${{ github.workspace }}/pg_upgrade_test
+
+    steps:
+      - name: Install Linux Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install pip postgresql-common libkrb5-dev
+
+      # Using e375302a until 1.10.1 get released including the following PR
+      # https://github.com/postgrespro/testgres/pull/125
+      - name: Checkout testgres
+        uses: actions/checkout@v4
+        with:
+          repository: 'postgrespro/testgres'
+          path: 'testgres'
+          ref: e375302a114cd4df3ceed54d6526f250c44c08e7
+
+      - name: Build and install testgres
+        run: |
+          cd testgres
+          python setup.py install --user
+
+      - name: Install PostgreSQL ${{ matrix.pg_version_old}} and ${{ matrix.pg_version_new }}
+        run: |
+          yes | sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh
+          echo "deb https://packagecloud.io/timescale/timescaledb/ubuntu/ $(lsb_release -c -s) main" | sudo tee /etc/apt/sources.list.d/timescaledb.list
+          wget --quiet -O - https://packagecloud.io/timescale/timescaledb/gpgkey | sudo apt-key add -
+          sudo apt-get update
+          sudo apt-get install -y \
+            postgresql-${{ matrix.pg_version_old }} postgresql-server-dev-${{ matrix.pg_version_old }} \
+            postgresql-${{ matrix.pg_version_new }} postgresql-server-dev-${{ matrix.pg_version_new }}
+          sudo apt-get install -y --no-install-recommends \
+            timescaledb-2-postgresql-${{ matrix.pg_version_old }} \
+            timescaledb-2-postgresql-${{ matrix.pg_version_new }}
+
+      - name: Checkout TimescaleDB
+        uses: actions/checkout@v4
+
+      - name: Build and install TimescaleDB on PostgreSQL ${{ matrix.pg_version_old}}
+        env:
+          BUILD_DIR: pg${{ matrix.pg_version_old }}
+        run: |
+          PATH="/usr/lib/postgresql/${{ matrix.pg_version_old }}/bin:$PATH"
+          ./bootstrap -DCMAKE_BUILD_TYPE=Release -DWARNINGS_AS_ERRORS=OFF -DASSERTIONS=ON -DLINTER=OFF -DGENERATE_DOWNGRADE_SCRIPT=OFF -DREGRESS_CHECKS=OFF -DTAP_CHECKS=OFF
+          make -j -C pg${{ matrix.pg_version_old }}
+          sudo make -j -C pg${{ matrix.pg_version_old }} install
+
+      - name: Build and install TimescaleDB on PostgreSQL ${{ matrix.pg_version_new}}
+        env:
+          BUILD_DIR: pg${{ matrix.pg_version_new }}
+        run: |
+          PATH="/usr/lib/postgresql/${{ matrix.pg_version_new }}/bin:$PATH"
+          ./bootstrap -DCMAKE_BUILD_TYPE=Release -DWARNINGS_AS_ERRORS=OFF -DASSERTIONS=ON -DLINTER=OFF -DGENERATE_DOWNGRADE_SCRIPT=OFF -DREGRESS_CHECKS=OFF -DTAP_CHECKS=OFF
+          make -j -C pg${{ matrix.pg_version_new }}
+          sudo make -j -C pg${{ matrix.pg_version_new }} install
+
+      - name: Run pg_upgrade test
+        env:
+          PGVERSIONOLD: ${{ matrix.pg_version_old }}
+          PGVERSIONNEW: ${{ matrix.pg_version_new }}
+          DIFFFILE: ${{ env.OUTPUT_DIR }}/upgrade_check.diff
+        run: |
+          scripts/test_pg_upgrade.py
+          diff -u \
+            "${OUTPUT_DIR}/post.pg${PGVERSIONOLD}.log" \
+            "${OUTPUT_DIR}/post.pg${PGVERSIONNEW}.log" | \
+            tee "${DIFFFILE}"
+          if [[ -s "${DIFFFILE}" ]]; then
+            echo "pg_upgrade test for ${PGVERSIONOLD} -> ${PGVERSIONNEW} failed"
+            exit 1
+          fi
+
+      - name: Show pg_upgrade diffs
+        if: always()
+        env:
+          DIFFFILE: ${{ env.OUTPUT_DIR }}/upgrade_check.diff
+          DIROLD: pg${{ matrix.pg_version_old }}
+          DIRNEW: pg${{ matrix.pg_version_new }}
+        run: |
+          cd ${OUTPUT_DIR}
+          cat ${DIFFFILE}
+          tar czf /tmp/pg_upgrade_artifacts.tgz \
+            ${DIFFFILE} \
+            ${OUTPUT_DIR}/*.log \
+            ${OUTPUT_DIR}/${DIROLD}/logs/* \
+            ${OUTPUT_DIR}/${DIRNEW}/logs/* \
+            ${OUTPUT_DIR}/${DIRNEW}/data/pg_upgrade_output.d/*
+
+      - name: Upload pg_upgrade logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: pg_upgrade logs from ${{ matrix.pg_version_old }} to ${{ matrix.pg_version_new }}
+          path: /tmp/pg_upgrade_artifacts.tgz

scripts/test_pg_upgrade.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+
+from shutil import rmtree
+from testgres import get_new_node, PostgresNode
+
+
+# Accessor functions
+def set_default_conf(node: PostgresNode, unix_socket_dir):
+    node.default_conf(fsync=True, allow_streaming=False, allow_logical=False)
+    node.append_conf(unix_socket_directories=unix_socket_dir)
+    node.append_conf(timezone="GMT")
+    node.append_conf(client_min_messages="warning")
+    node.append_conf(max_prepared_transactions="100")
+    node.append_conf(max_worker_processes="0")
+    node.append_conf(shared_preload_libraries="timescaledb")
+    node.append_conf("timescaledb.telemetry_level=off")
+
+
+def write_bytes_to_file(filename, nbytes):
+    with open(filename, "wb") as f:
+        f.write(nbytes)
+        f.close()
+
+
+def getenv_or_error(envvar):
+    return os.getenv(envvar) or sys.exit(f"Environment variable {envvar} not defined")
+
+
+def getenv_or_default(envvar, default):
+    return os.getenv(envvar) or default
+
+
+def initialize_node(working_dir, prefix, port, bin_dir, base_dir):
+    node = get_new_node(prefix=prefix, port=port, bin_dir=bin_dir, base_dir=base_dir)
+    node.init()
+    set_default_conf(node, working_dir)
+    return node
+
+
+# Globals
+pg_version_old = getenv_or_error("PGVERSIONOLD")
+pg_version_new = getenv_or_error("PGVERSIONNEW")
+
+pg_node_old = f"pg{pg_version_old}"
+pg_node_new = f"pg{pg_version_new}"
+
+pg_port_old = getenv_or_default("PGPORTOLD", "54321")
+pg_port_new = getenv_or_default("PGPORTNEW", "54322")
+
+test_version = getenv_or_default("TEST_VERSION", "v8")
+
+pg_bin_old = getenv_or_default("PGBINOLD", f"/usr/lib/postgresql/{pg_version_old}/bin")
+pg_bin_new = getenv_or_default("PGBINNEW", f"/usr/lib/postgresql/{pg_version_new}/bin")
+
+working_dir = getenv_or_default(
+    "OUTPUT_DIR",
+    f"/tmp/pg_upgrade_output/{pg_version_old}_to_{pg_version_new}",
+)
+
+pg_data_old = getenv_or_default("PGDATAOLD", f"{working_dir}/{pg_node_old}")
+pg_data_new = getenv_or_default("PGDATAOLD", f"{working_dir}/{pg_node_new}")
+
+pg_database_test = getenv_or_default("PGDATABASE", "pg_upgrade_test")
+
+if os.path.exists(working_dir):
+    rmtree(working_dir)
+os.makedirs(working_dir)
+
+# Real testing code
+print(f"Initializing nodes {pg_node_old} and {pg_node_new}")
+node_old = initialize_node(
+    working_dir, pg_node_old, pg_port_old, pg_bin_old, pg_data_old
+)
+node_old.start()
+
+node_new = initialize_node(
+    working_dir, pg_node_new, pg_port_new, pg_bin_new, pg_data_new
+)
+
+print(f"Creating {pg_database_test} database on node {pg_node_old}")
+node_old.safe_psql(filename="test/sql/updates/setup.roles.sql")
+node_old.safe_psql(query=f"CREATE DATABASE {pg_database_test};")
+node_old.safe_psql(dbname=pg_database_test, query="CREATE EXTENSION timescaledb;")
+node_old.safe_psql(dbname=pg_database_test, filename="test/sql/updates/pre.testing.sql")
+node_old.safe_psql(
+    dbname=pg_database_test,
+    filename=f"test/sql/updates/setup.{test_version}.sql",
+)
+node_old.safe_psql(dbname=pg_database_test, query="CHECKPOINT")
+node_old.safe_psql(dbname=pg_database_test, filename="test/sql/updates/setup.check.sql")
+
+# Run new psql over the old node to have the same psql output
+node_new.port = pg_port_old
+(code, old_out, old_err) = node_new.psql(
+    dbname=pg_database_test, filename="test/sql/updates/post.pg_upgrade.sql"
+)
+node_new.port = pg_port_new
+
+# Save output to log
+write_bytes_to_file(f"{working_dir}/post.{pg_node_old}.log", old_out)
+node_old.stop()
+
+print(f"Upgrading node {pg_node_old} to {pg_node_new}")
+res = node_new.upgrade_from(old_node=node_old, options=["--retain"])
+node_new.start()
+
+(code, new_out, new_err) = node_new.psql(
+    dbname=pg_database_test,
+    filename="test/sql/updates/post.pg_upgrade.sql",
+)
+
+# Save output to log
+write_bytes_to_file(f"{working_dir}/post.pg{pg_version_new}.log", new_out)
+node_new.stop()
+
+print(f"Finish upgrading node {pg_node_old} to {pg_node_new}")

test/sql/updates/post.pg_upgrade.sql

@@ -0,0 +1,10 @@
+-- This file and its contents are licensed under the Apache License 2.0.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-APACHE for a copy of the license.
+
+\pset format aligned
+\pset tuples_only off
+
+\ir post.catalog.sql
+\ir post.policies.sql
+\ir post.functions.sql