diff --git a/.github/workflows/changelog-check.yaml b/.github/workflows/changelog-check.yaml
index 48e7c96a1..892e75957 100644
--- a/.github/workflows/changelog-check.yaml
+++ b/.github/workflows/changelog-check.yaml
@@ -33,6 +33,7 @@ jobs:
         shell: bash --norc --noprofile {0}
         env:
           BODY: ${{ github.event.pull_request.body }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
         run: |
           folder=".unreleased"
 
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2f3f7c14e..95213fdc5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -20,7 +20,7 @@ This release includes these noteworthy features:
 
 **Features**
 * #5212 Allow pushdown of reference table joins
-* #5221 Improve Realtime Continuous Aggregate performance
+* #5261 Improve Realtime Continuous Aggregate performance
 * #5252 Improve unique constraint support on compressed hypertables
 * #5339 Support UPDATE/DELETE on compressed hypertables
 * #5344 Enable JOINS for Hierarchical Continuous Aggregates
diff --git a/scripts/check_changelog_format.py b/scripts/check_changelog_format.py
index d60223bbc..f495436aa 100755
--- a/scripts/check_changelog_format.py
+++ b/scripts/check_changelog_format.py
@@ -2,6 +2,7 @@
 
 import sys
 import re
+import os
 
 
 # Check if a line matches any of the specified patterns
@@ -17,13 +18,21 @@ def main():
     # Get the file name from the command line argument
     if len(sys.argv) > 1:
         file_name = sys.argv[1]
+        pr_number_seen = False
+        pr_num_str = f'#{os.environ["PR_NUMBER"]} '
         # Read the file and check non-empty lines
         with open(file_name, "r", encoding="utf-8") as file:
             for line in file:
                 line = line.strip()
+                pr_number_seen |= pr_num_str in line
                 if line and not is_valid_line(line):
                     print(f'Invalid entry in change log: "{line}"')
                     sys.exit(1)
+        if not pr_number_seen:
+            print(
+                f'Expected that the changelog contains a reference to the PR: "{pr_num_str}"'
+            )
+            sys.exit(1)
     else:
         print("Please provide a file name as a command-line argument.")
         sys.exit(1)