Fix pre-existing tools test failures: RPM signing no-op, docker PG-version drift, EOL images, and PG16/17/18 package-test retarget

.github/workflows/package-tests.yml

@@ -38,7 +38,7 @@ jobs:
           echo "project_version: ${{ github.event.inputs.project_version }}"
           # To be able to test pipeline without triggering with project_version parameter using workflow_dispatch parameter,
           # if workflow_dispatch parameter is empty, 12.1.5 parameter is set to execute pipeline.
-          [ -z ${PROJECT_VERSION} ] && export PROJECT_VERSION=12.1.5
+          [ -z ${PROJECT_VERSION} ] && export PROJECT_VERSION=14.1.0
           POSTGRES_VERSIONS=$(python -m packaging_automation.get_postgres_versions --project_version ${PROJECT_VERSION})
           echo "Postgres Version: ${POSTGRES_VERSIONS}"
           echo "::set-output name=pg_versions::${POSTGRES_VERSIONS}"
@@ -49,12 +49,12 @@ jobs:
       fail-fast: false
       matrix:
         platform:
-          - centos/8
           - el/9
           - ol/9
           - debian/bullseye
           - debian/bookworm
           - ubuntu/jammy
+          - ubuntu/noble
         pg: ${{ fromJson(needs.metadata.outputs.pg_versions) }}
     env:
       PLATFORM: ${{ matrix.platform }}
@@ -73,7 +73,7 @@ jobs:
         run: |
           export PROJECT_VERSION="${{ github.event.inputs.project_version }}"
           echo "Citus Version: ${PROJECT_VERSION} "
-          [ -z ${PROJECT_VERSION} ] && export PROJECT_VERSION=12.1.5
+          [ -z ${PROJECT_VERSION} ] && export PROJECT_VERSION=14.1.0
           python -m  packaging_automation.test_citus_package \
           --project_version "${PROJECT_VERSION}" \
           --os_release ${{ matrix.platform }} \

packaging_automation/citus_package.py

@@ -228,6 +228,12 @@ def sign_packages(
     output_path = f"{input_output_parameters.output_dir}/{sub_folder}"
     deb_files = glob.glob(f"{output_path}/*.deb", recursive=True)
     rpm_files = glob.glob(f"{output_path}/*.rpm", recursive=True)
+    if len(rpm_files) == 0 and len(deb_files) == 0:
+        print(
+            f"WARNING: sign_packages found no .rpm or .deb files under '{output_path}'. "
+            f"Nothing will be signed. If packages were expected here, the sign path does not "
+            f"match the build output path (check the output_dir handling in build_packages)."
+        )
     os.environ["PACKAGING_PASSPHRASE"] = signing_credentials.passphrase
     os.environ["PACKAGING_SECRET_KEY"] = signing_credentials.secret_key
 
@@ -434,9 +440,8 @@ def build_packages(
 
     docker_image_name = get_docker_image_name(platform)
     output_sub_folder = get_release_package_folder_name(os_name, os_version)
-    input_output_parameters.output_dir = (
-        f"{input_output_parameters.output_dir}/{output_sub_folder}"
-    )
+    base_output_dir = input_output_parameters.output_dir
+    input_output_parameters.output_dir = f"{base_output_dir}/{output_sub_folder}"
     for postgres_docker_extension in postgres_docker_extension_iterator:
         print(
             f"Package build for {os_name}-{os_version} for postgres {postgres_docker_extension} started... "
@@ -453,6 +458,12 @@ def build_packages(
             f"Package build for {os_name}-{os_version} for postgres {postgres_docker_extension} finished "
         )
 
+    # Restore the base output dir before signing. build_package mounts output_dir at
+    # /packages, so the packages are produced under "{base_output_dir}/{output_sub_folder}".
+    # sign_packages re-appends the sub_folder to output_dir, so it must receive the base dir
+    # (not the build-time mutated value); otherwise the sign path becomes
+    # "{base}/{sub_folder}/{sub_folder}", matches no packages, and signing is silently skipped.
+    input_output_parameters.output_dir = base_output_dir
     sign_packages(output_sub_folder, signing_credentials, input_output_parameters)
 
 

packaging_automation/common_tool_methods.py

@@ -645,8 +645,25 @@ def rpm_key_matches_summary(key: str, summary: str):
 
 
 def is_rpm_file_signed(file_path: str) -> bool:
-    result = run_with_output(f"rpm -K {file_path}")
-    return result.returncode == 0
+    # `rpm -K` / `--checksig` cannot discriminate signed from unsigned packages: it returns
+    # success ("digests OK") for an unsigned rpm as long as the digests verify. Query the
+    # signature header tags directly instead. Which tag carries the signature depends on the
+    # rpm version that produced it:
+    #   - legacy V3 signatures (e.g. rpm 4.11 / centos:7 signer) populate SIGPGP / SIGGPG
+    #   - header-only signatures (rpm >= 4.16 `rpm --addsign`) populate RSAHEADER
+    # The package is unsigned only when all of these render the literal string "(none)".
+    result = run_with_output(
+        f"rpm -qp --qf '%{{SIGPGP:pgpsig}}|%{{SIGGPG:pgpsig}}|%{{RSAHEADER:pgpsig}}' {file_path}"
+    )
+    if result.returncode != 0:
+        return False
+    output = (
+        result.stdout.decode("ascii", "replace")
+        if isinstance(result.stdout, bytes)
+        else result.stdout
+    )
+    signature_tags = output.strip().split("|")
+    return any(tag.strip() not in ("(none)", "") for tag in signature_tags)
 
 
 def verify_rpm_signature_in_dir(rpm_dir_path: str):

packaging_automation/tests/test_citus_package.py

@@ -1,3 +1,4 @@
+import glob
 import os
 
 import pathlib2
@@ -21,6 +22,7 @@
     delete_rpm_key_by_name,
     get_gpg_fingerprints_by_name,
     get_private_key_by_fingerprint_with_passphrase,
+    is_rpm_file_signed,
     run,
     transform_key_into_base64_str,
     verify_rpm_signature_in_dir,
@@ -59,7 +61,12 @@
 }
 
 TEST_GPG_KEY_NAME = "Citus Data <packaging@citusdata.com>"
-TEST_GPG_KEY_PASSPHRASE = os.getenv("PACKAGING_PASSPHRASE")
+# Use the literal passphrase baked into the throwaway test key
+# (tests/files/gpg/packaging_with_passphrase.gpg -> Passphrase: Citus123) rather
+# than the prod PACKAGING_PASSPHRASE secret, so this unit test stays self-contained
+# and immune to production signing-key/passphrase rotations. Matches the convention
+# already used in test_citus_package_utils.py.
+TEST_GPG_KEY_PASSPHRASE = "Citus123"
 GH_TOKEN = os.getenv("GH_TOKEN")
 PACKAGE_CLOUD_API_TOKEN = os.getenv("PACKAGE_CLOUD_API_TOKEN")
 REPO_CLIENT_SECRET = os.getenv("REPO_CLIENT_SECRET")
@@ -128,6 +135,18 @@ def test_build_packages():
     os_name, os_version = decode_os_and_release(PLATFORM)
     sub_folder = get_release_package_folder_name(os_name, os_version)
     release_output_folder = f"{BASE_OUTPUT_FOLDER}/{sub_folder}"
+    # Regression guard for the sign_packages path-doubling bug: the build output folder and the
+    # folder sign_packages signs in must resolve to the SAME path. If build_packages leaks its
+    # build-time output_dir mutation into sign_packages again, the sign path becomes a doubled
+    # "{sub_folder}/{sub_folder}", matches no packages, and signing is silently skipped, leaving
+    # the produced rpms unsigned. Assert every rpm actually produced at the build path carries a
+    # real signature (no-op for deb/pgxn platforms, which produce no rpm files here).
+    produced_rpms = glob.glob(f"{release_output_folder}/*.rpm")
+    for produced_rpm in produced_rpms:
+        assert is_rpm_file_signed(produced_rpm), (
+            f"Produced package '{produced_rpm}' is unsigned; sign_packages path-doubling "
+            f"regression detected (signing was silently skipped)."
+        )
     delete_all_gpg_keys_by_name(TEST_GPG_KEY_NAME)
 
     postgres_version_file_path = f"{PACKAGING_EXEC_FOLDER}/{POSTGRES_VERSION_FILE}"

packaging_automation/tests/test_citus_package_utils.py

@@ -142,7 +142,7 @@ def test_get_postgres_versions():
 def test_build_package_debian():
     input_output_parameters = InputOutputParameters.build(
         PACKAGING_EXEC_FOLDER,
-        f"{OUTPUT_FOLDER}/debian-stretch",
+        f"{OUTPUT_FOLDER}/debian-bookworm",
         output_validation=False,
     )
 
@@ -156,7 +156,7 @@ def test_build_package_debian():
     build_package(
         github_token=GH_TOKEN,
         build_type=BuildType.release,
-        docker_platform="debian-stretch",
+        docker_platform="debian-bookworm",
         postgres_version="all",
         input_output_parameters=input_output_parameters,
         is_test=True,
@@ -200,12 +200,12 @@ def test_sign_packages():
         PACKAGING_EXEC_FOLDER, f"{OUTPUT_FOLDER}", output_validation=False
     )
     sign_packages(
-        sub_folder="centos-8",
+        sub_folder="rpm_build",
         signing_credentials=signing_credentials,
         input_output_parameters=input_output_parameters,
     )
     sign_packages(
-        sub_folder="debian-stretch",
+        sub_folder="debian-bookworm",
         signing_credentials=signing_credentials,
         input_output_parameters=input_output_parameters,
     )

packaging_automation/tests/test_update_docker.py

@@ -14,17 +14,19 @@
     update_docker_file_for_latest_postgres,
     update_regular_docker_compose_file,
     update_docker_file_alpine,
-    update_docker_file_for_postgres15,
-    update_docker_file_for_postgres14,
+    update_docker_file_for_postgres16,
+    update_docker_file_for_postgres17,
+    update_docker_file_for_postgres18,
     update_changelog,
 )
 
 BASE_PATH = os.getenv("BASE_PATH", default=pathlib2.Path(__file__).parents[2])
 TEST_BASE_PATH = f"{BASE_PATH}/docker"
 PROJECT_VERSION = "12.0.0"
 
-POSTGRES_15_VERSION = "15.4"
-POSTGRES_14_VERSION = "14.9"
+POSTGRES_18_VERSION = "18.1"
+POSTGRES_17_VERSION = "17.6"
+POSTGRES_16_VERSION = "16.10"
 
 PROJECT_NAME = "citus"
 version_details = get_version_details(PROJECT_VERSION)
@@ -45,7 +47,7 @@ def teardown_module():
 
 def test_update_docker_file_for_latest_postgres():
     update_docker_file_for_latest_postgres(
-        PROJECT_VERSION, TEMPLATE_PATH, TEST_BASE_PATH, POSTGRES_14_VERSION
+        PROJECT_VERSION, TEMPLATE_PATH, TEST_BASE_PATH, POSTGRES_18_VERSION
     )
     with open(
         f"{TEST_BASE_PATH}/Dockerfile",
@@ -55,7 +57,7 @@ def test_update_docker_file_for_latest_postgres():
     ) as reader:
         content = reader.read()
         lines = content.splitlines()
-        assert lines[2].strip() == f"FROM postgres:{POSTGRES_14_VERSION}"
+        assert lines[2].strip() == f"FROM postgres:{POSTGRES_18_VERSION}"
         assert lines[3].strip() == f"ARG VERSION={PROJECT_VERSION}"
         assert (
             f"postgresql-$PG_MAJOR-{PROJECT_NAME}-"
@@ -83,7 +85,7 @@ def test_update_regular_docker_compose_file():
 
 def test_update_docker_file_alpine():
     update_docker_file_alpine(
-        PROJECT_VERSION, TEMPLATE_PATH, TEST_BASE_PATH, POSTGRES_14_VERSION
+        PROJECT_VERSION, TEMPLATE_PATH, TEST_BASE_PATH, POSTGRES_18_VERSION
     )
     with open(
         f"{TEST_BASE_PATH}/alpine/Dockerfile",
@@ -93,24 +95,24 @@ def test_update_docker_file_alpine():
     ) as reader:
         content = reader.read()
         lines = content.splitlines()
-        assert lines[2].strip() == f"FROM postgres:{POSTGRES_14_VERSION}-alpine"
+        assert lines[2].strip() == f"FROM postgres:{POSTGRES_18_VERSION}-alpine"
         assert lines[3].strip() == f"ARG VERSION={PROJECT_VERSION}"
         assert len(lines) == 58
 
 
-def test_update_docker_file_for_postgres14():
-    update_docker_file_for_postgres14(
-        PROJECT_VERSION, TEMPLATE_PATH, TEST_BASE_PATH, POSTGRES_14_VERSION
+def test_update_docker_file_for_postgres16():
+    update_docker_file_for_postgres16(
+        PROJECT_VERSION, TEMPLATE_PATH, TEST_BASE_PATH, POSTGRES_16_VERSION
     )
     with open(
-        f"{TEST_BASE_PATH}/postgres-14/Dockerfile",
+        f"{TEST_BASE_PATH}/postgres-16/Dockerfile",
         "r",
         encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING,
         errors=DEFAULT_UNICODE_ERROR_HANDLER,
     ) as reader:
         content = reader.read()
         lines = content.splitlines()
-        assert lines[2].strip() == f"FROM postgres:{POSTGRES_14_VERSION}"
+        assert lines[2].strip() == f"FROM postgres:{POSTGRES_16_VERSION}"
         assert lines[3].strip() == f"ARG VERSION={PROJECT_VERSION}"
         assert (
             f"postgresql-$PG_MAJOR-{PROJECT_NAME}-"
@@ -120,19 +122,41 @@ def test_update_docker_file_for_postgres14():
         assert len(lines) == 42
 
 
-def test_update_docker_file_for_postgres15():
-    update_docker_file_for_postgres15(
-        PROJECT_VERSION, TEMPLATE_PATH, TEST_BASE_PATH, POSTGRES_15_VERSION
+def test_update_docker_file_for_postgres17():
+    update_docker_file_for_postgres17(
+        PROJECT_VERSION, TEMPLATE_PATH, TEST_BASE_PATH, POSTGRES_17_VERSION
     )
     with open(
-        f"{TEST_BASE_PATH}/postgres-15/Dockerfile",
+        f"{TEST_BASE_PATH}/postgres-17/Dockerfile",
         "r",
         encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING,
         errors=DEFAULT_UNICODE_ERROR_HANDLER,
     ) as reader:
         content = reader.read()
         lines = content.splitlines()
-        assert lines[2].strip() == f"FROM postgres:{POSTGRES_15_VERSION}"
+        assert lines[2].strip() == f"FROM postgres:{POSTGRES_17_VERSION}"
+        assert lines[3].strip() == f"ARG VERSION={PROJECT_VERSION}"
+        assert (
+            f"postgresql-$PG_MAJOR-{PROJECT_NAME}-"
+            f"{version_details['major']}.{version_details['minor']}=$CITUS_VERSION"
+            in lines[21]
+        )
+        assert len(lines) == 42
+
+
+def test_update_docker_file_for_postgres18():
+    update_docker_file_for_postgres18(
+        PROJECT_VERSION, TEMPLATE_PATH, TEST_BASE_PATH, POSTGRES_18_VERSION
+    )
+    with open(
+        f"{TEST_BASE_PATH}/postgres-18/Dockerfile",
+        "r",
+        encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING,
+        errors=DEFAULT_UNICODE_ERROR_HANDLER,
+    ) as reader:
+        content = reader.read()
+        lines = content.splitlines()
+        assert lines[2].strip() == f"FROM postgres:{POSTGRES_18_VERSION}"
         assert lines[3].strip() == f"ARG VERSION={PROJECT_VERSION}"
         assert (
             f"postgresql-$PG_MAJOR-{PROJECT_NAME}-"
@@ -177,5 +201,6 @@ def test_update_changelog_without_postgres():
 
 def test_pkgvar_postgres_version_existence():
     config = dotenv_values(PKGVARS_FILE)
-    assert config["postgres_15_version"]
-    assert config["postgres_14_version"]
+    assert config["postgres_16_version"]
+    assert config["postgres_17_version"]
+    assert config["postgres_18_version"]