From 283018803cfc05d6f17f24a1d07b52409b86470b Mon Sep 17 00:00:00 2001 From: MutableFire Date: Thu, 2 Jul 2026 11:31:36 +0300 Subject: [PATCH] Speed up tests --- .github/workflows/build.yml | 51 ++++++++++++++++++++++++++++------- ci/build-gpdb.sh | 52 ------------------------------------ ci/install-runtime-deps.sh | 14 ++++++++++ ci/verify-opengpdb-define.sh | 7 ----- test/start-gp.sh | 36 +++++++++++++++++++++++++ 5 files changed, 92 insertions(+), 68 deletions(-) delete mode 100644 ci/build-gpdb.sh delete mode 100644 ci/verify-opengpdb-define.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ab185859..5964ad33 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -61,22 +61,55 @@ jobs: if: inputs.open_gpdb run: bash ci/free-disk-space.sh - - name: Build and install Greenplum - if: steps.cache-deps.outputs.cache-hit != 'true' - run: bash ci/build-gpdb.sh "${{ inputs.gpdb_repo }}" "${{ inputs.gpdb_ref }}" ${{ inputs.open_gpdb && 'open-gpdb' || '' }} + - name: Install Debian packages + run: | + TARGET_DIR="$HOME/local/gpdb" + EXPECTED_SCRIPT="greenplum_path.sh" + TEMP_DIR="$HOME/local/temp_install" + + if [ "${{ inputs.open_gpdb }}" == "true" ]; then + PACKAGE_URL="https://github.com/open-gpdb/gpdb/releases/download/6.29.9/greenplum-db-6_6.29.9-1-yandex.54459.ae09c99827_amd64.deb" + echo "Installing open-gpdb..." + else + PACKAGE_URL="https://github.com/GreengageDB/greengage/releases/download/6.29.2/greengage6.deb" + echo "Installing Greengage..." + fi + + mkdir -p "$TEMP_DIR" + wget -q "$PACKAGE_URL" -O /tmp/gp.deb + sudo apt-get remove -y python-is-python3 || true + sudo dpkg -x /tmp/gp.deb "$TEMP_DIR" + rm -f /tmp/gp.deb + + REAL_SOURCE=$(find "$TEMP_DIR" -type d -exec test -f {}/"$EXPECTED_SCRIPT" \; -print -quit 2>/dev/null) + if [ -z "$REAL_SOURCE" ]; then + FOUND_SCRIPT=$(find "$TEMP_DIR" -type f -name "*_path.sh" | head -n 1) + [ -n "$FOUND_SCRIPT" ] && REAL_SOURCE=$(dirname "$FOUND_SCRIPT") && EXPECTED_SCRIPT=$(basename "$FOUND_SCRIPT") + fi + + if [ -z "$REAL_SOURCE" ]; then + echo "ERROR: Installation directory not found" >&2; exit 1 + fi + + sudo rm -rf "$TARGET_DIR" + sudo mv "$REAL_SOURCE" "$TARGET_DIR" + sudo rm -rf "$TEMP_DIR" + + sudo chown -R "$USER:$USER" "$TARGET_DIR" + + if [ "$(basename "$EXPECTED_SCRIPT")" != "greenplum_path.sh" ]; then + ln -s "$EXPECTED_SCRIPT" "$TARGET_DIR/greenplum_path.sh" + fi + + source "$TARGET_DIR/greenplum_path.sh" + echo "Greenplum ready at: $GPHOME" - name: Build and install Arrow 15 - if: steps.cache-deps.outputs.cache-hit != 'true' run: bash ci/build-arrow.sh - name: Build and install gRPC - if: steps.cache-deps.outputs.cache-hit != 'true' run: bash ci/build-grpc.sh - - name: Verify open-gpdb defines OPENGPDB - if: inputs.open_gpdb - run: bash ci/verify-opengpdb-define.sh - - name: Build Tea run: bash ci/build-tea.sh diff --git a/ci/build-gpdb.sh b/ci/build-gpdb.sh deleted file mode 100644 index c64e8adf..00000000 --- a/ci/build-gpdb.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -# Build and install Greenplum 6 into $HOME/local/gpdb. -# -# Usage: build-gpdb.sh [open-gpdb] -# -# Passing "open-gpdb" as the third argument enables the fixups needed for the -# open-gpdb fork (see the inline comments below); arenadata/gpdb builds without -# them. -set -eo pipefail - -repo="$1" -ref="$2" -mode="${3:-}" - -git clone "$repo" -b "$ref" --depth 1 gpdb -cd gpdb -git submodule update --init - -configure_extra=() -if [ "$mode" = "open-gpdb" ]; then - # ORCA in open-gpdb still builds with -std=c++98/gnu++98, but the system - # Xerces-C 3.2 (Ubuntu 22.04) requires C++11 (char16_t); bump the ORCA C++ - # standard to gnu++14 (gnu++17 would reject ORCA's deprecated throw() specs). - # gnu++14 + -Wextra then surfaces -Wdeprecated-copy / -Wnonnull-compare which - # ORCA's -Werror turns fatal, so append -Wno-error=... last (wins over it). - for mk in src/backend/gpopt/gpopt.mk \ - src/backend/gporca/gporca.mk \ - src/backend/gporca/libgpos/src/common/Makefile; do - sed -i 's/-std=gnu++98/-std=gnu++14/g; s/-std=c++98/-std=gnu++14/g' "$mk" - printf '\noverride CPPFLAGS := $(CPPFLAGS) -Wno-error=deprecated-copy -Wno-error=nonnull-compare\n' >> "$mk" - done - # open-gpdb defaults --with-mdblocales=yes and hard-requires the mdblocales - # lib/header, which is not available on the CI image. - configure_extra+=(--without-mdblocales) -fi - -# TODO(gmusya): consider using --enable-cassert -./configure --with-perl --with-python --with-libxml --with-gssapi \ - --with-pythonsrc-ext "${configure_extra[@]}" --prefix="$HOME/local/gpdb" - -if [ "$mode" = "open-gpdb" ]; then - # src/common (FRONTEND) includes the backend-generated utils/errcodes.h but - # has no order-only dep on it, so -j8 races ("errcodes.h: No such file"). - # Force the generated header first. - make -C src/backend submake-errcodes -fi - -make -j8 -make -j8 install - -cd .. -rm -rf gpdb diff --git a/ci/install-runtime-deps.sh b/ci/install-runtime-deps.sh index 28ccdb5c..84b89a01 100644 --- a/ci/install-runtime-deps.sh +++ b/ci/install-runtime-deps.sh @@ -9,6 +9,20 @@ sudo apt-get install -y \ python2 \ redis-server \ software-properties-common + +if ! command -v pip >/dev/null 2>&1 || ! pip --version 2>&1 | grep -q "python 2"; then + echo "Installing pip for Python 2..." + curl -s https://bootstrap.pypa.io/pip/2.7/get-pip.py -o get-pip.py + python2 get-pip.py + rm get-pip.py +fi +echo "Installing psutil==5.9.8 for Python 2..." +pip install "psutil==5.9.8" +GP_PYTHON_LIB="$HOME/local/gpdb/lib/python" +SITE_PACKAGES=$(python2 -c "import site; print(site.getsitepackages()[0])") +echo "Adding Greenplum libs to Python 2 site-packages: $SITE_PACKAGES" +echo "$GP_PYTHON_LIB" | sudo tee "$SITE_PACKAGES/greenplum.pth" > /dev/null + sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test sudo apt-get install -y libstdc++6 sudo systemctl disable --now redis-server diff --git a/ci/verify-opengpdb-define.sh b/ci/verify-opengpdb-define.sh deleted file mode 100644 index 53d490ce..00000000 --- a/ci/verify-opengpdb-define.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -# tea relies on the OPENGPDB macro (defined in pg_config_manual.h) to select -# the open-gpdb sampling API; make sure the freshly built gpdb defines it. -set -ex - -grep -q '#define OPENGPDB' \ - "$HOME/local/gpdb/include/postgresql/server/pg_config_manual.h" diff --git a/test/start-gp.sh b/test/start-gp.sh index 97820ad7..b392173b 100755 --- a/test/start-gp.sh +++ b/test/start-gp.sh @@ -24,6 +24,40 @@ done # shellcheck disable=SC1091 source "$gp_root"/greenplum_path.sh +export GPHOME="$gp_root" +export PATH="$GPHOME/bin:$PATH" +export LD_LIBRARY_PATH="$GPHOME/lib:$LD_LIBRARY_PATH" +export PYTHONPATH="$GPHOME/lib/python:$PYTHONPATH" + +PYTHON_BIN=$(command -v python) +PYTHON_VER=$(python --version 2>&1) + +echo "DEBUG: Using python binary: $PYTHON_BIN" +echo "DEBUG: Python version: $PYTHON_VER" +echo "DEBUG: PYTHONPATH: $PYTHONPATH" + +if [[ ! "$PYTHON_VER" =~ "Python 2" ]]; then + echo "ERROR: Greenplum requires Python 2, but 'python' command resolves to: $PYTHON_VER" + echo "Attempting to force python2..." + if command -v python2 &> /dev/null; then + mkdir -p "$HOME/bin" + ln -sf "$(command -v python2)" "$HOME/bin/python" + export PATH="$HOME/bin:$PATH" + echo "DEBUG: Created symlink $HOME/bin/python -> python2" + echo "DEBUG: New python version: $(python --version 2>&1)" + else + echo "ERROR: python2 not found!" + exit 1 + fi +fi + +echo "DEBUG: Testing import..." +if ! python -c "import gppylib.mainUtils"; then + echo "ERROR: Import failed despite checks." + python -c "import sys; print(sys.path); import gppylib.mainUtils" + exit 1 +fi + mkdir "$config_dir" "$master_dir" "${seg_dirs[@]}" readonly hostlist="$config_dir/hostlist" echo "$HOSTNAME" >"$hostlist" @@ -46,8 +80,10 @@ IP_ALLOW=0.0.0.0/0 # Path for Greenplum mgmt utils and Greenplum binaries PATH=$src_root/test/bin:$GPHOME/bin:$PATH LD_LIBRARY_PATH=$GPHOME/lib:$LD_LIBRARY_PATH +PYTHONPATH=$GPHOME/lib/python:$PYTHONPATH export PATH export LD_LIBRARY_PATH +export PYTHONPATH export MASTER_DATA_DIRECTORY export TRUSTED_SHELL EOM