DO NOT MERGE: CI-3898: Add rules, build and allow-failure tests for adb-8.x

README.Ubuntu.bash

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-sudo apt-get update
-sudo apt-get install -y \
+apt-get update
+apt-get install -y \
 	bison \
 	ccache \
 	cmake \
@@ -10,15 +10,20 @@ sudo apt-get install -y \
 	git-core \
 	gcc \
 	g++ \
+	llvm \
+	clang \
 	inetutils-ping \
 	krb5-kdc \
 	krb5-admin-server \
 	libapr1-dev \
 	libbz2-dev \
+	libuv1-dev \
 	libcurl4-gnutls-dev \
 	libevent-dev \
 	libkrb5-dev \
 	libpam-dev \
+	libldap-common \
+	libldap-dev \
 	libperl-dev \
 	libreadline-dev \
 	libssl-dev \
@@ -35,12 +40,12 @@ sudo apt-get install -y \
 	pkg-config \
 	python3-dev \
 	python3-pip \
+	python3-psycopg2 \
 	python3-psutil \
-	python3-pygresql \
 	python3-yaml \
 	zlib1g-dev
 
-sudo tee -a /etc/sysctl.conf << EOF
+tee -a /etc/sysctl.conf << EOF
 kernel.shmmax = 5000000000000
 kernel.shmmni = 32768
 kernel.shmall = 40000000000
@@ -58,10 +63,10 @@ vm.overcommit_memory = 2
 vm.overcommit_ratio = 95
 EOF
 
-sudo sysctl -p
+sysctl -p
 
-sudo mkdir -p /etc/security/limits.d
-sudo tee -a /etc/security/limits.d/90-greenplum.conf << EOF
+mkdir -p /etc/security/limits.d
+tee -a /etc/security/limits.d/90-greenplum.conf << EOF
 * soft nofile 1048576
 * hard nofile 1048576
 * soft nproc 1048576

arenadata/.env

@@ -0,0 +1 @@
+COMPOSE_HTTP_TIMEOUT=400

arenadata/Dockerfile.linter

@@ -0,0 +1,12 @@
+FROM ubuntu:20.04
+
+RUN apt update && apt install -y git parallel clang-format-11 && \
+    ln -s /usr/bin/clang-format-11 /usr/bin/clang-format
+
+WORKDIR /opt/gpdb_src
+
+COPY . /opt/gpdb_src
+
+ENV CLANG_FORMAT=clang-format-11
+
+ENTRYPOINT src/tools/fmt gen && git diff --exit-code && src/tools/fmt chk

arenadata/Dockerfile.ubuntu

@@ -0,0 +1,54 @@
+FROM ubuntu:22.04 AS base
+
+COPY README.Ubuntu.bash ./
+
+RUN set -eux; \
+    export DEBIAN_FRONTEND=noninteractive; \
+    ./README.Ubuntu.bash; \
+    rm README.Ubuntu.bash; \
+# The en_US.UTF-8 locale is needed to run GPDB
+    locale-gen en_US.UTF-8; \
+# To run sshd directly, but not using `/etc/init.d/ssh start`
+    mkdir /run/sshd; \
+# Alter precedence in favor of IPv4 during resolving
+    echo 'precedence ::ffff:0:0/96  100' >> /etc/gai.conf; \
+# Packages for tests
+    apt install -y rsync iproute2 fakeroot lsof sudo python3.11 python3.11-dev openjdk-17-jdk libipc-run-perl; \
+    apt install -y protobuf-compiler libpq-dev libossp-uuid-dev; \
+# Install allure-behave for behave tests
+    python3 -m pip install psutil; \
+    pip3 install pytest gsutil behave~=1.2.6 coverage~=4.5 'mock<=5.0.0' allure-behave==2.4.0 protobuf==3.20.0; \
+    pip3 install PyGreSQL; \
+    pip3 cache purge
+
+WORKDIR /home/gpadmin
+
+RUN mkdir -p /gpadmin/.ssh/ && cat >> /gpadmin/.ssh/config <<-EOF
+Host *
+    StrictHostKeyChecking no
+EOF
+
+FROM base AS build
+
+COPY . gpdb_src
+
+RUN mkdir /home/gpadmin/bin_gpdb
+
+ENV TARGET_OS_VERSION=22
+ENV TARGET_OS=ubuntu
+ENV OUTPUT_ARTIFACT_DIR=bin_gpdb
+ENV SKIP_UNITTESTS=
+# To set default locale for unittests
+ENV LANG=en_US.UTF-8
+ENV CONFIGURE_FLAGS="--enable-debug-extensions --with-gssapi --with-openssl --enable-cassert --enable-debug --enable-depend --with-libxml --with-uuid=ossp --with-perl --enable-mapreduce --enable-orafce"
+
+# Compile with running mocking tests
+RUN bash /home/gpadmin/gpdb_src/concourse/scripts/compile_gpdb.bash
+
+FROM base AS code
+COPY . gpdb_src
+RUN rm -rf gpdb_src/.git/
+
+FROM base AS test
+COPY --from=code /home/gpadmin/gpdb_src gpdb_src
+COPY --from=build /home/gpadmin/bin_gpdb /home/gpadmin/bin_gpdb

arenadata/docker-compose.yaml

@@ -0,0 +1,56 @@
+---
+
+version: "3"
+services:
+  #image can be either hub.adsw.io/library/gpdb7_regress:${BRANCH_NAME}-x86-64 or
+  #hub.adsw.io/library/gpdb7_regress:${BRANCH_NAME}-ppc64le depending on the arch
+  cdw:
+    image: "${IMAGE}"
+    working_dir: /home/gpadmin
+    hostname: cdw
+    volumes:
+      - "$PWD/ssh_keys/id_rsa:/home/gpadmin/.ssh/id_rsa"
+      - "$PWD/ssh_keys/id_rsa.pub:/home/gpadmin/.ssh/id_rsa.pub"
+      - "$PWD/allure-results:/tmp/allure-results"
+    privileged: true
+    sysctls:
+      kernel.sem: 500 1024000 200 4096
+    init: true
+    entrypoint: >
+      sleep infinity
+  sdw1:
+    image: "${IMAGE}"
+    privileged: true
+    hostname: sdw1
+    volumes:
+      - "$PWD/ssh_keys/id_rsa:/home/gpadmin/.ssh/id_rsa"
+      - "$PWD/ssh_keys/id_rsa.pub:/home/gpadmin/.ssh/id_rsa.pub"
+    sysctls:
+      kernel.sem: 500 1024000 200 4096
+    init: true
+    entrypoint: >
+      sleep infinity
+  sdw2:
+    image: "${IMAGE}"
+    privileged: true
+    hostname: sdw2
+    volumes:
+      - "$PWD/ssh_keys/id_rsa:/home/gpadmin/.ssh/id_rsa"
+      - "$PWD/ssh_keys/id_rsa.pub:/home/gpadmin/.ssh/id_rsa.pub"
+    sysctls:
+      kernel.sem: 500 1024000 200 4096
+    init: true
+    entrypoint: >
+      sleep infinity
+  sdw3:
+    image: "${IMAGE}"
+    privileged: true
+    hostname: sdw3
+    volumes:
+      - "$PWD/ssh_keys/id_rsa:/home/gpadmin/.ssh/id_rsa"
+      - "$PWD/ssh_keys/id_rsa.pub:/home/gpadmin/.ssh/id_rsa.pub"
+    sysctls:
+      kernel.sem: 500 1024000 200 4096
+    init: true
+    entrypoint: >
+      sleep infinity

arenadata/scripts/behave_gpdb.bash

@@ -0,0 +1,52 @@
+#!/bin/bash -l
+
+set -eox pipefail
+
+CWDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../concourse/scripts" && pwd )"
+source "${CWDIR}/common.bash"
+
+function gen_env(){
+		cat > /opt/run_test.sh <<-EOF
+		set -ex
+
+		source /usr/local/greenplum-db-devel/greenplum_path.sh
+
+		source gpdb_src/gpAux/gpdemo/gpdemo-env.sh
+
+		if [[ ${FEATURE} == "gpexpand" ]]; then
+			mkdir -p /home/gpadmin/sqldump
+			wget -nv https://rt.adsw.io/artifactory/common/dump.sql.xz -O /home/gpadmin/sqldump/dump.sql.xz
+
+			xz -d /home/gpadmin/sqldump/dump.sql.xz
+		fi
+
+		cd "\${1}/gpdb_src/gpMgmt/"
+		BEHAVE_TAGS="${BEHAVE_TAGS}"
+		BEHAVE_FLAGS="${BEHAVE_FLAGS}"
+		if [ ! -z "\${BEHAVE_TAGS}" ]; then
+				make -f Makefile.behave behave tags=\${BEHAVE_TAGS}
+		else
+				flags="\${BEHAVE_FLAGS}" make -f Makefile.behave behave
+		fi
+	EOF
+
+		chmod a+x /opt/run_test.sh
+}
+
+function _main() {
+
+		if [ -z "${BEHAVE_TAGS}" ] && [ -z "${BEHAVE_FLAGS}" ]; then
+				echo "FATAL: BEHAVE_TAGS or BEHAVE_FLAGS not set"
+				exit 1
+		fi
+
+		# Run inside a subshell so it does not pollute the environment after
+		# sourcing greenplum_path
+		time (make_cluster)
+
+		time gen_env
+
+		time run_test
+}
+
+_main "$@"

arenadata/scripts/run_behave_tests.bash

@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+set -x -o pipefail
+
+behave_tests_dir="gpMgmt/test/behave/mgmt_utils"
+
+# TODO concourse_cluster tests are not stable
+# clusters="concourse_cluster ~concourse_cluster,demo_cluster"
+
+clusters="~concourse_cluster"
+
+if [ $# -eq 0 ]
+then
+  # TODO cross_subnet and gpssh tests are excluded
+  features=`ls $behave_tests_dir -1 | grep feature | grep -v -E "cross_subnet|gpssh" | sed 's/\.feature$//'`
+else
+  for feature in $@
+  do
+    if [ ! -f "$behave_tests_dir/$feature.feature" ]
+    then
+      echo "Feature '$feature' doesn't exists"
+      exit 1
+    fi
+  done
+  features=$@
+fi
+
+processes=3
+
+rm -rf allure-results
+mkdir allure-results -pm 777
+mkdir ssh_keys -p
+if [ ! -e "ssh_keys/id_rsa" ]
+then
+  ssh-keygen -P "" -f ssh_keys/id_rsa
+fi
+
+run_feature() {
+  local feature=$1
+  local cluster=$2
+  if [ $cluster = "concourse_cluster" ]; then
+    local project="${feature}_concourse"
+  else
+    local project="${feature}_demo"
+  fi
+  echo "Started $feature behave tests on cluster $cluster and project $project"
+  docker-compose -p $project -f arenadata/docker-compose.yaml --env-file arenadata/.env up -d
+  # Prepare ALL containers first
+  local services=$(docker-compose -p $project -f arenadata/docker-compose.yaml config --services | tr '\n' ' ')
+  for service in $services
+  do
+    docker-compose -p $project -f arenadata/docker-compose.yaml exec -T \
+      $service bash -c "mkdir -p /data/gpdata && chmod -R 777 /data &&
+        source gpdb_src/concourse/scripts/common.bash && install_gpdb &&
+       ./gpdb_src/concourse/scripts/setup_gpadmin_user.bash" &
+  done
+  wait
+
+  # Add host keys to known_hosts after containers setup
+  for service in $services
+  do
+    docker-compose -p $project -f arenadata/docker-compose.yaml exec -T \
+      $service bash -c "ssh-keyscan ${services/$service/} >> /home/gpadmin/.ssh/known_hosts" &
+  done
+  wait
+
+  docker-compose -p $project -f arenadata/docker-compose.yaml exec -T \
+    -e FEATURE="$feature" -e BEHAVE_FLAGS="--tags $feature --tags=$cluster \
+      -f behave_utils.arenadata.formatter:CustomFormatter \
+      -o non-existed-output \
+      -f allure_behave.formatter:AllureFormatter \
+      -o /tmp/allure-results"  \
+    cdw gpdb_src/arenadata/scripts/behave_gpdb.bash
+  status=$?
+
+  docker-compose -p $project -f arenadata/docker-compose.yaml --env-file arenadata/.env down -v
+
+  if [[ $status > 0 ]]; then echo "Feature $feature failed with exit code $status"; fi
+  exit $status
+}
+
+pids=""
+exits=0
+for feature in $features
+do
+  for cluster in $clusters
+  do
+     run_feature $feature $cluster &
+     pids+="$! "
+     if [[ $(jobs -r -p | wc -l) -ge $processes ]]; then
+        wait -n
+        ((exits += $?))
+     fi
+  done
+done
+for pid in $pids
+  do
+    wait $pid
+    exits=$((exits + $?))
+  done
+if [[ $exits > 0 ]]; then exit 1; fi