Skip to content

Commit 1680669

Browse files
committed
Fix the adb8 tests
Minor fixes for running tests on commit 32fba77 The Dockerfile and scripts for running behave tests are taken from adb-7.2.0. The README.Ubuntu.bash required for container build has also been updated. Additional packages and configuration flags have been added to Dockerfile.ubuntu SSH key verification is disabled, which increases the stability of behave tests. Disabled the inclusion of dependencies during the build in compile_gpdb.bash because we don't use it. In the script for running ORCA unit tests, the xerces build is disabled because it is not required. Fix the use of the old python version in mirrors_mgmt_utils. In explain_format, fix passing the test with ORCA in tests with jit. Changes are taken from 7e86a6b
1 parent c96d22c commit 1680669

13 files changed

Lines changed: 332 additions & 13 deletions

File tree

README.Ubuntu.bash

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/bin/bash
22

3-
sudo apt-get update
4-
sudo apt-get install -y \
3+
apt-get update
4+
apt-get install -y \
55
bison \
66
ccache \
77
cmake \
@@ -10,15 +10,20 @@ sudo apt-get install -y \
1010
git-core \
1111
gcc \
1212
g++ \
13+
llvm \
14+
clang \
1315
inetutils-ping \
1416
krb5-kdc \
1517
krb5-admin-server \
1618
libapr1-dev \
1719
libbz2-dev \
20+
libuv1-dev \
1821
libcurl4-gnutls-dev \
1922
libevent-dev \
2023
libkrb5-dev \
2124
libpam-dev \
25+
libldap-common \
26+
libldap-dev \
2227
libperl-dev \
2328
libreadline-dev \
2429
libssl-dev \
@@ -35,12 +40,12 @@ sudo apt-get install -y \
3540
pkg-config \
3641
python3-dev \
3742
python3-pip \
43+
python3-psycopg2 \
3844
python3-psutil \
39-
python3-pygresql \
4045
python3-yaml \
4146
zlib1g-dev
4247

43-
sudo tee -a /etc/sysctl.conf << EOF
48+
tee -a /etc/sysctl.conf << EOF
4449
kernel.shmmax = 5000000000000
4550
kernel.shmmni = 32768
4651
kernel.shmall = 40000000000
@@ -58,10 +63,10 @@ vm.overcommit_memory = 2
5863
vm.overcommit_ratio = 95
5964
EOF
6065

61-
sudo sysctl -p
66+
sysctl -p
6267

63-
sudo mkdir -p /etc/security/limits.d
64-
sudo tee -a /etc/security/limits.d/90-greenplum.conf << EOF
68+
mkdir -p /etc/security/limits.d
69+
tee -a /etc/security/limits.d/90-greenplum.conf << EOF
6570
* soft nofile 1048576
6671
* hard nofile 1048576
6772
* soft nproc 1048576

arenadata/.env

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
COMPOSE_HTTP_TIMEOUT=400

arenadata/Dockerfile.linter

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
FROM ubuntu:20.04
2+
3+
RUN apt update && apt install -y git parallel clang-format-11 && \
4+
ln -s /usr/bin/clang-format-11 /usr/bin/clang-format
5+
6+
WORKDIR /opt/gpdb_src
7+
8+
COPY . /opt/gpdb_src
9+
10+
ENV CLANG_FORMAT=clang-format-11
11+
12+
ENTRYPOINT src/tools/fmt gen && git diff --exit-code && src/tools/fmt chk

arenadata/Dockerfile.ubuntu

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
FROM ubuntu:22.04 AS base
2+
3+
COPY README.Ubuntu.bash ./
4+
5+
RUN set -eux; \
6+
export DEBIAN_FRONTEND=noninteractive; \
7+
./README.Ubuntu.bash; \
8+
rm README.Ubuntu.bash; \
9+
# The en_US.UTF-8 locale is needed to run GPDB
10+
locale-gen en_US.UTF-8; \
11+
# To run sshd directly, but not using `/etc/init.d/ssh start`
12+
mkdir /run/sshd; \
13+
# Alter precedence in favor of IPv4 during resolving
14+
echo 'precedence ::ffff:0:0/96 100' >> /etc/gai.conf; \
15+
# Packages for tests
16+
apt install -y rsync iproute2 fakeroot lsof sudo python3.11 python3.11-dev openjdk-17-jdk libipc-run-perl; \
17+
apt install -y protobuf-compiler libpq-dev libossp-uuid-dev; \
18+
# Install allure-behave for behave tests
19+
python3 -m pip install psutil; \
20+
pip3 install pytest gsutil behave~=1.2.6 coverage~=4.5 'mock<=5.0.0' allure-behave==2.4.0 protobuf==3.20.0; \
21+
pip3 install PyGreSQL; \
22+
pip3 cache purge
23+
24+
WORKDIR /home/gpadmin
25+
26+
RUN mkdir -p /gpadmin/.ssh/ && cat >> /gpadmin/.ssh/config <<-EOF
27+
Host *
28+
StrictHostKeyChecking no
29+
EOF
30+
31+
FROM base AS build
32+
33+
COPY . gpdb_src
34+
35+
RUN mkdir /home/gpadmin/bin_gpdb
36+
37+
ENV TARGET_OS_VERSION=22
38+
ENV TARGET_OS=ubuntu
39+
ENV OUTPUT_ARTIFACT_DIR=bin_gpdb
40+
ENV SKIP_UNITTESTS=
41+
# To set default locale for unittests
42+
ENV LANG=en_US.UTF-8
43+
ENV CONFIGURE_FLAGS="--enable-debug-extensions --with-gssapi --with-openssl --enable-cassert --enable-debug --enable-depend --with-libxml --with-uuid=ossp --with-perl --enable-mapreduce --enable-orafce"
44+
45+
# Compile with running mocking tests
46+
RUN bash /home/gpadmin/gpdb_src/concourse/scripts/compile_gpdb.bash
47+
48+
FROM base AS code
49+
COPY . gpdb_src
50+
RUN rm -rf gpdb_src/.git/
51+
52+
FROM base AS test
53+
COPY --from=code /home/gpadmin/gpdb_src gpdb_src
54+
COPY --from=build /home/gpadmin/bin_gpdb /home/gpadmin/bin_gpdb

arenadata/docker-compose.yaml

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
---
2+
3+
version: "3"
4+
services:
5+
#image can be either hub.adsw.io/library/gpdb7_regress:${BRANCH_NAME}-x86-64 or
6+
#hub.adsw.io/library/gpdb7_regress:${BRANCH_NAME}-ppc64le depending on the arch
7+
cdw:
8+
image: "${IMAGE}"
9+
working_dir: /home/gpadmin
10+
hostname: cdw
11+
volumes:
12+
- "$PWD/ssh_keys/id_rsa:/home/gpadmin/.ssh/id_rsa"
13+
- "$PWD/ssh_keys/id_rsa.pub:/home/gpadmin/.ssh/id_rsa.pub"
14+
- "$PWD/allure-results:/tmp/allure-results"
15+
privileged: true
16+
sysctls:
17+
kernel.sem: 500 1024000 200 4096
18+
init: true
19+
entrypoint: >
20+
sleep infinity
21+
sdw1:
22+
image: "${IMAGE}"
23+
privileged: true
24+
hostname: sdw1
25+
volumes:
26+
- "$PWD/ssh_keys/id_rsa:/home/gpadmin/.ssh/id_rsa"
27+
- "$PWD/ssh_keys/id_rsa.pub:/home/gpadmin/.ssh/id_rsa.pub"
28+
sysctls:
29+
kernel.sem: 500 1024000 200 4096
30+
init: true
31+
entrypoint: >
32+
sleep infinity
33+
sdw2:
34+
image: "${IMAGE}"
35+
privileged: true
36+
hostname: sdw2
37+
volumes:
38+
- "$PWD/ssh_keys/id_rsa:/home/gpadmin/.ssh/id_rsa"
39+
- "$PWD/ssh_keys/id_rsa.pub:/home/gpadmin/.ssh/id_rsa.pub"
40+
sysctls:
41+
kernel.sem: 500 1024000 200 4096
42+
init: true
43+
entrypoint: >
44+
sleep infinity
45+
sdw3:
46+
image: "${IMAGE}"
47+
privileged: true
48+
hostname: sdw3
49+
volumes:
50+
- "$PWD/ssh_keys/id_rsa:/home/gpadmin/.ssh/id_rsa"
51+
- "$PWD/ssh_keys/id_rsa.pub:/home/gpadmin/.ssh/id_rsa.pub"
52+
sysctls:
53+
kernel.sem: 500 1024000 200 4096
54+
init: true
55+
entrypoint: >
56+
sleep infinity

arenadata/scripts/behave_gpdb.bash

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
#!/bin/bash -l
2+
3+
set -eox pipefail
4+
5+
CWDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../concourse/scripts" && pwd )"
6+
source "${CWDIR}/common.bash"
7+
8+
function gen_env(){
9+
cat > /opt/run_test.sh <<-EOF
10+
set -ex
11+
12+
source /usr/local/greenplum-db-devel/greenplum_path.sh
13+
14+
source gpdb_src/gpAux/gpdemo/gpdemo-env.sh
15+
16+
if [[ ${FEATURE} == "gpexpand" ]]; then
17+
mkdir -p /home/gpadmin/sqldump
18+
wget -nv https://rt.adsw.io/artifactory/common/dump.sql.xz -O /home/gpadmin/sqldump/dump.sql.xz
19+
20+
xz -d /home/gpadmin/sqldump/dump.sql.xz
21+
fi
22+
23+
cd "\${1}/gpdb_src/gpMgmt/"
24+
BEHAVE_TAGS="${BEHAVE_TAGS}"
25+
BEHAVE_FLAGS="${BEHAVE_FLAGS}"
26+
if [ ! -z "\${BEHAVE_TAGS}" ]; then
27+
make -f Makefile.behave behave tags=\${BEHAVE_TAGS}
28+
else
29+
flags="\${BEHAVE_FLAGS}" make -f Makefile.behave behave
30+
fi
31+
EOF
32+
33+
chmod a+x /opt/run_test.sh
34+
}
35+
36+
function _main() {
37+
38+
if [ -z "${BEHAVE_TAGS}" ] && [ -z "${BEHAVE_FLAGS}" ]; then
39+
echo "FATAL: BEHAVE_TAGS or BEHAVE_FLAGS not set"
40+
exit 1
41+
fi
42+
43+
# Run inside a subshell so it does not pollute the environment after
44+
# sourcing greenplum_path
45+
time (make_cluster)
46+
47+
time gen_env
48+
49+
time run_test
50+
}
51+
52+
_main "$@"
Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
#!/usr/bin/env bash
2+
set -x -o pipefail
3+
4+
behave_tests_dir="gpMgmt/test/behave/mgmt_utils"
5+
6+
# TODO concourse_cluster tests are not stable
7+
# clusters="concourse_cluster ~concourse_cluster,demo_cluster"
8+
9+
clusters="~concourse_cluster"
10+
11+
if [ $# -eq 0 ]
12+
then
13+
# TODO cross_subnet and gpssh tests are excluded
14+
features=`ls $behave_tests_dir -1 | grep feature | grep -v -E "cross_subnet|gpssh" | sed 's/\.feature$//'`
15+
else
16+
for feature in $@
17+
do
18+
if [ ! -f "$behave_tests_dir/$feature.feature" ]
19+
then
20+
echo "Feature '$feature' doesn't exists"
21+
exit 1
22+
fi
23+
done
24+
features=$@
25+
fi
26+
27+
processes=3
28+
29+
rm -rf allure-results
30+
mkdir allure-results -pm 777
31+
mkdir ssh_keys -p
32+
if [ ! -e "ssh_keys/id_rsa" ]
33+
then
34+
ssh-keygen -P "" -f ssh_keys/id_rsa
35+
fi
36+
37+
run_feature() {
38+
local feature=$1
39+
local cluster=$2
40+
if [ $cluster = "concourse_cluster" ]; then
41+
local project="${feature}_concourse"
42+
else
43+
local project="${feature}_demo"
44+
fi
45+
echo "Started $feature behave tests on cluster $cluster and project $project"
46+
docker-compose -p $project -f arenadata/docker-compose.yaml --env-file arenadata/.env up -d
47+
# Prepare ALL containers first
48+
local services=$(docker-compose -p $project -f arenadata/docker-compose.yaml config --services | tr '\n' ' ')
49+
for service in $services
50+
do
51+
docker-compose -p $project -f arenadata/docker-compose.yaml exec -T \
52+
$service bash -c "mkdir -p /data/gpdata && chmod -R 777 /data &&
53+
source gpdb_src/concourse/scripts/common.bash && install_gpdb &&
54+
./gpdb_src/concourse/scripts/setup_gpadmin_user.bash" &
55+
done
56+
wait
57+
58+
# Add host keys to known_hosts after containers setup
59+
for service in $services
60+
do
61+
docker-compose -p $project -f arenadata/docker-compose.yaml exec -T \
62+
$service bash -c "ssh-keyscan ${services/$service/} >> /home/gpadmin/.ssh/known_hosts" &
63+
done
64+
wait
65+
66+
docker-compose -p $project -f arenadata/docker-compose.yaml exec -T \
67+
-e FEATURE="$feature" -e BEHAVE_FLAGS="--tags $feature --tags=$cluster \
68+
-f behave_utils.arenadata.formatter:CustomFormatter \
69+
-o non-existed-output \
70+
-f allure_behave.formatter:AllureFormatter \
71+
-o /tmp/allure-results" \
72+
cdw gpdb_src/arenadata/scripts/behave_gpdb.bash
73+
status=$?
74+
75+
docker-compose -p $project -f arenadata/docker-compose.yaml --env-file arenadata/.env down -v
76+
77+
if [[ $status > 0 ]]; then echo "Feature $feature failed with exit code $status"; fi
78+
exit $status
79+
}
80+
81+
pids=""
82+
exits=0
83+
for feature in $features
84+
do
85+
for cluster in $clusters
86+
do
87+
run_feature $feature $cluster &
88+
pids+="$! "
89+
if [[ $(jobs -r -p | wc -l) -ge $processes ]]; then
90+
wait -n
91+
((exits += $?))
92+
fi
93+
done
94+
done
95+
for pid in $pids
96+
do
97+
wait $pid
98+
exits=$((exits + $?))
99+
done
100+
if [[ $exits > 0 ]]; then exit 1; fi

concourse/scripts/compile_gpdb.bash

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ function _main() {
151151
unittest_check_gpdb
152152
fi
153153

154-
include_dependencies
154+
# include_dependencies
155155

156156
export_gpdb
157157
export_gpdb_extensions

concourse/scripts/unit_tests_gporca.bash

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ function test_orca
2727
function _main
2828
{
2929
mkdir gpdb_src/gpAux/ext
30-
build_xerces
30+
#build_xerces
3131
test_orca
3232
}
3333

gpMgmt/test/behave/mgmt_utils/steps/mirrors_mgmt_utils.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -481,7 +481,7 @@ def make_temp_dir_on_remote(context, hostname, tmp_base_dir_remote, mode='700'):
481481
raise Exception("tmp_base_dir cannot be empty")
482482

483483
tempfile_cmd = Command(name="Create temp directory on remote host",
484-
cmdStr=""" python -c "import tempfile; t=tempfile.mkdtemp(dir='{}');print(t)" """
484+
cmdStr=""" python3 -c "import tempfile; t=tempfile.mkdtemp(dir='{}');print(t)" """
485485
.format(tmp_base_dir_remote),
486486
remoteHost=hostname, ctxt=REMOTE)
487487
tempfile_cmd.run(validateAfter=True)

0 commit comments

Comments
 (0)