Skip to content

Commit aba83fa

Browse files
korydraughnkorydraughn
committed
[26] Add workflows for testing plugins against pre-release versions of iRODS
1 parent b6b04ce commit aba83fa

10 files changed

Lines changed: 1618 additions & 0 deletions

‎.github/workflows/build-and-test-plugin-with-irods-packages-debian.yml‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
# This workflow builds a plugin using pre-release iRODS packages and runs the
2+
# plugin tests using the irods/irods_testing_environment GitHub repository.
3+
#
4+
# This particular workflow targets Debian platforms.
5+
6+
name: build-and-test-plugin-with-irods-packages-debian
7+
8+
on:
9+
workflow_call:
10+
inputs:
11+
docker_image:
12+
description: The docker image and tag identifying the OS used to compile the plugin (e.g. debian:13).
13+
required: true
14+
type: string
15+
build_artifact_suffix:
16+
description: The name used to uniquely identify build output across workflow jobs (e.g. debian_13-main).
17+
required: true
18+
type: string
19+
plugin_package_directory:
20+
description: The directory name which is expected to hold the plugin packages (e.g. 'Debian gnu_linux_13'). Required by the irods_python_ci_utilities module.
21+
required: true
22+
type: string
23+
irods_testing_environment_project_os:
24+
description: The OS component of the project directory to test against. When joined with 'irods_testing_environment_project_db', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. debian-13).
25+
required: true
26+
type: string
27+
irods_testing_environment_project_db:
28+
description: The DB component of the project directory to test against. When joined with 'irods_testing_environment_project_os', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. postgres-16).
29+
required: true
30+
type: string
31+
cache_key_irods_packages:
32+
description: The key which potentially identifies a cached build of the irods/irods source code.
33+
required: true
34+
type: string
35+
cache_key_icommands_packages:
36+
description: The key which potentially identifies a cached build of the irods/irods_client_icommands source code.
37+
required: true
38+
type: string
39+
40+
defaults:
41+
run:
42+
shell: bash
43+
44+
jobs:
45+
build_plugin:
46+
name: Build plugin - ${{ inputs.docker_image }}
47+
48+
runs-on: ubuntu-latest
49+
container: ${{ inputs.docker_image }}
50+
51+
steps:
52+
- name: Checkout repository
53+
uses: actions/checkout@v5
54+
55+
- name: Install prerequisite packages
56+
run: |
57+
apt-get update -qq
58+
apt-get install -qq \
59+
build-essential \
60+
cmake \
61+
git \
62+
gnupg \
63+
lsb-release \
64+
python3-distro \
65+
python3-packaging \
66+
python3-pip \
67+
python3-setuptools \
68+
wget
69+
70+
- name: Set up iRODS externals package repository
71+
run: |
72+
mkdir -p /etc/apt/keyrings
73+
wget -qO - https://unstable.irods.org/irods-unstable-signing-key.asc | \
74+
gpg \
75+
--no-options \
76+
--no-default-keyring \
77+
--no-auto-check-trustdb \
78+
--homedir /dev/null \
79+
--no-keyring \
80+
--import-options import-export \
81+
--output /etc/apt/keyrings/renci-irods-unstable-archive-keyring.pgp \
82+
--import
83+
echo "deb [signed-by=/etc/apt/keyrings/renci-irods-unstable-archive-keyring.pgp arch=amd64] https://unstable.irods.org/apt/ $(lsb_release -sc) main" | \
84+
tee /etc/apt/sources.list.d/renci-irods-unstable.list
85+
86+
- name: Check cache for iRODS packages
87+
uses: actions/cache/restore@v4
88+
with:
89+
key: ${{ inputs.cache_key_irods_packages }}
90+
path: _build_irods/*.deb
91+
fail-on-cache-miss: 'true'
92+
93+
- name: Check cache for iCommands packages
94+
uses: actions/cache/restore@v4
95+
with:
96+
key: ${{ inputs.cache_key_icommands_packages }}
97+
path: _build_icommands/*.deb
98+
fail-on-cache-miss: 'true'
99+
100+
- name: Install iRODS development packages
101+
run: |
102+
apt-get update -qq
103+
apt-get install -qq \
104+
./_build_irods/irods-dev*.deb \
105+
./_build_irods/irods-runtime*.deb
106+
107+
- name: Install iRODS python CI utilities
108+
run: |
109+
# TODO(#18): Investigate how to remove the need for --break-system-packages.
110+
python3 -m pip install --break-system-packages git+https://github.com/irods/irods_python_ci_utilities.git@main
111+
112+
- name: Build and package
113+
run: |
114+
mkdir _build
115+
python3 irods_consortium_continuous_integration_build_hook.py --build_directory _build
116+
117+
- name: List contents of build directory
118+
run: ls -l _build
119+
120+
# The server and icommands packages were built and cached while inside of a docker container.
121+
# We cannot use actions/cache to get the server and icommands packages in the next job because
122+
# GHA ties the cache to the runner's OS. For this workflow, the run_tests job does not run inside
123+
# of a container. This leads to a cache miss, causing the workflow to fail. To get around that, we
124+
# use upload-artifact and download-artifact.
125+
- name: Upload iRODS packages for dependent jobs
126+
uses: actions/upload-artifact@v6
127+
with:
128+
name: irods-packages-prerelease-${{ inputs.build_artifact_suffix }}
129+
path: |
130+
_build/*.deb
131+
_build_irods/*.deb
132+
_build_icommands/*.deb
133+
overwrite: true
134+
135+
run_tests:
136+
needs: build_plugin
137+
138+
name: Test plugin - ${{ inputs.irods_testing_environment_project_os }}-${{ inputs.irods_testing_environment_project_db }}
139+
140+
runs-on: ubuntu-latest
141+
142+
steps:
143+
- name: Checkout repository
144+
uses: actions/checkout@v5
145+
146+
- name: Set up python for iRODS testing environment
147+
uses: actions/setup-python@v6
148+
with:
149+
python-version: '3.8'
150+
151+
- name: Download iRODS packages
152+
uses: actions/download-artifact@v7
153+
with:
154+
name: irods-packages-prerelease-${{ inputs.build_artifact_suffix }}
155+
path: build_artifacts
156+
157+
- name: Set up docker compose
158+
uses: docker/setup-compose-action@v1
159+
160+
- name: Set up iRODS testing environment
161+
run: |
162+
git clone -b main --depth 1 --single-branch https://github.com/irods/irods_testing_environment
163+
cd irods_testing_environment
164+
python3 -m venv venv_testing_env
165+
. venv_testing_env/bin/activate
166+
python --version
167+
pip install docker-compose GitPython
168+
pip install docker'<7' requests==2.26.0
169+
pip freeze
170+
171+
- name: Run tests
172+
run: |
173+
# Create a symlink to the directory containing the recently built plugin packages.
174+
# This is required by the irods_python_irods_ci_utilities module.
175+
ln -s build_artifacts/_build "${{ inputs.plugin_package_directory }}"
176+
177+
# Move the icommands package into the same directory containing the server packages.
178+
# This is required by the testing environment.
179+
mv build_artifacts/_build_icommands/*.deb build_artifacts/_build_irods
180+
181+
cd irods_testing_environment
182+
. venv_testing_env/bin/activate
183+
project_dir="${{ inputs.irods_testing_environment_project_os }}/${{ inputs.irods_testing_environment_project_os }}-${{ inputs.irods_testing_environment_project_db }}"
184+
python run_plugin_tests.py \
185+
"${GITHUB_REPOSITORY#${GITHUB_REPOSITORY_OWNER}/}" \
186+
--project-directory "projects/${project_dir}" \
187+
--irods-package-directory ../build_artifacts/_build_irods \
188+
--plugin-package-directory .. \
189+
--discard-logs \
190+
-v

‎.github/workflows/build-and-test-plugin-with-irods-packages-enterprise-linux.yml‎

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
# This workflow builds a plugin using pre-release iRODS packages and runs the
2+
# plugin tests using the irods/irods_testing_environment GitHub repository.
3+
#
4+
# This particular workflow targets Enterprise Linux platforms.
5+
6+
name: build-and-test-plugin-with-irods-packages-enterprise-linux
7+
8+
on:
9+
workflow_call:
10+
inputs:
11+
docker_image:
12+
description: The docker image and tag identifying the OS used to compile the plugin (e.g. rockylinux/rockylinux:10).
13+
required: true
14+
type: string
15+
build_artifact_suffix:
16+
description: The name used to uniquely identify build output across workflow jobs (e.g. rockylinux_10-main).
17+
required: true
18+
type: string
19+
plugin_package_directory:
20+
description: The directory name which is expected to hold the plugin packages (e.g. 'Rocky linux_10'). Required by the irods_python_ci_utilities module.
21+
required: true
22+
type: string
23+
irods_testing_environment_project_os:
24+
description: The OS component of the project directory to test against. When joined with 'irods_testing_environment_project_db', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. rockylinux-10).
25+
required: true
26+
type: string
27+
irods_testing_environment_project_db:
28+
description: The DB component of the project directory to test against. When joined with 'irods_testing_environment_project_os', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. postgres-16).
29+
required: true
30+
type: string
31+
cache_key_irods_packages:
32+
description: The key which potentially identifies a cached build of the irods/irods source code.
33+
required: true
34+
type: string
35+
cache_key_icommands_packages:
36+
description: The key which potentially identifies a cached build of the irods/irods_client_icommands source code.
37+
required: true
38+
type: string
39+
40+
defaults:
41+
run:
42+
shell: bash
43+
44+
jobs:
45+
build_plugin:
46+
name: Build plugin - ${{ inputs.docker_image }}
47+
48+
runs-on: ubuntu-latest
49+
container: ${{ inputs.docker_image }}
50+
51+
steps:
52+
- name: Checkout repository
53+
uses: actions/checkout@v5
54+
55+
- name: Install prerequisite packages
56+
run: |
57+
dnf update -y
58+
dnf install -y \
59+
dnf-plugins-core \
60+
epel-release \
61+
gcc-c++ \
62+
git \
63+
rpm-build \
64+
sudo \
65+
wget
66+
dnf install -y \
67+
cmake \
68+
python3-devel \
69+
python3-distro \
70+
python3-packaging \
71+
python3-pip
72+
73+
- name: Set up iRODS externals package repository
74+
run: |
75+
rpm --import https://unstable.irods.org/irods-unstable-signing-key.asc
76+
wget -qO - https://unstable.irods.org/renci-irods-unstable.yum.repo | tee /etc/yum.repos.d/renci-irods-unstable.yum.repo
77+
78+
- name: Check cache for iRODS packages
79+
uses: actions/cache/restore@v4
80+
with:
81+
key: ${{ inputs.cache_key_irods_packages }}
82+
path: _build_irods/*.rpm
83+
fail-on-cache-miss: 'true'
84+
85+
- name: Check cache for iCommands packages
86+
uses: actions/cache/restore@v4
87+
with:
88+
key: ${{ inputs.cache_key_icommands_packages }}
89+
path: _build_icommands/*.rpm
90+
fail-on-cache-miss: 'true'
91+
92+
- name: Install iRODS development packages
93+
run: |
94+
dnf update -y
95+
dnf install -y \
96+
./_build_irods/irods-devel*.rpm \
97+
./_build_irods/irods-runtime*.rpm
98+
99+
- name: Install iRODS python CI utilities
100+
run: |
101+
python3 -m pip install git+https://github.com/irods/irods_python_ci_utilities.git@main
102+
103+
- name: Build and package
104+
run: |
105+
mkdir _build
106+
python3 irods_consortium_continuous_integration_build_hook.py --build_directory _build
107+
108+
- name: List contents of build directory
109+
run: ls -l _build
110+
111+
# The server and icommands packages were built and cached while inside of a docker container.
112+
# We cannot use actions/cache to get the server and icommands packages in the next job because
113+
# GHA ties the cache to the runner's OS. For this workflow, the run_tests job does not run inside
114+
# of a container. This leads to a cache miss, causing the workflow to fail. To get around that, we
115+
# use upload-artifact and download-artifact.
116+
- name: Upload iRODS packages for dependent jobs
117+
uses: actions/upload-artifact@v6
118+
with:
119+
name: irods-packages-prerelease-${{ inputs.build_artifact_suffix }}
120+
path: |
121+
_build/*.rpm
122+
_build_irods/*.rpm
123+
_build_icommands/*.rpm
124+
overwrite: true
125+
126+
run_tests:
127+
needs: build_plugin
128+
129+
name: Test plugin - ${{ inputs.irods_testing_environment_project_os }}-${{ inputs.irods_testing_environment_project_db }}
130+
131+
runs-on: ubuntu-latest
132+
133+
steps:
134+
- name: Checkout repository
135+
uses: actions/checkout@v5
136+
137+
- name: Set up python for iRODS testing environment
138+
uses: actions/setup-python@v6
139+
with:
140+
python-version: '3.8'
141+
142+
- name: Download iRODS packages
143+
uses: actions/download-artifact@v7
144+
with:
145+
name: irods-packages-prerelease-${{ inputs.build_artifact_suffix }}
146+
path: build_artifacts
147+
148+
- name: Set up docker compose
149+
uses: docker/setup-compose-action@v1
150+
151+
- name: Set up iRODS testing environment
152+
run: |
153+
git clone -b main --depth 1 --single-branch https://github.com/irods/irods_testing_environment
154+
cd irods_testing_environment
155+
python3 -m venv venv_testing_env
156+
. venv_testing_env/bin/activate
157+
python --version
158+
pip install docker-compose GitPython
159+
pip install docker'<7' requests==2.26.0
160+
pip freeze
161+
162+
- name: Run tests
163+
run: |
164+
# Create a symlink to the directory containing the recently built plugin packages.
165+
# This is required by the irods_python_irods_ci_utilities module.
166+
ln -s build_artifacts/_build "${{ inputs.plugin_package_directory }}"
167+
168+
# Move the icommands package into the same directory containing the server packages.
169+
# This is required by the testing environment.
170+
mv build_artifacts/_build_icommands/*.rpm build_artifacts/_build_irods
171+
172+
cd irods_testing_environment
173+
. venv_testing_env/bin/activate
174+
project_dir="${{ inputs.irods_testing_environment_project_os }}/${{ inputs.irods_testing_environment_project_os }}-${{ inputs.irods_testing_environment_project_db }}"
175+
python run_plugin_tests.py \
176+
"${GITHUB_REPOSITORY#${GITHUB_REPOSITORY_OWNER}/}" \
177+
--project-directory "projects/${project_dir}" \
178+
--irods-package-directory ../build_artifacts/_build_irods \
179+
--plugin-package-directory .. \
180+
--discard-logs \
181+
-v

0 commit comments

Comments
 (0)