From d6de0737ca3da8243381161a77d55ac2826e161c Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Wed, 19 Nov 2025 09:27:21 -0600 Subject: [PATCH] Moved package build and deployment test workflows and scripts out of public buildscripts repo We no longer use these here so remove them. Ticket: ENT-13521 Changelog: none --- .../workflows/build-using-buildscripts.yml | 157 ------------------ .github/workflows/deployment-tests.yml | 145 ---------------- ci/Dockerfile-cfengine-build-package | 10 -- ci/Dockerfile-cfengine-deployment-tests | 3 - ci/README | 73 -------- ci/build.sh | 52 ------ ci/clean-build-package.sh | 14 -- ci/clean-deployment-tests.sh | 6 - ci/deployment-tests.sh | 63 ------- ci/docker-build-package.sh | 67 -------- ci/docker-deployment-tests.sh | 28 ---- ci/docker.sh | 1 - ci/setup-projects.sh | 61 ------- ci/setup.sh | 47 ------ ci/shell.sh | 2 - 15 files changed, 729 deletions(-) delete mode 100644 .github/workflows/build-using-buildscripts.yml delete mode 100644 .github/workflows/deployment-tests.yml delete mode 100644 ci/Dockerfile-cfengine-build-package delete mode 100644 ci/Dockerfile-cfengine-deployment-tests delete mode 100644 ci/README delete mode 100755 ci/build.sh delete mode 100755 ci/clean-build-package.sh delete mode 100755 ci/clean-deployment-tests.sh delete mode 100755 ci/deployment-tests.sh delete mode 100755 ci/docker-build-package.sh delete mode 100755 ci/docker-deployment-tests.sh delete mode 120000 ci/docker.sh delete mode 100755 ci/setup-projects.sh delete mode 100755 ci/setup.sh delete mode 100755 ci/shell.sh diff --git a/.github/workflows/build-using-buildscripts.yml b/.github/workflows/build-using-buildscripts.yml deleted file mode 100644 index 3de7ecd6d..000000000 --- a/.github/workflows/build-using-buildscripts.yml +++ /dev/null @@ -1,157 +0,0 @@ -name: Build dependencies and CFEngine hub package - -on: - workflow_call: - inputs: - additional_artifacts: - description: "Additional files or directories to include in artifacts" - default: "" - required: false - type: string - - secrets: - GH_ACTIONS_SSH_DEPLOY_KEY_ENTERPRISE_REPO: - required: true - GH_ACTIONS_SSH_DEPLOY_KEY_NOVA_REPO: - required: true - GH_ACTIONS_SSH_DEPLOY_KEY_MISSION_PORTAL_REPO: - required: true - GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE: - required: true - -permissions: - contents: read - -jobs: - build_cfengine_hub_package: - name: Build package - runs-on: ubuntu-22.04 - steps: - - name: Checkout Together Action - uses: actions/checkout@v3 - with: - repository: cfengine/together-javascript-action - ref: main - ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_TOGETHER_REPO }} - ssh-known-hosts: github.com - - - name: Action step - uses: ./ - id: together - with: - myToken: ${{ secrets.GITHUB_TOKEN }} - - - name: Checkout Core - uses: actions/checkout@v3 - with: - repository: cfengine/core - path: core - ref: ${{steps.together.outputs.core || github.base_ref}} - submodules: recursive - - - name: Checkout Masterfiles - uses: actions/checkout@v3 - with: - repository: cfengine/masterfiles - path: masterfiles - ref: ${{steps.together.outputs.masterfiles || github.base_ref}} - - - name: Checkout Buildscripts - uses: actions/checkout@v3 - with: - repository: cfengine/buildscripts - path: buildscripts - fetch-depth: 20 - - - name: Checkout Nova - uses: actions/checkout@v3 - with: - repository: cfengine/nova - path: nova - ref: ${{steps.together.outputs.nova || github.base_ref}} - ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_NOVA_REPO }} - ssh-known-hosts: github.com - - - name: Checkout Enterprise - uses: actions/checkout@v3 - with: - repository: cfengine/enterprise - path: enterprise - ref: ${{steps.together.outputs.enterprise || github.base_ref}} - submodules: recursive - ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_ENTERPRISE_REPO }} - ssh-known-hosts: github.com - - - name: Checkout Mission Portal - uses: actions/checkout@v3 - with: - repository: cfengine/mission-portal - path: mission-portal - ref: ${{steps.together.outputs.mission-portal || github.base_ref}} - submodules: recursive - ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_MISSION_PORTAL_REPO }} - ssh-known-hosts: github.com - - - name: get PACKAGE_SHA for package cache - run: echo "PACKAGE_SHA=$(buildscripts/ci/package-sha.sh)" | tee -a ${GITHUB_ENV} - - - name: get SHA of buildscripts/deps-packaging last commit - run: echo "DEPS_SHA=$(git log --pretty='format:%h' -1 -- .)" | tee -a ${GITHUB_ENV} - working-directory: buildscripts/deps-packaging - - - name: restore artifacts cache - uses: actions/cache/restore@v3 - with: - path: artifacts - key: artifacts-${{ env.PACKAGE_SHA }} - restore-keys: | - artifacts-${{ env.PACKAGE_SHA }} - - - name: Restore dependency cache - uses: actions/cache/restore@v3 - with: - path: cache - key: deps-${{ github.base_ref }}-${{ env.DEPS_SHA }} - restore-keys: | - deps-${{ github.base_ref }} - deps-master - deps - - - name: restore configured and built core and nova projects - uses: actions/cache/restore@v3 - with: - path: build - key: build-${{ env.PACKAGE_SHA }} - restore-keys: | - build-${{ env.PACKAGE_SHA }} - - - name: Build package in docker - env: - GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE: ${{ secrets.GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE }} - run: | - if [ ! -f artifacts/cfe*deb ]; then - buildscripts/ci/docker-build-package.sh ${{ inputs.additional_artifacts }} - fi - - - name: Save dependency cache - uses: actions/cache/save@v3 - with: - path: cache - key: deps-${{ github.base_ref }}-${{ env.DEPS_SHA }} - - - name: Save artifacts cache - uses: actions/cache/save@v3 - with: - path: | - artifacts - packages - key: artifacts-${{ env.PACKAGE_SHA }} - - - name: Save artifacts - if: success() || failure() - uses: actions/upload-artifact@v4 - with: - name: artifacts - path: | - artifacts - packages diff --git a/.github/workflows/deployment-tests.yml b/.github/workflows/deployment-tests.yml deleted file mode 100644 index ed0868e15..000000000 --- a/.github/workflows/deployment-tests.yml +++ /dev/null @@ -1,145 +0,0 @@ -name: Deployment tests for built package - -on: - workflow_call: - secrets: - GH_ACTIONS_SSH_DEPLOY_KEY_ENTERPRISE_REPO: - required: true - GH_ACTIONS_SSH_DEPLOY_KEY_NOVA_REPO: - required: true - GH_ACTIONS_SSH_DEPLOY_KEY_MISSION_PORTAL_REPO: - required: true - GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE: - required: true - -permissions: - contents: read - pull-requests: read - -jobs: - deployment_tests: - name: Run simple deployment tests - runs-on: ubuntu-22.04 - steps: - - name: Checkout Together Action - uses: actions/checkout@v3 - with: - repository: cfengine/together-javascript-action - ref: main - ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_TOGETHER_REPO }} - ssh-known-hosts: github.com - - - name: Action step - uses: ./ - id: together - with: - myToken: ${{ secrets.GITHUB_TOKEN }} - - - name: Checkout Core - uses: actions/checkout@v3 - with: - repository: cfengine/core - path: core - ref: ${{steps.together.outputs.core || github.base_ref}} - submodules: recursive - - - name: Checkout Masterfiles - uses: actions/checkout@v3 - with: - repository: cfengine/masterfiles - path: masterfiles - ref: ${{steps.together.outputs.masterfiles || github.base_ref}} - - - name: Checkout Buildscripts - uses: actions/checkout@v3 - with: - repository: cfengine/buildscripts - path: buildscripts - ref: ${{steps.together.outputs.buildscripts || github.base_ref}} - fetch-depth: 20 - - - name: Checkout Nova - uses: actions/checkout@v3 - with: - repository: cfengine/nova - path: nova - ref: ${{steps.together.outputs.nova || github.base_ref}} - ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_NOVA_REPO }} - ssh-known-hosts: github.com - - - name: Checkout Enterprise - uses: actions/checkout@v3 - with: - repository: cfengine/enterprise - path: enterprise - ref: ${{steps.together.outputs.enterprise || github.base_ref}} - submodules: recursive - ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_ENTERPRISE_REPO }} - ssh-known-hosts: github.com - - - name: Checkout Mission Portal - uses: actions/checkout@v3 - with: - repository: cfengine/mission-portal - path: mission-portal - ref: ${{steps.together.outputs.mission-portal || github.base_ref}} - submodules: recursive - ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_MISSION_PORTAL_REPO }} - ssh-known-hosts: github.com - - - name: get PACKAGE_SHA for package cache - run: echo "PACKAGE_SHA=$(buildscripts/ci/package-sha.sh)" | tee -a ${GITHUB_ENV} - - - name: get SHA of buildscripts/deps-packaging last commit - run: echo "DEPS_SHA=$(git log --pretty='format:%h' -1 -- .)" | tee -a ${GITHUB_ENV} - working-directory: buildscripts/deps-packaging - - - name: restore artifacts cache - uses: actions/cache/restore@v3 - with: - path: artifacts - key: artifacts-${{ env.PACKAGE_SHA }} - restore-keys: | - artifacts-${{ env.PACKAGE_SHA }} - - - name: Restore dependency cache - uses: actions/cache/restore@v3 - with: - path: cache - key: deps-${{ github.base_ref }}-${{ env.DEPS_SHA }} - restore-keys: | - deps-${{ github.base_ref }} - deps-master - deps - - - name: Build package in docker - id: build_package - env: - GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE: ${{ secrets.GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE }} - run: | - if [ ! -f artifacts/cfe*deb ]; then - buildscripts/ci/docker-build-package.sh - fi - - - name: Run deployment tests - run: buildscripts/ci/docker-deployment-tests.sh - - - name: Save dependency cache - uses: actions/cache/save@v3 - with: - path: cache - key: deps-${{ github.base_ref }}-${{ env.DEPS_SHA }} - - - name: Save artifacts cache - uses: actions/cache/save@v3 - with: - path: artifacts - key: artifacts-${{ env.PACKAGE_SHA }} - - - name: Save artifacts - if: success() || failure() - uses: actions/upload-artifact@v4 - with: - name: deployment-test-artifacts - path: | - artifacts diff --git a/ci/Dockerfile-cfengine-build-package b/ci/Dockerfile-cfengine-build-package deleted file mode 100644 index c38137afd..000000000 --- a/ci/Dockerfile-cfengine-build-package +++ /dev/null @@ -1,10 +0,0 @@ -FROM ubuntu:22.04 -RUN cat /etc/resolv.conf -RUN bash -c 'echo "test" >/dev/tcp/archive.ubuntu.com/80' -RUN apt-get update -y && apt-get install -y systemd wget sudo -# some ways of debugging update problems below, uncomment to try, see ENT-11871 for archive.ubuntu.com flakes -#RUN apt-get update -y -o Debug::Acquire::http=true -#RUN apt-get install -y -o Debug::Acquire::http=true systemd wget sudo -ADD setup.sh / -RUN /bin/bash -c '/setup.sh 2>&1 > setup.log' -CMD [ "/lib/systemd/systemd" ] diff --git a/ci/Dockerfile-cfengine-deployment-tests b/ci/Dockerfile-cfengine-deployment-tests deleted file mode 100644 index 59c54ed7b..000000000 --- a/ci/Dockerfile-cfengine-deployment-tests +++ /dev/null @@ -1,3 +0,0 @@ -FROM ubuntu:22.04 -RUN apt-get update -y && apt-get install -y systemd sudo -CMD [ "/lib/systemd/systemd" ] diff --git a/ci/README b/ci/README deleted file mode 100644 index f8b0a1342..000000000 --- a/ci/README +++ /dev/null @@ -1,73 +0,0 @@ -In this 'buildscripts/ci' directory you will find the means to build cfengine packages. -Note that you should ensure that the various projects are all on the same version: core, enterprise, masterfiles, nova, mission-portal all should be on say 3.21.x or 3.18.x or master. - -Currently a full build with no dependencies cached takes around 53 minutes with deps and packaging. - -Two options: containerize build or "normal" machine (such as virtual machine or actual hardware). - -The build is designed for ubuntu-22.04 but could be adjusted in various shell scripts for other platforms. (TODO: do this!) - -# containerized build -./clean.sh # cleans any leftover docker bits -./run.sh # builds a docker image, sets it up, runs it in the background, copies local cache into container, runs the build and saves the results -./shell.sh # runs bash in the container for debugging - -See /data/buildscripts/ci/build.sh for required environment variables and steps to try manually - -# virtual or real machine - -For virtual machine such as with vagrant, at $NTECH_ROOT (aka top-level directory containing all CFEngine repositories), init an ubuntu-22.04 vagrant machine so it has access to all your repositories. - -vagrant init ubuntu/focal64 -vagrant ssh - -# note however, that currently you will need to copy /vagrant to a non shared filesystem most likely as vboxsf type doesn't support required hard and soft links for packaging :( -set -ex -repos="\ -buildscripts \ -core \ -enterprise \ -masterfiles \ -mission-portal \ -nova \ -" - -mkdir -p $HOME/workspace -pwd -for repo in $repos; do - pwd - cp -R "$repo" $HOME/workspace -done - -# always be careful of trailing slashes with rsync, check it again if you are typing! -rsync -avz /vagrant/cache/ $HOME/.cache/ - - -And then do the following as you would on a real machine: - -cd $HOME/workspace -./buildscripts/ci/setup.sh # install needed dependencies/packages/etc -./buildscripts/ci/setup-projects.sh # items which must be done AFTER the container is created with setup.sh, in non-container case must be executed second -./buildscripts/ci/build.sh # run the build, should generate a package - -don't use ./clean.sh ./run.sh or ./shell.sh those are specifically for containerized/docker build - -If you are running selenium tests in mission-portal, you might need to do a `make -C mission-portal clean` before creating packages as the selenium tests copy some things from the distribution into the repository directory that cause problems with packaging, such as the api directory from nova. - -# manual debugging on-host - -cat buildscripts/ci/build.sh -# now export the environment variables there (and adjust as needed) -export BUILD_TYPE=DEBUG -export ESCAPETEST=yes -export TEST_MACHINE=chroot - -then take each step one-at-a-time or re-run as needed - -./buildscripts/build-scripts/install-dependencies - -Note that you can provide an argument to install-dependencies to just build ONE dependency, such as - -./buildscripts/build-scripts/install-dependencies lmdb - -If you change the version of a dependency the cache for that should be skipped and the dep rebuilt from scratch. diff --git a/ci/build.sh b/ci/build.sh deleted file mode 100755 index 384aa9ef8..000000000 --- a/ci/build.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -# build cfengine hub package -# $@ -- optional space separated paths to copy to artifacts - -set -ex - -additional_artifacts="$@" -export PROJECT=nova -export NO_CONFIGURE=1 -export BUILD_TYPE=DEBUG -export ESCAPETEST=yes -export EXPLICIT_ROLE=hub -export TEST_MACHINE=chroot - -set +x # hide secrets -eval $(ssh-agent -s) -if [ -z "$SECRET" ]; then - echo "Need sftp cache ssh secret key. Provide with SECRET env variable" - exit 1 -else - echo "$SECRET" | ssh-add - -fi -ssh-add -l -set -x # stop hiding secrets - -time ./buildscripts/build-scripts/install-dependencies -time ./buildscripts/build-scripts/configure # 3 minutes locally -time ./buildscripts/build-scripts/generate-source-tarballs # 1m49 -time ./buildscripts/build-scripts/compile -time sudo apt remove -y 'cfbuild*' || true -time sudo apt remove -y 'cfengine-*' || true -time sudo rm -rf /var/cfengine -time sudo rm -rf /opt/cfengine -time ./buildscripts/build-scripts/install-dependencies -time ./buildscripts/build-scripts/package - -sudo mkdir -p artifacts -sudo cp cfengine-nova-hub/*.deb artifacts/ || true -sudo cp cfengine-nova-hub/*.rpm artifacts/ || true - -for artifact_path in $additional_artifacts; do - sudo cp -r "$artifact_path" artifacts/ || true -done - - -# todo maybe save the cache cp -R ~/.cache buildscripts/ci/cache - -# clean up -time sudo apt remove -y 'cfbuild*' || true -time sudo apt remove -y 'cfengine-*' || true -time sudo rm -rf /var/cfengine -time sudo rm -rf /opt/cfengine diff --git a/ci/clean-build-package.sh b/ci/clean-build-package.sh deleted file mode 100755 index 43988ccdb..000000000 --- a/ci/clean-build-package.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -set -ex -# clean up docker stuff -name=cfengine-build-package -# TODO: a softer clean might get into the container and run ./buildscripts/build-scripts/clean-buildmachine -if docker ps | grep $name; then - docker stop $name -fi -if docker ps -a | grep $name; then - docker rm $name -fi -if docker images | grep $name; then - docker rmi -f $name -fi diff --git a/ci/clean-deployment-tests.sh b/ci/clean-deployment-tests.sh deleted file mode 100755 index 30baf84d1..000000000 --- a/ci/clean-deployment-tests.sh +++ /dev/null @@ -1,6 +0,0 @@ -# clean up docker stuff -name=cfengine-deployment-tests -# TODO: a softer clean might get into the container and run ./buildscripts/build-scripts/clean-buildmachine -docker stop $name -docker rm $name -docker rmi $name diff --git a/ci/deployment-tests.sh b/ci/deployment-tests.sh deleted file mode 100755 index 9f14240b8..000000000 --- a/ci/deployment-tests.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC2024 -# I am redirecting many sudo run commands to logfiles which can be owned by the non-priv user -# copied from mission-portal/ci/run.sh for selenium tests -# todo refactor to share some of this instead of copy/pasting -set -ex - -# find the dir one level up from here, home of all the repositories -COMPUTED_ROOT="$(readlink -e "$(dirname "$0")/../../")" -# NTECH_ROOT should be the same, but if available use it so user can do their own thing. -NTECH_ROOT=${NTECH_ROOT:-$COMPUTED_ROOT} -USER=${USER:-$(whoami)} - -# prepare artifacts dir -sudo mkdir -p "${NTECH_ROOT}/artifacts" -sudo chown "$USER" "${NTECH_ROOT}/artifacts" - -trap failure ERR -function failure() { - cd "${NTECH_ROOT}/artifacts" - if command cf-support; then - sudo cf-support --yes > $$.cfsupportlog 2>&1 || cat $$.cfsupportlog - else - tar cf "${NTECH_ROOT}/artifacts/CFEngine-Install.logs.tgz /var/log/CFEngine-Install*" - fi - rm $$.cfsupportlog -} - -if [ ! -d /var/cfengine ]; then - # ci and local buildscripts should place built packages in $NTECH_ROOT/artifacts - sudo dpkg -i "$NTECH_ROOT"/artifacts/cfengine-nova-hub*deb -fi - - - -AGENT_LOG="${NTECH_ROOT}/artifacts/agent.log" -if [ -f "$AGENT_LOG" ]; then - mv "$AGENT_LOG" "${AGENT_LOG}.$(date +%s)" -fi -mkdir -p "${NTECH_ROOT}/artifacts" -touch "$AGENT_LOG" -if [ ! -f /var/cfengine/policy_server.dat ]; then - sudo /var/cfengine/bin/cf-agent -B "$(hostname -I | awk ' {print $1}')" >>"$AGENT_LOG" 2>&1 -fi - -# make artifacts directory to be slurped by CI (jenkins, github, ...) -mkdir -p "${NTECH_ROOT}/artifacts" - -{ - sudo /var/cfengine/bin/cf-agent -KIf update.cf - sudo /var/cfengine/bin/cf-agent -KI - sudo /var/cfengine/bin/cf-agent -KI -} >>"$AGENT_LOG" 2>&1 - -if grep -i error "$AGENT_LOG" >/dev/null; then - echo "FAIL test, errors in $AGENT_LOG" - grep -i error "$AGENT_LOG" -fi - -apt-get -y install python3-psycopg2 -export REPORTING_TEST_DELAY=5 -cd "${NTECH_ROOT}/nova/tests/reporting" -python3 deployment_test.py diff --git a/ci/docker-build-package.sh b/ci/docker-build-package.sh deleted file mode 100755 index 68da103a7..000000000 --- a/ci/docker-build-package.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash -# run the build in a docker container -# $@ -- additional artifact paths to save -additional_artifacts="$@" -set -ex - -# find the dir two levels up from here, home of all the repositories -COMPUTED_ROOT="$(readlink -e "$(dirname "$0")/../../")" -# NTECH_ROOT should be the same, but if available use it so user can do their own thing. -NTECH_ROOT=${NTECH_ROOT:-$COMPUTED_ROOT} - -name=cfengine-build-package -label=PACKAGES_HUB_x86_64_linux_ubuntu_22 -export JOB_BASE_NAME=label=$label - - -docker build -t $name -f "${NTECH_ROOT}/buildscripts/ci/Dockerfile-$name" "${NTECH_ROOT}/buildscripts/ci" - -# add secret key to enable push up to sftp cache -set +x # hide secrets -if [ -n "$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE" ]; then - export SECRET="$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE" -else - if ! export SECRET="$(pass mystiko/developers/CFEngine/jenkins/jenkins_sftp_cache@github)"; then - echo "The sftp cache ssh secret key must be provided, either with environment variable GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE or access to mystiko path developers/CFEngine/jenkins/sftp-cache.sec" - exit 1 - fi -fi -set -x # done hiding secrets -# send in JOB_BASE_NAME to enable use of retrieved or generated deps cache -docker run -d --env SECRET --env JOB_BASE_NAME --privileged -v "${NTECH_ROOT}":/data --name $name $name - -# copy local caches to docker container -mkdir -p "${NTECH_ROOT}/artifacts" -mkdir -p "${NTECH_ROOT}/cache" - -# setup host key trust -pubkey="build-artifacts-cache.cloud.cfengine.com,138.68.18.72 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJhnAXjI9PMuRM3s0isYFH4SNZjKwq0E3VK+7YQKcL6aIxNhXjdJnNKAkh4MNlzZkLpFTYputUxKa1yPPrb5G/Y=" - -# ending with /. in srcpath copies contents to destpath -docker cp "${NTECH_ROOT}/cache/." $name:/root/.cache - -# in order for build-scripts/autogen to generate a revision file: -for i in core buildscripts buildscripts/deps-packaging enterprise nova masterfiles -do - docker exec -i $name bash -c "git config --global --add safe.directory /data/$i" -done - -# add build artifacts host public keys to container for use there -docker exec -i $name bash -c "mkdir -p ~/.ssh" -docker exec -i $name bash -c "echo $pubkey >> ~/.ssh/known_hosts" - -docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/setup-projects.sh' -docker exec -i $name bash -c "cd /data; ./buildscripts/ci/build.sh ${additional_artifacts}" - -# save back cache and artifacts to host for handling by CI and such -docker cp $name:/root/.cache/. "${NTECH_ROOT}/cache/" -docker exec $name find /data/artifacts -docker exec $name ps -efl # to see what is running, once the artifacts dir changed while we tried to tar it up and copy it -docker exec $name tar Ccf $(dirname /data) - $(basename /data/artifacts) | tar Cxf "${NTECH_ROOT}/artifacts/" - - -rc=1 # if we find no packages, fail -for f in artifacts/*.deb; do - [ -f "$f" ] && rc=0 - break -done -exit $rc diff --git a/ci/docker-deployment-tests.sh b/ci/docker-deployment-tests.sh deleted file mode 100755 index 4274b5293..000000000 --- a/ci/docker-deployment-tests.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# run the build in a docker container -set -ex - -# find the dir two levels up from here, home of all the repositories -COMPUTED_ROOT="$(readlink -e "$(dirname "$0")/../../")" -# NTECH_ROOT should be the same, but if available use it so user can do their own thing. -NTECH_ROOT=${NTECH_ROOT:-$COMPUTED_ROOT} - -name=cfengine-deployment-tests -# todo, check the image against the Dockerfile for up-to-date ness? -if ! docker images | grep $name; then - docker build -t $name -f "${NTECH_ROOT}/buildscripts/ci/Dockerfile-$name" . || true -fi - -# todo, check if already running and up-to-date? -# we want a fresh container, stop and remove any that exist by this $name -if docker ps -a | grep $name; then - docker ps -a | grep $name | awk '{print $1}' | xargs docker stop - docker ps -a | grep $name | awk '{print $1}' | xargs docker rm -fi -docker run -d --privileged -v "${NTECH_ROOT}":/data --name $name $name || true - -if [ ! -d "${NTECH_ROOT}/artifacts" ]; then - echo "${NTECH_ROOT}/artifacts directory should exist and have a cfengine-nova-hub package there" - exit 1 -fi -docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/deployment-tests.sh' diff --git a/ci/docker.sh b/ci/docker.sh deleted file mode 120000 index 04af95afb..000000000 --- a/ci/docker.sh +++ /dev/null @@ -1 +0,0 @@ -docker-build-package.sh \ No newline at end of file diff --git a/ci/setup-projects.sh b/ci/setup-projects.sh deleted file mode 100755 index aa77b362e..000000000 --- a/ci/setup-projects.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash - -echo "=== tool versions (npm, node, composer) ===" -which npm -npm --version -which node -node --version -which composer -composer --version - -set -ex -( -if test -f "mission-portal/public/scripts/package.json"; then - cd mission-portal/public/scripts - # install dependencies from npmjs - npm ci - # build react components - npm run build - # remove node_modules since the bundles are already built - # we do not need them to be presented in the package - rm -rf node_modules -fi -) - -# install composer and friends -( -if test -f "mission-portal/composer.json"; then - cd mission-portal - # install PHP dependencies from composer - composer install --no-dev -fi -) - -( -if test -f "nova/api/http/composer.json"; then - cd nova/api/http/ - # install PHP dependencies from composer - composer install --ignore-platform-reqs --no-dev -fi -) - -( -if test -f "mission-portal/public/themes/default/bootstrap/cfengine_theme.less"; then - cd mission-portal/public/themes/default/bootstrap - npx -p less lessc --compress ./cfengine_theme.less ./compiled/css/cfengine.less.css -fi -) - -( -if test -f "mission-portal/ldap/composer.json"; then - cd mission-portal/ldap - # install PHP dependencies from composer - composer install --no-dev -fi -) - -# packages needed for autogen are installed in setup.sh -PROJECT=nova ./buildscripts/build-scripts/autogen - -# remove unwanted dependencies -sudo apt-get -qy purge libltdl-dev libltdl7 #libtool diff --git a/ci/setup.sh b/ci/setup.sh deleted file mode 100755 index c90bc52c3..000000000 --- a/ci/setup.sh +++ /dev/null @@ -1,47 +0,0 @@ -# setup build host on ubuntu 22 -set -ex -PREFIX=/var/cfengine - -# Github Actions provides machines with various packages installed, -# what confuses our build system into thinking that it's an RPM distro. -sudo rm -f /bin/rpm - -# silence noise from debconf about frontends, Dialog, Readline -export DEBIAN_FRONTEND=noninteractive - -# Install dependencies -sudo apt-get update -qy - -# install apt-utils so that debconf can configure installed packages -sudo apt-get install -qy apt-utils - -# git is needed for build-scripts/autogen to determine revision for such things as deps-packaging -sudo apt-get install -qy git - -# python3-pip is needed for cfengine-nova-hub.deb packaging -sudo apt-get install -qy python3 python3-pip - -# Install psycopg2 -apt-get -y install python3-psycopg2 - -# install composer and friends -sudo --preserve-env apt-get -y install curl php-cli php-curl php-zip php-mbstring php-xml php-gd composer php-ldap -# packages needed for autogen -sudo apt-get -qy install git autoconf automake m4 make bison flex \ - binutils libtool gcc g++ libc-dev libpam0g-dev python3 psmisc - -# packages needed for buildscripts -sudo apt-get -qy install libncurses5 rsync -# packages needed for building -sudo apt-get -qy install bison flex binutils build-essential fakeroot ntp \ - dpkg-dev libpam0g-dev python3 debhelper pkg-config psmisc nfs-common - -# remove unwanted packages -sudo apt-get -qq purge apache* "postgresql*" redis* - -# packages needed for installing Mission portal dependencies -# remove any nodejs or node- packages currently in place -sudo apt-get remove -qy 'nodejs*' 'node-*' -# replace with exact version we want -wget -O - https://deb.nodesource.com/setup_20.x | sudo -E bash - -sudo apt-get install -qy nodejs diff --git a/ci/shell.sh b/ci/shell.sh deleted file mode 100755 index e5f5d8171..000000000 --- a/ci/shell.sh +++ /dev/null @@ -1,2 +0,0 @@ -# shell into the docker container -docker exec -it cfengine-build-package bash