Setup GH Action to create Self Hosted GH Runners on the STFC Cloud

.github/workflows/create-gh-action-runners.yaml

@@ -0,0 +1,107 @@
+name: Create Github Action Runners
+on: 
+  workflow-dispatch:
+    inputs:
+      runner_count:
+        description: "Number of Runners to Deploy"
+        required: true
+        default: 5
+
+      runner_flavor:
+        description: "The Flavor of VM to create. l3.nano (More Runners, Less Power) | l3.micro (Middle Ground) | l3.tiny (Less Runners, More Power)"
+        required: true
+        default: l3.nano
+
+jobs:
+  apply-terraform:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: self-hosted-gh-runners/static/terraform
+    outputs:
+      vm_ips: ${{ steps.check-vms.outputs.vm-ips }}
+    env:
+      OS_CLIENT_CONFIG_FILE: ${{ github.workspace }}/clouds.yaml
+      IMAGE_NAME: github-action-runner-image
+      FLAVOR_NAME: l3.nano
+      TF_VAR_runner_count: ${{ github.events.inputs.runner_count }}
+      TF_VAR_flavor_name: ${{ github.events.inputs.runner_flavor }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v6
+
+      - name: Create clouds.yaml
+        run: echo "${{ secrets.OPENSTACK_CLOUDS }}" >> ${{ github.workspace }}/clouds.yaml
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: 1.14.3
+          terraform_wrapper: false
+
+      - name: Get terraform.tfstate
+        run: |
+          vault login ${{ secrets.VAULT_TOKEN }}
+          vault kv get -address="https://secrets.isis.rl.ac.uk" -mount="fase-ci-cd" -field="tfstate" "tfstate" > terraform.tfstate
+
+      - name: Terraform init
+        run: terraform init
+
+      - name: Terraform plan
+        run: terraform plan -out=tfplan 1> /dev/null
+
+      - name: Terraform apply
+        run: terraform apply -auto-approve tfplan 1> /dev/null
+
+      - name: Create hosts.ini & Crete list of IPs
+        id: check-vms
+        run: |
+          echo $(terraform output -json vm_ips) | jq -r '.[]' >> ../ansible/hosts.ini \
+          && echo "vm_ips=$(echo $(terraform output -json vm_ips) | jq -r '.[]')" >> $GITHUB_OUTPUT
+
+      - name: Put terraform.tfstate
+        run: |
+          vault login ${{ secrets.VAULT_TOKEN }}
+          vault kv put -address="https://secrets.isis.rl.ac.uk" -mount="fase-ci-cd" "tfstate" tfstate="@terraform.tfstate"
+
+      - name: Setup SSH
+        env:
+          VM_IPS: ${{ steps.export.outputs.check-vms.vm_ips}}
+          FASE_SSH_KEY: ${{ secrets.FASE_SSH_KEY}}
+        run: |
+          mkdir -p ~/.ssh/
+          echo -e "$FASE_SSH_KEY" > ~/.ssh/id_rsa
+          chmod 400 ~/.ssh/id_rsa
+          eval $(ssh-agent -s)
+          ssh-add ~/.ssh/id_rsa
+          echo "$VM_IPS" >> ../ansible/hosts.txt
+          sed -i '$!s/$/,/' ../ansible/hosts.txt
+          ssh-keyscan -f ../ansible/hosts.txt >> ~/.ssh/known_hosts
+
+      - name: Configure the Runners
+        env: 
+          ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+        run: |
+          cat > /tmp/extra_vars.yml << EOF
+          ACCESS_TOKEN: "$ACCESS_TOKEN"
+          ORG: "${{ github.repository_owner }}"
+          EOF
+
+          ansible-playbook \ 
+          --private-key ~/.ssh/id_rsa \
+          -u ${{ secrets.ANSIBLE_DEPLOY_USER }} \
+          -i ../ansible/hosts.ini \ 
+          ../ansible/create-gh-action-runners.yaml \ 
+          --extra-vars "@/tmp/extra_vars.yml"
+
+          rm -f /tmp/extra_vars.yml
+
+      - name: Put hosts.ini
+        env:
+          VM_IPS: ${{ steps.export.outputs.check-vms.vm_ips}}
+        run: |
+          vault login ${{ secrets.VAULT_TOKEN }}
+          vault kv put -address="https://secrets.isis.rl.ac.uk" -mount="fase-ci-cd" "hosts" hosts="$VM_IPS"
+
+
+

.github/workflows/create-gh-runner-image.yaml

@@ -0,0 +1,82 @@
+name: create-gh-action-runner-image.yaml
+on:
+  workflow-dispatch:
+
+jobs:
+  packer:
+    runs-on: ubuntu-latest
+    defaults:
+      run: 
+        working-directory: self-hosted-gh-runners/static/packer
+    env:
+      OS_CLOUD: openstack
+      IMAGE_NAME: gh-action-runner-image
+      BASE_IMAGE_NAME: ubuntu-noble-24.04-nogui
+      FLAVOR_NAME: l3.nano
+      NETWORK_NAME: ua-ci-cd
+      FLOATING_IP: ${{ secrets.PACKER_FLOATING_IP }}
+      OS_CLIENT_CONFIG_FILE: ${{ github.workspace }}/clouds.yaml
+      GITHUB_ORG: ${{ github.repository_owner }}
+
+    outputs:
+      base_image_id: ${{ steps.export.outputs.base_image_id }}
+      flavor_id: ${{ steps.export.outputs.flavor_id }}
+      network_id: ${{ steps.export.outputs.network_id }}
+      floating_ip_id: ${{ steps.export.outputs.floating_ip_id }}
+      runner_version: ${{ steps.get-runner-version.outputs.runner-version }}
+
+    steps:
+    - name: Checkout repo
+      uses: actions/checkout@v6
+
+    - name: Install OpenStack CLI
+      run: |
+        sudo apt update
+        sudo apt install -y python3-pip
+        pip3 install python-openstackclient
+
+    - name: Create clouds.yaml
+      run: echo "${{ secrets.OPENSTACK_CLOUDS }}" >> ${{ github.workspace }}/clouds.yaml 
+
+    - name: Export Base Image, Build VM Flavor, Network & Floating IP IDs
+      id: export
+      run: |
+        echo "Export Base Image ID"
+        echo "base_image_id=$(openstack image show ${{ env.BASE_IMAGE_NAME }} -f value -c id)" >> "$GITHUB_OUTPUT"
+
+        echo "Export Build VM Flavor ID"
+        echo "flavor_id=$(openstack flavor show ${{ env.FLAVOR_NAME }} -f value -c id)" >> "$GITHUB_OUTPUT"
+
+        echo "Export Network ID"
+        echo "network_id=$(openstack network show ${{ env.NETWORK_NAME }} -f value -c id)" >> "$GITHUB_OUTPUT"
+
+        echo "Export Floating IP ID"
+        echo "floating_ip_id=$(openstack floating ip show ${{ env.FLOATING_IP }} -f value -c id)" >> "$GITHUB_OUTPUT"
+
+    - name: Get Latest Runner version
+      id: get-runner-version
+      run: |
+        echo "Get Latest Runner Version"
+        echo "runner_version=$(curl -s https://api.github.com/repos/actions/runner/releases/latest | grep '"tag_name":' | sed -E 's/.*"v?([^"]+)".*/\1/')" >> "$GITHUB_OUTPUT"
+
+    - name: Setup packer
+      uses: hashicorp/setup-packer@main
+      with:
+        version: latest
+
+    - name: Packer Init
+      run: packer init .
+
+    - name: Packer Validate & Build
+      env:
+        BASE_IMAGE_ID: ${{ steps.export.outputs.base_image_id }}
+        FLAVOR_ID: ${{ steps.export.outputs.flavor_id }}
+        NETWORK_ID: ${{ steps.export.outputs.network_id }}
+        FLOATING_IP_ID: ${{ steps.export.outputs.floating_ip_id }}
+        RUNNER_VERSION: ${{ steps.get-runner-version.outputs.runner_version }}
+      run: |
+        packer validate .
+        packer build .
+
+    - name: Delete old images
+      run: ./scripts/delete_old_images.sh

.github/workflows/destroy-gh-action-runners.yaml

@@ -0,0 +1,80 @@
+name: apply-sh-gh-runner-config
+on: 
+  workflow-dispatch:
+
+jobs:
+  apply-terraform:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: self-hosted-gh-runners/static/terraform
+    outputs:
+      vm_ips: ${{ steps.check-vms.outputs.vm-ips }}
+    env:
+      OS_CLIENT_CONFIG_FILE: ${{ github.workspace }}/clouds.yaml
+      ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v6
+
+      - name: Create clouds.yaml
+        run: echo "${{ secrets.OPENSTACK_CLOUDS }}" >> ${{ github.workspace }}/clouds.yaml
+
+      - name: Get terraform.tfstate
+        run: |
+          vault login ${{ secrets.VAULT_TOKEN }}
+          vault kv get -address="https://secrets.isis.rl.ac.uk" -mount="fase-ci-cd" -field="hosts" "hosts" >> ../ansible/hosts.txt
+
+      - name: Setup SSH
+        env:
+          VM_IPS: ${{ steps.export.outputs.check-vms.vm_ips}}
+          FASE_SSH_KEY: ${{ secrets.FASE_SSH_KEY}}
+        run: |
+          mkdir -p ~/.ssh/
+          echo -e "$FASE_SSH_KEY" > ~/.ssh/id_rsa
+          chmod 400 ~/.ssh/id_rsa
+          eval $(ssh-agent -s)
+          ssh-add ~/.ssh/id_rsa
+          echo "$VM_IPS" >> ../ansible/hosts.txt
+          sed -i '$!s/$/,/' ../ansible/hosts.txt
+          ssh-keyscan -f ../ansible/hosts.txt >> ~/.ssh/known_hosts
+
+      - name: Run GitHub runner setup playbook
+        env:
+          ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+        run: |
+          cat > /tmp/extra_vars.yml << EOF
+          ACCESS_TOKEN: "$ACCESS_TOKEN"
+          ORG: "${{ github.repository_owner }}"
+          EOF
+
+          ansible-playbook \
+          --private-key ~/.ssh/id_rsa \
+          -i ../ansible/hosts.ini \
+          ../ansible/destroy-gh-action-runners.yaml \
+          --extra-vars "@/tmp/extra_vars.yml"
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: 1.14.3
+          terraform_wrapper: false
+
+      - name: Get terraform.tfstate
+        run: |
+          vault login ${{ secrets.VAULT_TOKEN }}
+          vault kv get -address="https://secrets.isis.rl.ac.uk" -mount="fase-ci-cd" -field="tfstate" "tfstate" > terraform.tfstate
+
+      - name: Terraform init
+        run: terraform init
+
+      - name: Terraform destroy
+        run: terraform destroy -auto-approve 1> /dev/null
+
+      - name: Put terraform.tfstate
+        run: |
+          vault login ${{ secrets.VAULT_TOKEN }}
+          vault kv put -address="https://secrets.isis.rl.ac.uk" -mount="fase-ci-cd" "tfstate" tfstate="@terraform.tfstate"
+
+
+

.gitignore

@@ -0,0 +1,9 @@
+# Ignore any OpenStack cloud application creds
+clouds.yaml
+
+# Ignore any Terraform State / Init 
+terraform.*
+*.terraform*
+
+# Ignore Ansible hosts.ini
+hosts.ini

self-hosted-gh-runners/README.md

@@ -29,3 +29,10 @@ GitHub workflow_job webhook can trigger creation & destruction of VMs & runners
 6. remove runner
 7. terraform removes VM
 
+## Steps to implement
+
+1. Recreate the static deployment done via jenkins
+2. Test it, of course!
+
+3. Create a service to listen to workflow_job webook.
+4. Set the service to trigger terraform changes

self-hosted-gh-runners/runner-image/packer/image.pkr.hcl

@@ -0,0 +1,47 @@
+packer {
+    required_plugins {
+        openstack = {
+            version = "~> 1"
+            source = "github.com/hashicorp/openstack"
+        }
+        ansible = {
+            version = "~> 1.1.4"
+            source  = "github.com/hashicorp/ansible"
+        }
+    }
+}
+
+source "openstack" "gh_action_runner_image" {
+    ssh_username = "ubuntu"
+    cloud = "openstack"
+    source_image = var.source_image
+    flavor = var.flavor
+    networks = [var.network]
+    floating_ip = var.floating_ip
+    image_name = var.new_image
+    image_visibility = "private"
+    metadata = {
+        built_by = "packer"
+        team = "ua-ci-cd"
+        usage = "Self Hosted GH Action Runners"
+    }
+}
+
+build {
+    name = "openstack-image-build"
+    sources = ["source.openstack.gh_action_runner_image"]
+
+    provisioner "file" {
+        sources = ["scripts/configure.sh", "scripts/docker_prune.sh", "scripts/job_started.sh", "scripts/job_completed.sh", "scripts/remove.sh"]
+        destination = "~/"
+    }
+
+    provisioner "shell" {
+        script = "scripts/setup_vm.sh"
+    }
+
+    provisioner "ansible-local" {
+        playbook_file = "playbooks/install_dependencies.yaml"
+        extra_arguments = ["--extra-vars", "RUNNER_VERSION=${var.runner_version}"]
+    }  
+}

self-hosted-gh-runners/runner-image/packer/playbooks/install_dependencies.yaml

@@ -0,0 +1,41 @@
+- name: Install dependencies
+  remote_user: ubuntu
+  hosts: 127.0.0.1
+  connection: local
+
+  tasks:
+    - name: Install docker
+      become: true
+      ansible.builtin.apt:
+        name: docker.io
+        state: present
+
+    - name: Install docker buildkit features
+      become: true
+      ansible.builtin.apt:
+        name: docker-buildx
+        state: present
+
+    - name: Install GitHub CLI
+      become: true
+      ansible.builtin.apt:
+        name: gh
+        state: present
+
+    - name: Setup docker user
+      become: true
+      ansible.builtin.user:
+        name: ubuntu
+        groups: docker
+        append: yes
+
+    - name: Extract Runner
+      ansible.builtin.unarchive:
+        src: https://github.com/actions/runner/releases/download/v{{RUNNER_VERSION}}/actions-runner-linux-x64-{{RUNNER_VERSION}}.tar.gz
+        dest: ~/actions-runner
+        remote_src: yes
+
+    - name: Install dependencies
+      ansible.builtin.shell:
+        chdir: ~/actions-runner
+        cmd: sudo ./bin/installdependencies.sh