release.yaml

# .github/workflows/release.yml
on:
  workflow_call:
    inputs:
      name:
        required: true
        type: string
      target:
        required: true
        type: string
      additional_config:
        type: string
        required: false
        default: ""
      additional_profile:
        type: string
        required: false
        default: ""

      generated_model_path:
        type: string
        required: true

      snakemake_dir:
        type: string
        required: false
        default: "."

      main_runner:
        type: string
        required: false
        default: "aiinfn-lamarrsim-gpu"

      model_prefix:
        type: string
        required: false
        default: "trk"

    secrets:
        SECRETS_YAML:
            required: true

jobs:
  plan:
      runs-on: ubuntu-latest
      permissions:
        contents: write
        pull-requests: write

      steps:
        - name: Notify of the release in the PR
          uses: actions/github-script@v7
          with:
            script: |
              const releaseUrl = `https://github.com/${{ github.repository }}/releases/tag/${{ inputs.name }}-${{ env.RELEASE_TIMESTAMP }}`;
              const body = `
              ## :robot: A new training is being planned.

               * Name: ${{ inputs.name }}
               * Repository sub-dir: ${{ inputs.snakemake_dir }}
               * Snakemake targets: ${{ inputs.target }}
               * Selected runner: ${{ inputs.main_runner }}
              
              At the end of the training, the models will be released and this PR will be notified again.
              `;
              await github.rest.issues.createComment({
                issue_number: context.issue.number,
                owner: context.repo.owner,
                repo: context.repo.repo,
                body: body
              });

  train-and-validate:
    runs-on: ${{ inputs.main_runner }}
    timeout-minutes: 1500 # a bit more than one day
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Restore file timestamps
        run: |
            git log -1 --format="%at" -- notebooks/workflow/envs/lhcb.def

      - name: Retrieve secrets
        run: |
            mkdir -p /tmp/artifacts
            cat <<EOF > /tmp/artifacts/secrets.yaml
            ${{ secrets.SECRETS_YAML }}
            EOF

      - name: Snakemake workflow
        uses: landerlini/apptainer-gha-runner/workflow@main
        with:
            workdir: ${{ inputs.snakemake_dir }}
            target: ${{ inputs.target }}
            report: report-${{ inputs.name }}
            snakefile: workflow/Snakefile
            config: |
                secrets_file: /tmp/artifacts/secrets.yaml

                ${{ inputs.additional_config }}

            profile: |
                rerun-incomplete: True
                keep-going: True

                software-deployment-method: apptainer
                local-storage-prefix: /tmp
                apptainer-args: -B /home -B /cvmfs -B /tmp --nv --writable-tmpfs --no-mount /usr
                keep-storage-local-copies: true

                jobs: 1
                cores: 8

                default-resources:
                  mem_mb: 2000
                  millicpu: 1000

                resources:
                  mem_mb: 64000
                  millicpu: 32000
                  gpu: 1

                ${{ inputs.additional_profile }}

        
      - name: Upload raw model
        uses: actions/upload-artifact@v4
        with:
          name: compiled_model.c
          path: ${{ inputs.generated_model_path }}
        

  compress-transpiled-c:
    runs-on: ubuntu-latest
  #  container: harbor.cloud.infn.it/unpacked/aiinfn-gha-runner:latest
    needs: train-and-validate
    steps:
      - name: Install missing libraries
        run: |
          sudo apt-get update 
          sudo apt-get install -y xz-utils

      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          name: compiled_model.c
          path: ./artifact

      - name: Compress file
        run: |
            xz -9e -T0 -c -k "$(find ./artifact -type f -name '*.c' | head -n 1)" > compiled_model.c.xz
              
      - name: Upload compressed model
        uses: actions/upload-artifact@v4
        with:
          name: compiled_model.c.xz
          path: ./compiled_model.c.xz


  compile:
    needs: train-and-validate
    strategy:
      matrix:
        include:
          - runner: ubuntu-latest
            arch: x86_64_v2
            compiler: centos7-gcc9-opt 
            container: /cvmfs/lhcb.cern.ch/containers/os-base/centos7-devel/prod/amd64

          - runner: ubuntu-latest
            arch: x86_64_v2
            compiler: el9-gcc13-opt
            container: /cvmfs/lhcb.cern.ch/containers/os-base/alma9-devel/prod/amd64/

          - runner: ubuntu-latest
            arch: x86_64_v2
            compiler: el9-gcc13-dbg
            container: /cvmfs/lhcb.cern.ch/containers/os-base/alma9-devel/prod/amd64/

          - runner: ubuntu-24.04-arm
            arch: armv8.1_a
            compiler: el9-gcc13-opt 
            container: /cvmfs/lhcb.cern.ch/containers/os-base/alma9-devel/prod/arm64v8/



    runs-on: ${{ matrix.runner }}
    
    steps:
      - name: Mount CVMFS
        uses: cvmfs-contrib/github-action-cvmfs@v4

      - name: Build apptainer
        run: |
          git clone https://github.com/apptainer/apptainer.git
          cd apptainer
          git checkout v1.4.1
          ./mconfig --with-suid
          make -C builddir
          sudo make -C builddir install


      - name: Download transpiled model (uncompressed)
        uses: actions/download-artifact@v4
        with:
          name: compiled_model.c
          path: ./artifact
  
      - name: Compile ${{ inputs.name }} for ${{ matrix.arch }}-${{ matrix.compiler }}
        run: |
          cat <<EOF > compile_cmd.sh
          source /cvmfs/lhcb.cern.ch/lib/LbEnv -c ${{ matrix.arch }}-${{ matrix.compiler }}

          mkdir -p build
          gcc artifact/compiled_model.c \
              -O3 \
              -lm  \
              --shared  \
              -fPIC \
              -std=c99 \
              -o build/${{ inputs.model_prefix }}-${{ matrix.arch }}-${{ matrix.compiler }}.so
          EOF
          apptainer exec -B/home -B/cvmfs -B/tmp  \
              ${{ matrix.container }} \
              /bin/bash compile_cmd.sh

      - name: Upload ${{ inputs.model_prefix }}-${{ matrix.arch }}-${{ matrix.compiler }}
        uses: actions/upload-artifact@v4
        with:
          path: build/${{ inputs.model_prefix }}-${{ matrix.arch }}-${{ matrix.compiler }}.so
          name: ${{ matrix.arch }}-${{ matrix.compiler }}

  release:
      runs-on: ubuntu-latest
      needs: compile
      permissions:
        contents: write
        pull-requests: write
      steps:
        - name: Checkout repository
          uses: actions/checkout@v4

        - name: Download artifact
          uses: actions/download-artifact@v4
          with:
            path: ./artifact

        - name: Download artifact
          run: |
            mkdir -p artifact/test
            echo "test" > artifact/test/test.so


        - name: Install GitHub CLI
          run: sudo apt-get update && sudo apt-get install -y gh

        - name: Fetch latest merged PR description
          env:
            GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          run: |
            echo "RELEASE_TIMESTAMP=$(date +'%Y-%m-%dT%Hh%Mm%S')" >> $GITHUB_ENV
            gh pr list --head ${{ github.head_ref }} --limit 1 --json number,body \
              --jq '.[0].body' > pr_description.md

        - name: List artifacts
          env:
            GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          run: |
            find artifact/


        - name: Create GitHub Release
          env:
            GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          uses: ncipollo/release-action@v1
          with:
            tag: ${{ inputs.name }}-${{ env.RELEASE_TIMESTAMP }}
            name: "Models ${{ github.base_ref }} trained for ${{ inputs.name }}-${{ env.RELEASE_TIMESTAMP }}"
            bodyFile: pr_description.md
            artifacts: artifact/*/*.so,artifact/*/*.html,artifact/*/*.xz,artifact/*/*.zip
            artifactContentType: raw


        - name: Notify of the release in the PR
          uses: actions/github-script@v7
          with:
            script: |
              const releaseUrl = `https://github.com/${{ github.repository }}/releases/tag/${{ inputs.name }}-${{ env.RELEASE_TIMESTAMP }}`;
              const body = `
              ## :rocket: Models for ${{ inputs.name }} were released
              You can review the models developed in this PR in [Release ${{ inputs.name }}-${{ env.RELEASE_TIMESTAMP }}](${releaseUrl})
              `;
              await github.rest.issues.createComment({
                issue_number: context.issue.number,
                owner: context.repo.owner,
                repo: context.repo.repo,
                body: body
              });