ci.yml

name: "Testing ArkScript"

on:
  push:
    branches: [ dev ]
    paths-ignore:
      - '.github/workflows/benchmarks.yml'
      - '.github/workflows/docker.yml'
      - '.github/workflows/label.yml'
      - '.github/workflows/lizard.yml'
      - '.github/workflows/release.yml'
      - '.vscode/*.*'
      - 'examples/*.ark'
      - '.github/images/*.*'
      - '*.md'
      - 'docs/*.*'
      - 'Dockerfile'
      - '.dockerignore'
      - 'LICENCE'
      - '.gitignore'
  pull_request:

permissions:
  pull-requests: write  # Required for commenting on pull requests

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  BUILD_TYPE: Debug
  SQLITE_VERSION: 3390100  # 3.39.1
  EMSDK_VERSION: 4.0.10

jobs:
  check:
    name: Formatting Check
    runs-on: ubuntu-24.04
    strategy:
      matrix:
        path:
          - 'src'
          - 'include'
    steps:
      - uses: actions/checkout@v5
      - name: Run clang-format style check for C/C++/Protobuf programs.
        uses: jidicula/clang-format-action@6cd220de46c89139a0365edae93eee8eb30ca8fe  # v4.16.0
        with:
          clang-format-version: '19'
          check-path: ${{ matrix.path }}
          fallback-style: 'Mozilla' # optional

  check_ark:
    name: ArkScript formatting check
    runs-on: ubuntu-slim

    strategy:
      fail-fast: false
      matrix:
        path:
          - examples

    steps:
      - uses: actions/checkout@v5
      - name: Format check
        uses: ArkScript-lang/action-format@master
        with:
          folder: ${{ matrix.path }}

  build-with-emscripten:
    runs-on: ubuntu-24.04
    name: "Build with emscripten"
    needs: [ check ]

    steps:
      - uses: actions/checkout@v5
        with:
          submodules: recursive

      - name: Setup emsdk
        uses: pyodide/setup-emsdk@ca2dd8aef8c2a0e11743c5c36f0b430ddb694b5c  # v15
        with:
          version: ${{ env.EMSDK_VERSION }}
          actions-cache-folder: 'emsdk-cache'

      - name: Verify
        run: emcc -v

      - name: Compile
        shell: bash
        run: |
          emcmake cmake -Bbuild \
            -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \
            -DARK_SANITIZERS=Off \
            -DARK_COVERAGE=Off \
            -DARK_BUILD_EXE=Off \
            -DARK_BUILD_MODULES=Off \
            -DARK_TESTS=Off \
            -DARK_UNITY_BUILD=On
          cmake --build build --config ${{ env.BUILD_TYPE }} -j $(nproc)

  no-unity-building:
    runs-on: ubuntu-24.04
    name: "Build without Unity build"
    needs: [ check ]

    steps:
      - uses: actions/checkout@v5
        with:
          submodules: recursive

      - name: Setup compilers, dependencies, project and build
        uses: ./.github/workflows/setup-compilers
        with:
          os_name: ubuntu-24.04
          compiler: clang
          compiler_version: 16
          sanitizers: "On"
          with_deps: true
          unity: "Off"

  install-arkscript:
    runs-on: ${{ matrix.os }}
    name: Install ArkScript on ${{ matrix.os }}
    needs: [ check ]

    strategy:
      fail-fast: false
      matrix:
        os:
          - ubuntu-24.04
          - macos-14

    steps:
      - uses: actions/checkout@v5
        with:
          submodules: recursive

      - name: Setup compilers, dependencies, project and build
        uses: ./.github/workflows/setup-compilers
        with:
          os_name: ${{ matrix.os }}
          compiler: clang
          compiler_version: 16
          sanitizers: "On"
          with_deps: false

      - name: Install via cmake
        run: |
          mkdir -p /tmp/ArkScript
          cmake --install build --prefix /tmp/ArkScript
          # to ensure that the executable won't try to load libArkReactor.so/dylib from build/
          rm -rf build

      - name: Test installation
        run: |
          export ARKSCRIPT_PATH=/tmp/ArkScript/lib/Ark
          echo /tmp/ArkScript; ls /tmp/ArkScript
          echo /tmp/ArkScript/bin; ls /tmp/ArkScript/bin
          echo /tmp/ArkScript/lib; ls /tmp/ArkScript/lib
          DYLD_PRINT_LIBRARIES=1 /tmp/ArkScript/bin/arkscript -e '(import std.List) (print list:map " is a function")' || exit 1

  cli:
    runs-on: ${{ matrix.os }}
    name: Test ArkScript CLI on ${{ matrix.os }}
    needs: [ check ]

    strategy:
      fail-fast: false
      matrix:
        os:
          - ubuntu-24.04
          - macos-14

    steps:
      - uses: actions/checkout@v5
        with:
          submodules: recursive

      - name: Setup compilers, dependencies, project and build
        uses: ./.github/workflows/setup-compilers
        with:
          os_name: ${{ matrix.os }}
          compiler: clang
          compiler_version: 16
          sanitizers: "On"
          with_deps: false

      - name: Execute script normally
        run: |
          export ARKSCRIPT_PATH=$(pwd)/lib
          CODE='(import std.Sys) (if (!= ["1" "2" "3"] sys:args) { (print sys:args) (sys:exit 1) })'
          echo $CODE > a.ark
          rm -rf __arkscript__

          ./build/arkscript a.ark 1 2 3
          # the cache should be created
          if ! [ -d __arkscript__ ]; then
            echo "Cache folder doesn't exist"
            exit 1
          fi

      - name: -fno-cache should not create cache folder
        run: |
          export ARKSCRIPT_PATH=$(pwd)/lib
          CODE='(import std.Sys) (if (!= ["1" "2" "3"] sys:args) { (print sys:args) (sys:exit 1) })'
          echo $CODE > a.ark
          rm -rf __arkscript__

          ./build/arkscript -fno-cache a.ark 1 2 3
          # -fno-cache should prevent the cache creation
          if [ -d __arkscript__ ]; then
            echo "Cache folder exists"
            exit 1
          fi

      - name: Take input from stdin
        run: |
          export ARKSCRIPT_PATH=$(pwd)/lib
          CODE='(import std.Sys) (if (!= ["1" "2" "3"] sys:args) { (print sys:args) (sys:exit 1) })'
          echo $CODE > a.ark
          rm -rf __arkscript__

          ./build/arkscript - 1 2 3 <a.ark
          # we read from stdin, there should be no cache created
          if [ -d __arkscript__ ]; then
            echo "Cache folder exists"
            exit 1
          fi

      - name: Take input from CLI with heredoc
        run: |
          export ARKSCRIPT_PATH=$(pwd)/lib
          rm -rf __arkscript__

          ./build/arkscript - 1 2 3 << EOF
          # test comment, should be on its own line and not hide the following import
          (import std.Sys)
          (if (!= ["1" "2" "3"] sys:args)
            {
              (print sys:args)
              (sys:exit 1) })
          EOF
          # there should not be any cache generated when evaluating code from the CLI
          if [ -d __arkscript__ ]; then
            echo "Cache folder exists"
            exit 1
          fi

  check-super-insts-usage:
    runs-on: ubuntu-24.04
    name: Check Super Instructions usage
    needs: [ check ]

    steps:
      - uses: actions/checkout@v5
        with:
          submodules: recursive

      - name: Setup compilers, dependencies, project and build
        uses: ./.github/workflows/setup-compilers
        with:
          os_name: ubuntu-24.04
          compiler: clang
          compiler_version: 16
          sanitizers: "On"
          with_deps: false

      - uses: actions/setup-python@v6
        if: ${{ ! startsWith(matrix.config.name, 'Windows') }}
        with:
          python-version: '3.13'

      - run: |
          echo "SUPER_INSTS_REPORT<<EOF" >> $GITHUB_ENV
          python3 tools/ark_frequent_instructions.py super_insts_usage >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV

      - uses: 8BitJonny/gh-get-current-pr@4.0.0
        id: PR
        with:
          sha: ${{ github.event.pull_request.head.sha }}
          # Only return if PR is still open
          filterOutClosed: true

      - run: |
          echo "PR found: ${prFound}"
          echo "PR ${prNumber:-ERROR} ${prTitle:-NOTITLE}"
        env:
          # Direct access to common PR properties
          prNumber: ${{ steps.PR.outputs.number }}
          prTitle: ${{ steps.PR.outputs.pr_title }}
          prFound: ${{ steps.PR.outputs.pr_found }}

      - name: Find Comment
        uses: peter-evans/find-comment@v4
        if: steps.PR.outputs.pr_found == 'true'
        id: fc
        with:
          issue-number: ${{ github.event.pull_request.number }}
          comment-author: 'github-actions[bot]'
          body-includes: Super Instructions report

      - name: Create or update comment
        uses: peter-evans/create-or-update-comment@v5.0.0
        if: steps.PR.outputs.pr_found == 'true'
        with:
          comment-id: ${{ steps.fc.outputs.comment-id }}
          issue-number: ${{ github.event.pull_request.number }}
          body: |
            ## Super Instructions report
            ${{ env.SUPER_INSTS_REPORT }}
          edit-mode: replace

  build:
    runs-on: ${{ matrix.config.os }}
    name: ${{ matrix.config.name }}
    needs: [ check ]

    strategy:
      fail-fast: false
      matrix:
        config:
          - {
            os: ubuntu-24.04, name: "Ubuntu Clang 16",
            compiler: clang, compiler_version: 16, sanitizers: "On",
            artifact: "ubuntu-clang-16"
          }
          - {
            os: ubuntu-24.04, name: "Ubuntu Clang 16 (valgrind)",
            compiler: clang, compiler_version: 16, sanitizers: "Off",
            artifact: "ubuntu-clang-valgrind"
          }
          - {
            os: ubuntu-24.04, name: "Ubuntu GCC 14",
            compiler: gcc, compiler_version: 14, sanitizers: "On",
            artifact: "ubuntu-gcc-14"
          }
          - {
            os: windows-latest, name: "Windows VS 2022",
            compiler: msvc, compiler_version: "", sanitizers: "On",
            artifact: "windows-msvc-22"
          }
          - {
            os: macos-14, name: "MacOS Clang 16",
            artifact: "macos-clang-16",
            compiler: clang, compiler_version: 16, sanitizers: "On"
          }

    steps:
      - uses: actions/checkout@v5
        with:
          submodules: recursive

      - name: Setup compilers, dependencies, project and build
        uses: ./.github/workflows/setup-compilers
        with:
          os_name: ${{ matrix.config.os }}
          compiler: ${{ matrix.config.compiler }}
          compiler_version: ${{ matrix.config.compiler_version }}
          sanitizers: ${{ matrix.config.sanitizers }}
          with_deps: true

      - name: Organize files for upload
        shell: bash
        run: |
          mkdir -p artifact/lib/std
          # Linux/MacOS
          if [[ '${{ startsWith(matrix.config.os, 'ubuntu') }}' == 'true' ]] || [[ '${{ startsWith(matrix.config.os, 'macos') }}' == 'true' ]]; then
            cp build/arkscript artifact
            cp build/bin/libArkReactor.* artifact
          fi
          # Windows
          if [[ '${{ startsWith(matrix.config.os, 'windows') }}' == 'true' ]]; then
            cp build/$BUILD_TYPE/arkscript.exe artifact
            cp build/$BUILD_TYPE/ArkReactor.dll artifact
          fi
          # Generic
          cp lib/*.arkm artifact/lib
          cp lib/std/*.ark artifact/lib/std
          cp tests/unittests/*.arkm artifact/lib

      - name: Organize temp artifact
        shell: bash
        run: |
          mkdir -p temp/tests/unittests/
          if [[ '${{ startsWith(matrix.config.os, 'windows') }}' == 'true' ]]; then
            cp build/$BUILD_TYPE/unittests.exe temp/
          else
            cp build/unittests temp/
          fi
          cp tests/unittests/*.arkm temp/tests/unittests
          cp -r tests/unittests temp/tests/unittests

      - name: Upload artifact
        uses: actions/upload-artifact@v5
        with:
          name: ${{ matrix.config.artifact }}
          path: artifact

      - name: Upload temp artifact
        uses: actions/upload-artifact@v5
        with:
          name: temp-${{ matrix.config.artifact }}
          path: temp
          retention-days: 1

  coverage:
    runs-on: ubuntu-24.04
    name: Coverage

    steps:
      - uses: actions/checkout@v5
        with:
          submodules: recursive

      - name: Install dependencies
        run: |
          sudo apt-get install -yq -o Acquire::Retries=3 lcov
          lcov --version

      - name: Setup compilers, dependencies, project and build
        uses: ./.github/workflows/setup-compilers
        with:
          os_name: ubuntu-24.04
          compiler: clang
          compiler_version: 16
          sanitizers: On
          coverage: On
          with_deps: true

      - name: Run tests
        run: |
          export ASAN_OPTIONS=detect_odr_violation=0
          ./build/unittests

      - name: Gather coverage
        run: cmake --build build --target coverage

      - name: Coveralls
        uses: coverallsapp/github-action@master
        with:
          path-to-lcov: build/coverage.info
          github-token: ${{ secrets.GITHUB_TOKEN }}

  tests:
    runs-on: ${{ matrix.config.os }}
    name: Tests on ${{ matrix.config.name }}
    needs: [ build ]

    strategy:
      fail-fast: false
      matrix:
        config:
          - { os: ubuntu-24.04,  name: "Ubuntu Clang 16", artifact: "ubuntu-clang-16", compiler_version: 16, }
          - { os: ubuntu-24.04,  name: "Ubuntu GCC 14",   artifact: "ubuntu-gcc-14", compiler_version: 14, }
          - { os: windows-latest, name: "Windows VS 2022", artifact: "windows-msvc-22", compiler_version: 22, }
          - { os: macos-14,   name: "MacOS Clang 16",  artifact: "macos-clang-16", compiler_version: 16, }

    steps:
      - uses: actions/checkout@v5
        with:
          submodules: recursive

      - name: Setup tests
        uses: ./.github/workflows/setup-tests
        with:
          os_name: ${{ matrix.config.os }}
          compiler_version: ${{ matrix.config.compiler_version }}

      - name: Tests
        shell: bash
        run: |
          export ASAN_OPTIONS=detect_odr_violation=0
          ./unittests

      - uses: actions/setup-python@v6
        if: ${{ ! startsWith(matrix.config.name, 'Windows') }}
        with:
          python-version: '3.13'

      - name: Modules tests
        if: ${{ ! startsWith(matrix.config.name, 'Windows') }}
        shell: bash
        run: |
          export ASAN_OPTIONS=detect_odr_violation=0
          chmod u+x ./lib/modules/.github/run-tests
          (./lib/modules/.github/run-tests src)

      - name: REPL tests
        if: ${{ ! startsWith(matrix.config.name, 'Windows') }}
        shell: bash
        run: |
          pip install -r tests/repl/requirements.txt
          python3 tests/repl/test.py

  fuzzing:
    runs-on: ubuntu-24.04
    name: Fuzz testing
    continue-on-error: true

    container:
      image: aflplusplus/aflplusplus:v4.20c
      env:
        BUILD_FOLDER: afl
        FUZZER_SEED: 0
        FUZZ_TIME_SEC: 300
        FUZZER_TIMEOUT_EXEC_MS: 500
        AFL_NO_STARTUP_CALIBRATION: 1
        AFL_CMPLOG_ONLY_NEW: 1
        AFL_MAP_SIZE: 312000
      volumes:
        - ${{ github.workspace }}:/src
      options: --cpus 4

    steps:
      - uses: actions/checkout@v5
        with:
          submodules: recursive

      - name: Build with AFL
        run: |
          cd /src
          cmake -B${BUILD_FOLDER} \
            -DCMAKE_C_COMPILER=/AFLplusplus/afl-cc \
            -DCMAKE_CXX_COMPILER=/AFLplusplus/afl-c++ \
            -DCMAKE_BUILD_TYPE=Release \
            -DARK_BUILD_EXE=On \
            -DARK_SANITIZERS=On \
            -DARK_UNITY_BUILD=On
          cmake --build ${BUILD_FOLDER} --config Release -j 4

      - name: Fuzz
        run: |
          # ensure the executable works first
          ARKSCRIPT_PATH=$(pwd)/lib ${BUILD_FOLDER}/arkscript -e '(import std.List) (print list:map " is a function")' || exit 1
          # then we can fuzz. If the executable doesn't work, the fuzzer crashes with a cryptic error message
          afl-fuzz -i tests/fuzzing/corpus-cmin-tmin \
            -o output \
            -s $FUZZER_SEED \
            -t $FUZZER_TIMEOUT_EXEC_MS \
            -V $FUZZ_TIME_SEC \
            -- ${BUILD_FOLDER}/arkscript -L ./lib @@

      - uses: 8BitJonny/gh-get-current-pr@4.0.0
        id: PR
        with:
          sha: ${{ github.event.pull_request.head.sha }}
          # Only return if PR is still open
          filterOutClosed: true

      - run: |
          echo "PR found: ${prFound}"
          echo "PR ${prNumber:-ERROR} ${prTitle:-NOTITLE}"
        env:
          # Direct access to common PR properties
          prNumber: ${{ steps.PR.outputs.number }}
          prTitle: ${{ steps.PR.outputs.pr_title }}
          prFound: ${{ steps.PR.outputs.pr_found }}

      - name: Summarize for PR
        id: summary
        if: steps.PR.outputs.pr_found == 'true'
        shell: bash
        run: |
          echo "FUZZ_SUMMARY<<EOF" >> $GITHUB_ENV
          afl-whatsup -s -d output >> $GITHUB_ENV
          afl-showmap -C -i output -o /dev/null -- ./${BUILD_FOLDER}/arkscript -L ./lib @@ | sed -r "s/\x1B\[([0-9]{1,3}(;[0-9]{1,2};?)?)?[mGK]//g" | grep -v Reading | grep -v Scanning >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV

      - name: Summarize
        if: steps.PR.outputs.pr_found == 'false'
        shell: bash
        run: |
          afl-whatsup -s -d output
          afl-showmap -C -i output -o /dev/null -- ./${BUILD_FOLDER}/arkscript -L ./lib @@

      - name: Find Comment
        uses: peter-evans/find-comment@v4
        if: steps.PR.outputs.pr_found == 'true'
        id: fc
        with:
          issue-number: ${{ github.event.pull_request.number }}
          comment-author: 'github-actions[bot]'
          body-includes: Fuzzing report

      - name: Create or update comment
        uses: peter-evans/create-or-update-comment@v5.0.0
        if: steps.PR.outputs.pr_found == 'true'
        with:
          comment-id: ${{ steps.fc.outputs.comment-id }}
          issue-number: ${{ github.event.pull_request.number }}
          body: |
            ## Fuzzing report
            ${{ env.FUZZ_SUMMARY }}
          edit-mode: replace

      - name: Sort files for upload
        continue-on-error: true
        run: |
          mkdir crashes
          cp output/default/crashes/id:* crashes/
          (cd crashes && find . -type f -exec mv '{}' '{}'.ark \;)
          tar -czvf crashes.tar.gz crashes

      - name: Upload crashes
        continue-on-error: true
        uses: actions/upload-artifact@v5
        with:
          name: crashes
          path: crashes.tar.gz

  valgrind:
    runs-on: ubuntu-24.04
    name: Ubuntu Clang Valgrind
    needs: [ build ]

    steps:
      - uses: actions/checkout@v5
        with:
          submodules: recursive

      - name: Download artifact
        id: download
        uses: actions/download-artifact@v6
        with:
          name: "ubuntu-clang-valgrind"
          path: build

      - name: Update LLVM compilers
        shell: bash
        run: |
          mv build/lib/*.arkm lib/
          chmod u+x build/arkscript
          sudo apt-get update --fix-missing
          sudo apt-get install -y clang-16 lld-16 libc++-16-dev libc++abi-16-dev clang-tools-16

      - name: Install valgrind 3.24+
        shell: bash
        run: |
          valgrind_ver=3.24.0-3_amd64
          wget --no-check-certificate http://ftp.us.debian.org/debian/pool/main/v/valgrind/valgrind_${valgrind_ver}.deb
          sudo apt remove needrestart
          sudo apt install ./valgrind_${valgrind_ver}.deb

      - name: Valgrind checks for memory leaks
        shell: bash
        run: |
          # hack to close fd 155 and 158 as github leaves them open
          # thread: https://github.com/actions/runner-images/issues/3729
          # solution from: https://github.com/actions/runner-images/issues/3391#issuecomment-842139475
          if [ -d /proc/$$/fd/ ]; then
            for descriptor_path in /proc/$$/fd/*; do
                descriptor="$(basename "$descriptor_path")"
                # Don't close stdin/stderr/stdout (-gt 2)
                if [ $descriptor -gt 2 ]; then
                  exec {descriptor}<&-
                fi
            done
          fi
          valgrind --leak-check=full \
            --show-leak-kinds=all --show-reachable=yes \
            --track-origins=yes --track-fds=yes --trace-children=yes \
            --verbose -s \
            --demangle=yes \
            --error-exitcode=1 \
            build/arkscript -L ./lib tests/unittests/resources/LangSuite/unittests.ark valgrind