diff --git a/.github/workflows/config.yml b/.github/workflows/config.yml index 197dc166e..b497c44da 100644 --- a/.github/workflows/config.yml +++ b/.github/workflows/config.yml @@ -9,36 +9,69 @@ env: S2MS_API_KEY: ${{ secrets.S2MS_API_KEY }} jobs: + fetch-s2-versions: + name: Fetch SingleStore supported versions + runs-on: ubuntu-latest + outputs: + versions: ${{ steps.get_versions.outputs.versions }} + steps: + - name: Get supported versions of SingleStore + id: get_versions + uses: singlestore-labs/singlestore-supported-versions@main + with: + include_rc: true + + build-matrix: + name: Build test matrix + runs-on: ubuntu-latest + needs: fetch-s2-versions + outputs: + matrix: ${{ steps.compose.outputs.matrix }} + steps: + - name: Install jq + run: sudo apt-get update && sudo apt-get install -y jq + + - name: Compose matrix JSON + id: compose + env: + VERSIONS: ${{ needs.fetch-s2-versions.outputs.versions }} + run: | + scripts='[ + ".github/workflows/test_setup/run_functional_tests1.sh", + ".github/workflows/test_setup/run_functional_tests2.sh" + ]' + + version_rows=$(jq -cn --argjson versions "$VERSIONS" ' + $versions | map({ + name: ("SingleStore " + . + " connection test"), + singlestore_version: (.) + }) + ') + + rows=$(jq -cn --argjson versions "$version_rows" --argjson scripts "$scripts" ' + [ + $versions[] as $v | + $scripts[] as $s | + ($v + { func_test_script: $s }) + ] + ') + + matrix=$(jq -cn --argjson rows "$rows" '{ include: $rows }') + + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + test-ubuntu: + name: ${{ matrix.name }} runs-on: ubuntu-22.04 + needs: build-matrix strategy: fail-fast: false - matrix: - singlestore_image: - - singlestore/cluster-in-a-box:alma-8.5.6-b51bc5471a-4.0.17-1.17.8 - - singlestore/cluster-in-a-box:alma-8.7.12-483e5f8acb-4.1.0-1.17.15 - func_test_script: - - .github/workflows/test_setup/run_functional_tests1.sh - - .github/workflows/test_setup/run_functional_tests2.sh + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} steps: - uses: actions/checkout@v4 - name: Remove unnecessary pre-installed toolchains for free disk spaces - run: | - echo "=== BEFORE ===" - df -h - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf /usr/local/share/boost - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/hostedtoolcache/CodeQL - sudo rm -rf /opt/hostedtoolcache/Ruby - sudo rm -rf /opt/hostedtoolcache/Go - docker system prune -af || true - sudo apt-get clean - echo "=== AFTER ===" - df -h + run: ./.github/workflows/test_setup/free_disk_space.sh - name: Set up .NET uses: actions/setup-dotnet@v4 @@ -55,11 +88,19 @@ jobs: - name: Install Powershell run: sudo snap install powershell --classic + - name: Extra Docker cleanup before starting SingleStore + run: | + docker ps -a || true + docker rm -f singlestore-integration || true + docker system prune -af || true + docker volume prune -f || true + df -h + - name: Start SingleStore Cluster run: | ./.github/workflows/test_setup/setup_cluster.sh env: - SINGLESTORE_IMAGE: ${{ matrix.singlestore_image }} + SINGLESTORE_VERSION: ${{ matrix.singlestore_version }} - name: Copy test config run: | @@ -82,7 +123,7 @@ jobs: - name: Run Integration Tests run: dotnet test test/EFCore.SingleStore.IntegrationTests -f net8.0 -c Release --no-build - - name: Run Functional Tests ${{ matrix.singlestore_image }} - ${{ matrix.func_test_script }} + - name: Run Functional Tests ${{ matrix.singlestore_version }} - ${{ matrix.func_test_script }} run: ${{ matrix.func_test_script }} test-windows: diff --git a/.github/workflows/test_setup/free_disk_space.sh b/.github/workflows/test_setup/free_disk_space.sh new file mode 100755 index 000000000..f2bff8452 --- /dev/null +++ b/.github/workflows/test_setup/free_disk_space.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -euxo pipefail + +echo "=== BEFORE ===" +df -h + +sudo rm -rf /usr/share/dotnet +sudo rm -rf /opt/ghc +sudo rm -rf /usr/local/share/boost +sudo rm -rf "$AGENT_TOOLSDIRECTORY" +sudo rm -rf /usr/local/lib/android +sudo rm -rf /opt/hostedtoolcache/CodeQL +sudo rm -rf /opt/hostedtoolcache/Ruby +sudo rm -rf /opt/hostedtoolcache/Go + +sudo rm -rf /opt/hostedtoolcache/Java* +sudo rm -rf /opt/hostedtoolcache/Python* +sudo rm -rf /opt/hostedtoolcache/Node* +sudo rm -rf /opt/hostedtoolcache/PyPy* +sudo rm -rf /opt/hostedtoolcache/Swift* +sudo rm -rf /opt/hostedtoolcache/gradle* +sudo rm -rf /opt/hostedtoolcache/maven* +sudo rm -rf /opt/hostedtoolcache/Rust* +sudo rm -rf /opt/hostedtoolcache/Perl* +sudo rm -rf /opt/hostedtoolcache/llvm + +sudo rm -rf /usr/local/lib/node_modules || true +sudo rm -rf /usr/local/share/powershell || true +sudo rm -rf /usr/local/julia* || true +sudo rm -rf /usr/share/swift || true + +sudo rm -rf /var/lib/apt/lists/* || true +sudo rm -rf /var/cache/apt/* || true +sudo rm -rf /var/cache/man/* || true +sudo rm -rf /var/log/* || true +sudo rm -rf /tmp/* || true + +sudo apt-get update || true +sudo apt-get purge -y \ + azure-cli \ + google-cloud-cli \ + google-chrome-stable \ + firefox \ + powershell \ + mono-devel \ + hhvm \ + php* \ + dotnet-sdk-* \ + temurin-* \ + openjdk-* \ + mysql-client* \ + postgresql-client* \ + || true + +sudo apt-get autoremove -y || true +sudo apt-get clean || true + +docker system prune -af || true +docker volume prune -f || true + +echo "=== AFTER ===" +df -h diff --git a/.github/workflows/test_setup/setup_cluster.sh b/.github/workflows/test_setup/setup_cluster.sh index 0e3f6f9e3..5348a4a0b 100755 --- a/.github/workflows/test_setup/setup_cluster.sh +++ b/.github/workflows/test_setup/setup_cluster.sh @@ -1,8 +1,11 @@ #!/usr/bin/env bash set -eu -DEFAULT_IMAGE_NAME="memsql/cluster-in-a-box:centos-7.3.9-a7abc2ebd4-3.2.8-1.11.4" -IMAGE_NAME="${SINGLESTORE_IMAGE:-$DEFAULT_IMAGE_NAME}" +IMAGE_NAME="ghcr.io/singlestore-labs/singlestoredb-dev:latest" + +DEFAULT_SINGLESTORE_VERSION="8.9" +VERSION="${SINGLESTORE_VERSION:-$DEFAULT_SINGLESTORE_VERSION}" + CONTAINER_NAME="singlestore-integration" EXISTS=$(docker inspect ${CONTAINER_NAME} >/dev/null 2>&1 && echo 1 || echo 0) @@ -17,10 +20,11 @@ if [[ "${EXISTS}" -eq 1 ]]; then fi if [[ "${EXISTS}" -eq 0 ]]; then - docker run -i --init \ + docker run -d \ --name ${CONTAINER_NAME} \ - -e LICENSE_KEY=${LICENSE_KEY} \ + -e SINGLESTORE_LICENSE=${LICENSE_KEY} \ -e ROOT_PASSWORD=${SQL_USER_PASSWORD} \ + -e SINGLESTORE_VERSION=${VERSION} \ -p 3306:3306 -p 3307:3307 \ ${IMAGE_NAME} fi @@ -38,11 +42,18 @@ singlestore-wait-start() { done mysql -u root -h 127.0.0.1 -P 3306 -p"${SQL_USER_PASSWORD}" -e "create database if not exists singlestoretest" >/dev/null 2>/dev/null && \ mysql -u root -h 127.0.0.1 -P 3306 -p"${SQL_USER_PASSWORD}" -e "SET GLOBAL data_conversion_compatibility_level = '6.0'" >/dev/null 2>/dev/null + echo ". Success!" } singlestore-wait-start +mysql -u root -h 127.0.0.1 -P 3306 -p"${SQL_USER_PASSWORD}" --batch -N -e "\ + SET GLOBAL failover_on_low_disk = OFF; \ + SET GLOBAL minimal_disk_space = 0; \ + SET GLOBAL spilling_minimal_disk_space = 0; \ +" >/dev/null 2>/dev/null || true + echo echo "Ensuring child nodes are connected using container IP" CONTAINER_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${CONTAINER_NAME})