diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 165f041..c3d5b34 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -1,47 +1,104 @@ name: Checks - -on: - push: - pull_request: - -# only allow one job to run -concurrency: general_workflow +on: [push, pull_request] jobs: test: - runs-on: ubuntu-latest + strategy: + matrix: + ckan-version: ["2.11"] + dcor_control: ["develop", "pypi"] + fail-fast: false - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up Docker - uses: docker/setup-buildx-action@v2 + name: DCOR-CKAN ${{ matrix.ckan-version }} ${{ matrix.dcor_control }} + runs-on: ubuntu-latest + container: + image: ckan/ckan-dev:${{ matrix.ckan-version }} + options: --user root --hostname dcor-dev-docker - - name: Install Docker Compose - run: | - sudo curl -L "https://github.com/docker/compose/releases/download/v2.17.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose - sudo chmod +x /usr/local/bin/docker-compose - docker-compose --version # Verify installation + services: + solr: + image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr9 + postgres: + image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }} + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + options: >- + --health-cmd "pg_isready" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + redis: + image: redis:3 + minio: + image: raghavaalajangi95/dcor-minio:latest + # Note: + # The official MinIO setup uses standalone mode, which has limitations: + # - Certain 'mc admin' commands, such as user and policy management, are not supported. + # - Using these commands will result in errors like: + # "This 'admin' API is not supported by server in 'mode-server-fs'" + # - `minio/minio:edge-cicd` container from MinIO is a good alternative, + # but it is still not supporting 'mc admin policy' command and the container is not managed properly. + # References: + # - https://github.com/minio/minio/discussions/16099 + # - https://github.com/orgs/community/discussions/26688#discussioncomment-4183701 + # - https://stackoverflow.com/questions/64031598/creating-a-minios3-container-inside-a-github-actions-yml-file/64188150#64188150 + env: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + MINIO_NEW_USER: dcor + MINIO_NEW_SECRET_KEY: secretkey12345678 + options: >- + --health-cmd "curl -f http://localhost:9000/minio/health/ready || exit 1" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + ports: + - 9000:9000 + - 9001:9001 - - name: Start Docker Compose services - run: docker-compose --env-file docker-ci.env -f docker-compose.ci.yml up -d + env: + CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test + CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test + CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test + CKAN_SOLR_URL: http://solr:8983/solr/ckan + CKAN_REDIS_URL: redis://redis:6379/1 - - name: Wait for CKAN to be healthy - run: docker exec dcor_ckan bash -c "while ! wget -qO- http://localhost:5000; do sleep 5; done" + steps: + - uses: actions/checkout@v4 - - name: List running services (for debugging) - run: docker-compose -f docker-compose.ci.yml ps + - name: Install and setup DCOR + run: | + # pip3 install dcor_control + pip3 install git+https://github.com/DCOR-dev/dcor_control + # use test-core.ini as default + cp /srv/app/src/ckan/test-core.ini /srv/app/ckan.ini + # initialize CKAN database + ckan -c /srv/app/ckan.ini db init + # fix location of allowed datastore commands + # https://github.com/ckan/ckan/issues/8926 + mkdir -p /srv/app/ckanext/datastore/tests/ + wget -O /srv/app/ckanext/datastore/tests/allowed_functions.txt https://raw.githubusercontent.com/ckan/ckan/refs/heads/master/ckanext/datastore/allowed_functions.txt + # complement configuration + dcor inspect --assume-yes - - name: Run Tests - run: bash ./docker-run-tests.sh + - name: Install latest DCOR extensions from GitHub + if: matrix.dcor_control == 'develop' + run: | + dcor develop --yes + dcor inspect --assume-yes - - name: Upload Coverage to GitHub - uses: actions/upload-artifact@v4 - with: - name: coverage-report - path: coverage.xml + - name: Install extension and requirements + run: | + pip install -e . + pip install -r ./ckanext/dc_view/tests/requirements.txt - - name: Bring Down Docker Compose - if: always() - run: docker-compose -f docker-compose.ci.yml down + - name: Run tests + run: | + pytest -v --ckan-ini=/srv/app/ckan.ini \ + --cov=ckanext.dc_view \ + --cov-report=xml \ + --cov-append \ + --disable-warnings \ + ckanext/dc_view/tests/ diff --git a/docker-ci.env b/docker-ci.env deleted file mode 100644 index 530b968..0000000 --- a/docker-ci.env +++ /dev/null @@ -1,83 +0,0 @@ -# Host Ports -CKAN_PORT_HOST=5000 - -# CKAN databases -POSTGRES_USER=postgres -POSTGRES_PASSWORD=postgres -POSTGRES_DB=postgres -POSTGRES_HOST=db -CKAN_DB_USER=ckandbuser -CKAN_DB_PASSWORD=ckandbpassword -CKAN_DB=ckandb -DATASTORE_READONLY_USER=datastore_ro -DATASTORE_READONLY_PASSWORD=datastore -DATASTORE_DB=datastore -CKAN_SQLALCHEMY_URL=postgresql://ckandbuser:ckandbpassword@db/ckandb -CKAN_DATASTORE_WRITE_URL=postgresql://ckandbuser:ckandbpassword@db/datastore -CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore - -# Test database connections -TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test -TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test -TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test - -# Dev settings -USE_HTTPS_FOR_DEV=false - -# CKAN core -CKAN_SITE_ID=default -CKAN_SITE_URL=http://localhost:5000 -CKAN___BEAKER__SESSION__SECRET=CHANGE_ME -# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings -CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME -CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME -CKAN_SYSADMIN_NAME=ckan_admin -CKAN_SYSADMIN_PASSWORD=test1234 -CKAN_SYSADMIN_EMAIL=your_email@example.com -CKAN_STORAGE_PATH=/var/lib/ckan -CKAN_SMTP_SERVER=smtp.corporateict.domain:25 -CKAN_SMTP_STARTTLS=True -CKAN_SMTP_USER=user -CKAN_SMTP_PASSWORD=pass -CKAN_SMTP_MAIL_FROM=ckan@localhost -CKAN_MAX_UPLOAD_SIZE_MB=100 -TZ=UTC - -# Solr -SOLR_IMAGE_VERSION=2.10-solr9 -CKAN_SOLR_URL=http://solr:8983/solr/ckan -TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan - -# Redis -REDIS_VERSION=6 -CKAN_REDIS_URL=redis://redis:6379/1 -TEST_CKAN_REDIS_URL=redis://redis:6379/1 - -# Datapusher -DATAPUSHER_VERSION=0.0.20 -CKAN_DATAPUSHER_URL=http://datapusher:8800 -CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 - -# NGINX -NGINX_PORT=80 -NGINX_SSLPORT=443 - -# Extensions -CKAN__PLUGINS="image_view text_view datatables_view datastore datapusher envvars" -CKAN__HARVEST__MQ__TYPE=redis -CKAN__HARVEST__MQ__HOSTNAME=redis -CKAN__HARVEST__MQ__PORT=6379 -CKAN__HARVEST__MQ__REDIS_DB=1 - -# CKAN__AUTH__PUBLIC_USER_DETAILS = True - -# minio -MINIO_ROOT_USER=admin -MINIO_ROOT_PASSWORD=admin123 -MINIO_NEW_USER=dcor -MINIO_NEW_SECRET_KEY=secretkey12345678 - -CKAN__DCOR_OBJECT_STORE__ENDPOINT_URL = http://minio:9000 -CKAN__DCOR_OBJECT_STORE__ACCESS_KEY_ID = dcor -CKAN__DCOR_OBJECT_STORE__SECRET_ACCESS_KEY = secretkey12345678 - diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml deleted file mode 100644 index b27c0f8..0000000 --- a/docker-compose.ci.yml +++ /dev/null @@ -1,91 +0,0 @@ -volumes: - ckan_storage: - pg_data: - solr_data: - pip_cache: - site_packages: - vscode_server: - minio_data: - -services: - ckan: - # Use image from Raghava's dockerHub - image: raghavaalajangi95/dcor-ckan:latest - container_name: dcor_ckan - env_file: - docker-ci.env - links: - - db - - solr - - redis - - minio - # hostname is defined as a docker resource in `dcor_control` - hostname: dcor-dev-docker - ports: - - "0.0.0.0:${CKAN_PORT_HOST}:5000" - user: root # This will avoid permission errors - volumes: - - ckan_storage:/var/lib/ckan - # Map current GitHub runner working dir as src_extensions dir within - # the container (this will allow us to access the ckanext-dcor_theme - # directly inside src_extensions) - - ./:/srv/app/src_extensions - - pip_cache:/root/.cache/pip - - site_packages:/usr/lib/python3.10/site-packages - - vscode_server:/root/.vscode-server - restart: unless-stopped - healthcheck: - test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] - - datapusher: - image: ckan/ckan-base-datapusher:${DATAPUSHER_VERSION} - container_name: datapusher - restart: unless-stopped - healthcheck: - test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8800"] - - db: - # Use image from Raghava's dockerHub - image: raghavaalajangi95/dcor-postgresql:latest - container_name: postgres - env_file: - - docker-ci.env - volumes: - - pg_data:/var/lib/postgresql/data - restart: unless-stopped - healthcheck: - test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"] - - solr: - image: ckan/ckan-solr:${SOLR_IMAGE_VERSION} - container_name: solr - volumes: - - solr_data:/var/solr - restart: unless-stopped - healthcheck: - test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8983/solr/"] - - redis: - image: redis:${REDIS_VERSION} - container_name: redis - restart: unless-stopped - healthcheck: - test: ["CMD", "redis-cli", "-e", "QUIT"] - - minio: - # Use image from Raghava's dockerHub - image: raghavaalajangi95/dcor-minio:latest - container_name: minio - restart: unless-stopped - env_file: - - docker-ci.env - ports: - - "9000:9000" - - "9001:9001" - volumes: - - minio_data:/data-minio - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/ready"] - interval: 10s - timeout: 5s - retries: 5 diff --git a/docker-run-tests.sh b/docker-run-tests.sh deleted file mode 100644 index 848b2b6..0000000 --- a/docker-run-tests.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -# Fail immediately if any command fails -set -e - -CKAN_CONTAINER="dcor_ckan" # CKAN container name -EXTENSION_PATH="/srv/app/src_extensions/" - -# Create venv and install dependencies as a root user inside the container -docker exec -u root ${CKAN_CONTAINER} bash -c " - cd ${EXTENSION_PATH}; - - # Update dcor libraries - dcor update --yes; - - # Install ckanext-dc_view and its test requirements - pip install .; - pip install -r ./ckanext/dc_view/tests/requirements.txt; - - # Make sure everything is in order - dcor inspect --assume-yes; -" - -# Run tests on GitHub runner where container gets permissions from. -echo "Running tests in the virtual environment..." -docker exec ${CKAN_CONTAINER} bash -c " - cd ${EXTENSION_PATH}; - # Run coverage - coverage run --source=ckanext.dc_view --omit=*tests* -m pytest -p no:warnings ckanext; -" - -# Generate the XML report -docker exec ${CKAN_CONTAINER} bash -c " - cd ${EXTENSION_PATH}; - coverage xml; -" - -echo "Tests passed inside the container."