From e8589bc93dc348b8b9cd9e4eb987dd4dc09aec60 Mon Sep 17 00:00:00 2001 From: RaghavaAlajangi Date: Thu, 24 Apr 2025 11:23:42 +0200 Subject: [PATCH 01/38] enh: simplify CI workflow with test.ini approach --- .github/workflows/check.yml | 98 ++++++++++++++++++++++++------------- docker-ci.env | 83 ------------------------------- docker-compose.ci.yml | 91 ---------------------------------- docker-run-tests.sh | 38 -------------- docker-test.ini | 51 +++++++++++++++++++ 5 files changed, 114 insertions(+), 247 deletions(-) delete mode 100644 docker-ci.env delete mode 100644 docker-compose.ci.yml delete mode 100644 docker-run-tests.sh create mode 100644 docker-test.ini diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 165f041..9e1c868 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -1,47 +1,75 @@ name: Checks - -on: - push: - pull_request: - -# only allow one job to run -concurrency: general_workflow +on: [push, pull_request] jobs: test: + name: DCOR-CKAN runs-on: ubuntu-latest + container: + image: raghavaalajangi95/dcor-ckan:latest + options: --user root - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up Docker - uses: docker/setup-buildx-action@v2 - - - name: Install Docker Compose - run: | - sudo curl -L "https://github.com/docker/compose/releases/download/v2.17.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose - sudo chmod +x /usr/local/bin/docker-compose - docker-compose --version # Verify installation + services: + solr: + image: ckan/ckan-solr:2.11-solr9 + postgres: + image: raghavaalajangi95/dcor-postgresql:latest + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + options: | + --health-cmd pg_isready \ + --health-interval 10s \ + --health-timeout 5s \ + --health-retries 10 + redis: + image: redis:alpine + minio: + image: raghavaalajangi95/dcor-minio:latest + env: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + MINIO_NEW_USER: dcor + MINIO_NEW_SECRET_KEY: secretkey12345678 + options: | + --health-cmd "curl -f http://localhost:9000/minio/health/ready || exit 1" \ + --health-interval 10s \ + --health-timeout 5s \ + --health-retries 10 + ports: + - 9000:9000 - - name: Start Docker Compose services - run: docker-compose --env-file docker-ci.env -f docker-compose.ci.yml up -d + env: + CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test + CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test + CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test + CKAN_SOLR_URL: http://solr:8983/solr/ckan + CKAN_REDIS_URL: redis://redis:6379/1 - - name: Wait for CKAN to be healthy - run: docker exec dcor_ckan bash -c "while ! wget -qO- http://localhost:5000; do sleep 5; done" + steps: + - uses: actions/checkout@v4 - - name: List running services (for debugging) - run: docker-compose -f docker-compose.ci.yml ps + - name: Install requirements + run: | + pip install . + pip install -r ./ckanext/dc_view/tests/requirements.txt + # Replace default path to CKAN core config file with the one on the container + sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' docker-test.ini - - name: Run Tests - run: bash ./docker-run-tests.sh + - name: Setup extension + run: | + ckan -c test.ini db init - - name: Upload Coverage to GitHub - uses: actions/upload-artifact@v4 - with: - name: coverage-report - path: coverage.xml + - name: Wait for MinIO to be ready + run: | + until curl -f http://minio:9000/minio/health/ready; do sleep 2; done - - name: Bring Down Docker Compose - if: always() - run: docker-compose -f docker-compose.ci.yml down + - name: Run tests + run: | + pytest -v --ckan-ini=test.ini \ + --cov=ckanext.dc_view \ + --cov-report=xml \ + --cov-append \ + --disable-warnings \ + ckanext/dc_view/tests/ diff --git a/docker-ci.env b/docker-ci.env deleted file mode 100644 index 530b968..0000000 --- a/docker-ci.env +++ /dev/null @@ -1,83 +0,0 @@ -# Host Ports -CKAN_PORT_HOST=5000 - -# CKAN databases -POSTGRES_USER=postgres -POSTGRES_PASSWORD=postgres -POSTGRES_DB=postgres -POSTGRES_HOST=db -CKAN_DB_USER=ckandbuser -CKAN_DB_PASSWORD=ckandbpassword -CKAN_DB=ckandb -DATASTORE_READONLY_USER=datastore_ro -DATASTORE_READONLY_PASSWORD=datastore -DATASTORE_DB=datastore -CKAN_SQLALCHEMY_URL=postgresql://ckandbuser:ckandbpassword@db/ckandb -CKAN_DATASTORE_WRITE_URL=postgresql://ckandbuser:ckandbpassword@db/datastore -CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore - -# Test database connections -TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test -TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test -TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test - -# Dev settings -USE_HTTPS_FOR_DEV=false - -# CKAN core -CKAN_SITE_ID=default -CKAN_SITE_URL=http://localhost:5000 -CKAN___BEAKER__SESSION__SECRET=CHANGE_ME -# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings -CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME -CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME -CKAN_SYSADMIN_NAME=ckan_admin -CKAN_SYSADMIN_PASSWORD=test1234 -CKAN_SYSADMIN_EMAIL=your_email@example.com -CKAN_STORAGE_PATH=/var/lib/ckan -CKAN_SMTP_SERVER=smtp.corporateict.domain:25 -CKAN_SMTP_STARTTLS=True -CKAN_SMTP_USER=user -CKAN_SMTP_PASSWORD=pass -CKAN_SMTP_MAIL_FROM=ckan@localhost -CKAN_MAX_UPLOAD_SIZE_MB=100 -TZ=UTC - -# Solr -SOLR_IMAGE_VERSION=2.10-solr9 -CKAN_SOLR_URL=http://solr:8983/solr/ckan -TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan - -# Redis -REDIS_VERSION=6 -CKAN_REDIS_URL=redis://redis:6379/1 -TEST_CKAN_REDIS_URL=redis://redis:6379/1 - -# Datapusher -DATAPUSHER_VERSION=0.0.20 -CKAN_DATAPUSHER_URL=http://datapusher:8800 -CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 - -# NGINX -NGINX_PORT=80 -NGINX_SSLPORT=443 - -# Extensions -CKAN__PLUGINS="image_view text_view datatables_view datastore datapusher envvars" -CKAN__HARVEST__MQ__TYPE=redis -CKAN__HARVEST__MQ__HOSTNAME=redis -CKAN__HARVEST__MQ__PORT=6379 -CKAN__HARVEST__MQ__REDIS_DB=1 - -# CKAN__AUTH__PUBLIC_USER_DETAILS = True - -# minio -MINIO_ROOT_USER=admin -MINIO_ROOT_PASSWORD=admin123 -MINIO_NEW_USER=dcor -MINIO_NEW_SECRET_KEY=secretkey12345678 - -CKAN__DCOR_OBJECT_STORE__ENDPOINT_URL = http://minio:9000 -CKAN__DCOR_OBJECT_STORE__ACCESS_KEY_ID = dcor -CKAN__DCOR_OBJECT_STORE__SECRET_ACCESS_KEY = secretkey12345678 - diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml deleted file mode 100644 index b27c0f8..0000000 --- a/docker-compose.ci.yml +++ /dev/null @@ -1,91 +0,0 @@ -volumes: - ckan_storage: - pg_data: - solr_data: - pip_cache: - site_packages: - vscode_server: - minio_data: - -services: - ckan: - # Use image from Raghava's dockerHub - image: raghavaalajangi95/dcor-ckan:latest - container_name: dcor_ckan - env_file: - docker-ci.env - links: - - db - - solr - - redis - - minio - # hostname is defined as a docker resource in `dcor_control` - hostname: dcor-dev-docker - ports: - - "0.0.0.0:${CKAN_PORT_HOST}:5000" - user: root # This will avoid permission errors - volumes: - - ckan_storage:/var/lib/ckan - # Map current GitHub runner working dir as src_extensions dir within - # the container (this will allow us to access the ckanext-dcor_theme - # directly inside src_extensions) - - ./:/srv/app/src_extensions - - pip_cache:/root/.cache/pip - - site_packages:/usr/lib/python3.10/site-packages - - vscode_server:/root/.vscode-server - restart: unless-stopped - healthcheck: - test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] - - datapusher: - image: ckan/ckan-base-datapusher:${DATAPUSHER_VERSION} - container_name: datapusher - restart: unless-stopped - healthcheck: - test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8800"] - - db: - # Use image from Raghava's dockerHub - image: raghavaalajangi95/dcor-postgresql:latest - container_name: postgres - env_file: - - docker-ci.env - volumes: - - pg_data:/var/lib/postgresql/data - restart: unless-stopped - healthcheck: - test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"] - - solr: - image: ckan/ckan-solr:${SOLR_IMAGE_VERSION} - container_name: solr - volumes: - - solr_data:/var/solr - restart: unless-stopped - healthcheck: - test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8983/solr/"] - - redis: - image: redis:${REDIS_VERSION} - container_name: redis - restart: unless-stopped - healthcheck: - test: ["CMD", "redis-cli", "-e", "QUIT"] - - minio: - # Use image from Raghava's dockerHub - image: raghavaalajangi95/dcor-minio:latest - container_name: minio - restart: unless-stopped - env_file: - - docker-ci.env - ports: - - "9000:9000" - - "9001:9001" - volumes: - - minio_data:/data-minio - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/ready"] - interval: 10s - timeout: 5s - retries: 5 diff --git a/docker-run-tests.sh b/docker-run-tests.sh deleted file mode 100644 index 848b2b6..0000000 --- a/docker-run-tests.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -# Fail immediately if any command fails -set -e - -CKAN_CONTAINER="dcor_ckan" # CKAN container name -EXTENSION_PATH="/srv/app/src_extensions/" - -# Create venv and install dependencies as a root user inside the container -docker exec -u root ${CKAN_CONTAINER} bash -c " - cd ${EXTENSION_PATH}; - - # Update dcor libraries - dcor update --yes; - - # Install ckanext-dc_view and its test requirements - pip install .; - pip install -r ./ckanext/dc_view/tests/requirements.txt; - - # Make sure everything is in order - dcor inspect --assume-yes; -" - -# Run tests on GitHub runner where container gets permissions from. -echo "Running tests in the virtual environment..." -docker exec ${CKAN_CONTAINER} bash -c " - cd ${EXTENSION_PATH}; - # Run coverage - coverage run --source=ckanext.dc_view --omit=*tests* -m pytest -p no:warnings ckanext; -" - -# Generate the XML report -docker exec ${CKAN_CONTAINER} bash -c " - cd ${EXTENSION_PATH}; - coverage xml; -" - -echo "Tests passed inside the container." diff --git a/docker-test.ini b/docker-test.ini new file mode 100644 index 0000000..558cca9 --- /dev/null +++ b/docker-test.ini @@ -0,0 +1,51 @@ +[DEFAULT] +debug = false +smtp_server = localhost +error_email_from = paste@localhost + +[server:main] +use = egg:Paste#http +host = 0.0.0.0 +port = 5000 + +[app:main] +use = config:../ckan/test-core.ini + +# Insert any custom config settings to be used when running your extension's +# tests here. + +ckan.legacy_templates = false +ckan.plugins = envvars + +# Logging configuration +[loggers] +keys = root, ckan, sqlalchemy + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console + +[logger_ckan] +qualname = ckan +handlers = +level = INFO + +[logger_sqlalchemy] +handlers = +qualname = sqlalchemy.engine +level = WARN + +[handler_console] +class = StreamHandler +args = (sys.stdout,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s \ No newline at end of file From edd0f0fbb6d794fe08b5996fab27d9b99a26062c Mon Sep 17 00:00:00 2001 From: RaghavaAlajangi Date: Thu, 24 Apr 2025 11:35:10 +0200 Subject: [PATCH 02/38] fix: update MinIO health check command in CI workflow --- .github/workflows/check.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 9e1c868..41c222e 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -33,9 +33,9 @@ jobs: MINIO_NEW_USER: dcor MINIO_NEW_SECRET_KEY: secretkey12345678 options: | - --health-cmd "curl -f http://localhost:9000/minio/health/ready || exit 1" \ - --health-interval 10s \ - --health-timeout 5s \ + --health-cmd "pg_isready" + --health-interval 10s + --health-timeout 5s --health-retries 10 ports: - 9000:9000 From 427f6b35b19e59b31469e0ee4072da9544aaa626 Mon Sep 17 00:00:00 2001 From: RaghavaAlajangi Date: Thu, 24 Apr 2025 11:42:26 +0200 Subject: [PATCH 03/38] fix: correct health check commands for PostgreSQL and MinIO in CI workflow --- .github/workflows/check.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 41c222e..4de7f8a 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -19,9 +19,9 @@ jobs: POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres options: | - --health-cmd pg_isready \ - --health-interval 10s \ - --health-timeout 5s \ + --health-cmd "pg_isready" + --health-interval 10s + --health-timeout 5s --health-retries 10 redis: image: redis:alpine @@ -33,7 +33,7 @@ jobs: MINIO_NEW_USER: dcor MINIO_NEW_SECRET_KEY: secretkey12345678 options: | - --health-cmd "pg_isready" + --health-cmd "curl -f http://localhost:9000/minio/health/ready || exit 1" --health-interval 10s --health-timeout 5s --health-retries 10 From 0449c59a31a9708ad693b6d1499eb0df8aaee683 Mon Sep 17 00:00:00 2001 From: RaghavaAlajangi Date: Thu, 24 Apr 2025 11:48:56 +0200 Subject: [PATCH 04/38] fix: options block to a single line --- .github/workflows/check.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 4de7f8a..ed170e0 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -18,11 +18,7 @@ jobs: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres - options: | - --health-cmd "pg_isready" - --health-interval 10s - --health-timeout 5s - --health-retries 10 + options: --health-cmd "pg_isready" --health-interval 10s --health-timeout 5s --health-retries 10 redis: image: redis:alpine minio: @@ -32,11 +28,7 @@ jobs: MINIO_ROOT_PASSWORD: minioadmin MINIO_NEW_USER: dcor MINIO_NEW_SECRET_KEY: secretkey12345678 - options: | - --health-cmd "curl -f http://localhost:9000/minio/health/ready || exit 1" - --health-interval 10s - --health-timeout 5s - --health-retries 10 + options: --health-cmd "curl -f http://localhost:9000/minio/health/ready || exit 1" --health-interval 10s --health-timeout 5s --health-retries 10 ports: - 9000:9000 From a84291309730af777c98a06300f7657ccdb12ea0 Mon Sep 17 00:00:00 2001 From: RaghavaAlajangi Date: Thu, 24 Apr 2025 12:11:11 +0200 Subject: [PATCH 05/38] fix: add missing environment variables for PostgreSQL and MinIO services in CI workflow --- .github/workflows/check.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index ed170e0..fdd2d61 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -18,6 +18,12 @@ jobs: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres + CKAN_DB_USER: ckandbuser + CKAN_DB_PASSWORD: ckandbpassword + CKAN_DB: ckandb + DATASTORE_READONLY_USER: datastore_ro + DATASTORE_READONLY_PASSWORD: datastore + DATASTORE_DB: datastore options: --health-cmd "pg_isready" --health-interval 10s --health-timeout 5s --health-retries 10 redis: image: redis:alpine @@ -31,6 +37,7 @@ jobs: options: --health-cmd "curl -f http://localhost:9000/minio/health/ready || exit 1" --health-interval 10s --health-timeout 5s --health-retries 10 ports: - 9000:9000 + - 9001:9001 env: CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test From 06ba8f8f6a1fb157f6292b6b8ca3fbd286d9162a Mon Sep 17 00:00:00 2001 From: RaghavaAlajangi Date: Thu, 24 Apr 2025 12:14:05 +0200 Subject: [PATCH 06/38] fix: update CKAN command to use docker-test.ini for database initialization --- .github/workflows/check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index fdd2d61..74b37c5 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -58,7 +58,7 @@ jobs: - name: Setup extension run: | - ckan -c test.ini db init + ckan -c docker-test.ini db init - name: Wait for MinIO to be ready run: | From 657ace4b5b241a83d9cb821aba7700a8c8c4dcb1 Mon Sep 17 00:00:00 2001 From: RaghavaAlajangi Date: Thu, 24 Apr 2025 13:11:03 +0200 Subject: [PATCH 07/38] fix: install DCOR extensions in the DCOR-CKAN container --- .github/workflows/check.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 74b37c5..b802f5a 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -49,6 +49,11 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install DCOR extensions inside the DCOR-CKAN container + run: | + pip3 install dcor_control==0.12.6 + dcor inspect --assume-yes + - name: Install requirements run: | pip install . From adfece6a237c0eaf4d6e3e541811ccfc1f010a16 Mon Sep 17 00:00:00 2001 From: RaghavaAlajangi Date: Mon, 5 May 2025 11:46:44 +0200 Subject: [PATCH 08/38] fix: health check for CKAN before installing requirements --- .github/workflows/check.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index b802f5a..43df33b 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -49,12 +49,23 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Wait for CKAN to be healthy + run: | + for i in {1..60}; do + if curl -f http://localhost:5000; then + echo "CKAN is up!" + break + fi + echo "Waiting for CKAN to be ready..." + sleep 5 + done + - name: Install DCOR extensions inside the DCOR-CKAN container run: | pip3 install dcor_control==0.12.6 dcor inspect --assume-yes - - name: Install requirements + - name: Install extention & requirements run: | pip install . pip install -r ./ckanext/dc_view/tests/requirements.txt From aa557db4957e626763187f87e2b5e8e670dcdede Mon Sep 17 00:00:00 2001 From: RaghavaAlajangi Date: Tue, 6 May 2025 11:23:22 +0200 Subject: [PATCH 09/38] fix: update CKAN image to version 2.11 and add hostname display step --- .github/workflows/check.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 43df33b..ebada76 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -6,7 +6,7 @@ jobs: name: DCOR-CKAN runs-on: ubuntu-latest container: - image: raghavaalajangi95/dcor-ckan:latest + image: ckan/ckan:2.11:latest options: --user root services: @@ -49,6 +49,9 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Show hostname + run: hostname + - name: Wait for CKAN to be healthy run: | for i in {1..60}; do From 8f3f585354a42860445245b967533d7955bb415b Mon Sep 17 00:00:00 2001 From: RaghavaAlajangi Date: Tue, 6 May 2025 11:25:39 +0200 Subject: [PATCH 10/38] fix: update CKAN image to version 2.11.1 in workflow configuration --- .github/workflows/check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index ebada76..2cbb577 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -6,7 +6,7 @@ jobs: name: DCOR-CKAN runs-on: ubuntu-latest container: - image: ckan/ckan:2.11:latest + image: ckan/ckan-dev:2.11.1 options: --user root services: From f0cb710ab5deb06f7aeb31ea0bd525172f8214b7 Mon Sep 17 00:00:00 2001 From: RaghavaAlajangi Date: Tue, 6 May 2025 11:32:54 +0200 Subject: [PATCH 11/38] fix: add hostname option to CKAN container configuration --- .github/workflows/check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 2cbb577..a94d154 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest container: image: ckan/ckan-dev:2.11.1 - options: --user root + options: --user root --hostname dcor-docker services: solr: From d1f5696f9a2a06590ee8305efb05b09e325885cb Mon Sep 17 00:00:00 2001 From: RaghavaAlajangi Date: Tue, 6 May 2025 11:40:33 +0200 Subject: [PATCH 12/38] fix: remove hostname display and health check steps for CKAN in workflow --- .github/workflows/check.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index a94d154..ce132d8 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -49,20 +49,6 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Show hostname - run: hostname - - - name: Wait for CKAN to be healthy - run: | - for i in {1..60}; do - if curl -f http://localhost:5000; then - echo "CKAN is up!" - break - fi - echo "Waiting for CKAN to be ready..." - sleep 5 - done - - name: Install DCOR extensions inside the DCOR-CKAN container run: | pip3 install dcor_control==0.12.6 From e05538523477319e3d6ad6187c95ce39f664129c Mon Sep 17 00:00:00 2001 From: RaghavaAlajangi Date: Tue, 6 May 2025 11:41:35 +0200 Subject: [PATCH 13/38] fix: update test configuration to use docker-test.ini for pytest --- .github/workflows/check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index ce132d8..19af740 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -71,7 +71,7 @@ jobs: - name: Run tests run: | - pytest -v --ckan-ini=test.ini \ + pytest -v --ckan-ini=docker-test.ini \ --cov=ckanext.dc_view \ --cov-report=xml \ --cov-append \ From bfb6544114befd7496e4ded39bfbd0fa607f558a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Wed, 7 May 2025 10:30:05 +0200 Subject: [PATCH 14/38] ci: install latest dcor_control --- .github/workflows/check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 19af740..31f7a56 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -51,7 +51,7 @@ jobs: - name: Install DCOR extensions inside the DCOR-CKAN container run: | - pip3 install dcor_control==0.12.6 + pip3 install dcor_control dcor inspect --assume-yes - name: Install extention & requirements From 93cc5377b00d525fbddcae8b80ddf0be15d2edfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Wed, 7 May 2025 10:36:40 +0200 Subject: [PATCH 15/38] ci: set correct hostname --- .github/workflows/check.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 31f7a56..12d8b2d 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest container: image: ckan/ckan-dev:2.11.1 - options: --user root --hostname dcor-docker + options: --user root --hostname dcor-dev-docker services: solr: @@ -49,12 +49,12 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install DCOR extensions inside the DCOR-CKAN container + - name: Install DCOR extensions run: | pip3 install dcor_control dcor inspect --assume-yes - - name: Install extention & requirements + - name: Install extension and requirements run: | pip install . pip install -r ./ckanext/dc_view/tests/requirements.txt From a350b3b5053f910d4d11e4e159d25e2a7f4c9022 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Wed, 7 May 2025 10:44:37 +0200 Subject: [PATCH 16/38] ci: install latest dcor_control from gh --- .github/workflows/check.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 12d8b2d..bae5975 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -51,7 +51,8 @@ jobs: - name: Install DCOR extensions run: | - pip3 install dcor_control + # pip3 install dcor_control + pip3 install git+https://github.com/DCOR-dev/dcor_control dcor inspect --assume-yes - name: Install extension and requirements From 8b2404407aa5547180a7542fdda3f9e935fd6417 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Wed, 7 May 2025 10:53:30 +0200 Subject: [PATCH 17/38] enh: set correct docker postgres credentials --- .github/workflows/check.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index bae5975..cffd6b9 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -18,8 +18,8 @@ jobs: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres - CKAN_DB_USER: ckandbuser - CKAN_DB_PASSWORD: ckandbpassword + CKAN_DB_USER: ckan_default + CKAN_DB_PASSWORD: ckan_default_password CKAN_DB: ckandb DATASTORE_READONLY_USER: datastore_ro DATASTORE_READONLY_PASSWORD: datastore @@ -49,7 +49,7 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install DCOR extensions + - name: Install and setup DCOR run: | # pip3 install dcor_control pip3 install git+https://github.com/DCOR-dev/dcor_control From 87b2e7569a39fbf9812e7838f99a6ba4ef296269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Wed, 7 May 2025 10:57:55 +0200 Subject: [PATCH 18/38] ci: show ckan config --- .github/workflows/check.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index cffd6b9..2ddb4d3 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -53,6 +53,7 @@ jobs: run: | # pip3 install dcor_control pip3 install git+https://github.com/DCOR-dev/dcor_control + cat /srv/app/ckan.ini dcor inspect --assume-yes - name: Install extension and requirements From 194d1db7bd128a86da284badbaaafb89e68d9c29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Wed, 7 May 2025 11:38:04 +0200 Subject: [PATCH 19/38] ci: use latest ckan and postgres directly from ckan devs --- .github/workflows/check.yml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 2ddb4d3..fb3f5d5 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -3,30 +3,29 @@ on: [push, pull_request] jobs: test: - name: DCOR-CKAN + strategy: + matrix: + ckan-version: ["2.11"] + fail-fast: false + + name: DCOR-CKAN ${{ matrix.ckan-version }} runs-on: ubuntu-latest container: - image: ckan/ckan-dev:2.11.1 + image: ckan/ckan-dev:${{ matrix.ckan-version }} options: --user root --hostname dcor-dev-docker services: solr: - image: ckan/ckan-solr:2.11-solr9 + image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr9 postgres: - image: raghavaalajangi95/dcor-postgresql:latest + image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }} env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres - CKAN_DB_USER: ckan_default - CKAN_DB_PASSWORD: ckan_default_password - CKAN_DB: ckandb - DATASTORE_READONLY_USER: datastore_ro - DATASTORE_READONLY_PASSWORD: datastore - DATASTORE_DB: datastore options: --health-cmd "pg_isready" --health-interval 10s --health-timeout 5s --health-retries 10 redis: - image: redis:alpine + image: redis:3 minio: image: raghavaalajangi95/dcor-minio:latest env: From 67c4659e4e2b1a9ce8d1ce20cd332cf8f4dc0efb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Wed, 7 May 2025 12:19:08 +0200 Subject: [PATCH 20/38] ci: initialize db --- .github/workflows/check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index fb3f5d5..6c75933 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -52,7 +52,7 @@ jobs: run: | # pip3 install dcor_control pip3 install git+https://github.com/DCOR-dev/dcor_control - cat /srv/app/ckan.ini + ckan -c /srv/app/ckan.ini db init dcor inspect --assume-yes - name: Install extension and requirements From 0c07d159e10241652e0c3b91184c7933f6ce99b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Wed, 7 May 2025 12:25:17 +0200 Subject: [PATCH 21/38] ci: base everything on test-core.ini --- .github/workflows/check.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 6c75933..3ad5b19 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -52,6 +52,7 @@ jobs: run: | # pip3 install dcor_control pip3 install git+https://github.com/DCOR-dev/dcor_control + cp /srv/app/src/ckan/test-core.ini /srv/app/ckan.ini ckan -c /srv/app/ckan.ini db init dcor inspect --assume-yes @@ -59,8 +60,6 @@ jobs: run: | pip install . pip install -r ./ckanext/dc_view/tests/requirements.txt - # Replace default path to CKAN core config file with the one on the container - sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' docker-test.ini - name: Setup extension run: | @@ -72,7 +71,7 @@ jobs: - name: Run tests run: | - pytest -v --ckan-ini=docker-test.ini \ + pytest -v --ckan-ini=/srv/app/ckan.ini \ --cov=ckanext.dc_view \ --cov-report=xml \ --cov-append \ From 063f83fdbe5c94dab39db68835957962da765aab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Wed, 7 May 2025 12:34:50 +0200 Subject: [PATCH 22/38] ci: remove datastore config due to error --- .github/workflows/check.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 3ad5b19..fdc5fcf 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -40,8 +40,8 @@ jobs: env: CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test - CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test - CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test + #CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test + #CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test CKAN_SOLR_URL: http://solr:8983/solr/ckan CKAN_REDIS_URL: redis://redis:6379/1 From 09acf120be2d4554794ec14553a005871e0fd36d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Wed, 7 May 2025 12:42:15 +0200 Subject: [PATCH 23/38] ci: datastore allowed_functions.txt not found --- .github/workflows/check.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index fdc5fcf..642ed68 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -40,8 +40,8 @@ jobs: env: CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test - #CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test - #CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test + CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test + CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test CKAN_SOLR_URL: http://solr:8983/solr/ckan CKAN_REDIS_URL: redis://redis:6379/1 @@ -52,8 +52,13 @@ jobs: run: | # pip3 install dcor_control pip3 install git+https://github.com/DCOR-dev/dcor_control + # use test-core.ini as default cp /srv/app/src/ckan/test-core.ini /srv/app/ckan.ini - ckan -c /srv/app/ckan.ini db init + # initialize CKAN database + ckan -c /srv/app/ckan.ini db init + # fix location of allowed datastore commands + wget -O /srv/app/ckanext/datastore/tests/allowed_functions.txt https://raw.githubusercontent.com/ckan/ckan/refs/heads/master/ckanext/datastore/allowed_functions.txt + # complement configuration dcor inspect --assume-yes - name: Install extension and requirements From 2094518db13279ab4f57874efa00eeb935200bad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Wed, 7 May 2025 12:42:33 +0200 Subject: [PATCH 24/38] ci: datastore allowed_functions.txt not found --- .github/workflows/check.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 642ed68..ca40cd9 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -57,6 +57,7 @@ jobs: # initialize CKAN database ckan -c /srv/app/ckan.ini db init # fix location of allowed datastore commands + mkdir -p /srv/app/ckanext/datastore/tests/ wget -O /srv/app/ckanext/datastore/tests/allowed_functions.txt https://raw.githubusercontent.com/ckan/ckan/refs/heads/master/ckanext/datastore/allowed_functions.txt # complement configuration dcor inspect --assume-yes From 9913e356a2627d21eb97d1ab35d9249e38a6372e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Wed, 7 May 2025 12:46:36 +0200 Subject: [PATCH 25/38] ci: remove bad db init and remove docker-test.ini --- .github/workflows/check.yml | 4 --- docker-test.ini | 51 ------------------------------------- 2 files changed, 55 deletions(-) delete mode 100644 docker-test.ini diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index ca40cd9..c135209 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -67,10 +67,6 @@ jobs: pip install . pip install -r ./ckanext/dc_view/tests/requirements.txt - - name: Setup extension - run: | - ckan -c docker-test.ini db init - - name: Wait for MinIO to be ready run: | until curl -f http://minio:9000/minio/health/ready; do sleep 2; done diff --git a/docker-test.ini b/docker-test.ini deleted file mode 100644 index 558cca9..0000000 --- a/docker-test.ini +++ /dev/null @@ -1,51 +0,0 @@ -[DEFAULT] -debug = false -smtp_server = localhost -error_email_from = paste@localhost - -[server:main] -use = egg:Paste#http -host = 0.0.0.0 -port = 5000 - -[app:main] -use = config:../ckan/test-core.ini - -# Insert any custom config settings to be used when running your extension's -# tests here. - -ckan.legacy_templates = false -ckan.plugins = envvars - -# Logging configuration -[loggers] -keys = root, ckan, sqlalchemy - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console - -[logger_ckan] -qualname = ckan -handlers = -level = INFO - -[logger_sqlalchemy] -handlers = -qualname = sqlalchemy.engine -level = WARN - -[handler_console] -class = StreamHandler -args = (sys.stdout,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s \ No newline at end of file From 21a9d37214c26eff5c4d5e9065ac69dad95ea214 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Wed, 7 May 2025 13:05:17 +0200 Subject: [PATCH 26/38] ci: curl not available --- .github/workflows/check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index c135209..772054c 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -69,7 +69,7 @@ jobs: - name: Wait for MinIO to be ready run: | - until curl -f http://minio:9000/minio/health/ready; do sleep 2; done + until wget -q http://minio:9000/minio/health/ready; do sleep 2; done - name: Run tests run: | From 1c1e87fd99211e85dad2dfc4549cf8fb90f8f3c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Wed, 7 May 2025 13:20:00 +0200 Subject: [PATCH 27/38] ci: try to fix extension module not found --- .github/workflows/check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 772054c..b74ff2b 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -64,7 +64,7 @@ jobs: - name: Install extension and requirements run: | - pip install . + pip install -e . pip install -r ./ckanext/dc_view/tests/requirements.txt - name: Wait for MinIO to be ready From 705e3f39b07bcf5b2266e5e77347776b9fb70596 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Wed, 7 May 2025 13:25:02 +0200 Subject: [PATCH 28/38] docs --- .github/workflows/check.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index b74ff2b..fc91253 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -57,6 +57,7 @@ jobs: # initialize CKAN database ckan -c /srv/app/ckan.ini db init # fix location of allowed datastore commands + # https://github.com/ckan/ckan/issues/8926 mkdir -p /srv/app/ckanext/datastore/tests/ wget -O /srv/app/ckanext/datastore/tests/allowed_functions.txt https://raw.githubusercontent.com/ckan/ckan/refs/heads/master/ckanext/datastore/allowed_functions.txt # complement configuration From 84f286aa10ae830380d05c9f0f1ee10f36fab6ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Wed, 7 May 2025 13:28:55 +0200 Subject: [PATCH 29/38] ci: expand matrix --- .github/workflows/check.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index fc91253..2413fc5 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -5,7 +5,8 @@ jobs: test: strategy: matrix: - ckan-version: ["2.11"] + ckan-version: ["2.11", "2.10"] + dcor_control: ["develop", "pypi"] fail-fast: false name: DCOR-CKAN ${{ matrix.ckan-version }} @@ -63,6 +64,12 @@ jobs: # complement configuration dcor inspect --assume-yes + - name: Install latest DCOR extensions from GitHub + if: matrix.dcor_control == 'develop' + run: | + dcor develop --yes + dcor inspect --assume-yes + - name: Install extension and requirements run: | pip install -e . From a4224f1b3f6f40c12b22eb94773a53f5610abed8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Wed, 7 May 2025 13:33:20 +0200 Subject: [PATCH 30/38] ci: remove ckan 2.10 --- .github/workflows/check.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 2413fc5..dcf733f 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -5,11 +5,11 @@ jobs: test: strategy: matrix: - ckan-version: ["2.11", "2.10"] + ckan-version: ["2.11"] dcor_control: ["develop", "pypi"] fail-fast: false - name: DCOR-CKAN ${{ matrix.ckan-version }} + name: DCOR-CKAN ${{ matrix.ckan-version }} ${{ matrix.dcor_control }} runs-on: ubuntu-latest container: image: ckan/ckan-dev:${{ matrix.ckan-version }} From a25fc0911918baf13eae98d5ab036a2c3be52e96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Wed, 7 May 2025 13:39:54 +0200 Subject: [PATCH 31/38] ci: docs --- .github/workflows/check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index dcf733f..276c200 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -70,7 +70,7 @@ jobs: dcor develop --yes dcor inspect --assume-yes - - name: Install extension and requirements + - name: Install current extension and its testing requirements run: | pip install -e . pip install -r ./ckanext/dc_view/tests/requirements.txt From fcc73aee028c0e071d676ff7e2a7e08926b239e3 Mon Sep 17 00:00:00 2001 From: RaghavaAlajangi Date: Fri, 9 May 2025 10:45:33 +0200 Subject: [PATCH 32/38] ci: use minio official image and configure it in CI for transparency --- .github/workflows/check.yml | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 276c200..22d6800 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -28,13 +28,17 @@ jobs: redis: image: redis:3 minio: - image: raghavaalajangi95/dcor-minio:latest + image: minio/minio:latest env: MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin - MINIO_NEW_USER: dcor - MINIO_NEW_SECRET_KEY: secretkey12345678 - options: --health-cmd "curl -f http://localhost:9000/minio/health/ready || exit 1" --health-interval 10s --health-timeout 5s --health-retries 10 + options: >- + --entrypoint "minio" + --health-cmd "curl -f http://localhost:9000/minio/health/ready || exit 1" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + server --console-address :9001 /data ports: - 9000:9000 - 9001:9001 @@ -45,10 +49,26 @@ jobs: CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test CKAN_SOLR_URL: http://solr:8983/solr/ckan CKAN_REDIS_URL: redis://redis:6379/1 + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + MINIO_NEW_USER: dcor + MINIO_NEW_USER_KEY: secretkey12345678 + steps: - uses: actions/checkout@v4 + - name: Configure MinIO (user, bucket, policy) + run: | + curl -sO https://dl.min.io/client/mc/release/linux-amd64/mc + chmod +x mc && mv mc /usr/local/bin/mc + + mc alias set local http://minio:9000 "${MINIO_ROOT_USER}" "${MINIO_ROOT_PASSWORD}" + mc admin user add local "${MINIO_NEW_USER}" "${MINIO_NEW_SECRET_KEY}" || echo "User already exists" + mc admin policy attach local readwrite --user="${MINIO_NEW_USER}" || echo "Policy already attached" + mc mb local/ckan || echo "Bucket already exists" + mc policy set download local/ckan/storage || echo "Policy already set" + - name: Install and setup DCOR run: | # pip3 install dcor_control From f1c5f0fdf0a28840b20a85e82760efd2b8565846 Mon Sep 17 00:00:00 2001 From: RaghavaAlajangi Date: Fri, 9 May 2025 11:33:34 +0200 Subject: [PATCH 33/38] ci: update MinIO image to edge-cicd and remove entrypoint command --- .github/workflows/check.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 22d6800..f943c99 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -28,17 +28,15 @@ jobs: redis: image: redis:3 minio: - image: minio/minio:latest + image: minio/minio:edge-cicd env: MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin options: >- - --entrypoint "minio" --health-cmd "curl -f http://localhost:9000/minio/health/ready || exit 1" --health-interval 10s --health-timeout 5s --health-retries 10 - server --console-address :9001 /data ports: - 9000:9000 - 9001:9001 From 3899786f438e7f88bca24e6dd4cccd462bb7da75 Mon Sep 17 00:00:00 2001 From: RaghavaAlajangi Date: Fri, 9 May 2025 13:08:46 +0200 Subject: [PATCH 34/38] ci: update MinIO client download method --- .github/workflows/check.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index f943c99..d7ef3e2 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -24,7 +24,11 @@ jobs: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres - options: --health-cmd "pg_isready" --health-interval 10s --health-timeout 5s --health-retries 10 + options: >- + --health-cmd "pg_isready" + --health-interval 10s + --health-timeout 5s + --health-retries 10 redis: image: redis:3 minio: @@ -58,7 +62,7 @@ jobs: - name: Configure MinIO (user, bucket, policy) run: | - curl -sO https://dl.min.io/client/mc/release/linux-amd64/mc + wget -q https://dl.min.io/client/mc/release/linux-amd64/mc chmod +x mc && mv mc /usr/local/bin/mc mc alias set local http://minio:9000 "${MINIO_ROOT_USER}" "${MINIO_ROOT_PASSWORD}" @@ -93,10 +97,6 @@ jobs: pip install -e . pip install -r ./ckanext/dc_view/tests/requirements.txt - - name: Wait for MinIO to be ready - run: | - until wget -q http://minio:9000/minio/health/ready; do sleep 2; done - - name: Run tests run: | pytest -v --ckan-ini=/srv/app/ckan.ini \ From b71582f1eb9e1a1bed2b506ef37150c65e80fb8b Mon Sep 17 00:00:00 2001 From: RaghavaAlajangi Date: Fri, 9 May 2025 13:25:49 +0200 Subject: [PATCH 35/38] ci: update MinIO secret key environment variable name --- .github/workflows/check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index d7ef3e2..d7187f6 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -54,7 +54,7 @@ jobs: MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin MINIO_NEW_USER: dcor - MINIO_NEW_USER_KEY: secretkey12345678 + MINIO_NEW_SECRET_KEY: secretkey12345678 steps: From ffa3db2884d26ec78d75674b28376bda03b083f6 Mon Sep 17 00:00:00 2001 From: RaghavaAlajangi Date: Fri, 9 May 2025 14:38:13 +0200 Subject: [PATCH 36/38] ci: configure MinIO to run in distributed mode with console address --- .github/workflows/check.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index d7187f6..431954d 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -41,6 +41,8 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 10 + # Run MinIO in distributed mode, not filesystem mode + server --console-address ":9001" http://localhost:9000/data ports: - 9000:9000 - 9001:9001 From ce18883e41020f7ed7b3e81966c5614e6dfb6d7e Mon Sep 17 00:00:00 2001 From: RaghavaAlajangi Date: Fri, 9 May 2025 15:19:45 +0200 Subject: [PATCH 37/38] ci: remove MinIO distributed mode command and comment out policy attachment --- .github/workflows/check.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 431954d..342553f 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -41,8 +41,6 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 10 - # Run MinIO in distributed mode, not filesystem mode - server --console-address ":9001" http://localhost:9000/data ports: - 9000:9000 - 9001:9001 @@ -69,7 +67,7 @@ jobs: mc alias set local http://minio:9000 "${MINIO_ROOT_USER}" "${MINIO_ROOT_PASSWORD}" mc admin user add local "${MINIO_NEW_USER}" "${MINIO_NEW_SECRET_KEY}" || echo "User already exists" - mc admin policy attach local readwrite --user="${MINIO_NEW_USER}" || echo "Policy already attached" + # mc admin policy attach local readwrite --user="${MINIO_NEW_USER}" || echo "Policy already attached" mc mb local/ckan || echo "Bucket already exists" mc policy set download local/ckan/storage || echo "Policy already set" From 34b79adc9d007ae39fa2cdac553135338d07d645 Mon Sep 17 00:00:00 2001 From: RaghavaAlajangi Date: Fri, 9 May 2025 16:45:29 +0200 Subject: [PATCH 38/38] ci: update MinIO configuration and remove user management commands --- .github/workflows/check.yml | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 342553f..c3d5b34 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -32,10 +32,23 @@ jobs: redis: image: redis:3 minio: - image: minio/minio:edge-cicd + image: raghavaalajangi95/dcor-minio:latest + # Note: + # The official MinIO setup uses standalone mode, which has limitations: + # - Certain 'mc admin' commands, such as user and policy management, are not supported. + # - Using these commands will result in errors like: + # "This 'admin' API is not supported by server in 'mode-server-fs'" + # - `minio/minio:edge-cicd` container from MinIO is a good alternative, + # but it is still not supporting 'mc admin policy' command and the container is not managed properly. + # References: + # - https://github.com/minio/minio/discussions/16099 + # - https://github.com/orgs/community/discussions/26688#discussioncomment-4183701 + # - https://stackoverflow.com/questions/64031598/creating-a-minios3-container-inside-a-github-actions-yml-file/64188150#64188150 env: MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin + MINIO_NEW_USER: dcor + MINIO_NEW_SECRET_KEY: secretkey12345678 options: >- --health-cmd "curl -f http://localhost:9000/minio/health/ready || exit 1" --health-interval 10s @@ -51,26 +64,10 @@ jobs: CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test CKAN_SOLR_URL: http://solr:8983/solr/ckan CKAN_REDIS_URL: redis://redis:6379/1 - MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin - MINIO_NEW_USER: dcor - MINIO_NEW_SECRET_KEY: secretkey12345678 - steps: - uses: actions/checkout@v4 - - name: Configure MinIO (user, bucket, policy) - run: | - wget -q https://dl.min.io/client/mc/release/linux-amd64/mc - chmod +x mc && mv mc /usr/local/bin/mc - - mc alias set local http://minio:9000 "${MINIO_ROOT_USER}" "${MINIO_ROOT_PASSWORD}" - mc admin user add local "${MINIO_NEW_USER}" "${MINIO_NEW_SECRET_KEY}" || echo "User already exists" - # mc admin policy attach local readwrite --user="${MINIO_NEW_USER}" || echo "Policy already attached" - mc mb local/ckan || echo "Bucket already exists" - mc policy set download local/ckan/storage || echo "Policy already set" - - name: Install and setup DCOR run: | # pip3 install dcor_control @@ -92,7 +89,7 @@ jobs: dcor develop --yes dcor inspect --assume-yes - - name: Install current extension and its testing requirements + - name: Install extension and requirements run: | pip install -e . pip install -r ./ckanext/dc_view/tests/requirements.txt