diff --git a/.env b/.env index 0e03944b..976001fa 100644 --- a/.env +++ b/.env @@ -52,11 +52,7 @@ CKAN_DATASTORE_WRITE_URL=postgresql://datastore:pass@datastore/datastore CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:pass@datastore/datastore ## Search Settings -CKAN_SITE_ID=inventory -CKAN_SOLR_URL=http://solr:8983/solr/ckan -CKAN_SOLR_BASE_URL=http://solr:8983 -CKAN_SOLR_USER=catalog -CKAN_SOLR_PASSWORD='Bleeding-Edge' +CKAN_SOLR_URL=http://placeholder-value.local ## Redis settings CKAN_REDIS_URL=redis://redis:6379/0 diff --git a/.snyk b/.snyk index 676df7c5..d277ee9c 100644 --- a/.snyk +++ b/.snyk @@ -4,26 +4,6 @@ language-settings: python: "3.10" # ignores vulnerabilities until expiry date; change duration by modifying expiry date ignore: - SNYK-PYTHON-CKAN-16322869: - - '*': - reason: >- - Ticket created https://github.com/GSA/data.gov/issues/5941 - expires: 2026-06-04T05:30:00.000Z - SNYK-PYTHON-CKAN-16322864: - - '*': - reason: >- - Ticket created https://github.com/GSA/data.gov/issues/5941 - expires: 2026-06-04T05:30:00.000Z - SNYK-PYTHON-CKAN-16322872: - - '*': - reason: >- - Ticket created https://github.com/GSA/data.gov/issues/5941 - expires: 2026-06-04T05:30:00.000Z - SNYK-PYTHON-CKAN-16427188: - - '*': - reason: >- - Ticket created https://github.com/GSA/data.gov/issues/5941 - expires: 2026-06-04T05:30:00.000Z SNYK-PYTHON-PIP-16964647: - '*': reason: >- diff --git a/docker-compose.yml b/docker-compose.yml index d4f8d21d..9549e027 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,6 @@ services: depends_on: - datastore - db - - solr - redis - localstack-container ports: @@ -37,17 +36,6 @@ services: DB_CKAN_PASSWORD: pass DB_CKAN_DB: ckan - solr: - image: ghcr.io/gsa/catalog.data.gov.solr:8-stunnel-root - command: /app/solr/local_setup.sh - ports: - - "8983:8983" - deploy: - replicas: 1 - volumes: - - solr_data:/var/solr - - .:/app - redis: image: redis:alpine @@ -70,4 +58,3 @@ services: - "./tmp/localstack:/var/lib/localstack" volumes: ckan: - solr_data: diff --git a/e2e/cypress/integration/ckan_extensions.cy.js b/e2e/cypress/integration/ckan_extensions.cy.js index 9c5bdd49..2f9fe39c 100644 --- a/e2e/cypress/integration/ckan_extensions.cy.js +++ b/e2e/cypress/integration/ckan_extensions.cy.js @@ -2,7 +2,7 @@ describe('CKAN Extensions', () => { it('Uses CKAN 2.11', () => { cy.request('/api/action/status_show').should((response) => { expect(response.body).to.have.property('success', true); - expect(response.body.result).to.have.property('ckan_version', '2.11.4'); + expect(response.body.result).to.have.property('ckan_version', '2.11.5'); }); }); @@ -14,11 +14,11 @@ describe('CKAN Extensions', () => { expect(installed_extensions).to.include('xloader'); expect(installed_extensions).to.include('s3filestore'); expect(installed_extensions).to.include('envvars'); - expect(installed_extensions).to.include('datastore'); expect(installed_extensions).to.include('datagov_inventory'); expect(installed_extensions).to.include('dcat_usmetadata'); expect(installed_extensions).to.include('usmetadata'); expect(installed_extensions).to.include('datajson'); + expect(installed_extensions).to.include('pgsearch'); // TODO: Re-integrate saml2auth when automated testing is created for it // expect(installed_extensions).to.include('saml2auth'); }); diff --git a/e2e/cypress/integration/organizations.cy.js b/e2e/cypress/integration/organizations.cy.js index 4c467656..0a887110 100644 --- a/e2e/cypress/integration/organizations.cy.js +++ b/e2e/cypress/integration/organizations.cy.js @@ -1,6 +1,7 @@ describe('Organization', () => { before(() => { cy.create_token(); + cy.delete_organization('cypress-test-org') }) beforeEach(() => { diff --git a/e2e/cypress/support/command.js b/e2e/cypress/support/command.js index 158a0907..c734bd4e 100644 --- a/e2e/cypress/support/command.js +++ b/e2e/cypress/support/command.js @@ -2,6 +2,28 @@ require('cypress-downloadfile/lib/downloadFileCommand'); import Chance from 'chance'; const chance = new Chance(); +function get_token_jti(api_token) { + const token_payload = api_token.split('.')[1].replace(/-/g, '+').replace(/_/g, '/'); + const decoded_payload = JSON.parse(atob(token_payload)); + return decoded_payload.jti; +} + +function api_headers(extra_headers = {}) { + const token_data = Cypress.env('token_data'); + const csrf_token = Cypress.$('meta[name="_csrf_token"]').attr('content'); + const headers = { + 'X-CKAN-API-Key': token_data.api_token, + 'Content-Type': 'application/json', + ...extra_headers, + }; + + if (csrf_token) { + headers['X-CSRFToken'] = csrf_token; + } + + return headers; +} + function verify_element_exists() { cy.get('td') .eq(4) @@ -70,13 +92,11 @@ Cypress.Commands.add('create_token', (tokenName) => { cy.get('body').then($body => { cy.get('#name').type('cypress token'); cy.get('button[value="create"]').click(); - // find the token in tag and save it for later use - // find the token id (jti) somewhere in the form - cy.get('div.alert-success code').invoke('text').then((text1) => { - cy.get('form[action^="/user/' + userName +'/api-tokens/"]').invoke('attr', 'action').then((text2) => { - const jti = text2.split('/')[4] - Cypress.env('token_data', { api_token: text1, jti: jti }); - }) + // Find the token in the success alert and save its JWT id for revocation. + cy.get('div.alert-success code').invoke('text').then((api_token) => { + api_token = api_token.trim(); + const jti = get_token_jti(api_token); + Cypress.env('token_data', { api_token: api_token, jti: jti }); }); cy.log('cypress token created.'); }); @@ -98,11 +118,10 @@ Cypress.Commands.add('revoke_token', (tokenName) => { cy.request({ url: '/api/3/action/api_token_revoke', method: 'POST', - headers: { - 'X-CKAN-API-Key': token_data.api_token, - 'Content-Type': 'application/json' - }, + headers: api_headers(), body: {jti: token_data.jti} + }).then(() => { + Cypress.env('token_data', null); }); }); @@ -135,15 +154,10 @@ Cypress.Commands.add('create_organization', (orgName, orgDesc, extras = null) => * for testing or to visit the organization creation page * :RETURN null: */ - const token_data = Cypress.env('token_data'); - let request_obj = { url: '/api/action/organization_create', method: 'POST', - headers: { - 'X-CKAN-API-Key': token_data.api_token, - 'Content-Type': 'application/json' - }, + headers: api_headers(), body: { description: orgDesc, title: orgName, @@ -173,16 +187,11 @@ Cypress.Commands.add('delete_organization', (orgName) => { * :PARAM orgName String: Name of the organization to purge from the current state * :RETURN null: */ - const token_data = Cypress.env('token_data'); - cy.request({ url: '/api/action/organization_delete', method: 'POST', failOnStatusCode: false, - headers: { - 'X-CKAN-API-Key': token_data.api_token, - 'Content-Type': 'application/json' - }, + headers: api_headers(), body: { id: orgName? orgName: 'test-organization' }, @@ -192,10 +201,7 @@ Cypress.Commands.add('delete_organization', (orgName) => { url: '/api/action/organization_purge', method: 'POST', failOnStatusCode: false, - headers: { - 'X-CKAN-API-Key': token_data.api_token, - 'Content-Type': 'application/json' - }, + headers: api_headers(), body: { id: orgName? orgName: 'test-organization' }, @@ -208,16 +214,11 @@ Cypress.Commands.add('create_user', (userName, userEmail, userPassword) => { * Method to create an user via CKAN API * :RETURN null: */ - const token_data = Cypress.env('token_data'); - let request_obj = { url: '/api/action/user_create', method: 'POST', failOnStatusCode: false, - headers: { - 'X-CKAN-API-Key': token_data.api_token, - 'Content-Type': 'application/json' - }, + headers: api_headers(), body: { name: userName, email: userEmail, @@ -248,16 +249,11 @@ Cypress.Commands.add('assign_user', (orgName, userName, userRole) => { * Method to assign an organization role to an user via CKAN API * :RETURN null: */ - const token_data = Cypress.env('token_data'); - let request_obj = { url: '/api/action/organization_member_create', method: 'POST', failOnStatusCode: true, - headers: { - 'X-CKAN-API-Key': token_data.api_token, - 'Content-Type': 'application/json' - }, + headers: api_headers(), body: { id: orgName, username: userName, @@ -275,15 +271,10 @@ Cypress.Commands.add('delete_user', (userName) => { * Method to delete an user via CKAN API * :RETURN null: */ - const token_data = Cypress.env('token_data'); - let request_obj = { method: 'POST', failOnStatusCode: false, - headers: { - 'X-CKAN-API-Key': token_data.api_token, - 'Content-Type': 'application/json' - }, + headers: api_headers(), body: { id: userName }, @@ -301,15 +292,11 @@ Cypress.Commands.add('delete_dataset', (datasetName) => { * :PARAM datasetName String: Name of the dataset to purge from the current state * :RETURN null: */ - const token_data = Cypress.env('token_data'); cy.request({ url: '/api/action/dataset_purge', method: 'POST', failOnStatusCode: false, - headers: { - 'X-CKAN-API-Key': token_data.api_token, - 'Content-Type': 'application/json' - }, + headers: api_headers(), body: { id: datasetName, }, @@ -317,15 +304,13 @@ Cypress.Commands.add('delete_dataset', (datasetName) => { }); Cypress.Commands.add('create_dataset', (ckan_dataset) => { - const token_data = Cypress.env('token_data'); var options = { method: 'POST', url: '/api/3/action/package_create', - headers: { + headers: api_headers({ 'cache-control': 'no-cache', 'content-type': 'application/json', - 'X-CKAN-API-Key': token_data.api_token, - }, + }), body: JSON.stringify(ckan_dataset), }; diff --git a/requirements.in.txt b/requirements.in.txt index a83c591e..f20c6fc6 100644 --- a/requirements.in.txt +++ b/requirements.in.txt @@ -1,4 +1,4 @@ -git+https://github.com/GSA/ckan.git@ckan-2.11.4-nosolr#egg=ckan +git+https://github.com/GSA/ckan.git@ckan-2.11.5-nosolr#egg=ckan # TODO https://github.com/GSA/datagov-deploy/issues/2794 git+https://github.com/GSA/ckanext-pgsearch.git@main#egg=ckanext-pgsearch git+https://github.com/GSA/ckanext-saml2auth.git@ckan-2-11-datagov#egg=ckanext-saml2auth @@ -12,48 +12,49 @@ ckanext-envvars>=0.0.3 newrelic gunicorn -# CKAN core library dependency upgrade pin -# lxml==4.9.1 -lxml>=6.1.0 -# bleach==3.3.0 -# Jinja2==2.11.3 - # Add necessary CKAN core libraries alembic==1.13.2 Babel==2.15.0 bleach==6.1.0 -blinker==1.8.2 +blinker==1.9.0 +certifi>=2024.7.4 +click==8.1.7 dominate==2.9.1 +#fedgen==1.0.0 git+https://github.com/GSA/python-feedgen.git@main -Flask==3.0.3 +Flask==3.1.3 Flask-Babel==4.0.0 Flask-Login==0.6.3 Flask-Session==0.8.0 Flask-WTF==1.2.1 -# Markdown==3.6 -Markdown==3.8.1 -msgspec==0.18.6 +greenlet==3.3.1 +Jinja2==3.1.6 +#lxml==6.0.2 +lxml>=6.1.0 +Markdown==3.10.2 +msgspec===0.20.0 packaging==24.1 passlib==1.7.4 polib==1.2.0 -psycopg2==2.9.9 -# PyJWT==2.8.0 +psycopg2==2.9.11 +# PyJWT==2.12.1 PyJWT==2.13.0 pyparsing==3.1.2 python-magic==0.4.27 -pysolr==3.9.0 +pysolr==3.11.0 python-dateutil==2.9.0.post0 pytz pyyaml==6.0.1 +requests==2.33.0 rq==1.16.2 simplejson==3.19.2 SQLAlchemy[mypy]==1.4.52 sqlparse==0.5.4 +#typing_extensions==4.12.2 typing_extensions>=4.13.2 tzlocal==5.2 webassets==2.0 -# Werkzeug[watchdog]==3.1.5 -Werkzeug[watchdog]==3.1.6 +Werkzeug[watchdog]==3.1.8 zope.interface==6.4post2 # # ckanext-saml2 dependencies @@ -84,17 +85,11 @@ importlib-resources<6.0 zipp>=3.19.1 gevent>=24.10.1 -pyparsing # need to avoid solr missing module error on cloud.gov - - -certifi>=2024.7.4 setuptools~=78.1.1 # Pin MarkupSafe to avoid button issue data.gov/issues/4954 for logged in user # https://github.com/GSA/data.gov/issues/4954 MarkupSafe==2.* -jinja2>=3.1.6 urllib3>=2.5.0 cryptography>=46.0.7 -requests>=2.33.0 diff --git a/requirements.txt b/requirements.txt index d6faef73..8676100d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,41 +4,41 @@ async-timeout==5.0.1 attrs==26.1.0 Babel==2.15.0 bleach==6.1.0 -blinker==1.8.2 -boto3==1.43.18 -botocore==1.43.18 +blinker==1.9.0 +boto3==1.43.22 +botocore==1.43.22 cachelib==0.14.0 certifi==2026.5.20 cffi==2.0.0 chardet==7.4.3 charset-normalizer==3.4.7 -ckan @ git+https://github.com/GSA/ckan.git@1af1607e4f74fe4855b4ae4814c60fff1b2d199c +ckan @ git+https://github.com/GSA/ckan.git@1a46b19a3343c589289275e8babca28a3b16af5a ckanext-datajson==0.1.28 ckanext-dcat-usmetadata==0.6.3 ckanext-envvars==0.0.6 -ckanext-pgsearch @ git+https://github.com/GSA/ckanext-pgsearch.git@2d5fe7023cc9caf7192da67e648b7fd9e484ca87 +ckanext-pgsearch @ git+https://github.com/GSA/ckanext-pgsearch.git@48492c63fbaed8bcd892194b22192ffb404f2902 ckanext-s3filestore @ git+https://github.com/keitaroinc/ckanext-s3filestore.git@e972ae3c36489dcfc716f75063ac64c13cde1146 ckanext-saml2auth @ git+https://github.com/GSA/ckanext-saml2auth.git@062f26b19a525139bd613ef3654ca9dc64e0a096 ckanext-usmetadata==0.3.3 -e git+https://github.com/ckan/ckanext-xloader.git@11eb3e64867ac9aa3cab95236e3eed520f601012#egg=ckanext_xloader ckantoolkit==0.0.7 -click==8.4.1 +click==8.1.7 cryptography==48.0.0 defusedxml==0.7.1 dominate==2.9.1 elementpath==5.1.1 et_xmlfile==2.0.0 feedgen @ git+https://github.com/GSA/python-feedgen.git@b1fe34a7e14ebdaf6f415cdee5855c3c77305f16 -Flask==3.0.3 +Flask==3.1.3 flask-babel==4.0.0 Flask-Login==0.6.3 Flask-Session==0.8.0 Flask-WTF==1.2.1 gevent==26.5.0 -greenlet==3.5.1 +greenlet==3.3.1 gunicorn==26.0.0 html5lib==1.1 -idna==3.17 +idna==3.18 ijson==3.5.0 importlib-resources==5.13.0 itsdangerous==2.2.0 @@ -51,13 +51,13 @@ librt==0.11.0 linear-tsv==1.1.0 lxml==6.1.1 Mako==1.3.12 -Markdown==3.8.1 +Markdown==3.10.2 MarkupSafe==2.1.5 messytables==0.15.2 -msgspec==0.18.6 +msgspec==0.20.0 mypy==2.1.0 mypy_extensions==1.1.0 -newrelic==13.0.1 +newrelic==13.1.0 openpyxl==3.1.5 packaging==24.1 passlib==1.7.4 @@ -65,19 +65,19 @@ pathspec==1.1.1 pika==1.4.1 pip==26.1.1 polib==1.2.0 -psycopg2==2.9.9 +psycopg2==2.9.11 pycparser==3.0 PyJWT==2.13.0 pyOpenSSL==26.2.0 pyparsing==3.1.2 pysaml2==7.3.1 -pysolr==3.9.0 +pysolr==3.11.0 python-dateutil==2.9.0.post0 python-magic==0.4.27 pytz==2026.2 PyYAML==6.0.1 redis==8.0.0 -requests==2.34.2 +requests==2.33.0 rfc3987==1.3.8 rq==1.16.2 s3transfer==0.18.0 @@ -98,7 +98,7 @@ urllib3==2.7.0 watchdog==6.0.0 webassets==2.0 webencodings==0.5.1 -Werkzeug==3.1.6 +Werkzeug==3.1.8 wheel==0.46.2 WTForms==3.2.2 xlrd==2.0.2 diff --git a/start.sh b/start.sh index f0ce8eb5..a4040713 100755 --- a/start.sh +++ b/start.sh @@ -5,15 +5,12 @@ set -e function wait_for () { local host=$1 local port=$2 - + while ! nc -z -w 5 "$host" "$port"; do sleep 1 done } -echo -n "Waiting for Solr..." -wait_for solr 8983 -echo "ok" echo -n "Waiting for DB..." wait_for db 5432 echo "ok"