Ckan 2-11-5

.env

@@ -52,11 +52,7 @@ CKAN_DATASTORE_WRITE_URL=postgresql://datastore:pass@datastore/datastore
 CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:pass@datastore/datastore
 
 ## Search Settings
-CKAN_SITE_ID=inventory
-CKAN_SOLR_URL=http://solr:8983/solr/ckan
-CKAN_SOLR_BASE_URL=http://solr:8983
-CKAN_SOLR_USER=catalog
-CKAN_SOLR_PASSWORD='Bleeding-Edge'
+CKAN_SOLR_URL=http://placeholder-value.local
 
 ## Redis settings
 CKAN_REDIS_URL=redis://redis:6379/0

.snyk

@@ -4,26 +4,6 @@ language-settings:
   python: "3.10"
 # ignores vulnerabilities until expiry date; change duration by modifying expiry date
 ignore:
-  SNYK-PYTHON-CKAN-16322869:
-    - '*':
-        reason: >-
-          Ticket created https://github.com/GSA/data.gov/issues/5941
-        expires: 2026-06-04T05:30:00.000Z
-  SNYK-PYTHON-CKAN-16322864:
-    - '*':
-        reason: >-
-          Ticket created https://github.com/GSA/data.gov/issues/5941
-        expires: 2026-06-04T05:30:00.000Z
-  SNYK-PYTHON-CKAN-16322872:
-    - '*':
-        reason: >-
-          Ticket created https://github.com/GSA/data.gov/issues/5941
-        expires: 2026-06-04T05:30:00.000Z
-  SNYK-PYTHON-CKAN-16427188:
-    - '*':
-        reason: >-
-          Ticket created https://github.com/GSA/data.gov/issues/5941
-        expires: 2026-06-04T05:30:00.000Z
   SNYK-PYTHON-PIP-16964647:
     - '*':
         reason: >-

docker-compose.yml

@@ -6,7 +6,6 @@ services:
     depends_on:
       - datastore
       - db
-      - solr
       - redis
       - localstack-container
     ports:
@@ -37,17 +36,6 @@ services:
       DB_CKAN_PASSWORD: pass
       DB_CKAN_DB: ckan
 
-  solr:
-    image: ghcr.io/gsa/catalog.data.gov.solr:8-stunnel-root
-    command: /app/solr/local_setup.sh
-    ports:
-      - "8983:8983"
-    deploy:
-      replicas: 1
-    volumes:
-      - solr_data:/var/solr
-      - .:/app
-
   redis:
     image: redis:alpine
 
@@ -70,4 +58,3 @@ services:
       - "./tmp/localstack:/var/lib/localstack"
 volumes:
   ckan:
-  solr_data:

e2e/cypress/integration/ckan_extensions.cy.js

@@ -2,7 +2,7 @@ describe('CKAN Extensions', () => {
     it('Uses CKAN 2.11', () => {
         cy.request('/api/action/status_show').should((response) => {
             expect(response.body).to.have.property('success', true);
-            expect(response.body.result).to.have.property('ckan_version', '2.11.4');
+            expect(response.body.result).to.have.property('ckan_version', '2.11.5');
         });
     });
 
@@ -14,11 +14,11 @@ describe('CKAN Extensions', () => {
             expect(installed_extensions).to.include('xloader');
             expect(installed_extensions).to.include('s3filestore');
             expect(installed_extensions).to.include('envvars');
-            expect(installed_extensions).to.include('datastore');
             expect(installed_extensions).to.include('datagov_inventory');
             expect(installed_extensions).to.include('dcat_usmetadata');
             expect(installed_extensions).to.include('usmetadata');
             expect(installed_extensions).to.include('datajson');
+            expect(installed_extensions).to.include('pgsearch');
             // TODO: Re-integrate saml2auth when automated testing is created for it
             // expect(installed_extensions).to.include('saml2auth');
         });

e2e/cypress/integration/organizations.cy.js

@@ -1,6 +1,7 @@
 describe('Organization', () => {
     before(() => {
         cy.create_token();
+        cy.delete_organization('cypress-test-org')
     })
 
     beforeEach(() => {

e2e/cypress/support/command.js

@@ -2,6 +2,28 @@ require('cypress-downloadfile/lib/downloadFileCommand');
 import Chance from 'chance';
 const chance = new Chance();
 
+function get_token_jti(api_token) {
+    const token_payload = api_token.split('.')[1].replace(/-/g, '+').replace(/_/g, '/');
+    const decoded_payload = JSON.parse(atob(token_payload));
+    return decoded_payload.jti;
+}
+
+function api_headers(extra_headers = {}) {
+    const token_data = Cypress.env('token_data');
+    const csrf_token = Cypress.$('meta[name="_csrf_token"]').attr('content');
+    const headers = {
+        'X-CKAN-API-Key': token_data.api_token,
+        'Content-Type': 'application/json',
+        ...extra_headers,
+    };
+
+    if (csrf_token) {
+        headers['X-CSRFToken'] = csrf_token;
+    }
+
+    return headers;
+}
+
 function verify_element_exists() {
     cy.get('td')
         .eq(4)
@@ -70,13 +92,11 @@ Cypress.Commands.add('create_token', (tokenName) => {
     cy.get('body').then($body => {
         cy.get('#name').type('cypress token');
         cy.get('button[value="create"]').click();
-        // find the token in <code> tag and save it for later use
-        // find the token id (jti) somewhere in the form
-        cy.get('div.alert-success code').invoke('text').then((text1) => {
-            cy.get('form[action^="/user/' + userName +'/api-tokens/"]').invoke('attr', 'action').then((text2) => {
-                const jti = text2.split('/')[4]
-                Cypress.env('token_data', { api_token: text1, jti: jti });
-            })
+        // Find the token in the success alert and save its JWT id for revocation.
+        cy.get('div.alert-success code').invoke('text').then((api_token) => {
+            api_token = api_token.trim();
+            const jti = get_token_jti(api_token);
+            Cypress.env('token_data', { api_token: api_token, jti: jti });
         });
         cy.log('cypress token created.');
     });
@@ -98,11 +118,10 @@ Cypress.Commands.add('revoke_token', (tokenName) => {
     cy.request({
         url: '/api/3/action/api_token_revoke',
         method: 'POST',
-        headers: {
-            'X-CKAN-API-Key': token_data.api_token,
-            'Content-Type': 'application/json'
-        },
+        headers: api_headers(),
         body: {jti: token_data.jti}
+    }).then(() => {
+        Cypress.env('token_data', null);
     });
 });
 
@@ -135,15 +154,10 @@ Cypress.Commands.add('create_organization', (orgName, orgDesc, extras = null) =>
      *      for testing or to visit the organization creation page
      * :RETURN null:
      */
-    const token_data = Cypress.env('token_data');
-
     let request_obj = {
         url: '/api/action/organization_create',
         method: 'POST',
-        headers: {
-            'X-CKAN-API-Key': token_data.api_token,
-            'Content-Type': 'application/json'
-        },
+        headers: api_headers(),
         body: {
             description: orgDesc,
             title: orgName,
@@ -173,16 +187,11 @@ Cypress.Commands.add('delete_organization', (orgName) => {
      * :PARAM orgName String: Name of the organization to purge from the current state
      * :RETURN null:
      */
-    const token_data = Cypress.env('token_data');
-
     cy.request({
         url: '/api/action/organization_delete',
         method: 'POST',
         failOnStatusCode: false,
-        headers: {
-            'X-CKAN-API-Key': token_data.api_token,
-            'Content-Type': 'application/json'
-        },
+        headers: api_headers(),
         body: {
             id: orgName? orgName: 'test-organization'
         },
@@ -192,10 +201,7 @@ Cypress.Commands.add('delete_organization', (orgName) => {
         url: '/api/action/organization_purge',
         method: 'POST',
         failOnStatusCode: false,
-        headers: {
-            'X-CKAN-API-Key': token_data.api_token,
-            'Content-Type': 'application/json'
-        },
+        headers: api_headers(),
         body: {
             id: orgName? orgName: 'test-organization'
         },
@@ -208,16 +214,11 @@ Cypress.Commands.add('create_user', (userName, userEmail, userPassword) => {
      * Method to create an user via CKAN API
      * :RETURN null:
      */
-    const token_data = Cypress.env('token_data');
-
     let request_obj = {
         url: '/api/action/user_create',
         method: 'POST',
         failOnStatusCode: false,
-        headers: {
-            'X-CKAN-API-Key': token_data.api_token,
-            'Content-Type': 'application/json'
-        },
+        headers: api_headers(),
         body: {
             name: userName,
             email: userEmail,
@@ -248,16 +249,11 @@ Cypress.Commands.add('assign_user', (orgName, userName, userRole) => {
      * Method to assign an organization role to an user via CKAN API
      * :RETURN null:
      */
-    const token_data = Cypress.env('token_data');
-
     let request_obj = {
         url: '/api/action/organization_member_create',
         method: 'POST',
         failOnStatusCode: true,
-        headers: {
-            'X-CKAN-API-Key': token_data.api_token,
-            'Content-Type': 'application/json'
-        },
+        headers: api_headers(),
         body: {
             id: orgName,
             username: userName,
@@ -275,15 +271,10 @@ Cypress.Commands.add('delete_user', (userName) => {
      * Method to delete an user via CKAN API
      * :RETURN null:
      */
-    const token_data = Cypress.env('token_data');
-
     let request_obj = {
         method: 'POST',
         failOnStatusCode: false,
-        headers: {
-            'X-CKAN-API-Key': token_data.api_token,
-            'Content-Type': 'application/json'
-        },
+        headers: api_headers(),
         body: {
             id: userName
         },
@@ -301,31 +292,25 @@ Cypress.Commands.add('delete_dataset', (datasetName) => {
      * :PARAM datasetName String: Name of the dataset to purge from the current state
      * :RETURN null:
      */
-    const token_data = Cypress.env('token_data');
     cy.request({
         url: '/api/action/dataset_purge',
         method: 'POST',
         failOnStatusCode: false,
-        headers: {
-            'X-CKAN-API-Key': token_data.api_token,
-            'Content-Type': 'application/json'
-        },
+        headers: api_headers(),
         body: {
             id: datasetName,
         },
     });
 });
 
 Cypress.Commands.add('create_dataset', (ckan_dataset) => {
-    const token_data = Cypress.env('token_data');
     var options = {
         method: 'POST',
         url: '/api/3/action/package_create',
-        headers: {
+        headers: api_headers({
             'cache-control': 'no-cache',
             'content-type': 'application/json',
-            'X-CKAN-API-Key': token_data.api_token,
-        },
+        }),
         body: JSON.stringify(ckan_dataset),
     };
 

requirements.in.txt

@@ -1,4 +1,4 @@
-git+https://github.com/GSA/ckan.git@ckan-2.11.4-nosolr#egg=ckan
+git+https://github.com/GSA/ckan.git@ckan-2.11.5-nosolr#egg=ckan
 # TODO https://github.com/GSA/datagov-deploy/issues/2794
 git+https://github.com/GSA/ckanext-pgsearch.git@main#egg=ckanext-pgsearch
 git+https://github.com/GSA/ckanext-saml2auth.git@ckan-2-11-datagov#egg=ckanext-saml2auth
@@ -12,48 +12,49 @@ ckanext-envvars>=0.0.3
 newrelic
 gunicorn
 
-# CKAN core library dependency upgrade pin
-# lxml==4.9.1
-lxml>=6.1.0
-# bleach==3.3.0
-# Jinja2==2.11.3
-
 # Add necessary CKAN core libraries
 alembic==1.13.2
 Babel==2.15.0
 bleach==6.1.0
-blinker==1.8.2
+blinker==1.9.0
+certifi>=2024.7.4
+click==8.1.7
 dominate==2.9.1
+#fedgen==1.0.0
 git+https://github.com/GSA/python-feedgen.git@main
-Flask==3.0.3
+Flask==3.1.3
 Flask-Babel==4.0.0
 Flask-Login==0.6.3
 Flask-Session==0.8.0
 Flask-WTF==1.2.1
-# Markdown==3.6
-Markdown==3.8.1
-msgspec==0.18.6
+greenlet==3.3.1
+Jinja2==3.1.6
+#lxml==6.0.2
+lxml>=6.1.0
+Markdown==3.10.2
+msgspec===0.20.0
 packaging==24.1
 passlib==1.7.4
 polib==1.2.0
-psycopg2==2.9.9
-# PyJWT==2.8.0
+psycopg2==2.9.11
+# PyJWT==2.12.1
 PyJWT==2.13.0
 pyparsing==3.1.2
 python-magic==0.4.27
-pysolr==3.9.0
+pysolr==3.11.0
 python-dateutil==2.9.0.post0
 pytz
 pyyaml==6.0.1
+requests==2.33.0
 rq==1.16.2
 simplejson==3.19.2
 SQLAlchemy[mypy]==1.4.52
 sqlparse==0.5.4
+#typing_extensions==4.12.2
 typing_extensions>=4.13.2
 tzlocal==5.2
 webassets==2.0
-# Werkzeug[watchdog]==3.1.5
-Werkzeug[watchdog]==3.1.6
+Werkzeug[watchdog]==3.1.8
 zope.interface==6.4post2
 
 # # ckanext-saml2 dependencies
@@ -84,17 +85,11 @@ importlib-resources<6.0
 zipp>=3.19.1
 gevent>=24.10.1
 
-pyparsing # need to avoid solr missing module error on cloud.gov
-
-
-certifi>=2024.7.4
 setuptools~=78.1.1
 
 # Pin MarkupSafe to avoid button issue data.gov/issues/4954 for logged in user
 # https://github.com/GSA/data.gov/issues/4954
 MarkupSafe==2.*
 
-jinja2>=3.1.6
 urllib3>=2.5.0
 cryptography>=46.0.7
-requests>=2.33.0