1150 get local plans data into provide tool

.github/workflows/test.yml

@@ -23,6 +23,7 @@ jobs:
     environment: development
     env:
       DOCKER_REPO: ${{ secrets.DEPLOY_DOCKER_REPOSITORY }}
+      TESTCONTAINERS_REUSE_ENABLE: true
     steps:
       - uses: actions/checkout@v4
       - name: Install AWS CLI

config/default.yaml

@@ -116,9 +116,14 @@ datasetsConfig:
       base: infrastructure funding statement
       variable: statement
 # Any dataset that has one of these three previous rules set, will be displayed in the LPA Dashboard to be checked.
+organisationTypes:
+  - local-authority
+  - national-park-authority
+  - development-corporation
+  - local-planning-group
 provisionReasons:
   - statutory
-  - prospective  
+  - prospective
   - expected
   - encouraged
 tablePageLength: 50

config/production.yaml

@@ -1,3 +1,7 @@
+organisationTypes:
+  - local-authority
+  - national-park-authority
+  - development-corporation
 asyncRequestApi: {
     url: http://production-pub-async-api-lb-636110663.eu-west-2.elb.amazonaws.com
 }

config/staging.yaml

@@ -1,3 +1,7 @@
+organisationTypes:
+  - local-authority
+  - national-park-authority
+  - development-corporation
 asyncRequestApi: {
     url: http://staging-pub-async-api-lb-12493311.eu-west-2.elb.amazonaws.com
 }

src/assets/scss/index.scss

@@ -93,7 +93,7 @@ $govuk-global-styles: true;
 }
 
 .big-number {
-  @include govuk-font($size: 80, $weight: bold);
+  @include govuk-font($size: 48, $weight: bold);
   display: block;
 }
 

src/filters/filters.js

@@ -32,7 +32,8 @@ export function statusToTagClass (status) {
 const { govukMarkdown, govukDateTime } = xGovFilters
 
 const addFilters = (nunjucksEnv) => {
-  nunjucksEnv.addFilter('datasetSlugToReadableName', datasetSlugToReadableName)
+  // Wrapper function for datasetSlugToReadableName filter to allow working with async loading of the mapping data
+  nunjucksEnv.addFilter('datasetSlugToReadableName', (...args) => datasetSlugToReadableName(...args))
 
   nunjucksEnv.addFilter('govukMarkdown', govukMarkdown)
   nunjucksEnv.addFilter('govukDateTime', govukDateTime)

src/middleware/common.middleware.js

@@ -110,7 +110,7 @@ export const takeResourceIdFromParams = (req) => {
 
 export const fetchOrgInfo = fetchOne({
   query: ({ params }) => {
-    return `SELECT name, organisation, entity, statistical_geography FROM organisation WHERE organisation = '${params.lpa}'`
+    return `SELECT name, organisation, entity, statistical_geography, dataset FROM organisation WHERE organisation = '${params.lpa}'`
   },
   result: 'orgInfo'
 })
@@ -212,8 +212,8 @@ export const prepareAuthority = async (req, res, next) => {
       return next()
     }
 
-    // Second query: Check for 'some' quality
-    const someResult = await platformApi.fetchEntities({
+    // Second query: Check for 'some' quality, need all, if stroing alternateEntityList
+    const someResult = await platformApi.fetchAllEntities({
       organisation_entity: orgInfo.entity,
       dataset: params.dataset,
       quality: 'some'
@@ -377,7 +377,7 @@ export const fetchSpecification = fetchOne({
 // Fall back dataset fields if no specification found
 
 export const fetchDatasetFields = fetchMany({
-  query: ({ req }) => `select field from dataset_field where dataset = '${req.dataset.collection}'`,
+  query: ({ req }) => `select field from dataset_field where dataset = '${req.dataset.dataset}'`,
   result: 'datasetFields'
 })
 
@@ -1179,3 +1179,75 @@ export const fetchEntityIssueCountsPerformanceDb = fetchMany({
   result: 'entityIssueCounts',
   dataset: FetchOptions.performanceDb
 })
+
+/**
+ * Middleware. Fetches all local-planning-group entities from the Platform API in a single call and derives two outputs:
+ *. - takes org code and:
+ * - req.parentGroup {Object[]|null} - If this org is a member of any planning group(s), returns an array of those
+ *   groups with { entity, name, organisation }. Null if this org belongs to no planning groups.
+ *
+ * - req.planningGroupMembers {Object[]|null} - If this org IS a planning group, returns an array of its member
+ *   organisations with { organisation, name }, where name is resolved via a parallel Platform API lookup.
+ *   Falls back to the org code as name if the lookup fails. Null if this org is not a planning group.
+ */
+export const fetchLocalPlanningGroups = async (req, res, next) => {
+  try {
+    const { formattedData: allGroups } = await platformApi.fetchAllEntities({ prefix: 'local-planning-group' })
+    // Remove all end-dated planning groups
+    const today = new Date().toISOString().slice(0, 10)
+    const groups = allGroups.filter(g => !g['end-date'] || g['end-date'] > today)
+    const orgCode = req.orgInfo.organisation
+
+    // Find any groups this org is within the organisations field.
+    const parentMatches = groups.filter(g => (g.organisations || '').split(';').includes(orgCode))
+    req.parentGroup = parentMatches.length > 0
+      ? parentMatches.map(g => ({ entity: g['organisation-entity'], name: g.name, organisation: `${g.prefix}:${g.reference}` }))
+      : null
+
+    // Look to see if this org is a local-planning group
+    const ownGroup = groups.find(g => String(g['organisation-entity']) === String(req.orgInfo.entity))
+
+    // if group get all members, and resolve their names in a single call to the Platform API, then map back to the org codes
+    if (ownGroup) {
+      const orgCodes = (ownGroup.organisations || '').split(';').filter(Boolean)
+      const { flat: allOrgs } = await platformApi.fetchOrganisations()
+      const nameMap = new Map(allOrgs.map(o => [o.organisation, o.name]))
+      req.planningGroupMembers = orgCodes.map(organisation => ({
+        organisation,
+        name: nameMap.get(organisation) ?? organisation
+      }))
+    } else {
+      req.planningGroupMembers = null
+    }
+  } catch (error) {
+    logger.warn({ message: `fetchLocalPlanningGroups(): ${error.message}`, type: types.App })
+    req.parentGroup = null
+    req.planningGroupMembers = null
+  }
+  next()
+}
+
+/**
+ * Fetches provision records for the current organisation and any planning groups it belongs to,
+ * filtered to the current dataset. Includes the organisation name via a JOIN on the organisation table.
+ *
+ * Requires: req.params.lpa, req.params.dataset, req.parentGroup (set by fetchLocalPlanningGroups)
+ * Sets: req.provisions — array of { dataset, project, provision_reason, organisation, name }
+ *
+ * TODO: Does it need fetchMany any more, would allow an append of Org Name to fetchLocalPlanningGroups result
+ */
+export const fetchProvisionsByOrgsAndDatasets = fetchMany({
+  query: ({ params, req }) => {
+    const orgs = [params.lpa]
+    if (req.parentGroup) {
+      orgs.push(...req.parentGroup.map(g => g.organisation))
+    }
+    const inClause = orgs.map(o => `'${o}'`).join(', ')
+    return /* sql */ `select p.dataset, p.project, p.provision_reason, p.organisation, o.name
+       from provision p
+       left join organisation o on o.organisation = p.organisation
+       where p.organisation IN (${inClause})
+       AND p.dataset = '${params.dataset}'`
+  },
+  result: 'provisions'
+})

src/middleware/datasetOverview.middleware.js

@@ -4,7 +4,7 @@
  * @description Middleware for dataset overview page (under /oranisations/:lpa/:dataset/overview)
  */
 
-import { fetchDatasetPlatformInfo, fetchEntityIssueCountsPerformanceDb, fetchOrgInfo, fetchResources, fetchSources, logPageError, processSpecificationMiddlewares, expectationFetcher, expectations, noop, processAuthoritativeMiddlewares } from './common.middleware.js'
+import { fetchDatasetPlatformInfo, fetchEntityIssueCountsPerformanceDb, fetchOrgInfo, fetchResources, fetchSources, logPageError, processSpecificationMiddlewares, expectationFetcher, expectations, noop, processAuthoritativeMiddlewares, fetchLocalPlanningGroups, fetchProvisionsByOrgsAndDatasets } from './common.middleware.js'
 import { fetchOne, fetchMany, onlyIf, renderTemplate, FetchOptions, FetchOneFallbackPolicy } from './middleware.builders.js'
 import { getDeadlineHistory, requiredDatasets } from '../utils/utils.js'
 import logger from '../utils/logger.js'
@@ -186,7 +186,7 @@ export const fetchEntityCount = fetchOne({
  * @param {Function} next - Express next middleware function
  */
 export const prepareDatasetOverviewTemplateParams = (req, res, next) => {
-  const { orgInfo, entityCount, sources, dataset, entityIssueCounts, notice, authority, alternateSources, uniqueDatasetFields, expectationOutOfBounds = [] } = req
+  const { orgInfo, entityCount, sources, dataset, entityIssueCounts, notice, authority, alternateSources, uniqueDatasetFields, expectationOutOfBounds = [], provisions = [], parentGroup } = req
 
   let endpointErrorIssues = 0
   const endpoints = sources
@@ -231,6 +231,10 @@ export const prepareDatasetOverviewTemplateParams = (req, res, next) => {
     : ''
   const downloadUrl = config.downloadUrl + `/${encodeURIComponent(dataset.dataset)}.csv?organisation-entity=${encodeURIComponent(orgInfo.entity)}&quality=${encodeURIComponent(authority)}${fieldsParams ? '&' + fieldsParams : ''}`
 
+  const planningGroupProvisions = provisions.length > 1
+    ? provisions.filter(p => p.organisation !== req.params.lpa)
+    : []
+
   req.templateParams = {
     downloadUrl,
     authority,
@@ -239,6 +243,8 @@ export const prepareDatasetOverviewTemplateParams = (req, res, next) => {
     dataset,
     taskCount,
     alternateSources,
+    planningGroupProvisions: planningGroupProvisions.length > 0 ? planningGroupProvisions : undefined,
+    parentGroup,
     stats: {
       numberOfRecords: entityCount.entity_count,
       endpoints
@@ -259,6 +265,8 @@ const getDatasetOverview = renderTemplate(
 
 export default [
   fetchOrgInfo,
+  fetchLocalPlanningGroups,
+  fetchProvisionsByOrgsAndDatasets,
   fetchDatasetPlatformInfo,
   fetchColumnSummary,
   fetchResources,

src/middleware/datasetTaskList.middleware.js

@@ -21,6 +21,8 @@ import {
   fetchEntityCount,
   fetchEntityIssueCounts,
   fetchEntryIssueCounts,
+  fetchLocalPlanningGroups,
+  fetchProvisionsByOrgsAndDatasets,
   fetchOrgInfo, fetchResources, fetchSources,
   logPageError,
   noop,
@@ -206,13 +208,17 @@ export const prepareTasks = (req, res, next) => {
  * @param {*} next
  */
 export const prepareDatasetTaskListTemplateParams = (req, res, next) => {
-  const { taskList, dataset, orgInfo: organisation, authority } = req
+  const { taskList, dataset, orgInfo: organisation, authority, provisions } = req
+  const planningGroupProvisions = provisions?.length > 1
+    ? provisions.filter(p => p.organisation !== req.params.lpa)
+    : []
 
   req.templateParams = {
     taskList,
     organisation,
     authority,
-    dataset
+    dataset,
+    planningGroupProvisions: planningGroupProvisions.length > 0 ? planningGroupProvisions : undefined
   }
   next()
 }
@@ -226,6 +232,8 @@ const getDatasetTaskList = renderTemplate({
 export default [
   validateOrgAndDatasetQueryParams,
   fetchOrgInfo,
+  fetchLocalPlanningGroups,
+  fetchProvisionsByOrgsAndDatasets,
   fetchSources,
   fetchDatasetInfo,
   fetchResources,

src/middleware/dataview.middleware.js

@@ -3,6 +3,8 @@ import {
   createPaginationTemplateParams,
   extractJsonFieldFromEntities,
   fetchDatasetInfo,
+  fetchLocalPlanningGroups,
+  fetchProvisionsByOrgsAndDatasets,
   fetchOrgInfo,
   processAuthoritativeMiddlewares,
   processSpecificationMiddlewares,
@@ -81,7 +83,7 @@ export const constructTableParams = (req, res, next) => {
 }
 
 export const prepareTemplateParams = (req, res, next) => {
-  const { orgInfo, dataset, tableParams, pagination, dataRange, entityIssueCounts, authority, alternateSources, uniqueDatasetFields } = req
+  const { orgInfo, dataset, tableParams, pagination, dataRange, entityIssueCounts, authority, alternateSources, uniqueDatasetFields, provisions } = req
 
   // Hard code task count for 'some' authority
   const taskCount = authority !== 'some' ? (entityIssueCounts ? entityIssueCounts.length : 0) : 1
@@ -91,6 +93,10 @@ export const prepareTemplateParams = (req, res, next) => {
     : ''
   const downloadUrl = config.downloadUrl + `/${encodeURIComponent(dataset.dataset)}.csv?organisation-entity=${encodeURIComponent(orgInfo.entity)}&quality=${encodeURIComponent(authority)}${fieldsParams ? '&' + fieldsParams : ''}`
 
+  const planningGroupProvisions = provisions?.length > 1
+    ? provisions.filter(p => p.organisation !== req.params.lpa)
+    : []
+
   req.templateParams = {
     downloadUrl,
     organisation: orgInfo,
@@ -100,7 +106,8 @@ export const prepareTemplateParams = (req, res, next) => {
     tableParams,
     pagination,
     dataRange,
-    alternateSources
+    alternateSources,
+    planningGroupProvisions: planningGroupProvisions.length > 0 ? planningGroupProvisions : undefined
   }
   next()
 }
@@ -117,6 +124,8 @@ export default [
 
   getSetBaseSubPath(['data']),
   fetchOrgInfo,
+  fetchLocalPlanningGroups,
+  fetchProvisionsByOrgsAndDatasets,
   fetchDatasetInfo,
 
   fetchResources,

src/middleware/getStarted.middleware.js

@@ -1,17 +1,27 @@
-import { fetchDatasetInfo, fetchOrgInfo, logPageError, prepareAuthority } from './common.middleware.js'
+import { fetchDatasetInfo, fetchLocalPlanningGroups, fetchProvisionsByOrgsAndDatasets, fetchOrgInfo, logPageError, prepareAuthority } from './common.middleware.js'
 import { renderTemplate } from './middleware.builders.js'
 
 export const getGetStarted = renderTemplate({
   templateParams (req) {
-    const { orgInfo: organisation, dataset, authority } = req
-    return { organisation, dataset, authority }
+    const { orgInfo: organisation, dataset, authority, provisions } = req
+    const planningGroupProvisions = provisions?.length > 1
+      ? provisions.filter(p => p.organisation !== req.params.lpa)
+      : []
+    return {
+      organisation,
+      dataset,
+      authority,
+      planningGroupProvisions: planningGroupProvisions.length > 0 ? planningGroupProvisions : undefined
+    }
   },
   template: 'organisations/get-started.html',
   handlerName: 'getStarted'
 })
 
 export default [
   fetchOrgInfo,
+  fetchLocalPlanningGroups,
+  fetchProvisionsByOrgsAndDatasets,
   fetchDatasetInfo,
   prepareAuthority,
   getGetStarted,

src/middleware/lpa-overview.middleware.js

@@ -4,7 +4,7 @@
  * @description Middleware for oragnisation (LPA) overview page
  */
 
-import { expectationFetcher, expectations, fetchEndpointSummary, fetchOrgInfo, logPageError, noop, setAvailableDatasets, fetchEntityIssueCountsPerformanceDb } from './common.middleware.js'
+import { expectationFetcher, expectations, fetchEndpointSummary, fetchOrgInfo, logPageError, noop, setAvailableDatasets, fetchEntityIssueCountsPerformanceDb, fetchLocalPlanningGroups } from './common.middleware.js'
 import { fetchMany, renderTemplate, parallel } from './middleware.builders.js'
 import { getDeadlineHistory, requiredDatasets } from '../utils/utils.js'
 import _ from 'lodash'
@@ -246,7 +246,7 @@ export function prepareDatasetObjects (req, res, next) {
  * @returns {void}
  */
 export function prepareOverviewTemplateParams (req, res, next) {
-  const { orgInfo: organisation, provisions, datasets, availableDatasets } = req
+  const { orgInfo: organisation, provisions, datasets, availableDatasets, parentGroup, planningGroupMembers } = req
 
   const provisionData = new Map()
   for (const provision of provisions ?? []) {
@@ -268,7 +268,6 @@ export function prepareOverviewTemplateParams (req, res, next) {
   })
 
   const isODPMember = provisions.findIndex((p) => p.project === 'open-digital-planning') >= 0
-  const totalDatasets = datasets.length
   const [datasetsWithEndpoints, datasetsWithIssues, datasetsWithErrors] = datasets.reduce(orgStatsReducer, [0, 0, 0])
   const datasetsByReason = _.groupBy(datasets, (ds) => {
     const reason = provisionData.get(ds.dataset)?.provision_reason
@@ -285,6 +284,7 @@ export function prepareOverviewTemplateParams (req, res, next) {
         return 'other'
     }
   })
+  const totalDatasets = (datasetsByReason.statutory?.length ?? 0) + (datasetsByReason.expected?.length ?? 0) + (datasetsByReason.prospective?.length ?? 0)
 
   for (const coll of Object.values(datasetsByReason)) {
     coll.sort((a, b) => a.dataset.localeCompare(b.dataset))
@@ -297,7 +297,9 @@ export function prepareOverviewTemplateParams (req, res, next) {
     datasetsWithEndpoints,
     datasetsWithIssues,
     datasetsWithErrors,
-    isODPMember
+    isODPMember,
+    parentGroup,
+    planningGroupMembers
   }
 
   next()
@@ -453,6 +455,7 @@ const fetchOutOfBoundsExpectations = expectationFetcher({
 export default [
   fetchOrgInfo,
   parallel([
+    fetchLocalPlanningGroups,
     fetchEndpointSummary,
     fetchEntityIssueCountsPerformanceDb,
     fetchProvisions