Merge pull request #1118 from digital-land/1116-lpa-dashboard-generalisation-2---show-all-datasets-in-development

Datasets used are now loaded dynamically based on a query to provision table

Author: pooleycodes

config/default.yaml

@@ -98,6 +98,11 @@ datasetsConfig:
     entityDisplayName:
       base: listed building
       variable: outline
+# Any dataset that has one of these three previous rules set, will be displayed in the LPA Dashboard to be checked.
+provisionReasons:
+  - statutory
+  - prospective  
+  - expected
 tablePageLength: 50
 jira: {
   requestTypeId: 1

src/routes/schemas.js

@@ -116,7 +116,7 @@ const IssueSpecification = v.optional(v.strictObject({
 }))
 
 const OrgField = v.strictObject({ name: NonEmptyString, organisation: NonEmptyString, statistical_geography: v.optional(v.string()), entity: v.optional(v.integer()) })
-const DatasetNameField = v.strictObject({ name: NonEmptyString, dataset: NonEmptyString, collection: NonEmptyString })
+const DatasetNameField = v.strictObject({ name: NonEmptyString, dataset: NonEmptyString, collection: v.string() })
 const DatasetItem = v.strictObject({
   endpointCount: v.optional(v.number()),
   status: v.enum(datasetStatusEnum),

src/utils/datasetLoader.js

@@ -9,7 +9,7 @@ export async function getRedisClient () {
   if (!config.redis) return null
 
   if (!redisClient) {
-    const urlPrefix = 'redis'
+    const urlPrefix = `redis${config.redis.secure ? 's' : ''}`
     redisClient = createClient({
       url: `${urlPrefix}://${config.redis.host}:${config.redis.port}`
     })
@@ -35,6 +35,7 @@ export async function getRedisClient () {
 
 const CACHE_TTL = 300 // 5min
 
+// TODO: future removal of this function in favour of using datasetNameSlug and datasetSubjectLoaded instead.
 export async function fetchDatasetNames (datasetKeys) {
   if (!datasetKeys?.length) return {}
   const params = new URLSearchParams()

src/utils/datasetSubjectLoader.js

@@ -0,0 +1,197 @@
+import fetchDatasetsRequiredForLocalAuthority from './datasetteQueries/fetchDatasetsFromProvisions.js'
+import { datasetSlugToReadableName } from './datasetSlugToReadableName.js'
+import logger from './logger.js'
+import config from '../../config/index.js'
+import { getDatasetCollectionSlugNameMapping } from './datasetteQueries/fetchDatasetCollections.js'
+import { getRedisClient } from './datasetLoader.js'
+
+// Use a short TTL in development
+const CACHE_TTL = (['development', 'local'].includes(config.environment)) ? 60 : 60 * 60 // 1 minute or 1 hour depending on environment
+
+const fallbackDataSubjects = {
+  'article-4-direction': {
+    available: true,
+    dataSets: [
+      {
+        value: 'article-4-direction',
+        text: 'Article 4 direction',
+        available: true
+      },
+      {
+        value: 'article-4-direction-area',
+        text: 'Article 4 direction area',
+        available: true
+      }
+    ]
+  },
+  'brownfield-land': {
+    available: true,
+    dataSets: [
+      {
+        value: 'brownfield-land',
+        text: 'Brownfield land',
+        available: true
+      },
+      {
+        value: 'brownfield-site',
+        text: 'Brownfield site',
+        available: false
+      }
+    ]
+  },
+  'conservation-area': {
+    available: true,
+    dataSets: [
+      {
+        value: 'conservation-area',
+        text: 'Conservation area',
+        available: true
+      },
+      {
+        value: 'conservation-area-document',
+        text: 'Conservation area document',
+        available: true
+      }
+    ]
+  },
+  'listed-building': {
+    available: true,
+    dataSets: [
+      {
+        value: 'listed-building',
+        text: 'Listed building',
+        available: false
+      },
+      {
+        value: 'listed-building-grade',
+        text: 'Listed building grade',
+        available: false
+      },
+      {
+        value: 'listed-building-outline',
+        text: 'Listed building outline',
+        available: true
+      }
+    ]
+  },
+  'tree-preservation-order': {
+    available: true,
+    dataSets: [
+      {
+        value: 'tree',
+        text: 'Tree',
+        available: true,
+        requiresGeometryTypeSelection: true
+      },
+      {
+        value: 'tree-preservation-order',
+        text: 'Tree preservation order',
+        available: true
+      },
+      {
+        value: 'tree-preservation-zone',
+        text: 'Tree preservation zone',
+        available: true
+      }
+    ]
+  }
+}
+
+/* Assign datasets to their collections if a mapping exists; otherwise, group them under 'other', also a check here is done to see if dataset table exists.
+* Special handling: e.g., the 'tree' dataset requires geometry type selection
+* The fallbackDataSubjects demos what is being built.
+*/
+export async function makeDatasetSubjectMap (nameMap) {
+  const dataSubjectMapping = await getDatasetCollectionSlugNameMapping(nameMap)
+
+  if (!dataSubjectMapping) {
+    logger.warn('Failed to fetch dataset collection mapping, using fallback')
+    return fallbackDataSubjects
+  }
+
+  const subjects = {}
+  for (const key of Object.keys(nameMap)) {
+    const dataset = {
+      value: key,
+      text: nameMap[key],
+      available: true
+    }
+
+    // Add special handling for any: currenlty only for tree dataset
+    if (key === 'tree') {
+      dataset.requiresGeometryTypeSelection = true
+    }
+
+    // Only add if there is a collection mapping, if mapping is to empty collection '', make key ='other'
+    const mapping = dataSubjectMapping.get(key)
+    if (mapping === undefined) {
+      continue
+    }
+    const subjectKey = mapping === '' ? 'other' : mapping
+
+    // Initialize the subject if it doesn't exist
+    if (!subjects[subjectKey]) {
+      subjects[subjectKey] = {
+        available: true,
+        dataSets: []
+      }
+    }
+
+    subjects[subjectKey].dataSets.push(dataset)
+  }
+
+  return subjects
+}
+
+// Build data subjects by fetching dataset keys from provision table instead of hard coding.
+export async function buildDataSubjects () {
+  // First try to fetch dataset keys(slugs) from provisions table (i.e.the keys(datasets) that the LA is expected to provide based on provision rules)
+  let datasetKeys = []
+  try {
+    datasetKeys = await fetchDatasetsRequiredForLocalAuthority()
+  } catch (error) {
+    logger.warn('buildDataSubjects: Error fetching dataset keys roll back to defaults')
+    datasetKeys = Object.keys(config.datasetsConfig)
+  }
+
+  // Use existing datasetSlugToReadableName to create lookup of dataset keys to readable names
+  const nameMap = {}
+  for (const key of datasetKeys) {
+    nameMap[key] = datasetSlugToReadableName(key)
+  }
+
+  return makeDatasetSubjectMap(nameMap)
+}
+
+// Get data subject map, with Redis caching
+export async function getDataSubjectMap () {
+  let dataSubjectMap = {}
+
+  const cacheKey = 'dataset:dataSubjectMap'
+  const client = await getRedisClient()
+
+  if (client) {
+    try {
+      const cached = await client.get(cacheKey)
+      if (cached) {
+        dataSubjectMap = JSON.parse(cached)
+        return dataSubjectMap
+      }
+    } catch (err) {
+      logger.warn(`datasetSubjectLoader/redis get error: ${err.message}`)
+    }
+  }
+
+  // fallback → fetch fresh
+  logger.info('getDataSubjectMap: Dataset subject map cache miss, rebuilding data subject map')
+  dataSubjectMap = await buildDataSubjects()
+
+  if (client) {
+    try {
+      await client.setEx(cacheKey, CACHE_TTL, JSON.stringify(dataSubjectMap))
+    } catch (err) {
+      logger.warn(`datasetSubjectLoader/redis set error: ${err.message}`)
+    }
+  }
+  return dataSubjectMap
+}

src/utils/datasetteQueries/fetchDatasetCollections.js

@@ -0,0 +1,40 @@
+import datasette from '../../services/datasette.js'
+import logger from '../logger.js'
+import { types } from '../logging.js'
+
+// Temporary utility to get dataset collection slug-name mapping from Datasette, likely/hopefully don't need collection names in the future.
+export const getDatasetCollectionSlugNameMapping = async (nameMap) => {
+  try {
+    const datasets = Object.keys(nameMap).map(dataset => `'${dataset}'`).join(',')
+    const datasetSlugNameTable = await datasette.runQuery(
+      `SELECT dataset, collection FROM dataset WHERE dataset IN (${datasets})`
+    )
+
+    // If no dataset table exists for a dataset, remove from datasetSlugNameMapping
+    const validDatasets = []
+
+    for (const row of datasetSlugNameTable.formattedData) {
+      try {
+        // Test if the dataset table exists by running a simple query, for example datasets such as development-plan-geography exist in dataset, but no table created yet in datasette
+        await datasette.runQuery('SELECT 1 FROM entity LIMIT 1', row.dataset)
+        validDatasets.push(row)
+      } catch (error) {
+        logger.info(`Dataset table '${row.dataset}' does not exist, removing from mapping`, {
+          dataset: row.dataset,
+          errorMessage: error.message,
+          type: types.DataFetch
+        })
+        // Don't add to validDatasets
+      }
+    }
+
+    const datasetMapping = new Map()
+    validDatasets.forEach(row => {
+      datasetMapping.set(row.dataset, row.collection)
+    })
+    return datasetMapping
+  } catch (error) {
+    logger.error(`Failed to fetch dataset=>collection mapping: ${error.message}`)
+    return null
+  }
+}

src/utils/datasetteQueries/fetchDatasetsFromProvisions.js

@@ -0,0 +1,33 @@
+import datasette from '../../services/datasette.js'
+import logger from '../logger.js'
+import config from '../../../config/index.js'
+
+export const fetchDatasetsRequiredForLocalAuthority = async () => {
+  const sql = 'SELECT DISTINCT dataset, provision_reason FROM provision'
+
+  try {
+    const response = await datasette.runQuery(sql)
+
+    const filteredProvisions = response.formattedData.filter(p =>
+      // Use config-driven provision reasons
+      config.provisionReasons.includes(p.provision_reason)
+    )
+
+    let availableDatasets
+    // Test environment currently uses fall back
+    if (config.environment === 'development' || config.environment === 'local') {
+      logger.info('fetchDatasetsForLocalAuthority: Using filtered provisions for available datasets in development/local environment')
+      availableDatasets = filteredProvisions.map(p => p.dataset)
+    } else {
+      logger.info('fetchDatasetsForLocalAuthority: Using hard coded datasets for available datasets')
+      availableDatasets = Object.keys(config.datasetsConfig)
+    }
+
+    return availableDatasets
+  } catch (error) {
+    logger.warn(`fetchDatasetsForLocalAuthority: Error fetching dataset info: ${error.message}`, error)
+    throw error
+  }
+}
+
+export default fetchDatasetsRequiredForLocalAuthority