diff --git a/web/client/api/ArcGIS.js b/web/client/api/ArcGIS.js index 1c1115365c..5bf7999246 100644 --- a/web/client/api/ArcGIS.js +++ b/web/client/api/ArcGIS.js @@ -10,7 +10,11 @@ import axios from '../libs/ajax'; import Proj4js from 'proj4'; import { reprojectBbox } from '../utils/CoordinatesUtils'; import trimEnd from 'lodash/trimEnd'; -import { isFeatureServerUrl, esriGeometryTypeToGeoJSON } from '../utils/ArcGISUtils'; +import { + isFeatureServerUrl, + esriGeometryTypeToGeoJSON, + esriFieldTypeToPrimitive +} from '../utils/ArcGISUtils'; let _cache = {}; @@ -222,3 +226,164 @@ export const getCapabilities = (url, startPosition, maxRecords, text, info) => { return { numberOfRecordsMatched, numberOfRecordsReturned, records }; }); }; + +const buildFeatureLayerUrl = (url, name) => { + const baseUrl = trimEnd(url, '/'); + const layerId = name !== undefined && name !== null ? name : 0; + return `${baseUrl}/${layerId}`; +}; + +let _schemaCache = {}; +let _pendingSchemaFetch = {}; +/** + * Fetch a FeatureService layer's schema + * @param {string} url FeatureService root url + * @param {string|number} name layer id within the service + * @param {object} [opts] + * @param {string} [opts.authSourceId] security source id forwarded to axios + * @param {boolean} [opts.force] bypass cache + * @returns {object} promise resolving with + * - {array} fields - raw ESRI fields metadata + * - {object} properties - sampled `{name: 0|''}` primitives + * - {string} geometryType - GeoJSON geometry type + */ +export const getFeatureLayerSchema = (url, name, opts = {}) => { + const layerUrl = buildFeatureLayerUrl(url, name); + if (!opts.force && _schemaCache[layerUrl]) { + return Promise.resolve(_schemaCache[layerUrl]); + } + if (_pendingSchemaFetch[layerUrl]) { + return _pendingSchemaFetch[layerUrl]; + } + const request = axios.get(layerUrl, { + params: { f: 'json' }, + _msAuthSourceId: opts.authSourceId + }).then(({ data }) => { + const fields = Array.isArray(data?.fields) ? data.fields : []; + // Sample primitive value so getVectorLayerAttributes' isNumber/isString filter accepts it. + const properties = fields.reduce((acc, f) => { + if (!f?.name) return acc; + const primitive = esriFieldTypeToPrimitive(f.type); + if (primitive === 'number') acc[f.name] = 0; + else if (primitive === 'string') acc[f.name] = ''; + return acc; + }, {}); + const result = { + fields, + properties, + geometryType: data?.geometryType + ? esriGeometryTypeToGeoJSON(data.geometryType) + : undefined + }; + _schemaCache[layerUrl] = result; + delete _pendingSchemaFetch[layerUrl]; + return result; + }).catch((e) => { + delete _pendingSchemaFetch[layerUrl]; + throw e; + }); + _pendingSchemaFetch[layerUrl] = request; + return request; +}; + +const DEFAULT_FEATURE_PAGE_SIZE = 1000; + +/** + * Paginated fetch of features from a FeatureService layer as GeoJSON. + * @param {string} url FeatureService root url + * @param {string|number} name layer id within the service + * @param {object} [opts] + * @param {object} [opts.params] extra query params merged into the request + * @param {number} [opts.pageSize] resultRecordCount per page (default 1000) + * @param {number} [opts.maxRecordCount] caps the page size (server-side limit) + * @param {number} [opts.maxFeatures] maximum number of features to return + * @param {string} [opts.authSourceId] security source id forwarded to axios + * @returns {object} promise resolving with a GeoJSON FeatureCollection + * - {string} type - always `FeatureCollection` + * - {array} features - accumulated GeoJSON features + */ +export const fetchFeatureLayerCollection = (url, name, opts = {}) => { + const queryUrl = `${buildFeatureLayerUrl(url, name)}/query`; + const baseParams = { + where: '1=1', + outFields: '*', + outSR: 4326, + f: 'geojson', + ...(opts.params || {}) + }; + // Use server-advertised maxRecordCount as page size (may exceed 1000) + const recordCount = opts.pageSize + || opts.maxRecordCount + || DEFAULT_FEATURE_PAGE_SIZE; + const allFeatures = []; + const seenIds = new Set(); + const fetchPage = (offset) => { + return axios.get(queryUrl, { + params: { + ...baseParams, + resultOffset: offset, + resultRecordCount: recordCount + }, + _msAuthSourceId: opts.authSourceId + }).then((response) => { + const data = response?.data; + const pageFeatures = data?.features || []; + const newFeatures = pageFeatures.filter((f) => { + const id = f.id ?? f.properties?.OBJECTID; + if (id !== null && id !== undefined && seenIds.has(id)) return false; + if (id !== null && id !== undefined) seenIds.add(id); + return true; + }); + if (newFeatures.length) { + allFeatures.push(...newFeatures); + } + const reachedCap = opts.maxFeatures && allFeatures.length >= opts.maxFeatures; + const exceeded = data?.exceededTransferLimit + || data?.properties?.exceededTransferLimit; + if (!reachedCap && exceeded && newFeatures.length > 0) { + return fetchPage(offset + pageFeatures.length); + } + const features = opts.maxFeatures + ? allFeatures.slice(0, opts.maxFeatures) + : allFeatures; + return { type: 'FeatureCollection', features }; + }).catch(() => ({ type: 'FeatureCollection', features: allFeatures })); + }; + return fetchPage(0); +}; + +const DEFAULT_CLASSIFICATION_SAMPLE = 5000; +let _classificationCache = {}; + +/** + * Fetch a feature collection suitable for client-side + * classification of a FeatureService layer + * @param {object} layer layer config + * @param {object} [opts] + * @param {number} [opts.sampleSize] maximum number of features to return + * @param {boolean} [opts.force] force re-fetch + * @returns {object} promise resolving with a GeoJSON FeatureCollection + * - {string} type - always `FeatureCollection` + * - {array} features - fetched features + */ +export const queryFeatureLayerForClassification = (layer, opts = {}) => { + const layerUrl = buildFeatureLayerUrl(layer?.url, layer?.name); + const sampleSize = opts.sampleSize || DEFAULT_CLASSIFICATION_SAMPLE; + const cacheKey = `${layerUrl}::${sampleSize}`; + if (!opts.force && _classificationCache[cacheKey]) { + return _classificationCache[cacheKey]; + } + const request = fetchFeatureLayerCollection(layer?.url, layer?.name, { + authSourceId: layer?.security?.sourceId, + maxRecordCount: layer?.maxRecordCount, + maxFeatures: sampleSize + }).then((collection) => { + // Empty result usually means a rejected request — remove so next call retries. + if (!collection?.features?.length) { + delete _classificationCache[cacheKey]; + } + return collection; + }); + _classificationCache[cacheKey] = request; + return request; +}; diff --git a/web/client/api/__tests__/ArcGIS-test.js b/web/client/api/__tests__/ArcGIS-test.js index 715f5640c8..d489c1b84e 100644 --- a/web/client/api/__tests__/ArcGIS-test.js +++ b/web/client/api/__tests__/ArcGIS-test.js @@ -5,7 +5,13 @@ * This source code is licensed under the BSD-style license found in the * LICENSE file in the root directory of this source tree. */ -import { getCapabilities, getLayerMetadata } from '../ArcGIS'; +import { + getCapabilities, + getLayerMetadata, + getFeatureLayerSchema, + fetchFeatureLayerCollection, + queryFeatureLayerForClassification +} from '../ArcGIS'; import expect from 'expect'; import axios from '../../libs/ajax'; import MockAdapter from 'axios-mock-adapter'; @@ -271,4 +277,252 @@ describe('Test ArcGIS API', () => { .catch(done); }); }); + describe('getFeatureLayerSchema', () => { + beforeEach(done => { + mockAxios = new MockAdapter(axios); + setTimeout(done); + }); + afterEach(done => { + mockAxios.restore(); + setTimeout(done); + }); + it('should fetch schema and map ESRI field types to sample primitives', (done) => { + mockAxios.onGet().reply(() => [200, { + geometryType: 'esriGeometryPolygon', + fields: [ + { name: 'OBJECTID', alias: 'OID', type: 'esriFieldTypeOID' }, + { name: 'pop', type: 'esriFieldTypeInteger' }, + { name: 'name', type: 'esriFieldTypeString' }, + { name: 'shape', type: 'esriFieldTypeGeometry' }, + { name: 'updated', type: 'esriFieldTypeDate' } + ] + }]); + getFeatureLayerSchema('/arcgis/rest/services/Schema1/FeatureServer', 0) + .then((schema) => { + expect(schema.geometryType).toBe('MultiPolygon'); + expect(schema.fields.length).toBe(5); + expect(schema.properties).toEqual({ + OBJECTID: 0, + pop: 0, + name: '' + }); + done(); + }) + .catch(done); + }); + it('should cache schema and dedupe concurrent in-flight requests', (done) => { + let callCount = 0; + mockAxios.onGet().reply(() => { + callCount++; + return [200, { fields: [{ name: 'a', type: 'esriFieldTypeInteger' }] }]; + }); + const url = '/arcgis/rest/services/Schema2/FeatureServer'; + const p1 = getFeatureLayerSchema(url, 0); + const p2 = getFeatureLayerSchema(url, 0); + Promise.all([p1, p2]) + .then(([s1, s2]) => { + expect(callCount).toBe(1); + expect(s1).toBe(s2); + return getFeatureLayerSchema(url, 0); + }) + .then(() => { + expect(callCount).toBe(1); + done(); + }) + .catch(done); + }); + it('should reject and not cache on transport error', (done) => { + // First attempt fails -> rejection should propagate AND cache must stay empty + // so the second attempt re-issues the request + mockAxios.onGet().replyOnce(() => [500, {}]); + mockAxios.onGet().replyOnce(() => [200, { fields: [{ name: 'ok', type: 'esriFieldTypeInteger' }] }]); + getFeatureLayerSchema('/arcgis/rest/services/Schema3/FeatureServer', 0) + .then(() => done(new Error('expected rejection'))) + .catch(() => getFeatureLayerSchema('/arcgis/rest/services/Schema3/FeatureServer', 0)) + .then((schema) => { + expect(schema.fields[0].name).toBe('ok'); + done(); + }) + .catch(done); + }); + it('should default name to 0 when undefined', (done) => { + let requestUrl; + mockAxios.onGet().reply((config) => { + requestUrl = config.url; + return [200, { fields: [] }]; + }); + getFeatureLayerSchema('/arcgis/rest/services/Schema4/FeatureServer/', undefined) + .then(() => { + expect(requestUrl).toBe('/arcgis/rest/services/Schema4/FeatureServer/0'); + done(); + }) + .catch(done); + }); + }); + describe('fetchFeatureLayerCollection', () => { + beforeEach(done => { + mockAxios = new MockAdapter(axios); + setTimeout(done); + }); + afterEach(done => { + mockAxios.restore(); + setTimeout(done); + }); + it('should send canonical query params (where, outFields, outSR, f) plus extras', (done) => { + let receivedParams; + mockAxios.onGet().reply((config) => { + receivedParams = config.params; + return [200, { features: [] }]; + }); + fetchFeatureLayerCollection('/svc/Test/FeatureServer/', 0, { + params: { geometry: '0,0,1,1', geometryType: 'esriGeometryEnvelope' }, + authSourceId: 'auth1' + }).then(() => { + expect(receivedParams.where).toBe('1=1'); + expect(receivedParams.outFields).toBe('*'); + expect(receivedParams.outSR).toBe(4326); + expect(receivedParams.f).toBe('geojson'); + expect(receivedParams.geometry).toBe('0,0,1,1'); + expect(receivedParams.geometryType).toBe('esriGeometryEnvelope'); + done(); + }).catch(done); + }); + it('should follow exceededTransferLimit pagination and de-dup by OBJECTID', (done) => { + const pages = [ + { + exceededTransferLimit: true, + features: [ + { id: 1, properties: { OBJECTID: 1, v: 'a' } }, + { id: 2, properties: { OBJECTID: 2, v: 'b' } } + ] + }, + { + exceededTransferLimit: true, + features: [ + // duplicate of page 1; should be dropped + { id: 2, properties: { OBJECTID: 2, v: 'b' } }, + { id: 3, properties: { OBJECTID: 3, v: 'c' } } + ] + }, + { features: [] } + ]; + let pageIdx = 0; + mockAxios.onGet().reply(() => [200, pages[pageIdx++]]); + fetchFeatureLayerCollection('/svc/Pag/FeatureServer/', 0) + .then((collection) => { + expect(collection.type).toBe('FeatureCollection'); + expect(collection.features.map((f) => f.properties.OBJECTID)).toEqual([1, 2, 3]); + done(); + }) + .catch(done); + }); + it('should stop at maxFeatures cap and slice the result', (done) => { + mockAxios.onGet().reply(() => [200, { + exceededTransferLimit: true, + features: [ + { id: 1, properties: { OBJECTID: 1 } }, + { id: 2, properties: { OBJECTID: 2 } }, + { id: 3, properties: { OBJECTID: 3 } } + ] + }]); + fetchFeatureLayerCollection('/svc/Cap/FeatureServer/', 0, { maxFeatures: 2 }) + .then((collection) => { + expect(collection.features.length).toBe(2); + expect(collection.features[0].properties.OBJECTID).toBe(1); + expect(collection.features[1].properties.OBJECTID).toBe(2); + done(); + }) + .catch(done); + }); + it('should swallow request errors and resolve with what was accumulated', (done) => { + let attempt = 0; + mockAxios.onGet().reply(() => { + attempt++; + if (attempt === 1) { + return [200, { + exceededTransferLimit: true, + features: [{ id: 1, properties: { OBJECTID: 1 } }] + }]; + } + return [500, {}]; + }); + fetchFeatureLayerCollection('/svc/Err/FeatureServer/', 0) + .then((collection) => { + expect(collection.type).toBe('FeatureCollection'); + expect(collection.features.length).toBe(1); + done(); + }) + .catch(done); + }); + it('should use server maxRecordCount as page size (not silently capped at 1000)', (done) => { + let receivedRecordCount; + mockAxios.onGet().reply((config) => { + receivedRecordCount = config.params.resultRecordCount; + return [200, { features: [] }]; + }); + fetchFeatureLayerCollection('/svc/Big/FeatureServer/', 0, { maxRecordCount: 2000 }) + .then(() => { + expect(receivedRecordCount).toBe(2000); + done(); + }) + .catch(done); + }); + }); + describe('queryFeatureLayerForClassification', () => { + beforeEach(done => { + mockAxios = new MockAdapter(axios); + setTimeout(done); + }); + afterEach(done => { + mockAxios.restore(); + setTimeout(done); + }); + it('should cap at sampleSize and cache the result for repeat calls', (done) => { + let callCount = 0; + mockAxios.onGet().reply(() => { + callCount++; + return [200, { + features: Array.from({ length: 3 }, (_, i) => ({ + id: i + 1, + properties: { OBJECTID: i + 1, v: i } + })) + }]; + }); + const layer = { + url: '/svc/Cls1/FeatureServer/', + name: 0 + }; + queryFeatureLayerForClassification(layer, { sampleSize: 2 }) + .then((c1) => { + expect(c1.features.length).toBe(2); + return queryFeatureLayerForClassification(layer, { sampleSize: 2 }); + }) + .then((c2) => { + expect(callCount).toBe(1); + expect(c2.features.length).toBe(2); + done(); + }) + .catch(done); + }); + it('should invalidate cache when collection comes back empty', (done) => { + let callCount = 0; + mockAxios.onGet().reply(() => { + callCount++; + if (callCount === 1) return [200, { features: [] }]; + return [200, { features: [{ id: 1, properties: { OBJECTID: 1 } }] }]; + }); + const layer = { url: '/svc/Cls2/FeatureServer/', name: 0 }; + queryFeatureLayerForClassification(layer) + .then((c1) => { + expect(c1.features.length).toBe(0); + return queryFeatureLayerForClassification(layer); + }) + .then((c2) => { + expect(callCount).toBe(2); + expect(c2.features.length).toBe(1); + done(); + }) + .catch(done); + }); + }); }); diff --git a/web/client/api/catalog/ArcGIS.js b/web/client/api/catalog/ArcGIS.js index 1dbfc89894..147d7ec88f 100644 --- a/web/client/api/catalog/ArcGIS.js +++ b/web/client/api/catalog/ArcGIS.js @@ -40,6 +40,7 @@ const recordToLayer = (record, { layerBaseConfig }) => { ? { ...(record.geometryType && { geometryType: record.geometryType }), ...(record.maxRecordCount && { maxRecordCount: record.maxRecordCount }), + ...(record.fields && { fields: record.fields }), strategy: 'tile' } : { diff --git a/web/client/api/catalog/__tests__/ArcGIS-test.js b/web/client/api/catalog/__tests__/ArcGIS-test.js index 1063c6eb07..a187bea96e 100644 --- a/web/client/api/catalog/__tests__/ArcGIS-test.js +++ b/web/client/api/catalog/__tests__/ArcGIS-test.js @@ -150,4 +150,39 @@ describe('Test ArcGIS Catalog API', () => { } done(); }); + it('should forward fields from FeatureServer record to layer config', (done) => { + const testRecord = { + name: 0, + title: 'Layer with fields', + url: 'https://test.arcgis.com/rest/services/Test/FeatureServer', + geometryType: 'MultiPolygon', + fields: [ + { name: 'OBJECTID', alias: 'OID', type: 'esriFieldTypeOID' }, + { name: 'pop', type: 'esriFieldTypeInteger' } + ] + }; + try { + const layer = getLayerFromRecord(testRecord, { layerBaseConfig: {} }); + expect(layer.type).toBe('arcgis-feature'); + expect(layer.fields).toEqual(testRecord.fields); + } catch (e) { + return done(e); + } + return done(); + }); + it('should not add fields key when record has no fields', (done) => { + const testRecord = { + name: 0, + title: 'Layer without fields', + url: 'https://test.arcgis.com/rest/services/Test/FeatureServer', + geometryType: 'Point' + }; + try { + const layer = getLayerFromRecord(testRecord, { layerBaseConfig: {} }); + expect('fields' in layer).toBe(false); + } catch (e) { + return done(e); + } + return done(); + }); }); diff --git a/web/client/components/map/cesium/plugins/ArcGISFeatureLayer.js b/web/client/components/map/cesium/plugins/ArcGISFeatureLayer.js index cb06dcfb18..b28bed6999 100644 --- a/web/client/components/map/cesium/plugins/ArcGISFeatureLayer.js +++ b/web/client/components/map/cesium/plugins/ArcGISFeatureLayer.js @@ -8,7 +8,6 @@ import * as Cesium from 'cesium'; import isEqual from 'lodash/isEqual'; -import trimEnd from 'lodash/trimEnd'; import Layers from '../../../../utils/cesium/Layers'; import { @@ -18,50 +17,7 @@ import { import { applyDefaultStyleToVectorLayer } from '../../../../utils/StyleUtils'; import GeoJSONStyledFeatures from '../../../../utils/cesium/GeoJSONStyledFeatures'; import TiledBillboardCollection from '../../../../utils/cesium/TiledBillboardCollection'; -import axios from '../../../../libs/ajax'; - -const buildQueryUrl = (options) => { - const baseUrl = trimEnd(options.url, '/'); - const layerId = options.name !== undefined ? options.name : '0'; - return `${baseUrl}/${layerId}/query`; -}; - -const DEFAULT_PAGE_SIZE = 1000; - -const fetchPaginatedFeatures = (url, baseParams, authSourceId, pageSize) => { - const recordCount = pageSize || DEFAULT_PAGE_SIZE; - const allFeatures = []; - const seenIds = new Set(); - const fetchPage = (offset) => { - return axios.get(url, { - params: { ...baseParams, resultOffset: offset, resultRecordCount: recordCount }, - _msAuthSourceId: authSourceId - }).then(({ data }) => { - const newFeatures = (data?.features || []).filter(f => { - const id = f.id ?? f.properties?.OBJECTID; - if (id !== null && id !== undefined && seenIds.has(id)) return false; - if (id !== null && id !== undefined) seenIds.add(id); - return true; - }); - if (newFeatures.length) { - allFeatures.push(...newFeatures); - } - const exceeded = data?.exceededTransferLimit - || data?.properties?.exceededTransferLimit; - if (exceeded && newFeatures.length > 0) { - return fetchPage(offset + (data?.features?.length || 0)); - } - return { - type: 'FeatureCollection', - features: allFeatures - }; - }).catch(() => ({ - type: 'FeatureCollection', - features: allFeatures - })); - }; - return fetchPage(0); -}; +import { fetchFeatureLayerCollection } from '../../../../api/ArcGIS'; const getEffectiveStrategy = (options) => options?.strategy || 'tile'; @@ -69,28 +25,26 @@ const isPointGeometry = (options) => !options?.geometryType || ['Point', 'MultiP const createLoader = (options) => { const strategy = getEffectiveStrategy(options); - const baseParams = { - where: '1=1', - outFields: '*', - outSR: 4326, - f: 'geojson' + const callOptions = { + authSourceId: options.security?.sourceId, + maxRecordCount: options.maxRecordCount }; - if (strategy === 'bbox' || strategy === 'tile') { return (extent) => { const [xmin, ymin, xmax, ymax] = extent; - return fetchPaginatedFeatures(buildQueryUrl(options), { - ...baseParams, - geometry: `${xmin},${ymin},${xmax},${ymax}`, - geometryType: 'esriGeometryEnvelope', - spatialRel: 'esriSpatialRelIntersects', - inSR: 4326 - }, options.security?.sourceId, options.maxRecordCount).then((data) => ({ data })); + return fetchFeatureLayerCollection(options.url, options.name, { + ...callOptions, + params: { + geometry: `${xmin},${ymin},${xmax},${ymax}`, + geometryType: 'esriGeometryEnvelope', + spatialRel: 'esriSpatialRelIntersects', + inSR: 4326 + } + }).then((data) => ({ data })); }; } - return () => fetchPaginatedFeatures( - buildQueryUrl(options), baseParams, options.security?.sourceId, options.maxRecordCount - ).then((data) => ({ data })); + return () => fetchFeatureLayerCollection(options.url, options.name, callOptions) + .then((data) => ({ data })); }; const applyStyle = (styledFeatures, options, features) => { diff --git a/web/client/components/map/openlayers/plugins/ArcGISFeatureLayer.js b/web/client/components/map/openlayers/plugins/ArcGISFeatureLayer.js index f243e7f751..98df9c737a 100644 --- a/web/client/components/map/openlayers/plugins/ArcGISFeatureLayer.js +++ b/web/client/components/map/openlayers/plugins/ArcGISFeatureLayer.js @@ -7,7 +7,6 @@ */ import isEqual from 'lodash/isEqual'; -import trimEnd from 'lodash/trimEnd'; import VectorSource from 'ol/source/Vector'; import VectorLayer from 'ol/layer/Vector'; @@ -17,57 +16,9 @@ import GeoJSON from 'ol/format/GeoJSON'; import { getStyle } from '../VectorStyle'; import Layers from '../../../../utils/openlayers/Layers'; -import axios from '../../../../libs/ajax'; import { reprojectBbox } from '../../../../utils/CoordinatesUtils'; import { applyDefaultStyleToVectorLayer } from '../../../../utils/StyleUtils'; - -const buildQueryUrl = (options) => { - const baseUrl = trimEnd(options.url, '/'); - const layerId = options.name !== undefined ? options.name : '0'; - return `${baseUrl}/${layerId}/query`; -}; - -const DEFAULT_PAGE_SIZE = 1000; - -const fetchPaginatedFeatures = (url, baseParams, authSourceId, pageSize) => { - const recordCount = pageSize || DEFAULT_PAGE_SIZE; - const allFeatures = []; - const seenIds = new Set(); - const fetchPage = (offset) => { - return axios.get(url, { - params: { - ...baseParams, - resultOffset: offset, - resultRecordCount: recordCount - }, - _msAuthSourceId: authSourceId - }).then(response => { - const data = response?.data; - const newFeatures = (data?.features || []).filter(f => { - const id = f.id ?? f.properties?.OBJECTID; - if (id !== null && id !== undefined && seenIds.has(id)) return false; - if (id !== null && id !== undefined) seenIds.add(id); - return true; - }); - if (newFeatures.length) { - allFeatures.push(...newFeatures); - } - const exceeded = data?.exceededTransferLimit - || data?.properties?.exceededTransferLimit; - if (exceeded && newFeatures.length > 0) { - return fetchPage(offset + (data?.features?.length || 0)); - } - return { - type: 'FeatureCollection', - features: allFeatures - }; - }).catch(() => ({ - type: 'FeatureCollection', - features: allFeatures - })); - }; - return fetchPage(0); -}; +import { fetchFeatureLayerCollection } from '../../../../api/ArcGIS'; const getStrategy = (options) => { if (options.strategy === 'all') { @@ -83,27 +34,20 @@ const getEffectiveStrategy = (options) => options?.strategy || 'tile'; const createLoader = (source, options) => (extent, resolution, projection, success, failure) => { const projCode = projection.getCode(); - const params = { - where: '1=1', - outFields: '*', - outSR: 4326, - f: 'geojson' - }; const strategy = getEffectiveStrategy(options); + const params = {}; if (strategy === 'bbox' || strategy === 'tile') { - const bbox4326 = reprojectBbox(extent, projCode, 'EPSG:4326'); - const [xmin, ymin, xmax, ymax] = bbox4326; + const [xmin, ymin, xmax, ymax] = reprojectBbox(extent, projCode, 'EPSG:4326'); params.geometry = `${xmin},${ymin},${xmax},${ymax}`; params.geometryType = 'esriGeometryEnvelope'; params.spatialRel = 'esriSpatialRelIntersects'; params.inSR = 4326; } - fetchPaginatedFeatures( - buildQueryUrl(options), + fetchFeatureLayerCollection(options.url, options.name, { params, - options.security?.sourceId, - options.maxRecordCount - ) + authSourceId: options.security?.sourceId, + maxRecordCount: options.maxRecordCount + }) .then((collection) => { const features = source.getFormat().readFeatures(collection, { dataProjection: 'EPSG:4326', diff --git a/web/client/plugins/styleeditor/VectorStyleEditor.jsx b/web/client/plugins/styleeditor/VectorStyleEditor.jsx index fe29ed4e68..a5cfb9c800 100644 --- a/web/client/plugins/styleeditor/VectorStyleEditor.jsx +++ b/web/client/plugins/styleeditor/VectorStyleEditor.jsx @@ -27,6 +27,10 @@ import { import { getCapabilities } from '../../api/ThreeDTiles'; import { describeFeatureType } from '../../api/WFS'; import { classificationVector } from '../../api/StyleEditor'; +import { + getFeatureLayerSchema, + queryFeatureLayerForClassification +} from '../../api/ArcGIS'; import SLDService from '../../api/SLDService'; import { classifyGeoJSON, availableMethods } from '../../api/GeoJSONClassification'; import { getLayerJSONFeature } from '../../observables/wfs'; @@ -96,10 +100,16 @@ const capabilitiesRequest = { return featureProps; }) : Promise.resolve({}), - 'arcgis-feature': (layer) => Promise.resolve({ + 'arcgis-feature': (layer) => getFeatureLayerSchema(layer.url, layer.name, { + authSourceId: layer?.security?.sourceId + }).then(({ properties, geometryType, fields }) => ({ + properties, + geometryType: geometryType || layer.geometryType, + fields + })).catch(() => ({ geometryType: layer.geometryType, properties: {} - }) + })) }; function VectorStyleEditor({ @@ -193,7 +203,7 @@ function VectorStyleEditor({ (request ? request(layer) : Promise.resolve(layer)) - .then(({ properties, format, geometryType } = {}) => { + .then(({ properties, format, geometryType, fields } = {}) => { const newLayer = { ...layer, properties: { @@ -201,7 +211,8 @@ function VectorStyleEditor({ ...layer.properties }, format: format ? format : layer.format, - geometryType: geometryType ? geometryType : layer.geometryType + geometryType: geometryType ? geometryType : layer.geometryType, + fields: fields || layer.fields }; return newLayer; }) @@ -210,6 +221,7 @@ function VectorStyleEditor({ properties, format, geometryType, + fields, style: updatedStyle } = {}) => { if (isMounted.current) { @@ -220,6 +232,7 @@ function VectorStyleEditor({ properties, format, geometryType, + ...(fields !== undefined && { fields }), style: newStyle }); setLoading(false); @@ -253,7 +266,11 @@ function VectorStyleEditor({ }); } if (layer.type === 'arcgis-feature') { - return Promise.resolve({ type: 'FeatureCollection', features: layer.features || [] }); + return queryFeatureLayerForClassification(layer) + .then((collection) => { + geojson.current = collection; + return collection; + }); } return Promise.resolve({ type: 'FeatureCollection', features: [] }); } diff --git a/web/client/utils/ArcGISUtils.js b/web/client/utils/ArcGISUtils.js index 042ed4ec0d..f417df737c 100644 --- a/web/client/utils/ArcGISUtils.js +++ b/web/client/utils/ArcGISUtils.js @@ -44,6 +44,51 @@ const ESRI_GEOMETRY_TYPE_MAP = { */ export const esriGeometryTypeToGeoJSON = (esriType) => ESRI_GEOMETRY_TYPE_MAP[esriType] || 'GeometryCollection'; +const ESRI_NUMERIC_FIELD_TYPES = [ + 'esriFieldTypeOID', + 'esriFieldTypeSmallInteger', + 'esriFieldTypeInteger', + 'esriFieldTypeBigInteger', + 'esriFieldTypeSingle', + 'esriFieldTypeDouble' +]; +const ESRI_STRING_FIELD_TYPES = [ + 'esriFieldTypeString', + 'esriFieldTypeGUID', + 'esriFieldTypeGlobalID', + 'esriFieldTypeXML' +]; + +/** + * Map an ESRI field type to primitive type + * @param {string} esriFieldType ESRI field type + * @return {'number'|'string'|null} + */ +export const esriFieldTypeToPrimitive = (esriFieldType) => { + if (ESRI_NUMERIC_FIELD_TYPES.includes(esriFieldType)) { + return 'number'; + } + if (ESRI_STRING_FIELD_TYPES.includes(esriFieldType)) { + return 'string'; + } + return null; +}; + +/** + * Build an attribute list from an ESRI FeatureService fields. + * @param {Object[]} fields ESRI fields metadata + * @return {Object[]} attributes with `attribute`, `label`, and `type` + */ +export const esriFieldsToAttributes = (fields = []) => { + return fields + .map((f) => { + const type = esriFieldTypeToPrimitive(f?.type); + if (!type) return null; + return { attribute: f.name, label: f.alias || f.name, type }; + }) + .filter(Boolean); +}; + /** * Return all the sub layers ids given a layer id and layers structure * @param {string|number} id identifier of the starting layer diff --git a/web/client/utils/StyleEditorUtils.js b/web/client/utils/StyleEditorUtils.js index d2abb10952..e95b2abeaa 100644 --- a/web/client/utils/StyleEditorUtils.js +++ b/web/client/utils/StyleEditorUtils.js @@ -30,6 +30,7 @@ import url from 'url'; import { baseTemplates, customTemplates } from './styleeditor/stylesTemplates'; import { getStyleParser } from './VectorStyleUtils'; import { applyDefaultToLocalizedString } from '../components/I18N/LocalizedString'; +import { esriFieldsToAttributes } from './ArcGISUtils'; import xml2js from 'xml2js'; const xmlBuilder = new xml2js.Builder(); @@ -627,6 +628,9 @@ export function getAttributes(properties, fields = []) { * @return {array|null} returns an array of attributes */ export function getVectorLayerAttributes(layer) { + if (layer?.type === 'arcgis-feature' && Array.isArray(layer?.fields) && layer.fields.length) { + return esriFieldsToAttributes(layer.fields); + } if (!layer?.properties) { return null; } diff --git a/web/client/utils/__tests__/ArcGISUtils-test.js b/web/client/utils/__tests__/ArcGISUtils-test.js index 123f7daf48..a47b3fbc50 100644 --- a/web/client/utils/__tests__/ArcGISUtils-test.js +++ b/web/client/utils/__tests__/ArcGISUtils-test.js @@ -14,7 +14,9 @@ import { getLayerIds, getQueryLayerIds, esriToGeoJSONFeature, - esriGeometryTypeToGeoJSON + esriGeometryTypeToGeoJSON, + esriFieldTypeToPrimitive, + esriFieldsToAttributes } from '../ArcGISUtils'; const layers = [ @@ -117,6 +119,43 @@ describe('ArcGISUtils', () => { expect(getQueryLayerIds(6, layers)).toEqual(['6']); expect(getQueryLayerIds(7, layers)).toEqual(['8', '9']); }); + it('esriFieldTypeToPrimitive', () => { + expect(esriFieldTypeToPrimitive('esriFieldTypeOID')).toBe('number'); + expect(esriFieldTypeToPrimitive('esriFieldTypeSmallInteger')).toBe('number'); + expect(esriFieldTypeToPrimitive('esriFieldTypeInteger')).toBe('number'); + expect(esriFieldTypeToPrimitive('esriFieldTypeBigInteger')).toBe('number'); + expect(esriFieldTypeToPrimitive('esriFieldTypeSingle')).toBe('number'); + expect(esriFieldTypeToPrimitive('esriFieldTypeDouble')).toBe('number'); + expect(esriFieldTypeToPrimitive('esriFieldTypeString')).toBe('string'); + expect(esriFieldTypeToPrimitive('esriFieldTypeGUID')).toBe('string'); + expect(esriFieldTypeToPrimitive('esriFieldTypeGlobalID')).toBe('string'); + expect(esriFieldTypeToPrimitive('esriFieldTypeXML')).toBe('string'); + // Geometry/Date/Blob/Raster/unknown - null + expect(esriFieldTypeToPrimitive('esriFieldTypeGeometry')).toBe(null); + expect(esriFieldTypeToPrimitive('esriFieldTypeDate')).toBe(null); + expect(esriFieldTypeToPrimitive('esriFieldTypeBlob')).toBe(null); + expect(esriFieldTypeToPrimitive('esriFieldTypeRaster')).toBe(null); + expect(esriFieldTypeToPrimitive(undefined)).toBe(null); + expect(esriFieldTypeToPrimitive('undefined')).toBe(null); + }); + it('esriFieldsToAttributes', () => { + const fields = [ + { name: 'OBJECTID', alias: 'OID', type: 'esriFieldTypeOID' }, + { name: 'name', alias: 'Site name', type: 'esriFieldTypeString' }, + { name: 'pop', type: 'esriFieldTypeInteger' }, + // Geometry should be filtered out + { name: 'shape', type: 'esriFieldTypeGeometry' }, + // Date should be filtered out + { name: 'date', type: 'esriFieldTypeDate' } + ]; + expect(esriFieldsToAttributes(fields)).toEqual([ + { attribute: 'OBJECTID', label: 'OID', type: 'number' }, + { attribute: 'name', label: 'Site name', type: 'string' }, + { attribute: 'pop', label: 'pop', type: 'number' } + ]); + expect(esriFieldsToAttributes()).toEqual([]); + expect(esriFieldsToAttributes([])).toEqual([]); + }); it('esriToGeoJSONFeature', () => { expect(esriToGeoJSONFeature()) .toEqual({ type: 'Feature', properties: {}, geometry: null }); diff --git a/web/client/utils/__tests__/StyleEditorUtils-test.js b/web/client/utils/__tests__/StyleEditorUtils-test.js index cc2663c54f..366fce1f68 100644 --- a/web/client/utils/__tests__/StyleEditorUtils-test.js +++ b/web/client/utils/__tests__/StyleEditorUtils-test.js @@ -1177,6 +1177,41 @@ describe('StyleEditorUtils test', () => { } ]); }); + it('should return attributes for arcgis-feature using ESRI fields when present', () => { + const layer = { + type: 'arcgis-feature', + fields: [ + { name: 'OBJECTID', alias: 'OID', type: 'esriFieldTypeOID' }, + { name: 'name', alias: 'Site', type: 'esriFieldTypeString' }, + { name: 'pop', type: 'esriFieldTypeInteger' }, + // Geometry/Date filtered out + { name: 'shape', type: 'esriFieldTypeGeometry' }, + { name: 'updated', type: 'esriFieldTypeDate' } + ] + }; + expect(getVectorLayerAttributes(layer)).toEqual([ + { attribute: 'OBJECTID', label: 'OID', type: 'number' }, + { attribute: 'name', label: 'Site', type: 'string' }, + { attribute: 'pop', label: 'pop', type: 'number' } + ]); + }); + it('should return attributes for arcgis-feature falling back to properties when no fields present', () => { + const layer = { + type: 'arcgis-feature', + properties: { + name: 'Site A', + pop: 1234 + } + }; + expect(getVectorLayerAttributes(layer)).toEqual([ + { attribute: 'name', label: 'name', type: 'string' }, + { attribute: 'pop', label: 'pop', type: 'number' } + ]); + }); + it('should return null for arcgis-feature with neither fields nor properties', () => { + expect(getVectorLayerAttributes({ type: 'arcgis-feature' })).toBe(null); + expect(getVectorLayerAttributes({ type: 'arcgis-feature', fields: [] })).toBe(null); + }); it('should return the geometry type with getVectorLayerGeometryType function given a 3d tiles layer config with format', () => { expect(getVectorLayerGeometryType({ type: '3dtiles', format: 'pnts' })).toBe('pointcloud'); expect(getVectorLayerGeometryType({ type: '3dtiles' })).toBe('polyhedron'); diff --git a/web/client/utils/cesium/TiledBillboardCollection.js b/web/client/utils/cesium/TiledBillboardCollection.js index c9374f6067..a8ca87661f 100644 --- a/web/client/utils/cesium/TiledBillboardCollection.js +++ b/web/client/utils/cesium/TiledBillboardCollection.js @@ -535,6 +535,7 @@ TiledBillboardCollection.prototype.setOpacity = function(opacity) { TiledBillboardCollection.prototype.setStyleFunction = function(newStyle) { // Update the stored style this._style = newStyle; + this._styleOptions = { ...this._styleOptions, style: newStyle }; // Update existing billboards with new style Object.keys(this._tileCache).forEach(tileId => { diff --git a/web/client/utils/cesium/__tests__/TiledBillboardCollection-test.js b/web/client/utils/cesium/__tests__/TiledBillboardCollection-test.js index 62261f0896..53f955c1ce 100644 --- a/web/client/utils/cesium/__tests__/TiledBillboardCollection-test.js +++ b/web/client/utils/cesium/__tests__/TiledBillboardCollection-test.js @@ -321,6 +321,27 @@ describe('TiledBillboardCollection', () => { expect(collection._style).toEqual(newStyle); }); + it('should sync style on newly loaded tiles', () => { + const initialStyle = { symbolizers: [{ kind: 'Fill', color: '#ff0000' }] }; + const styleOptions = { geometryType: 'MultiPolygon', style: initialStyle }; + const collection = new TiledBillboardCollection({ + map: mockMap, + tileType: 'feature', + style: initialStyle, + styleOptions + }); + expect(collection._styleOptions.style).toEqual(initialStyle); + + const newStyle = { symbolizers: [{ kind: 'Fill', color: '#00ff00' }] }; + collection.setStyleFunction(newStyle); + expect(collection._style).toEqual(newStyle); + expect(collection._styleOptions.style).toEqual(newStyle); + // other styleOptions keys must be preserved + expect(collection._styleOptions.geometryType).toBe('MultiPolygon'); + // caller's original styleOptions object must NOT be mutated + expect(styleOptions.style).toEqual(initialStyle); + }); + it('should clean up on destroy', () => { const collection = new TiledBillboardCollection({ map: mockMap,