feat: add limit/offset pagination to all array-returning graphql queries

src/db/AreaTypes.ts

@@ -262,4 +262,6 @@ export enum OperationType {
 
 export interface BulkAreasGQLQueryInput {
   ancestors: string[]
+  limit?: number
+  offset?: number
 }

src/db/ChangeLogType.ts

@@ -71,12 +71,18 @@ export interface GetHistoryInputFilterType {
   userUuid: string
   fromDate: Date
   toDate: Date
+  limit?: number
+  offset?: number
 }
 
 export interface GetAreaHistoryInputFilterType {
   areaId: string
+  limit?: number
+  offset?: number
 }
 
 export interface GetOrganizationHistoryInputFilterType {
   orgId: MUUID
+  limit?: number
+  offset?: number
 }

src/db/TickTypes.ts

@@ -111,4 +111,13 @@ export interface TickEditFilterType {
 export interface TickUserSelectors {
   userId?: MUUID
   username?: string
+  limit?: number
+  offset?: number
+}
+
+export interface TickByClimbSelectors {
+  climbId: string
+  userId?: string
+  limit?: number
+  offset?: number
 }

src/graphql/area/AreaQueries.ts

@@ -15,8 +15,12 @@ const AreaQueries = {
 
   bulkAreas: async (_: any, params, { dataSources }: GQLContext): Promise<AreaType[]> => {
     const { areas } = dataSources
-    const { ancestors } = params as BulkAreasGQLQueryInput
-    return await areas.bulkDownloadAreas(ancestors)
+    const { ancestors, limit, offset } = params as BulkAreasGQLQueryInput
+    const DEFAULT_LIMIT = 500
+    const MAX_LIMIT = 2000
+    const safeLimit = Math.min(limit ?? DEFAULT_LIMIT, MAX_LIMIT)
+    const safeOffset = offset ?? 0
+    return await areas.bulkDownloadAreas(ancestors, safeLimit, safeOffset)
   }
 }
 

src/graphql/history/HistoryQueries.ts

@@ -7,29 +7,38 @@ import {
 } from '../../db/ChangeLogType.js'
 import { GQLContext } from '../../types.js'
 
+const DEFAULT_LIMIT = 50
+const MAX_LIMIT = 500
+
 const HistoryQueries = {
   getChangeHistory: async (_, { filter }, { dataSources }: GQLContext): Promise<any> => {
     const { history } = dataSources
-    const { uuidList }: GetHistoryInputFilterType = filter ?? {}
+    const { uuidList, limit, offset }: GetHistoryInputFilterType = filter ?? {}
     // Note: userUuid, fromDate, toDate filters don't currently work.
     // Note: though we pull uuidList, we don't use it either.
 
     // Convert array of uuid in string to UUID[]
     const muidList = uuidList?.map(entry => muid.from(entry)) ?? []
-    return await history.getChangeSets(muidList)
+    const safeLimit = Math.min(limit ?? DEFAULT_LIMIT, MAX_LIMIT)
+    const safeOffset = offset ?? 0
+    return await history.getChangeSets(muidList, safeLimit, safeOffset)
   },
 
   getAreaHistory: async (_, { filter }, { dataSources }: GQLContext): Promise<any> => {
     const { history } = dataSources
-    const { areaId }: GetAreaHistoryInputFilterType = filter ?? {}
-    const id = muid.from(areaId)
-    return await history.getAreaChangeSets(id)
+    const { areaId, limit, offset }: GetAreaHistoryInputFilterType = filter ?? {}
+    const id = areaId != null ? muid.from(areaId) : undefined
+    const safeLimit = Math.min(limit ?? DEFAULT_LIMIT, MAX_LIMIT)
+    const safeOffset = offset ?? 0
+    return await history.getAreaChangeSets(id, safeLimit, safeOffset)
   },
 
   getOrganizationHistory: async (_, { filter }, { dataSources }: GQLContext): Promise<any> => {
     const { history } = dataSources
-    const { orgId }: GetOrganizationHistoryInputFilterType = filter ?? {}
-    return await history.getOrganizationChangeSets(orgId)
+    const { orgId, limit, offset }: GetOrganizationHistoryInputFilterType = filter ?? {}
+    const safeLimit = Math.min(limit ?? DEFAULT_LIMIT, MAX_LIMIT)
+    const safeOffset = offset ?? 0
+    return await history.getOrganizationChangeSets(orgId, safeLimit, safeOffset)
   }
 }
 

src/graphql/organization/OrganizationQueries.ts

@@ -15,15 +15,17 @@ const OrganizationQueries = {
 
   organizations: async (
     _,
-    { filter, sort, limit = 40 }: { filter?: OrganizationGQLFilter, sort?: Sort, limit?: number },
+    { filter, sort, limit = 40, offset = 0 }: { filter?: OrganizationGQLFilter, sort?: Sort, limit?: number, offset?: number },
     { dataSources }: GQLContext
   ) => {
     const { organizations }: { organizations: OrganizationDataSource } = dataSources
+    const MAX_LIMIT = 500
+    const safeLimit = Math.min(limit, MAX_LIMIT)
     const filtered = await organizations.findOrganizationsByFilter(filter)
     if (sort != null) {
-      return await filtered.collation({ locale: 'en' }).sort(sort).limit(limit).toArray()
+      return await filtered.collation({ locale: 'en' }).sort(sort).skip(offset).limit(safeLimit).toArray()
     } else {
-      return await filtered.limit(limit).toArray()
+      return await filtered.skip(offset).limit(safeLimit).toArray()
     }
   }
 }

src/graphql/resolvers.ts

@@ -75,12 +75,16 @@ const resolvers = {
 
     areas: async (
       _,
-      { filter, sort }: { filter?: GQLFilter, sort?: Sort },
+      { filter, sort, limit, offset }: { filter?: GQLFilter, sort?: Sort, limit?: number, offset?: number },
       { dataSources }: GQLContext
     ) => {
       const { areas } = dataSources
+      const DEFAULT_LIMIT = 50
+      const MAX_LIMIT = 500
+      const safeLimit = Math.min(limit ?? DEFAULT_LIMIT, MAX_LIMIT)
+      const safeOffset = offset ?? 0
       const filtered = await areas.findAreasByFilter(filter)
-      return filtered.collation({ locale: 'en' }).sort(sort).toArray()
+      return filtered.collation({ locale: 'en' }).sort(sort).skip(safeOffset).limit(safeLimit).toArray()
     },
 
     area: async (_: any,

src/graphql/schema/Area.gql

@@ -1,12 +1,12 @@
 type Query {
   area(uuid: ID): Area
-  areas(filter: Filter, sort: Sort): [Area]
+  areas(filter: Filter, sort: Sort, limit: Int, offset: Int): [Area]
 
   """
   Bulk download an area and its children starting from ancestors paths (inclusive).
   To keep payload at a reasonable size ancestors must have at least 2 elements.
   """
-  bulkAreas(ancestors: [String!]!): [Area]
+  bulkAreas(ancestors: [String!]!, limit: Int, offset: Int): [Area]
 
   stats: Stats
   cragsNear(

src/graphql/schema/History.gql

@@ -3,14 +3,20 @@ input AllHistoryFilter {
   userUuid: ID
   fromDate: Date
   toDate: Date
+  limit: Int
+  offset: Int
 }
 
 input AreaHistoryFilter {
   areaId: ID
+  limit: Int
+  offset: Int
 }
 
 input OrganizationHistoryFilter {
   orgId: MUUID
+  limit: Int
+  offset: Int
 }
 
 type UpdateDescription {

src/graphql/schema/Organization.gql

@@ -1,6 +1,6 @@
 type Query {
   organization(muuid: MUUID): Organization
-  organizations(filter: OrgFilter, sort: OrgSort, limit: Int): [Organization]
+  organizations(filter: OrgFilter, sort: OrgSort, limit: Int, offset: Int): [Organization]
 }
 
 "A climbing area, wall or crag"

src/graphql/schema/Tick.gql

@@ -2,12 +2,12 @@ type Query {
   """
   Gets all of the users current ticks by their Auth-0 userId or username
   """
-  userTicks(userId: MUUID, username: String): [TickType]
+  userTicks(userId: MUUID, username: String, limit: Int, offset: Int): [TickType]
   """
   Gets all of the users current ticks for a specific climb by their
   Auth-0 userId and Open-Beta ClimbId
   """
-  userTicksByClimbId(userId: String, climbId: String): [TickType]
+  userTicksByClimbId(userId: String, climbId: String, limit: Int, offset: Int): [TickType]
 }
 
 type Mutation {

src/graphql/tick/TickQueries.ts

@@ -1,15 +1,23 @@
-import { TickType, TickUserSelectors } from '../../db/TickTypes'
+import { TickByClimbSelectors, TickType, TickUserSelectors } from '../../db/TickTypes'
 import type TickDataSource from '../../model/TickDataSource'
 
+const DEFAULT_LIMIT = 50
+const MAX_LIMIT = 500
+
 const TickQueries = {
   userTicks: async (_, input: TickUserSelectors, { dataSources }): Promise<TickType[] | null> => {
     const { ticks }: { ticks: TickDataSource } = dataSources
-    return await ticks.ticksByUser(input)
+    const { limit, offset, ...selectors } = input
+    const safeLimit = Math.min(limit ?? DEFAULT_LIMIT, MAX_LIMIT)
+    const safeOffset = offset ?? 0
+    return await ticks.ticksByUser(selectors, safeLimit, safeOffset)
   },
-  userTicksByClimbId: async (_, input, { dataSources }): Promise<TickType[] | null> => {
+  userTicksByClimbId: async (_, input: TickByClimbSelectors, { dataSources }): Promise<TickType[] | null> => {
     const { ticks }: { ticks: TickDataSource } = dataSources
-    const { climbId, userId } = input
-    return await ticks.ticksByUserIdAndClimb(climbId, userId)
+    const { climbId, userId, limit, offset } = input
+    const safeLimit = Math.min(limit ?? DEFAULT_LIMIT, MAX_LIMIT)
+    const safeOffset = offset ?? 0
+    return await ticks.ticksByUserIdAndClimb(climbId, userId, safeLimit, safeOffset)
   }
 }
 

src/model/AreaDataSource.ts

@@ -317,7 +317,7 @@ export default class AreaDataSource extends MongoDataSource<AreaType> {
     return await this.areaModel.find(filter).lean()
   }
 
-  async bulkDownloadAreas (ancestors: string[]): Promise<AreaType[]> {
+  async bulkDownloadAreas (ancestors: string[], limit: number = 500, offset: number = 0): Promise<AreaType[]> {
     if (ancestors.length < 2) {
       throw new GraphQLError('Must provide at least 2 ancestors.', {
         extensions: {
@@ -326,10 +326,23 @@ export default class AreaDataSource extends MongoDataSource<AreaType> {
       })
     }
     const ancestorsCSV = ancestors.join(',')
-    const [leafAreas, nonLeafAreas] = await Promise.all([
-      this.findDescendantsByPath(ancestorsCSV, true),
-      this.findDescendantsByPath(ancestorsCSV, false)
-    ])
-    return nonLeafAreas.concat(leafAreas)
+    return await this.findDescendantsByPathPaginated(ancestorsCSV, limit, offset)
+  }
+
+  /**
+   * Find all descendants (inclusive) starting from path with pagination
+   * @param path comma-separated _id's of area
+   * @param limit maximum number of results
+   * @param offset number of results to skip
+   * @returns array of areas
+   */
+  async findDescendantsByPathPaginated (path: string, limit: number = 500, offset: number = 0): Promise<AreaType[]> {
+    const regex = new RegExp(`^${path}`)
+    const filter: any = { ancestors: regex, _deleting: { $eq: null } }
+    return await this.collection
+      .find(filter)
+      .skip(offset)
+      .limit(limit)
+      .toArray()
   }
 }

src/model/AreaHistoryDatasource.ts

@@ -6,7 +6,7 @@ import { getChangeLogModel } from '../db/index.js'
 export class AreaHistoryDataSource extends MongoDataSource<ChangeLogType> {
   changelogModel = getChangeLogModel()
 
-  async getChangeSetsByUuid (areaUuid?: MUUID): Promise<AreaChangeLogType[]> {
+  async getChangeSetsByUuid (areaUuid?: MUUID, limit: number = 50, offset: number = 0): Promise<AreaChangeLogType[]> {
     let rs
     if (areaUuid == null) {
       // No area id specified: return all changes
@@ -22,6 +22,12 @@ export class AreaHistoryDataSource extends MongoDataSource<ChangeLogType> {
           $sort: {
             createdAt: -1
           }
+        },
+        {
+          $skip: offset
+        },
+        {
+          $limit: limit
         }
       ])
       return rs as AreaChangeLogType[]
@@ -53,6 +59,12 @@ export class AreaHistoryDataSource extends MongoDataSource<ChangeLogType> {
             $sort: {
               createdAt: -1
             }
+          },
+          {
+            $skip: offset
+          },
+          {
+            $limit: limit
           }
         ])
       return rs2

src/model/ChangeLogDataSource.ts

@@ -63,27 +63,35 @@ export default class ChangeLogDataSource extends MongoDataSource<ChangeLogType>
     return this
   }
 
-  async getAreaChangeSets (areaUuid?: MUUID): Promise<AreaChangeLogType[]> {
-    return await AreaHistoryDataSource.getInstance().getChangeSetsByUuid(areaUuid)
+  async getAreaChangeSets (areaUuid?: MUUID, limit: number = 50, offset: number = 0): Promise<AreaChangeLogType[]> {
+    return await AreaHistoryDataSource.getInstance().getChangeSetsByUuid(areaUuid, limit, offset)
   }
 
-  async getOrganizationChangeSets (orgId?: MUUID): Promise<OrganizationChangeLogType[]> {
-    return await OrganizationHistoryDataSource.getInstance().getChangeSetsByOrgId(orgId)
+  async getOrganizationChangeSets (orgId?: MUUID, limit: number = 50, offset: number = 0): Promise<OrganizationChangeLogType[]> {
+    return await OrganizationHistoryDataSource.getInstance().getChangeSetsByOrgId(orgId, limit, offset)
   }
 
   /**
    * Return all changes.  For now just handle Area type.
    * @param uuidList optional filter
+   * @param limit maximum number of results (default 50)
+   * @param offset number of results to skip (default 0)
    * @returns change sets
    */
-  async getChangeSets (uuidList: MUUID[]): Promise<Array<AreaChangeLogType | ClimbChangeLogType | OrganizationChangeLogType>> {
+  async getChangeSets (uuidList: MUUID[], limit: number = 50, offset: number = 0): Promise<Array<AreaChangeLogType | ClimbChangeLogType | OrganizationChangeLogType>> {
     return await this.changeLogModel.aggregate([
       {
         $sort: {
           createdAt: -1
         }
+      },
+      {
+        $skip: offset
+      },
+      {
+        $limit: limit
       }
-    ]).limit(500)
+    ])
   }
 
   async _testRemoveAll (): Promise<void> {

src/model/OrganizationHistoryDatasource.ts

@@ -6,7 +6,7 @@ import { getChangeLogModel } from '../db/index.js'
 export class OrganizationHistoryDataSource extends MongoDataSource<OrganizationChangeLogType> {
   changelogModel = getChangeLogModel()
 
-  async getChangeSetsByOrgId (orgId?: MUUID): Promise<OrganizationChangeLogType[]> {
+  async getChangeSetsByOrgId (orgId?: MUUID, limit: number = 50, offset: number = 0): Promise<OrganizationChangeLogType[]> {
     let rs
     if (orgId == null) {
       // No orgId specified: return all changes
@@ -22,6 +22,12 @@ export class OrganizationHistoryDataSource extends MongoDataSource<OrganizationC
           $sort: {
             createdAt: -1
           }
+        },
+        {
+          $skip: offset
+        },
+        {
+          $limit: limit
         }
       ])
       return rs as OrganizationChangeLogType[]
@@ -53,6 +59,12 @@ export class OrganizationHistoryDataSource extends MongoDataSource<OrganizationC
             $sort: {
               createdAt: -1
             }
+          },
+          {
+            $skip: offset
+          },
+          {
+            $limit: limit
           }
         ])
       return rs2