reworking queries to include pagination

Author: MattReimer

pydex/classes/RiverscapesAPI.py

@@ -3,6 +3,7 @@
 from typing import Dict, List, Generator, Tuple
 import webbrowser
 import re
+import math
 import time
 import concurrent.futures
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
@@ -490,8 +491,29 @@ def get_project_full(self, project_id: str) -> RiverscapesProject:
             _type_: _description_
         """
         qry = self.load_query('getProjectFull')
-        results = self.run_query(qry, {"id": project_id})
-        return RiverscapesProject(results['data']['project'])
+        
+        limit = 500
+        offset = 0
+        
+        results = self.run_query(qry, {"id": project_id, "fileLimit": limit, "fileOffset": offset})
+        project_data = results['data']['project']
+        
+        files_meta = project_data.get('files')
+        if files_meta and 'total' in files_meta:
+            total = files_meta.get('total', 0)
+            items = files_meta.get('items', [])
+            
+            if total > limit:
+                # Calculate how many extra pages we need to fetch
+                num_pages = math.ceil(total / limit)
+                # We already grabbed page 1 (offset 0), so fetch from page 2
+                for page in range(1, num_pages):
+                    page_offset = page * limit
+                    page_results = self.run_query(qry, {"id": project_id, "fileLimit": limit, "fileOffset": page_offset})
+                    page_items = page_results['data']['project'].get('files', {}).get('items', [])
+                    items.extend(page_items)
+                
+        return RiverscapesProject(project_data)
 
     def get_project_files(self, project_id: str) -> List[Dict[str, any]]:
         """ This returns the file listing with everything you need to download project files
@@ -504,8 +526,34 @@ def get_project_files(self, project_id: str) -> List[Dict[str, any]]:
             _type_: _description_
         """
         qry = self.load_query('projectFiles')
-        results = self.run_query(qry, {"projectId": project_id})
-        return results['data']['project']['files']
+        
+        offset = 0
+        limit = 500
+        all_files = []
+        
+        # Initial fetch
+        results = self.run_query(qry, {
+            "projectId": project_id,
+            "limit": limit,
+            "offset": offset
+        })
+        
+        files_meta = results['data']['project']['files']
+        total = files_meta['total']
+        all_files.extend(files_meta['items'])
+        
+        if total > limit:
+            num_pages = math.ceil(total / limit)
+            for page in range(1, num_pages):
+                page_offset = page * limit
+                results = self.run_query(qry, {
+                    "projectId": project_id,
+                    "limit": limit,
+                    "offset": page_offset
+                })
+                all_files.extend(results['data']['project']['files']['items'])
+                
+        return all_files
 
     def get_project_types(self) -> Dict[str, RiverscapesProjectType]:
         """_summary_

pydex/graphql/queries/getProjectFull.graphql

@@ -1,4 +1,4 @@
-query getProjectFull($id: ID!) {
+query getProjectFull($id: ID!, $fileLimit: Int = 500, $fileOffset: Int = 0) {
   project(id: $id) {
     ...dbobj
     bounds {
@@ -54,10 +54,13 @@ query getProjectFull($id: ID!) {
     tags
     totalSize
     visibility
-    files {
-      localPath
-      size
-      contentType
+    files(limit: $fileLimit, offset: $fileOffset) {
+      items {
+        localPath
+        size
+        contentType
+      }
+      ...paginated
     }
     tree {
       defaultView

pydex/graphql/queries/projectFiles.graphql

@@ -1,13 +1,15 @@
-query projectFiles ($projectId: ID!) {
+query projectFiles ($projectId: ID!, $limit: Int!, $offset: Int!) {
     project(id: $projectId) {
         id
-        files {
-            size
-            contentType
-            downloadUrl
-            localPath
-            etag
+        files(limit: $limit, offset: $offset) {
+            total
+            items {
+                size
+                contentType
+                downloadUrl
+                localPath
+                etag
+            }
         }
-        id
     }
 }

pydex/graphql/riverscapes.schema.graphql

@@ -1135,7 +1135,7 @@ type Project implements DBObj & HasOwner & HasStar {
   deleted: Boolean!
   description: String!
   dirty: Boolean!
-  files: [FileDownloadMeta!]!
+  files(limit: Int!, offset: Int!): PaginatedFileDownloadMeta!
 
   """Banner image (optional)"""
   heroImage: HeroImage
@@ -2396,6 +2396,7 @@ type User implements DBObj & UserInterface {
   description: String!
   hivebriteProfile: HivebriteProfile
   id: ID!
+  initialized: Boolean
   jobTitle: String
   lastLogin: DateTime!
   location: String
@@ -2466,6 +2467,7 @@ interface UserInterface {
   description: String!
   hivebriteProfile: HivebriteProfile
   id: ID!
+  initialized: Boolean
 
   """Simple job title string field"""
   jobTitle: String