Search all 1,500+ publications via auto-pagination, remove limit param

Author: kks32

dapi/client.py

@@ -368,20 +368,19 @@ def search(
         pi: Optional[str] = None,
         keyword: Optional[str] = None,
         publication_type: Optional[str] = None,
-        limit: int = 100,
         output: str = "df",
     ):
-        """Search published datasets with optional filters.
+        """Search all published datasets with optional filters.
 
-        All filters are AND-combined and case-insensitive.
+        Searches across all ~1,500 published datasets. Filters are AND-combined
+        and case-insensitive.
 
         Args:
             query (str, optional): General search across title, description, keywords, PI.
             pi (str, optional): Filter by PI name.
             keyword (str, optional): Filter by keyword.
             publication_type (str, optional): Filter by type: "simulation",
                 "experimental", "field_recon", "other", "hybrid_simulation".
-            limit (int, optional): Max publications to fetch. Defaults to 100.
             output (str, optional): "df" for DataFrame (default), "list" for dicts.
 
         Returns:
@@ -400,7 +399,6 @@ def search(
             pi=pi,
             keyword=keyword,
             publication_type=publication_type,
-            limit=limit,
             output=output,
         )
 

dapi/publications.py

@@ -9,6 +9,7 @@
 
 _DS_PUBLICATIONS_API = "https://designsafe-ci.org/api/publications/v2/"
 _PUBLISHED_SYSTEM_ID = "designsafe.storage.published"
+_PAGE_SIZE = 100
 
 
 def _get_auth_headers(t: Tapis) -> Dict[str, str]:
@@ -17,6 +18,28 @@ def _get_auth_headers(t: Tapis) -> Dict[str, str]:
     return {"X-Tapis-Token": token, "Authorization": f"Bearer {token}"}
 
 
+def _fetch_all_publications(headers: Dict[str, str]) -> List[Dict]:
+    """Fetch all publications using pagination."""
+    all_pubs = []
+    offset = 0
+    while True:
+        resp = requests.get(
+            _DS_PUBLICATIONS_API,
+            headers=headers,
+            params={"limit": _PAGE_SIZE, "offset": offset},
+            timeout=30,
+        )
+        resp.raise_for_status()
+        data = resp.json()
+        batch = data.get("result", [])
+        total = data.get("total", 0)
+        all_pubs.extend(batch)
+        if len(batch) < _PAGE_SIZE or len(all_pubs) >= total:
+            break
+        offset += _PAGE_SIZE
+    return all_pubs
+
+
 def _pi_display(pi: Optional[Dict]) -> str:
     """Format PI dict as display name."""
     if not pi:
@@ -93,11 +116,11 @@ def search_publications(
     pi: Optional[str] = None,
     keyword: Optional[str] = None,
     publication_type: Optional[str] = None,
-    limit: int = 100,
     output: str = "df",
 ) -> Union[pd.DataFrame, List[Dict]]:
-    """Search published datasets with optional filters.
+    """Search all published datasets with optional filters.
 
+    Fetches all publications (~1,500) and filters client-side.
     All filters are case-insensitive and combined with AND logic.
     At least one filter (query, pi, keyword, or publication_type) must be provided.
 
@@ -108,7 +131,6 @@ def search_publications(
         keyword (str, optional): Filter by keyword (partial match against keywords list).
         publication_type (str, optional): Filter by type: "simulation", "experimental",
             "field_recon", "other", "hybrid_simulation".
-        limit (int, optional): Max publications to fetch before filtering. Defaults to 100.
         output (str, optional): "df" for DataFrame (default), "list" for list of dicts.
 
     Returns:
@@ -128,19 +150,12 @@ def search_publications(
 
     headers = _get_auth_headers(t)
     try:
-        resp = requests.get(
-            _DS_PUBLICATIONS_API,
-            headers=headers,
-            params={"limit": limit, "offset": 0},
-            timeout=30,
-        )
-        resp.raise_for_status()
+        all_pubs = _fetch_all_publications(headers)
     except requests.RequestException as e:
         raise FileOperationError(f"Failed to search publications: {e}") from e
 
-    data = resp.json()
     matches = []
-    for p in data.get("result", []):
+    for p in all_pubs:
         # Apply filters with AND logic
         if query:
             searchable = " ".join(

docs/publications.md

@@ -41,8 +41,8 @@ ds.publications.search(publication_type="simulation")
 # Combine filters (AND logic)
 ds.publications.search(keyword="storm surge", publication_type="simulation")
 
-# Increase search pool (API returns 100 by default)
-ds.publications.search(keyword="earthquake", limit=500)
+# Combine filters
+ds.publications.search(keyword="earthquake", publication_type="experimental")
 ```
 
 ## Get publication details
@@ -79,7 +79,7 @@ DataFrame columns: `name`, `type`, `size`, `lastModified`, `path`.
 
 ## How it works
 
-1. **Publication listing and search**: dapi queries the DesignSafe portal API (`/api/publications/v2/`). The API returns all publications; search filtering is done client-side.
+1. **Publication listing and search**: dapi queries the DesignSafe portal API (`/api/publications/v2/`). For search, all ~1,500 publications are fetched via pagination (~3 seconds) and filtered client-side.
 
 2. **Publication detail**: dapi queries `/api/publications/v2/PRJ-XXXX/` which returns metadata in the `tree.children[0].value` structure, including DOIs, authors, and data types.