diff --git a/surfsense_backend/app/connectors/bookstack_connector.py b/surfsense_backend/app/connectors/bookstack_connector.py index 138f2a826..04eac5679 100644 --- a/surfsense_backend/app/connectors/bookstack_connector.py +++ b/surfsense_backend/app/connectors/bookstack_connector.py @@ -155,12 +155,77 @@ def make_api_request( except requests.exceptions.RequestException as e: raise Exception(f"BookStack API request failed: {e!s}") from e - def get_all_pages(self, count: int = 500) -> list[dict[str, Any]]: + def get_all_shelves(self, count: int = 500) -> list[dict[str, Any]]: + """ + Fetch all shelves from BookStack with pagination. + + Args: + count: Number of records per request (max 500) + + Returns: + List of shelf objects + """ + all_shelves = [] + offset = 0 + + while True: + params = { + "count": min(count, 500), + "offset": offset, + } + + result = self.make_api_request("shelves", params) + + if not isinstance(result, dict) or "data" not in result: + raise Exception("Invalid response from BookStack API") + + shelves = result["data"] + all_shelves.extend(shelves) + + logger.info(f"Fetched {len(shelves)} shelves (offset: {offset})") + + total = result.get("total", 0) + if offset + len(shelves) >= total: + break + + offset += len(shelves) + + logger.info(f"Total shelves fetched: {len(all_shelves)}") + return all_shelves + + def build_book_to_shelf_map(self) -> dict[int, int]: + """ + Build a mapping from book_id to shelf_id. + + Fetches all shelves and their book listings to create + a lookup table used for filtering pages by shelf. + + Returns: + Dict mapping book_id -> shelf_id + """ + book_to_shelf = {} + shelves = self.get_all_shelves() + + for shelf in shelves: + shelf_id = shelf["id"] + shelf_detail = self.make_api_request(f"shelves/{shelf_id}") + if isinstance(shelf_detail, dict): + for book in shelf_detail.get("books", []): + book_to_shelf[book["id"]] = shelf_id + + return book_to_shelf + + def get_all_pages( + self, + count: int = 500, + excluded_shelf_ids: list[int] | None = None, + ) -> list[dict[str, Any]]: """ Fetch all pages from BookStack with pagination. Args: count: Number of records per request (max 500) + excluded_shelf_ids: Optional list of shelf IDs whose pages should be excluded Returns: List of page objects @@ -195,6 +260,15 @@ def get_all_pages(self, count: int = 500) -> list[dict[str, Any]]: offset += len(pages) + # Filter by excluded shelves if specified + if excluded_shelf_ids: + book_to_shelf = self.build_book_to_shelf_map() + excluded = set(excluded_shelf_ids) + all_pages = [ + p for p in all_pages + if book_to_shelf.get(p.get("book_id")) not in excluded + ] + logger.info(f"Total pages fetched: {len(all_pages)}") return all_pages @@ -268,6 +342,7 @@ def get_pages_by_date_range( start_date: str, end_date: str, count: int = 500, + excluded_shelf_ids: list[int] | None = None, ) -> tuple[list[dict[str, Any]], str | None]: """ Fetch pages updated within a specific date range. @@ -278,6 +353,7 @@ def get_pages_by_date_range( start_date: Start date in YYYY-MM-DD format end_date: End date in YYYY-MM-DD format (currently unused, for future use) count: Number of records per request (max 500) + excluded_shelf_ids: Optional list of shelf IDs whose pages should be excluded Returns: Tuple of (list of page objects, error message or None) @@ -316,6 +392,15 @@ def get_pages_by_date_range( offset += len(pages) + # Filter by excluded shelves if specified + if excluded_shelf_ids and all_pages: + book_to_shelf = self.build_book_to_shelf_map() + excluded = set(excluded_shelf_ids) + all_pages = [ + p for p in all_pages + if book_to_shelf.get(p.get("book_id")) not in excluded + ] + if not all_pages: return [], f"No pages found updated after {start_date}" diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index b241aa2fb..ef61bb6fd 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -143,6 +143,53 @@ async def list_github_repositories( ) from e +class BookStackCredentialsRequest(BaseModel): + """Request model for BookStack API credentials.""" + base_url: str = Field(..., description="BookStack instance base URL") + token_id: str = Field(..., description="BookStack API Token ID") + token_secret: str = Field(..., description="BookStack API Token Secret") + + +@router.post("/bookstack/shelves", response_model=list[dict[str, Any]]) +async def list_bookstack_shelves( + creds: BookStackCredentialsRequest, + user: User = Depends(current_active_user), +): + """ + Fetches all shelves from a BookStack instance. + Used by the frontend to let users select which shelves to exclude from indexing. + """ + try: + from app.connectors.bookstack_connector import BookStackConnector + + client = BookStackConnector( + base_url=creds.base_url, + token_id=creds.token_id, + token_secret=creds.token_secret, + ) + shelves = client.get_all_shelves() + + result = [] + for shelf in shelves: + detail = client.make_api_request(f"shelves/{shelf['id']}") + books = detail.get("books", []) if isinstance(detail, dict) else [] + result.append({ + "id": shelf["id"], + "name": shelf["name"], + "book_count": len(books), + "books": [{"id": b["id"], "name": b["name"]} for b in books], + }) + return result + except ValueError as e: + logger.error(f"BookStack credential validation failed for user {user.id}: {e!s}") + raise HTTPException(status_code=400, detail=f"Invalid BookStack credentials: {e!s}") from e + except Exception as e: + logger.error(f"Failed to fetch BookStack shelves for user {user.id}: {e!s}") + raise HTTPException( + status_code=500, detail=f"Failed to fetch BookStack shelves: {e!s}" + ) from e + + @router.post("/search-source-connectors", response_model=SearchSourceConnectorRead) async def create_search_source_connector( connector: SearchSourceConnectorCreate, diff --git a/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py b/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py index bf3aaa35f..579b698e2 100644 --- a/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py @@ -104,6 +104,9 @@ async def index_bookstack_pages( bookstack_token_id = connector.config.get("BOOKSTACK_TOKEN_ID") bookstack_token_secret = connector.config.get("BOOKSTACK_TOKEN_SECRET") + # Optional: shelf IDs to exclude from indexing + excluded_shelf_ids = connector.config.get("BOOKSTACK_EXCLUDED_SHELF_IDS", []) + if ( not bookstack_base_url or not bookstack_token_id @@ -148,7 +151,9 @@ async def index_bookstack_pages( # Get pages within date range try: pages, error = bookstack_client.get_pages_by_date_range( - start_date=start_date_str, end_date=end_date_str + start_date=start_date_str, + end_date=end_date_str, + excluded_shelf_ids=excluded_shelf_ids, ) if error: diff --git a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/bookstack-connect-form.tsx b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/bookstack-connect-form.tsx index b2a6b0b25..bbfb79bda 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/bookstack-connect-form.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/bookstack-connect-form.tsx @@ -1,12 +1,15 @@ "use client"; import { zodResolver } from "@hookform/resolvers/zod"; -import { Info } from "lucide-react"; +import { Info, Loader2, RefreshCw, X } from "lucide-react"; import type { FC } from "react"; -import { useRef, useState } from "react"; +import { useCallback, useRef, useState } from "react"; import { useForm } from "react-hook-form"; import * as z from "zod"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; import { Form, FormControl, @@ -27,6 +30,7 @@ import { } from "@/components/ui/select"; import { Switch } from "@/components/ui/switch"; import { EnumConnectorName } from "@/contracts/enums/connector"; +import { connectorsApiService } from "@/lib/apis/connectors-api.service"; import { DateRangeSelector } from "../../components/date-range-selector"; import { getConnectorBenefits } from "../connector-benefits"; import type { ConnectFormProps } from "../index"; @@ -46,12 +50,25 @@ const bookstackConnectorFormSchema = z.object({ type BookStackConnectorFormValues = z.infer; +interface BookStackShelf { + id: number; + name: string; + book_count: number; + books: { id: number; name: string }[]; +} + export const BookStackConnectForm: FC = ({ onSubmit, isSubmitting }) => { const isSubmittingRef = useRef(false); const [startDate, setStartDate] = useState(undefined); const [endDate, setEndDate] = useState(undefined); const [periodicEnabled, setPeriodicEnabled] = useState(false); const [frequencyMinutes, setFrequencyMinutes] = useState("1440"); + const [shelves, setShelves] = useState([]); + const [excludedShelfIds, setExcludedShelfIds] = useState([]); + const [loadingShelves, setLoadingShelves] = useState(false); + const [shelvesError, setShelvesError] = useState(null); + const [shelvesLoaded, setShelvesLoaded] = useState(false); + const form = useForm({ resolver: zodResolver(bookstackConnectorFormSchema), defaultValues: { @@ -62,8 +79,42 @@ export const BookStackConnectForm: FC = ({ onSubmit, isSubmitt }, }); + const fetchShelves = useCallback(async () => { + const values = form.getValues(); + if (!values.base_url || !values.token_id || !values.token_secret) { + setShelvesError("Please fill in Base URL, Token ID, and Token Secret first."); + return; + } + + setLoadingShelves(true); + setShelvesError(null); + + try { + const data = await connectorsApiService.listBookStackShelves( + values.base_url, + values.token_id, + values.token_secret, + ) as BookStackShelf[]; + setShelves(data); + setShelvesLoaded(true); + } catch (err) { + setShelvesError(err instanceof Error ? err.message : "Failed to fetch shelves"); + setShelves([]); + setShelvesLoaded(false); + } finally { + setLoadingShelves(false); + } + }, [form]); + + const toggleShelfExclusion = (shelfId: number) => { + setExcludedShelfIds((prev) => + prev.includes(shelfId) + ? prev.filter((id) => id !== shelfId) + : [...prev, shelfId] + ); + }; + const handleSubmit = async (values: BookStackConnectorFormValues) => { - // Prevent multiple submissions if (isSubmittingRef.current || isSubmitting) { return; } @@ -77,6 +128,7 @@ export const BookStackConnectForm: FC = ({ onSubmit, isSubmitt BOOKSTACK_BASE_URL: values.base_url, BOOKSTACK_TOKEN_ID: values.token_id, BOOKSTACK_TOKEN_SECRET: values.token_secret, + BOOKSTACK_EXCLUDED_SHELF_IDS: excludedShelfIds, }, is_indexable: true, is_active: true, @@ -203,6 +255,95 @@ export const BookStackConnectForm: FC = ({ onSubmit, isSubmitt )} /> + {/* Shelf Exclusion Picker */} +
+
+
+

Shelf Filter

+

+ Optionally exclude shelves from indexing. Click "Load Shelves" after entering your credentials. +

+
+ +
+ + {shelvesError && ( +

{shelvesError}

+ )} + + {shelvesLoaded && shelves.length > 0 && ( +
+ {shelves.map((shelf) => { + const isExcluded = excludedShelfIds.includes(shelf.id); + return ( + + ); + })} +
+ )} + + {shelvesLoaded && shelves.length === 0 && ( +

No shelves found in this BookStack instance.

+ )} + + {excludedShelfIds.length > 0 && shelvesLoaded && ( +
+ Excluding: + {excludedShelfIds.map((id) => { + const shelf = shelves.find((s) => s.id === id); + return ( + toggleShelfExclusion(id)} + > + {shelf?.name || `ID ${id}`} + + + ); + })} +
+ )} +
+ {/* Indexing Configuration */}

Indexing Configuration

diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/bookstack-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/bookstack-config.tsx index ca287c652..a182efe28 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/bookstack-config.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/bookstack-config.tsx @@ -1,16 +1,27 @@ "use client"; -import { KeyRound } from "lucide-react"; +import { KeyRound, Loader2, RefreshCw, X } from "lucide-react"; import type { FC } from "react"; -import { useEffect, useState } from "react"; +import { useCallback, useEffect, useState } from "react"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; +import { connectorsApiService } from "@/lib/apis/connectors-api.service"; import type { ConnectorConfigProps } from "../index"; export interface BookStackConfigProps extends ConnectorConfigProps { onNameChange?: (name: string) => void; } +interface BookStackShelf { + id: number; + name: string; + book_count: number; + books: { id: number; name: string }[]; +} + export const BookStackConfig: FC = ({ connector, onConfigChange, @@ -26,18 +37,66 @@ export const BookStackConfig: FC = ({ (connector.config?.BOOKSTACK_TOKEN_SECRET as string) || "" ); const [name, setName] = useState(connector.name || ""); + const [shelves, setShelves] = useState([]); + const [excludedShelfIds, setExcludedShelfIds] = useState( + (connector.config?.BOOKSTACK_EXCLUDED_SHELF_IDS as number[]) || [] + ); + const [loadingShelves, setLoadingShelves] = useState(false); + const [shelvesError, setShelvesError] = useState(null); + const [shelvesLoaded, setShelvesLoaded] = useState(false); // Update values when connector changes useEffect(() => { const url = (connector.config?.BOOKSTACK_BASE_URL as string) || ""; const id = (connector.config?.BOOKSTACK_TOKEN_ID as string) || ""; const secret = (connector.config?.BOOKSTACK_TOKEN_SECRET as string) || ""; + const excluded = (connector.config?.BOOKSTACK_EXCLUDED_SHELF_IDS as number[]) || []; setBaseUrl(url); setTokenId(id); setTokenSecret(secret); + setExcludedShelfIds(excluded); setName(connector.name || ""); }, [connector.config, connector.name]); + const fetchShelves = useCallback(async () => { + if (!baseUrl || !tokenId || !tokenSecret) { + setShelvesError("Please fill in Base URL, Token ID, and Token Secret first."); + return; + } + + setLoadingShelves(true); + setShelvesError(null); + + try { + const data = await connectorsApiService.listBookStackShelves( + baseUrl, + tokenId, + tokenSecret, + ) as BookStackShelf[]; + setShelves(data); + setShelvesLoaded(true); + } catch (err) { + setShelvesError(err instanceof Error ? err.message : "Failed to fetch shelves"); + setShelves([]); + setShelvesLoaded(false); + } finally { + setLoadingShelves(false); + } + }, [baseUrl, tokenId, tokenSecret]); + + const toggleShelfExclusion = (shelfId: number) => { + const newExcluded = excludedShelfIds.includes(shelfId) + ? excludedShelfIds.filter((id) => id !== shelfId) + : [...excludedShelfIds, shelfId]; + setExcludedShelfIds(newExcluded); + if (onConfigChange) { + onConfigChange({ + ...connector.config, + BOOKSTACK_EXCLUDED_SHELF_IDS: newExcluded, + }); + } + }; + const handleBaseUrlChange = (value: string) => { setBaseUrl(value); if (onConfigChange) { @@ -145,6 +204,105 @@ export const BookStackConfig: FC = ({
+ + {/* Shelf Exclusion Picker */} +
+
+
+

Shelf Filter

+

+ Select which shelves to include in indexing. Unchecked shelves will be excluded. +

+
+ +
+ + {shelvesError && ( +

{shelvesError}

+ )} + + {shelvesLoaded && shelves.length > 0 && ( +
+ {shelves.map((shelf) => { + const isExcluded = excludedShelfIds.includes(shelf.id); + return ( + + ); + })} +
+ )} + + {!shelvesLoaded && excludedShelfIds.length > 0 && ( +
+ Currently excluding shelf IDs: + {excludedShelfIds.map((id) => ( + + {id} + + ))} +

+ Click "Load Shelves" to see shelf names and modify exclusions. +

+
+ )} + + {excludedShelfIds.length > 0 && shelvesLoaded && ( +
+ Excluding: + {excludedShelfIds.map((id) => { + const shelf = shelves.find((s) => s.id === id); + return ( + toggleShelfExclusion(id)} + > + {shelf?.name || `ID ${id}`} + + + ); + })} +
+ )} +
); }; diff --git a/surfsense_web/lib/apis/connectors-api.service.ts b/surfsense_web/lib/apis/connectors-api.service.ts index fafe1a8fa..0f64d4223 100644 --- a/surfsense_web/lib/apis/connectors-api.service.ts +++ b/surfsense_web/lib/apis/connectors-api.service.ts @@ -378,6 +378,23 @@ class ConnectorsApiService { listDiscordChannelsResponse ); }; + + // ============================================================================= + // BookStack Connector Methods + // ============================================================================= + + /** + * List BookStack shelves for shelf exclusion picker + */ + listBookStackShelves = async (baseUrl: string, tokenId: string, tokenSecret: string) => { + return baseApiService.post(`/api/v1/bookstack/shelves`, undefined, { + body: { + base_url: baseUrl, + token_id: tokenId, + token_secret: tokenSecret, + }, + }); + }; } export type { SlackChannel, DiscordChannel };