diff --git a/docs/userGuide/makingTheSiteSearchable.md b/docs/userGuide/makingTheSiteSearchable.md index a1e92e96f4..6e83cf5213 100644 --- a/docs/userGuide/makingTheSiteSearchable.md +++ b/docs/userGuide/makingTheSiteSearchable.md @@ -50,10 +50,6 @@ MarkBind now supports [Pagefind](https://pagefind.app/), a static low-bandwidth This is a beta feature and will be refined in future updates. To use it, you must have enableSearch: true in your site.json (this is the default). - -The Pagefind index is currently only generated during a full site build (e.g., markbind build). It will not repeatedly update during live reload (markbind serve) when you modify pages. You must restart the server (re-run markbind serve) or rebuild to refresh the search index. - - To add the Pagefind search bar to your page, simply insert the following element where you want it to appear: ```md diff --git a/packages/cli/src/util/serveUtil.ts b/packages/cli/src/util/serveUtil.ts index eec7886aa8..8022cf4702 100755 --- a/packages/cli/src/util/serveUtil.ts +++ b/packages/cli/src/util/serveUtil.ts @@ -35,7 +35,8 @@ const addHandler = (site: any, onePagePath?: boolean) => (filePath: string): voi } Promise.resolve().then(async () => { if (site.isFilepathAPage(filePath) || site.isDependencyOfPage(filePath)) { - return site.rebuildSourceFiles(); + await site.rebuildSourceFiles(); + return await site.updatePagefindIndex(filePath); } return site.buildAsset(filePath); }).catch((err: Error) => { @@ -59,7 +60,8 @@ const changeHandler = (site: any, onePagePath?: boolean) => (filePath: string): return site.reloadSiteConfig(); } if (site.isDependencyOfPage(filePath)) { - return site.rebuildAffectedSourceFiles(filePath); + await site.rebuildAffectedSourceFiles(filePath); + return await site.updatePagefindIndex(filePath); } return site.buildAsset(filePath); }).catch((err: Error) => { @@ -80,7 +82,8 @@ const removeHandler = (site: any, onePagePath?: boolean) => (filePath: string): } Promise.resolve().then(async () => { if (site.isFilepathAPage(filePath) || site.isDependencyOfPage(filePath)) { - return site.rebuildSourceFiles(); + await site.rebuildSourceFiles(); + return await site.indexSiteWithPagefind(); } return site.removeAsset(filePath); }).catch((err: Error) => { diff --git a/packages/core/src/Site/SiteGenerationManager.ts b/packages/core/src/Site/SiteGenerationManager.ts index a5fce2f302..3d9162a1f2 100644 --- a/packages/core/src/Site/SiteGenerationManager.ts +++ b/packages/core/src/Site/SiteGenerationManager.ts @@ -84,6 +84,9 @@ export class SiteGenerationManager { currentOpenedPages: string[]; toRebuild: Set; + // Pagefind index state (kept in memory for serve mode for incremental updates) + pagefindIndex: any; + constructor(rootPath: string, outputPath: string, onePagePath: string, forceReload = false, siteConfigPath = SITE_CONFIG_NAME, isDevMode: any, backgroundBuildMode: boolean, postBackgroundBuildFunc: () => void) { @@ -105,6 +108,9 @@ export class SiteGenerationManager { : ''; this.currentOpenedPages = []; this.toRebuild = new Set(); + + // Pagefind index state (kept in memory for serve mode for incremental updates) + this.pagefindIndex = null; } configure(siteAssets: SiteAssetsManager, sitePages: SitePagesManager) { @@ -317,7 +323,15 @@ export class SiteGenerationManager { await this.siteAssets.copyMaterialIconsAsset(); await this.writeSiteData(); if (this.siteConfig.enableSearch) { - const indexingSucceeded = await this.indexSiteWithPagefind(); + let indexingSucceeded: boolean; + if (this.onePagePath) { + const builtPages = this.sitePages.pages.filter(page => + fs.existsSync(page.pageConfig.resultPath), + ); + indexingSucceeded = await this.updatePagefindIndex(builtPages); + } else { + indexingSucceeded = await this.indexSiteWithPagefind(); + } this.sitePages.pagefindIndexingSucceeded = indexingSucceeded; } this.calculateBuildTimeForGenerate(startTime, lazyWebsiteGenerationString); @@ -440,6 +454,11 @@ export class SiteGenerationManager { this._setTimestampVariable(); await this.runPageGenerationTasks([pageGenerationTask]); await this.writeSiteData(); + + if (this.siteConfig.enableSearch && this.pagefindIndex) { + await this.updatePagefindIndex(pagesToRebuild); + } + SiteGenerationManager.calculateBuildTimeForRebuildPagesBeingViewed(startTime); } catch (err) { await SiteGenerationManager.rejectHandler(err, [this.tempPath, this.outputPath]); @@ -466,6 +485,11 @@ export class SiteGenerationManager { const isCompleted = await this.generatePagesMarkedToRebuild(); if (isCompleted) { logger.info('Background building completed!'); + + if (this.siteConfig.enableSearch) { + await this.indexSiteWithPagefind(); + } + this.postBackgroundBuildFunc(); } } @@ -872,29 +896,49 @@ export class SiteGenerationManager { ); /** - * Indexes all the pages of the site using pagefind. - * @returns true if indexing succeeded and pagefind assets were written, false otherwise. - */ + * Initializes a new Pagefind index with proper configuration. + * @returns The created index object + */ + private async initializePagefindIndex(): Promise { + const { createIndex } = pagefind; + const pagefindConfig = this.siteConfig.pagefind || {}; + + const createIndexOptions: Record = { + keepIndexUrl: true, + verbose: true, + logfile: 'debug.log', + }; + + if (pagefindConfig.exclude_selectors) { + createIndexOptions.excludeSelectors = pagefindConfig.exclude_selectors; + } + + const { index } = await createIndex(createIndexOptions); + return index; + } + + /** + * Indexes all the pages of the site using pagefind. + * Performs a full rebuild of the search index. + * @returns true if indexing succeeded and pagefind assets were written, false otherwise. + */ async indexSiteWithPagefind(): Promise { const startTime = new Date(); logger.info('Creating Pagefind Search Index...'); try { - const { createIndex, close } = pagefind; - - const pagefindConfig = this.siteConfig.pagefind || {}; - - const createIndexOptions: Record = { - keepIndexUrl: true, - verbose: true, - logfile: 'debug.log', - }; - - if (pagefindConfig.exclude_selectors) { - createIndexOptions.excludeSelectors = pagefindConfig.exclude_selectors; + // Clean up existing in-memory index if it exists + if (this.pagefindIndex) { + await this.pagefindIndex.deleteIndex(); + this.pagefindIndex = null; } - const { index } = await createIndex(createIndexOptions); + const index = await this.initializePagefindIndex(); + const { close } = pagefind; + if (index) { + // Store index in memory for incremental updates in serve mode + this.pagefindIndex = index; + // Filter pages that should be indexed (searchable !== false) const searchablePages = this.sitePages.pages.filter( page => page.pageConfig.searchable, @@ -939,10 +983,22 @@ export class SiteGenerationManager { logger.info(`Pagefind indexed ${totalPageCount} pages in ${totalTime}s`); const pagefindOutputPath = path.join(this.outputPath, TEMPLATE_SITE_ASSET_FOLDER_NAME, 'pagefind'); + // Clear output directory before writing + await fs.emptyDir(pagefindOutputPath); await fs.ensureDir(pagefindOutputPath); await index.writeFiles({ outputPath: pagefindOutputPath }); logger.info(`Pagefind assets written to ${pagefindOutputPath}`); - await close(); + + // Only close the index in build/deploy mode; keep it in memory for serve mode + // Detect serve mode by checking if postBackgroundBuildFunc has a name (named function = serve) + const isServeMode = this.postBackgroundBuildFunc.name !== ''; + const shouldClose = !isServeMode; + + if (shouldClose) { + await close(); + this.pagefindIndex = null; + } + return true; } logger.error('Pagefind failed to create index'); @@ -954,6 +1010,57 @@ export class SiteGenerationManager { } } + /** + * Updates the search index for changed pages only (incremental update). + * Requires the index to be kept in memory from a prior indexSiteWithPagefind() call. + * @param pages Array of pages that were modified/added + * @returns true if update succeeded, false otherwise + */ + async updatePagefindIndex(pages: Page[]): Promise { + if (!this.pagefindIndex) { + logger.info('Pagefind index not in memory, auto-creating...'); + this.pagefindIndex = await this.initializePagefindIndex(); + } + + const pagefindOutputPath = path.join(this.outputPath, TEMPLATE_SITE_ASSET_FOLDER_NAME, 'pagefind'); + + try { + const searchablePages = pages.filter(page => page.pageConfig.searchable); + + await Promise.all(searchablePages.map(async (page) => { + try { + const content = await fs.readFile(page.pageConfig.resultPath, 'utf8'); + const relativePath = path.relative(this.outputPath, page.pageConfig.resultPath); + + return this.pagefindIndex.addHTMLFile({ + sourcePath: relativePath, + content, + }); + } catch (err) { + const pageResultPath = page.pageConfig.resultPath; + logger.warn(`Skipping index update for ${pageResultPath}: file not built yet`); + return null; + } + })); + + const { files } = await this.pagefindIndex.getFiles(); + await fs.emptyDir(pagefindOutputPath); + + const pagefindFiles: { path: string; content: Uint8Array }[] = files; + await Promise.all(pagefindFiles.map(async (file) => { + const filePath = path.join(pagefindOutputPath, file.path); + await fs.ensureDir(path.dirname(filePath)); + return fs.writeFile(filePath, Buffer.from(file.content)); + })); + + logger.info(`Updated Pagefind index for ${searchablePages.length} page(s)`); + return true; + } catch (error) { + logger.error(`Failed to update Pagefind index: ${error}`); + return false; + } + } + async reloadSiteConfig() { if (this.backgroundBuildMode) { this.stopOngoingBuilds(); diff --git a/packages/core/src/Site/index.ts b/packages/core/src/Site/index.ts index 763cc025d0..533d7cdb40 100644 --- a/packages/core/src/Site/index.ts +++ b/packages/core/src/Site/index.ts @@ -86,6 +86,18 @@ export class Site { return this.generationManager.rebuildSourceFiles(); } + async indexSiteWithPagefind(): Promise { + return this.generationManager.indexSiteWithPagefind(); + } + + async updatePagefindIndex(filePaths: string | string[]): Promise { + const paths = Array.isArray(filePaths) ? filePaths : [filePaths]; + const pages = this.generationManager.sitePages.pages.filter(page => + paths.some(p => page.pageConfig.sourcePath === p), + ); + return this.generationManager.updatePagefindIndex(pages); + } + buildAsset(filePaths: string | string[]) { return this.assetsManager.buildAsset(filePaths); } diff --git a/packages/core/test/unit/Site/Site.test.ts b/packages/core/test/unit/Site/Site.test.ts index 8523920322..5cce07873e 100644 --- a/packages/core/test/unit/Site/Site.test.ts +++ b/packages/core/test/unit/Site/Site.test.ts @@ -37,6 +37,9 @@ jest.mock('../../../src/Site/SiteGenerationManager', () => ({ buildSourceFiles: jest.fn(), rebuildSourceFiles: jest.fn(), reloadSiteConfig: jest.fn(), + updatePagefindIndex: jest.fn().mockResolvedValue(true), + indexSiteWithPagefind: jest.fn().mockResolvedValue(true), + sitePages: { pages: [] }, })), })); @@ -120,6 +123,23 @@ test('Site rebuildSourceFiles delegates to SiteGenerationManager', () => { expect(site.generationManager.rebuildSourceFiles).toHaveBeenCalled(); }); +test('Site updatePagefindIndex delegates to SiteGenerationManager', async () => { + const site = new Site(...siteArguments); + const mockPage = { pageConfig: { sourcePath: 'test.md', resultPath: '_site/test.html', searchable: true } }; + const mockPages = [mockPage]; + site.generationManager.sitePages = { pages: mockPages } as any; + + await site.updatePagefindIndex('test.md'); + expect(site.generationManager.updatePagefindIndex).toHaveBeenCalledWith(mockPages); +}); + +test('Site indexSiteWithPagefind delegates to SiteGenerationManager', async () => { + const site = new Site(...siteArguments); + + await site.indexSiteWithPagefind(); + expect(site.generationManager.indexSiteWithPagefind).toHaveBeenCalled(); +}); + test('Site reloadSiteConfig delegates to SiteGenerationManager', async () => { const site = new Site(...siteArguments); await site.reloadSiteConfig(); diff --git a/packages/core/test/unit/Site/SiteGenerationManager.test.ts b/packages/core/test/unit/Site/SiteGenerationManager.test.ts index 02745f543b..8adde4dff7 100644 --- a/packages/core/test/unit/Site/SiteGenerationManager.test.ts +++ b/packages/core/test/unit/Site/SiteGenerationManager.test.ts @@ -367,6 +367,191 @@ describe('SiteGenerationManager', () => { }); }); + describe('updatePagefindIndex', () => { + beforeEach(() => { + const json = { + ...PAGE_NJK, + 'site.json': SITE_JSON_DEFAULT, + '_site/index.html': 'Test page', + '_site/page1.html': 'Page 1', + }; + mockFs.vol.fromJSON(json, rootPath); + }); + + test('should auto-create index when pagefindIndex is null', async () => { + generationManager.pagefindIndex = null; + generationManager.siteConfig = { pagefind: undefined } as any; + const mockIndex = createMockIndex({ page_count: 0, errors: [] }); + const createIndexSpy = jest.spyOn(pagefind, 'createIndex').mockResolvedValue( + { index: mockIndex } as any, + ); + + await generationManager.updatePagefindIndex([]); + + expect(createIndexSpy).toHaveBeenCalled(); + createIndexSpy.mockRestore(); + }); + + test('should call addHTMLFile for each searchable page', async () => { + const mockIndex = createMockIndex({ page_count: 1, errors: [] }, { errors: [] }); + generationManager.pagefindIndex = mockIndex; + generationManager.siteConfig = { pagefind: undefined } as any; + + const pages = [ + { pageConfig: { resultPath: path.join(outputPath, 'index.html'), searchable: true } }, + { pageConfig: { resultPath: path.join(outputPath, 'page1.html'), searchable: true } }, + ] as any; + + await generationManager.updatePagefindIndex(pages); + + expect(mockIndex.addHTMLFile).toHaveBeenCalledTimes(2); + expect(mockIndex.addHTMLFile).toHaveBeenCalledWith({ + sourcePath: 'index.html', + content: 'Test page', + }); + expect(mockIndex.addHTMLFile).toHaveBeenCalledWith({ + sourcePath: 'page1.html', + content: 'Page 1', + }); + }); + + test('should skip non-searchable pages', async () => { + const mockIndex = createMockIndex({ page_count: 1, errors: [] }, { errors: [] }); + generationManager.pagefindIndex = mockIndex; + generationManager.siteConfig = { pagefind: undefined } as any; + + const pages = [ + { pageConfig: { resultPath: path.join(outputPath, 'index.html'), searchable: false } }, + { pageConfig: { resultPath: path.join(outputPath, 'page1.html'), searchable: true } }, + ] as any; + + await generationManager.updatePagefindIndex(pages); + + expect(mockIndex.addHTMLFile).toHaveBeenCalledTimes(1); + expect(mockIndex.addHTMLFile).toHaveBeenCalledWith({ + sourcePath: 'page1.html', + content: 'Page 1', + }); + }); + + test('should log error and return false on failure', async () => { + const mockIndex = createMockIndex({ page_count: 1, errors: [] }, { errors: [] }); + (mockIndex.addHTMLFile as jest.Mock).mockRejectedValue(new Error('Index error')); + generationManager.pagefindIndex = mockIndex; + generationManager.siteConfig = { pagefind: undefined } as any; + const errorSpy = jest.spyOn(logger, 'error').mockImplementation(); + + const failData = { resultPath: path.join(outputPath, 'index.html'), searchable: true }; + const failPages = [{ pageConfig: failData }] as any; + + const result = await generationManager.updatePagefindIndex(failPages); + + expect(result).toBe(false); + expect(errorSpy).toHaveBeenCalledWith('Failed to update Pagefind index: Error: Index error'); + + errorSpy.mockRestore(); + }); + + test('should skip pages that do not exist when updating index', async () => { + const mockIndex = createMockIndex({ page_count: 1, errors: [] }, { errors: [] }); + generationManager.pagefindIndex = mockIndex; + generationManager.siteConfig = { enableSearch: true, pagefind: undefined } as any; + + const missingPageConfig = { resultPath: path.join(outputPath, 'nonexistent.html'), searchable: true }; + const pages = [{ pageConfig: missingPageConfig }] as any; + + const warnSpy = jest.spyOn(logger, 'warn').mockImplementation(); + + await generationManager.updatePagefindIndex(pages); + + expect(warnSpy).toHaveBeenCalled(); + warnSpy.mockRestore(); + }); + }); + + describe('Lazy serve pagefind indexing', () => { + beforeEach(() => { + const json = { + ...PAGE_NJK, + 'site.json': SITE_JSON_DEFAULT, + '_site/index.html': 'Test page', + }; + mockFs.vol.fromJSON(json, rootPath); + }); + + test('_rebuildPagesBeingViewed should NOT update index when enableSearch is false', async () => { + generationManager.siteConfig = { enableSearch: false } as any; + const resultPath = path.join(outputPath, 'index.html'); + const pageConfig = { sourcePath: 'index.md', resultPath, searchable: true }; + generationManager.sitePages.pages = [{ pageConfig }] as any; + + const runTaskSpy = jest.spyOn(generationManager, 'runPageGenerationTasks') + .mockResolvedValue(undefined as any); + const writeDataSpy = jest.spyOn(generationManager, 'writeSiteData') + .mockResolvedValue(undefined as any); + const updateIndexSpy = jest.spyOn(generationManager, 'updatePagefindIndex'); + + await generationManager._rebuildPagesBeingViewed(['index']); + + expect(updateIndexSpy).not.toHaveBeenCalled(); + runTaskSpy.mockRestore(); + writeDataSpy.mockRestore(); + updateIndexSpy.mockRestore(); + }); + + test('_rebuildPagesBeingViewed should NOT update index when pagefindIndex is null', async () => { + generationManager.siteConfig = { enableSearch: true } as any; + generationManager.pagefindIndex = null; + const resultPath = path.join(outputPath, 'index.html'); + const pageConfig = { sourcePath: 'index.md', resultPath, searchable: true }; + generationManager.sitePages.pages = [{ pageConfig }] as any; + + const runTaskSpy = jest.spyOn(generationManager, 'runPageGenerationTasks') + .mockResolvedValue(undefined as any); + const writeDataSpy = jest.spyOn(generationManager, 'writeSiteData') + .mockResolvedValue(undefined as any); + const updateIndexSpy = jest.spyOn(generationManager, 'updatePagefindIndex'); + + await generationManager._rebuildPagesBeingViewed(['index']); + + expect(updateIndexSpy).not.toHaveBeenCalled(); + runTaskSpy.mockRestore(); + writeDataSpy.mockRestore(); + updateIndexSpy.mockRestore(); + }); + + test('_backgroundBuild should call indexSiteWithPagefind when enableSearch is true', async () => { + generationManager.siteConfig = { enableSearch: true } as any; + generationManager.toRebuild = new Set(['page1', 'page2']); + + const genPagesSpy = jest.spyOn(generationManager, 'generatePagesMarkedToRebuild') + .mockResolvedValue(true); + const indexSpy = jest.spyOn(generationManager, 'indexSiteWithPagefind') + .mockResolvedValue(true); + + await generationManager._backgroundBuildNotViewedFiles(); + + expect(indexSpy).toHaveBeenCalled(); + genPagesSpy.mockRestore(); + indexSpy.mockRestore(); + }); + + test('_backgroundBuild should NOT call indexSiteWithPagefind when enableSearch is false', async () => { + generationManager.siteConfig = { enableSearch: false } as any; + generationManager.toRebuild = new Set(['page1', 'page2']); + + const genPagesSpy = jest.spyOn(generationManager, 'generatePagesMarkedToRebuild') + .mockResolvedValue(true); + const indexSpy = jest.spyOn(generationManager, 'indexSiteWithPagefind'); + + await generationManager._backgroundBuildNotViewedFiles(); + + expect(indexSpy).not.toHaveBeenCalled(); + genPagesSpy.mockRestore(); + indexSpy.mockRestore(); + }); + }); + test('collectBaseUrl should collect baseurls correctly for sub nested subsites', async () => { const json = { ...PAGE_NJK, diff --git a/packages/core/test/unit/utils/data.ts b/packages/core/test/unit/utils/data.ts index a55a4a6ccb..b9e24e5f87 100644 --- a/packages/core/test/unit/utils/data.ts +++ b/packages/core/test/unit/utils/data.ts @@ -112,6 +112,8 @@ export interface MockIndex { addDirectory: ReturnType; addHTMLFile: ReturnType; writeFiles: ReturnType; + deleteIndex?: ReturnType; + getFiles?: ReturnType; } export interface MockPagefind { @@ -132,6 +134,8 @@ export function createMockIndex( addDirectory: jest.fn().mockResolvedValue(result), addHTMLFile: jest.fn().mockResolvedValue(htmlFileResult), writeFiles: jest.fn().mockResolvedValue(undefined), + deleteIndex: jest.fn().mockResolvedValue(undefined), + getFiles: jest.fn().mockResolvedValue({ files: [] }), }; }