feat/persistent-cache

src/services/index.ts

@@ -6,3 +6,4 @@ export * from './npm-registry'
 export * from './jsdelivr-registry'
 export * from './changelog-fetcher'
 export * from './version-checker'
+export * from './persistent-cache'

src/services/jsdelivr-registry.ts

@@ -2,6 +2,7 @@ import { Pool, request } from 'undici'
 import * as semver from 'semver'
 import { CACHE_TTL, JSDELIVR_CDN_URL, MAX_CONCURRENT_REQUESTS, REQUEST_TIMEOUT } from '../config'
 import { getAllPackageData } from './npm-registry'
+import { persistentCache } from './persistent-cache'
 import { OnBatchReadyCallback } from '../types'
 
 // Create a persistent connection pool for jsDelivr CDN with optimal settings
@@ -129,15 +130,32 @@ export async function getAllPackageDataFromJsdelivr(
   const fetchPackageWithFallback = async (packageName: string): Promise<void> => {
     const currentVersion = currentVersions?.get(packageName)
 
-    // Try to get from cache first
-    const cached = packageCache.get(packageName)
-    if (cached && Date.now() - cached.timestamp < CACHE_TTL) {
-      packageData.set(packageName, cached.data)
+    // Try to get from in-memory cache first (fastest)
+    const memoryCached = packageCache.get(packageName)
+    if (memoryCached && Date.now() - memoryCached.timestamp < CACHE_TTL) {
+      packageData.set(packageName, memoryCached.data)
       completedCount++
       if (onProgress) {
         onProgress(packageName, completedCount, total)
       }
-      addToBatch(packageName, cached.data)
+      addToBatch(packageName, memoryCached.data)
+      return
+    }
+
+    // Try persistent disk cache (fast, survives restarts)
+    const diskCached = persistentCache.get(packageName)
+    if (diskCached) {
+      // Also populate in-memory cache for subsequent accesses
+      packageCache.set(packageName, {
+        data: diskCached,
+        timestamp: Date.now(),
+      })
+      packageData.set(packageName, diskCached)
+      completedCount++
+      if (onProgress) {
+        onProgress(packageName, completedCount, total)
+      }
+      addToBatch(packageName, diskCached)
       return
     }
 
@@ -169,10 +187,13 @@ export async function getAllPackageDataFromJsdelivr(
 
         if (result) {
           packageData.set(packageName, result)
+          // Cache in memory
           packageCache.set(packageName, {
             data: result,
             timestamp: Date.now(),
           })
+          // Cache to disk for persistence
+          persistentCache.set(packageName, result)
           addToBatch(packageName, result)
         }
 
@@ -196,11 +217,13 @@ export async function getAllPackageDataFromJsdelivr(
         allVersions: allVersions.sort(semver.rcompare),
       }
 
-      // Cache the result
+      // Cache the result in memory
       packageCache.set(packageName, {
         data: result,
         timestamp: Date.now(),
       })
+      // Cache to disk for persistence
+      persistentCache.set(packageName, result)
 
       packageData.set(packageName, result)
       completedCount++
@@ -217,10 +240,13 @@ export async function getAllPackageDataFromJsdelivr(
 
         if (result) {
           packageData.set(packageName, result)
+          // Cache in memory
           packageCache.set(packageName, {
             data: result,
             timestamp: Date.now(),
           })
+          // Cache to disk for persistence
+          persistentCache.set(packageName, result)
           addToBatch(packageName, result)
         }
       } catch (npmError) {
@@ -240,6 +266,9 @@ export async function getAllPackageDataFromJsdelivr(
   // Flush any remaining batch items
   flushBatch()
 
+  // Flush persistent cache to disk
+  persistentCache.flush()
+
   // Clear the progress line and show completion time if no custom progress handler
   if (!onProgress) {
     process.stdout.write('\r' + ' '.repeat(80) + '\r')

src/services/npm-registry.ts

@@ -1,5 +1,6 @@
 import * as semver from 'semver'
 import { CACHE_TTL, NPM_REGISTRY_URL, REQUEST_TIMEOUT } from '../config'
+import { persistentCache } from './persistent-cache'
 
 // In-memory cache for package data
 interface CacheEntry {
@@ -15,10 +16,21 @@ const packageCache = new Map<string, CacheEntry>()
 async function fetchPackageFromRegistry(
   packageName: string
 ): Promise<{ latestVersion: string; allVersions: string[] }> {
-  // Check cache first
-  const cached = packageCache.get(packageName)
-  if (cached && Date.now() - cached.timestamp < CACHE_TTL) {
-    return cached.data
+  // Check in-memory cache first (fastest)
+  const memoryCached = packageCache.get(packageName)
+  if (memoryCached && Date.now() - memoryCached.timestamp < CACHE_TTL) {
+    return memoryCached.data
+  }
+
+  // Check persistent disk cache (fast, survives restarts)
+  const diskCached = persistentCache.get(packageName)
+  if (diskCached) {
+    // Also populate in-memory cache for subsequent accesses
+    packageCache.set(packageName, {
+      data: diskCached,
+      timestamp: Date.now(),
+    })
+    return diskCached
   }
 
   try {
@@ -70,11 +82,13 @@ async function fetchPackageFromRegistry(
         allVersions,
       }
 
-      // Cache the result
+      // Cache the result in memory
       packageCache.set(packageName, {
         data: result,
         timestamp: Date.now(),
       })
+      // Cache to disk for persistence
+      persistentCache.set(packageName, result)
 
       return result
     } finally {
@@ -120,6 +134,9 @@ export async function getAllPackageData(
   // Wait for all requests to complete
   await Promise.all(allPromises)
 
+  // Flush persistent cache to disk
+  persistentCache.flush()
+
   // Clear the progress line and show completion time if no custom progress handler
   if (!onProgress) {
     process.stdout.write('\r' + ' '.repeat(80) + '\r')