diff --git a/gateway/sds_gateway/static/css/visualizations/waterfall.css b/gateway/sds_gateway/static/css/visualizations/waterfall.css index 0fedf7db..dfd773bc 100644 --- a/gateway/sds_gateway/static/css/visualizations/waterfall.css +++ b/gateway/sds_gateway/static/css/visualizations/waterfall.css @@ -54,6 +54,21 @@ margin-right: 0; } +.waterfall-loading-overlay { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + background-color: rgba(255, 255, 255, 0.8); + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + z-index: 10; + backdrop-filter: blur(2px); +} + #waterfallOverlayCanvas { position: absolute; top: 0; @@ -65,7 +80,7 @@ /* Button states */ .btn:disabled { opacity: 0.6; - cursor: not-allowed; + cursor: default; } /* Loading states */ diff --git a/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallAPIClient.js b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallAPIClient.js new file mode 100644 index 00000000..1ca529e6 --- /dev/null +++ b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallAPIClient.js @@ -0,0 +1,156 @@ +/** + * Waterfall API Client + * Handles all API requests for waterfall visualization + */ + +class WaterfallAPIClient { + constructor(captureUuid) { + this.captureUuid = captureUuid; + } + + /** + * Get CSRF token from form input + */ + getCSRFToken() { + const token = document.querySelector("[name=csrfmiddlewaretoken]"); + return token ? token.value : ""; + } + + /** + * Get create waterfall endpoint URL + */ + _getCreateWaterfallEndpoint() { + return `/api/v1/visualizations/${this.captureUuid}/create_waterfall/`; + } + + /** + * Get waterfall status endpoint URL + */ + _getWaterfallStatusEndpoint(jobId) { + return `/api/v1/visualizations/${this.captureUuid}/waterfall_status/?job_id=${jobId}`; + } + + /** + * Get waterfall metadata endpoint URL + */ + _getWaterfallMetadataEndpoint(jobId) { + return `/api/v1/visualizations/${this.captureUuid}/waterfall_metadata/?job_id=${jobId}`; + } + + /** + * Get waterfall result endpoint URL + */ + _getWaterfallResultEndpoint(jobId, startIndex = null, endIndex = null) { + let url = `/api/v1/visualizations/${this.captureUuid}/download_waterfall/?job_id=${jobId}`; + const params = new URLSearchParams(); + if (startIndex !== null) { + params.append("start_index", startIndex); + } + if (endIndex !== null) { + params.append("end_index", endIndex); + } + const queryString = params.toString(); + if (queryString) { + url += `&${queryString}`; + } + return url; + } + + /** + * Get post-processing status for a capture + */ + async getPostProcessingStatus() { + const response = await fetch( + `/api/latest/assets/captures/${this.captureUuid}/post_processing_status/`, + ); + + if (!response.ok) { + throw new Error( + `Failed to get post-processing status: ${response.status}`, + ); + } + + return await response.json(); + } + + /** + * Get waterfall metadata + */ + async getWaterfallMetadata(jobId) { + const response = await fetch(this._getWaterfallMetadataEndpoint(jobId), { + headers: { + "X-CSRFToken": this.getCSRFToken(), + }, + }); + + if (!response.ok) { + throw new Error(`Failed to load waterfall metadata: ${response.status}`); + } + + return await response.json(); + } + + /** + * Load a range of waterfall slices + */ + async loadWaterfallRange(jobId, startIndex, endIndex) { + const response = await fetch( + this._getWaterfallResultEndpoint(jobId, startIndex, endIndex), + { + headers: { + "X-CSRFToken": this.getCSRFToken(), + }, + }, + ); + + if (!response.ok) { + throw new Error(`Failed to load waterfall range: ${response.status}`); + } + + const result = await response.json(); + return result.data || []; + } + + /** + * Create a waterfall processing job + */ + async createWaterfallJob() { + const response = await fetch(this._getCreateWaterfallEndpoint(), { + method: "POST", + headers: { + "X-CSRFToken": this.getCSRFToken(), + }, + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + const data = await response.json(); + + if (!data.uuid) { + throw new Error("Waterfall job ID not found"); + } + + return data; + } + + /** + * Get waterfall job status + */ + async getWaterfallJobStatus(jobId) { + const response = await fetch(this._getWaterfallStatusEndpoint(jobId), { + headers: { + "X-CSRFToken": this.getCSRFToken(), + }, + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + return await response.json(); + } +} + +export { WaterfallAPIClient }; diff --git a/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallCacheManager.js b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallCacheManager.js new file mode 100644 index 00000000..9cdc6075 --- /dev/null +++ b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallCacheManager.js @@ -0,0 +1,153 @@ +/** + * Waterfall Cache Manager + * Manages loading and caching of waterfall slice data + */ + +export class WaterfallCacheManager { + constructor(totalSlices) { + this.totalSlices = totalSlices; + this.loadedSlices = new Set(); // Track which slice indices are loaded + this.loadingRanges = new Set(); // Track which ranges are currently loading + this.sliceData = new Map(); // Map of slice index -> parsed slice data + this.maxConcurrentLoads = 3; + } + + /** + * Check if a specific slice is loaded + */ + isSliceLoaded(index) { + return this.loadedSlices.has(index); + } + + /** + * Check if a range of slices is fully loaded + */ + isRangeLoaded(startIndex, endIndex) { + for (let i = startIndex; i < endIndex; i++) { + if (i >= this.totalSlices) break; + if (!this.loadedSlices.has(i)) { + return false; + } + } + return true; + } + + /** + * Get loaded slices for a range (may include nulls for unloaded slices) + */ + getRangeSlices(startIndex, endIndex) { + const slices = []; + for (let i = startIndex; i < endIndex; i++) { + if (i >= this.totalSlices) break; + slices.push(this.sliceData.get(i) || null); + } + return slices; + } + + /** + * Add loaded slices to the cache + * + * @param {number} startIndex + * @param {Array} parsedSlices - Array of parsed slice data + */ + addLoadedSlices(startIndex, slices) { + for (let i = 0; i < slices.length; i++) { + const sliceIndex = startIndex + i; + if (sliceIndex >= this.totalSlices) break; + + this.sliceData.set(sliceIndex, slices[i]); + this.loadedSlices.add(sliceIndex); + } + } + + /** + * Get missing ranges that need to be loaded + */ + getMissingRanges(startIndex, endIndex) { + const missingRanges = []; + let rangeStart = null; + + for (let i = startIndex; i < endIndex; i++) { + if (i >= this.totalSlices) break; + + if (!this.loadedSlices.has(i)) { + if (rangeStart === null) { + rangeStart = i; + } + } else { + if (rangeStart !== null) { + // End of missing range + missingRanges.push([rangeStart, i]); + rangeStart = null; + } + } + } + + if (rangeStart !== null) { + missingRanges.push([rangeStart, endIndex]); + } + + return missingRanges; + } + + /** + * Check if an exact range is currently being loaded + */ + isRangeLoading(startIndex, endIndex) { + const rangeKey = `${startIndex}-${endIndex}`; + return this.loadingRanges.has(rangeKey); + } + + /** + * Check if the given range is fully contained by any ranges that are currently being loaded + */ + isRangeContainedByLoadingRange(startIndex, endIndex) { + for (const rangeKey of this.loadingRanges) { + const [loadingStart, loadingEnd] = rangeKey.split("-").map(Number); + + if (startIndex >= loadingStart && endIndex <= loadingEnd) { + return true; + } + } + return false; + } + + /** + * Mark a range as loading + */ + markRangeLoading(startIndex, endIndex) { + const rangeKey = `${startIndex}-${endIndex}`; + this.loadingRanges.add(rangeKey); + } + + /** + * Mark a range as finished loading + */ + markRangeLoaded(startIndex, endIndex) { + const rangeKey = `${startIndex}-${endIndex}`; + this.loadingRanges.delete(rangeKey); + } + + /** + * Get the number of loaded slices + */ + getLoadedCount() { + return this.loadedSlices.size; + } + + /** + * Clear all cached data + */ + clear() { + this.loadedSlices.clear(); + this.loadingRanges.clear(); + this.sliceData.clear(); + } + + /** + * Update total slices (e.g., when metadata is loaded) + */ + setTotalSlices(totalSlices) { + this.totalSlices = totalSlices; + } +} diff --git a/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallControls.js b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallControls.js index 87fd28c2..ac675725 100644 --- a/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallControls.js +++ b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallControls.js @@ -15,6 +15,7 @@ class WaterfallControls { this.totalSlices = 0; this.waterfallWindowStart = 0; this.hoveredSliceIndex = null; + this.isLoading = false; // Constants this.WATERFALL_WINDOW_SIZE = WATERFALL_WINDOW_SIZE; @@ -244,18 +245,28 @@ class WaterfallControls { scrollIndicatorBelow.classList.toggle("visible", canScrollDown); } - // Update button states + // Update button states if not disabled if (scrollUpBtn) { - scrollUpBtn.disabled = !canScrollUp; - scrollUpBtn.classList.toggle("disabled", !canScrollUp); + const shouldDisable = !canScrollUp || this.isLoading; + scrollUpBtn.disabled = shouldDisable; + scrollUpBtn.classList.toggle("disabled", shouldDisable); } if (scrollDownBtn) { - scrollDownBtn.disabled = !canScrollDown; - scrollDownBtn.classList.toggle("disabled", !canScrollDown); + const shouldDisable = !canScrollDown || this.isLoading; + scrollDownBtn.disabled = shouldDisable; + scrollDownBtn.classList.toggle("disabled", shouldDisable); } } + /** + * Set loading state for scroll buttons + */ + setLoading(isLoading) { + this.isLoading = isLoading; + this.updateScrollIndicators(); + } + /** * Start playback animation */ @@ -430,6 +441,8 @@ class WaterfallControls { } handleScrollUp() { + if (this.isLoading) return; + // Move the window up to show more recent slices const newWindowStart = Math.min( this.totalSlices - this.WATERFALL_WINDOW_SIZE, @@ -459,6 +472,8 @@ class WaterfallControls { } handleScrollDown() { + if (this.isLoading) return; + // Move the window down to show older slices const newWindowStart = Math.max( 0, diff --git a/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallRenderer.js b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallRenderer.js index 7b3e2b05..77b08759 100644 --- a/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallRenderer.js +++ b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallRenderer.js @@ -105,12 +105,23 @@ class WaterfallRenderer { /** * Render the waterfall plot + * waterfallData: Array of slices + * totalSlices: Total number of slices in the dataset + * startSliceIndex: The actual slice index of the first slice in waterfallData */ - renderWaterfall(waterfallData, totalSlices) { - if (!this.ctx || !this.canvas || waterfallData.length === 0) return; + renderWaterfall(waterfallData, totalSlices, startSliceIndex) { + if ( + !this.ctx || + !this.canvas || + !waterfallData || + waterfallData.length === 0 + ) { + return; + } - // Store total slices for overlay updates + // Store total slices and starting index for overlay updates this.totalSlices = totalSlices; + this.waterfallWindowStart = startSliceIndex; // Clear canvas this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height); @@ -118,24 +129,23 @@ class WaterfallRenderer { // Calculate dimensions with margins const plotHeight = this.canvas.height - this.TOP_MARGIN - this.BOTTOM_MARGIN; - const maxVisibleSlices = Math.min(totalSlices, this.WATERFALL_WINDOW_SIZE); + const maxVisibleSlices = Math.min( + waterfallData.length, + this.WATERFALL_WINDOW_SIZE, + ); const sliceHeight = plotHeight / maxVisibleSlices; - // Calculate which slices to display - const startSliceIndex = this.waterfallWindowStart; - // Draw waterfall slices from bottom to top - for (let i = 0; i < this.WATERFALL_WINDOW_SIZE; i++) { - const sliceIndex = startSliceIndex + i; - if (sliceIndex >= totalSlices) break; + for (let i = 0; i < waterfallData.length && i < maxVisibleSlices; i++) { + const slice = waterfallData[i]; + if (!slice || !slice.data) { + continue; + } - const slice = waterfallData[sliceIndex]; - if (slice?.data) { - // Calculate Y position - const y = this.BOTTOM_MARGIN + (maxVisibleSlices - 1 - i) * sliceHeight; + // Calculate Y position + const y = this.BOTTOM_MARGIN + (maxVisibleSlices - 1 - i) * sliceHeight; - this.drawWaterfallSlice(slice.data, y, sliceHeight, this.canvas.width); - } + this.drawWaterfallSlice(slice.data, y, sliceHeight, this.canvas.width); } // Update the overlay with highlights and index legend diff --git a/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallVisualization.js b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallVisualization.js index aed6e53b..214fac28 100644 --- a/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallVisualization.js +++ b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallVisualization.js @@ -4,12 +4,14 @@ */ import { generateErrorMessage, setupErrorDisplay } from "../errorHandler.js"; +import { WaterfallAPIClient } from "./WaterfallAPIClient.js"; +import { WaterfallCacheManager } from "./WaterfallCacheManager.js"; import { DEFAULT_COLOR_MAP, ERROR_MESSAGES, - get_create_waterfall_endpoint, - get_waterfall_result_endpoint, - get_waterfall_status_endpoint, + PREFETCH_DISTANCE, + PREFETCH_TRIGGER, + WATERFALL_WINDOW_SIZE, } from "./constants.js"; class WaterfallVisualization { @@ -22,13 +24,18 @@ class WaterfallVisualization { this.controls = null; // Data state - this.waterfallData = []; - this.parsedWaterfallData = []; // Cache parsed data to avoid re-parsing this.totalSlices = 0; this.scaleMin = null; this.scaleMax = null; this.isLoading = false; + // Cache manager for slice loading + this.cacheManager = null; + this.jobId = null; // Store job ID for range requests + + // API client + this.apiClient = new WaterfallAPIClient(captureUuid); + // Processing state this.isGenerating = false; this.currentJobId = null; @@ -102,6 +109,8 @@ class WaterfallVisualization { this.waterfallRenderer.setWaterfallWindowStart(waterfallWindowStart); if (windowChanged) { + // Check if we need to load more slices for the new window + this.ensureSlicesLoaded(waterfallWindowStart); this.renderWaterfall(); } @@ -188,7 +197,7 @@ class WaterfallVisualization { } // Re-render if we have data - if (this.parsedWaterfallData && this.parsedWaterfallData.length > 0) { + if (this.cacheManager && this.totalSlices > 0) { this.render(); } } @@ -197,7 +206,7 @@ class WaterfallVisualization { * Render the waterfall visualization */ render() { - if (!this.parsedWaterfallData || this.parsedWaterfallData.length === 0) { + if (this.totalSlices === 0) { return; } @@ -207,38 +216,108 @@ class WaterfallVisualization { // Show visualization components this.showVisualizationComponents(); + // Ensure slices for current window are loaded + this.ensureSlicesLoaded(this.waterfallWindowStart); + // Render waterfall with cached parsed data this.renderWaterfall(); - // Render periodogram - this.renderPeriodogram(); + // Render periodogram (only if slice is loaded) + if ( + this.cacheManager && + this.currentSliceIndex >= 0 && + this.currentSliceIndex < this.totalSlices && + this.cacheManager.sliceData.get(this.currentSliceIndex) + ) { + this.renderPeriodogram(); + } } /** * Render the periodogram chart */ renderPeriodogram() { - if (!this.periodogramChart || this.parsedWaterfallData.length === 0) return; - this.periodogramChart.renderPeriodogram( - this.parsedWaterfallData[this.currentSliceIndex], - ); + if (!this.periodogramChart || !this.cacheManager) return; + + // Get slice from cache manager + const slice = this.cacheManager.sliceData.get(this.currentSliceIndex); + if (!slice || !slice.data) { + return; // Slice not loaded yet + } + + this.periodogramChart.renderPeriodogram(slice); } /** * Render only the waterfall plot (for color map changes) */ renderWaterfall() { - if (!this.parsedWaterfallData || this.parsedWaterfallData.length === 0) { + if (this.totalSlices === 0 || !this.cacheManager) { + return; + } + + // Check if window is fully loaded + const isFullyLoaded = this.isWindowFullyLoaded(); + + if (!isFullyLoaded) { + // Show loading overlay if window is not fully loaded + this.showLoadingOverlay(); + return; + } + + // Hide loading overlay + this.hideLoadingOverlay(); + + // Get slices for the current window + const waterfallWindowEnd = Math.min( + this.waterfallWindowStart + WATERFALL_WINDOW_SIZE, + this.totalSlices, + ); + + // Since window is fully loaded, all slices should be available + const loadedSlices = this.cacheManager.getRangeSlices( + this.waterfallWindowStart, + waterfallWindowEnd, + ); + if (loadedSlices.some((slice) => slice === null)) { + this.showError("Failed to load waterfall data"); return; } - // Render waterfall with cached parsed data - let renderer handle slicing + // Render with fully-loaded data this.waterfallRenderer.renderWaterfall( - this.parsedWaterfallData, + loadedSlices, this.totalSlices, + this.waterfallWindowStart, ); } + /** + * Show loading overlay + */ + showLoadingOverlay() { + const overlay = document.getElementById("waterfallLoadingOverlay"); + if (overlay) { + overlay.classList.remove("d-none"); + } + if (this.controls) { + this.controls.setLoading(true); + } + } + + /** + * Hide loading overlay + */ + hideLoadingOverlay() { + const overlay = document.getElementById("waterfallLoadingOverlay"); + if (overlay) { + overlay.classList.add("d-none"); + } + if (this.controls) { + this.controls.setLoading(false); + } + } + /** * Update only slice highlights without re-rendering everything */ @@ -404,16 +483,7 @@ class WaterfallVisualization { this.showLoading(true); // First, get the post-processing status to check if waterfall data is available - const statusResponse = await fetch( - `/api/latest/assets/captures/${this.captureUuid}/post_processing_status/`, - ); - if (!statusResponse.ok) { - throw new Error( - `Failed to get post-processing status: ${statusResponse.status}`, - ); - } - - const statusData = await statusResponse.json(); + const statusData = await this.apiClient.getPostProcessingStatus(); const waterfallData = statusData.post_processed_data.find( (data) => data.processing_type === "waterfall" && @@ -426,39 +496,25 @@ class WaterfallVisualization { return; } - // Get the waterfall data file - const dataResponse = await fetch( - `/api/latest/assets/captures/${this.captureUuid}/download_post_processed_data/?processing_type=waterfall`, - ); + // Store job ID for range requests + this.jobId = waterfallData.uuid; - if (!dataResponse.ok) { - throw new Error( - `Failed to download waterfall data: ${dataResponse.status}`, - ); - } + // Get metadata first to know total slices and power bounds + await this.loadWaterfallMetadata(); - const waterfallJson = await dataResponse.json(); - this.waterfallData = waterfallJson; - this.totalSlices = waterfallJson.length; - - // Parse all waterfall data once and cache it - this.parsedWaterfallData = this.waterfallData.map((slice) => ({ - ...slice, - data: this.parseWaterfallData(slice.data), - })); + // Initialize cache manager + this.cacheManager = new WaterfallCacheManager(this.totalSlices); - // Calculate power bounds from all data - this.calculatePowerBounds(); + // Load initial window of slices + await this.loadWaterfallRange( + 0, + Math.min(WATERFALL_WINDOW_SIZE, this.totalSlices), + ); this.isLoading = false; this.showLoading(false); - this.controls.setTotalSlices(this.totalSlices); - this.waterfallRenderer.setScaleBounds(this.scaleMin, this.scaleMax); - this.periodogramChart.updateYAxisBounds(this.scaleMin, this.scaleMax); - this.updateColorLegend(); - - this.render(); + // Initial render will happen after first range loads } catch (error) { console.error("Failed to load waterfall data:", error); this.isLoading = false; @@ -479,6 +535,185 @@ class WaterfallVisualization { } } + /** + * Load waterfall metadata + */ + async loadWaterfallMetadata() { + if (!this.jobId) { + throw new Error("Job ID not available"); + } + + const metadata = await this.apiClient.getWaterfallMetadata(this.jobId); + this.totalSlices = metadata.slices_processed || 0; + this.scaleMin = metadata.power_scale_min ?? -130.0; + this.scaleMax = metadata.power_scale_max ?? 0.0; + + // Update visualization bounds + if (this.waterfallRenderer) { + this.waterfallRenderer.setScaleBounds(this.scaleMin, this.scaleMax); + } + if (this.periodogramChart) { + this.periodogramChart.updateYAxisBounds(this.scaleMin, this.scaleMax); + } + if (this.controls) { + this.controls.setTotalSlices(this.totalSlices); + } + this.updateColorLegend(); + } + + /** + * Load a range of waterfall slices + */ + async loadWaterfallRange(startIndex, endIndex) { + if (!this.jobId || !this.cacheManager) { + throw new Error("Job ID or cache manager not available"); + } + + // Clamp indices + const clampedStart = Math.max(0, startIndex); + const clampedEnd = Math.min(endIndex, this.totalSlices); + + if (clampedStart >= clampedEnd) { + return; + } + + // Check if this range is already loaded or loading + if (this.cacheManager.isRangeLoading(clampedStart, clampedEnd)) { + return; // Already loading this range + } + + if (this.cacheManager.isRangeLoaded(clampedStart, clampedEnd)) { + return; // Already loaded + } + + this.cacheManager.markRangeLoading(clampedStart, clampedEnd); + + try { + const slices = await this.apiClient.loadWaterfallRange( + this.jobId, + clampedStart, + clampedEnd, + ); + + // Parse slices + const parsedSlices = slices.map((slice) => ({ + ...slice, + data: this.parseWaterfallData(slice.data), + })); + + // Store in cache manager + this.cacheManager.addLoadedSlices(clampedStart, parsedSlices); + + // Trigger render + this.render(); + } finally { + this.cacheManager.markRangeLoaded(clampedStart, clampedEnd); + } + } + + /** + * Ensure slices for a given window or slice index are loaded + * Uses separate prefetch trigger threshold and prefetch distance: + * - Trigger threshold: Only prefetch when within PREFETCH_TRIGGER of unfetched data + * - Prefetch distance: Once triggered, load up to PREFETCH_DISTANCE on both sides + */ + async ensureSlicesLoaded(windowStart) { + if (this.totalSlices === 0 || !this.jobId || !this.cacheManager) { + return; + } + + const windowEnd = windowStart + WATERFALL_WINDOW_SIZE; + + // Prefetch trigger threshold: check the range around the window + const triggerStart = Math.max(windowStart - PREFETCH_TRIGGER, 0); + const triggerEnd = Math.min(windowEnd + PREFETCH_TRIGGER, this.totalSlices); + + // Check if there's any missing data within the trigger threshold + const missingRangesInTrigger = this.cacheManager.getMissingRanges( + triggerStart, + triggerEnd, + ); + + // Only proceed with prefetching if we're approaching unfetched data + if (missingRangesInTrigger.length === 0) { + // No missing data within trigger threshold, no need to prefetch + return; + } + + // Check if all missing ranges in the trigger area are already being loaded + const allMissingRangesAreLoading = missingRangesInTrigger.every( + ([missingStart, missingEnd]) => + this.cacheManager.isRangeContainedByLoadingRange( + missingStart, + missingEnd, + ), + ); + + if (allMissingRangesAreLoading) { + // All missing ranges in trigger area are already being loaded, don't start a new request + return; + } + + // Prefetch distance: load up to PREFETCH_DISTANCE on both sides when triggered + const prefetchStart = Math.max(windowStart - PREFETCH_DISTANCE, 0); + const prefetchEnd = Math.min( + windowEnd + PREFETCH_DISTANCE, + this.totalSlices, + ); + + // Get all missing ranges in the prefetch range + const missingRanges = this.cacheManager.getMissingRanges( + prefetchStart, + prefetchEnd, + ); + + // Load missing ranges (limit concurrent requests) + const maxConcurrent = this.cacheManager.maxConcurrentLoads; + let activeLoads = 0; + const loadQueue = [...missingRanges]; + + const processQueue = async () => { + while (loadQueue.length > 0 || activeLoads > 0) { + if (activeLoads < maxConcurrent && loadQueue.length > 0) { + const [start, end] = loadQueue.shift(); + activeLoads++; + this.loadWaterfallRange(start, end) + .catch((error) => { + console.error(`Error loading range ${start}-${end}:`, error); + }) + .finally(() => { + activeLoads--; + }); + } else { + // Wait a bit before checking again + await new Promise((resolve) => setTimeout(resolve, 50)); + } + } + }; + + // Start loading (don't await to avoid blocking) + processQueue(); + } + + /** + * Check if the current window is fully loaded + */ + isWindowFullyLoaded() { + if (this.totalSlices === 0 || !this.cacheManager) { + return false; + } + + const waterfallWindowEnd = Math.min( + this.waterfallWindowStart + WATERFALL_WINDOW_SIZE, + this.totalSlices, + ); + + return this.cacheManager.isRangeLoaded( + this.waterfallWindowStart, + waterfallWindowEnd, + ); + } + /** * Trigger waterfall processing when data is not available */ @@ -505,25 +740,7 @@ class WaterfallVisualization { */ async createWaterfallJob() { try { - const response = await fetch( - get_create_waterfall_endpoint(this.captureUuid), - { - method: "POST", - headers: { - "X-CSRFToken": this.getCSRFToken(), - }, - }, - ); - - if (!response.ok) { - throw new Error(`HTTP ${response.status}: ${response.statusText}`); - } - - const data = await response.json(); - - if (!data.uuid) { - throw new Error("Waterfall job ID not found"); - } + const data = await this.apiClient.createWaterfallJob(); this.currentJobId = data.uuid; // Start polling for status @@ -554,21 +771,10 @@ class WaterfallVisualization { if (!this.currentJobId) return; try { - const response = await fetch( - get_waterfall_status_endpoint(this.captureUuid, this.currentJobId), - { - headers: { - "X-CSRFToken": this.getCSRFToken(), - }, - }, + const data = await this.apiClient.getWaterfallJobStatus( + this.currentJobId, ); - if (!response.ok) { - throw new Error(`HTTP ${response.status}: ${response.statusText}`); - } - - const data = await response.json(); - if (data.processing_status === "completed") { // Job completed, stop polling and fetch result this.stopStatusPolling(); @@ -601,41 +807,24 @@ class WaterfallVisualization { */ async fetchWaterfallResult() { try { - const response = await fetch( - get_waterfall_result_endpoint(this.captureUuid, this.currentJobId), - { - headers: { - "X-CSRFToken": this.getCSRFToken(), - }, - }, - ); - - if (!response.ok) { - throw new Error(`HTTP ${response.status}: ${response.statusText}`); - } - - const waterfallJson = await response.json(); + // Store job ID for range requests + this.jobId = this.currentJobId; - this.waterfallData = waterfallJson; - this.totalSlices = waterfallJson.length; + // Get metadata first to know total slices and power bounds + await this.loadWaterfallMetadata(); - // Parse all waterfall data once and cache it - this.parsedWaterfallData = this.waterfallData.map((slice) => ({ - ...slice, - data: this.parseWaterfallData(slice.data), - })); + // Initialize cache manager + this.cacheManager = new WaterfallCacheManager(this.totalSlices); - // Calculate power bounds from all data - this.calculatePowerBounds(); + // Load initial window of slices + await this.loadWaterfallRange( + 0, + Math.min(WATERFALL_WINDOW_SIZE, this.totalSlices), + ); this.setGeneratingState(false); - this.controls.setTotalSlices(this.totalSlices); - this.waterfallRenderer.setScaleBounds(this.scaleMin, this.scaleMax); - this.periodogramChart.updateYAxisBounds(this.scaleMin, this.scaleMax); - this.updateColorLegend(); - - this.render(); + // Initial render will happen after first range loads } catch (error) { console.error("Error fetching waterfall result:", error); this.showError("Failed to fetch waterfall result"); @@ -654,14 +843,6 @@ class WaterfallVisualization { this.showLoading(isGenerating); } - /** - * Get CSRF token from form input - */ - getCSRFToken() { - const token = document.querySelector("[name=csrfmiddlewaretoken]"); - return token ? token.value : ""; - } - /** * Parse base64 waterfall data */ @@ -681,59 +862,6 @@ class WaterfallVisualization { } } - /** - * Calculate power bounds from all waterfall data - */ - calculatePowerBounds() { - if (this.parsedWaterfallData.length === 0) { - // Fallback to default bounds if no data - this.scaleMin = -130; - this.scaleMax = 0; - return; - } - - let globalMin = Number.POSITIVE_INFINITY; - let globalMax = Number.NEGATIVE_INFINITY; - - // Iterate through all parsed slices to find global min/max - for (const slice of this.parsedWaterfallData) { - if (slice.data && slice.data.length > 0) { - const sliceMin = Math.min(...slice.data); - const sliceMax = Math.max(...slice.data); - - globalMin = Math.min(globalMin, sliceMin); - globalMax = Math.max(globalMax, sliceMax); - } - } - - // If we found valid data, use it; otherwise fall back to defaults - if ( - globalMin !== Number.POSITIVE_INFINITY && - globalMax !== Number.NEGATIVE_INFINITY - ) { - // Add a small margin (5%) to the bounds for better visualization - const margin = (globalMax - globalMin) * 0.05; - this.scaleMin = globalMin - margin; - this.scaleMax = globalMax + margin; - } else { - // Fallback to default bounds - this.scaleMin = -130; - this.scaleMax = 0; - } - } - - /** - * Get waterfall data for a specific range - */ - getWaterfallRange(startIndex, endIndex) { - const start = Math.max(0, startIndex); - const end = Math.min(this.totalSlices, endIndex); - - if (start >= end) return []; - - return this.parsedWaterfallData.slice(start, end); - } - /** * Show/hide loading state */ @@ -826,8 +954,9 @@ class WaterfallVisualization { */ showError(message, errorDetail = null) { // Clear data state first - this.waterfallData = []; - this.parsedWaterfallData = []; + if (this.cacheManager) { + this.cacheManager.clear(); + } this.totalSlices = 0; this.scaleMin = null; this.scaleMax = null; diff --git a/gateway/sds_gateway/static/js/visualizations/waterfall/constants.js b/gateway/sds_gateway/static/js/visualizations/waterfall/constants.js index 6b1cb965..0fc06614 100644 --- a/gateway/sds_gateway/static/js/visualizations/waterfall/constants.js +++ b/gateway/sds_gateway/static/js/visualizations/waterfall/constants.js @@ -22,21 +22,12 @@ export const CANVASJS_RIGHT_MARGIN = 10; export const WATERFALL_TOP_MARGIN = 5; export const WATERFALL_BOTTOM_MARGIN = 5; -// Window size constants +// Waterfall window constants export const WATERFALL_WINDOW_SIZE = 100; - -// API endpoints -export const get_create_waterfall_endpoint = (capture_uuid) => { - return `/api/v1/visualizations/${capture_uuid}/create_waterfall/`; -}; - -export const get_waterfall_status_endpoint = (capture_uuid, job_id) => { - return `/api/v1/visualizations/${capture_uuid}/waterfall_status/?job_id=${job_id}`; -}; - -export const get_waterfall_result_endpoint = (capture_uuid, job_id) => { - return `/api/v1/visualizations/${capture_uuid}/download_waterfall/?job_id=${job_id}`; -}; +// How close to unfetched data to trigger a prefetch +export const PREFETCH_TRIGGER = 2 * WATERFALL_WINDOW_SIZE; +// How much data around the current window to prefetch +export const PREFETCH_DISTANCE = 4 * WATERFALL_WINDOW_SIZE; export const ERROR_MESSAGES = { NO_CAPTURE: "No capture data found", diff --git a/gateway/sds_gateway/templates/visualizations/waterfall.html b/gateway/sds_gateway/templates/visualizations/waterfall.html index ecfe0be2..039c0b4b 100644 --- a/gateway/sds_gateway/templates/visualizations/waterfall.html +++ b/gateway/sds_gateway/templates/visualizations/waterfall.html @@ -127,6 +127,12 @@