From 8d8400568923938abfd89239738e2ff453165018 Mon Sep 17 00:00:00 2001 From: Connor Howington Date: Mon, 3 Nov 2025 18:22:48 -0500 Subject: [PATCH 1/5] Implement waterfall data streaming --- .../static/css/visualizations/waterfall.css | 15 + .../waterfall/WaterfallAPIClient.js | 156 +++++++ .../waterfall/WaterfallCacheManager.js | 139 ++++++ .../waterfall/WaterfallRenderer.js | 42 +- .../waterfall/WaterfallVisualization.js | 428 +++++++++++------- .../js/visualizations/waterfall/constants.js | 13 - .../templates/visualizations/waterfall.html | 6 + .../sds_gateway/visualizations/api_views.py | 217 ++++++++- .../visualizations/processing/waterfall.py | 23 +- 9 files changed, 819 insertions(+), 220 deletions(-) create mode 100644 gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallAPIClient.js create mode 100644 gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallCacheManager.js diff --git a/gateway/sds_gateway/static/css/visualizations/waterfall.css b/gateway/sds_gateway/static/css/visualizations/waterfall.css index 0fedf7db..b87d67e7 100644 --- a/gateway/sds_gateway/static/css/visualizations/waterfall.css +++ b/gateway/sds_gateway/static/css/visualizations/waterfall.css @@ -54,6 +54,21 @@ margin-right: 0; } +.waterfall-loading-overlay { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + background-color: rgba(255, 255, 255, 0.8); + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + z-index: 10; + backdrop-filter: blur(2px); +} + #waterfallOverlayCanvas { position: absolute; top: 0; diff --git a/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallAPIClient.js b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallAPIClient.js new file mode 100644 index 00000000..1ca529e6 --- /dev/null +++ b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallAPIClient.js @@ -0,0 +1,156 @@ +/** + * Waterfall API Client + * Handles all API requests for waterfall visualization + */ + +class WaterfallAPIClient { + constructor(captureUuid) { + this.captureUuid = captureUuid; + } + + /** + * Get CSRF token from form input + */ + getCSRFToken() { + const token = document.querySelector("[name=csrfmiddlewaretoken]"); + return token ? token.value : ""; + } + + /** + * Get create waterfall endpoint URL + */ + _getCreateWaterfallEndpoint() { + return `/api/v1/visualizations/${this.captureUuid}/create_waterfall/`; + } + + /** + * Get waterfall status endpoint URL + */ + _getWaterfallStatusEndpoint(jobId) { + return `/api/v1/visualizations/${this.captureUuid}/waterfall_status/?job_id=${jobId}`; + } + + /** + * Get waterfall metadata endpoint URL + */ + _getWaterfallMetadataEndpoint(jobId) { + return `/api/v1/visualizations/${this.captureUuid}/waterfall_metadata/?job_id=${jobId}`; + } + + /** + * Get waterfall result endpoint URL + */ + _getWaterfallResultEndpoint(jobId, startIndex = null, endIndex = null) { + let url = `/api/v1/visualizations/${this.captureUuid}/download_waterfall/?job_id=${jobId}`; + const params = new URLSearchParams(); + if (startIndex !== null) { + params.append("start_index", startIndex); + } + if (endIndex !== null) { + params.append("end_index", endIndex); + } + const queryString = params.toString(); + if (queryString) { + url += `&${queryString}`; + } + return url; + } + + /** + * Get post-processing status for a capture + */ + async getPostProcessingStatus() { + const response = await fetch( + `/api/latest/assets/captures/${this.captureUuid}/post_processing_status/`, + ); + + if (!response.ok) { + throw new Error( + `Failed to get post-processing status: ${response.status}`, + ); + } + + return await response.json(); + } + + /** + * Get waterfall metadata + */ + async getWaterfallMetadata(jobId) { + const response = await fetch(this._getWaterfallMetadataEndpoint(jobId), { + headers: { + "X-CSRFToken": this.getCSRFToken(), + }, + }); + + if (!response.ok) { + throw new Error(`Failed to load waterfall metadata: ${response.status}`); + } + + return await response.json(); + } + + /** + * Load a range of waterfall slices + */ + async loadWaterfallRange(jobId, startIndex, endIndex) { + const response = await fetch( + this._getWaterfallResultEndpoint(jobId, startIndex, endIndex), + { + headers: { + "X-CSRFToken": this.getCSRFToken(), + }, + }, + ); + + if (!response.ok) { + throw new Error(`Failed to load waterfall range: ${response.status}`); + } + + const result = await response.json(); + return result.data || []; + } + + /** + * Create a waterfall processing job + */ + async createWaterfallJob() { + const response = await fetch(this._getCreateWaterfallEndpoint(), { + method: "POST", + headers: { + "X-CSRFToken": this.getCSRFToken(), + }, + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + const data = await response.json(); + + if (!data.uuid) { + throw new Error("Waterfall job ID not found"); + } + + return data; + } + + /** + * Get waterfall job status + */ + async getWaterfallJobStatus(jobId) { + const response = await fetch(this._getWaterfallStatusEndpoint(jobId), { + headers: { + "X-CSRFToken": this.getCSRFToken(), + }, + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + return await response.json(); + } +} + +export { WaterfallAPIClient }; diff --git a/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallCacheManager.js b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallCacheManager.js new file mode 100644 index 00000000..0df455db --- /dev/null +++ b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallCacheManager.js @@ -0,0 +1,139 @@ +/** + * Waterfall Cache Manager + * Manages loading and caching of waterfall slice data + */ + +export class WaterfallCacheManager { + constructor(totalSlices) { + this.totalSlices = totalSlices; + this.loadedSlices = new Set(); // Track which slice indices are loaded + this.loadingRanges = new Set(); // Track which ranges are currently loading + this.sliceData = new Map(); // Map of slice index -> parsed slice data + this.maxConcurrentLoads = 3; + } + + /** + * Check if a specific slice is loaded + */ + isSliceLoaded(index) { + return this.loadedSlices.has(index); + } + + /** + * Check if a range of slices is fully loaded + */ + isRangeLoaded(startIndex, endIndex) { + for (let i = startIndex; i < endIndex; i++) { + if (i >= this.totalSlices) break; + if (!this.loadedSlices.has(i)) { + return false; + } + } + return true; + } + + /** + * Get loaded slices for a range (may include nulls for unloaded slices) + */ + getRangeSlices(startIndex, endIndex) { + const slices = []; + for (let i = startIndex; i < endIndex; i++) { + if (i >= this.totalSlices) break; + slices.push(this.sliceData.get(i) || null); + } + return slices; + } + + /** + * Add loaded slices to the cache + * + * @param {number} startIndex + * @param {Array} parsedSlices - Array of parsed slice data + */ + addLoadedSlices(startIndex, slices) { + for (let i = 0; i < slices.length; i++) { + const sliceIndex = startIndex + i; + if (sliceIndex >= this.totalSlices) break; + + this.sliceData.set(sliceIndex, slices[i]); + this.loadedSlices.add(sliceIndex); + } + } + + /** + * Get missing ranges that need to be loaded + */ + getMissingRanges(startIndex, endIndex) { + const missingRanges = []; + let rangeStart = null; + + for (let i = startIndex; i < endIndex; i++) { + if (i >= this.totalSlices) break; + + if (!this.loadedSlices.has(i)) { + if (rangeStart === null) { + rangeStart = i; + } + } else { + if (rangeStart !== null) { + // End of missing range + missingRanges.push([rangeStart, i]); + rangeStart = null; + } + } + } + + if (rangeStart !== null) { + missingRanges.push([rangeStart, endIndex]); + } + + return missingRanges; + } + + /** + * Check if a range is currently being loaded + */ + isRangeLoading(startIndex, endIndex) { + const rangeKey = `${startIndex}-${endIndex}`; + return this.loadingRanges.has(rangeKey); + } + + /** + * Mark a range as loading + */ + markRangeLoading(startIndex, endIndex) { + const rangeKey = `${startIndex}-${endIndex}`; + this.loadingRanges.add(rangeKey); + } + + /** + * Mark a range as finished loading + */ + markRangeLoaded(startIndex, endIndex) { + const rangeKey = `${startIndex}-${endIndex}`; + this.loadingRanges.delete(rangeKey); + } + + /** + * Get the number of loaded slices + */ + getLoadedCount() { + return this.loadedSlices.size; + } + + /** + * Clear all cached data + */ + clear() { + this.loadedSlices.clear(); + this.loadingRanges.clear(); + this.sliceData.clear(); + } + + /** + * Update total slices (e.g., when metadata is loaded) + */ + setTotalSlices(totalSlices) { + this.totalSlices = totalSlices; + } +} diff --git a/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallRenderer.js b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallRenderer.js index 7b3e2b05..77b08759 100644 --- a/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallRenderer.js +++ b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallRenderer.js @@ -105,12 +105,23 @@ class WaterfallRenderer { /** * Render the waterfall plot + * waterfallData: Array of slices + * totalSlices: Total number of slices in the dataset + * startSliceIndex: The actual slice index of the first slice in waterfallData */ - renderWaterfall(waterfallData, totalSlices) { - if (!this.ctx || !this.canvas || waterfallData.length === 0) return; + renderWaterfall(waterfallData, totalSlices, startSliceIndex) { + if ( + !this.ctx || + !this.canvas || + !waterfallData || + waterfallData.length === 0 + ) { + return; + } - // Store total slices for overlay updates + // Store total slices and starting index for overlay updates this.totalSlices = totalSlices; + this.waterfallWindowStart = startSliceIndex; // Clear canvas this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height); @@ -118,24 +129,23 @@ class WaterfallRenderer { // Calculate dimensions with margins const plotHeight = this.canvas.height - this.TOP_MARGIN - this.BOTTOM_MARGIN; - const maxVisibleSlices = Math.min(totalSlices, this.WATERFALL_WINDOW_SIZE); + const maxVisibleSlices = Math.min( + waterfallData.length, + this.WATERFALL_WINDOW_SIZE, + ); const sliceHeight = plotHeight / maxVisibleSlices; - // Calculate which slices to display - const startSliceIndex = this.waterfallWindowStart; - // Draw waterfall slices from bottom to top - for (let i = 0; i < this.WATERFALL_WINDOW_SIZE; i++) { - const sliceIndex = startSliceIndex + i; - if (sliceIndex >= totalSlices) break; + for (let i = 0; i < waterfallData.length && i < maxVisibleSlices; i++) { + const slice = waterfallData[i]; + if (!slice || !slice.data) { + continue; + } - const slice = waterfallData[sliceIndex]; - if (slice?.data) { - // Calculate Y position - const y = this.BOTTOM_MARGIN + (maxVisibleSlices - 1 - i) * sliceHeight; + // Calculate Y position + const y = this.BOTTOM_MARGIN + (maxVisibleSlices - 1 - i) * sliceHeight; - this.drawWaterfallSlice(slice.data, y, sliceHeight, this.canvas.width); - } + this.drawWaterfallSlice(slice.data, y, sliceHeight, this.canvas.width); } // Update the overlay with highlights and index legend diff --git a/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallVisualization.js b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallVisualization.js index aed6e53b..0391bdee 100644 --- a/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallVisualization.js +++ b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallVisualization.js @@ -4,12 +4,12 @@ */ import { generateErrorMessage, setupErrorDisplay } from "../errorHandler.js"; +import { WaterfallAPIClient } from "./WaterfallAPIClient.js"; +import { WaterfallCacheManager } from "./WaterfallCacheManager.js"; import { DEFAULT_COLOR_MAP, ERROR_MESSAGES, - get_create_waterfall_endpoint, - get_waterfall_result_endpoint, - get_waterfall_status_endpoint, + WATERFALL_WINDOW_SIZE, } from "./constants.js"; class WaterfallVisualization { @@ -22,13 +22,18 @@ class WaterfallVisualization { this.controls = null; // Data state - this.waterfallData = []; - this.parsedWaterfallData = []; // Cache parsed data to avoid re-parsing this.totalSlices = 0; this.scaleMin = null; this.scaleMax = null; this.isLoading = false; + // Cache manager for slice loading + this.cacheManager = null; + this.jobId = null; // Store job ID for range requests + + // API client + this.apiClient = new WaterfallAPIClient(captureUuid); + // Processing state this.isGenerating = false; this.currentJobId = null; @@ -102,6 +107,8 @@ class WaterfallVisualization { this.waterfallRenderer.setWaterfallWindowStart(waterfallWindowStart); if (windowChanged) { + // Check if we need to load more slices for the new window + this.ensureSlicesLoaded(waterfallWindowStart); this.renderWaterfall(); } @@ -188,7 +195,7 @@ class WaterfallVisualization { } // Re-render if we have data - if (this.parsedWaterfallData && this.parsedWaterfallData.length > 0) { + if (this.cacheManager && this.totalSlices > 0) { this.render(); } } @@ -197,7 +204,7 @@ class WaterfallVisualization { * Render the waterfall visualization */ render() { - if (!this.parsedWaterfallData || this.parsedWaterfallData.length === 0) { + if (this.totalSlices === 0) { return; } @@ -207,38 +214,98 @@ class WaterfallVisualization { // Show visualization components this.showVisualizationComponents(); + // Ensure slices for current window are loaded + this.ensureSlicesLoaded(this.waterfallWindowStart); + // Render waterfall with cached parsed data this.renderWaterfall(); - // Render periodogram - this.renderPeriodogram(); + // Render periodogram (only if slice is loaded) + if ( + this.cacheManager && + this.currentSliceIndex >= 0 && + this.currentSliceIndex < this.totalSlices && + this.cacheManager.sliceData.get(this.currentSliceIndex) + ) { + this.renderPeriodogram(); + } } /** * Render the periodogram chart */ renderPeriodogram() { - if (!this.periodogramChart || this.parsedWaterfallData.length === 0) return; - this.periodogramChart.renderPeriodogram( - this.parsedWaterfallData[this.currentSliceIndex], - ); + if (!this.periodogramChart || !this.cacheManager) return; + + // Get slice from cache manager + const slice = this.cacheManager.sliceData.get(this.currentSliceIndex); + if (!slice || !slice.data) { + return; // Slice not loaded yet + } + + this.periodogramChart.renderPeriodogram(slice); } /** * Render only the waterfall plot (for color map changes) */ renderWaterfall() { - if (!this.parsedWaterfallData || this.parsedWaterfallData.length === 0) { + if (this.totalSlices === 0 || !this.cacheManager) { + return; + } + + // Check if window is fully loaded + const isFullyLoaded = this.isWindowFullyLoaded(); + + if (!isFullyLoaded) { + // Show loading overlay if window is not fully loaded + this.showLoadingOverlay(); + return; + } + + // Hide loading overlay + this.hideLoadingOverlay(); + + // Get slices for the current window + const windowSize = WATERFALL_WINDOW_SIZE; + const startIndex = this.waterfallWindowStart; + const endIndex = Math.min(startIndex + windowSize, this.totalSlices); + + // Since window is fully loaded, all slices should be available + const loadedSlices = this.cacheManager.getRangeSlices(startIndex, endIndex); + if (loadedSlices.some((slice) => slice === null)) { + this.showError("Failed to load waterfall data"); return; } - // Render waterfall with cached parsed data - let renderer handle slicing + // Render with fully-loaded data this.waterfallRenderer.renderWaterfall( - this.parsedWaterfallData, + loadedSlices, this.totalSlices, + startIndex, ); } + /** + * Show loading overlay + */ + showLoadingOverlay() { + const overlay = document.getElementById("waterfallLoadingOverlay"); + if (overlay) { + overlay.classList.remove("d-none"); + } + } + + /** + * Hide loading overlay + */ + hideLoadingOverlay() { + const overlay = document.getElementById("waterfallLoadingOverlay"); + if (overlay) { + overlay.classList.add("d-none"); + } + } + /** * Update only slice highlights without re-rendering everything */ @@ -404,16 +471,7 @@ class WaterfallVisualization { this.showLoading(true); // First, get the post-processing status to check if waterfall data is available - const statusResponse = await fetch( - `/api/latest/assets/captures/${this.captureUuid}/post_processing_status/`, - ); - if (!statusResponse.ok) { - throw new Error( - `Failed to get post-processing status: ${statusResponse.status}`, - ); - } - - const statusData = await statusResponse.json(); + const statusData = await this.apiClient.getPostProcessingStatus(); const waterfallData = statusData.post_processed_data.find( (data) => data.processing_type === "waterfall" && @@ -426,39 +484,25 @@ class WaterfallVisualization { return; } - // Get the waterfall data file - const dataResponse = await fetch( - `/api/latest/assets/captures/${this.captureUuid}/download_post_processed_data/?processing_type=waterfall`, - ); - - if (!dataResponse.ok) { - throw new Error( - `Failed to download waterfall data: ${dataResponse.status}`, - ); - } + // Store job ID for range requests + this.jobId = waterfallData.uuid; - const waterfallJson = await dataResponse.json(); - this.waterfallData = waterfallJson; - this.totalSlices = waterfallJson.length; + // Get metadata first to know total slices and power bounds + await this.loadWaterfallMetadata(); - // Parse all waterfall data once and cache it - this.parsedWaterfallData = this.waterfallData.map((slice) => ({ - ...slice, - data: this.parseWaterfallData(slice.data), - })); + // Initialize cache manager + this.cacheManager = new WaterfallCacheManager(this.totalSlices); - // Calculate power bounds from all data - this.calculatePowerBounds(); + // Load initial window of slices + await this.loadWaterfallRange( + 0, + Math.min(WATERFALL_WINDOW_SIZE, this.totalSlices), + ); this.isLoading = false; this.showLoading(false); - this.controls.setTotalSlices(this.totalSlices); - this.waterfallRenderer.setScaleBounds(this.scaleMin, this.scaleMax); - this.periodogramChart.updateYAxisBounds(this.scaleMin, this.scaleMax); - this.updateColorLegend(); - - this.render(); + // Initial render will happen after first range loads } catch (error) { console.error("Failed to load waterfall data:", error); this.isLoading = false; @@ -479,6 +523,144 @@ class WaterfallVisualization { } } + /** + * Load waterfall metadata + */ + async loadWaterfallMetadata() { + if (!this.jobId) { + throw new Error("Job ID not available"); + } + + const metadata = await this.apiClient.getWaterfallMetadata(this.jobId); + this.totalSlices = metadata.slices_processed || 0; + this.scaleMin = metadata.power_scale_min ?? -130.0; + this.scaleMax = metadata.power_scale_max ?? 0.0; + + // Update visualization bounds + if (this.waterfallRenderer) { + this.waterfallRenderer.setScaleBounds(this.scaleMin, this.scaleMax); + } + if (this.periodogramChart) { + this.periodogramChart.updateYAxisBounds(this.scaleMin, this.scaleMax); + } + if (this.controls) { + this.controls.setTotalSlices(this.totalSlices); + } + this.updateColorLegend(); + } + + /** + * Load a range of waterfall slices + */ + async loadWaterfallRange(startIndex, endIndex) { + if (!this.jobId || !this.cacheManager) { + throw new Error("Job ID or cache manager not available"); + } + + // Clamp indices + startIndex = Math.max(0, startIndex); + endIndex = Math.min(endIndex, this.totalSlices); + + if (startIndex >= endIndex) { + return; + } + + // Check if this range is already loaded or loading + if (this.cacheManager.isRangeLoading(startIndex, endIndex)) { + return; // Already loading this range + } + + if (this.cacheManager.isRangeLoaded(startIndex, endIndex)) { + return; // Already loaded + } + + this.cacheManager.markRangeLoading(startIndex, endIndex); + + try { + const slices = await this.apiClient.loadWaterfallRange( + this.jobId, + startIndex, + endIndex, + ); + + // Parse slices + const parsedSlices = slices.map((slice) => ({ + ...slice, + data: this.parseWaterfallData(slice.data), + })); + + // Store in cache manager + this.cacheManager.addLoadedSlices(startIndex, parsedSlices); + + // Trigger render + this.render(); + } finally { + this.cacheManager.markRangeLoaded(startIndex, endIndex); + } + } + + /** + * Ensure slices for a given window or slice index are loaded + */ + async ensureSlicesLoaded(windowStart) { + if (this.totalSlices === 0 || !this.jobId || !this.cacheManager) { + return; + } + + // Calculate the range we need to display + const windowSize = WATERFALL_WINDOW_SIZE; + const startIndex = Math.max(0, windowStart); + const endIndex = Math.min(windowStart + windowSize * 2, this.totalSlices); // Load extra for prefetching + + // Get missing ranges from cache manager + const missingRanges = this.cacheManager.getMissingRanges( + startIndex, + endIndex, + ); + + // Load missing ranges (limit concurrent requests) + const maxConcurrent = this.cacheManager.maxConcurrentLoads; + let activeLoads = 0; + const loadQueue = [...missingRanges]; + + const processQueue = async () => { + while (loadQueue.length > 0 || activeLoads > 0) { + if (activeLoads < maxConcurrent && loadQueue.length > 0) { + const [start, end] = loadQueue.shift(); + activeLoads++; + this.loadWaterfallRange(start, end) + .catch((error) => { + console.error(`Error loading range ${start}-${end}:`, error); + }) + .finally(() => { + activeLoads--; + }); + } else { + // Wait a bit before checking again + await new Promise((resolve) => setTimeout(resolve, 50)); + } + } + }; + + // Start loading (don't await to avoid blocking) + processQueue(); + } + + /** + * Check if the current window is fully loaded + */ + isWindowFullyLoaded() { + if (this.totalSlices === 0 || !this.cacheManager) { + return false; + } + + const windowSize = WATERFALL_WINDOW_SIZE; + const startIndex = this.waterfallWindowStart; + const endIndex = Math.min(startIndex + windowSize, this.totalSlices); + + return this.cacheManager.isRangeLoaded(startIndex, endIndex); + } + /** * Trigger waterfall processing when data is not available */ @@ -505,25 +687,7 @@ class WaterfallVisualization { */ async createWaterfallJob() { try { - const response = await fetch( - get_create_waterfall_endpoint(this.captureUuid), - { - method: "POST", - headers: { - "X-CSRFToken": this.getCSRFToken(), - }, - }, - ); - - if (!response.ok) { - throw new Error(`HTTP ${response.status}: ${response.statusText}`); - } - - const data = await response.json(); - - if (!data.uuid) { - throw new Error("Waterfall job ID not found"); - } + const data = await this.apiClient.createWaterfallJob(); this.currentJobId = data.uuid; // Start polling for status @@ -554,21 +718,10 @@ class WaterfallVisualization { if (!this.currentJobId) return; try { - const response = await fetch( - get_waterfall_status_endpoint(this.captureUuid, this.currentJobId), - { - headers: { - "X-CSRFToken": this.getCSRFToken(), - }, - }, + const data = await this.apiClient.getWaterfallJobStatus( + this.currentJobId, ); - if (!response.ok) { - throw new Error(`HTTP ${response.status}: ${response.statusText}`); - } - - const data = await response.json(); - if (data.processing_status === "completed") { // Job completed, stop polling and fetch result this.stopStatusPolling(); @@ -601,41 +754,24 @@ class WaterfallVisualization { */ async fetchWaterfallResult() { try { - const response = await fetch( - get_waterfall_result_endpoint(this.captureUuid, this.currentJobId), - { - headers: { - "X-CSRFToken": this.getCSRFToken(), - }, - }, - ); + // Store job ID for range requests + this.jobId = this.currentJobId; - if (!response.ok) { - throw new Error(`HTTP ${response.status}: ${response.statusText}`); - } + // Get metadata first to know total slices and power bounds + await this.loadWaterfallMetadata(); - const waterfallJson = await response.json(); + // Initialize cache manager + this.cacheManager = new WaterfallCacheManager(this.totalSlices); - this.waterfallData = waterfallJson; - this.totalSlices = waterfallJson.length; - - // Parse all waterfall data once and cache it - this.parsedWaterfallData = this.waterfallData.map((slice) => ({ - ...slice, - data: this.parseWaterfallData(slice.data), - })); - - // Calculate power bounds from all data - this.calculatePowerBounds(); + // Load initial window of slices + await this.loadWaterfallRange( + 0, + Math.min(WATERFALL_WINDOW_SIZE, this.totalSlices), + ); this.setGeneratingState(false); - this.controls.setTotalSlices(this.totalSlices); - this.waterfallRenderer.setScaleBounds(this.scaleMin, this.scaleMax); - this.periodogramChart.updateYAxisBounds(this.scaleMin, this.scaleMax); - this.updateColorLegend(); - - this.render(); + // Initial render will happen after first range loads } catch (error) { console.error("Error fetching waterfall result:", error); this.showError("Failed to fetch waterfall result"); @@ -654,14 +790,6 @@ class WaterfallVisualization { this.showLoading(isGenerating); } - /** - * Get CSRF token from form input - */ - getCSRFToken() { - const token = document.querySelector("[name=csrfmiddlewaretoken]"); - return token ? token.value : ""; - } - /** * Parse base64 waterfall data */ @@ -681,59 +809,6 @@ class WaterfallVisualization { } } - /** - * Calculate power bounds from all waterfall data - */ - calculatePowerBounds() { - if (this.parsedWaterfallData.length === 0) { - // Fallback to default bounds if no data - this.scaleMin = -130; - this.scaleMax = 0; - return; - } - - let globalMin = Number.POSITIVE_INFINITY; - let globalMax = Number.NEGATIVE_INFINITY; - - // Iterate through all parsed slices to find global min/max - for (const slice of this.parsedWaterfallData) { - if (slice.data && slice.data.length > 0) { - const sliceMin = Math.min(...slice.data); - const sliceMax = Math.max(...slice.data); - - globalMin = Math.min(globalMin, sliceMin); - globalMax = Math.max(globalMax, sliceMax); - } - } - - // If we found valid data, use it; otherwise fall back to defaults - if ( - globalMin !== Number.POSITIVE_INFINITY && - globalMax !== Number.NEGATIVE_INFINITY - ) { - // Add a small margin (5%) to the bounds for better visualization - const margin = (globalMax - globalMin) * 0.05; - this.scaleMin = globalMin - margin; - this.scaleMax = globalMax + margin; - } else { - // Fallback to default bounds - this.scaleMin = -130; - this.scaleMax = 0; - } - } - - /** - * Get waterfall data for a specific range - */ - getWaterfallRange(startIndex, endIndex) { - const start = Math.max(0, startIndex); - const end = Math.min(this.totalSlices, endIndex); - - if (start >= end) return []; - - return this.parsedWaterfallData.slice(start, end); - } - /** * Show/hide loading state */ @@ -826,8 +901,9 @@ class WaterfallVisualization { */ showError(message, errorDetail = null) { // Clear data state first - this.waterfallData = []; - this.parsedWaterfallData = []; + if (this.cacheManager) { + this.cacheManager.clear(); + } this.totalSlices = 0; this.scaleMin = null; this.scaleMax = null; diff --git a/gateway/sds_gateway/static/js/visualizations/waterfall/constants.js b/gateway/sds_gateway/static/js/visualizations/waterfall/constants.js index 6b1cb965..60a3e304 100644 --- a/gateway/sds_gateway/static/js/visualizations/waterfall/constants.js +++ b/gateway/sds_gateway/static/js/visualizations/waterfall/constants.js @@ -25,19 +25,6 @@ export const WATERFALL_BOTTOM_MARGIN = 5; // Window size constants export const WATERFALL_WINDOW_SIZE = 100; -// API endpoints -export const get_create_waterfall_endpoint = (capture_uuid) => { - return `/api/v1/visualizations/${capture_uuid}/create_waterfall/`; -}; - -export const get_waterfall_status_endpoint = (capture_uuid, job_id) => { - return `/api/v1/visualizations/${capture_uuid}/waterfall_status/?job_id=${job_id}`; -}; - -export const get_waterfall_result_endpoint = (capture_uuid, job_id) => { - return `/api/v1/visualizations/${capture_uuid}/download_waterfall/?job_id=${job_id}`; -}; - export const ERROR_MESSAGES = { NO_CAPTURE: "No capture data found", API_ERROR: "API request failed", diff --git a/gateway/sds_gateway/templates/visualizations/waterfall.html b/gateway/sds_gateway/templates/visualizations/waterfall.html index ecfe0be2..039c0b4b 100644 --- a/gateway/sds_gateway/templates/visualizations/waterfall.html +++ b/gateway/sds_gateway/templates/visualizations/waterfall.html @@ -127,6 +127,12 @@
Controls
+
+ +
Loading waterfall data...
+

diff --git a/gateway/sds_gateway/visualizations/api_views.py b/gateway/sds_gateway/visualizations/api_views.py index b9f447d6..62be4906 100644 --- a/gateway/sds_gateway/visualizations/api_views.py +++ b/gateway/sds_gateway/visualizations/api_views.py @@ -1,5 +1,7 @@ """API views for the visualizations app.""" +import json + from django.http import FileResponse from django.shortcuts import get_object_or_404 from drf_spectacular.utils import OpenApiExample @@ -593,6 +595,75 @@ def create_waterfall(self, request: Request, pk: str | None = None) -> Response: status=status.HTTP_500_INTERNAL_SERVER_ERROR, ) + @extend_schema( + summary="Get waterfall metadata", + description="Get metadata for a waterfall visualization", + parameters=[ + OpenApiParameter( + name="capture_uuid", + type=str, + location=OpenApiParameter.PATH, + description="UUID of the capture", + ), + OpenApiParameter( + name="job_id", + type=str, + location=OpenApiParameter.QUERY, + description="UUID of the processing job", + ), + ], + responses={ + 200: OpenApiResponse( + description="Waterfall metadata", + ), + 404: OpenApiResponse(description="Job not found"), + }, + ) + @action(detail=True, methods=["get"], url_path="waterfall_metadata") + def get_waterfall_metadata( + self, request: Request, pk: str | None = None + ) -> Response: + """ + Get metadata for a waterfall visualization. + """ + job_id = self.get_request_param(request, "job_id", from_query=True) + if not job_id: + return Response( + {"error": "job_id parameter is required"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + # Get the post-processed data object + processed_data = get_object_or_404( + PostProcessedData, + uuid=job_id, + capture__uuid=pk, + processing_type=ProcessingType.Waterfall.value, + ) + + if processed_data.processing_status != ProcessingStatus.Completed.value: + return Response( + {"error": "Waterfall processing not completed"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + # Get metadata, with defaults if not present + metadata = processed_data.metadata or {} + + if not metadata.get("total_slices"): + # Count the number of slices in the JSON file + with open(processed_data.data_file) as file: + num_slices = len(json.load(file)) + + metadata["slices_processed"] = num_slices + processed_data.metadata = metadata + processed_data.save() + + return Response( + metadata, + status=status.HTTP_200_OK, + ) + @extend_schema( summary="Get waterfall status", description="Get the status of a waterfall generation job", @@ -651,7 +722,10 @@ def get_waterfall_status(self, request: Request, pk: str | None = None) -> Respo @extend_schema( summary="Download waterfall result", - description="Download the generated waterfall data", + description=( + "Download the generated waterfall data. Supports range queries " + "with start_index and end_index parameters." + ), parameters=[ OpenApiParameter( name="capture_uuid", @@ -665,11 +739,35 @@ def get_waterfall_status(self, request: Request, pk: str | None = None) -> Respo location=OpenApiParameter.QUERY, description="UUID of the processing job", ), + OpenApiParameter( + name="start_index", + type=int, + location=OpenApiParameter.QUERY, + required=False, + description=( + "Start index for range query (0-based). " + "If omitted, returns all data." + ), + ), + OpenApiParameter( + name="end_index", + type=int, + location=OpenApiParameter.QUERY, + required=False, + description=( + "End index for range query (exclusive). " + "If omitted, returns all data from start_index." + ), + ), ], responses={ - 200: OpenApiResponse(description="Waterfall data file"), + 200: OpenApiResponse( + description="Waterfall data file or JSON response with range" + ), 404: OpenApiResponse(description="Result not found"), - 400: OpenApiResponse(description="Processing not completed"), + 400: OpenApiResponse( + description="Processing not completed or invalid range" + ), }, ) @action(detail=True, methods=["get"], url_path="download_waterfall") @@ -678,6 +776,11 @@ def download_waterfall( ) -> Response | FileResponse: """ Download the generated waterfall data. + + Supports range queries: + - If start_index and/or end_index are provided, returns only the + requested slice range + - Otherwise, returns the full dataset (for backward compatibility) """ job_id = self.get_request_param(request, "job_id", from_query=True) if not job_id: @@ -686,8 +789,8 @@ def download_waterfall( status=status.HTTP_400_BAD_REQUEST, ) - # Get the processing job - processing_job = get_object_or_404( + # Get the post-processed data object + processed_data = get_object_or_404( PostProcessedData, uuid=job_id, capture__uuid=pk, @@ -695,27 +798,113 @@ def download_waterfall( ) try: - if processing_job.processing_status != ProcessingStatus.Completed.value: + if processed_data.processing_status != ProcessingStatus.Completed.value: return Response( {"error": "Waterfall processing not completed"}, status=status.HTTP_400_BAD_REQUEST, ) - if not processing_job.data_file: + if not processed_data.data_file: return Response( {"error": "No waterfall file found"}, status=status.HTTP_404_NOT_FOUND, ) - # Return the file - file_response = FileResponse( - processing_job.data_file, content_type="application/json" + # Check for range query parameters + start_index = self.get_request_param( + request, "start_index", from_query=True ) - file_response["Content-Disposition"] = ( - f'attachment; filename="waterfall_{pk}.json"' - ) - return file_response # noqa: TRY300 + end_index = self.get_request_param(request, "end_index", from_query=True) + # If no range parameters specified, return full file + # (backward compatibility) + if start_index is None and end_index is None: + file_response = FileResponse( + processed_data.data_file, content_type="application/json" + ) + file_response["Content-Disposition"] = ( + f'attachment; filename="waterfall_{pk}.json"' + ) + return file_response + + # Read the JSON file for range queries + processed_data.data_file.seek(0) + waterfall_data = json.load(processed_data.data_file) + + # Get total slices from metadata if available, otherwise from data length + total_slices = len(waterfall_data) + if processed_data.metadata and "total_slices" in processed_data.metadata: + total_slices = processed_data.metadata["total_slices"] + elif ( + processed_data.metadata + and "slices_processed" in processed_data.metadata + ): + total_slices = processed_data.metadata["slices_processed"] + + # Handle range query + if start_index is not None or end_index is not None: + try: + start_idx = int(start_index) if start_index is not None else 0 + end_idx = int(end_index) if end_index is not None else total_slices + except (ValueError, TypeError): + return Response( + { + "error": ( + "Invalid start_index or end_index. Must be integers." + ) + }, + status=status.HTTP_400_BAD_REQUEST, + ) + + # Validate range + if start_idx < 0 or end_idx < 0: + return Response( + {"error": ("start_index and end_index must be non-negative")}, + status=status.HTTP_400_BAD_REQUEST, + ) + if start_idx > end_idx: + return Response( + { + "error": ( + "start_index must be less than or equal to end_index" + ) + }, + status=status.HTTP_400_BAD_REQUEST, + ) + if start_idx >= total_slices: + error_msg = ( + f"start_index {start_idx} is out of range. " + f"Total slices: {total_slices}" + ) + return Response( + {"error": error_msg}, + status=status.HTTP_400_BAD_REQUEST, + ) + + # Clamp end_index to total_slices + end_idx = min(end_idx, total_slices) + + # Slice the data + sliced_data = waterfall_data[start_idx:end_idx] + + # Return JSON response with range info + return Response( + { + "data": sliced_data, + "start_index": start_idx, + "end_index": end_idx, + "total_slices": total_slices, + "count": len(sliced_data), + }, + status=status.HTTP_200_OK, + ) + + except json.JSONDecodeError as e: + log.error(f"Error parsing waterfall JSON: {e}") + return Response( + {"error": "Invalid JSON data in waterfall file"}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) except Exception as e: # noqa: BLE001 log.error(f"Error downloading waterfall: {e}") return Response( diff --git a/gateway/sds_gateway/visualizations/processing/waterfall.py b/gateway/sds_gateway/visualizations/processing/waterfall.py index cfe2365d..2ea3483e 100644 --- a/gateway/sds_gateway/visualizations/processing/waterfall.py +++ b/gateway/sds_gateway/visualizations/processing/waterfall.py @@ -185,10 +185,29 @@ def convert_drf_to_waterfall_json( msg = "No valid waterfall slices found" raise SourceDataError(msg) + # Calculate power bounds from all slices + global_min = float("inf") + global_max = float("-inf") + + for slice_data in waterfall_data: + # Decode the base64 data to calculate bounds + data_bytes = base64.b64decode(slice_data["data"]) + power_spectrum_db = np.frombuffer(data_bytes, dtype=np.float32) + + slice_min = float(np.min(power_spectrum_db)) + slice_max = float(np.max(power_spectrum_db)) + + global_min = min(global_min, slice_min) + global_max = max(global_max, slice_max) + + power_scale_min = global_min if global_min != float("inf") else None + power_scale_max = global_max if global_max != float("-inf") else None + # Log final summary logger.info( f"Waterfall processing complete: {len(waterfall_data)} slices processed, " - f"{skipped_slices} slices skipped due to data issues" + f"{skipped_slices} slices skipped due to data issues. " + f"Power bounds: [{power_scale_min:.2f}, {power_scale_max:.2f}] dB" ) metadata = { @@ -202,6 +221,8 @@ def convert_drf_to_waterfall_json( "fft_size": base_params.fft_size, "samples_per_slice": SAMPLES_PER_SLICE, "channel": channel, + "power_scale_min": power_scale_min, + "power_scale_max": power_scale_max, } return { From b7cb174b0543c84d6c48d3ac33810945fc817ee5 Mon Sep 17 00:00:00 2001 From: Connor Howington Date: Tue, 4 Nov 2025 20:19:11 -0500 Subject: [PATCH 2/5] Improve data streaming during playback and navigation --- .../static/css/visualizations/waterfall.css | 2 +- .../waterfall/WaterfallCacheManager.js | 16 ++++- .../waterfall/WaterfallControls.js | 25 +++++-- .../waterfall/WaterfallVisualization.js | 68 +++++++++++++++++-- .../js/visualizations/waterfall/constants.js | 4 +- 5 files changed, 100 insertions(+), 15 deletions(-) diff --git a/gateway/sds_gateway/static/css/visualizations/waterfall.css b/gateway/sds_gateway/static/css/visualizations/waterfall.css index b87d67e7..dfd773bc 100644 --- a/gateway/sds_gateway/static/css/visualizations/waterfall.css +++ b/gateway/sds_gateway/static/css/visualizations/waterfall.css @@ -80,7 +80,7 @@ /* Button states */ .btn:disabled { opacity: 0.6; - cursor: not-allowed; + cursor: default; } /* Loading states */ diff --git a/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallCacheManager.js b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallCacheManager.js index 0df455db..9cdc6075 100644 --- a/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallCacheManager.js +++ b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallCacheManager.js @@ -91,13 +91,27 @@ export class WaterfallCacheManager { } /** - * Check if a range is currently being loaded + * Check if an exact range is currently being loaded */ isRangeLoading(startIndex, endIndex) { const rangeKey = `${startIndex}-${endIndex}`; return this.loadingRanges.has(rangeKey); } + /** + * Check if the given range is fully contained by any ranges that are currently being loaded + */ + isRangeContainedByLoadingRange(startIndex, endIndex) { + for (const rangeKey of this.loadingRanges) { + const [loadingStart, loadingEnd] = rangeKey.split("-").map(Number); + + if (startIndex >= loadingStart && endIndex <= loadingEnd) { + return true; + } + } + return false; + } + /** * Mark a range as loading */ diff --git a/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallControls.js b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallControls.js index 87fd28c2..ac675725 100644 --- a/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallControls.js +++ b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallControls.js @@ -15,6 +15,7 @@ class WaterfallControls { this.totalSlices = 0; this.waterfallWindowStart = 0; this.hoveredSliceIndex = null; + this.isLoading = false; // Constants this.WATERFALL_WINDOW_SIZE = WATERFALL_WINDOW_SIZE; @@ -244,18 +245,28 @@ class WaterfallControls { scrollIndicatorBelow.classList.toggle("visible", canScrollDown); } - // Update button states + // Update button states if not disabled if (scrollUpBtn) { - scrollUpBtn.disabled = !canScrollUp; - scrollUpBtn.classList.toggle("disabled", !canScrollUp); + const shouldDisable = !canScrollUp || this.isLoading; + scrollUpBtn.disabled = shouldDisable; + scrollUpBtn.classList.toggle("disabled", shouldDisable); } if (scrollDownBtn) { - scrollDownBtn.disabled = !canScrollDown; - scrollDownBtn.classList.toggle("disabled", !canScrollDown); + const shouldDisable = !canScrollDown || this.isLoading; + scrollDownBtn.disabled = shouldDisable; + scrollDownBtn.classList.toggle("disabled", shouldDisable); } } + /** + * Set loading state for scroll buttons + */ + setLoading(isLoading) { + this.isLoading = isLoading; + this.updateScrollIndicators(); + } + /** * Start playback animation */ @@ -430,6 +441,8 @@ class WaterfallControls { } handleScrollUp() { + if (this.isLoading) return; + // Move the window up to show more recent slices const newWindowStart = Math.min( this.totalSlices - this.WATERFALL_WINDOW_SIZE, @@ -459,6 +472,8 @@ class WaterfallControls { } handleScrollDown() { + if (this.isLoading) return; + // Move the window down to show older slices const newWindowStart = Math.max( 0, diff --git a/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallVisualization.js b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallVisualization.js index 0391bdee..6ac276c2 100644 --- a/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallVisualization.js +++ b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallVisualization.js @@ -9,6 +9,8 @@ import { WaterfallCacheManager } from "./WaterfallCacheManager.js"; import { DEFAULT_COLOR_MAP, ERROR_MESSAGES, + PREFETCH_DISTANCE, + PREFETCH_TRIGGER, WATERFALL_WINDOW_SIZE, } from "./constants.js"; @@ -294,6 +296,9 @@ class WaterfallVisualization { if (overlay) { overlay.classList.remove("d-none"); } + if (this.controls) { + this.controls.setLoading(true); + } } /** @@ -304,6 +309,9 @@ class WaterfallVisualization { if (overlay) { overlay.classList.add("d-none"); } + if (this.controls) { + this.controls.setLoading(false); + } } /** @@ -601,21 +609,67 @@ class WaterfallVisualization { /** * Ensure slices for a given window or slice index are loaded + * Uses separate prefetch trigger threshold and prefetch distance: + * - Trigger threshold: Only prefetch when within windowSize * PREFETCH_TRIGGER of unfetched data + * - Prefetch distance: Once triggered, load up to windowSize * PREFETCH_DISTANCE on both sides */ async ensureSlicesLoaded(windowStart) { if (this.totalSlices === 0 || !this.jobId || !this.cacheManager) { return; } - // Calculate the range we need to display - const windowSize = WATERFALL_WINDOW_SIZE; - const startIndex = Math.max(0, windowStart); - const endIndex = Math.min(windowStart + windowSize * 2, this.totalSlices); // Load extra for prefetching + const windowEnd = windowStart + WATERFALL_WINDOW_SIZE; + + // Prefetch trigger threshold: check the range around the window + const triggerStart = Math.max( + windowStart - WATERFALL_WINDOW_SIZE * PREFETCH_TRIGGER, + 0, + ); + const triggerEnd = Math.min( + windowEnd + WATERFALL_WINDOW_SIZE * PREFETCH_TRIGGER, + this.totalSlices, + ); + + // Check if there's any missing data within the trigger threshold + const missingRangesInTrigger = this.cacheManager.getMissingRanges( + triggerStart, + triggerEnd, + ); + + // Only proceed with prefetching if we're approaching unfetched data + if (missingRangesInTrigger.length === 0) { + // No missing data within trigger threshold, no need to prefetch + return; + } + + // Check if all missing ranges in the trigger area are already being loaded + const allMissingRangesAreLoading = missingRangesInTrigger.every( + ([missingStart, missingEnd]) => + this.cacheManager.isRangeContainedByLoadingRange( + missingStart, + missingEnd, + ), + ); + + if (allMissingRangesAreLoading) { + // All missing ranges in trigger area are already being loaded, don't start a new request + return; + } + + // Prefetch distance: load up to WATERFALL_WINDOW_SIZE * PREFETCH_DISTANCE on both sides when triggered + const prefetchStart = Math.max( + windowStart - WATERFALL_WINDOW_SIZE * PREFETCH_DISTANCE, + 0, + ); + const prefetchEnd = Math.min( + windowEnd + WATERFALL_WINDOW_SIZE * PREFETCH_DISTANCE, + this.totalSlices, + ); - // Get missing ranges from cache manager + // Get all missing ranges in the prefetch range const missingRanges = this.cacheManager.getMissingRanges( - startIndex, - endIndex, + prefetchStart, + prefetchEnd, ); // Load missing ranges (limit concurrent requests) diff --git a/gateway/sds_gateway/static/js/visualizations/waterfall/constants.js b/gateway/sds_gateway/static/js/visualizations/waterfall/constants.js index 60a3e304..826b2588 100644 --- a/gateway/sds_gateway/static/js/visualizations/waterfall/constants.js +++ b/gateway/sds_gateway/static/js/visualizations/waterfall/constants.js @@ -22,8 +22,10 @@ export const CANVASJS_RIGHT_MARGIN = 10; export const WATERFALL_TOP_MARGIN = 5; export const WATERFALL_BOTTOM_MARGIN = 5; -// Window size constants +// Waterfall window constants export const WATERFALL_WINDOW_SIZE = 100; +export const PREFETCH_TRIGGER = 2; +export const PREFETCH_DISTANCE = 4; export const ERROR_MESSAGES = { NO_CAPTURE: "No capture data found", From a639f64909cee3136de5cabb1b2acd898cff67dd Mon Sep 17 00:00:00 2001 From: Connor Howington Date: Wed, 5 Nov 2025 16:09:39 -0500 Subject: [PATCH 3/5] Update constants usage --- .../waterfall/WaterfallVisualization.js | 23 ++++++------------- .../js/visualizations/waterfall/constants.js | 6 +++-- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallVisualization.js b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallVisualization.js index 6ac276c2..178ec5a9 100644 --- a/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallVisualization.js +++ b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallVisualization.js @@ -610,8 +610,8 @@ class WaterfallVisualization { /** * Ensure slices for a given window or slice index are loaded * Uses separate prefetch trigger threshold and prefetch distance: - * - Trigger threshold: Only prefetch when within windowSize * PREFETCH_TRIGGER of unfetched data - * - Prefetch distance: Once triggered, load up to windowSize * PREFETCH_DISTANCE on both sides + * - Trigger threshold: Only prefetch when within PREFETCH_TRIGGER of unfetched data + * - Prefetch distance: Once triggered, load up to PREFETCH_DISTANCE on both sides */ async ensureSlicesLoaded(windowStart) { if (this.totalSlices === 0 || !this.jobId || !this.cacheManager) { @@ -621,14 +621,8 @@ class WaterfallVisualization { const windowEnd = windowStart + WATERFALL_WINDOW_SIZE; // Prefetch trigger threshold: check the range around the window - const triggerStart = Math.max( - windowStart - WATERFALL_WINDOW_SIZE * PREFETCH_TRIGGER, - 0, - ); - const triggerEnd = Math.min( - windowEnd + WATERFALL_WINDOW_SIZE * PREFETCH_TRIGGER, - this.totalSlices, - ); + const triggerStart = Math.max(windowStart - PREFETCH_TRIGGER, 0); + const triggerEnd = Math.min(windowEnd + PREFETCH_TRIGGER, this.totalSlices); // Check if there's any missing data within the trigger threshold const missingRangesInTrigger = this.cacheManager.getMissingRanges( @@ -656,13 +650,10 @@ class WaterfallVisualization { return; } - // Prefetch distance: load up to WATERFALL_WINDOW_SIZE * PREFETCH_DISTANCE on both sides when triggered - const prefetchStart = Math.max( - windowStart - WATERFALL_WINDOW_SIZE * PREFETCH_DISTANCE, - 0, - ); + // Prefetch distance: load up to PREFETCH_DISTANCE on both sides when triggered + const prefetchStart = Math.max(windowStart - PREFETCH_DISTANCE, 0); const prefetchEnd = Math.min( - windowEnd + WATERFALL_WINDOW_SIZE * PREFETCH_DISTANCE, + windowEnd + PREFETCH_DISTANCE, this.totalSlices, ); diff --git a/gateway/sds_gateway/static/js/visualizations/waterfall/constants.js b/gateway/sds_gateway/static/js/visualizations/waterfall/constants.js index 826b2588..0fc06614 100644 --- a/gateway/sds_gateway/static/js/visualizations/waterfall/constants.js +++ b/gateway/sds_gateway/static/js/visualizations/waterfall/constants.js @@ -24,8 +24,10 @@ export const WATERFALL_BOTTOM_MARGIN = 5; // Waterfall window constants export const WATERFALL_WINDOW_SIZE = 100; -export const PREFETCH_TRIGGER = 2; -export const PREFETCH_DISTANCE = 4; +// How close to unfetched data to trigger a prefetch +export const PREFETCH_TRIGGER = 2 * WATERFALL_WINDOW_SIZE; +// How much data around the current window to prefetch +export const PREFETCH_DISTANCE = 4 * WATERFALL_WINDOW_SIZE; export const ERROR_MESSAGES = { NO_CAPTURE: "No capture data found", From e09f8851f07940727bb7ae055340ad4dd50f098b Mon Sep 17 00:00:00 2001 From: Connor Howington Date: Wed, 5 Nov 2025 16:10:16 -0500 Subject: [PATCH 4/5] Fix variable reassignments --- .../waterfall/WaterfallVisualization.js | 46 +++++++++++-------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallVisualization.js b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallVisualization.js index 178ec5a9..214fac28 100644 --- a/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallVisualization.js +++ b/gateway/sds_gateway/static/js/visualizations/waterfall/WaterfallVisualization.js @@ -269,12 +269,16 @@ class WaterfallVisualization { this.hideLoadingOverlay(); // Get slices for the current window - const windowSize = WATERFALL_WINDOW_SIZE; - const startIndex = this.waterfallWindowStart; - const endIndex = Math.min(startIndex + windowSize, this.totalSlices); + const waterfallWindowEnd = Math.min( + this.waterfallWindowStart + WATERFALL_WINDOW_SIZE, + this.totalSlices, + ); // Since window is fully loaded, all slices should be available - const loadedSlices = this.cacheManager.getRangeSlices(startIndex, endIndex); + const loadedSlices = this.cacheManager.getRangeSlices( + this.waterfallWindowStart, + waterfallWindowEnd, + ); if (loadedSlices.some((slice) => slice === null)) { this.showError("Failed to load waterfall data"); return; @@ -284,7 +288,7 @@ class WaterfallVisualization { this.waterfallRenderer.renderWaterfall( loadedSlices, this.totalSlices, - startIndex, + this.waterfallWindowStart, ); } @@ -566,29 +570,29 @@ class WaterfallVisualization { } // Clamp indices - startIndex = Math.max(0, startIndex); - endIndex = Math.min(endIndex, this.totalSlices); + const clampedStart = Math.max(0, startIndex); + const clampedEnd = Math.min(endIndex, this.totalSlices); - if (startIndex >= endIndex) { + if (clampedStart >= clampedEnd) { return; } // Check if this range is already loaded or loading - if (this.cacheManager.isRangeLoading(startIndex, endIndex)) { + if (this.cacheManager.isRangeLoading(clampedStart, clampedEnd)) { return; // Already loading this range } - if (this.cacheManager.isRangeLoaded(startIndex, endIndex)) { + if (this.cacheManager.isRangeLoaded(clampedStart, clampedEnd)) { return; // Already loaded } - this.cacheManager.markRangeLoading(startIndex, endIndex); + this.cacheManager.markRangeLoading(clampedStart, clampedEnd); try { const slices = await this.apiClient.loadWaterfallRange( this.jobId, - startIndex, - endIndex, + clampedStart, + clampedEnd, ); // Parse slices @@ -598,12 +602,12 @@ class WaterfallVisualization { })); // Store in cache manager - this.cacheManager.addLoadedSlices(startIndex, parsedSlices); + this.cacheManager.addLoadedSlices(clampedStart, parsedSlices); // Trigger render this.render(); } finally { - this.cacheManager.markRangeLoaded(startIndex, endIndex); + this.cacheManager.markRangeLoaded(clampedStart, clampedEnd); } } @@ -699,11 +703,15 @@ class WaterfallVisualization { return false; } - const windowSize = WATERFALL_WINDOW_SIZE; - const startIndex = this.waterfallWindowStart; - const endIndex = Math.min(startIndex + windowSize, this.totalSlices); + const waterfallWindowEnd = Math.min( + this.waterfallWindowStart + WATERFALL_WINDOW_SIZE, + this.totalSlices, + ); - return this.cacheManager.isRangeLoaded(startIndex, endIndex); + return this.cacheManager.isRangeLoaded( + this.waterfallWindowStart, + waterfallWindowEnd, + ); } /** From fad766c19cb5c6c27d47fd9f8a4dd3837cfebc9f Mon Sep 17 00:00:00 2001 From: Connor Howington Date: Wed, 5 Nov 2025 17:01:37 -0500 Subject: [PATCH 5/5] Add tests for new viz API logic --- .../visualizations/tests/test_views.py | 509 ++++++++++++++++++ 1 file changed, 509 insertions(+) diff --git a/gateway/sds_gateway/visualizations/tests/test_views.py b/gateway/sds_gateway/visualizations/tests/test_views.py index ea63d6f3..ec22c034 100644 --- a/gateway/sds_gateway/visualizations/tests/test_views.py +++ b/gateway/sds_gateway/visualizations/tests/test_views.py @@ -157,6 +157,11 @@ def setUp(self) -> None: kwargs={"pk": self.capture.uuid}, ) + self.get_waterfall_metadata_url = reverse( + "api:visualizations-get-waterfall-metadata", + kwargs={"pk": self.capture.uuid}, + ) + def test_create_waterfall_api_requires_authentication(self) -> None: """Test that create_waterfall API requires authentication.""" response = self.client.post(self.create_waterfall_url) @@ -328,6 +333,510 @@ def test_download_waterfall_api_success(self) -> None: content = b"".join(response.streaming_content).decode() assert content == test_content + def test_get_waterfall_metadata_api_requires_authentication(self) -> None: + """Test that get_waterfall_metadata API requires authentication.""" + response = self.client.get(self.get_waterfall_metadata_url) + assert response.status_code == status.HTTP_403_FORBIDDEN + + def test_get_waterfall_metadata_api_missing_job_id(self) -> None: + """Test that missing job_id parameter returns 400 for API.""" + self.client.force_login(self.user) + + response = self.client.get(self.get_waterfall_metadata_url) + assert response.status_code == status.HTTP_400_BAD_REQUEST + + data = response.json() + assert "error" in data + assert "job_id parameter is required" in data["error"] + + def test_get_waterfall_metadata_api_success_with_existing_metadata(self) -> None: + """Test successful waterfall metadata retrieval with existing metadata.""" + self.client.force_login(self.user) + + # Create test file content (array of slices) + test_slices = [{"freq": i, "data": [1, 2, 3]} for i in range(10)] + test_content = json.dumps(test_slices) + test_file = SimpleUploadedFile( + "waterfall_test.json", + test_content.encode(), + content_type="application/json", + ) + + # Create completed waterfall with metadata already set + waterfall_data = PostProcessedData.objects.create( + capture=self.capture, + processing_type=ProcessingType.Waterfall.value, + processing_parameters={}, + processing_status=ProcessingStatus.Completed.value, + metadata={"total_slices": 10, "sample_rate": 1000000}, + data_file=test_file, + ) + + response = self.client.get( + self.get_waterfall_metadata_url, + {"job_id": str(waterfall_data.uuid)}, + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json() + assert data["total_slices"] == 10 + assert data["sample_rate"] == 1000000 + + def test_get_waterfall_metadata_api_calculates_slices_from_file(self) -> None: + """Test that metadata slices are calculated from file if not present.""" + self.client.force_login(self.user) + + # Create test file content (array of slices) + test_slices = [{"freq": i, "data": [1, 2, 3]} for i in range(15)] + test_content = json.dumps(test_slices) + test_file = SimpleUploadedFile( + "waterfall_test.json", + test_content.encode(), + content_type="application/json", + ) + + # Create completed waterfall without total_slices in metadata + waterfall_data = PostProcessedData.objects.create( + capture=self.capture, + processing_type=ProcessingType.Waterfall.value, + processing_parameters={}, + processing_status=ProcessingStatus.Completed.value, + metadata={"sample_rate": 1000000}, # No total_slices + data_file=test_file, + ) + + response = self.client.get( + self.get_waterfall_metadata_url, + {"job_id": str(waterfall_data.uuid)}, + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json() + assert data["slices_processed"] == 15 + assert data["sample_rate"] == 1000000 + + # Verify that the metadata was saved back to the object + waterfall_data.refresh_from_db() + assert waterfall_data.metadata["slices_processed"] == 15 + + def test_get_waterfall_metadata_api_job_not_found(self) -> None: + """Test that non-existent job returns 404.""" + self.client.force_login(self.user) + + fake_job_id = "00000000-0000-0000-0000-000000000000" + response = self.client.get( + self.get_waterfall_metadata_url, {"job_id": fake_job_id} + ) + assert response.status_code == status.HTTP_404_NOT_FOUND + + def test_get_waterfall_metadata_api_processing_not_completed(self) -> None: + """Test that incomplete processing returns 400.""" + self.client.force_login(self.user) + + # Create waterfall processing job that's still processing + waterfall_data = PostProcessedData.objects.create( + capture=self.capture, + processing_type=ProcessingType.Waterfall.value, + processing_parameters={}, + processing_status=ProcessingStatus.Processing.value, + metadata={}, + ) + + response = self.client.get( + self.get_waterfall_metadata_url, + {"job_id": str(waterfall_data.uuid)}, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + data = response.json() + assert "error" in data + assert "Waterfall processing not completed" in data["error"] + + def test_get_waterfall_metadata_api_capture_not_owned(self) -> None: + """Test that users cannot get metadata for others' captures.""" + other_user = User.objects.create( + email="otheruser@example.com", + password="testpassword", # noqa: S106 + is_approved=True, + ) + + self.client.force_login(self.user) + + # Create waterfall data for other user's capture + other_capture = Capture.objects.create( + capture_type=CaptureType.DigitalRF, + channel="ch0", + index_name="other-index", + owner=other_user, + top_level_dir="/other/dir", + ) + + waterfall_data = PostProcessedData.objects.create( + capture=other_capture, + processing_type=ProcessingType.Waterfall.value, + processing_parameters={}, + processing_status=ProcessingStatus.Completed.value, + metadata={"total_slices": 10}, + ) + + other_capture_url = reverse( + "api:visualizations-get-waterfall-metadata", + kwargs={"pk": other_capture.uuid}, + ) + + response = self.client.get( + other_capture_url, {"job_id": str(waterfall_data.uuid)} + ) + assert response.status_code == status.HTTP_404_NOT_FOUND + + def test_download_waterfall_api_with_start_index(self) -> None: + """Test download_waterfall with start_index parameter.""" + self.client.force_login(self.user) + + # Create test file content with multiple slices + test_slices = [ + {"index": i, "data": [i * 10 + j for j in range(5)]} for i in range(20) + ] + test_content = json.dumps(test_slices) + test_file = SimpleUploadedFile( + "waterfall_test.json", + test_content.encode(), + content_type="application/json", + ) + + waterfall_data = PostProcessedData.objects.create( + capture=self.capture, + processing_type=ProcessingType.Waterfall.value, + processing_parameters={}, + processing_status=ProcessingStatus.Completed.value, + metadata={"total_slices": 20}, + data_file=test_file, + ) + + # Request slices from index 5 to the end + response = self.client.get( + self.download_waterfall_url, + {"job_id": str(waterfall_data.uuid), "start_index": 5}, + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json() + assert "data" in data + assert "start_index" in data + assert "end_index" in data + assert "total_slices" in data + assert "count" in data + + assert data["start_index"] == 5 + assert data["end_index"] == 20 + assert data["total_slices"] == 20 + assert data["count"] == 15 + assert len(data["data"]) == 15 + assert data["data"][0]["index"] == 5 + assert data["data"][-1]["index"] == 19 + + def test_download_waterfall_api_with_end_index(self) -> None: + """Test download_waterfall with end_index parameter.""" + self.client.force_login(self.user) + + # Create test file content with multiple slices + test_slices = [ + {"index": i, "data": [i * 10 + j for j in range(5)]} for i in range(20) + ] + test_content = json.dumps(test_slices) + test_file = SimpleUploadedFile( + "waterfall_test.json", + test_content.encode(), + content_type="application/json", + ) + + waterfall_data = PostProcessedData.objects.create( + capture=self.capture, + processing_type=ProcessingType.Waterfall.value, + processing_parameters={}, + processing_status=ProcessingStatus.Completed.value, + metadata={"total_slices": 20}, + data_file=test_file, + ) + + # Request slices from start to index 10 + response = self.client.get( + self.download_waterfall_url, + {"job_id": str(waterfall_data.uuid), "end_index": 10}, + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json() + assert data["start_index"] == 0 + assert data["end_index"] == 10 + assert data["total_slices"] == 20 + assert data["count"] == 10 + assert len(data["data"]) == 10 + assert data["data"][0]["index"] == 0 + assert data["data"][-1]["index"] == 9 + + def test_download_waterfall_api_with_range(self) -> None: + """Test download_waterfall with both start_index and end_index.""" + self.client.force_login(self.user) + + # Create test file content with multiple slices + test_slices = [ + {"index": i, "data": [i * 10 + j for j in range(5)]} for i in range(20) + ] + test_content = json.dumps(test_slices) + test_file = SimpleUploadedFile( + "waterfall_test.json", + test_content.encode(), + content_type="application/json", + ) + + waterfall_data = PostProcessedData.objects.create( + capture=self.capture, + processing_type=ProcessingType.Waterfall.value, + processing_parameters={}, + processing_status=ProcessingStatus.Completed.value, + metadata={"total_slices": 20}, + data_file=test_file, + ) + + # Request slices from index 5 to 15 + response = self.client.get( + self.download_waterfall_url, + { + "job_id": str(waterfall_data.uuid), + "start_index": 5, + "end_index": 15, + }, + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json() + assert data["start_index"] == 5 + assert data["end_index"] == 15 + assert data["total_slices"] == 20 + assert data["count"] == 10 + assert len(data["data"]) == 10 + assert data["data"][0]["index"] == 5 + assert data["data"][-1]["index"] == 14 + + def test_download_waterfall_api_invalid_range_start_greater_than_end(self) -> None: + """Test that start_index > end_index returns 400.""" + self.client.force_login(self.user) + + # Create test file content + test_slices = [{"index": i} for i in range(10)] + test_content = json.dumps(test_slices) + test_file = SimpleUploadedFile( + "waterfall_test.json", + test_content.encode(), + content_type="application/json", + ) + + waterfall_data = PostProcessedData.objects.create( + capture=self.capture, + processing_type=ProcessingType.Waterfall.value, + processing_parameters={}, + processing_status=ProcessingStatus.Completed.value, + metadata={"total_slices": 10}, + data_file=test_file, + ) + + response = self.client.get( + self.download_waterfall_url, + { + "job_id": str(waterfall_data.uuid), + "start_index": 7, + "end_index": 5, + }, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + data = response.json() + assert "error" in data + assert "start_index must be less than or equal to end_index" in data["error"] + + def test_download_waterfall_api_start_index_out_of_bounds(self) -> None: + """Test that start_index >= total_slices returns 400.""" + self.client.force_login(self.user) + + # Create test file content + test_slices = [{"index": i} for i in range(10)] + test_content = json.dumps(test_slices) + test_file = SimpleUploadedFile( + "waterfall_test.json", + test_content.encode(), + content_type="application/json", + ) + + waterfall_data = PostProcessedData.objects.create( + capture=self.capture, + processing_type=ProcessingType.Waterfall.value, + processing_parameters={}, + processing_status=ProcessingStatus.Completed.value, + metadata={"total_slices": 10}, + data_file=test_file, + ) + + response = self.client.get( + self.download_waterfall_url, + { + "job_id": str(waterfall_data.uuid), + "start_index": 10, + }, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + data = response.json() + assert "error" in data + assert "start_index 10 is out of range" in data["error"] + assert "Total slices: 10" in data["error"] + + def test_download_waterfall_api_negative_indices(self) -> None: + """Test that negative indices return 400.""" + self.client.force_login(self.user) + + # Create test file content + test_slices = [{"index": i} for i in range(10)] + test_content = json.dumps(test_slices) + test_file = SimpleUploadedFile( + "waterfall_test.json", + test_content.encode(), + content_type="application/json", + ) + + waterfall_data = PostProcessedData.objects.create( + capture=self.capture, + processing_type=ProcessingType.Waterfall.value, + processing_parameters={}, + processing_status=ProcessingStatus.Completed.value, + metadata={"total_slices": 10}, + data_file=test_file, + ) + + response = self.client.get( + self.download_waterfall_url, + { + "job_id": str(waterfall_data.uuid), + "start_index": -1, + }, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + data = response.json() + assert "error" in data + assert "must be non-negative" in data["error"] + + def test_download_waterfall_api_invalid_index_type(self) -> None: + """Test that non-integer indices return 400.""" + self.client.force_login(self.user) + + # Create test file content + test_slices = [{"index": i} for i in range(10)] + test_content = json.dumps(test_slices) + test_file = SimpleUploadedFile( + "waterfall_test.json", + test_content.encode(), + content_type="application/json", + ) + + waterfall_data = PostProcessedData.objects.create( + capture=self.capture, + processing_type=ProcessingType.Waterfall.value, + processing_parameters={}, + processing_status=ProcessingStatus.Completed.value, + metadata={"total_slices": 10}, + data_file=test_file, + ) + + response = self.client.get( + self.download_waterfall_url, + { + "job_id": str(waterfall_data.uuid), + "start_index": "not_a_number", + }, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + data = response.json() + assert "error" in data + assert "Must be integers" in data["error"] + + def test_download_waterfall_api_end_index_clamped_to_total(self) -> None: + """Test that end_index is clamped to total_slices.""" + self.client.force_login(self.user) + + # Create test file content + test_slices = [{"index": i} for i in range(10)] + test_content = json.dumps(test_slices) + test_file = SimpleUploadedFile( + "waterfall_test.json", + test_content.encode(), + content_type="application/json", + ) + + waterfall_data = PostProcessedData.objects.create( + capture=self.capture, + processing_type=ProcessingType.Waterfall.value, + processing_parameters={}, + processing_status=ProcessingStatus.Completed.value, + metadata={"total_slices": 10}, + data_file=test_file, + ) + + # Request end_index beyond total_slices + response = self.client.get( + self.download_waterfall_url, + { + "job_id": str(waterfall_data.uuid), + "start_index": 5, + "end_index": 20, + }, + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json() + assert data["start_index"] == 5 + assert data["end_index"] == 10 # Clamped to total_slices + assert data["total_slices"] == 10 + assert data["count"] == 5 + assert len(data["data"]) == 5 + + def test_download_waterfall_api_backward_compatibility_no_params(self) -> None: + """Test that download_waterfall without range params returns full file.""" + self.client.force_login(self.user) + + # Create test file content + test_slices = [{"index": i} for i in range(10)] + test_content = json.dumps(test_slices) + test_file = SimpleUploadedFile( + "waterfall_test.json", + test_content.encode(), + content_type="application/json", + ) + + waterfall_data = PostProcessedData.objects.create( + capture=self.capture, + processing_type=ProcessingType.Waterfall.value, + processing_parameters={}, + processing_status=ProcessingStatus.Completed.value, + metadata={}, + data_file=test_file, + ) + + # Request without range parameters (backward compatibility) + response = self.client.get( + self.download_waterfall_url, + {"job_id": str(waterfall_data.uuid)}, + ) + + assert response.status_code == status.HTTP_200_OK + assert response.get("Content-Type") == "application/json" + assert "attachment" in response.get("Content-Disposition", "") + + # Verify it returns the full file content (not JSON response) + content = b"".join(response.streaming_content).decode() + assert content == test_content + class SpectrogramVisualizationViewTestCases(TestCase): """Test cases for SpectrogramVisualizationView."""