optimal Memory CICD

Author: josep-bravo

.github/workflows/ci-cd.yml

@@ -153,26 +153,50 @@ jobs:
               
               # Set up test environment
               export PYTHONDONTWRITEBYTECODE=1
+              export PYTHONHASHSEED=0
               export SPEECH_RECOGNITION_VENV='/opt/ros_python_env'
               export AI_VENV='/opt/ros_python_env'
+              # Reduce Python memory footprint
+              export MALLOC_TRIM_THRESHOLD_=100000
+              export MALLOC_MMAP_THRESHOLD_=131072
               
               # Run tests with coverage for Python packages (run each package separately)
               echo '🧪 Running tests with coverage...'
               PY_PACKAGES_LIST='${{ env.PY_PACKAGES }}'
               
               for pkg in \${PY_PACKAGES_LIST}; do
                 echo \"Testing package: \$pkg\"
+                
+                # Force garbage collection before each package
+                python3 -c \"import gc; gc.collect()\" 2>/dev/null || true
+                
+                # Run tests with memory-optimized settings
                 colcon test \
                   --packages-select \$pkg \
                   --event-handlers console_direct+ \
-                  --pytest-args --cov=\$pkg --cov-report=html --cov-report=term --cov-report=lcov --cov-report=xml:coverage.xml \
+                  --pytest-args \
+                    -p no:cacheprovider \
+                    --cov=\$pkg \
+                    --cov-report=html \
+                    --cov-report=term \
+                    --cov-report=lcov \
+                    --cov-report=xml:coverage.xml \
                   --return-code-on-test-failure || true
                 
                 # Copy lcov file immediately after each package test
                 if [ -f \"build/\${pkg}/coverage.lcov\" ]; then
                   cp \"build/\${pkg}/coverage.lcov\" /ci_workspace/coverage-py-\${pkg}.lcov
                   echo \"✅ Copied build/\${pkg}/coverage.lcov → /ci_workspace/coverage-py-\${pkg}.lcov\"
                 fi
+                
+                # Clean up test artifacts to free memory between packages
+                echo \"🧹 Cleaning up after \${pkg} tests...\"
+                rm -rf build/\${pkg}/.pytest_cache 2>/dev/null || true
+                rm -rf build/\${pkg}/__pycache__ 2>/dev/null || true
+                python3 -c \"import gc; gc.collect()\" 2>/dev/null || true
+                
+                # Brief pause to allow memory to be freed
+                sleep 2
               done
               
               # Copy build artifacts back to mounted directory for artifact upload

Docker/quick_test_coverage.sh

@@ -304,7 +304,9 @@ if [ ${#ALL_PKG_LIST[@]} -gt 0 ]; then
         for pkg in "${PY_PACKAGES[@]}"; do
             PYTEST_ARGS_ARRAY+=("--cov=$pkg")
         done
+        # Memory-optimized settings to prevent OOM (Error 137)
         PYTEST_ARGS_ARRAY+=("--cov-report=html" "--cov-report=term" "--cov-report=xml")
+        PYTEST_ARGS_ARRAY+=("-p" "no:cacheprovider")  # Disable caching to reduce memory
     fi
 
     # Convert array to proper string for colcon

src/audio_stream_manager/pytest.ini

@@ -3,7 +3,8 @@ testpaths = test
 python_files = test_*.py
 python_classes = Test*
 python_functions = test_*
-addopts = -v --tb=short
+# Memory-optimized settings to prevent OOM (Error 137)
+addopts = -v --tb=short -p no:logging -p no:warnings --maxfail=50
 markers =
     unit: Unit tests
     integration: Integration tests

src/audio_stream_manager/test/conftest.py

@@ -2,10 +2,12 @@
 Test configuration for audio_stream_manager package.
 Ensures proper Python environment setup for audio dependencies.
 """
+
+import gc
 import os
 import sys
-import pytest
 
+import pytest
 
 # Setup environment for audio_stream_manager (uses ros_python_env)
 VENV_PATH = os.environ.get("AI_VENV", "/opt/ros_python_env")
@@ -26,28 +28,28 @@
 def setup_audio_mocks():
     """Setup mock dependencies for audio testing"""
     from unittest.mock import MagicMock
-    
+
     # Mock sounddevice if not available
     try:
         import sounddevice
     except ImportError:
-        sys.modules['sounddevice'] = MagicMock()
+        sys.modules["sounddevice"] = MagicMock()
         print("[pytest] Mocked sounddevice")
-    
+
     # Mock numpy if not available (should be available but just in case)
     try:
         import numpy
     except ImportError:
-        sys.modules['numpy'] = MagicMock()
+        sys.modules["numpy"] = MagicMock()
         print("[pytest] Mocked numpy")
 
     # Mock ROS if not available
     try:
         import rclpy
     except ImportError:
-        sys.modules['rclpy'] = MagicMock()
-        sys.modules['rclpy.node'] = MagicMock()
-        sys.modules['rclpy.parameter'] = MagicMock()
+        sys.modules["rclpy"] = MagicMock()
+        sys.modules["rclpy.node"] = MagicMock()
+        sys.modules["rclpy.parameter"] = MagicMock()
         print("[pytest] Mocked rclpy")
 
 
@@ -67,9 +69,23 @@ def setup_tests():
 # Basic pytest configuration
 def pytest_configure(config):
     """Configure pytest for audio stream manager tests"""
-    config.addinivalue_line(
-        "markers", "unit: Unit tests for audio components"
-    )
-    config.addinivalue_line(
-        "markers", "integration: Integration tests with ROS"
-    )
+    config.addinivalue_line("markers", "unit: Unit tests for audio components")
+    config.addinivalue_line("markers", "integration: Integration tests with ROS")
+
+
+# Memory optimization hooks
+@pytest.fixture(autouse=True)
+def cleanup_after_test():
+    """Force garbage collection after each test to prevent memory buildup"""
+    yield
+    gc.collect()
+
+
+def pytest_runtest_teardown(item, nextitem):
+    """Force garbage collection between tests to free memory"""
+    gc.collect()
+
+
+def pytest_sessionfinish(session, exitstatus):
+    """Final cleanup after all tests complete"""
+    gc.collect()

src/speech_recognition/pytest.ini

@@ -3,7 +3,8 @@ testpaths = test
 python_files = test_*.py
 python_classes = Test*
 python_functions = test_*
-addopts = -v --tb=short
+# Memory-optimized settings to prevent OOM (Error 137)
+addopts = -v --tb=short -p no:logging -p no:warnings --maxfail=50
 # Exclude weights directory and any __pycache__ from collection
 norecursedirs = weights __pycache__ .git .pytest_cache build install log speech_recognition/weights
 collect_ignore = ["weights", "speech_recognition/weights", "speech_recognition/weights/"]

src/speech_recognition/test/conftest.py

@@ -2,13 +2,17 @@
 Test configuration for speech_recognition package.
 Ensures proper Python environment setup for speech/AI dependencies.
 """
+
+import gc
 import os
 import sys
 from unittest.mock import MagicMock
 
+import pytest
+
 # Mock silero_vad.data immediately to prevent import errors during collection
 mock_silero_data = MagicMock()
-sys.modules['silero_vad.data'] = mock_silero_data
+sys.modules["silero_vad.data"] = mock_silero_data
 
 # Setup environment for speech_recognition (uses unified ros_python_env)
 DIARIZATION_ENV_PATH = "/opt/ros_python_env"
@@ -28,12 +32,13 @@
 # Mock heavy AI/ML dependencies for testing
 def setup_ml_mocks():
     """Setup mock dependencies for ML/AI testing without requiring full models"""
-    from unittest.mock import MagicMock, patch
-    
+    from unittest.mock import MagicMock
+
     # Mock torch and torchaudio
     try:
         import torch
         import torchaudio
+
         print(f"[pytest] Using real PyTorch: {torch.__version__}")
     except ImportError:
         print("[pytest] Mocking PyTorch dependencies")
@@ -43,13 +48,13 @@ def setup_ml_mocks():
         mock_torch.hub.load = MagicMock()
         mock_torch.tensor = MagicMock()
         mock_torch.jit.load = MagicMock()
-        
+
         mock_torchaudio = MagicMock()
         mock_torchaudio.functional.resample = MagicMock()
-        
-        sys.modules['torch'] = mock_torch
-        sys.modules['torchaudio'] = mock_torchaudio
-    
+
+        sys.modules["torch"] = mock_torch
+        sys.modules["torchaudio"] = mock_torchaudio
+
     # Mock faster_whisper
     try:
         import faster_whisper
@@ -58,8 +63,8 @@ def setup_ml_mocks():
         mock_whisper = MagicMock()
         mock_whisper.WhisperModel = MagicMock()
         mock_whisper.BatchedInferencePipeline = MagicMock()
-        sys.modules['faster_whisper'] = mock_whisper
-    
+        sys.modules["faster_whisper"] = mock_whisper
+
     # Mock diart
     try:
         import diart
@@ -69,37 +74,37 @@ def setup_ml_mocks():
         mock_diart.SpeakerDiarization = MagicMock()
         mock_diart.SpeakerDiarizationConfig = MagicMock()
         mock_diart.models = MagicMock()
-        sys.modules['diart'] = mock_diart
-        sys.modules['diart.models'] = mock_diart.models
-    
+        sys.modules["diart"] = mock_diart
+        sys.modules["diart.models"] = mock_diart.models
+
     # Mock pyannote
     try:
         import pyannote
     except ImportError:
         print("[pytest] Mocking pyannote")
         mock_pyannote = MagicMock()
         mock_pyannote.core.Annotation = MagicMock()
-        sys.modules['pyannote'] = mock_pyannote
-        sys.modules['pyannote.core'] = mock_pyannote.core
-    
+        sys.modules["pyannote"] = mock_pyannote
+        sys.modules["pyannote.core"] = mock_pyannote.core
+
     # Mock openwakeword
     try:
         import openwakeword
     except ImportError:
         print("[pytest] Mocking openwakeword")
-        sys.modules['openwakeword'] = MagicMock()
-    
+        sys.modules["openwakeword"] = MagicMock()
+
     # Mock silero_vad module that's causing the import error
     mock_silero = MagicMock()
     mock_silero.model.load_silero_vad = MagicMock()
     mock_silero.utils_vad.init_jit_model = MagicMock()
     mock_silero.utils_vad.OnnxWrapper = MagicMock()
     mock_silero.data = MagicMock()  # Mock the missing data submodule
-    sys.modules['silero_vad'] = mock_silero
-    sys.modules['silero_vad.model'] = mock_silero.model
-    sys.modules['silero_vad.utils_vad'] = mock_silero.utils_vad
-    sys.modules['silero_vad.data'] = mock_silero.data
-    
+    sys.modules["silero_vad"] = mock_silero
+    sys.modules["silero_vad.model"] = mock_silero.model
+    sys.modules["silero_vad.utils_vad"] = mock_silero.utils_vad
+    sys.modules["silero_vad.data"] = mock_silero.data
+
     print("[pytest] ML dependency mocking completed")
 
 
@@ -110,12 +115,24 @@ def setup_ml_mocks():
 # Basic pytest configuration
 def pytest_configure(config):
     """Configure pytest for speech recognition tests"""
-    config.addinivalue_line(
-        "markers", "unit: Unit tests for speech components"
-    )
-    config.addinivalue_line(
-        "markers", "integration: Integration tests with ROS"
-    )
-    config.addinivalue_line(
-        "markers", "slow: Tests that require model loading"
-    )
+    config.addinivalue_line("markers", "unit: Unit tests for speech components")
+    config.addinivalue_line("markers", "integration: Integration tests with ROS")
+    config.addinivalue_line("markers", "slow: Tests that require model loading")
+
+
+# Memory optimization hooks
+@pytest.fixture(autouse=True)
+def cleanup_after_test():
+    """Force garbage collection after each test to prevent memory buildup"""
+    yield
+    gc.collect()
+
+
+def pytest_runtest_teardown(item, nextitem):
+    """Force garbage collection between tests to free memory"""
+    gc.collect()
+
+
+def pytest_sessionfinish(session, exitstatus):
+    """Final cleanup after all tests complete"""
+    gc.collect()