Enable single-precision particle builds; add SP-particles CI matrix entries

.github/workflows/cuda-ci.yml

@@ -18,11 +18,20 @@ concurrency:
 jobs:
   cuda-build:
     runs-on: ubuntu-22.04
-    name: CUDA v${{matrix.cuda_ver}} - Particles ${{matrix.particles}}
+    name: CUDA v${{matrix.cuda_ver}} - Mesh ${{matrix.mesh_precision}} Particles ${{matrix.particles}} ${{matrix.particles_precision}}
     strategy:
       matrix:
         cuda_pkg: [12-2]
         particles: [OFF, ON]
+        particles_precision: [DOUBLE, SINGLE]
+        mesh_precision: [DOUBLE, SINGLE]
+        exclude:
+          - particles: OFF
+            particles_precision: SINGLE
+          # SP-mesh entries are temporarily disabled because the upstream
+          # SP-mesh build currently fails on non-particle code. Remove the
+          # guard below to re-enable SP-mesh + DP/SP particles once fixed.
+          - mesh_precision: SINGLE
         include:
           - cuda_ver: "12.2"
             cuda_pkg: 12-2
@@ -71,6 +80,8 @@ jobs:
             -DERF_ENABLE_MPI:BOOL=ON \
             -DERF_ENABLE_CUDA:BOOL=ON \
             -DERF_ENABLE_PARTICLES:BOOL=${{matrix.particles}} \
+            -DERF_PRECISION:STRING=${{matrix.mesh_precision}} \
+            -DERF_PARTICLES_PRECISION:STRING=${{matrix.particles_precision}} \
             -DERF_ENABLE_UNIT_TESTS:BOOL=ON .
           echo "Building with $(nproc) processes"
           cmake --build build-${{matrix.cuda_pkg}} --parallel $(nproc) \

.github/workflows/gcc.yml

@@ -8,12 +8,21 @@ concurrency:
 
 jobs:
   library:
-    name: GNU@9.3 C++17 Release - Particles ${{matrix.particles}}
+    name: GNU@9.3 C++17 Release - Mesh ${{matrix.mesh_precision}} Particles ${{matrix.particles}} ${{matrix.particles_precision}}
     runs-on: ubuntu-22.04
     # env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual"}
     strategy:
       matrix:
         particles: [OFF, ON]
+        particles_precision: [DOUBLE, SINGLE]
+        mesh_precision: [DOUBLE, SINGLE]
+        exclude:
+          - particles: OFF
+            particles_precision: SINGLE
+          # SP-mesh entries are temporarily disabled because the upstream
+          # SP-mesh build currently fails on non-particle code. Remove the
+          # guard below to re-enable SP-mesh + DP/SP particles once fixed.
+          - mesh_precision: SINGLE
 
     steps:
 
@@ -45,6 +54,8 @@ jobs:
           -DERF_DIM:STRING=3 \
           -DERF_ENABLE_MPI:BOOL=ON \
           -DERF_ENABLE_PARTICLES:BOOL=${{matrix.particles}} \
+          -DERF_PRECISION:STRING=${{matrix.mesh_precision}} \
+          -DERF_PARTICLES_PRECISION:STRING=${{matrix.particles_precision}} \
           -DERF_ENABLE_TESTS:BOOL=ON \
           -DERF_ENABLE_ALL_WARNINGS:BOOL=ON \
           -DERF_ENABLE_FCOMPARE:BOOL=ON \
@@ -73,17 +84,20 @@ jobs:
         exit ${return}
 
     - name: Check Unit Test Registration
+      if: matrix.particles_precision == 'DOUBLE' && matrix.mesh_precision == 'DOUBLE'
       run: |
         Tests/Unit/erf_unit_tests --gtest_list_tests | tee ${{runner.workspace}}/unit-test-list.txt
         grep -Eq '^[A-Za-z0-9_].*\.$' ${{runner.workspace}}/unit-test-list.txt
       working-directory: ${{runner.workspace}}/ERF/build
 
     - name: CMake Unit Tests
+      if: matrix.particles_precision == 'DOUBLE' && matrix.mesh_precision == 'DOUBLE'
       run: |
         ctest -L unit -VV
       working-directory: ${{runner.workspace}}/ERF/build
 
     - name: CMake Regression Tests # see file ERF/Tests/CTestList.cmake
+      if: matrix.particles_precision == 'DOUBLE' && matrix.mesh_precision == 'DOUBLE'
       run: |
         ctest -L regression -VV
       working-directory: ${{runner.workspace}}/ERF/build

.github/workflows/hip.yml

@@ -20,7 +20,7 @@ jobs:
   # https://gitlab.kitware.com/cmake/cmake/-/issues/21968
   # https://github.com/ROCm-Developer-Tools/HIP/issues/2246
   Build-And-Test-HIP:
-    name: HIP ROCm GFortran@9.3 C++17 - Particles ${{matrix.particles}}
+    name: HIP ROCm GFortran@9.3 C++17 - Mesh ${{matrix.mesh_precision}} Particles ${{matrix.particles}} ${{matrix.particles_precision}}
     runs-on: ubuntu-22.04
     # Have to have -Wno-deprecated-declarations due to deprecated atomicAddNoRet
     # Have to have -Wno-gnu-zero-variadic-macro-arguments to avoid
@@ -37,6 +37,15 @@ jobs:
     strategy:
       matrix:
         particles: [OFF, ON]
+        particles_precision: [DOUBLE, SINGLE]
+        mesh_precision: [DOUBLE, SINGLE]
+        exclude:
+          - particles: OFF
+            particles_precision: SINGLE
+          # SP-mesh entries are temporarily disabled because the upstream
+          # SP-mesh build currently fails on non-particle code. Remove the
+          # guard below to re-enable SP-mesh + DP/SP particles once fixed.
+          - mesh_precision: SINGLE
 
     steps:
 
@@ -85,6 +94,8 @@ jobs:
           -DERF_ENABLE_HIP:BOOL=ON \
           -DAMReX_AMD_ARCH=gfx908 \
           -DERF_ENABLE_PARTICLES:BOOL=${{matrix.particles}} \
+          -DERF_PRECISION:STRING=${{matrix.mesh_precision}} \
+          -DERF_PARTICLES_PRECISION:STRING=${{matrix.particles_precision}} \
           -DERF_ENABLE_TESTS:BOOL=ON \
           -DERF_ENABLE_ALL_WARNINGS:BOOL=ON \
           -DERF_ENABLE_FCOMPARE:BOOL=ON \

.github/workflows/macos.yml

@@ -8,12 +8,21 @@ concurrency:
 
 jobs:
   MacOS-Clang:
-    name: Apple Clang@11.0 - Particles ${{matrix.particles}}
+    name: Apple Clang@11.0 - Mesh ${{matrix.mesh_precision}} Particles ${{matrix.particles}} ${{matrix.particles_precision}}
     runs-on: macos-latest
     # env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual"}
     strategy:
       matrix:
         particles: [OFF, ON]
+        particles_precision: [DOUBLE, SINGLE]
+        mesh_precision: [DOUBLE, SINGLE]
+        exclude:
+          - particles: OFF
+            particles_precision: SINGLE
+          # SP-mesh entries are temporarily disabled because the upstream
+          # SP-mesh build currently fails on non-particle code. Remove the
+          # guard below to re-enable SP-mesh + DP/SP particles once fixed.
+          - mesh_precision: SINGLE
 
     steps:
 
@@ -43,6 +52,8 @@ jobs:
           -DERF_DIM:STRING=3 \
           -DERF_ENABLE_MPI:BOOL=TRUE \
           -DERF_ENABLE_PARTICLES:BOOL=${{matrix.particles}} \
+          -DERF_PRECISION:STRING=${{matrix.mesh_precision}} \
+          -DERF_PARTICLES_PRECISION:STRING=${{matrix.particles_precision}} \
           -DERF_ENABLE_TESTS:BOOL=TRUE \
           -DERF_ENABLE_ALL_WARNINGS:BOOL=TRUE \
           -DERF_ENABLE_FCOMPARE:BOOL=TRUE \
@@ -72,17 +83,20 @@ jobs:
         exit ${return}
 
     - name: Check Unit Test Registration
+      if: matrix.particles_precision == 'DOUBLE' && matrix.mesh_precision == 'DOUBLE'
       run: |
         Tests/Unit/erf_unit_tests --gtest_list_tests | tee ${{runner.workspace}}/unit-test-list.txt
         grep -Eq '^[A-Za-z0-9_].*\.$' ${{runner.workspace}}/unit-test-list.txt
       working-directory: ${{runner.workspace}}/ERF/build
 
     - name: CMake Unit Tests
+      if: matrix.particles_precision == 'DOUBLE' && matrix.mesh_precision == 'DOUBLE'
       run: |
         ctest -L unit -VV
       working-directory: ${{runner.workspace}}/ERF/build
 
     - name: CMake Regression Tests # see file ERF/Tests/CTestList.cmake
+      if: matrix.particles_precision == 'DOUBLE' && matrix.mesh_precision == 'DOUBLE'
       run: |
         ctest -L regression -VV
       working-directory: ${{runner.workspace}}/ERF/build

.github/workflows/sycl.yml

@@ -17,11 +17,20 @@ concurrency:
 
 jobs:
   Build-And-Test-SYCL:
-    name: oneAPI SYCL - Particles ${{matrix.particles}}
+    name: oneAPI SYCL - Mesh ${{matrix.mesh_precision}} Particles ${{matrix.particles}} ${{matrix.particles_precision}}
     runs-on: ubuntu-24.04
     strategy:
       matrix:
         particles: [OFF, ON]
+        particles_precision: [DOUBLE, SINGLE]
+        mesh_precision: [DOUBLE, SINGLE]
+        exclude:
+          - particles: OFF
+            particles_precision: SINGLE
+          # SP-mesh entries are temporarily disabled because the upstream
+          # SP-mesh build currently fails on non-particle code. Remove the
+          # guard below to re-enable SP-mesh + DP/SP particles once fixed.
+          - mesh_precision: SINGLE
 
     steps:
 
@@ -72,6 +81,8 @@ jobs:
           -DERF_ENABLE_TESTS:BOOL=ON \
           -DERF_ENABLE_ALL_WARNINGS:BOOL=ON \
           -DERF_ENABLE_PARTICLES:BOOL=${{matrix.particles}} \
+          -DERF_PRECISION:STRING=${{matrix.mesh_precision}} \
+          -DERF_PARTICLES_PRECISION:STRING=${{matrix.particles_precision}} \
           -DCMAKE_C_COMPILER=$(which icx) \
           -DCMAKE_CXX_COMPILER=$(which icpx) \
           -DCMAKE_CXX_STANDARD=17 \

.github/workflows/windows-mpi.yml

@@ -8,13 +8,22 @@ concurrency:
 
 jobs:
   WIN64-MSVC:
-    name: WIN64-MSVC - MPI ${{ matrix.mpi }} - Particles ${{ matrix.particles }}
+    name: WIN64-MSVC - MPI ${{ matrix.mpi }} - Mesh ${{ matrix.mesh_precision }} Particles ${{ matrix.particles }} ${{ matrix.particles_precision }}
     runs-on: windows-latest
     strategy:
       fail-fast: false
       matrix:
         particles: [OFF, ON]
+        particles_precision: [DOUBLE, SINGLE]
+        mesh_precision: [DOUBLE, SINGLE]
         mpi: [ON]
+        exclude:
+          - particles: OFF
+            particles_precision: SINGLE
+          # SP-mesh entries are temporarily disabled because the upstream
+          # SP-mesh build currently fails on non-particle code. Remove the
+          # guard below to re-enable SP-mesh + DP/SP particles once fixed.
+          - mesh_precision: SINGLE
 
     steps:
       - name: Checkout (with submodules)
@@ -39,6 +48,9 @@ jobs:
           cmake -S . -B build `
             -DCMAKE_BUILD_TYPE=Release `
             -DERF_ENABLE_MPI=ON `
+            -DERF_ENABLE_PARTICLES:BOOL=${{ matrix.particles }} `
+            -DERF_PRECISION:STRING=${{ matrix.mesh_precision }} `
+            -DERF_PARTICLES_PRECISION:STRING=${{ matrix.particles_precision }} `
             -DMPI_CXX_LIB_NAMES=msmpi `
             -DMPI_C_LIB_NAMES=msmpi `
             -DMPI_msmpi_LIBRARY="$env:MSMPI_LIB\msmpi.lib" `
@@ -157,7 +169,7 @@ jobs:
       - name: Upload artifact (from build tree)
         uses: actions/upload-artifact@v4
         with:
-          name: ERF-win64-Release-mpi-${{ matrix.mpi }}-particles-${{ matrix.particles }}
+          name: ERF-win64-Release-mpi-${{ matrix.mpi }}-mesh-${{ matrix.mesh_precision }}-particles-${{ matrix.particles }}-${{ matrix.particles_precision }}
           path: |
             build
           if-no-files-found: warn

.github/workflows/windows.yml

@@ -8,11 +8,20 @@ concurrency:
 
 jobs:
   WIN32-MSVC10:
-    name: WIN32 MSVC@19.29 - Particles ${{matrix.particles}}
+    name: WIN32 MSVC@19.29 - Mesh ${{matrix.mesh_precision}} Particles ${{matrix.particles}} ${{matrix.particles_precision}}
     runs-on: windows-latest
     strategy:
       matrix:
         particles: [OFF, ON]
+        particles_precision: [DOUBLE, SINGLE]
+        mesh_precision: [DOUBLE, SINGLE]
+        exclude:
+          - particles: OFF
+            particles_precision: SINGLE
+          # SP-mesh entries are temporarily disabled because the upstream
+          # SP-mesh build currently fails on non-particle code. Remove the
+          # guard below to re-enable SP-mesh + DP/SP particles once fixed.
+          - mesh_precision: SINGLE
 
     steps:
 
@@ -28,6 +37,8 @@ jobs:
           -DERF_DIM:STRING=3 `
           -DERF_ENABLE_MPI:BOOL=OFF `
           -DERF_ENABLE_PARTICLES:BOOL=${{matrix.particles}} `
+          -DERF_PRECISION:STRING=${{matrix.mesh_precision}} `
+          -DERF_PARTICLES_PRECISION:STRING=${{matrix.particles_precision}} `
           -DERF_ENABLE_TESTS:BOOL=ON `
           -DERF_ENABLE_ALL_WARNINGS:BOOL=OFF `
           -DERF_ENABLE_FCOMPARE:BOOL=ON `
@@ -38,6 +49,7 @@ jobs:
         cmake --build build --config Debug --parallel 2 --verbose
 
     - name: Check Unit Test Registration
+      if: matrix.particles_precision == 'DOUBLE' && matrix.mesh_precision == 'DOUBLE'
       shell: pwsh
       run: |
         $out = & "${{runner.workspace}}/ERF/build/Tests/Unit/Debug/erf_unit_tests.exe" --gtest_list_tests
@@ -49,11 +61,13 @@ jobs:
       working-directory: ${{runner.workspace}}/ERF/build
 
     - name: CMake Unit Tests
+      if: matrix.particles_precision == 'DOUBLE' && matrix.mesh_precision == 'DOUBLE'
       run: |
         ctest -C Debug -L unit -VV
       working-directory: ${{runner.workspace}}/ERF/build
 
     - name: CMake Regression Tests # see file ERF/Tests/CTestList.cmake
+      if: matrix.particles_precision == 'DOUBLE' && matrix.mesh_precision == 'DOUBLE'
       run: |
         ctest -C Debug -L regression -VV
       working-directory: ${{runner.workspace}}/ERF/build

CMake/SetAmrexOptions.cmake

@@ -5,6 +5,7 @@ set(AMReX_PIC OFF)
 set(AMReX_MPI ${ERF_ENABLE_MPI})
 set(AMReX_OMP ${ERF_ENABLE_OPENMP})
 set(AMReX_PRECISION "${ERF_PRECISION}" CACHE STRING "Floating point precision" FORCE)
+set(AMReX_PARTICLES_PRECISION "${ERF_PARTICLES_PRECISION}" CACHE STRING "Particle floating point precision" FORCE)
 set(AMReX_EB OFF)
 set(AMReX_FORTRAN_INTERFACES OFF)
 set(AMReX_AMRDATA OFF)

CMakeLists.txt

@@ -136,6 +136,7 @@ option(ERF_ENABLE_HDF5 "Enable HDF5 IO" ${ERF_ENABLE_NETCDF})
 option(ERF_ENABLE_PARTICLES "Enable Lagrangian particles" OFF)
 option(ERF_ENABLE_FCOMPARE "Enable building fcompare when not testing" OFF)
 set(ERF_PRECISION "DOUBLE" CACHE STRING "Floating point precision SINGLE or DOUBLE")
+set(ERF_PARTICLES_PRECISION "${ERF_PRECISION}" CACHE STRING "Particle floating point precision SINGLE or DOUBLE")
 
 option(ERF_ENABLE_RRTMGP "Enable RTE-RRTMGP Radiation" OFF)
 option(ERF_ENABLE_ML_UPHYS_DIAGNOSTICS "Enable ML UPHYS diagnostics in SDM" OFF)
@@ -233,8 +234,21 @@ else()
       endif()
     else()
       set(CMAKE_PREFIX_PATH ${AMREX_DIR} ${CMAKE_PREFIX_PATH})
+      if(ERF_PRECISION STREQUAL "SINGLE")
+        set(_amrex_mesh_precision_component "SINGLE")
+      else()
+        set(_amrex_mesh_precision_component "DOUBLE")
+      endif()
+      if(ERF_PARTICLES_PRECISION STREQUAL "SINGLE")
+        set(_amrex_particle_precision_component "PSINGLE")
+      else()
+        set(_amrex_particle_precision_component "PDOUBLE")
+      endif()
       list(APPEND AMREX_COMPONENTS
-        "3D" "PIC" "PARTICLES" "PDOUBLE" "DOUBLE" "LSOLVERS")
+        "3D" "PIC" "PARTICLES"
+        "${_amrex_particle_precision_component}"
+        "${_amrex_mesh_precision_component}"
+        "LSOLVERS")
       if (ERF_ENABLE_MPI)
         list(APPEND AMREX_COMPONENTS "MPI")
       endif()

Docs/sphinx_doc/Particles.rst

@@ -51,6 +51,27 @@ One must also set
 
 in the inputs file or on the command line at runtime.
 
+Particle Precision
+~~~~~~~~~~~~~~~~~~
+
+Particle floating-point data (positions, velocities, and any user-defined
+real-typed attributes) can be stored in single precision independently of
+the mesh-data precision. With cmake, set
+
+::
+
+   -DERF_PARTICLES_PRECISION:STRING=SINGLE \
+
+(default is ``DOUBLE``, which matches ``ERF_PRECISION``). This forwards to
+``AMReX_PARTICLES_PRECISION`` and defines ``AMREX_SINGLE_PRECISION_PARTICLES``.
+With gmake, set
+
+::
+
+   USE_SINGLE_PRECISION_PARTICLES = TRUE
+
+in the GNUmakefile.
+
 Initialization
 --------------
 

Docs/sphinx_doc/buildingSystems.rst

@@ -548,6 +548,14 @@ CMake can also generate makefiles for the Ninja build system for faster compilat
      - Enables support for Lagrangian particles
      - OFF
      - ON/OFF
+   * - ``ERF_PRECISION``
+     - Floating-point precision for mesh data
+     - DOUBLE
+     - SINGLE/DOUBLE
+   * - ``ERF_PARTICLES_PRECISION``
+     - Floating-point precision for particle data; defaults to ``ERF_PRECISION``
+     - DOUBLE
+     - SINGLE/DOUBLE
    * - ``ERF_ENABLE_MULTIBLOCK``
      - Enables multiblock capability
      - OFF