diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 63b2924cfc8..e5cd9019559 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -415,6 +415,21 @@ jobs: DOCKER_CERTS_UPDATE_COMMAND: "update-ca-trust" HOST_CONFIG: /spack-generated-wave-solver-only.cmake + - name: Rockylinux CUDA on T4 (8, clang 17.0.6, cuda 12.9.1) + # GitHub-hosted runner with 1x T4 GPU (16GB VRAM) + BUILD_AND_TEST_CLI_ARGS: "--no-install-schema" + CMAKE_BUILD_TYPE: Release + BUILD_GENERATOR: "--ninja" + ENABLE_HYPRE_DEVICE: CUDA + ENABLE_HYPRE: ON + ENABLE_TRILINOS: OFF + GEOS_ENABLE_BOUNDS_CHECK: OFF + DOCKER_REPOSITORY: geosx/rockylinux8-clang17-cuda12.9.1 + RUNS_ON: GPU + NPROC: 8 + DOCKER_RUN_ARGS: "--cpus=8 --memory=128g --runtime=nvidia --gpus all" + HOST_CONFIG: /spack-generated.cmake + #- name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, cuda 12.4.0,) # BUILD_AND_TEST_CLI_ARGS: "--no-run-unit-tests --no-install-schema" # BUILD_GENERATOR: "--ninja" diff --git a/src/coreComponents/physicsSolvers/solidMechanics/contact/SolidMechanicsAugmentedLagrangianContact.cpp b/src/coreComponents/physicsSolvers/solidMechanics/contact/SolidMechanicsAugmentedLagrangianContact.cpp index df469f26ea1..dbf13d324c8 100644 --- a/src/coreComponents/physicsSolvers/solidMechanics/contact/SolidMechanicsAugmentedLagrangianContact.cpp +++ b/src/coreComponents/physicsSolvers/solidMechanics/contact/SolidMechanicsAugmentedLagrangianContact.cpp @@ -35,6 +35,7 @@ #include "constitutive/contact/FrictionSelector.hpp" #include "constitutive/solid/PorousSolid.hpp" #include "constitutive/solid/SolidFields.hpp" +#include "finiteElement/FiniteElementDiscretization.hpp" #include "mesh/DomainPartition.hpp" namespace geos @@ -48,10 +49,6 @@ SolidMechanicsAugmentedLagrangianContact::SolidMechanicsAugmentedLagrangianConta Group * const parent ): ContactSolverBase( name, parent ) { - - m_faceTypeToFiniteElements.insert( {"Quadrilateral", std::make_unique< finiteElement::H1_QuadrilateralFace_Lagrange1_GaussLegendre2 >()} ); - m_faceTypeToFiniteElements.insert( {"Triangle", std::make_unique< finiteElement::H1_TriangleFace_Lagrange1_Gauss1 >()} ); - registerWrapper( viewKeyStruct::simultaneousString(), &m_simultaneous ). setInputFlag( InputFlags::OPTIONAL ). setApplyDefaultValue( 1 ). @@ -167,6 +164,54 @@ void SolidMechanicsAugmentedLagrangianContact::registerDataOnMesh( dataRepositor } +void SolidMechanicsAugmentedLagrangianContact::initializePostInitialConditionsPreSubGroups() +{ + ContactSolverBase::initializePostInitialConditionsPreSubGroups(); + + DomainPartition & domain = this->getGroupByPath< DomainPartition >( "/Problem/domain" ); + validateTetrahedralQuadrature( domain.getMeshBodies() ); +} + +void SolidMechanicsAugmentedLagrangianContact::validateTetrahedralQuadrature( Group & meshBodies ) +{ + string const discretizationName = getDiscretizationName(); + + NumericalMethodsManager const & numericalMethodManager = + this->getGroupByPath< DomainPartition >( "/Problem/domain" ).getNumericalMethodManager(); + FiniteElementDiscretizationManager const & feDiscretizationManager = + numericalMethodManager.getFiniteElementDiscretizationManager(); + FiniteElementDiscretization const & feDiscretization = + feDiscretizationManager.getGroup< FiniteElementDiscretization >( discretizationName ); + + + integer const useHighOrderQuadrature = + feDiscretization.getReference< integer >( "useHighOrderQuadratureRule" ); + + bool hasTetrahedra = false; + forDiscretizationOnMeshTargets( meshBodies, [&]( string const &, + MeshLevel const & mesh, + string_array const & regionNames ) + { + ElementRegionManager const & elemManager = mesh.getElemManager(); + elemManager.forElementRegions< CellElementRegion >( regionNames, [&]( localIndex const, + CellElementRegion const & region ) + { + region.forElementSubRegions< CellElementSubRegion >( [&]( CellElementSubRegion const & subRegion ) + { + if( subRegion.getElementType() == ElementType::Tetrahedron ) + { + hasTetrahedra = true; + } + } ); + } ); + } ); + + GEOS_ERROR_IF( hasTetrahedra && useHighOrderQuadrature != 1, + GEOS_FMT( "{}: Tetrahedral meshes require useHighOrderQuadratureRule=\"1\" for correct integration of bubble contributions. " + "Please add this attribute to your FiniteElements/{} XML block.", + getName(), discretizationName ) ); +} + void SolidMechanicsAugmentedLagrangianContact::setupDofs( DomainPartition const & domain, DofManager & dofManager ) const { @@ -219,6 +264,29 @@ void SolidMechanicsAugmentedLagrangianContact::setupSystem( DomainPartition & do PhysicsSolverBase::setupSystem( domain, dofManager, localMatrix, rhs, solution, setSparsity ); } +void SolidMechanicsAugmentedLagrangianContact::postInputInitialization() +{ + ContactSolverBase::postInputInitialization(); + + DomainPartition & domain = this->getGroupByPath< DomainPartition >( "/Problem/domain" ); + NumericalMethodsManager const & numericalMethodManager = domain.getNumericalMethodManager(); + FiniteElementDiscretizationManager const & feDiscretizationManager = + numericalMethodManager.getFiniteElementDiscretizationManager(); + FiniteElementDiscretization const & feDiscretization = + feDiscretizationManager.getGroup< FiniteElementDiscretization >( getDiscretizationName() ); + + m_faceTypeToFiniteElements.insert( {"Quadrilateral", feDiscretization.factory( ElementType::Quadrilateral )} ); + m_faceTypeToFiniteElements.insert( {"Triangle", feDiscretization.factory( ElementType::Triangle )} ); + + GEOS_LOG_RANK_0( GEOS_FMT( "{} using finite element discretization {}:", + getName(), getDiscretizationName() ) ); + for( auto const & [name, fePtr] : m_faceTypeToFiniteElements ) + { + GEOS_LOG_RANK_0( GEOS_FMT( " {} face elements: {} quadrature points", + name, fePtr->getNumQuadraturePoints() ) ); + } +} + void SolidMechanicsAugmentedLagrangianContact::setSparsityPattern( DomainPartition & domain, DofManager & dofManager, CRSMatrix< real64, globalIndex > & GEOS_UNUSED_PARAM( localMatrix ), diff --git a/src/coreComponents/physicsSolvers/solidMechanics/contact/SolidMechanicsAugmentedLagrangianContact.hpp b/src/coreComponents/physicsSolvers/solidMechanics/contact/SolidMechanicsAugmentedLagrangianContact.hpp index af1dc42caf8..c3758556deb 100644 --- a/src/coreComponents/physicsSolvers/solidMechanics/contact/SolidMechanicsAugmentedLagrangianContact.hpp +++ b/src/coreComponents/physicsSolvers/solidMechanics/contact/SolidMechanicsAugmentedLagrangianContact.hpp @@ -47,8 +47,12 @@ class SolidMechanicsAugmentedLagrangianContact : public ContactSolverBase */ string getCatalogName() const override { return catalogName(); } + virtual void postInputInitialization() override; + virtual void registerDataOnMesh( dataRepository::Group & meshBodies ) override final; + virtual void initializePostInitialConditionsPreSubGroups() override; + virtual void setupDofs( DomainPartition const & domain, DofManager & dofManager ) const override; @@ -213,6 +217,12 @@ class SolidMechanicsAugmentedLagrangianContact : public ContactSolverBase private: + /** + * @brief Validate that tetrahedral meshes use high-order quadrature rules + * @param meshBodies the group containing the mesh bodies + */ + void validateTetrahedralQuadrature( Group & meshBodies ); + /** * @brief add the number of non-zero elements induced by the coupling between * nodal and bubble displacement. diff --git a/src/coreComponents/physicsSolvers/solidMechanics/contact/SolidMechanicsLagrangeContactBubbleStab.cpp b/src/coreComponents/physicsSolvers/solidMechanics/contact/SolidMechanicsLagrangeContactBubbleStab.cpp index ff06db81bfb..d73fb84619a 100644 --- a/src/coreComponents/physicsSolvers/solidMechanics/contact/SolidMechanicsLagrangeContactBubbleStab.cpp +++ b/src/coreComponents/physicsSolvers/solidMechanics/contact/SolidMechanicsLagrangeContactBubbleStab.cpp @@ -27,7 +27,7 @@ #include "physicsSolvers/solidMechanics/contact/kernels/SolidMechanicsContactFaceBubbleKernels.hpp" #include "physicsSolvers/solidMechanics/contact/LogLevelsInfo.hpp" #include "physicsSolvers/LogLevelsInfo.hpp" - +#include "finiteElement/FiniteElementDiscretization.hpp" #include "constitutive/contact/FrictionSelector.hpp" #include "fieldSpecification/FieldSpecificationManager.hpp" @@ -43,9 +43,6 @@ SolidMechanicsLagrangeContactBubbleStab::SolidMechanicsLagrangeContactBubbleStab Group * const parent ): ContactSolverBase( name, parent ) { - m_faceTypeToFiniteElements.insert( {"Quadrilateral", std::make_unique< finiteElement::H1_QuadrilateralFace_Lagrange1_GaussLegendre2 >()} ); - m_faceTypeToFiniteElements.insert( {"Triangle", std::make_unique< finiteElement::H1_TriangleFace_Lagrange1_Gauss1 >()} ); - LinearSolverParameters & linSolParams = m_linearSolverParameters.get(); linSolParams.mgr.strategy = LinearSolverParameters::MGR::StrategyType::lagrangianContactMechanicsBubbleStab; linSolParams.mgr.separateComponents = true; @@ -134,6 +131,54 @@ void SolidMechanicsLagrangeContactBubbleStab::registerDataOnMesh( Group & meshBo } ); } +void SolidMechanicsLagrangeContactBubbleStab::initializePostInitialConditionsPreSubGroups() +{ + ContactSolverBase::initializePostInitialConditionsPreSubGroups(); + + DomainPartition & domain = this->getGroupByPath< DomainPartition >( "/Problem/domain" ); + validateTetrahedralQuadrature( domain.getMeshBodies() ); +} + +void SolidMechanicsLagrangeContactBubbleStab::validateTetrahedralQuadrature( Group & meshBodies ) +{ + string const discretizationName = getDiscretizationName(); + + NumericalMethodsManager const & numericalMethodManager = + this->getGroupByPath< DomainPartition >( "/Problem/domain" ).getNumericalMethodManager(); + FiniteElementDiscretizationManager const & feDiscretizationManager = + numericalMethodManager.getFiniteElementDiscretizationManager(); + FiniteElementDiscretization const & feDiscretization = + feDiscretizationManager.getGroup< FiniteElementDiscretization >( discretizationName ); + + + integer const useHighOrderQuadrature = + feDiscretization.getReference< integer >( "useHighOrderQuadratureRule" ); + + bool hasTetrahedra = false; + forDiscretizationOnMeshTargets( meshBodies, [&]( string const &, + MeshLevel const & mesh, + string_array const & regionNames ) + { + ElementRegionManager const & elemManager = mesh.getElemManager(); + elemManager.forElementRegions< CellElementRegion >( regionNames, [&]( localIndex const, + CellElementRegion const & region ) + { + region.forElementSubRegions< CellElementSubRegion >( [&]( CellElementSubRegion const & subRegion ) + { + if( subRegion.getElementType() == ElementType::Tetrahedron ) + { + hasTetrahedra = true; + } + } ); + } ); + } ); + + GEOS_ERROR_IF( hasTetrahedra && useHighOrderQuadrature != 1, + GEOS_FMT( "{}: Tetrahedral meshes require useHighOrderQuadratureRule=\"1\" for correct integration of bubble contributions. " + "Please add this attribute to your FiniteElements/{} XML block.", + getName(), discretizationName ) ); +} + void SolidMechanicsLagrangeContactBubbleStab::setupDofs( DomainPartition const & domain, DofManager & dofManager ) const { @@ -192,6 +237,29 @@ void SolidMechanicsLagrangeContactBubbleStab::setupDofs( DomainPartition const & meshTargets ); } +void SolidMechanicsLagrangeContactBubbleStab::postInputInitialization() +{ + ContactSolverBase::postInputInitialization(); + + DomainPartition & domain = this->getGroupByPath< DomainPartition >( "/Problem/domain" ); + NumericalMethodsManager const & numericalMethodManager = domain.getNumericalMethodManager(); + FiniteElementDiscretizationManager const & feDiscretizationManager = + numericalMethodManager.getFiniteElementDiscretizationManager(); + FiniteElementDiscretization const & feDiscretization = + feDiscretizationManager.getGroup< FiniteElementDiscretization >( getDiscretizationName() ); + + m_faceTypeToFiniteElements.insert( {"Quadrilateral", feDiscretization.factory( ElementType::Quadrilateral )} ); + m_faceTypeToFiniteElements.insert( {"Triangle", feDiscretization.factory( ElementType::Triangle )} ); + + GEOS_LOG_RANK_0( GEOS_FMT( "{} using finite element discretization {}:", + getName(), getDiscretizationName() ) ); + for( auto const & [name, fePtr] : m_faceTypeToFiniteElements ) + { + GEOS_LOG_RANK_0( GEOS_FMT( " {} face elements: {} quadrature points", + name, fePtr->getNumQuadraturePoints() ) ); + } +} + void SolidMechanicsLagrangeContactBubbleStab::setupSystem( DomainPartition & domain, DofManager & dofManager, CRSMatrix< real64, globalIndex > & localMatrix, diff --git a/src/coreComponents/physicsSolvers/solidMechanics/contact/SolidMechanicsLagrangeContactBubbleStab.hpp b/src/coreComponents/physicsSolvers/solidMechanics/contact/SolidMechanicsLagrangeContactBubbleStab.hpp index 6c8562044e4..d17b43a3c2d 100644 --- a/src/coreComponents/physicsSolvers/solidMechanics/contact/SolidMechanicsLagrangeContactBubbleStab.hpp +++ b/src/coreComponents/physicsSolvers/solidMechanics/contact/SolidMechanicsLagrangeContactBubbleStab.hpp @@ -50,6 +50,10 @@ class SolidMechanicsLagrangeContactBubbleStab : public ContactSolverBase */ string getCatalogName() const override { return catalogName(); } + virtual void postInputInitialization() override; + + virtual void initializePostInitialConditionsPreSubGroups() override; + virtual void registerDataOnMesh( Group & MeshBodies ) override final; real64 solverStep( real64 const & time_n, @@ -200,6 +204,12 @@ class SolidMechanicsLagrangeContactBubbleStab : public ContactSolverBase private: + /** + * @brief Validate that tetrahedral meshes use high-order quadrature rules + * @param meshBodies the group containing the mesh bodies + */ + void validateTetrahedralQuadrature( Group & meshBodies ); + /** * @brief add the number of non-zero elements induced by the coupling between * nodal and bubble displacement.