Phasechange

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,30 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Compile using
+2. Run example
+3. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Environment**
+ - OS: [e.g. MacOS Monterey]
+ - Compiler versions [e.g., gfortran 11.0.2 and open-mpi 4.1]
+
+**Additional context**
+Add any other context about the problem here.

.github/PULL_REQUEST_TEMPLATE/pull_request_template.md

@@ -0,0 +1,35 @@
+# Description
+
+Please include a summary of the changes and the related issue. Please also include relevant motivation and context. List any dependencies that are required for this change.
+
+Fixes # (issue)
+
+## Type of change
+
+Please delete options that are not relevant.
+
+- [ ] Bug fix (non-breaking change which fixes an issue)
+- [ ] New feature (non-breaking change which adds functionality)
+- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
+- [ ] This change requires a documentation update
+
+# How Has This Been Tested?
+
+Please describe the tests that you ran to verify your changes. 
+Provide instructions so we can reproduce. 
+Please also list any relevant details for your test configuration
+
+- [ ] Test A
+- [ ] Test B
+
+**Test Configuration**:
+* What computers and compilers did you use to test this:
+
+# Checklist:
+
+- [ ] I have performed a self-review of my code
+- [ ] I have commented my code, particularly in hard-to-understand areas
+- [ ] I have made corresponding changes to the documentation
+- [ ] I have added tests that prove my fix is effective or that my feature works
+- [ ] New and existing tests pass locally with my changes
+- [ ] Any dependent changes have been merged and published in downstream modules

.github/workflows/ci.yml

@@ -0,0 +1,117 @@
+name: CI
+
+on:
+  push:
+    branches: [ master, GPU ]
+    paths:
+      - '**.f90'
+      - '**.fpp'
+      - '**.sh'
+      - '**.py'
+      - '**.yaml'
+      - '**.yml'
+      - 'golden.txt'
+      - 'Makefile'
+      - 'makefile'
+
+  pull_request:
+    branches: [ master, GPU ]
+
+  workflow_dispatch:
+
+jobs:
+  CI:
+    strategy:
+      matrix:
+        include:
+        - os:    ubuntu-latest
+          nvhpc: 22.3
+          cuda:  11.6
+        - os:    macos-latest
+          gcc:   11
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Clone MFC
+        uses: actions/checkout@v2
+
+      - name: Post-Clone Setup
+        run: |
+          chmod +x ./mfc.sh
+
+      - name: Setup & Environment Variables
+        if:   matrix.os == 'macos-latest'
+        run: |
+          echo "HOMEBREW_CC=gcc-${{ matrix.gcc }}"  >> $GITHUB_ENV
+          echo "HOMEBREW_CXX=g++-${{ matrix.gcc }}" >> $GITHUB_ENV
+          echo "OMPI_MPICC=gcc-${{ matrix.gcc }}"   >> $GITHUB_ENV
+          echo "OMPI_CXX=g++-${{ matrix.gcc }}"     >> $GITHUB_ENV
+          echo "OMPI_FC=gfortran-${{ matrix.gcc }}" >> $GITHUB_ENV
+
+      - name: Install Dependencies (via Aptitude)
+        if:   matrix.os == 'ubuntu-latest'
+        run: >-
+          sudo apt install -y tar wget make cmake gcc g++ python3 openmpi-*
+          python python-dev python3-dev python3-venv libopenmpi-dev
+
+      - name: Install Nvidia HPC SDK
+        if:   matrix.os == 'ubuntu-latest' && github.ref == 'refs/heads/GPU'
+        run: |
+          echo 'deb [trusted=yes] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list
+          sudo apt-get update  -y
+          sudo apt-get install -y nvhpc-$(echo "${{ matrix.nvhpc }}" | tr '.' '-')
+          NVARCH=`uname -s`_`uname -m`
+          NVCOMPILERS="/opt/nvidia/hpc_sdk"
+          VERSION="${{ matrix.nvhpc }}"
+          CUDA_VERSION="${{ matrix.cuda }}"
+          echo "PATH=$NVCOMPILERS/$NVARCH/$VERSION/compilers/bin:$NVCOMPILERS/$NVARCH/$VERSION/comm_libs/mpi/bin:$PATH" >> $GITHUB_ENV
+          echo "CUDA_HOME=$NVCOMPILERS/$NVARCH/$VERSION/cuda/" >> $GITHUB_ENV
+          echo "CUDA_DIR=$NVCOMPILERS/$NVARCH/$VERSION/cuda/" >> $GITHUB_ENV
+          echo "LD_LIBRARY_PATH=$NVCOMPILERS/$NVARCH/$VERSION/cuda/$CUDA_VERSION/lib64:$LD_LIBRARY_PATH" >> $GITHUB_ENV
+
+      - name: Install Dependencies (via Brew)
+        if: matrix.os == 'macos-latest'
+        run: |
+          brew install wget make python make cmake coreutils
+          brew install gcc@${{ matrix.gcc }}
+          brew install --build-from-source open-mpi
+
+      - name: Print MPI Configuration
+        run: |
+          mpif90 --version
+          mpif90 --showme
+          mpicc --version
+          mpicc --showme
+          mpicxx --version
+          mpicxx --showme
+
+      - name: Build MFC (CPU)
+        run:  ./mfc.sh build -m release-cpu -j $(nproc)
+
+      - name: Test MFC (CPU)
+        run:  ./mfc.sh test -j $(nproc)
+
+      - name: Print MPI Configuration
+        run: |
+          mpif90 --version
+          mpif90 --showme
+          mpicc --version
+          mpicc --showme
+          mpicxx --version
+          mpicxx --showme
+
+      - name: Build MFC (GPU)
+        if:   github.ref == 'refs/heads/GPU'
+        run:  ./mfc.sh build -m release-gpu -j $(nproc)
+
+      - name: Upload logs
+        uses: actions/upload-artifact@v2
+        if:   always()
+        with:
+          name: ${{ matrix.os }}
+          path: |
+            build/**/*.log
+            tests/**/*.dat
+            tests/**/*.out
+            tests/**/*.inp
+            tests/**/*.txt
+            tests/*.txt

.gitignore

@@ -1,35 +1,72 @@
-doc/Doxygen/*/html
-doc/Doxygen/*logo*
-doc/Doxygen/*icon*
+build/
+tags
+src/*/tags
+
+__pycache__
+
 src/*/*.mod
 src/*/*.o
 src/pre_process_code/pre_process
 src/simulation_code/simulation
 src/post_process_code/post_process
 src/master_scripts/*.pyc
+src/*/.build
 src/master_scripts/__*
 make_doc.sh
 cases/
+
 .DS_Store
+
 tests/*/D/*
 tests/*/p*
 tests/*/D_*
+tests/*/*.inp
 tests/*/*.inf
 tests/*/*.inp
 tests/*/*.dat
 tests/*/silo*
 tests/*/restart*
 tests/*/*.out
 tests/*/binary
-example_cases/*/D/*
-example_cases/*/p*
-example_cases/*/D_*
-example_cases/*/*.inf
-example_cases/*/*.inp
-example_cases/*/*.dat
-example_cases/*/silo*
-example_cases/*/restart*
-example_cases/*/*.out
-example_cases/*/binary
-lib/fftw-3.3.8
-lib/lapack-3.4.2
+tests/**/*.log
+tests/*.txt
+tests/*/**
+!**/golden.txt
+
+samples/*batch/*/
+samples/*/D/*
+samples/*/p*
+samples/*/D_*
+samples/*/*.inf
+samples/*/*.inp
+samples/*/*.dat
+samples/*/*.o*
+samples/*/silo*
+samples/*/restart*
+samples/*/*.out
+samples/*/binary
+samples/*/pre_process.sh
+samples/*/simulation.sh
+samples/*/post_process.sh
+
+doc/*/MFC_user_guide.bbl
+doc/*/MFC_user_guide.blg
+doc/*/MFC_user_guide.aux
+doc/*/MFC_user_guide.toc
+doc/*/MFC_user_guide.fls
+doc/*/MFC_user_guide.synctex.gz
+doc/*/MFC_user_guide.log
+doc/*/MFC_user_guide.fdb_latexmk
+
+doc/Doxygen/*/html
+doc/Doxygen/*icon*
+tests_hypoelastic/*/D/*
+tests_hypoelastic/*/p*
+tests_hypoelastic/*/D_*
+tests_hypoelastic/*/*.inf
+tests_hypoelastic/*/*.inp
+tests_hypoelastic/*/*.dat
+tests_hypoelastic/*/silo*
+tests_hypoelastic/*/restart*
+tests_hypoelastic/*/*.out
+tests_hypoelastic/*/binary

.gitlab-ci.yml

@@ -0,0 +1,93 @@
+# CI Workflow for MFC running Ascent @ ORNL through GitLab.
+
+# GitLab: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html
+#  - User Set:
+#    - $GIT_STRATEGY: none -> Runner should not clone repository before executing script
+#  - User Read:
+#    - $CI_PROJECT_NAME   -> e.g MFC, MFC-develop, ...
+#    - $CI_REPOSITORY_URL -> e.g https://gitlab-ci-token:[MASKED]@code.ornl.gov/ecpcitest/cfd154/MFC-develop.git
+#    - $CI_COMMIT_BRANCH  -> e.g GPU, master, ...
+# ORNL's Ascent:
+#  - User Set:
+#    - $SCHEDULER_PARAMETERS
+#  - User Read:
+#    - $MEMBERWORK -> e.g /gpfs/wolf/scratch/henrylb
+
+stages:
+  - clone # Clone MFC after removing artifacts from previous run(s). 
+  - cpu   # Build MFC & Test it on/with CPUs
+  - gpu   # Build MFC & Test it on/with GPUs
+
+# Variables for all jobs
+variables:
+  # Tell runner to not clone MFC
+  GIT_STRATEGY: none 
+  # Ascent Job Scheduler parameters
+  SCHEDULER_PARAMETERS: "-P CFD154 -nnodes 1 -W 00:30"
+
+# This script runs before each job's main script is executed.
+# It defines useful environment variables used at runtime.
+before_script:
+  # Check "$CI_PROJECT_NAME" contains "MFC" so we are sure
+  # we won't clone into or delete from an unsuspecting directory
+  # due to an unfortunate error.
+  - echo "[CI] Running fin $(pwd):"
+  - if [[ ! "$CI_PROJECT_NAME" == *"MFC"* ]]; then exit 1; fi;
+
+  # Define MFC directory path
+  - CI_MFC_DIR="$MEMBERWORK/cfd154/.ci/$CI_PROJECT_NAME/$CI_COMMIT_BRANCH"
+
+# Clone MFC after removing artifacts from previous run(s). 
+clone-job:
+  stage: clone
+  script:
+    # Clean MFC directory if it exists (from a previous run)
+    - if [ -d "$CI_MFC_DIR" ]; then rm -rf "$CI_MFC_DIR"; fi 
+    # Create empty MFC directory
+    - mkdir -p "$CI_MFC_DIR"
+
+    # Clone MFC into MFC directory & cd into it
+    - git clone --single-branch --branch "$CI_COMMIT_BRANCH" "$CI_REPOSITORY_URL" "$CI_MFC_DIR/"
+    - cd "$CI_MFC_DIR" && echo "[CI] Running in $(pwd):"
+  tags:
+    - nobatch
+
+cpu-build-job:
+  stage: cpu
+  script:
+    - cd "$CI_MFC_DIR" && echo "[CI] Running in $(pwd):"
+    - . ./mfc.sh load -c a -m g
+    - ./mfc.sh build -j 8 -m release-cpu
+  tags:
+    - nobatch
+
+cpu-test-job:
+  stage: cpu
+  needs:
+    - cpu-build-job
+  script:
+    - cd "$CI_MFC_DIR" && echo "[CI] Running in $(pwd):"
+    - . ./mfc.sh load -c a -m g
+    - ./mfc.sh test -j 8
+  tags:
+    - batch
+
+gpu-build-job:
+  stage: gpu
+  script:
+    - cd "$CI_MFC_DIR" && echo "[CI] Running in $(pwd):"
+    - . ./mfc.sh load -c a -m g
+    - ./mfc.sh build -j 8 -m release-gpu
+  tags:
+    - nobatch
+
+gpu-test-job:
+  stage: gpu
+  needs:
+    - gpu-build-job
+  script:
+    - cd "$CI_MFC_DIR" && echo "[CI] Running in $(pwd):"
+    - . ./mfc.sh load -c a -m g
+    - ./mfc.sh test -j 8
+  tags:
+    - batch

.mfc/___current___

@@ -0,0 +1 @@
+/gpfs/home/mrodri97/MFC-develop/bootstrap/../.mfc/release-cpu