SPC

configure.ac

@@ -3,7 +3,7 @@
 # Copyright (c) 2004-2009 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2015 The University of Tennessee and The University
+# Copyright (c) 2004-2018 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
 # Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
@@ -287,6 +287,24 @@ AS_IF([test "$enable_oshmem" != "no"], [project_oshmem_amc=true], [project_oshme
 m4_ifndef([project_oshmem], [project_oshmem_amc=false])
 AM_CONDITIONAL([PROJECT_OSHMEM], [test "$project_oshmem_amc" = "true"])
 
+# Enable/Disable Software-Based Performance Counters Capability
+AC_ARG_ENABLE(spc,
+    AC_HELP_STRING([--enable-spc],
+                   [Enable software-based performance counters capability (default: disabled)]))
+if test "$enable_spc" = "yes"; then
+    AC_MSG_RESULT([yes])
+    SPC_ENABLE=1
+else
+    AC_MSG_RESULT([no])
+    SPC_ENABLE=0
+fi
+AC_DEFINE_UNQUOTED([SPC_ENABLE],
+                   [$SPC_ENABLE],
+                   [If the software-based performance counters capability should be enabled.])
+AM_CONDITIONAL(SPC_ENABLE, test "$SPC_ENABLE" = "1")
+
+AS_IF([test "$enable_spc" != "no"], [project_spc_amc=true], [project_spc_amc=false])
+
 if test "$enable_binaries" = "no" && test "$enable_dist" = "yes"; then
     AC_MSG_WARN([--disable-binaries is incompatible with --enable dist])
     AC_MSG_ERROR([Cannot continue])
@@ -1427,7 +1445,7 @@ AC_CONFIG_FILES([
     test/util/Makefile
 ])
 
-m4_ifdef([project_ompi], [AC_CONFIG_FILES([test/monitoring/Makefile])])
+m4_ifdef([project_ompi], [AC_CONFIG_FILES([test/monitoring/Makefile test/spc/Makefile])])
 
 AC_CONFIG_FILES([contrib/dist/mofed/debian/rules],
                 [chmod +x contrib/dist/mofed/debian/rules])

examples/Makefile

@@ -2,7 +2,7 @@
 # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
+# Copyright (c) 2004-2018 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
 # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -67,14 +67,15 @@ EXAMPLES = \
 	 oshmem_circular_shift \
 	 oshmem_max_reduction \
 	 oshmem_strided_puts \
-	 oshmem_symmetric_data
+	 oshmem_symmetric_data \
+	 spc_example
 
 
 # Default target.  Always build the C MPI examples.  Only build the
 # others if we have the appropriate Open MPI / OpenSHMEM language
 # bindings.
 
-all: hello_c ring_c connectivity_c
+all: hello_c ring_c connectivity_c spc_example
 	@ if which ompi_info >/dev/null 2>&1 ; then \
 	    $(MAKE) mpi; \
 	fi

examples/Makefile.include

@@ -3,7 +3,7 @@
 # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
+# Copyright (c) 2004-2018 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
 # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -57,4 +57,5 @@ EXTRA_DIST += \
         examples/oshmem_strided_puts.c \
         examples/oshmem_symmetric_data.c \
         examples/Hello.java \
-        examples/Ring.java
+        examples/Ring.java \
+        examples/spc_example.c

examples/spc_example.c

@@ -0,0 +1,121 @@
+/*
+ * Copyright (c) 2018      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ *
+ * Simple example usage of SPCs through MPI_T.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <mpi.h>
+
+/* Sends 'num_messages' messages of 'message_size' bytes from rank 0 to rank 1.
+ * All messages are send synchronously and with the same tag in MPI_COMM_WORLD.
+ */
+void message_exchange(int num_messages, int message_size)
+{
+    int i, rank;
+    /* Use calloc to initialize data to 0's */
+    char *data = (char*)calloc(message_size, sizeof(char));
+    MPI_Status status;
+
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+    if(rank == 0) {
+        for(i = 0; i < num_messages; i++)
+            MPI_Send(data, message_size, MPI_BYTE, 1, 123, MPI_COMM_WORLD);
+    } else if(rank == 1) {
+        for(i = 0; i < num_messages; i++)
+            MPI_Recv(data, message_size, MPI_BYTE, 0, 123, MPI_COMM_WORLD, &status);
+    }
+
+    free(data);
+}
+
+int main(int argc, char **argv)
+{
+    int num_messages, message_size;
+
+    if(argc < 3) {
+        printf("Usage: mpirun -np 2 --mca mpi_spc_attach all --mca mpi_spc_dump_enabled true ./test [num_messages] [message_size]\n");
+        return -1;
+    } else {
+        num_messages = atoi(argv[1]);
+        message_size = atoi(argv[2]);
+    }
+
+    int i, rank, size, provided, num, name_len, desc_len, verbosity, bind, var_class, readonly, continuous, atomic, count, index;
+    MPI_Datatype datatype;
+    MPI_T_enum enumtype;
+    MPI_Comm comm;
+    char name[256], description[256];
+
+    /* Counter names to be read by ranks 0 and 1 */
+    char counter_names[2][40];
+    sprintf(counter_names[0], "runtime_spc_OMPI_BYTES_SENT_USER");
+    sprintf(counter_names[1], "runtime_spc_OMPI_BYTES_RECEIVED_USER");
+
+    MPI_Init(NULL, NULL);
+    MPI_T_init_thread(MPI_THREAD_SINGLE, &provided);
+
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &size);
+    if(size != 2) {
+        fprintf(stderr, "ERROR: This test should be run with two MPI processes.\n");
+        return -1;
+    }
+
+    /* Determine the MPI_T pvar indices for the OMPI_BYTES_SENT/RECIEVED_USER SPCs */
+    index = -1;
+    MPI_T_pvar_get_num(&num);
+    for(i = 0; i < num; i++) {
+        name_len = desc_len = 256;
+        PMPI_T_pvar_get_info(i, name, &name_len, &verbosity,
+                             &var_class, &datatype, &enumtype, description, &desc_len, &bind,
+                             &readonly, &continuous, &atomic);
+        if(strcmp(name, counter_names[rank]) == 0) {
+            index = i;
+            printf("[%d] %s -> %s\n", rank, name, description);
+        }
+    }
+
+    /* Make sure we found the counters */
+    if(index == -1) {
+        fprintf(stderr, "ERROR: Couldn't find the appropriate SPC counter in the MPI_T pvars.\n");
+        return -1;
+    }
+
+    int ret;
+    long long value;
+
+    MPI_T_pvar_session session;
+    MPI_T_pvar_handle handle;
+    /* Create the MPI_T sessions/handles for the counters and start the counters */
+    ret = MPI_T_pvar_session_create(&session);
+    ret = MPI_T_pvar_handle_alloc(session, index, NULL, &handle, &count);
+    ret = MPI_T_pvar_start(session, handle);
+
+    message_exchange(num_messages, message_size);
+
+    ret = MPI_T_pvar_read(session, handle, &value);
+    /* Print the counter values in order by rank */
+    for(i = 0; i < 2; i++) {
+        if(i == rank) {
+            printf("[%d] Value Read: %lld\n", rank, value);
+            fflush(stdout);
+        }
+        MPI_Barrier(MPI_COMM_WORLD);
+    }
+    /* Stop the MPI_T session, free the handle, and then free the session */
+    ret = MPI_T_pvar_stop(session, handle);
+    ret = MPI_T_pvar_handle_free(session, &handle);
+    ret = MPI_T_pvar_session_free(&session);
+
+    MPI_T_finalize();
+    MPI_Finalize();
+
+    return 0;
+}

ompi/mca/pml/ob1/pml_ob1.c

@@ -36,6 +36,7 @@
 #include "opal_stdint.h"
 #include "opal/mca/btl/btl.h"
 #include "opal/mca/btl/base/base.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #include "ompi/mca/pml/pml.h"
 #include "ompi/mca/pml/base/base.h"

ompi/mca/pml/ob1/pml_ob1_isend.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2015 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -28,6 +28,7 @@
 #include "pml_ob1_sendreq.h"
 #include "pml_ob1_recvreq.h"
 #include "ompi/peruse/peruse-internal.h"
+#include "ompi/runtime/ompi_spc.h"
 
 /**
  * Single usage request. As we allow recursive calls (as an
@@ -119,6 +120,11 @@ static inline int mca_pml_ob1_send_inline (const void *buf, size_t count,
     rc = mca_bml_base_sendi (bml_btl, &convertor, &match, OMPI_PML_OB1_MATCH_HDR_LEN,
                              size, MCA_BTL_NO_ORDER, MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP,
                              MCA_PML_OB1_HDR_TYPE_MATCH, NULL);
+
+    if(OPAL_LIKELY(rc == OPAL_SUCCESS)) {
+        SPC_USER_OR_MPI(tag, (long long)size, OMPI_BYTES_SENT_USER, OMPI_BYTES_SENT_MPI);
+    }
+
     if (count > 0) {
         opal_convertor_cleanup (&convertor);
     }

ompi/mca/pml/ob1/pml_ob1_recvfrag.c

@@ -39,6 +39,7 @@
 #include "ompi/mca/pml/pml.h"
 #include "ompi/peruse/peruse-internal.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #include "pml_ob1.h"
 #include "pml_ob1_comm.h"
@@ -314,9 +315,8 @@ check_cantmatch_for_match(mca_pml_ob1_comm_proc_t *proc)
 {
     mca_pml_ob1_recv_frag_t *frag = proc->frags_cant_match;
 
-    if( (NULL != frag) && (frag->hdr.hdr_match.hdr_seq == proc->expected_sequence) ) {
+    if( (NULL != frag) && (frag->hdr.hdr_match.hdr_seq == proc->expected_sequence) )
         return remove_head_from_ordered_list(&proc->frags_cant_match);
-    }
     return NULL;
 }
 
@@ -388,6 +388,7 @@ void mca_pml_ob1_recv_frag_callback_match(mca_btl_base_module_t* btl,
             MCA_PML_OB1_RECV_FRAG_ALLOC(frag);
             MCA_PML_OB1_RECV_FRAG_INIT(frag, hdr, segments, num_segments, btl);
             append_frag_to_ordered_list(&proc->frags_cant_match, frag, proc->expected_sequence);
+            SPC_RECORD(OMPI_OUT_OF_SEQUENCE, 1);
             OB1_MATCHING_UNLOCK(&comm->matching_lock);
             return;
         }
@@ -453,6 +454,8 @@ void mca_pml_ob1_recv_frag_callback_match(mca_btl_base_module_t* btl,
                                    &iov_count,
                                    &bytes_received );
             match->req_bytes_received = bytes_received;
+            SPC_USER_OR_MPI(match->req_recv.req_base.req_ompi.req_status.MPI_TAG, (long long)bytes_received,
+                            OMPI_BYTES_RECEIVED_USER, OMPI_BYTES_RECEIVED_MPI);
             /*
              *  Unpacking finished, make the user buffer unaccessable again.
              */
@@ -777,6 +780,11 @@ match_one(mca_btl_base_module_t *btl,
           mca_pml_ob1_comm_proc_t *proc,
           mca_pml_ob1_recv_frag_t* frag)
 {
+#if SPC_ENABLE == 1
+    opal_timer_t timer = 0;
+#endif
+    SPC_TIMER_START(OMPI_MATCH_TIME, &timer);
+
     mca_pml_ob1_recv_request_t *match;
     mca_pml_ob1_comm_t *comm = (mca_pml_ob1_comm_t *)comm_ptr->c_pml_comm;
 
@@ -814,19 +822,25 @@ match_one(mca_btl_base_module_t *btl,
                                                        num_segments);
                 /* this frag is already processed, so we want to break out
                    of the loop and not end up back on the unexpected queue. */
+                SPC_TIMER_STOP(OMPI_MATCH_TIME, &timer);
                 return NULL;
             }
 
             PERUSE_TRACE_COMM_EVENT(PERUSE_COMM_MSG_MATCH_POSTED_REQ,
                                     &(match->req_recv.req_base), PERUSE_RECV);
+            SPC_TIMER_STOP(OMPI_MATCH_TIME, &timer);
             return match;
         }
 
         /* if no match found, place on unexpected queue */
         append_frag_to_list(&proc->unexpected_frags, btl, hdr, segments,
                             num_segments, frag);
+        SPC_RECORD(OMPI_UNEXPECTED, 1);
+        SPC_RECORD(OMPI_UNEXPECTED_IN_QUEUE, 1);
+        SPC_UPDATE_WATERMARK(OMPI_MAX_UNEXPECTED_IN_QUEUE, OMPI_UNEXPECTED_IN_QUEUE);
         PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_MSG_INSERT_IN_UNEX_Q, comm_ptr,
                                hdr->hdr_src, hdr->hdr_tag, PERUSE_RECV);
+        SPC_TIMER_STOP(OMPI_MATCH_TIME, &timer);
         return NULL;
     } while(true);
 }
@@ -920,6 +934,11 @@ static int mca_pml_ob1_recv_frag_match( mca_btl_base_module_t *btl,
         MCA_PML_OB1_RECV_FRAG_ALLOC(frag);
         MCA_PML_OB1_RECV_FRAG_INIT(frag, hdr, segments, num_segments, btl);
         append_frag_to_ordered_list(&proc->frags_cant_match, frag, next_msg_seq_expected);
+
+        SPC_RECORD(OMPI_OUT_OF_SEQUENCE, 1);
+        SPC_RECORD(OMPI_OOS_IN_QUEUE, 1);
+        SPC_UPDATE_WATERMARK(OMPI_MAX_OOS_IN_QUEUE, OMPI_OOS_IN_QUEUE);
+
         OB1_MATCHING_UNLOCK(&comm->matching_lock);
         return OMPI_SUCCESS;
     }