Merge pull request #37 from JeffersonLab/evio-6-dev-tmp

Evio 6 dev tmp

Author: jonzarling

.github/workflows/c_tests.yml

@@ -17,7 +17,7 @@ jobs:
         run: cmake -S . -B build -DMAKE_EXAMPLES=1
 
       - name: Build
-        run: cmake --build build -- -j$(nproc)
+        run: cmake --build build --target install --parallel
 
       - name: Run tests
         working-directory: build

.gitignore

@@ -29,6 +29,7 @@ build/
 jbuild/
 lib/
 bin/
+tmp/
 doxyerrors.log
 target/
 doc/javadoc
@@ -37,3 +38,5 @@ doc/doxygen/C
 doc/source/
 config.log
 
+# Ignore typical install directory type
+Linux-*/

CMakeLists.txt

@@ -1,22 +1,26 @@
+# See instructions in README.md for building and installing
+
+### VERSION DEFINITIONS ###
 cmake_minimum_required(VERSION 3.22)
 project(evio VERSION 6.1.0 LANGUAGES C CXX)
-
-# C/C++ standard and build options
 set(CMAKE_C_STANDARD 11)
-set(CMAKE_C_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_STANDARD 20)
+# C/C++ build options
+set(CMAKE_C_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_DEBUG_POSTFIX -dbg)
-add_compile_options(-Wall)
+add_compile_options(-Wall) # Enable all warnings
 
-# Build options (and defaults)
+# Includes
+include(GNUInstallDirs)
+include(FindPackageHandleStandardArgs) # find_package_handle_standard_args()
+include(CTest) 
+
+# Build option parameters (and defaults)
 option(C_ONLY                "SKIP building C++ library, build C only" OFF)
 option(MAKE_EXAMPLES         "Build example/test programs" OFF)
 option(USE_FILESYSTEMLIB     "Use C++ <filesystem> instead of Boost" OFF)
 option(DISRUPTOR_FETCH       "Allow CMake to download Disruptor if not found" ON)
-include(GNUInstallDirs)
-include(FindPackageHandleStandardArgs) # find_package_handle_standard_args()
-include(CTest) 
 
 # Add custom find_package for Disruptor
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules")
@@ -51,7 +55,10 @@ elseif (DEFINED ENV{CODA})
 else()
     # Use default CMAKE_INSTALL_PREFIX
     set(INSTALL_DIR_DEFINED 1)
+    set(CMAKE_INSTALL_PREFIX ${CMAKE_CURRENT_SOURCE_DIR}/${ARCH})
+    set(CMAKE_INSTALL_INCLUDEDIR ${CMAKE_INSTALL_PREFIX}/include)
     message(STATUS "Installing to default location: ${CMAKE_INSTALL_PREFIX}")
+    message(STATUS "Installing includes to default location: ${CMAKE_INSTALL_INCLUDEDIR}")
 endif()
 
 # Boost libs
@@ -98,12 +105,13 @@ file(GLOB C_LIB_FILES "src/libsrc/*.c")
 # C++ source files
 file(GLOB CPP_LIB_FILES "src/libsrc++/*.cpp")
 file(GLOB CPP_HEADER_FILES "src/libsrc++/*.h")
-# A few extras required
+# C++ utility files
+file(GLOB CPP_UTILS_FILES "src/utils/cpp/*.cpp")
+# A few extras required for examples
 if(MAKE_EXAMPLES)
+    list(APPEND CPP_HEADER_FILES src/test/cpp/EvioTestHelper.h)
     file(GLOB TEST "src/test/cpp/*.cpp")
     file(GLOB TESTC "src/test/c/*.c")
-    list(APPEND CPP_HEADER_FILES src/test/cpp/EvioTestHelper.h)
-    # list(APPEND CPP_LIB_FILES src/test/cpp/EvioTestHelper.h)
 endif()
 
 # BUILD C++ LIBRARY (unless otherwise specified)
@@ -142,6 +150,25 @@ if(NOT C_ONLY)
             ${DISRUPTOR_INCLUDE_DIR}
     )
 
+    # Build utility programs
+    foreach(fileName ${CPP_UTILS_FILES})
+        # Get file name with no directory or extension as executable name
+        get_filename_component(execName ${fileName} NAME_WE)
+        # Create executable from file
+        add_executable(${execName} ${fileName})
+        # Put debug extension on if applicable
+        set_target_properties(${execName} PROPERTIES DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX})
+        # Needs these libs
+        target_link_libraries(${execName} eviocc pthread ${Boost_LIBRARIES} ${LZ4_LIBRARY} expat dl z m)
+
+        # Only install if installation directory has been defined
+        if(DEFINED INSTALL_DIR_DEFINED)
+            message(STATUS "Installing utility executable: ${execName}")
+            # Install into bin/utils dir
+            install(TARGETS ${execName} RUNTIME DESTINATION bin/utils)
+        endif()
+    endforeach()
+
     # Add the C++ tests/examples
     if(MAKE_EXAMPLES)
         foreach(fileName ${TEST})

README.md

@@ -1,40 +1,50 @@
 # **EVIO 6 SOFTWARE PACKAGE**
 
-EVIO stands for EVent Input/Output, a unique data format developed by Jefferson Lab. 
-It was created by the Data Acquisition (DAQ) group and is maintained by the 
-Experimental Physics Software and Computing Infrastructure (EPSCI) group at Thomas 
-Jefferson National Accelerator Facility (JLab).
+EVIO stands for EVent Input/Output, a unique data format developed by Jefferson Lab used by typical detector readout systems at the lab. This software repository allows one to read & write `.evio` and `.ev` format data, within either a C/C++ or Java programming environment.
 
-This software repository allows one to read & write `.evio` and `.ev` format data, 
-within either a C/C++ or Java programming environment.
+# **Useful Links**
+
+Documentation on GitHub:
+
+* [All Links](https://jeffersonlab.github.io/evio)
+* [User's Guide PDF](https://jeffersonlab.github.io/evio/doc-6.0/users_guide/evio_Users_Guide.pdf)
+* [EVIO Data Format Reference](https://jeffersonlab.github.io/evio/doc-6.0/format_guide/evio_Formats.pdf)
+
+Software Library Documentation:
+
+* [Javadoc for Java Library](https://jeffersonlab.github.io/evio/doc-6.0/javadoc/index.html)
+* [Doxygen for C Library](https://jeffersonlab.github.io/evio/doc-6.0/doxygen/C/html/index.html)
+* [Doxygen for C++ Libary](https://jeffersonlab.github.io/evio/doc-6.0/doxygen/CC/html/index.html)
 
 # **Getting Started**
 
 ## **C/C++ Library**
 
-To build C/C++ code from this repository:
+The C and C++ libraries are build using `cmake`. To build C/C++ code from this repository:
 
     git clone https://github.com/JeffersonLab/evio/
     cd evio; mkdir build
     cmake -S . -B build
-    cmake --build build --parallel
+    cmake --build build --target install --parallel
 
-Note that during the cmake configure step (first of two `cmake` commands above), one can
-toggle the following special flags:
+Note that during the cmake configure step (first of two `cmake` commands above), one can also include the following special flags:
 
 * `C_ONLY` : build C lib only, skip C++ (default `-DC_ONLY=0`)
 * `MAKE_EXAMPLES`: build example/test programs (default `-DMAKE_EXAMPLES=0`)
 * `USE_FILESYSTEMLIB`: ue C++17 <filesystem> instead of Boost (default `-DUSE_FILESYSTEMLIB=0`)
 * `DISRUPTOR_FETCH`: allow CMake to download Disruptor if not found (default `-DDISRUPTOR_FETCH=1`)
+* `CODA_INSTALL`: installs in this base directory. If not used,
+then the env variable $CODA location is next checked. Otherwise defaults to \${CMAKE_HOST_SYSTEM_NAME}-\${CMAKE_HOST_SYSTEM_PROCESSOR}, typically something like `[evio_directory]/Linux-x86_64`.
 
 One can still also use `scons` instead of cmake to build the evio C/C++ library, though this feature
 will not be supported in future releases.
 
 ### Prerequisites
 
-C++ 17 or higher, `cmake`, `lz4`, `boost_system`, `boost_thread`, and `boost_chrono`. If LZ4 is not
-already configured, it can be installed from [LZ4 on github](https://github.com/lz4/lz4). The boost
-libraries are typically system-specific. 
+C++ 17 or higher, `cmake`, `lz4`, `boost_system`, `boost_thread`, and `boost_chrono`. Compilation can 
+be done using `clang` or `gcc` (gcc 11 or higher recommended). If LZ4 is not
+already configured, it can be installed from [LZ4 on github](https://github.com/lz4/lz4). Installation of boost
+libraries are typically system-specific (e.g. using a command like `yum`, `dbn`, `rpm`, `apt-get`, etc.). 
 
 ## **Java Library**
 
@@ -57,23 +67,12 @@ Requires Maven (`mvn`) and an installation of Java on your system.
 **Running on "ifarm" at JLab will not work unless you install java yourself**. Note that the default java versions on the farm will be too old to 
 work. See downloads from [OpenJDK](https://openjdk.org/install/) or [Oracle](https://www.oracle.com/java/technologies/javase/jdk17-archive-downloads.html).
 
------------------------------
 
-# **Useful Links**
-
-----------------------------
+# **Further Information**
 
-Documentation on GitHub:
 
-* [All Documentation](https://jeffersonlab.github.io/evio)
-* [User's Guide PDF](https://jeffersonlab.github.io/evio/doc-6.0/users_guide/evio_Users_Guide.pdf)
-* [EVIO Data Format Reference](https://jeffersonlab.github.io/evio/doc-6.0/format_guide/evio_Formats.pdf)
+The EVIO package was created by the Data Acquisition (DAQ) group and is maintained by the Experimental Physics Software and Computing Infrastructure (EPSCI) group at the Thomas Jefferson National Accelerator Facility (JLab). It has been developed by many authors over the years.
 
-Software Library Documentation:
-
-* [Javadoc for Java Library](https://jeffersonlab.github.io/evio/doc-6.0/javadoc/index.html)
-* [Doxygen for C Library](https://jeffersonlab.github.io/evio/doc-6.0/doxygen/C/html/index.html)
-* [Doxygen for C++ Libary](https://jeffersonlab.github.io/evio/doc-6.0/doxygen/CC/html/index.html)
 
 Other Links:
 * [EVIO Event Viewer on GitHub](https://github.com/JeffersonLab/JEventViewer)
@@ -86,10 +85,8 @@ well as the software used to read and write to these respective `.evio` and `.hi
 More information on the HIPO data format can be found at https://github.com/gavalian/hipo,
 or from the CLAS12 Software Project Coordinator.
 
-----------------------------
+Contact: Jon Zarling (jzarling@jlab.org)
 
 # **Copyright**
 
-----------------------------
-
 For any issues regarding use and copyright, read the [license](LICENSE.txt) file.

java/jars/README.md

@@ -1,5 +1,5 @@
 #  **Java JARs Folder**
 
-This folder contains any jar file dependencies required by the EVIO Java library that are not retrieved directly from the Maven repository. At present, only the disruptor library needs to be added from here.
+This folder contains jar files used for executing EVIO code using the Java API. The main jar file was made using Java 17.
 
-Note that the disruptor 4.0.0 jar file included comes from a [JeffersonLab fork](https://github.com/JeffersonLab/disruptor) of the original lmax Java library. This forked repository adds an additional function (SpinCountBackoffWaitStrategy) that is required in the evio Java Library.
+It also includes dependencies required by the EVIO Java library that are not retrieved directly from the Maven repository. At present, only the disruptor dependency needs to be added from here. Note that the disruptor 4.0.0 jar file included comes from a [JeffersonLab fork](https://github.com/JeffersonLab/disruptor) of the original lmax Java library. This forked repository adds an additional function (SpinCountBackoffWaitStrategy) that is required in the evio Java Library.

src/README.md

@@ -5,5 +5,6 @@
 | [libsrc](libsrc)       | EVIO C library  |
 | [libsrc++](libsrc++)   | EVIO C++ library  |
 | [main/java](main/java) | EVIO Java Library  |
-| [test](test)           | For internal testing of Java, C, C++ libraries. |
+| [test](test)           | For internal testing of Java, C, C++ libraries.     |
+| [utils](utils)         | Standalone programs, e.g. utilities to merge files, convert formats, attempt recovery, etc. |
 

src/libsrc++/EventParser.cpp

@@ -227,9 +227,11 @@ namespace  evio {
         // --which will be interpreted by the various "get data" methods.
         auto & bytes = structure->getRawBytes();
         ByteOrder byteOrder = structure->getByteOrder();
-
         if (bytes.empty()) {
-            throw EvioException("Null data in structure");
+            // throw EvioException("Null data in structure");
+            // printf("WARNING: no data inside structure! Skipping... \n");
+            // printf("   structure %s \n", structure->getHeader()->toString().c_str());
+            return;
         }
 
         size_t length = bytes.size();

src/libsrc++/EvioReaderV4.cpp

@@ -1090,7 +1090,7 @@ namespace evio {
         uint32_t eventDataSizeBytes = 4*(length - 1);
 
         try {
-            auto *bytes = new uint8_t[eventDataSizeBytes];
+            auto bytes = std::vector<uint8_t>(eventDataSizeBytes);
 
             uint32_t bytesToGo = eventDataSizeBytes;
             uint32_t offset = 0;
@@ -1106,7 +1106,7 @@ namespace evio {
                                               blkBytesRemaining : bytesToGo;
 
                     // Read in bytes remaining in internal buffer
-                    byteBuffer->getBytes(bytes + offset, bytesToReadNow);
+                    byteBuffer->getBytes(bytes.data() + offset, bytesToReadNow);
                     offset               += bytesToReadNow;
                     bytesToGo            -= bytesToReadNow;
                     blkBytesRemaining    -= bytesToReadNow;
@@ -1123,12 +1123,14 @@ namespace evio {
                         blkBytesRemaining = blockBytesRemaining();
                     }
                 }
+
+                
             }
 
             // Last (perhaps only) read
-            byteBuffer->getBytes(bytes + offset, bytesToGo);
+            byteBuffer->getBytes(bytes.data() + offset, bytesToGo);
 //std::cout << "nextEvent: eventDataSizeByte = " <<  eventDataSizeBytes << std::endl;
-            event->setRawBytes(bytes, eventDataSizeBytes);
+            event->setRawBytes(bytes.data(), eventDataSizeBytes);
             event->setByteOrder(byteOrder); // add this to track endianness, timmer
             // Don't worry about dictionaries here as version must be 1-3
             event->setEventNumber(++eventNumber);

src/main/java/org/jlab/coda/jevio/EvioReader.java

@@ -215,6 +215,7 @@ public EvioReader(File file, boolean checkRecNumSeq, boolean sequential, boolean
 
         // Parse file header to find the file's endianness & evio version #
         if (findEvioVersion() != ReadStatus.SUCCESS) {
+            rFile.close();
             throw new EvioException("Failed reading first block header");
         }
 

src/main/java/org/jlab/coda/jevio/EvioReaderUnsyncV4.java

@@ -1046,7 +1046,7 @@ else if (bytesInBuf % 32768 == 0) {
                     // Check block size, attempt to recover if flag set
                     // (otherwise return exception with a hint to set flag)
                     // System.out.println("blkSize BEFORE = " + blkSize);
-                    if(doHeaderRecoveryCheck && fileSize - fileChannel.position() >= 10*4) {
+                    if(doHeaderRecoveryCheck && (fileSize - fileChannel.position()) >= 10*4 && fileChannel.position() > 20*4) {
 
                         int expectedMagicPos = 27; // in words
                         int words_to_skip = 0; // words_to_skip = foundMagicPos - expectedMagicPos