Make LMDB an optional dependency

CMakeLists.txt

@@ -225,8 +225,8 @@ endif()
 set(ROCAL_DEBIAN_PACKAGE_LIST  "hip-runtime-amd, openmp-extras-runtime, mivisionx, libturbojpeg:amd64 | libturbojpeg0")
 set(ROCAL_RPM_PACKAGE_LIST     "hip-runtime-amd, openmp-extras-runtime, mivisionx")
 # Set the dev dependent packages
-set(ROCAL_DEBIAN_DEV_PACKAGE_LIST  "half, hip-dev, openmp-extras-dev, mivisionx-dev, liblmdb-dev, libprotobuf-dev, libturbojpeg0-dev")
-set(ROCAL_RPM_DEV_PACKAGE_LIST  "half, hip-devel, openmp-extras-devel, mivisionx-devel, lmdb-devel, protobuf-devel")
+set(ROCAL_DEBIAN_DEV_PACKAGE_LIST  "half, hip-dev, openmp-extras-dev, mivisionx-dev, libprotobuf-dev, libturbojpeg0-dev")
+set(ROCAL_RPM_DEV_PACKAGE_LIST  "half, hip-devel, openmp-extras-devel, mivisionx-devel, protobuf-devel")
 
 # Add OS specific dependencies
 if (EXISTS "/etc/os-release")

README.md

@@ -111,11 +111,6 @@ rocAL can be currently used to perform the following operations either with rand
   sudo apt install libprotobuf-dev
   ```
 
-* [LMBD Library](http://www.lmdb.tech/doc/)
-  ```shell
-  sudo apt install liblmdb-dev
-  ```
-
 * [TurboJPEG](https://libjpeg-turbo.org/) - Version `2.0` or higher
   ```shell
   sudo apt install libturbojpeg0-dev
@@ -132,6 +127,10 @@ rocAL can be currently used to perform the following operations either with rand
   ```shell
   sudo apt install python3-wheel
   ```
+* [LMDB Library](http://www.lmdb.tech/doc/) - **Optional**: needed only for Caffe/Caffe2 LMDB reader support
+  ```shell
+  sudo apt install liblmdb-dev
+  ```
 
 * rocDecode - **Optional** for source install, but required for package install
   ```shell

rocAL-setup.py

@@ -241,7 +241,7 @@ def install_packages(linuxFlag, linuxSystemInstall, linuxSystemInstall_check, pa
 coreDebianPackages = [
     'nasm',
     'yasm',
-    'liblmdb-dev',
+    #'liblmdb-dev',  # optional: needed for Caffe/Caffe2 LMDB reader support
     #'rapidjson-dev',
     'libsndfile1-dev', # for audio features
     'python3-dev',
@@ -266,7 +266,7 @@ def install_packages(linuxFlag, linuxSystemInstall, linuxSystemInstall_check, pa
 coreRPMPackages = [
     'nasm',
     'yasm',
-    'lmdb-devel',
+    #'lmdb-devel',  # optional: needed for Caffe/Caffe2 LMDB reader support
     'jsoncpp-devel',
     #'rapidjson-devel',
     str(libsndFile), # for audio features

rocAL/CMakeLists.txt

@@ -175,10 +175,11 @@ if(NOT Threads_FOUND)
     set(BUILD_ROCAL false)
     message("-- ${Yellow}NOTE: rocAL library requires Threads, Not Found ${ColourReset}")
 endif()
-# LMDB
-if(NOT LMDB_FOUND)
-    set(BUILD_ROCAL false)
-    message("-- ${Yellow}NOTE: rocAL library requires LMDB, Not Found ${ColourReset}")	
+# LMDB (optional)
+if(LMDB_FOUND)
+    message("-- ${White}rocAL built with LMDB - Caffe/Caffe2 LMDB reader support enabled${ColourReset}")
+else()
+    message("-- ${Yellow}NOTE: rocAL built without LMDB - Caffe/Caffe2 LMDB readers will not be supported${ColourReset}")
 endif()
 # RapidJSON
 if(NOT RapidJSON_FOUND)
@@ -211,9 +212,11 @@ if(${BUILD_ROCAL})
     # Protobuf
     include_directories(${PROTOBUF_INCLUDE_DIRS})
     set(LINK_LIBRARY_LIST ${LINK_LIBRARY_LIST} ${PROTOBUF_LIBRARIES})
-    # LMDB
-    include_directories(${LMDB_INCLUDE_DIRS})	
-    set(LINK_LIBRARY_LIST ${LINK_LIBRARY_LIST} ${LMDB_LIBRARIES})	
+    # LMDB (optional)
+    if(LMDB_FOUND)
+        include_directories(${LMDB_INCLUDE_DIRS})
+        set(LINK_LIBRARY_LIST ${LINK_LIBRARY_LIST} ${LMDB_LIBRARIES})
+    endif()	
     # RapidJSON
     include_directories(${RapidJSON_INCLUDE_DIRS})
     # Filesystem
@@ -344,6 +347,14 @@ if(${BUILD_ROCAL})
         message("-- ${Yellow}NOTE: rocAL built without Audio support - Audio Functionalities will not be enabled${ColourReset}")
     endif()
 
+    # LMDB (optional)
+    if(LMDB_FOUND)
+        target_compile_definitions(${PROJECT_NAME} PUBLIC -DROCAL_LMDB)
+    else()
+        target_compile_definitions(${PROJECT_NAME} PUBLIC -DROCAL_LMDB=0)
+        message("-- ${Yellow}NOTE: rocAL built without LMDB - Caffe/Caffe2 LMDB readers will not be supported${ColourReset}")
+    endif()
+
     # LibTar
     if(LIBTAR_FOUND)
         include_directories(${LIBTAR_INCLUDE_DIRS})

rocAL/include/meta_data/caffe_meta_data_reader.h

@@ -21,6 +21,7 @@ THE SOFTWARE.
 */
 
 #pragma once
+#ifdef ROCAL_LMDB
 #include <dirent.h>
 #include <lmdb.h>
 
@@ -63,3 +64,4 @@ class CaffeMetaDataReader : public MetaDataReader {
     MDB_txn* _mdb_txn;
     MDB_cursor* _mdb_cursor;
 };
+#endif  // ROCAL_LMDB

rocAL/include/meta_data/caffe_meta_data_reader_detection.h

@@ -21,6 +21,7 @@ THE SOFTWARE.
 */
 
 #pragma once
+#ifdef ROCAL_LMDB
 #include <dirent.h>
 
 #include <list>
@@ -66,3 +67,4 @@ class CaffeMetaDataReaderDetection : public MetaDataReader {
     MDB_txn* _mdb_txn;
     MDB_cursor* _mdb_cursor;
 };
+#endif  // ROCAL_LMDB

rocAL/include/readers/image/caffe2_lmdb_record_reader.h

@@ -21,6 +21,7 @@ THE SOFTWARE.
 */
 
 #pragma once
+#ifdef ROCAL_LMDB
 #include <dirent.h>
 #include <google/protobuf/message_lite.h>
 
@@ -97,3 +98,4 @@ class Caffe2LMDBRecordReader : public Reader {
     MDB_cursor* _read_mdb_cursor;
     void open_env_for_read_image();
 };
+#endif  // ROCAL_LMDB

rocAL/include/readers/image/caffe_lmdb_record_reader.h

@@ -21,6 +21,7 @@ THE SOFTWARE.
 */
 
 #pragma once
+#ifdef ROCAL_LMDB
 #include <dirent.h>
 #include <google/protobuf/message_lite.h>
 #include <lmdb.h>
@@ -95,3 +96,4 @@ class CaffeLMDBRecordReader : public Reader {
     void open_env_for_read_image();
     std::shared_ptr<MetaDataReader> _meta_data_reader = nullptr;
 };
+#endif  // ROCAL_LMDB

rocAL/include/readers/image/image_reader.h

@@ -27,18 +27,22 @@ THE SOFTWARE.
 #include <vector>
 #include <random>
 
+#ifdef ROCAL_LMDB
 #include <lmdb.h>
+#endif
 #include "meta_data/meta_data_reader.h"
 #include "readers/video/video_properties.h"
 #include "pipeline/tensor.h"
 #include "pipeline/enum_registry.h"
 
+#ifdef ROCAL_LMDB
 #define CHECK_LMDB_RETURN_STATUS(status)                                                          \
     do {                                                                                          \
         if (status != MDB_SUCCESS)                                                                \
             THROW("LMDB error, " + std::string(__FILE__) + ":" + std::to_string(__LINE__) + " " + \
                   #status + ":" + std::string(mdb_strerror(status)));                             \
     } while (0)
+#endif
 
 enum class StorageType {
     FILE_SYSTEM = 0,

rocAL/source/api/rocal_api_data_loaders.cpp

@@ -462,6 +462,7 @@ rocalJpegCaffe2LMDBRecordSource(
     Tensor* output = nullptr;
     auto context = static_cast<Context*>(p_context);
     try {
+#ifdef ROCAL_LMDB
         bool use_input_dimension = (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE) || (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE_RESTRICTED);
         bool decoder_keep_original = (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE_RESTRICTED) || (decode_size_policy == ROCAL_USE_MAX_SIZE_RESTRICTED);
         DecoderType decType = DecoderType::TURBO_JPEG;  // default
@@ -496,6 +497,9 @@ rocalJpegCaffe2LMDBRecordSource(
             auto actual_output = context->master_graph->create_tensor(info, is_output);
             context->master_graph->add_node<CopyNode>({output}, {actual_output});
         }
+#else
+        THROW("Caffe2 LMDB reader is not enabled (rocAL built without LMDB support)")
+#endif
 
     } catch (const std::exception& e) {
         ROCAL_PRINT_EXCEPTION(context, e);
@@ -521,6 +525,7 @@ rocalJpegCaffe2LMDBRecordSourceSingleShard(
     Tensor* output = nullptr;
     auto context = static_cast<Context*>(p_context);
     try {
+#ifdef ROCAL_LMDB
         bool use_input_dimension = (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE) || (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE_RESTRICTED);
         bool decoder_keep_original = (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE_RESTRICTED) || (decode_size_policy == ROCAL_USE_MAX_SIZE_RESTRICTED);
         DecoderType decType = DecoderType::TURBO_JPEG;  // default
@@ -558,6 +563,9 @@ rocalJpegCaffe2LMDBRecordSourceSingleShard(
             auto actual_output = context->master_graph->create_tensor(info, is_output);
             context->master_graph->add_node<CopyNode>({output}, {actual_output});
         }
+#else
+        THROW("Caffe2 LMDB reader is not enabled (rocAL built without LMDB support)")
+#endif
 
     } catch (const std::exception& e) {
         ROCAL_PRINT_EXCEPTION(context, e);
@@ -582,6 +590,7 @@ rocalJpegCaffeLMDBRecordSource(
     Tensor* output = nullptr;
     auto context = static_cast<Context*>(p_context);
     try {
+#ifdef ROCAL_LMDB
         bool use_input_dimension = (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE) || (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE_RESTRICTED);
         bool decoder_keep_original = (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE_RESTRICTED) || (decode_size_policy == ROCAL_USE_MAX_SIZE_RESTRICTED);
         DecoderType decType = DecoderType::TURBO_JPEG;  // default
@@ -618,6 +627,9 @@ rocalJpegCaffeLMDBRecordSource(
             auto actual_output = context->master_graph->create_tensor(info, is_output);
             context->master_graph->add_node<CopyNode>({output}, {actual_output});
         }
+#else
+        THROW("Caffe LMDB reader is not enabled (rocAL built without LMDB support)")
+#endif
 
     } catch (const std::exception& e) {
         ROCAL_PRINT_EXCEPTION(context, e);
@@ -643,6 +655,7 @@ rocalJpegCaffeLMDBRecordSourceSingleShard(
     Tensor* output = nullptr;
     auto context = static_cast<Context*>(p_context);
     try {
+#ifdef ROCAL_LMDB
         bool use_input_dimension = (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE) || (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE_RESTRICTED);
         bool decoder_keep_original = (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE_RESTRICTED) || (decode_size_policy == ROCAL_USE_MAX_SIZE_RESTRICTED);
         DecoderType decType = DecoderType::TURBO_JPEG;  // default
@@ -680,6 +693,9 @@ rocalJpegCaffeLMDBRecordSourceSingleShard(
             auto actual_output = context->master_graph->create_tensor(info, is_output);
             context->master_graph->add_node<CopyNode>({output}, {actual_output});
         }
+#else
+        THROW("Caffe LMDB reader is not enabled (rocAL built without LMDB support)")
+#endif
 
     } catch (const std::exception& e) {
         ROCAL_PRINT_EXCEPTION(context, e);
@@ -708,6 +724,7 @@ rocalJpegCaffeLMDBRecordSourcePartialSingleShard(
     Tensor* output = nullptr;
     auto context = static_cast<Context*>(p_context);
     try {
+#ifdef ROCAL_LMDB
         bool use_input_dimension = (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE) || (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE_RESTRICTED);
 
         if (shard_count < 1)
@@ -746,6 +763,9 @@ rocalJpegCaffeLMDBRecordSourcePartialSingleShard(
             auto actual_output = context->master_graph->create_tensor(info, is_output);
             context->master_graph->add_node<CopyNode>({output}, {actual_output});
         }
+#else
+        THROW("Caffe LMDB reader is not enabled (rocAL built without LMDB support)")
+#endif
 
     } catch (const std::exception& e) {
         ROCAL_PRINT_EXCEPTION(context, e);
@@ -774,6 +794,7 @@ rocalJpegCaffe2LMDBRecordSourcePartialSingleShard(
     Tensor* output = nullptr;
     auto context = static_cast<Context*>(p_context);
     try {
+#ifdef ROCAL_LMDB
         bool use_input_dimension = (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE) || (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE_RESTRICTED);
 
         if (shard_count < 1)
@@ -811,6 +832,9 @@ rocalJpegCaffe2LMDBRecordSourcePartialSingleShard(
             auto actual_output = context->master_graph->create_tensor(info, is_output);
             context->master_graph->add_node<CopyNode>({output}, {actual_output});
         }
+#else
+        THROW("Caffe2 LMDB reader is not enabled (rocAL built without LMDB support)")
+#endif
 
     } catch (const std::exception& e) {
         ROCAL_PRINT_EXCEPTION(context, e);

rocAL/source/api/rocal_api_info.cpp

@@ -101,38 +101,55 @@ RocalMetaData
     rocalCreateCaffe2LMDBLabelReader(RocalContext p_context, const char *source_path, bool is_output) {
     if (!p_context)
         THROW("Invalid rocal context passed to rocalCreateCaffe2LMDBLabelReader")
-
+#ifdef ROCAL_LMDB
     auto context = static_cast<Context *>(p_context);
     return context->master_graph->create_caffe2_lmdb_record_meta_data_reader(source_path, MetaDataReaderType::CAFFE2_META_DATA_READER, MetaDataType::Label);
+#else
+    THROW("Caffe2 LMDB reader is not enabled (rocAL built without LMDB support)")
+    return nullptr;
+#endif
 }
 
 RocalMetaData
     ROCAL_API_CALL
     rocalCreateCaffe2LMDBReaderDetection(RocalContext p_context, const char *source_path, bool is_output) {
     if (!p_context)
         THROW("Invalid rocal context passed to rocalCreateCaffe2LMDBReaderDetection")
+#ifdef ROCAL_LMDB
     auto context = static_cast<Context *>(p_context);
-
     return context->master_graph->create_caffe2_lmdb_record_meta_data_reader(source_path, MetaDataReaderType::CAFFE2_DETECTION_META_DATA_READER, MetaDataType::BoundingBox);
+#else
+    THROW("Caffe2 LMDB reader is not enabled (rocAL built without LMDB support)")
+    return nullptr;
+#endif
 }
 
 RocalMetaData
     ROCAL_API_CALL
     rocalCreateCaffeLMDBLabelReader(RocalContext p_context, const char *source_path) {
     if (!p_context)
         THROW("Invalid rocal context passed to rocalCreateCaffeLMDBLabelReader")
+#ifdef ROCAL_LMDB
     auto context = static_cast<Context *>(p_context);
     return context->master_graph->create_caffe_lmdb_record_meta_data_reader(source_path, MetaDataReaderType::CAFFE_META_DATA_READER, MetaDataType::Label);
+#else
+    THROW("Caffe LMDB reader is not enabled (rocAL built without LMDB support)")
+    return nullptr;
+#endif
 }
 
 RocalMetaData
     ROCAL_API_CALL
     rocalCreateCaffeLMDBReaderDetection(RocalContext p_context, const char *source_path) {
     if (!p_context)
         THROW("Invalid rocal context passed to rocalCreateCaffeLMDBReaderDetection")
+#ifdef ROCAL_LMDB
     auto context = static_cast<Context *>(p_context);
-
     return context->master_graph->create_caffe_lmdb_record_meta_data_reader(source_path, MetaDataReaderType::CAFFE_DETECTION_META_DATA_READER, MetaDataType::BoundingBox);
+#else
+    THROW("Caffe LMDB reader is not enabled (rocAL built without LMDB support)")
+    return nullptr;
+#endif
 }
 
 size_t ROCAL_API_CALL rocalIsEmpty(RocalContext p_context) {

rocAL/source/meta_data/caffe2_meta_data_reader.cpp

@@ -20,6 +20,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
+#ifdef ROCAL_LMDB
 #include "meta_data/caffe2_meta_data_reader.h"
 
 #include <google/protobuf/message_lite.h>
@@ -168,3 +169,4 @@ void Caffe2MetaDataReader::release() {
 
 Caffe2MetaDataReader::Caffe2MetaDataReader() {
 }
+#endif  // ROCAL_LMDB

rocAL/source/meta_data/caffe2_meta_data_reader_detection.cpp

@@ -20,6 +20,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
+#ifdef ROCAL_LMDB
 #include "meta_data/caffe2_meta_data_reader_detection.h"
 
 #include <google/protobuf/message_lite.h>
@@ -208,3 +209,4 @@ void Caffe2MetaDataReaderDetection::release() {
 
 Caffe2MetaDataReaderDetection::Caffe2MetaDataReaderDetection() {
 }
+#endif  // ROCAL_LMDB

rocAL/source/meta_data/caffe_meta_data_reader.cpp

@@ -20,6 +20,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
+#ifdef ROCAL_LMDB
 #include "meta_data/caffe_meta_data_reader.h"
 
 #include <string.h>
@@ -136,3 +137,4 @@ void CaffeMetaDataReader::read_lmdb_record(std::string _path, uint file_byte_siz
     mdb_txn_abort(_mdb_txn);
     mdb_env_close(_mdb_env);
 }
+#endif  // ROCAL_LMDB

rocAL/source/meta_data/caffe_meta_data_reader_detection.cpp

@@ -20,6 +20,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
+#ifdef ROCAL_LMDB
 #include <iostream>
 #include <fstream>
 #include <utility>
@@ -185,4 +186,5 @@ void CaffeMetaDataReaderDetection::release() {
 }
 
 CaffeMetaDataReaderDetection::CaffeMetaDataReaderDetection() {
-}
+}
+#endif  // ROCAL_LMDB