diff --git a/CMakeLists.txt b/CMakeLists.txt index 7c625d200..3a5be1f61 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -225,8 +225,8 @@ endif() set(ROCAL_DEBIAN_PACKAGE_LIST "hip-runtime-amd, openmp-extras-runtime, mivisionx, libturbojpeg:amd64 | libturbojpeg0") set(ROCAL_RPM_PACKAGE_LIST "hip-runtime-amd, openmp-extras-runtime, mivisionx") # Set the dev dependent packages -set(ROCAL_DEBIAN_DEV_PACKAGE_LIST "half, hip-dev, openmp-extras-dev, mivisionx-dev, liblmdb-dev, libprotobuf-dev, libturbojpeg0-dev") -set(ROCAL_RPM_DEV_PACKAGE_LIST "half, hip-devel, openmp-extras-devel, mivisionx-devel, lmdb-devel, protobuf-devel") +set(ROCAL_DEBIAN_DEV_PACKAGE_LIST "half, hip-dev, openmp-extras-dev, mivisionx-dev, libprotobuf-dev, libturbojpeg0-dev") +set(ROCAL_RPM_DEV_PACKAGE_LIST "half, hip-devel, openmp-extras-devel, mivisionx-devel, protobuf-devel") # Add OS specific dependencies if (EXISTS "/etc/os-release") diff --git a/README.md b/README.md index 509cdbc93..8d97e5464 100644 --- a/README.md +++ b/README.md @@ -111,11 +111,6 @@ rocAL can be currently used to perform the following operations either with rand sudo apt install libprotobuf-dev ``` -* [LMBD Library](http://www.lmdb.tech/doc/) - ```shell - sudo apt install liblmdb-dev - ``` - * [TurboJPEG](https://libjpeg-turbo.org/) - Version `2.0` or higher ```shell sudo apt install libturbojpeg0-dev @@ -132,6 +127,10 @@ rocAL can be currently used to perform the following operations either with rand ```shell sudo apt install python3-wheel ``` +* [LMDB Library](http://www.lmdb.tech/doc/) - **Optional**: needed only for Caffe/Caffe2 LMDB reader support + ```shell + sudo apt install liblmdb-dev + ``` * rocDecode - **Optional** for source install, but required for package install ```shell diff --git a/rocAL-setup.py b/rocAL-setup.py index 4f1974d20..35d667d01 100644 --- a/rocAL-setup.py +++ b/rocAL-setup.py @@ -241,7 +241,7 @@ def install_packages(linuxFlag, linuxSystemInstall, linuxSystemInstall_check, pa coreDebianPackages = [ 'nasm', 'yasm', - 'liblmdb-dev', + #'liblmdb-dev', # optional: needed for Caffe/Caffe2 LMDB reader support #'rapidjson-dev', 'libsndfile1-dev', # for audio features 'python3-dev', @@ -266,7 +266,7 @@ def install_packages(linuxFlag, linuxSystemInstall, linuxSystemInstall_check, pa coreRPMPackages = [ 'nasm', 'yasm', - 'lmdb-devel', + #'lmdb-devel', # optional: needed for Caffe/Caffe2 LMDB reader support 'jsoncpp-devel', #'rapidjson-devel', str(libsndFile), # for audio features diff --git a/rocAL/CMakeLists.txt b/rocAL/CMakeLists.txt index e8ed651e5..b4a148d33 100644 --- a/rocAL/CMakeLists.txt +++ b/rocAL/CMakeLists.txt @@ -175,10 +175,11 @@ if(NOT Threads_FOUND) set(BUILD_ROCAL false) message("-- ${Yellow}NOTE: rocAL library requires Threads, Not Found ${ColourReset}") endif() -# LMDB -if(NOT LMDB_FOUND) - set(BUILD_ROCAL false) - message("-- ${Yellow}NOTE: rocAL library requires LMDB, Not Found ${ColourReset}") +# LMDB (optional) +if(LMDB_FOUND) + message("-- ${White}rocAL built with LMDB - Caffe/Caffe2 LMDB reader support enabled${ColourReset}") +else() + message("-- ${Yellow}NOTE: rocAL built without LMDB - Caffe/Caffe2 LMDB readers will not be supported${ColourReset}") endif() # RapidJSON if(NOT RapidJSON_FOUND) @@ -211,9 +212,11 @@ if(${BUILD_ROCAL}) # Protobuf include_directories(${PROTOBUF_INCLUDE_DIRS}) set(LINK_LIBRARY_LIST ${LINK_LIBRARY_LIST} ${PROTOBUF_LIBRARIES}) - # LMDB - include_directories(${LMDB_INCLUDE_DIRS}) - set(LINK_LIBRARY_LIST ${LINK_LIBRARY_LIST} ${LMDB_LIBRARIES}) + # LMDB (optional) + if(LMDB_FOUND) + include_directories(${LMDB_INCLUDE_DIRS}) + set(LINK_LIBRARY_LIST ${LINK_LIBRARY_LIST} ${LMDB_LIBRARIES}) + endif() # RapidJSON include_directories(${RapidJSON_INCLUDE_DIRS}) # Filesystem @@ -344,6 +347,14 @@ if(${BUILD_ROCAL}) message("-- ${Yellow}NOTE: rocAL built without Audio support - Audio Functionalities will not be enabled${ColourReset}") endif() + # LMDB (optional) + if(LMDB_FOUND) + target_compile_definitions(${PROJECT_NAME} PUBLIC -DROCAL_LMDB) + else() + target_compile_definitions(${PROJECT_NAME} PUBLIC -DROCAL_LMDB=0) + message("-- ${Yellow}NOTE: rocAL built without LMDB - Caffe/Caffe2 LMDB readers will not be supported${ColourReset}") + endif() + # LibTar if(LIBTAR_FOUND) include_directories(${LIBTAR_INCLUDE_DIRS}) diff --git a/rocAL/include/meta_data/caffe_meta_data_reader.h b/rocAL/include/meta_data/caffe_meta_data_reader.h index 28ed28dda..f2b8d054f 100644 --- a/rocAL/include/meta_data/caffe_meta_data_reader.h +++ b/rocAL/include/meta_data/caffe_meta_data_reader.h @@ -21,6 +21,7 @@ THE SOFTWARE. */ #pragma once +#ifdef ROCAL_LMDB #include #include @@ -63,3 +64,4 @@ class CaffeMetaDataReader : public MetaDataReader { MDB_txn* _mdb_txn; MDB_cursor* _mdb_cursor; }; +#endif // ROCAL_LMDB diff --git a/rocAL/include/meta_data/caffe_meta_data_reader_detection.h b/rocAL/include/meta_data/caffe_meta_data_reader_detection.h index c19679933..0c56a5459 100644 --- a/rocAL/include/meta_data/caffe_meta_data_reader_detection.h +++ b/rocAL/include/meta_data/caffe_meta_data_reader_detection.h @@ -21,6 +21,7 @@ THE SOFTWARE. */ #pragma once +#ifdef ROCAL_LMDB #include #include @@ -66,3 +67,4 @@ class CaffeMetaDataReaderDetection : public MetaDataReader { MDB_txn* _mdb_txn; MDB_cursor* _mdb_cursor; }; +#endif // ROCAL_LMDB diff --git a/rocAL/include/readers/image/caffe2_lmdb_record_reader.h b/rocAL/include/readers/image/caffe2_lmdb_record_reader.h index 7557231c3..5c4f648f7 100644 --- a/rocAL/include/readers/image/caffe2_lmdb_record_reader.h +++ b/rocAL/include/readers/image/caffe2_lmdb_record_reader.h @@ -21,6 +21,7 @@ THE SOFTWARE. */ #pragma once +#ifdef ROCAL_LMDB #include #include @@ -97,3 +98,4 @@ class Caffe2LMDBRecordReader : public Reader { MDB_cursor* _read_mdb_cursor; void open_env_for_read_image(); }; +#endif // ROCAL_LMDB diff --git a/rocAL/include/readers/image/caffe_lmdb_record_reader.h b/rocAL/include/readers/image/caffe_lmdb_record_reader.h index d81e3ea8f..a8e622ac3 100644 --- a/rocAL/include/readers/image/caffe_lmdb_record_reader.h +++ b/rocAL/include/readers/image/caffe_lmdb_record_reader.h @@ -21,6 +21,7 @@ THE SOFTWARE. */ #pragma once +#ifdef ROCAL_LMDB #include #include #include @@ -95,3 +96,4 @@ class CaffeLMDBRecordReader : public Reader { void open_env_for_read_image(); std::shared_ptr _meta_data_reader = nullptr; }; +#endif // ROCAL_LMDB diff --git a/rocAL/include/readers/image/image_reader.h b/rocAL/include/readers/image/image_reader.h index 256705145..689e6a165 100644 --- a/rocAL/include/readers/image/image_reader.h +++ b/rocAL/include/readers/image/image_reader.h @@ -27,18 +27,22 @@ THE SOFTWARE. #include #include +#ifdef ROCAL_LMDB #include +#endif #include "meta_data/meta_data_reader.h" #include "readers/video/video_properties.h" #include "pipeline/tensor.h" #include "pipeline/enum_registry.h" +#ifdef ROCAL_LMDB #define CHECK_LMDB_RETURN_STATUS(status) \ do { \ if (status != MDB_SUCCESS) \ THROW("LMDB error, " + std::string(__FILE__) + ":" + std::to_string(__LINE__) + " " + \ #status + ":" + std::string(mdb_strerror(status))); \ } while (0) +#endif enum class StorageType { FILE_SYSTEM = 0, diff --git a/rocAL/source/api/rocal_api_data_loaders.cpp b/rocAL/source/api/rocal_api_data_loaders.cpp index c478351da..fb0a9622b 100644 --- a/rocAL/source/api/rocal_api_data_loaders.cpp +++ b/rocAL/source/api/rocal_api_data_loaders.cpp @@ -462,6 +462,7 @@ rocalJpegCaffe2LMDBRecordSource( Tensor* output = nullptr; auto context = static_cast(p_context); try { +#ifdef ROCAL_LMDB bool use_input_dimension = (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE) || (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE_RESTRICTED); bool decoder_keep_original = (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE_RESTRICTED) || (decode_size_policy == ROCAL_USE_MAX_SIZE_RESTRICTED); DecoderType decType = DecoderType::TURBO_JPEG; // default @@ -496,6 +497,9 @@ rocalJpegCaffe2LMDBRecordSource( auto actual_output = context->master_graph->create_tensor(info, is_output); context->master_graph->add_node({output}, {actual_output}); } +#else + THROW("Caffe2 LMDB reader is not enabled (rocAL built without LMDB support)") +#endif } catch (const std::exception& e) { ROCAL_PRINT_EXCEPTION(context, e); @@ -521,6 +525,7 @@ rocalJpegCaffe2LMDBRecordSourceSingleShard( Tensor* output = nullptr; auto context = static_cast(p_context); try { +#ifdef ROCAL_LMDB bool use_input_dimension = (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE) || (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE_RESTRICTED); bool decoder_keep_original = (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE_RESTRICTED) || (decode_size_policy == ROCAL_USE_MAX_SIZE_RESTRICTED); DecoderType decType = DecoderType::TURBO_JPEG; // default @@ -558,6 +563,9 @@ rocalJpegCaffe2LMDBRecordSourceSingleShard( auto actual_output = context->master_graph->create_tensor(info, is_output); context->master_graph->add_node({output}, {actual_output}); } +#else + THROW("Caffe2 LMDB reader is not enabled (rocAL built without LMDB support)") +#endif } catch (const std::exception& e) { ROCAL_PRINT_EXCEPTION(context, e); @@ -582,6 +590,7 @@ rocalJpegCaffeLMDBRecordSource( Tensor* output = nullptr; auto context = static_cast(p_context); try { +#ifdef ROCAL_LMDB bool use_input_dimension = (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE) || (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE_RESTRICTED); bool decoder_keep_original = (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE_RESTRICTED) || (decode_size_policy == ROCAL_USE_MAX_SIZE_RESTRICTED); DecoderType decType = DecoderType::TURBO_JPEG; // default @@ -618,6 +627,9 @@ rocalJpegCaffeLMDBRecordSource( auto actual_output = context->master_graph->create_tensor(info, is_output); context->master_graph->add_node({output}, {actual_output}); } +#else + THROW("Caffe LMDB reader is not enabled (rocAL built without LMDB support)") +#endif } catch (const std::exception& e) { ROCAL_PRINT_EXCEPTION(context, e); @@ -643,6 +655,7 @@ rocalJpegCaffeLMDBRecordSourceSingleShard( Tensor* output = nullptr; auto context = static_cast(p_context); try { +#ifdef ROCAL_LMDB bool use_input_dimension = (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE) || (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE_RESTRICTED); bool decoder_keep_original = (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE_RESTRICTED) || (decode_size_policy == ROCAL_USE_MAX_SIZE_RESTRICTED); DecoderType decType = DecoderType::TURBO_JPEG; // default @@ -680,6 +693,9 @@ rocalJpegCaffeLMDBRecordSourceSingleShard( auto actual_output = context->master_graph->create_tensor(info, is_output); context->master_graph->add_node({output}, {actual_output}); } +#else + THROW("Caffe LMDB reader is not enabled (rocAL built without LMDB support)") +#endif } catch (const std::exception& e) { ROCAL_PRINT_EXCEPTION(context, e); @@ -708,6 +724,7 @@ rocalJpegCaffeLMDBRecordSourcePartialSingleShard( Tensor* output = nullptr; auto context = static_cast(p_context); try { +#ifdef ROCAL_LMDB bool use_input_dimension = (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE) || (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE_RESTRICTED); if (shard_count < 1) @@ -746,6 +763,9 @@ rocalJpegCaffeLMDBRecordSourcePartialSingleShard( auto actual_output = context->master_graph->create_tensor(info, is_output); context->master_graph->add_node({output}, {actual_output}); } +#else + THROW("Caffe LMDB reader is not enabled (rocAL built without LMDB support)") +#endif } catch (const std::exception& e) { ROCAL_PRINT_EXCEPTION(context, e); @@ -774,6 +794,7 @@ rocalJpegCaffe2LMDBRecordSourcePartialSingleShard( Tensor* output = nullptr; auto context = static_cast(p_context); try { +#ifdef ROCAL_LMDB bool use_input_dimension = (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE) || (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE_RESTRICTED); if (shard_count < 1) @@ -811,6 +832,9 @@ rocalJpegCaffe2LMDBRecordSourcePartialSingleShard( auto actual_output = context->master_graph->create_tensor(info, is_output); context->master_graph->add_node({output}, {actual_output}); } +#else + THROW("Caffe2 LMDB reader is not enabled (rocAL built without LMDB support)") +#endif } catch (const std::exception& e) { ROCAL_PRINT_EXCEPTION(context, e); diff --git a/rocAL/source/api/rocal_api_info.cpp b/rocAL/source/api/rocal_api_info.cpp index 560658e64..882b7d64f 100644 --- a/rocAL/source/api/rocal_api_info.cpp +++ b/rocAL/source/api/rocal_api_info.cpp @@ -101,9 +101,13 @@ RocalMetaData rocalCreateCaffe2LMDBLabelReader(RocalContext p_context, const char *source_path, bool is_output) { if (!p_context) THROW("Invalid rocal context passed to rocalCreateCaffe2LMDBLabelReader") - +#ifdef ROCAL_LMDB auto context = static_cast(p_context); return context->master_graph->create_caffe2_lmdb_record_meta_data_reader(source_path, MetaDataReaderType::CAFFE2_META_DATA_READER, MetaDataType::Label); +#else + THROW("Caffe2 LMDB reader is not enabled (rocAL built without LMDB support)") + return nullptr; +#endif } RocalMetaData @@ -111,9 +115,13 @@ RocalMetaData rocalCreateCaffe2LMDBReaderDetection(RocalContext p_context, const char *source_path, bool is_output) { if (!p_context) THROW("Invalid rocal context passed to rocalCreateCaffe2LMDBReaderDetection") +#ifdef ROCAL_LMDB auto context = static_cast(p_context); - return context->master_graph->create_caffe2_lmdb_record_meta_data_reader(source_path, MetaDataReaderType::CAFFE2_DETECTION_META_DATA_READER, MetaDataType::BoundingBox); +#else + THROW("Caffe2 LMDB reader is not enabled (rocAL built without LMDB support)") + return nullptr; +#endif } RocalMetaData @@ -121,8 +129,13 @@ RocalMetaData rocalCreateCaffeLMDBLabelReader(RocalContext p_context, const char *source_path) { if (!p_context) THROW("Invalid rocal context passed to rocalCreateCaffeLMDBLabelReader") +#ifdef ROCAL_LMDB auto context = static_cast(p_context); return context->master_graph->create_caffe_lmdb_record_meta_data_reader(source_path, MetaDataReaderType::CAFFE_META_DATA_READER, MetaDataType::Label); +#else + THROW("Caffe LMDB reader is not enabled (rocAL built without LMDB support)") + return nullptr; +#endif } RocalMetaData @@ -130,9 +143,13 @@ RocalMetaData rocalCreateCaffeLMDBReaderDetection(RocalContext p_context, const char *source_path) { if (!p_context) THROW("Invalid rocal context passed to rocalCreateCaffeLMDBReaderDetection") +#ifdef ROCAL_LMDB auto context = static_cast(p_context); - return context->master_graph->create_caffe_lmdb_record_meta_data_reader(source_path, MetaDataReaderType::CAFFE_DETECTION_META_DATA_READER, MetaDataType::BoundingBox); +#else + THROW("Caffe LMDB reader is not enabled (rocAL built without LMDB support)") + return nullptr; +#endif } size_t ROCAL_API_CALL rocalIsEmpty(RocalContext p_context) { diff --git a/rocAL/source/meta_data/caffe2_meta_data_reader.cpp b/rocAL/source/meta_data/caffe2_meta_data_reader.cpp index 85adc29b5..0528421cc 100644 --- a/rocAL/source/meta_data/caffe2_meta_data_reader.cpp +++ b/rocAL/source/meta_data/caffe2_meta_data_reader.cpp @@ -20,6 +20,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +#ifdef ROCAL_LMDB #include "meta_data/caffe2_meta_data_reader.h" #include @@ -168,3 +169,4 @@ void Caffe2MetaDataReader::release() { Caffe2MetaDataReader::Caffe2MetaDataReader() { } +#endif // ROCAL_LMDB diff --git a/rocAL/source/meta_data/caffe2_meta_data_reader_detection.cpp b/rocAL/source/meta_data/caffe2_meta_data_reader_detection.cpp index 8f9e9725a..c4c03cb3f 100644 --- a/rocAL/source/meta_data/caffe2_meta_data_reader_detection.cpp +++ b/rocAL/source/meta_data/caffe2_meta_data_reader_detection.cpp @@ -20,6 +20,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +#ifdef ROCAL_LMDB #include "meta_data/caffe2_meta_data_reader_detection.h" #include @@ -208,3 +209,4 @@ void Caffe2MetaDataReaderDetection::release() { Caffe2MetaDataReaderDetection::Caffe2MetaDataReaderDetection() { } +#endif // ROCAL_LMDB diff --git a/rocAL/source/meta_data/caffe_meta_data_reader.cpp b/rocAL/source/meta_data/caffe_meta_data_reader.cpp index a91d55941..fed525855 100644 --- a/rocAL/source/meta_data/caffe_meta_data_reader.cpp +++ b/rocAL/source/meta_data/caffe_meta_data_reader.cpp @@ -20,6 +20,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +#ifdef ROCAL_LMDB #include "meta_data/caffe_meta_data_reader.h" #include @@ -136,3 +137,4 @@ void CaffeMetaDataReader::read_lmdb_record(std::string _path, uint file_byte_siz mdb_txn_abort(_mdb_txn); mdb_env_close(_mdb_env); } +#endif // ROCAL_LMDB diff --git a/rocAL/source/meta_data/caffe_meta_data_reader_detection.cpp b/rocAL/source/meta_data/caffe_meta_data_reader_detection.cpp index bde3082c2..ed5cbe5b6 100644 --- a/rocAL/source/meta_data/caffe_meta_data_reader_detection.cpp +++ b/rocAL/source/meta_data/caffe_meta_data_reader_detection.cpp @@ -20,6 +20,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +#ifdef ROCAL_LMDB #include #include #include @@ -185,4 +186,5 @@ void CaffeMetaDataReaderDetection::release() { } CaffeMetaDataReaderDetection::CaffeMetaDataReaderDetection() { -} \ No newline at end of file +} +#endif // ROCAL_LMDB \ No newline at end of file diff --git a/rocAL/source/meta_data/meta_data_reader_factory.cpp b/rocAL/source/meta_data/meta_data_reader_factory.cpp index 706280d81..778ecde89 100644 --- a/rocAL/source/meta_data/meta_data_reader_factory.cpp +++ b/rocAL/source/meta_data/meta_data_reader_factory.cpp @@ -24,10 +24,12 @@ THE SOFTWARE. #include +#ifdef ROCAL_LMDB #include "meta_data/caffe2_meta_data_reader.h" #include "meta_data/caffe2_meta_data_reader_detection.h" #include "meta_data/caffe_meta_data_reader.h" #include "meta_data/caffe_meta_data_reader_detection.h" +#endif #include "meta_data/cifar10_meta_data_reader.h" #include "meta_data/coco_meta_data_reader.h" #include "meta_data/coco_meta_data_reader_key_points.h" @@ -111,6 +113,7 @@ std::shared_ptr create_meta_data_reader(const MetaDataConfig& co meta_data_reader->init(config, meta_data_batch); return meta_data_reader; } break; +#ifdef ROCAL_LMDB case MetaDataReaderType::CAFFE_META_DATA_READER: { if (config.type() != MetaDataType::Label) THROW("CAFFE_META_DATA_READER can only be used to load labels") @@ -143,6 +146,13 @@ std::shared_ptr create_meta_data_reader(const MetaDataConfig& co meta_data_reader->init(config, meta_data_batch); return meta_data_reader; } break; +#else + case MetaDataReaderType::CAFFE_META_DATA_READER: + case MetaDataReaderType::CAFFE_DETECTION_META_DATA_READER: + case MetaDataReaderType::CAFFE2_META_DATA_READER: + case MetaDataReaderType::CAFFE2_DETECTION_META_DATA_READER: + THROW("LMDB meta data reader is not enabled (rocAL built without LMDB support)") +#endif case MetaDataReaderType::MXNET_META_DATA_READER: { if (config.type() != MetaDataType::Label) THROW("MXNetMetaDataReader can only be used to load labels") diff --git a/rocAL/source/readers/image/caffe2_lmdb_record_reader.cpp b/rocAL/source/readers/image/caffe2_lmdb_record_reader.cpp index 34d745494..e2463a1bb 100644 --- a/rocAL/source/readers/image/caffe2_lmdb_record_reader.cpp +++ b/rocAL/source/readers/image/caffe2_lmdb_record_reader.cpp @@ -20,6 +20,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +#ifdef ROCAL_LMDB #include "readers/image/caffe2_lmdb_record_reader.h" #include #include @@ -266,3 +267,4 @@ void Caffe2LMDBRecordReader::read_image(unsigned char *buff, std::string file_na mdb_cursor_close(_read_mdb_cursor); _read_mdb_cursor = nullptr; } +#endif // ROCAL_LMDB diff --git a/rocAL/source/readers/image/caffe_lmdb_record_reader.cpp b/rocAL/source/readers/image/caffe_lmdb_record_reader.cpp index 72cf7cdb1..9e7130fbc 100644 --- a/rocAL/source/readers/image/caffe_lmdb_record_reader.cpp +++ b/rocAL/source/readers/image/caffe_lmdb_record_reader.cpp @@ -20,6 +20,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +#ifdef ROCAL_LMDB #include "readers/image/caffe_lmdb_record_reader.h" #include "pipeline/commons.h" @@ -260,3 +261,4 @@ void CaffeLMDBRecordReader::read_image(unsigned char *buff, std::string file_nam mdb_cursor_close(_read_mdb_cursor); _read_mdb_cursor = nullptr; } +#endif // ROCAL_LMDB diff --git a/rocAL/source/readers/image/reader_factory.cpp b/rocAL/source/readers/image/reader_factory.cpp index c7ceafff2..6b8e42ee6 100644 --- a/rocAL/source/readers/image/reader_factory.cpp +++ b/rocAL/source/readers/image/reader_factory.cpp @@ -26,8 +26,10 @@ THE SOFTWARE. #include #include "readers/file_source_reader.h" +#ifdef ROCAL_LMDB #include "readers/image/caffe2_lmdb_record_reader.h" #include "readers/image/caffe_lmdb_record_reader.h" +#endif #include "readers/image/cifar10_data_reader.h" #include "readers/image/coco_file_source_reader.h" #include "readers/image/external_source_reader.h" @@ -69,6 +71,7 @@ std::shared_ptr create_reader(ReaderConfig config) { throw std::runtime_error("CFar10 data reader cannot access the storage"); return ret; } break; +#ifdef ROCAL_LMDB case StorageType::CAFFE_LMDB_RECORD: { auto ret = std::make_shared(); if (ret->initialize(config) != Reader::Status::OK) @@ -81,6 +84,11 @@ std::shared_ptr create_reader(ReaderConfig config) { throw std::runtime_error("Caffe2LMDBRecordReader cannot access the storage"); return ret; } break; +#else + case StorageType::CAFFE_LMDB_RECORD: + case StorageType::CAFFE2_LMDB_RECORD: + throw std::runtime_error("LMDB reader is not enabled (rocAL built without LMDB support)"); +#endif case StorageType::MXNET_RECORDIO: { auto ret = std::make_shared(); if (ret->initialize(config) != Reader::Status::OK) diff --git a/rocAL_pybind/amd/rocal/decoders.py b/rocAL_pybind/amd/rocal/decoders.py index ab125b2c1..d8735e102 100644 --- a/rocAL_pybind/amd/rocal/decoders.py +++ b/rocAL_pybind/amd/rocal/decoders.py @@ -95,6 +95,8 @@ def image(*inputs, user_feature_key_map=None, path='', file_root='', annotations Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) elif (reader == "Caffe2Reader" or reader == "Caffe2ReaderDetection"): + if not hasattr(b, 'caffe2ImageDecoderShard'): + raise RuntimeError("rocAL was built without LMDB support. Caffe2 LMDB decoders are not available.") kwargs_pybind = { "source_path": path, "color_format": output_type, @@ -112,6 +114,8 @@ def image(*inputs, user_feature_key_map=None, path='', file_root='', annotations Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) elif reader == "CaffeReader" or reader == "CaffeReaderDetection": + if not hasattr(b, 'caffeImageDecoderShard'): + raise RuntimeError("rocAL was built without LMDB support. Caffe LMDB decoders are not available.") kwargs_pybind = { "source_path": path, "color_format": output_type, @@ -285,6 +289,8 @@ def image_random_crop(*inputs, user_feature_key_map=None, path='', file_root='', crop_output_image = b.tfImageDecoder( Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) elif (reader == "CaffeReader" or reader == "CaffeReaderDetection"): + if not hasattr(b, 'caffeImageDecoderPartialShard'): + raise RuntimeError("rocAL was built without LMDB support. Caffe LMDB decoders are not available.") kwargs_pybind = { "source_path": path, "color_format": output_type, @@ -303,6 +309,8 @@ def image_random_crop(*inputs, user_feature_key_map=None, path='', file_root='', crop_output_image = b.caffeImageDecoderPartialShard( Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) elif (reader == "Caffe2Reader" or reader == "Caffe2ReaderDetection"): + if not hasattr(b, 'caffe2ImageDecoderShard'): + raise RuntimeError("rocAL was built without LMDB support. Caffe2 LMDB decoders are not available.") kwargs_pybind = { "source_path": path, "color_format": output_type, @@ -398,6 +406,8 @@ def image_slice(*inputs, file_root='', path='', annotations_file='', shard_id=0, image_decoder_slice = b.cocoImageDecoderSliceShard( Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) elif (reader == "CaffeReader" or reader == "CaffeReaderDetection"): + if not hasattr(b, 'caffeImageDecoderPartialShard'): + raise RuntimeError("rocAL was built without LMDB support. Caffe LMDB decoders are not available.") kwargs_pybind = { "source_path": path, "color_format": output_type, @@ -417,6 +427,8 @@ def image_slice(*inputs, file_root='', path='', annotations_file='', shard_id=0, image_decoder_slice = b.caffeImageDecoderPartialShard( Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) elif (reader == "Caffe2Reader" or reader == "Caffe2ReaderDetection"): + if not hasattr(b, 'caffe2ImageDecoderShard'): + raise RuntimeError("rocAL was built without LMDB support. Caffe2 LMDB decoders are not available.") kwargs_pybind = { "source_path": path, "color_format": output_type, diff --git a/rocAL_pybind/amd/rocal/readers.py b/rocAL_pybind/amd/rocal/readers.py index e239f5f14..d4d771f98 100644 --- a/rocAL_pybind/amd/rocal/readers.py +++ b/rocAL_pybind/amd/rocal/readers.py @@ -138,6 +138,8 @@ def caffe(path, bbox=False, stick_to_shard=False, pad_last_batch=False): @return caffe reader meta data, bboxes, and labels. """ + if not hasattr(b, 'caffeReader'): + raise RuntimeError("rocAL was built without LMDB support. Caffe LMDB readers are not available.") # Output bboxes = [] labels = [] @@ -168,6 +170,8 @@ def caffe2(path, bbox=False, stick_to_shard=False, pad_last_batch=False): @return caffe2 reader meta data, bboxes, and labels. """ + if not hasattr(b, 'caffe2Reader'): + raise RuntimeError("rocAL was built without LMDB support. Caffe2 LMDB readers are not available.") # Output bboxes = [] labels = [] diff --git a/rocAL_pybind/rocal_pybind.cpp b/rocAL_pybind/rocal_pybind.cpp index 13ceb2f9d..b6a0f1a5d 100644 --- a/rocAL_pybind/rocal_pybind.cpp +++ b/rocAL_pybind/rocal_pybind.cpp @@ -886,10 +886,12 @@ py::class_(m, "rocalListOfTensorList") m.def("setOutputs", &rocalSetOutputs); m.def("tfReader", &rocalCreateTFReader, py::return_value_policy::reference); m.def("tfReaderDetection", &rocalCreateTFReaderDetection, py::return_value_policy::reference); +#ifdef ROCAL_LMDB m.def("caffeReader", &rocalCreateCaffeLMDBLabelReader, py::return_value_policy::reference); m.def("caffe2Reader", &rocalCreateCaffe2LMDBLabelReader, py::return_value_policy::reference); m.def("caffeReaderDetection", &rocalCreateCaffeLMDBReaderDetection, py::return_value_policy::reference); m.def("caffe2ReaderDetection", &rocalCreateCaffe2LMDBReaderDetection, py::return_value_policy::reference); +#endif m.def("mxnetReader", &rocalCreateMXNetReader, py::return_value_policy::reference); m.def("webDatasetReader", &rocalCreateWebDatasetReader, py::return_value_policy::reference); m.def("isEmpty", &rocalIsEmpty); @@ -1108,6 +1110,7 @@ py::class_(m, "rocalListOfTensorList") py::return_value_policy::reference); m.def("tfImageDecoder", &rocalJpegTFRecordSource, "Reads file from the source given and decodes it according to the policy only for TFRecords", py::return_value_policy::reference); +#ifdef ROCAL_LMDB m.def("caffeImageDecoder", &rocalJpegCaffeLMDBRecordSource, "Reads file from the source given and decodes it according to the policy", py::return_value_policy::reference); m.def("caffeImageDecoderShard", &rocalJpegCaffeLMDBRecordSourceSingleShard, "Reads file from the source given and decodes it according to the shard id and number of shards", @@ -1120,6 +1123,7 @@ py::class_(m, "rocalListOfTensorList") py::return_value_policy::reference); m.def("caffe2ImageDecoderPartialShard", &rocalJpegCaffe2LMDBRecordSourcePartialSingleShard, "Reads file from the source given and partially decodes it according to the shard id and number of shards", py::return_value_policy::reference); +#endif m.def("fusedDecoderCrop", &rocalFusedJpegCrop, "Reads file from the source and decodes them partially to output random crops", py::return_value_policy::reference); m.def("fusedDecoderCropShard", &rocalFusedJpegCropSingleShard, "Reads file from the source and decodes them partially to output random crops",