diff --git a/CMakeLists.txt b/CMakeLists.txt index e3aeefac7..932ee0111 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -629,7 +629,7 @@ if(OCOS_ENABLE_VISION) if (NOT OCOS_ENABLE_DLIB) message(FATAL_ERROR "Vision operators require DLIB to be enabled.") # for now, we need dlib for image processing endif() - include(ext_imgcodecs) + include(ext_imgcodecs) target_include_directories(ocos_operators PUBLIC ${libPNG_SOURCE_DIR} ${libJPEG_SOURCE_DIR}) target_link_libraries(ocos_operators PUBLIC ${PNG_LIBRARY} ${JPEG_LIBRARY}) endif() @@ -763,9 +763,14 @@ if(OCOS_BUILD_SHARED_LIB) source_group(TREE ${PROJECT_SOURCE_DIR} FILES ${shared_TARGET_SRC}) standardize_output_folder(extensions_shared) - if(LINUX OR ANDROID) - set_property(TARGET extensions_shared APPEND_STRING PROPERTY LINK_FLAGS - " -Wl,--version-script -Wl,${PROJECT_SOURCE_DIR}/shared/ortcustomops.ver") + if(LINUX OR ANDROID OR CMAKE_SYSTEM_NAME STREQUAL "AIX") + if(CMAKE_SYSTEM_NAME STREQUAL "AIX") + set_property(TARGET extensions_shared APPEND_STRING PROPERTY LINK_FLAGS + " -Wl,-bE:${PROJECT_SOURCE_DIR}/shared/ortcustomops.exp") + else() + set_property(TARGET extensions_shared APPEND_STRING PROPERTY LINK_FLAGS + " -Wl,--version-script -Wl,${PROJECT_SOURCE_DIR}/shared/ortcustomops.ver") + endif() # strip if not a debug build if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") set_property(TARGET extensions_shared APPEND_STRING PROPERTY LINK_FLAGS " -Wl,-s") diff --git a/cmake/externals/blingfire-aix.patch b/cmake/externals/blingfire-aix.patch new file mode 100644 index 000000000..b864c775a --- /dev/null +++ b/cmake/externals/blingfire-aix.patch @@ -0,0 +1,34 @@ +--- ./blingfireclient.library/inc/FAConfig_orig.h 2026-05-19 03:27:38.241882009 -0500 ++++ ./blingfireclient.library/inc/FAConfig.h 2026-05-19 03:29:23.720979389 -0500 +@@ -11,6 +11,7 @@ + #define NEW new + #endif + ++#include + #if _MSC_VER < 1400 + # ifndef fopen_s + # define fopen_s(ppF,pN,pA) int(0); *ppF = fopen(pN,pA); +@@ -23,13 +24,12 @@ + #include + #include + +-#ifdef BLING_FIRE_MAC ++#if defined(BLING_FIRE_MAC) || defined(_AIX) + #include + #else + #include + #endif + +-#include + #include + #include + #include +@@ -79,7 +79,7 @@ + + #include + +-/* because we have stdint.h ++/* because we have stdint.h + #ifndef __int32 + #define __int32 int32_t + #endif diff --git a/cmake/externals/blingfire.cmake b/cmake/externals/blingfire.cmake index 8c7eefe1c..f4799b975 100644 --- a/cmake/externals/blingfire.cmake +++ b/cmake/externals/blingfire.cmake @@ -3,9 +3,10 @@ FetchContent_Declare( GIT_REPOSITORY https://github.com/microsoft/BlingFire.git GIT_TAG 0831265c1aca95ca02eca5bf1155e4251e545328 EXCLUDE_FROM_ALL - PATCH_COMMAND git checkout . && git apply --ignore-space-change --ignore-whitespace ${PROJECT_SOURCE_DIR}/cmake/externals/blingfire_cmake.patch) - + PATCH_COMMAND git checkout . && git apply --ignore-space-change --ignore-whitespace ${PROJECT_SOURCE_DIR}/cmake/externals/blingfire_cmake.patch && git apply --ignore-space-change --ignore-whitespace ${PROJECT_SOURCE_DIR}/cmake/externals/blingfire-aix.patch + ) FetchContent_MakeAvailable(Blingfire) + set_target_properties(bingfirtinydll_static PROPERTIES FOLDER externals/bingfire) set_target_properties(fsaClientTiny PROPERTIES FOLDER externals/bingfire) diff --git a/cmake/externals/blingfire_cmake.patch b/cmake/externals/blingfire_cmake.patch index 2bdb2b239..01a873765 100644 --- a/cmake/externals/blingfire_cmake.patch +++ b/cmake/externals/blingfire_cmake.patch @@ -5,6 +5,18 @@ index 5d47b3a..dd36aaf 100644 @@ -1,4 +1,4 @@ -cmake_minimum_required (VERSION 3.0) +cmake_minimum_required (VERSION 3.5) - + set (BLING_FIRE_VERSION_MAJOR 1) set (BLING_FIRE_VERSION_MINOR 0) + +--- ./CMakeLists_orig.txt 2026-05-23 03:43:50.580750573 -0500 ++++ ./CMakeLists.txt 2026-05-23 03:44:19.454763579 -0500 +@@ -107,7 +107,7 @@ + IF(${dirname} STREQUAL "any_test") + target_link_libraries(${dirname} ${CMAKE_DL_LIBS}) + ENDIF() +- IF (APPLE) ++ IF (APPLE OR CMAKE_SYSTEM_NAME MATCHES "AIX") + target_link_libraries(${dirname} iconv) + ENDIF() + ENDIF() diff --git a/cmake/externals/dr_libs.cmake b/cmake/externals/dr_libs.cmake index f7811b4dc..c68b0f562 100644 --- a/cmake/externals/dr_libs.cmake +++ b/cmake/externals/dr_libs.cmake @@ -2,6 +2,7 @@ FetchContent_Declare(dr_libs URL https://github.com/mackron/dr_libs/archive/660795b2834aebb2217c9849d668b6e4bd4ef810.zip URL_HASH SHA1=af554b21dcd1ab3c7c8d946638682a2cbccf3e16 SOURCE_SUBDIR not_set + PATCH_COMMAND git apply --whitespace=fix --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/cmake/externals/dr_libs_aix.patch ) FetchContent_MakeAvailable(dr_libs) diff --git a/cmake/externals/dr_libs_aix.patch b/cmake/externals/dr_libs_aix.patch new file mode 100644 index 000000000..100ee1e08 --- /dev/null +++ b/cmake/externals/dr_libs_aix.patch @@ -0,0 +1,13 @@ +--- ./dr_mp3.h 2026-03-28 10:41:13.180505219 -0500 ++++ ./dr_mp3.h 2026-03-28 10:41:06.567807627 -0500 +@@ -86,6 +86,10 @@ + #ifndef dr_mp3_h + #define dr_mp3_h + ++#if defined(_AIX) && defined(hz) ++#undef hz ++#endif ++ + #ifdef __cplusplus + extern "C" { + #endif diff --git a/cmake/externals/opencv.cmake b/cmake/externals/opencv.cmake index 56b850da0..5511dabb6 100644 --- a/cmake/externals/opencv.cmake +++ b/cmake/externals/opencv.cmake @@ -149,8 +149,8 @@ FetchContent_Declare( -DBUILD_TESTS:BOOL=FALSE -DBUILD_SHARED_LIBS:BOOL=FALSE -DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_CURRENT_BINARY_DIR}/opencv - PATCH_COMMAND git checkout . && git apply --whitespace=fix --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/cmake/externals/opencv-no-rtti.patch EXCLUDE_FROM_ALL + PATCH_COMMAND git checkout . && git apply --whitespace=fix --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/cmake/externals/opencv-no-rtti.patch && git apply --whitespace=fix --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/cmake/externals/rand_aix.patch && git apply --whitespace=fix --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/cmake/externals/opencv_libpng_aix.patch ) FetchContent_MakeAvailable(opencv) diff --git a/cmake/externals/opencv_libpng_aix.patch b/cmake/externals/opencv_libpng_aix.patch new file mode 100644 index 000000000..fab77b800 --- /dev/null +++ b/cmake/externals/opencv_libpng_aix.patch @@ -0,0 +1,19 @@ +--- a/3rdparty/libpng/CMakeLists.txt ++++ b/3rdparty/libpng/CMakeLists.txt +@@ -56,11 +56,12 @@ else() + endif() + +-if(PPC64LE OR PPC64) +- # VSX3 features are backwards compatible +- if(";${CPU_BASELINE_FINAL};" MATCHES "VSX.*" +- AND NOT PPC64) ++# Enable PowerPC VSX optimizations on PowerPC systems (Linux) and AIX ++if(PPC64LE OR PPC64 OR CMAKE_SYSTEM_NAME MATCHES "AIX") ++ # Enable VSX on PowerPC (AIX and Linux) ++ if(";${CPU_BASELINE_FINAL};" MATCHES "VSX.*" OR CMAKE_SYSTEM_NAME MATCHES "AIX") + list(APPEND lib_srcs powerpc/powerpc_init.c powerpc/filter_vsx_intrinsics.c) + add_definitions(-DPNG_POWERPC_VSX_OPT=2) ++ message(STATUS "libpng: PowerPC VSX optimizations enabled") + else() + add_definitions(-DPNG_POWERPC_VSX_OPT=0) + endif() diff --git a/cmake/externals/rand_aix.patch b/cmake/externals/rand_aix.patch new file mode 100644 index 000000000..7988d1d24 --- /dev/null +++ b/cmake/externals/rand_aix.patch @@ -0,0 +1,46 @@ + +--- ./modules/core/src/rand.cpp 2026-03-25 07:34:27.666925971 -0500 ++++ ./modules/core/src/rand.cpp 2026-03-25 07:40:40.661543262 -0500 +@@ -48,6 +48,10 @@ + + #include "precomp.hpp" + ++#if defined(_AIX) && defined(hz) ++#undef hz ++#endif ++ + namespace cv + { + +--- ./modules/core/src/system.cpp 2026-03-25 07:50:38.785508231 -0500 ++++ ./modules/core/src/system.cpp 2026-03-25 07:50:25.791118063 -0500 +@@ -130,7 +130,7 @@ + # include + #if defined __QNXNTO__ + # include +-#else ++#elif !defined (_AIX) + # include + #endif + #if defined __ANDROID__ || defined __linux__ +@@ -146,8 +146,8 @@ + #endif + + +-#if (defined __ppc64__ || defined __PPC64__) && defined __unix__ ++#if (defined __ppc64__ || defined __PPC64__) && defined __unix__ && !defined _AIX + # include "sys/auxv.h" + # ifndef AT_HWCAP2 + # define AT_HWCAP2 26 + # endif +@@ -615,7 +615,7 @@ + #ifdef __mips_msa + have[CV_CPU_MSA] = true; + #endif + +- #if (defined __ppc64__ || defined __PPC64__) && defined __unix__ ++ #if (defined __ppc64__ || defined __PPC64__) && defined __unix__ && !defined _AIX + unsigned int hwcap = getauxval(AT_HWCAP); + if (hwcap & PPC_FEATURE_HAS_VSX) { + hwcap = getauxval(AT_HWCAP2); + diff --git a/cmake/externals/sentencepieceproject.cmake b/cmake/externals/sentencepieceproject.cmake index dfef57654..464796599 100644 --- a/cmake/externals/sentencepieceproject.cmake +++ b/cmake/externals/sentencepieceproject.cmake @@ -39,7 +39,9 @@ endif() # PB files was seperated as another patch file to avoid the patch file too large to be reviewed. set(spm_patches "${PROJECT_SOURCE_DIR}/cmake/externals/sentencepieceproject_cmake.patch" - "${PROJECT_SOURCE_DIR}/cmake/externals/sentencepieceproject_pb.patch") + "${PROJECT_SOURCE_DIR}/cmake/externals/sentencepieceproject_pb.patch" + "${PROJECT_SOURCE_DIR}/cmake/externals/sentencepieceproject_aix.patch" + "${PROJECT_SOURCE_DIR}/cmake/externals/sentencepieceproject_enum.patch") set(spm_patch_command git checkout . && git apply --ignore-space-change --ignore-whitespace ${spm_patches}) diff --git a/cmake/externals/sentencepieceproject_aix.patch b/cmake/externals/sentencepieceproject_aix.patch new file mode 100644 index 000000000..f6382a1ed --- /dev/null +++ b/cmake/externals/sentencepieceproject_aix.patch @@ -0,0 +1,13 @@ +--- ./src/util.h_orig 2026-03-13 04:45:26.898234633 -0500 ++++ ./src/util.h 2026-03-13 05:08:47.892826822 -0500 +@@ -36,7 +36,9 @@ + #include + #endif + +-#if !defined(__APPLE__) && !defined(_WIN32) ++#if defined(_AIX) ++#define IS_BIG_ENDIAN ++#elif !defined(__APPLE__) && !defined(_WIN32) + #include + #if BYTE_ORDER == __BIG_ENDIAN + #define IS_BIG_ENDIAN diff --git a/cmake/externals/sentencepieceproject_enum.patch b/cmake/externals/sentencepieceproject_enum.patch new file mode 100644 index 000000000..0778568b4 --- /dev/null +++ b/cmake/externals/sentencepieceproject_enum.patch @@ -0,0 +1,11 @@ +--- ./src/unicode_script.h_orig 2026-04-06 23:28:59.276085358 -0500 ++++ ./src/unicode_script.h 2026-04-06 23:29:39.906085511 -0500 +@@ -19,7 +19,7 @@ + + namespace sentencepiece { + namespace unicode_script { +-enum ScriptType { ++enum ScriptType : int32_t { + U_Adlam, + U_Ahom, + U_Anatolian_Hieroglyphs, diff --git a/operators/text/vector_to_string.hpp b/operators/text/vector_to_string.hpp index 12e91af06..e13c80602 100644 --- a/operators/text/vector_to_string.hpp +++ b/operators/text/vector_to_string.hpp @@ -13,7 +13,7 @@ namespace std { template struct hash> { - size_t operator()(const vector& __vector) const noexcept; + size_t operator()(const vector& __vec) const noexcept; }; } // namespace std diff --git a/operators/tokenizer/ugm_kernels.hpp b/operators/tokenizer/ugm_kernels.hpp index 990dfa9a8..2eed3b746 100644 --- a/operators/tokenizer/ugm_kernels.hpp +++ b/operators/tokenizer/ugm_kernels.hpp @@ -89,11 +89,22 @@ struct SpmUgmTokenizer { // std::cout << int(charsmap_data_[i]) << " "; // } + if (charsmap_data_.size() < sizeof(uint32_t)) { + return OrtxStatus(extError_t::kOrtxErrorCorruptData, "Precompiled charsmap is too short to contain XCDA header."); + } size_t charsmap_offset = 0; // First four bytes of precompiled_charsmap contains length of binary // blob containing XOR-compressed compact double array (XCDA) entries - uint32_t xcda_blob_size = *(const uint32_t*)&charsmap_data_[0]; + // The data is stored in little-endian format, so we need to convert on big-endian systems + uint32_t xcda_blob_size; + std::memcpy(&xcda_blob_size, &charsmap_data_[0], sizeof(xcda_blob_size)); + + // Convert from little-endian to host byte order + #if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + xcda_blob_size = __builtin_bswap32(xcda_blob_size); + #endif + charsmap_offset += sizeof(xcda_blob_size); if (xcda_blob_size + charsmap_offset >= charsmap_data_.size()) { return OrtxStatus(extError_t::kOrtxErrorCorruptData, "Index out of array bounds in precompiled charsmap!"); @@ -296,7 +307,7 @@ struct SpmUgmTokenizer { "Invalid UTF-8 encoding detected in input string at position " + std::to_string(-validation_result)); } - + std::string normalized; if (case_encoder_) { normalized = NmtNormalize(input); @@ -637,7 +648,12 @@ struct SpmUgmTokenizer { if (index >= xcda_array_size_) { // Fix #2: off-by-one correction ORTX_CXX_API_THROW("[UgmTok]Index out of array bounds in XCDA array!", ORT_RUNTIME_EXCEPTION); } - return xcda_array_[index]; + uint32_t value = xcda_array_[index]; + // Convert from little-endian to host byte order + #if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + value = __builtin_bswap32(value); + #endif + return value; } const uint32_t* xcda_array_; size_t xcda_array_size_; @@ -1051,4 +1067,4 @@ class SpmUgmDecoder { std::set special_token_ids_; }; -} // namespace ort_extensions \ No newline at end of file +} // namespace ort_extensions diff --git a/shared/api/nemo_mel_spectrogram.h b/shared/api/nemo_mel_spectrogram.h index 7033169ac..c07a9408d 100644 --- a/shared/api/nemo_mel_spectrogram.h +++ b/shared/api/nemo_mel_spectrogram.h @@ -9,6 +9,10 @@ #include #include +#if defined _AIX && defined (hz) +#undef hz +#endif + namespace nemo_mel { struct NemoMelConfig { diff --git a/shared/ortcustomops.exp b/shared/ortcustomops.exp new file mode 100644 index 000000000..e07771273 --- /dev/null +++ b/shared/ortcustomops.exp @@ -0,0 +1,3 @@ +RegisterCustomOps +AddExternalCustomOp +GetActiveOrtAPIVersion