From 3358e18c761dae9f74cb7b46cdb0b9876b47fe46 Mon Sep 17 00:00:00 2001 From: tim Date: Mon, 18 Jul 2016 11:52:36 -0700 Subject: [PATCH 1/7] update version number to 2.1 --- oss_src/unity/lib/version_number.hpp | 2 +- oss_src/unity/python/setup.py | 2 +- oss_src/unity/python/sframe/version_info.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/oss_src/unity/lib/version_number.hpp b/oss_src/unity/lib/version_number.hpp index 16303e5a..a46f0bcb 100644 --- a/oss_src/unity/lib/version_number.hpp +++ b/oss_src/unity/lib/version_number.hpp @@ -6,4 +6,4 @@ * of the BSD license. See the LICENSE file for details. */ -#define __UNITY_VERSION__ "1.10"//#{{VERSION_STRING}} +#define __UNITY_VERSION__ "2.1"//#{{VERSION_STRING}} diff --git a/oss_src/unity/python/setup.py b/oss_src/unity/python/setup.py index f56d7c5f..d8e3f5fc 100644 --- a/oss_src/unity/python/setup.py +++ b/oss_src/unity/python/setup.py @@ -129,7 +129,7 @@ def run(self): sys.stderr.write(msg) sys.exit(1) - version_number='1.9'#{{VERSION_STRING}} + version_number='2.1'#{{VERSION_STRING}} setup( name="SFrame", version=version_number, diff --git a/oss_src/unity/python/sframe/version_info.py b/oss_src/unity/python/sframe/version_info.py index b5b325d3..fd03ad9c 100644 --- a/oss_src/unity/python/sframe/version_info.py +++ b/oss_src/unity/python/sframe/version_info.py @@ -7,6 +7,6 @@ ''' # python egg version -__VERSION__ = '1.9'#{{VERSION_STRING}} -version = '1.9'#{{VERSION_STRING}} +__VERSION__ = '2.1'#{{VERSION_STRING}} +version = '2.1'#{{VERSION_STRING}} build_number = '0'#{{BUILD_NUMBER}} From 92e3614130bf8adfc3bbe39b573d1a3ec5a09b89 Mon Sep 17 00:00:00 2001 From: Yucheng Low Date: Tue, 19 Jul 2016 14:56:26 -0700 Subject: [PATCH 2/7] fixed memory leak in nanomsg conversion --- oss_src/cppipc/common/message_types.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/oss_src/cppipc/common/message_types.cpp b/oss_src/cppipc/common/message_types.cpp index 937deeaa..7dafc4fa 100644 --- a/oss_src/cppipc/common/message_types.cpp +++ b/oss_src/cppipc/common/message_types.cpp @@ -20,7 +20,7 @@ namespace cppipc { void call_message::clear() { if (!zmqbodyused) { - if (!body) free((void*)body); + if (body) free((void*)body); } body = NULL; objectid = 0; @@ -77,8 +77,6 @@ void call_message::emit(nanosockets::zmq_msg_vector& msg) { z_body->assign(body, bodylen); } - // we are giving away the body pointer - body = NULL; clear(); } @@ -86,7 +84,7 @@ void call_message::emit(nanosockets::zmq_msg_vector& msg) { void reply_message::clear() { if (!zmqbodyused) { - if (!body) free((void*)body); + if (body) free((void*)body); } body = NULL; bodylen = 0; @@ -137,8 +135,6 @@ void reply_message::emit(nanosockets::zmq_msg_vector& msg) { z_body->assign(body, bodylen); } - // we are giving away the body pointer - body = NULL; clear(); } From 51a288c79f42ffaf0dfa4b44d11ea3ea95102ad2 Mon Sep 17 00:00:00 2001 From: Zach Nation Date: Tue, 19 Jul 2016 15:40:01 -0700 Subject: [PATCH 3/7] Some URL changes/updates Change-Id: Iab6d0f6088d4070815606f41b2194291b7acfed2 --- oss_src/lambda/worker_pool.hpp | 2 -- oss_src/unity/python/README.txt | 2 +- .../python/doc/source/_themes/sphinx_rtd_theme/layout.html | 2 -- oss_src/unity/python/sframe/data_structures/sframe.py | 3 +-- oss_src/unity/sdk/doxygen/header.html | 1 - 5 files changed, 2 insertions(+), 8 deletions(-) diff --git a/oss_src/lambda/worker_pool.hpp b/oss_src/lambda/worker_pool.hpp index f891e7b3..7ef8edac 100644 --- a/oss_src/lambda/worker_pool.hpp +++ b/oss_src/lambda/worker_pool.hpp @@ -481,8 +481,6 @@ class worker_pool { logprogress_stream << "All operations will proceed as normal, but " << "lambda operations will not be able to use all " << "available cores." << std::endl; - logprogress_stream << "To help us diagnose this issue, please send " - << "the log file to product-feedback@turi.com." << std::endl; logprogress_stream << "(The location of the log file is printed at the " << "start of the GraphLab server)." << std::endl; logstream(LOG_ERROR) << "Less than " << num_workers << " successfully started." diff --git a/oss_src/unity/python/README.txt b/oss_src/unity/python/README.txt index cb3f4cd2..2961b554 100644 --- a/oss_src/unity/python/README.txt +++ b/oss_src/unity/python/README.txt @@ -7,7 +7,7 @@ References The code for this package is located here: https://github.com/turi-code/SFrame A lot more documentation regarding the SFrame is available here: -https://turi.com and on our Forum at http://forum.turi.com +https://turi.com Contributors ============ diff --git a/oss_src/unity/python/doc/source/_themes/sphinx_rtd_theme/layout.html b/oss_src/unity/python/doc/source/_themes/sphinx_rtd_theme/layout.html index 347e9c9a..a8f566a5 100644 --- a/oss_src/unity/python/doc/source/_themes/sphinx_rtd_theme/layout.html +++ b/oss_src/unity/python/doc/source/_themes/sphinx_rtd_theme/layout.html @@ -73,8 +73,6 @@ {# TypeKit for museo sans #} - {# Turi metrics #} - diff --git a/oss_src/unity/python/sframe/data_structures/sframe.py b/oss_src/unity/python/sframe/data_structures/sframe.py index 521f50bc..7d6825b2 100644 --- a/oss_src/unity/python/sframe/data_structures/sframe.py +++ b/oss_src/unity/python/sframe/data_structures/sframe.py @@ -179,8 +179,7 @@ def __rdd_support_init__(sprk_ctx,graphlab_util_ref): unity = glconnect.get_unity() STAGING_DIR = unity.get_current_cache_file_location() if STAGING_DIR is None: - raise RuntimeError("Could not retrieve local staging directory! \ - Please contact us on http://forum.turi.com.") + raise RuntimeError("Could not retrieve local staging directory!") else: raise RuntimeError("Your spark context's master is '" + str(sprk_ctx.master) + diff --git a/oss_src/unity/sdk/doxygen/header.html b/oss_src/unity/sdk/doxygen/header.html index 963cb58d..7a0adcb0 100644 --- a/oss_src/unity/sdk/doxygen/header.html +++ b/oss_src/unity/sdk/doxygen/header.html @@ -8,7 +8,6 @@ $projectname: $title $title - $treeview From ccedb11695cbb3f54580679f02864f0d2afde41e Mon Sep 17 00:00:00 2001 From: Yucheng Low Date: Wed, 20 Jul 2016 10:35:03 -0700 Subject: [PATCH 4/7] fix egg making on windows. --- oss_local_scripts/make_egg.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/oss_local_scripts/make_egg.sh b/oss_local_scripts/make_egg.sh index 7d9318bf..3ea565e9 100755 --- a/oss_local_scripts/make_egg.sh +++ b/oss_local_scripts/make_egg.sh @@ -234,14 +234,13 @@ package_egg() { dist_type="bdist_wheel" fi + VERSION_NUMBER=`python -c "import sframe; print(sframe.version)"` + rm -rf sframe/libgcc_s_seh-1.dll \ sframe/libstdc++-6.dll \ sframe/cython/libgcc_s_seh-1.dll \ sframe/cython/libstdc++-6.dll - push_ld_library_path - - VERSION_NUMBER=`python -c "import sframe; print(sframe.version)"` ${PYTHON_EXECUTABLE} setup.py ${dist_type} # This produced an egg/wheel starting with SFrame-${VERSION_NUMBER} under dist/ cd ${WORKSPACE} @@ -287,6 +286,9 @@ package_egg() { # Install the egg and do a sanity check $PIP_EXECUTABLE install --force-reinstall --ignore-installed ${EGG_PATH} unset PYTHONPATH + if [[ $OSTYPE == msys ]]; then + $PYTHON_EXECUTABLE -c "import graphlab; graphlab.get_dependencies()" || true + fi $PYTHON_EXECUTABLE -c "import sframe; sframe.SArray(range(100)).apply(lambda x: x)" pop_ld_library_path From 20f1d6232fe8a868ede4ff7e57c7e66043dd776b Mon Sep 17 00:00:00 2001 From: Yucheng Low Date: Wed, 20 Jul 2016 12:18:25 -0700 Subject: [PATCH 5/7] stupid error --- oss_local_scripts/make_egg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oss_local_scripts/make_egg.sh b/oss_local_scripts/make_egg.sh index 3ea565e9..6acfc880 100755 --- a/oss_local_scripts/make_egg.sh +++ b/oss_local_scripts/make_egg.sh @@ -287,7 +287,7 @@ package_egg() { $PIP_EXECUTABLE install --force-reinstall --ignore-installed ${EGG_PATH} unset PYTHONPATH if [[ $OSTYPE == msys ]]; then - $PYTHON_EXECUTABLE -c "import graphlab; graphlab.get_dependencies()" || true + $PYTHON_EXECUTABLE -c "import sframe; sframe.get_dependencies()" || true fi $PYTHON_EXECUTABLE -c "import sframe; sframe.SArray(range(100)).apply(lambda x: x)" From 1f19742e8f89e22e7e89c809c7752e1a69cf5ba0 Mon Sep 17 00:00:00 2001 From: Yucheng Low Date: Wed, 20 Jul 2016 17:07:00 -0700 Subject: [PATCH 6/7] Made explicit destructors for a bunch of interface / proxy objects. --- oss_src/cppipc/common/object_factory.cpp | 4 ++++ oss_src/cppipc/ipc_object_base.hpp | 5 +++- oss_src/cppipc/magic_macros.hpp | 24 ++++++++++++++++++- oss_src/unity/lib/CMakeLists.txt | 1 + oss_src/unity/lib/api/destructor_impl.cpp | 17 +++++++++++++ .../unity/lib/api/unity_global_interface.hpp | 2 +- .../unity/lib/api/unity_graph_interface.hpp | 2 +- .../api/unity_sarray_builder_interface.hpp | 2 +- .../unity/lib/api/unity_sarray_interface.hpp | 2 +- .../api/unity_sframe_builder_interface.hpp | 2 +- .../unity/lib/api/unity_sframe_interface.hpp | 2 +- .../unity/lib/api/unity_sketch_interface.hpp | 2 +- 12 files changed, 56 insertions(+), 9 deletions(-) create mode 100644 oss_src/unity/lib/api/destructor_impl.cpp diff --git a/oss_src/cppipc/common/object_factory.cpp b/oss_src/cppipc/common/object_factory.cpp index 630881f4..452b64ae 100644 --- a/oss_src/cppipc/common/object_factory.cpp +++ b/oss_src/cppipc/common/object_factory.cpp @@ -5,3 +5,7 @@ * This software may be modified and distributed under the terms * of the BSD license. See the LICENSE file for details. */ +#include +namespace cppipc { +ipc_object_base::~ipc_object_base() { } +} diff --git a/oss_src/cppipc/ipc_object_base.hpp b/oss_src/cppipc/ipc_object_base.hpp index 544c4da5..846fd214 100644 --- a/oss_src/cppipc/ipc_object_base.hpp +++ b/oss_src/cppipc/ipc_object_base.hpp @@ -15,7 +15,10 @@ */ namespace cppipc { -class EXPORT ipc_object_base: public std::enable_shared_from_this { }; +class EXPORT ipc_object_base: public std::enable_shared_from_this { + public: + virtual ~ipc_object_base(); +}; } // cppipc diff --git a/oss_src/cppipc/magic_macros.hpp b/oss_src/cppipc/magic_macros.hpp index 0d5382c8..f7122847 100644 --- a/oss_src/cppipc/magic_macros.hpp +++ b/oss_src/cppipc/magic_macros.hpp @@ -107,6 +107,19 @@ * }; * \endcode */ +#define GENERATE_INTERFACE_NO_INLINE_DESTRUCTOR(base_name, proxy_name, functions) \ + class base_name : public cppipc::ipc_object_base { \ + public: \ + typedef proxy_name proxy_object_type; \ + virtual ~base_name(); \ + inline virtual void save(graphlab::oarchive& oarc) const {} \ + inline virtual void load(graphlab::iarchive& iarc) {} \ + BOOST_PP_SEQ_FOR_EACH(__GENERATE_BASE__, _, __ADD_PARENS__(functions)) \ + REGISTRATION_BEGIN(base_name) \ + BOOST_PP_SEQ_FOR_EACH(__GENERATE_REGISTRATION__, base_name, __ADD_PARENS__(functions)) \ + REGISTRATION_END \ + }; + #define GENERATE_INTERFACE(base_name, proxy_name, functions) \ class base_name : public cppipc::ipc_object_base { \ public: \ @@ -120,6 +133,8 @@ REGISTRATION_END \ }; +#define GENERATE_BASE_DESTRUCTOR(base_name) \ + base_name::~base_name() { } @@ -292,13 +307,20 @@ class proxy_name; \ GENERATE_INTERFACE(base_name, proxy_name, functions) +#define GENERATE_INTERFACE_AND_PROXY_NO_INLINE_DESTRUCTOR(base_name, proxy_name, functions) \ + class proxy_name; \ + GENERATE_INTERFACE_NO_INLINE_DESTRUCTOR(base_name, proxy_name, functions) #else #define GENERATE_INTERFACE_AND_PROXY(base_name, proxy_name, functions) \ class proxy_name; \ GENERATE_INTERFACE(base_name, proxy_name, functions) \ - GENERATE_PROXY(base_name, proxy_name, functions) + GENERATE_PROXY(base_name, proxy_name, functions) +#define GENERATE_INTERFACE_AND_PROXY_NO_INLINE_DESTRUCTOR(base_name, proxy_name, functions) \ + class proxy_name; \ + GENERATE_INTERFACE_NO_INLINE_DESTRUCTOR(base_name, proxy_name, functions) \ + GENERATE_PROXY(base_name, proxy_name, functions) #endif #endif diff --git a/oss_src/unity/lib/CMakeLists.txt b/oss_src/unity/lib/CMakeLists.txt index 60d04b10..9d0907b2 100644 --- a/oss_src/unity/lib/CMakeLists.txt +++ b/oss_src/unity/lib/CMakeLists.txt @@ -3,6 +3,7 @@ project(unity) make_library(unity_core SOURCES api/function_closure_info.cpp + api/destructor_impl.cpp variant_converter.cpp variant.cpp unity_global.cpp diff --git a/oss_src/unity/lib/api/destructor_impl.cpp b/oss_src/unity/lib/api/destructor_impl.cpp new file mode 100644 index 00000000..4f8a1597 --- /dev/null +++ b/oss_src/unity/lib/api/destructor_impl.cpp @@ -0,0 +1,17 @@ +#include "unity_global_interface.hpp" +#include "unity_graph_interface.hpp" +#include "unity_sarray_builder_interface.hpp" +#include "unity_sarray_interface.hpp" +#include "unity_sframe_builder_interface.hpp" +#include "unity_sframe_interface.hpp" +#include "unity_sketch_interface.hpp" + +namespace graphlab { +GENERATE_BASE_DESTRUCTOR(unity_global_base) +GENERATE_BASE_DESTRUCTOR(unity_sgraph_base) +GENERATE_BASE_DESTRUCTOR(unity_sarray_builder_base) +GENERATE_BASE_DESTRUCTOR(unity_sarray_base) +GENERATE_BASE_DESTRUCTOR(unity_sframe_builder_base) +GENERATE_BASE_DESTRUCTOR(unity_sframe_base) +GENERATE_BASE_DESTRUCTOR(unity_sketch_base) +} diff --git a/oss_src/unity/lib/api/unity_global_interface.hpp b/oss_src/unity/lib/api/unity_global_interface.hpp index c31c65e1..fa0cbf13 100644 --- a/oss_src/unity/lib/api/unity_global_interface.hpp +++ b/oss_src/unity/lib/api/unity_global_interface.hpp @@ -48,7 +48,7 @@ class unity_global_base { typedef std::map global_configuration_type; - GENERATE_INTERFACE_AND_PROXY(unity_global_base, unity_global_proxy, + GENERATE_INTERFACE_AND_PROXY_NO_INLINE_DESTRUCTOR(unity_global_base, unity_global_proxy, (std::vector, list_toolkit_functions, ) (std::vector, list_toolkit_classes, ) (global_configuration_type, describe_toolkit_function, (std::string)) diff --git a/oss_src/unity/lib/api/unity_graph_interface.hpp b/oss_src/unity/lib/api/unity_graph_interface.hpp index b3898a9c..c4bd3735 100644 --- a/oss_src/unity/lib/api/unity_graph_interface.hpp +++ b/oss_src/unity/lib/api/unity_graph_interface.hpp @@ -52,7 +52,7 @@ class unity_sgraph_base { #endif -GENERATE_INTERFACE_AND_PROXY(unity_sgraph_base, unity_graph_proxy, +GENERATE_INTERFACE_AND_PROXY_NO_INLINE_DESTRUCTOR(unity_sgraph_base, unity_graph_proxy, (options_map_t, summary, ) (std::vector, get_vertex_fields, (size_t)) (std::vector, get_edge_fields, (size_t)(size_t)) diff --git a/oss_src/unity/lib/api/unity_sarray_builder_interface.hpp b/oss_src/unity/lib/api/unity_sarray_builder_interface.hpp index ab5e8962..5ddf2e2d 100644 --- a/oss_src/unity/lib/api/unity_sarray_builder_interface.hpp +++ b/oss_src/unity/lib/api/unity_sarray_builder_interface.hpp @@ -15,7 +15,7 @@ namespace graphlab { class unity_sarray_base; -GENERATE_INTERFACE_AND_PROXY(unity_sarray_builder_base, unity_sarray_builder_proxy, +GENERATE_INTERFACE_AND_PROXY_NO_INLINE_DESTRUCTOR(unity_sarray_builder_base, unity_sarray_builder_proxy, (void, init, (size_t)(size_t)(flex_type_enum)) (void, append, (const flexible_type&)(size_t)) (void, append_multiple, (const std::vector&)(size_t)) diff --git a/oss_src/unity/lib/api/unity_sarray_interface.hpp b/oss_src/unity/lib/api/unity_sarray_interface.hpp index 7279b8ca..6aed7efc 100644 --- a/oss_src/unity/lib/api/unity_sarray_interface.hpp +++ b/oss_src/unity/lib/api/unity_sarray_interface.hpp @@ -19,7 +19,7 @@ namespace graphlab { class unity_sframe_base; typedef std::map func_options_map; -GENERATE_INTERFACE_AND_PROXY(unity_sarray_base, unity_sarray_proxy, +GENERATE_INTERFACE_AND_PROXY_NO_INLINE_DESTRUCTOR(unity_sarray_base, unity_sarray_proxy, (void, construct_from_vector, (const std::vector&)(flex_type_enum)) (void, construct_from_const, (const flexible_type&)(size_t)(flex_type_enum)) (void, construct_from_sarray_index, (std::string)) diff --git a/oss_src/unity/lib/api/unity_sframe_builder_interface.hpp b/oss_src/unity/lib/api/unity_sframe_builder_interface.hpp index dca64dea..2ca33e2a 100644 --- a/oss_src/unity/lib/api/unity_sframe_builder_interface.hpp +++ b/oss_src/unity/lib/api/unity_sframe_builder_interface.hpp @@ -9,7 +9,7 @@ namespace graphlab { class unity_sframe_base; -GENERATE_INTERFACE_AND_PROXY(unity_sframe_builder_base, unity_sframe_builder_proxy, +GENERATE_INTERFACE_AND_PROXY_NO_INLINE_DESTRUCTOR(unity_sframe_builder_base, unity_sframe_builder_proxy, (void, init, (size_t)(size_t)(std::vector)(std::vector)(std::string)) (void, append, (const std::vector&)(size_t)) (void, append_multiple, (const std::vector>&)(size_t)) diff --git a/oss_src/unity/lib/api/unity_sframe_interface.hpp b/oss_src/unity/lib/api/unity_sframe_interface.hpp index 9981de91..cc0eeec7 100644 --- a/oss_src/unity/lib/api/unity_sframe_interface.hpp +++ b/oss_src/unity/lib/api/unity_sframe_interface.hpp @@ -24,7 +24,7 @@ typedef std::map csv_parsing_config_map; typedef std::map string_map; typedef std::map> csv_parsing_errors; -GENERATE_INTERFACE_AND_PROXY(unity_sframe_base, unity_sframe_proxy, +GENERATE_INTERFACE_AND_PROXY_NO_INLINE_DESTRUCTOR(unity_sframe_base, unity_sframe_proxy, (void, construct_from_dataframe, (const dataframe_t&)) (void, construct_from_sframe_index, (std::string)) (csv_parsing_errors, construct_from_csvs, (std::string)(csv_parsing_config_map)(str_flex_type_map)) diff --git a/oss_src/unity/lib/api/unity_sketch_interface.hpp b/oss_src/unity/lib/api/unity_sketch_interface.hpp index 7d55cb82..2e5ed4d7 100644 --- a/oss_src/unity/lib/api/unity_sketch_interface.hpp +++ b/oss_src/unity/lib/api/unity_sketch_interface.hpp @@ -22,7 +22,7 @@ class unity_sketch_base; typedef std::pair item_count; typedef std::map> sub_sketch_map; -GENERATE_INTERFACE_AND_PROXY(unity_sketch_base, unity_sketch_proxy, +GENERATE_INTERFACE_AND_PROXY_NO_INLINE_DESTRUCTOR(unity_sketch_base, unity_sketch_proxy, (void, construct_from_sarray, (std::shared_ptr)(bool)(const std::vector&)) (double, get_quantile, (double)) (double, frequency_count, (flexible_type)) From f1551ffcb448f9af2526772ddd44d6f370c9a4b7 Mon Sep 17 00:00:00 2001 From: Yucheng Low Date: Thu, 21 Jul 2016 16:44:00 -0700 Subject: [PATCH 7/7] nanomsg should not retry. --- oss_src/nanosockets/async_request_socket.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/oss_src/nanosockets/async_request_socket.cpp b/oss_src/nanosockets/async_request_socket.cpp index 6ca3d201..492c932c 100644 --- a/oss_src/nanosockets/async_request_socket.cpp +++ b/oss_src/nanosockets/async_request_socket.cpp @@ -108,8 +108,11 @@ int async_request_socket::request_master(zmq_msg_vector& msgs, int async_request_socket::create_socket(size_t i) { if (sockets[i].z_socket == -1) { sockets[i].z_socket = nn_socket(AF_SP, NN_REQ); + int resendintl = 2147483647; + int rc = nn_setsockopt(sockets[i].z_socket, NN_REQ, NN_REQ_RESEND_IVL , &(resendintl), sizeof(resendintl)); + assert(rc == 0); set_conservative_socket_parameters(sockets[i].z_socket); - int rc = nn_connect(sockets[i].z_socket, server.c_str()); + rc = nn_connect(sockets[i].z_socket, server.c_str()); if (rc == -1) { print_zmq_error("Unexpected error on connection"); return rc;