Merge branch 'master' into torchscript_extensions

DvirDukhan · web-flow · commit 13dfd20bd49e · 2020-12-14T14:16:26.000+02:00
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -331,7 +331,7 @@ platform-build-defs: &platform-build-defs
 
 after-platform-builds: &after-platform-builds
   requires:
-    - build-centos7
+    # - build-centos7
     - build-bionic
     - build-xenial
 
@@ -358,7 +358,7 @@ workflows:
       - platform-build:
           name: build-xenial
           platform: xenial
-          <<: *on-any-branch
+          <<: *platform-build-defs
       - coverage:
           <<: *on-any-branch
           <<: *after-linter
diff --git a/opt/readies b/opt/readies
@@ -1 +1 @@
-Subproject commit 0f048e34a03cc860313e827dc5a31afe8d9918fd
+Subproject commit b6d26cf551f0f648cc1b04fa60daa4959cfbc796
diff --git a/opt/system-setup.py b/opt/system-setup.py
@@ -5,8 +5,9 @@
 import argparse
 
 HERE = os.path.abspath(os.path.dirname(__file__))
+READIES = os.path.join(HERE, "readies")
 ROOT = os.path.abspath(os.path.join(HERE, ".."))
-sys.path.insert(0, os.path.join(HERE, "readies"))
+sys.path.insert(0, READIES)
 import paella
 
 #----------------------------------------------------------------------------------------------
@@ -19,9 +20,6 @@ def common_first(self):
         self.install_downloaders()
         self.setup_pip()
         self.pip_install("wheel virtualenv")
-        # if self.osnick == 'xenial': 
-        #     self.pip_install("setuptools --upgrade")
-        #     self.pip_install("-IU --force-reinstall setuptools")
 
         if self.os == 'linux':
             self.install("ca-certificates")
@@ -32,17 +30,6 @@ def debian_compat(self):
         self.install("gawk")
         self.install("build-essential")
         self.install("libssl-dev")
-        self.run("""
-        apt remove -y --purge --auto-remove cmake
-        version=3.19
-        build=0
-        mkdir ~/temp
-        cd ~/temp
-        wget https://cmake.org/files/v$version/cmake-$version.$build-Linux-x86_64.sh 
-        mkdir /opt/cmake
-        sh cmake-$version.$build-Linux-x86_64.sh --prefix=/opt/cmake --skip-license
-        ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake
-        """)
         self.install("clang-format")
         self.install("python3-regex")
         self.install("python3-psutil python3-networkx python3-numpy") # python3-skimage
@@ -53,22 +40,11 @@ def redhat_compat(self):
         self.run("%s/readies/bin/enable-utf8" % HERE)
 
         self.group_install("'Development Tools'")
-
-        self.install("centos-release-scl")
-        self.install("devtoolset-8")
-        self.run("cp /opt/rh/devtoolset-8/enable /etc/profile.d/scl-devtoolset-8.sh")
-
+        self.run("%s/bin/getgcc --modern" % READIES)
         self.install("llvm-toolset-7")
 
-        paella.mkdir_p("%s/profile.d" % ROOT)
-        self.run("cp /opt/rh/devtoolset-8/enable %s/profile.d/scl-devtoolset-8.sh" % ROOT)
-
-        self.run("""
-            dir=$(mktemp -d /tmp/tar.XXXXXX)
-            (cd $dir; wget -q -O tar.tgz http://redismodules.s3.amazonaws.com/gnu/gnu-tar-1.32-x64-centos7.tgz; tar -xzf tar.tgz -C /; )
-            rm -rf $dir
-            """)
-
+        if self.arch == 'x64':
+            self.install_linux_gnu_tar()
 
         if not self.dist == "amzn":
             self.install("epel-release")
@@ -79,30 +55,23 @@ def redhat_compat(self):
             self.install("python3-devel")
             self.pip_install("psutil")
 
-        self.install("cmake3")
-        self.run("ln -sf `command -v cmake3` /usr/local/bin/cmake")
-
         self.install_git_lfs_on_linux()
 
     def fedora(self):
         self.group_install("'Development Tools'")
-        self.install("cmake")
-        self.run("ln -sf `command -v cmake3` /usr/local/bin/cmake")
         self.install("python3 python3-psutil python3-networkx")
         self.install("clang")
         self.install_git_lfs_on_linux()
 
     def macos(self):
-        if sh('xcode-select -p') == '':
-            fatal("Xcode tools are not installed. Please run xcode-select --install.")
-
         self.install_gnu_utils()
-        self.install("cmake")
         self.install("git-lfs")
         self.install("redis")
         self.install("clang-format")
 
     def common_last(self):
+        self.run("%s/bin/getcmake" % READIES)
+
         self.run("python3 -m pip uninstall -y ramp-packer RLTest || true")
         # redis-py-cluster should be installed from git due to redis-py dependency
         self.pip_install("--no-cache-dir git+https://github.com/Grokzen/redis-py-cluster.git@master")
diff --git a/src/background_workers.c b/src/background_workers.c
@@ -178,7 +178,7 @@ void *RedisAI_Run_ThreadMain(void *arg) {
                         RedisAI_RunInfo *orig = rinfo->orig_copy;
                         long long dagRefCount = RAI_DagRunInfoFreeShallowCopy(rinfo);
                         if (dagRefCount == 0) {
-                            RedisAI_OnFinishCtx finish_ctx = (RedisAI_RunInfo *)orig;
+                            RedisAI_OnFinishCtx *finish_ctx = orig;
                             orig->OnFinish(finish_ctx, orig->private_data);
                         }
 
@@ -417,7 +417,7 @@ void *RedisAI_Run_ThreadMain(void *arg) {
                         RedisAI_RunInfo *orig = rinfo->orig_copy;
                         long long dagRefCount = RAI_DagRunInfoFreeShallowCopy(rinfo);
                         if (dagRefCount == 0) {
-                            RedisAI_OnFinishCtx finish_ctx = (RedisAI_RunInfo *)orig;
+                            RedisAI_OnFinishCtx *finish_ctx = orig;
                             orig->OnFinish(finish_ctx, orig->private_data);
                         }
                     } else {
@@ -449,7 +449,7 @@ void *RedisAI_Run_ThreadMain(void *arg) {
                 // If the reference count for the DAG is zero and the client is still around,
                 // then we actually unblock the client
                 if (dagRefCount == 0) {
-                    RedisAI_OnFinishCtx finish_ctx = (RedisAI_RunInfo *)orig;
+                    RedisAI_OnFinishCtx *finish_ctx = orig;
                     orig->OnFinish(finish_ctx, orig->private_data);
                 }
             }
diff --git a/src/dag.c b/src/dag.c
@@ -34,6 +34,7 @@
 
 #include "model.h"
 #include "redisai.h"
+#include "background_workers.h"
 #include "rmutil/alloc.h"
 #include "rmutil/args.h"
 #include "run_info.h"
@@ -889,7 +890,7 @@ void RedisAI_Disconnected(RedisModuleCtx *ctx, RedisModuleBlockedClient *bc) {
     RedisModule_Log(ctx, "warning", "Blocked client %p disconnected!", (void *)bc);
 }
 
-// Parse the DAG run command and return true if it is a valid command to execute.
+// Parse the DAG run command and return REDISMODULE_OK only if it is a valid command to execute.
 static int DAG_CommandParser(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, int dagMode,
                              RedisAI_RunInfo **rinfo_ptr) {
 
@@ -1229,7 +1230,7 @@ static int DAG_CommandParser(RedisModuleCtx *ctx, RedisModuleString **argv, int
 
 // Add Shallow copies of the DAG run info to the devices' queues.
 // Return REDISMODULE_OK in case of success, REDISMODULE_ERR if (at least) one insert op had failed.
-static bool DAG_InsertDAGToQueue(RedisAI_RunInfo *rinfo) {
+static int DAG_InsertDAGToQueue(RedisAI_RunInfo *rinfo) {
     const char **devices = array_new(const char *, 10);
 
     for (long long i = 0; i < array_len(rinfo->dagOps); i++) {
@@ -1280,7 +1281,7 @@ static bool DAG_InsertDAGToQueue(RedisAI_RunInfo *rinfo) {
             array_free(rinfo_copies);
             array_free(run_queues_info);
             RAI_SetError(rinfo->err, RAI_EDAGRUN, "ERR Queue not initialized for device");
-            rinfo->OnFinish((RedisAI_OnFinishCtx)rinfo, rinfo->private_data);
+            rinfo->OnFinish((RedisAI_OnFinishCtx *)rinfo, rinfo->private_data);
             return REDISMODULE_ERR;
         }
         run_queues_info = array_append(run_queues_info, run_queue_info);
@@ -1302,7 +1303,7 @@ static bool DAG_InsertDAGToQueue(RedisAI_RunInfo *rinfo) {
     return REDISMODULE_OK;
 }
 
-void DAG_ReplyAndUnblock(RedisAI_OnFinishCtx ctx, void *private_data) {
+void DAG_ReplyAndUnblock(RedisAI_OnFinishCtx *ctx, void *private_data) {
 
     RedisAI_RunInfo *rinfo = (RedisAI_RunInfo *)ctx;
     if (rinfo->client)
diff --git a/src/dag.h b/src/dag.h
@@ -175,6 +175,6 @@ int RedisAI_ProcessDagRunCommand(RedisModuleCtx *ctx, RedisModuleString **argv,
  * @param ctx Context object that contains errors and results
  * @param private_data is a pointer to the DAG run info struct
  */
-void DAG_ReplyAndUnblock(RedisAI_OnFinishCtx ctx, void *private_data);
+void DAG_ReplyAndUnblock(RedisAI_OnFinishCtx *ctx, void *private_data);
 
 #endif /* SRC_DAG_H_ */
diff --git a/src/model.c b/src/model.c
@@ -8,6 +8,7 @@
  */
 
 #include "model.h"
+#include "version.h"
 #include "backends.h"
 #include "backends/util.h"
 #include "model_struct.h"
diff --git a/src/redisai.c b/src/redisai.c
@@ -1,3 +1,8 @@
+
+#ifdef __linux__
+#define _GNU_SOURCE
+#endif
+
 #define REDISMODULE_MAIN
 #include "redismodule.h"
 #include "tensor.h"
@@ -32,6 +37,12 @@
 #define REDISAI_GIT_SHA "unknown"
 #endif
 
+#ifdef __linux__
+#ifndef RUSAGE_THREAD
+#define RUSAGE_THREAD 1
+#endif
+#endif
+
 int redisMajorVersion;
 int redisMinorVersion;
 int redisPatchVersion;
@@ -986,10 +997,31 @@ void RAI_moduleInfoFunc(RedisModuleInfoCtx *ctx, int for_crash_report) {
     RedisModule_InfoAddFieldLongLong(ctx, "inter_op_parallelism", getBackendsInterOpParallelism());
     RedisModule_InfoAddFieldLongLong(ctx, "intra_op_parallelism", getBackendsIntraOpParallelism());
     struct rusage self_ru, c_ru;
+
     // Return resource usage statistics for the calling process,
     // which is the sum of resources used by all threads in the
     // process
     getrusage(RUSAGE_SELF, &self_ru);
+
+    // Return resource usage statistics for the calling thread
+    // which in this case is Redis/RedisAI main thread
+    // RUSAGE_THREAD is Linux-specific.
+    sds main_thread_used_cpu_sys = sdsempty();
+    sds main_thread_used_cpu_user = sdsempty();
+#if (defined(__linux__) && defined(RUSAGE_THREAD))
+    struct rusage main_thread_ru;
+    getrusage(RUSAGE_THREAD, &main_thread_ru);
+    main_thread_used_cpu_sys =
+        sdscatprintf(main_thread_used_cpu_sys, "%ld.%06ld", (long)main_thread_ru.ru_stime.tv_sec,
+                     (long)self_ru.ru_stime.tv_usec);
+    main_thread_used_cpu_user =
+        sdscatprintf(main_thread_used_cpu_user, "%ld.%06ld", (long)main_thread_ru.ru_utime.tv_sec,
+                     (long)self_ru.ru_utime.tv_usec);
+#else
+    sdscatprintf(main_thread_used_cpu_sys, "N/A");
+    sdscatprintf(main_thread_used_cpu_user, "N/A");
+#endif
+
     // Return resource usage statistics for all of its
     // terminated child processes
     getrusage(RUSAGE_CHILDREN, &c_ru);
@@ -1006,6 +1038,8 @@ void RAI_moduleInfoFunc(RedisModuleInfoCtx *ctx, int for_crash_report) {
     RedisModule_InfoAddFieldCString(ctx, "self_used_cpu_user", self_used_cpu_user);
     RedisModule_InfoAddFieldCString(ctx, "children_used_cpu_sys", children_used_cpu_sys);
     RedisModule_InfoAddFieldCString(ctx, "children_used_cpu_user", children_used_cpu_user);
+    RedisModule_InfoAddFieldCString(ctx, "main_thread_used_cpu_sys", main_thread_used_cpu_sys);
+    RedisModule_InfoAddFieldCString(ctx, "main_thread_used_cpu_user", main_thread_used_cpu_user);
 
     AI_dictIterator *iter = AI_dictGetSafeIterator(run_queues);
     AI_dictEntry *entry = AI_dictNext(iter);
@@ -1018,7 +1052,7 @@ void RAI_moduleInfoFunc(RedisModuleInfoCtx *ctx, int for_crash_report) {
                 struct timespec ts;
                 clockid_t cid;
                 sds queue_used_cpu_total = sdscatprintf(
-                    sdsempty(), "queue_%s_bthread_#%d_used_cpu_total", queue_name, i + 1);
+                    sdsempty(), "queue_%s_bthread_n%d_used_cpu_total", queue_name, i + 1);
                 sds bthread_used_cpu_total = sdsempty();
 #if (!defined(_POSIX_C_SOURCE) && !defined(_XOPEN_SOURCE)) || defined(_DARWIN_C_SOURCE) ||         \
     defined(__cplusplus)
@@ -1034,7 +1068,7 @@ void RAI_moduleInfoFunc(RedisModuleInfoCtx *ctx, int for_crash_report) {
                     } else {
                         bthread_used_cpu_total =
                             sdscatprintf(bthread_used_cpu_total, "%ld.%06ld", (long)ts.tv_sec,
-                                         (long)(ts.tv_nsec / 1000000));
+                                         (long)(ts.tv_nsec / 1000));
                     }
                 }
                 RedisModule_InfoAddFieldCString(ctx, queue_used_cpu_total, bthread_used_cpu_total);
diff --git a/src/redisai.h b/src/redisai.h
@@ -1,14 +1,11 @@
 #ifndef SRC_REDISAI_H_
 #define SRC_REDISAI_H_
 
-#include "background_workers.h"
-#include "model_struct.h"
-#include "redismodule.h"
-#include "util/dict.h"
-#include "version.h"
 #include <stdbool.h>
+#include "redismodule.h"
 
-#define MODULE_API_FUNC(x) (*x)
+#define REDISAI_LLAPI_VERSION 1
+#define MODULE_API_FUNC(x)    (*x)
 
 #ifndef REDISAI_H_INCLUDE
 typedef struct RAI_Tensor RAI_Tensor;
@@ -18,6 +15,8 @@ typedef struct RAI_Script RAI_Script;
 typedef struct RAI_ModelRunCtx RAI_ModelRunCtx;
 typedef struct RAI_ScriptRunCtx RAI_ScriptRunCtx;
 typedef struct RAI_Error RAI_Error;
+typedef struct RAI_ModelOpts RAI_ModelOpts;
+typedef struct RAI_OnFinishCtx RAI_OnFinishCtx;
 #endif
 
 #define REDISAI_BACKEND_TENSORFLOW  0
diff --git a/src/redismodule.h b/src/redismodule.h
@@ -20,7 +20,7 @@ extern "C" {
 /* API versions. */
 #define REDISMODULE_APIVER_1 1
 
-/* Version of the RedisModuleTypeMethods structure. Once the RedisModuleTypeMethods 
+/* Version of the RedisModuleTypeMethods structure. Once the RedisModuleTypeMethods
  * structure is changed, this version number needs to be changed synchronistically. */
 #define REDISMODULE_TYPE_METHOD_VERSION 3
 
diff --git a/src/run_info.h b/src/run_info.h
@@ -63,18 +63,20 @@ int RAI_InitDagOp(RAI_DagOp **result);
  */
 void RAI_FreeDagOp(RAI_DagOp *dagOp);
 
+typedef struct RedisAI_RunInfo RedisAI_RunInfo;
+
 /**
  * This structure contains the context data at the end of the execution.
  * user can access results and errors through LLAPI.
  */
-typedef void *RedisAI_OnFinishCtx;
+typedef RedisAI_RunInfo RedisAI_OnFinishCtx;
 
 /**
  * @brief User defined callback to execute at the end of the run.
  * @param ctx parameter includes the running results and errors.
  * @param private_data is an optional pointer to the user's private data.
  */
-typedef void (*RAI_OnFinishCB)(RedisAI_OnFinishCtx ctx, void *private_data);
+typedef void (*RAI_OnFinishCB)(RedisAI_OnFinishCtx *ctx, void *private_data);
 
 /**
  * This structure represents the context in which RedisAI blocking commands
@@ -84,8 +86,6 @@ typedef void (*RAI_OnFinishCB)(RedisAI_OnFinishCtx ctx, void *private_data);
  * but only the fields needed in a given operation.
  */
 
-typedef struct RedisAI_RunInfo RedisAI_RunInfo;
-
 struct RedisAI_RunInfo {
     RedisModuleBlockedClient *client;
     int single_op_dag;
diff --git a/src/tensor.c b/src/tensor.c
@@ -9,7 +9,9 @@
 
 #include "tensor.h"
 #include "err.h"
+#include "arr.h"
 #include "redisai.h"
+#include "version.h"
 #include "rmutil/alloc.h"
 #include "tensor_struct.h"
 #include "util/dict.h"
diff --git a/tests/flow/tests.sh b/tests/flow/tests.sh
@@ -79,6 +79,7 @@ valgrind_config() {
 #----------------------------------------------------------------------------------------------
 
 run_tests() {
+  make -C ../module
 	local title="$1"
 	[[ ! -z $title ]] && { $ROOT/opt/readies/bin/sep -0; printf "Tests with $title:\n\n"; }
 	cd $ROOT/tests/flow
diff --git a/tests/flow/tests_common.py b/tests/flow/tests_common.py
@@ -301,7 +301,7 @@ def test_info_modules(env):
     env.assertEqual( 'ai_self_used_cpu_user' in ret, True )
     env.assertEqual( 'ai_children_used_cpu_sys' in ret, True )
     env.assertEqual( 'ai_children_used_cpu_user' in ret, True )
-    env.assertEqual( 'ai_queue_CPU_bthread_#1_used_cpu_total' in ret, True )
+    env.assertEqual( 'ai_queue_CPU_bthread_n1_used_cpu_total' in ret, True )
 
 def test_lua_multi(env):
     con = env.getConnection()
diff --git a/tests/flow/tests_llapi.py b/tests/flow/tests_llapi.py
diff --git a/tests/module/LLAPI.c b/tests/module/LLAPI.c
diff --git a/tests/module/Makefile b/tests/module/Makefile

Original file line number	Diff line number	Diff line change
`@@ -178,7 +178,7 @@ void RedisAI_Run_ThreadMain(void arg) {`
`178`	`178`	`RedisAI_RunInfo *orig = rinfo->orig_copy;`
`179`	`179`	`long long dagRefCount = RAI_DagRunInfoFreeShallowCopy(rinfo);`
`180`	`180`	`if (dagRefCount == 0) {`
`181`		`- RedisAI_OnFinishCtx finish_ctx = (RedisAI_RunInfo *)orig;`
	`181`	`+ RedisAI_OnFinishCtx *finish_ctx = orig;`
`182`	`182`	`orig->OnFinish(finish_ctx, orig->private_data);`
`183`	`183`	`}`
`184`	`184`
`@@ -417,7 +417,7 @@ void RedisAI_Run_ThreadMain(void arg) {`
`417`	`417`	`RedisAI_RunInfo *orig = rinfo->orig_copy;`
`418`	`418`	`long long dagRefCount = RAI_DagRunInfoFreeShallowCopy(rinfo);`
`419`	`419`	`if (dagRefCount == 0) {`
`420`		`- RedisAI_OnFinishCtx finish_ctx = (RedisAI_RunInfo *)orig;`
	`420`	`+ RedisAI_OnFinishCtx *finish_ctx = orig;`
`421`	`421`	`orig->OnFinish(finish_ctx, orig->private_data);`
`422`	`422`	`}`
`423`	`423`	`} else {`
`@@ -449,7 +449,7 @@ void RedisAI_Run_ThreadMain(void arg) {`
`449`	`449`	`// If the reference count for the DAG is zero and the client is still around,`
`450`	`450`	`// then we actually unblock the client`
`451`	`451`	`if (dagRefCount == 0) {`
`452`		`- RedisAI_OnFinishCtx finish_ctx = (RedisAI_RunInfo *)orig;`
	`452`	`+ RedisAI_OnFinishCtx *finish_ctx = orig;`
`453`	`453`	`orig->OnFinish(finish_ctx, orig->private_data);`
`454`	`454`	`}`
`455`	`455`	`}`