Merge pull request #531 from RedisAI/Create_test_module_for_LLAPI

alonre24 · web-flow · commit e3f75e47b9e3 · 2020-12-07T14:26:42.000+02:00
Create new test module that uses the LLAPI
diff --git a/src/background_workers.c b/src/background_workers.c
@@ -178,7 +178,7 @@ void *RedisAI_Run_ThreadMain(void *arg) {
                         RedisAI_RunInfo *orig = rinfo->orig_copy;
                         long long dagRefCount = RAI_DagRunInfoFreeShallowCopy(rinfo);
                         if (dagRefCount == 0) {
-                            RedisAI_OnFinishCtx finish_ctx = (RedisAI_RunInfo *)orig;
+                            RedisAI_OnFinishCtx *finish_ctx = orig;
                             orig->OnFinish(finish_ctx, orig->private_data);
                         }
 
@@ -417,7 +417,7 @@ void *RedisAI_Run_ThreadMain(void *arg) {
                         RedisAI_RunInfo *orig = rinfo->orig_copy;
                         long long dagRefCount = RAI_DagRunInfoFreeShallowCopy(rinfo);
                         if (dagRefCount == 0) {
-                            RedisAI_OnFinishCtx finish_ctx = (RedisAI_RunInfo *)orig;
+                            RedisAI_OnFinishCtx *finish_ctx = orig;
                             orig->OnFinish(finish_ctx, orig->private_data);
                         }
                     } else {
@@ -449,7 +449,7 @@ void *RedisAI_Run_ThreadMain(void *arg) {
                 // If the reference count for the DAG is zero and the client is still around,
                 // then we actually unblock the client
                 if (dagRefCount == 0) {
-                    RedisAI_OnFinishCtx finish_ctx = (RedisAI_RunInfo *)orig;
+                    RedisAI_OnFinishCtx *finish_ctx = orig;
                     orig->OnFinish(finish_ctx, orig->private_data);
                 }
             }
diff --git a/src/dag.c b/src/dag.c
@@ -34,6 +34,7 @@
 
 #include "model.h"
 #include "redisai.h"
+#include "background_workers.h"
 #include "rmutil/alloc.h"
 #include "rmutil/args.h"
 #include "run_info.h"
@@ -889,7 +890,7 @@ void RedisAI_Disconnected(RedisModuleCtx *ctx, RedisModuleBlockedClient *bc) {
     RedisModule_Log(ctx, "warning", "Blocked client %p disconnected!", (void *)bc);
 }
 
-// Parse the DAG run command and return true if it is a valid command to execute.
+// Parse the DAG run command and return REDISMODULE_OK only if it is a valid command to execute.
 static int DAG_CommandParser(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, int dagMode,
                              RedisAI_RunInfo **rinfo_ptr) {
 
@@ -1229,7 +1230,7 @@ static int DAG_CommandParser(RedisModuleCtx *ctx, RedisModuleString **argv, int
 
 // Add Shallow copies of the DAG run info to the devices' queues.
 // Return REDISMODULE_OK in case of success, REDISMODULE_ERR if (at least) one insert op had failed.
-static bool DAG_InsertDAGToQueue(RedisAI_RunInfo *rinfo) {
+static int DAG_InsertDAGToQueue(RedisAI_RunInfo *rinfo) {
     const char **devices = array_new(const char *, 10);
 
     for (long long i = 0; i < array_len(rinfo->dagOps); i++) {
@@ -1280,7 +1281,7 @@ static bool DAG_InsertDAGToQueue(RedisAI_RunInfo *rinfo) {
             array_free(rinfo_copies);
             array_free(run_queues_info);
             RAI_SetError(rinfo->err, RAI_EDAGRUN, "ERR Queue not initialized for device");
-            rinfo->OnFinish((RedisAI_OnFinishCtx)rinfo, rinfo->private_data);
+            rinfo->OnFinish((RedisAI_OnFinishCtx *)rinfo, rinfo->private_data);
             return REDISMODULE_ERR;
         }
         run_queues_info = array_append(run_queues_info, run_queue_info);
@@ -1302,7 +1303,7 @@ static bool DAG_InsertDAGToQueue(RedisAI_RunInfo *rinfo) {
     return REDISMODULE_OK;
 }
 
-void DAG_ReplyAndUnblock(RedisAI_OnFinishCtx ctx, void *private_data) {
+void DAG_ReplyAndUnblock(RedisAI_OnFinishCtx *ctx, void *private_data) {
 
     RedisAI_RunInfo *rinfo = (RedisAI_RunInfo *)ctx;
     if (rinfo->client)
diff --git a/src/dag.h b/src/dag.h
@@ -175,6 +175,6 @@ int RedisAI_ProcessDagRunCommand(RedisModuleCtx *ctx, RedisModuleString **argv,
  * @param ctx Context object that contains errors and results
  * @param private_data is a pointer to the DAG run info struct
  */
-void DAG_ReplyAndUnblock(RedisAI_OnFinishCtx ctx, void *private_data);
+void DAG_ReplyAndUnblock(RedisAI_OnFinishCtx *ctx, void *private_data);
 
 #endif /* SRC_DAG_H_ */
diff --git a/src/model.c b/src/model.c
@@ -8,6 +8,7 @@
  */
 
 #include "model.h"
+#include "version.h"
 #include "backends.h"
 #include "backends/util.h"
 #include "model_struct.h"
diff --git a/src/redisai.h b/src/redisai.h
@@ -1,14 +1,11 @@
 #ifndef SRC_REDISAI_H_
 #define SRC_REDISAI_H_
 
-#include "background_workers.h"
-#include "model_struct.h"
-#include "redismodule.h"
-#include "util/dict.h"
-#include "version.h"
 #include <stdbool.h>
+#include "redismodule.h"
 
-#define MODULE_API_FUNC(x) (*x)
+#define REDISAI_LLAPI_VERSION 1
+#define MODULE_API_FUNC(x)    (*x)
 
 #ifndef REDISAI_H_INCLUDE
 typedef struct RAI_Tensor RAI_Tensor;
@@ -18,6 +15,8 @@ typedef struct RAI_Script RAI_Script;
 typedef struct RAI_ModelRunCtx RAI_ModelRunCtx;
 typedef struct RAI_ScriptRunCtx RAI_ScriptRunCtx;
 typedef struct RAI_Error RAI_Error;
+typedef struct RAI_ModelOpts RAI_ModelOpts;
+typedef struct RAI_OnFinishCtx RAI_OnFinishCtx;
 #endif
 
 #define REDISAI_BACKEND_TENSORFLOW  0
diff --git a/src/redismodule.h b/src/redismodule.h
@@ -20,7 +20,7 @@ extern "C" {
 /* API versions. */
 #define REDISMODULE_APIVER_1 1
 
-/* Version of the RedisModuleTypeMethods structure. Once the RedisModuleTypeMethods 
+/* Version of the RedisModuleTypeMethods structure. Once the RedisModuleTypeMethods
  * structure is changed, this version number needs to be changed synchronistically. */
 #define REDISMODULE_TYPE_METHOD_VERSION 3
 
diff --git a/src/run_info.h b/src/run_info.h
@@ -63,18 +63,20 @@ int RAI_InitDagOp(RAI_DagOp **result);
  */
 void RAI_FreeDagOp(RAI_DagOp *dagOp);
 
+typedef struct RedisAI_RunInfo RedisAI_RunInfo;
+
 /**
  * This structure contains the context data at the end of the execution.
  * user can access results and errors through LLAPI.
  */
-typedef void *RedisAI_OnFinishCtx;
+typedef RedisAI_RunInfo RedisAI_OnFinishCtx;
 
 /**
  * @brief User defined callback to execute at the end of the run.
  * @param ctx parameter includes the running results and errors.
  * @param private_data is an optional pointer to the user's private data.
  */
-typedef void (*RAI_OnFinishCB)(RedisAI_OnFinishCtx ctx, void *private_data);
+typedef void (*RAI_OnFinishCB)(RedisAI_OnFinishCtx *ctx, void *private_data);
 
 /**
  * This structure represents the context in which RedisAI blocking commands
@@ -84,8 +86,6 @@ typedef void (*RAI_OnFinishCB)(RedisAI_OnFinishCtx ctx, void *private_data);
  * but only the fields needed in a given operation.
  */
 
-typedef struct RedisAI_RunInfo RedisAI_RunInfo;
-
 struct RedisAI_RunInfo {
     RedisModuleBlockedClient *client;
     int single_op_dag;
diff --git a/src/tensor.c b/src/tensor.c
@@ -9,7 +9,9 @@
 
 #include "tensor.h"
 #include "err.h"
+#include "arr.h"
 #include "redisai.h"
+#include "version.h"
 #include "rmutil/alloc.h"
 #include "tensor_struct.h"
 #include "util/dict.h"
diff --git a/tests/flow/tests.sh b/tests/flow/tests.sh
@@ -79,6 +79,7 @@ valgrind_config() {
 #----------------------------------------------------------------------------------------------
 
 run_tests() {
+  make -C ../module
 	local title="$1"
 	[[ ! -z $title ]] && { $ROOT/opt/readies/bin/sep -0; printf "Tests with $title:\n\n"; }
 	cd $ROOT/tests/flow
diff --git a/tests/flow/tests_llapi.py b/tests/flow/tests_llapi.py
@@ -0,0 +1,18 @@
+import redis
+
+from includes import *
+import os
+
+'''
+python -m RLTest --test tests_llapi.py --module path/to/redisai.so
+'''
+goal_dir = os.path.join(os.getcwd(), "../module/LLAPI.so")
+TEST_MODULE_PATH = os.path.abspath(goal_dir)
+
+def test_basic_check(env):
+
+    con = env.getConnection()
+    ret = con.execute_command("MODULE", "LOAD", TEST_MODULE_PATH)
+    env.assertEqual(ret, b'OK')
+    ret = con.execute_command("RAI_llapi.basic_check")
+    env.assertEqual(ret, b'OK')
diff --git a/tests/module/LLAPI.c b/tests/module/LLAPI.c
@@ -0,0 +1,39 @@
+
+#define REDISMODULE_MAIN
+
+#include "../../src/redisai.h"
+#include <errno.h>
+#include <string.h>
+
+int RAI_llapi_basic_check(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+	REDISMODULE_NOT_USED(argv);
+
+	if (argc>1) {
+		RedisModule_WrongArity(ctx);
+		return REDISMODULE_OK;
+	}
+
+	RAI_Error *err;
+	RedisAI_InitError(&err);
+	if(RedisAI_GetErrorCode(err) == RedisAI_ErrorCode_OK)
+		return RedisModule_ReplyWithSimpleString(ctx, "OK");
+	return RedisModule_ReplyWithError(ctx, "ERROR");
+}
+
+int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+	REDISMODULE_NOT_USED(argv);
+	REDISMODULE_NOT_USED(argc);
+
+	if(RedisModule_Init(ctx, "RAI_llapi", 1, REDISMODULE_APIVER_1) ==
+	   REDISMODULE_ERR)
+		return REDISMODULE_ERR;
+
+	if(RedisAI_Initialize(ctx) != REDISMODULE_OK)
+		RedisModule_Log(ctx, "warning",
+		  "could not initialize RedisAI api, running without AI support.");
+
+	if(RedisModule_CreateCommand(ctx, "RAI_llapi.basic_check", RAI_llapi_basic_check, "",
+	  0, 0, 0) == REDISMODULE_ERR)
+		return REDISMODULE_ERR;
+	return REDISMODULE_OK;
+}
diff --git a/tests/module/Makefile b/tests/module/Makefile
@@ -0,0 +1,33 @@
+
+# find the OS
+uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not')
+
+# Compile flags for linux / osx
+ifeq ($(uname_S),Linux)
+	SHOBJ_CFLAGS ?= -W -Wall -fno-common -g -ggdb -std=c99 -O2
+	SHOBJ_LDFLAGS ?= -shared
+else
+	SHOBJ_CFLAGS ?= -W -Wall -dynamic -fno-common -g -ggdb -std=c99 -O2
+	SHOBJ_LDFLAGS ?= -bundle -undefined dynamic_lookup
+endif
+
+TEST_MODULES = LLAPI.so
+
+.PHONY: all
+
+all: $(TEST_MODULES)
+
+32bit:
+	$(MAKE) CFLAGS="-m32" LDFLAGS="-melf_i386"
+
+%.o: %.c
+	$(CC) -I../../src -DREDIS_MODULE_TARGET -DREDISMODULE_EXPERIMENTAL_API $(SHOBJ_CFLAGS) -fPIC -c $< -o $@
+
+%.so: %.o
+	$(CC) -o $@ LLAPI.o $(SHOBJ_LDFLAGS) -lc -lm
+	chmod +x LLAPI.so
+
+.PHONY: clean
+
+clean:
+	rm -f $(TEST_MODULES) $(TEST_MODULES:.so=.xo)

Original file line number	Diff line number	Diff line change
`@@ -178,7 +178,7 @@ void RedisAI_Run_ThreadMain(void arg) {`
`178`	`178`	`RedisAI_RunInfo *orig = rinfo->orig_copy;`
`179`	`179`	`long long dagRefCount = RAI_DagRunInfoFreeShallowCopy(rinfo);`
`180`	`180`	`if (dagRefCount == 0) {`
`181`		`- RedisAI_OnFinishCtx finish_ctx = (RedisAI_RunInfo *)orig;`
	`181`	`+ RedisAI_OnFinishCtx *finish_ctx = orig;`
`182`	`182`	`orig->OnFinish(finish_ctx, orig->private_data);`
`183`	`183`	`}`
`184`	`184`
`@@ -417,7 +417,7 @@ void RedisAI_Run_ThreadMain(void arg) {`
`417`	`417`	`RedisAI_RunInfo *orig = rinfo->orig_copy;`
`418`	`418`	`long long dagRefCount = RAI_DagRunInfoFreeShallowCopy(rinfo);`
`419`	`419`	`if (dagRefCount == 0) {`
`420`		`- RedisAI_OnFinishCtx finish_ctx = (RedisAI_RunInfo *)orig;`
	`420`	`+ RedisAI_OnFinishCtx *finish_ctx = orig;`
`421`	`421`	`orig->OnFinish(finish_ctx, orig->private_data);`
`422`	`422`	`}`
`423`	`423`	`} else {`
`@@ -449,7 +449,7 @@ void RedisAI_Run_ThreadMain(void arg) {`
`449`	`449`	`// If the reference count for the DAG is zero and the client is still around,`
`450`	`450`	`// then we actually unblock the client`
`451`	`451`	`if (dagRefCount == 0) {`
`452`		`- RedisAI_OnFinishCtx finish_ctx = (RedisAI_RunInfo *)orig;`
	`452`	`+ RedisAI_OnFinishCtx *finish_ctx = orig;`
`453`	`453`	`orig->OnFinish(finish_ctx, orig->private_data);`
`454`	`454`	`}`
`455`	`455`	`}`