Documenting string and bool tensor types + validate bool blob (#842)

Add documentation for BOOL and STRING tensor + add overflow verification for bool tensor blob (+ test it)

Author: alonre24

docs/commands.md

@@ -50,11 +50,38 @@ OK
 ```
 
 !!! note "Uninitialized Tensor Values"
-    As both `BLOB` and `VALUES` are optional arguments, it is possible to use the `AI.TENSORSET` to create an uninitialized tensor.
+    As both `BLOB` and `VALUES` are optional arguments, it is possible to use the `AI.TENSORSET` to create an uninitialized tensor (it will contain zeros at all entries).
 
 !!! important "Using `BLOB` is preferable to `VALUES`"
     While it is possible to set the tensor using binary data or numerical values, it is recommended that you use the `BLOB` option. It requires fewer resources and performs better compared to specifying the values discretely.
 
+###Boolean Tensors
+The possible values for a tensor of type `BOOL` are `0` and `1`. The size of every bool element in a blob should be 1 byte.   
+
+**Examples**
+
+Here are two ways of creating the following boolean tensor: $\begin{equation*} A = \begin{bmatrix} 0 & 1 \\ 0 & 1 \\ \end{bmatrix} \end{equation*}$
+```
+redis> AI.TENSORSET my_bool_tensor BOOL 2 2 VALUES 0 1 0 1
+OK
+redis> AI.TENSORSET my_bool_tensor BOOL 2 2 BLOB "\x00\x01\x00\x01"
+OK
+```
+
+###String Tensors
+String tensors are tensors in which every element is a single utf-8 string (may or may not be null-terminated). A string element can be at any length, and it cannot contain another null character except for the last one if it is a null-terminated string.
+A string tensor blob contains the encoded string elements concatenated, where the null character serves as a delimiter. Note that the size of string tensor blob equals to the total size of its elements, and it is not determined given the tensor's shapes (unlike in the rest of tensor types) 
+
+**Examples**
+
+Here are two ways of creating the same 2X2 string tensor:
+```
+redis> AI.TENSORSET my_str_tensor STRING 2 2 VALUES first second third fourth
+OK
+redis> AI.TENSORSET my_bool_tensor STRING 2 2 BLOB "first\x00second\x00third\x00fourth\x00"
+OK
+```
+
 ## AI.TENSORGET
 The **`AI.TENSORGET`** command returns a tensor stored as key's value.
 
@@ -81,51 +108,69 @@ Depending on the specified reply format:
     1. The tensor's shape as an Array consisting of an item per dimension
  * **BLOB**: the tensor's binary data as a String. If used together with the **META** option, the binary data string will put after the metadata in the array reply.
  * **VALUES**: Array containing the numerical representation of the tensor's data. If used together with the **META** option, the binary data string will put after the metadata in the array reply.
-* Default: **META** and **BLOB** are returned by default, in case that non of the arguments above is specified. 
+* Default: **META** and **BLOB** are returned by default, in case that none of the arguments above is specified. 
 
 
 **Examples**
 
-Given a tensor value stored at the 'mytensor' key:
+Given tensor values stored at 'my_tensor' and _my_str_tensor keys:
 
 ```
-redis> AI.TENSORSET mytensor FLOAT 2 2 VALUES 1 2 3 4
+redis> AI.TENSORSET my_tensor FLOAT 2 2 VALUES 1 2 3 4
+OK
+redis> AI.TENSORSET my_str_tensor STRING 2 2 VALUES first second third fourth
 OK
 ```
 
-The following shows how to retrieve the tensor's metadata:
+The following shows how to retrieve the tensors' metadata:
 
 ```
-redis> AI.TENSORGET mytensor META
+redis> AI.TENSORGET my_tensor META
 1) "dtype"
 2) "FLOAT"
 3) "shape"
+4) 1) (integer) 2
+   2) (integer) 2
+
+redis> AI.TENSORGET my_str_tensor META
+1) "dtype"
+2) "STRING"
+3) "shape"
 4) 1) (integer) 2
    2) (integer) 2
 ```
 
-The following shows how to retrieve the tensor's values as an Array:
+The following shows how to retrieve the tensors' values as an Array:
 
 ```
-redis> AI.TENSORGET mytensor VALUES
+redis> AI.TENSORGET my_tensor VALUES
 1) "1"
 2) "2"
 3) "3"
 4) "4"
+
+redis> AI.TENSORGET my_str_tensor VALUES
+1) "first"
+2) "second"
+3) "third"
+4) "fourth"
 ```
 
-The following shows how to retrieve the tensor's binary data as a String:
+The following shows how to retrieve the tensors' binary data as a String:
 
 ```
-redis> AI.TENSORGET mytensor BLOB
+redis> AI.TENSORGET my_tensor BLOB
 "\x00\x00\x80?\x00\x00\x00@\x00\x00@@\x00\x00\x80@"
+
+redis> AI.TENSORGET my_str_tensor BLOB
+"first\x00second\x00third\x00fourth\x00"
 ```
 
 
-The following shows how the combine the retrieval of the tensor's metadata, and the tensor's values as an Array:
+The following shows how the combine the retrieval of the tensors' metadata, and the tensors' values as an Array:
 
 ```
-redis> AI.TENSORGET mytensor META VALUES
+redis> AI.TENSORGET my_tensor META VALUES
 1) "dtype"
 2) "FLOAT"
 3) "shape"
@@ -136,19 +181,40 @@ redis> AI.TENSORGET mytensor META VALUES
    2) "2"
    3) "3"
    4) "4"
+
+redis> AI.TENSORGET my_str_tensor META VALUES
+1) "dtype"
+2) "STRING"
+3) "shape"
+4) 1) (integer) 2
+   2) (integer) 2
+5) "values"
+6) 1) "first"
+   2) "second"
+   3) "third"
+   4) "fourth"
 ```
 
-The following shows how the combine the retrieval of the tensor's metadata, and binary data as a String:
+The following shows how the combine the retrieval of the tensors' metadata, and binary data as a String:
 
 ```
-redis> AI.TENSORGET mytensor META BLOB
+redis> AI.TENSORGET my_tensor META BLOB
 1) "dtype"
 2) "FLOAT"
 3) "shape"
 4) 1) (integer) 2
    2) (integer) 2
 5) "blob"
 6) "\x00\x00\x80?\x00\x00\x00@\x00\x00@@\x00\x00\x80@"
+
+redis> AI.TENSORGET my_str_tensor META BLOB
+1) "dtype"
+2) "STRING"
+3) "shape"
+4) 1) (integer) 2
+   2) (integer) 2
+5) "blob"
+6) "first\x00second\x00third\x00fourth\x00"
 ```
 
 !!! important "Using `BLOB` is preferable to `VALUES`"

src/redis_ai_objects/tensor.c

@@ -122,6 +122,21 @@ static int _RAI_TensorFillWithValues(int argc, RedisModuleString **argv, RAI_Ten
     return REDISMODULE_OK;
 }
 
+static int _RAI_TensorParseBooleansBlob(const char *tensor_blob, size_t blob_len, size_t tensor_len,
+                                        RAI_Error *err) {
+
+    // if we encounter a non-boolean value - return an error
+    for (size_t i = 0; i < blob_len - 1; i++) {
+        if (tensor_blob[i] != 0 && tensor_blob[i] != 1) {
+            if (err) {
+                RAI_SetError(err, RAI_ETENSORSET, "ERR BOOL tensor elements must be 0 or 1");
+            }
+            return REDISMODULE_ERR;
+        }
+    }
+    return REDISMODULE_OK;
+}
+
 // This will populate the offsets array with the start position of every string element in the blob
 static int _RAI_TensorParseStringsBlob(const char *tensor_blob, size_t blob_len, size_t tensor_len,
                                        uint64_t *offsets, RAI_Error *err) {
@@ -300,6 +315,13 @@ RAI_Tensor *RAI_TensorCreateFromBlob(DLDataType data_type, const size_t *dims, i
                          "ERR data length does not match tensor shape and type");
             return NULL;
         }
+        if (data_type.code == kDLBool) {
+            if (_RAI_TensorParseBooleansBlob(tensor_blob, blob_len, tensor_len, err) !=
+                REDISMODULE_OK) {
+                RAI_TensorFree(new_tensor);
+                return NULL;
+            }
+        }
     }
 
     // Copy the blob. We must copy instead of increasing the ref count since we don't have
@@ -656,6 +678,10 @@ int RAI_TensorSetData(RAI_Tensor *t, const char *data, size_t len) {
         }
         RedisModule_Free(RAI_TensorData(t));
         t->tensor.dl_tensor.data = RedisModule_Alloc(len);
+    } else if (data_type.code == kDLBool) {
+        if (_RAI_TensorParseBooleansBlob(data, len, RAI_TensorLength(t), NULL) != REDISMODULE_OK) {
+            return 0;
+        }
     }
     memcpy(RAI_TensorData(t), data, len);
     t->blobSize = len;

tests/flow/tests_common.py

@@ -119,6 +119,10 @@ def test_common_tensorset_error_replies(env):
     check_error_message(env, con, "Number of string elements in data blob does not match tensor length",
                         'AI.TENSORSET', 'z{0}', 'STRING', 2, 'BLOB', 'C-string\0followed by a non C-string')
 
+    # ERR in bool tensor blob - element is not 0/1
+    check_error_message(env, con, "BOOL tensor elements must be 0 or 1",
+                        'AI.TENSORSET', 'z{0}', 'BOOL', 2, 'BLOB', "\x02\x01")
+
 
 def test_common_tensorget(env):
     con = get_connection(env, '{0}')

tests/module/LLAPI.c

@@ -313,13 +313,21 @@ int RAI_llapi_CreateTensor(RedisModuleCtx *ctx, RedisModuleString **argv, int ar
 
     // create an empty tensor and validate that in contains zeros
     t = RedisAI_TensorCreate("INT8", dims, n_dims);
-    int8_t expected_blob[8] = {0};
+    int8_t expected_blob[4] = {0};
     if (t == NULL || RedisAI_TensorLength(t) != dims[0] * dims[1] ||
         memcmp(RedisAI_TensorData(t), expected_blob, 4) != 0) {
         return RedisModule_ReplyWithSimpleString(ctx, "empty tensor create test failed");
     }
     RedisAI_TensorFree(t);
 
+    // create an invalid bool tensor
+    t = RedisAI_TensorCreate("BOOL", dims, n_dims);
+    uint8_t data_blob[4] = {2, 0, 0, 0}; // This value is invalid for bool type
+    if (RedisAI_TensorSetData(t, (const char *)data_blob, 4) != 0) {
+        return RedisModule_ReplyWithSimpleString(ctx, "invalid bool tensor data set test failed");
+    }
+    RedisAI_TensorFree(t);
+
     // This should fail since the blob contains only one null-terminated string, while the tensor's
     // len should be 4.
     RAI_Tensor *t1 = RedisAI_TensorCreate("STRING", dims, n_dims);