Merge branch 'master' into backends_upgrade

Author: DvirDukhan

.circleci/config.yml

@@ -17,7 +17,7 @@ commands:
       - run:
           name: Setup automation
           command: |
-            PIP=1 ./opt/readies/bin/getpy3
+            ./opt/readies/bin/getpy3
             ./opt/system-setup.py
 
   build-steps:
@@ -125,6 +125,7 @@ jobs:
       - run:
           name: Submodule checkout
           command: git submodule update --init --recursive
+      - setup-automation
       - run:
           name: lint
           command: |
@@ -201,7 +202,63 @@ jobs:
       - run:
           name: Test with valgrind
           command: |
-            make -C opt test VALGRIND=1
+            make -C opt test VALGRIND=1 CLUSTER=0 AOF=0
+          no_output_timeout: 120m
+
+  valgrind-cluster:
+    docker:
+      - image: redisfab/rmbuilder:6.0.5-x64-buster
+    steps:
+      - checkout
+      - run:
+          name: Submodule checkout
+          command: git submodule update --init --recursive
+      - restore_cache:
+          keys:
+            - build-dependencies-{{ checksum "get_deps.sh" }}
+          # If no exact match is found will get dependencies from source
+      - setup-automation
+      - run:
+          name: Install dependencies
+          command: |
+            ./opt/readies/bin/getredis -v 6 --valgrind --force
+            ./get_deps.sh cpu
+      - run:
+          name: Build for valgrind with cluster
+          command: |
+            make -C opt all VALGRIND=1 SHOW=1
+      - run:
+          name: Test with valgrind and cluster
+          command: |
+            make -C opt test VALGRIND=1 GEN=0 AOF=0
+          no_output_timeout: 120m
+
+  valgrind-AOF:
+    docker:
+      - image: redisfab/rmbuilder:6.0.5-x64-buster
+    steps:
+      - checkout
+      - run:
+          name: Submodule checkout
+          command: git submodule update --init --recursive
+      - restore_cache:
+          keys:
+            - build-dependencies-{{ checksum "get_deps.sh" }}
+          # If no exact match is found will get dependencies from source
+      - setup-automation
+      - run:
+          name: Install dependencies
+          command: |
+            ./opt/readies/bin/getredis -v 6 --valgrind --force
+            ./get_deps.sh cpu
+      - run:
+          name: Build for valgrind with AOF
+          command: |
+            make -C opt all VALGRIND=1 SHOW=1
+      - run:
+          name: Test with valgrind and AOF
+          command: |
+            make -C opt test VALGRIND=1 GEN=0 CLUSTER=0
           no_output_timeout: 120m
 
   build-macos:
@@ -406,7 +463,13 @@ workflows:
           <<: *on-any-branch
           <<: *after-linter
       - valgrind:
-          <<: *on-master
+          <<: *on-any-branch
+          <<: *after-linter
+      - valgrind-cluster:
+          <<: *on-integ-branch
+          <<: *after-linter
+      - valgrind-AOF:
+          <<: *on-integ-branch
           <<: *after-linter
       - build-and-test-gpu:
           <<: *on-any-branch

docs/commands.md

@@ -660,14 +660,14 @@ Because `AI.DAGRUN` provides the `PERSIST` option it is flagged as a 'write' com
     Refer to the Redis [`READONLY` command](https://redis.io/commands/readonly) for further information about read-only cluster replicas.
 
 ## AI.INFO
-The **`AI.INFO`** command returns information about the execution a model or a script.
+The **`AI.INFO`** command returns general module information or information about the execution a model or a script.
 
 Runtime information is collected each time that [`AI.MODELRUN`](#aimodelrun) or [`AI.SCRIPTRUN`](#aiscriptrun) is called. The information is stored locally by the executing RedisAI engine, so when deployed in a cluster each shard stores its own runtime information.
 
 **Redis API**
 
 ```
-AI.INFO <key> [RESETSTAT]
+AI.INFO [<key>] [RESETSTAT]
 ```
 
 _Arguments_
@@ -677,7 +677,15 @@ _Arguments_
 
 _Return_
 
-An array with alternating entries that represent the following key-value pairs:
+For a module genernal information: An array with alternating entries that represent the following key-value pairs:
+
+* **Version**: a string showing the current module version.
+* **Low level API Version**:  a string showing the current module's low level api version.
+* **RDB Encoding version**: a string showing the current module's RDB encoding version.
+* **TensorFlow version**: a string showing the current loaded TesnorFlow backend version.
+* **ONNX version**: a string showing the current loaded ONNX Runtime backend version.
+
+For model or script runtime information: An array with alternating entries that represent the following key-value pairs:
 
 * **KEY**: a String of the name of the key storing the model or script value
 * **TYPE**: a String of the type of value (i.e. 'MODEL' or 'SCRIPT')

opt/readies

@@ -26,6 +26,13 @@
    obj:*/libtorch.so.*
 }
 
+{
+   ignore_unversioned_libs
+   Memcheck:Overlap
+   ...
+   obj:*/libtorch_cpu.so*
+}
+
 {
    ignore_unversioned_libs
    Memcheck:Leak

opt/redis_valgrind.sup

@@ -18,33 +18,28 @@ def __init__(self, nop=False):
 
     def common_first(self):
         self.install_downloaders()
-        self.setup_pip()
-        self.pip_install("wheel virtualenv")
+        self.pip_install("wheel")
 
-        if self.os == 'linux':
-            self.install("ca-certificates")
-        self.install("git unzip wget patchelf")
+        self.install("git unzip patchelf")
         self.install("coreutils") # for realpath
 
     def debian_compat(self):
         self.install("gawk")
         self.install("build-essential")
         self.install("libssl-dev")
-        self.install("clang-format")
         self.install("python3-regex")
         self.install("python3-psutil python3-networkx python3-numpy")
-        if self.arch == 'arm64v8' or self.arch == 'arm32v7':
+        if self.platform.is_arm():
             self.install("python3-dev") # python3-skimage
         self.install("libmpich-dev libopenblas-dev") # for libtorch
         self.install_git_lfs_on_linux()
 
     def redhat_compat(self):
         self.install("redhat-lsb-core")
-        self.run("%s/readies/bin/enable-utf8" % HERE)
+        self.run("%s/bin/enable-utf8" % READIES)
 
-        self.group_install("'Development Tools'")
         self.run("%s/bin/getgcc --modern" % READIES)
-        self.install("llvm-toolset-7")
+        # self.install("llvm-toolset-7")
 
         if self.arch == 'x64':
             self.install_linux_gnu_tar()
@@ -63,7 +58,6 @@ def redhat_compat(self):
     def fedora(self):
         self.group_install("'Development Tools'")
         self.install("python3 python3-psutil python3-networkx")
-        self.install("clang")
         self.install_git_lfs_on_linux()
 
     def linux_last(self):
@@ -73,18 +67,13 @@ def macos(self):
         self.install_gnu_utils()
         self.install("git-lfs")
         self.install("redis")
-        self.install("clang-format")
 
     def common_last(self):
+        self.run("%s/bin/getclang --format" % READIES)
         self.run("%s/bin/getcmake" % READIES)
 
-        self.run("python3 -m pip uninstall -y ramp-packer RLTest || true")
-        # redis-py-cluster should be installed from git due to redis-py dependency
-        self.pip_install("--no-cache-dir git+https://github.com/Grokzen/redis-py-cluster.git@master")
-        self.pip_install("--no-cache-dir git+https://github.com/RedisLabsModules/RLTest.git@master")
-        self.pip_install("--no-cache-dir git+https://github.com/RedisLabs/RAMP@master")
+        self.run("{PYTHON} {READIES}/bin/getrmpytools".format(PYTHON=self.python, READIES=READIES))
 
-        self.pip_install("-r %s/readies/paella/requirements.txt" % HERE)
         self.pip_install("-r %s/tests/flow/test_requirements.txt" % ROOT)
 
         self.pip_install("awscli")

opt/system-setup.py

@@ -16,6 +16,7 @@ file (GLOB BACKEND_COMMON_SRC
         err.c
         util/dict.c
         tensor.c
+        util/string_utils.c
         serialization/ai_datatypes.c)
 
 ADD_LIBRARY(redisai_obj OBJECT