Merge branch 'master' into serialization_and_short_reads

Author: DvirDukhan

.circleci/config.yml

@@ -12,6 +12,14 @@ commands:
       - setup_remote_docker:
           docker_layer_caching: true
 
+  setup-automation:
+    steps:
+      - run:
+          name: Setup automation
+          command: |
+            PIP=1 ./opt/readies/bin/getpy3
+            ./opt/system-setup.py
+
   build-steps:
     parameters:
       platform:
@@ -25,11 +33,10 @@ commands:
           keys:
           - v1-dependencies-{{ checksum "get_deps.sh" }}
           # If no exact match is found will get dependencies from source
+      - setup-automation
       - run:
           name: Install dependencies
           command: |
-            ./opt/readies/bin/getpy3
-            BREW_NO_UPDATE=1 ./opt/system-setup.py
             ./opt/readies/bin/getredis -v 6 --force
             ./get_deps.sh cpu
       - save_cache:
@@ -60,8 +67,6 @@ commands:
             - artifacts/*.zip
             - artifacts/*.tgz
             - artifacts/*.tar
-            # - artifacts/shapshots/*.zip
-            # - artifacts/shapshots/*.tgz
       - store_artifacts:
           path: tests/logs
 
@@ -98,8 +103,6 @@ commands:
             - artifacts/*.zip
             - artifacts/*.tgz
             - artifacts/*.tar
-            # - artifacts/shapshots/*.zip
-            # - artifacts/shapshots/*.tgz
       - store_artifacts:
           path: test/logs
 
@@ -156,11 +159,10 @@ jobs:
           keys:
           - build-dependencies-{{ checksum "get_deps.sh" }}
           # If no exact match is found will get dependencies from source
+      - setup-automation
       - run:
           name: Install dependencies
           command: |
-            ./opt/readies/bin/getpy3
-            ./opt/system-setup.py
             ./opt/readies/bin/getredis -v 6 --valgrind --force
             ./get_deps.sh cpu
       - run:
@@ -174,6 +176,34 @@ jobs:
             make -C opt cov-upload
           no_output_timeout: 30m
 
+  valgrind:
+    docker:
+      - image: redisfab/rmbuilder:6.0.5-x64-buster
+    steps:
+      - checkout
+      - run:
+          name: Submodule checkout
+          command: git submodule update --init --recursive 
+      - restore_cache:
+          keys:
+          - build-dependencies-{{ checksum "get_deps.sh" }}
+          # If no exact match is found will get dependencies from source
+      - setup-automation
+      - run:
+          name: Install dependencies
+          command: |
+            ./opt/readies/bin/getredis -v 6 --valgrind --force
+            ./get_deps.sh cpu
+      - run:
+          name: Build for valgrind
+          command: |
+            make -C opt all VALGRIND=1 SHOW=1
+      - run:
+          name: Test with valgrind
+          command: |
+            make -C opt test VALGRIND=1
+          no_output_timeout: 120m
+
   build-macos:
     macos:
       xcode: 11.3.0
@@ -322,7 +352,6 @@ on-master-and-version-tags: &on-master-and-version-tags
     tags:
       only: /^v[0-9].*/
 
-
 platform-build-defs: &platform-build-defs
   requires:
     - build-debian
@@ -362,6 +391,9 @@ workflows:
       - coverage:
           <<: *on-any-branch
           <<: *after-linter
+      - valgrind:
+          <<: *on-master
+          <<: *after-linter
       - build-and-test-gpu:
           <<: *on-any-branch
           <<: *after-linter

README.md

@@ -136,7 +136,7 @@ RedisAI currently supports PyTorch (libtorch), Tensorflow (libtensorflow), Tenso
 | 0.4.0   | 1.2.0   | 1.14.0     | None   | 0.5.0         |
 | 0.9.0   | 1.3.1   | 1.14.0     | 2.0.0  | 1.0.0         |
 | 1.0.0   | 1.5.0   | 1.15.0     | 2.0.0  | 1.2.0         |
-| master  | 1.5.0   | 1.15.0     | 2.0.0  | 1.2.0         |
+| master  | 1.7.0   | 1.15.0     | 2.0.0  | 1.2.0         |
 
 Note: Keras and TensorFlow 2.x are supported through graph freezing. See [this script](https://github.com/RedisAI/RedisAI/blob/master/tests/test_data/tf2-minimal.py) to see how to export a frozen graph from Keras and TensorFlow 2.x. Note that a frozen graph will be executed using the TensorFlow 1.15 backend. Should any 2.0 ops be not supported on the 1.15 after freezing, please open an Issue.
 
@@ -152,4 +152,4 @@ Got questions? Feel free to ask at the [RedisAI Forum](https://forum.redislabs.c
 
 Redis Source Available License Agreement - see [LICENSE](LICENSE)
 
-Copyright 2020, [Tensorwerk, Inc](https://tensorwerk.com) & [Redis Labs, Inc](https://redislabs.com)
+Copyright 2020, [Redis Labs, Inc](https://redislabs.com)

opt/Makefile

@@ -231,15 +231,29 @@ export SLAVES ?= 1
 export AOF ?= 1
 export CLUSTER ?= 1
 
+define UNIT_TESTS
+$(SHOW)echo Running unit tests ...
+$(SHOW)\
+BINDIR=$(realpath $(BINDIR)) \
+$(ROOT)/tests/unit/tests.sh
+endef
+
+define FLOW_TESTS
+$(SHOW)echo Running flow tests ...
+$(SHOW)\
+DEVICE=$(DEVICE) \
+MODULE=$(realpath $(INSTALLED_TARGET)) \
+CLUSTER=$(CLUSTER) \
+GEN=$(GEN) AOF=$(AOF) SLAVES=$(SLAVES) \
+VALGRIND=$(VALGRIND) \
+$(ROOT)/tests/flow/tests.sh
+endef
+
 unit_tests: build
-	@echo "$(BINDIR)"
 	$(COVERAGE_RESET)
-	$(SHOW)\
-		BINDIR=$(realpath $(BINDIR)) \
-		$(ROOT)/tests/unit/tests.sh
+	$(UNIT_TESTS)
 	$(COVERAGE_COLLECT_REPORT)
 
-
 flow_tests: build
 	$(COVERAGE_RESET)
 	$(SHOW)\
@@ -250,10 +264,12 @@ flow_tests: build
 		VALGRIND=$(VALGRIND) \
 		REDIS=$(REDIS) \
 		$(ROOT)/tests/flow/tests.sh
+
 	$(COVERAGE_COLLECT_REPORT)
 
 test: build
 	$(COVERAGE_RESET)
+	$(UNIT_TESTS)
 	$(SHOW)\
 		BINDIR=$(realpath $(BINDIR)) \
 		$(ROOT)/tests/unit/tests.sh
@@ -267,6 +283,8 @@ test: build
 		$(ROOT)/tests/flow/tests.sh
 	$(COVERAGE_COLLECT_REPORT)
 
+#----------------------------------------------------------------------------------------------
+
 valgrind:
 	$(SHOW)$(ROOT)/tests/flow/valgrind.sh $(realpath $(INSTALLED_TARGET))
 

opt/redis_valgrind.sup

@@ -43,3 +43,28 @@
    Memcheck:Value8
    fun:lzf_compress
 }
+
+{
+   <dlopen>
+   Memcheck:Leak
+   fun:malloc
+   ...
+   fun:dlopen@@GLIBC_2.2.5
+   ...
+   fun:RAI_LoadBackend
+}
+
+{
+   <tf-operator new>
+   Memcheck:Leak
+   ...
+   fun:clone
+}
+
+{
+   <malloc>
+   Memcheck:Leak
+   fun:malloc
+   ...
+   fun:clone
+}

opt/system-setup.py

@@ -66,6 +66,9 @@ def fedora(self):
         self.install("clang")
         self.install_git_lfs_on_linux()
 
+    def linux_last(self):
+        self.install("valgrind")
+
     def macos(self):
         self.install_gnu_utils()
         self.install("git-lfs")

src/CMakeLists.txt

@@ -23,10 +23,13 @@ ADD_LIBRARY(redisai_obj OBJECT
         util/queue.c
         util/string_utils.c
         redisai.c
+        command_parser.c
         run_info.c
         background_workers.c
         config.c
-        dag.c
+        DAG/dag.c
+        DAG/dag_parser.c
+        modelRun_ctx.c
         backends.c
         backends/util.c
         model.c