Skip to content

Commit 56ec03c

Browse files
author
DvirDukhan
authored
Merge branch 'master' into serialization_and_short_reads
2 parents 5b5392b + 740bbba commit 56ec03c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+935
-276
lines changed

.circleci/config.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -331,7 +331,7 @@ platform-build-defs: &platform-build-defs
331331

332332
after-platform-builds: &after-platform-builds
333333
requires:
334-
- build-centos7
334+
# - build-centos7
335335
- build-bionic
336336
- build-xenial
337337

@@ -358,7 +358,7 @@ workflows:
358358
- platform-build:
359359
name: build-xenial
360360
platform: xenial
361-
<<: *on-any-branch
361+
<<: *platform-build-defs
362362
- coverage:
363363
<<: *on-any-branch
364364
<<: *after-linter

.dockerignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
/bin/
22
/deps/
3+
/docs/
34
/install*
45
/build/
56
.venv/

Dockerfile.jetson

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
# BUILD redisfab/redisai:${VERSION}-jetson-${ARCH}-${OSNICK}
2+
3+
ARG REDIS_VER=6.0.9
4+
5+
# OSNICK=bionic|centos7|centos6
6+
ARG OSNICK=bionic
7+
8+
# OS=ubuntu18.04|ubuntu16.04|centos7
9+
ARG OS=ubuntu18.04
10+
11+
# ARCH=arm64v8|arm32v7
12+
ARG ARCH=arm64v8
13+
14+
ARG CUDA_VER=10.2-cudnn7
15+
16+
ARG L4T_VER=r32.4.4
17+
18+
ARG PACK=0
19+
ARG TEST=0
20+
21+
#----------------------------------------------------------------------------------------------
22+
FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis
23+
FROM redisfab/jetpack:4.4.1-arm64v8-l4t as builder
24+
25+
ARG OSNICK
26+
ARG OS
27+
ARG ARCH
28+
ARG REDIS_VER
29+
ARG CUDA_VER
30+
ARG L4T_VER
31+
32+
RUN echo "Building for ${OSNICK} (${OS}) for ${ARCH} [with Redis ${REDIS_VER}]"
33+
34+
ENV NVIDIA_VISIBLE_DEVICES all
35+
ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
36+
37+
WORKDIR /build
38+
COPY --from=redis /usr/local/ /usr/local/
39+
40+
COPY ./opt/ opt/
41+
COPY ./tests/flow/test_requirements.txt tests/flow/
42+
43+
RUN PIP=1 FORCE=1 ./opt/readies/bin/getpy3
44+
RUN ./opt/system-setup.py
45+
46+
ARG DEPS_ARGS="GPU=1 JETSON=1 WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0"
47+
COPY ./get_deps.sh .
48+
# RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh gpu; else env $DEPS_ARGS ./get_deps.sh gpu; fi
49+
RUN set -e ;\
50+
env $DEPS_ARGS ./get_deps.sh
51+
52+
ARG BUILD_ARGS="GPU=1 JETSON=1 WITH_TF=1 WITH_PT=1 WITH_TFLITE=0 WITH_ORT=0"
53+
54+
ADD ./ /build
55+
RUN bash -c "set -e ;\
56+
. ./opt/readies/bin/sourced ./profile.d ;\
57+
make -C opt build $BUILD_ARGS SHOW=1"
58+
59+
ARG PACK
60+
ARG TEST
61+
62+
RUN mkdir -p bin/artifacts
63+
RUN set -e ;\
64+
if [ "$PACK" = "1" ]; then make -C opt pack GPU=1; fi
65+
66+
RUN set -e ;\
67+
if [ "$TEST" = "1" ]; then \
68+
TEST= make -C opt test GPU=1 $BUILD_ARGS NO_LFS=1 ;\
69+
if [[ -d test/logs ]]; then \
70+
tar -C test/logs -czf bin/artifacts/test-logs-gpu.tgz . ;\
71+
fi ;\
72+
fi
73+
74+
#----------------------------------------------------------------------------------------------
75+
# FROM nvidia/cuda:${CUDA_VER}-runtime-${OS}
76+
# FROM nvidia/cuda-arm64:11.1-runtime-ubuntu18.04
77+
# FROM nvcr.io/nvidia/l4t-base:${L4T_VER}
78+
FROM redisfab/jetpack:4.4.1-arm64v8-l4t
79+
80+
ARG OS
81+
ARG L4T_VER
82+
83+
RUN if [ ! -z $(command -v apt-get) ]; then apt-get -qq update; apt-get -q install -y libgomp1; fi
84+
RUN if [ ! -z $(command -v yum) ]; then yum install -y libgomp; fi
85+
86+
ENV REDIS_MODULES /usr/lib/redis/modules
87+
RUN mkdir -p $REDIS_MODULES/
88+
89+
COPY --from=redis /usr/local/ /usr/local/
90+
COPY --from=builder /build/install-gpu/ $REDIS_MODULES/
91+
92+
COPY --from=builder /build/bin/artifacts/ /var/opt/redislabs/artifacts
93+
94+
# COPY --from=builder /usr/local/etc/redis/redis.conf /usr/local/etc/redis/
95+
96+
WORKDIR /data
97+
EXPOSE 6379
98+
# CMD ["/usr/local/bin/redis-server", "/usr/local/etc/redis/redis.conf", "--loadmodule", "/usr/lib/redis/modules/redisai.so"]
99+
CMD ["/usr/local/bin/redis-server", "--loadmodule", "/usr/lib/redis/modules/redisai.so"]

get_deps.sh

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ if [[ $1 == --help || $1 == help ]]; then
1616
Argument variables:
1717
CPU=1 Get CPU dependencies
1818
GPU=1 Get GPU dependencies
19+
JETSON=1 Get Jetson Nano dependencies
1920
VERBOSE=1 Print commands
2021
FORCE=1 Download even if present
2122
WITH_DLPACK=0 Skip dlpack
@@ -106,6 +107,9 @@ if [[ $WITH_TF != 0 ]]; then
106107
LIBTF_URL_BASE=https://storage.googleapis.com/tensorflow/libtensorflow
107108
elif [[ $ARCH == arm64v8 ]]; then
108109
TF_ARCH=arm64
110+
if [[ $JETSON == 1 ]]; then
111+
TF_BUILD="gpu-jetson"
112+
fi
109113
LIBTF_URL_BASE=https://s3.amazonaws.com/redismodules/tensorflow
110114
elif [[ $ARCH == arm32v7 ]]; then
111115
TF_ARCH=arm
@@ -205,7 +209,11 @@ if [[ $WITH_PT != 0 ]]; then
205209
if [[ $GPU != 1 ]]; then
206210
PT_BUILD=cpu
207211
else
208-
PT_BUILD=cu101
212+
if [[ $JETSON == 1 ]]; then
213+
PT_BUILD=cu102-jetson
214+
else
215+
PT_BUILD=cu101
216+
fi
209217
fi
210218
if [[ $ARCH == x64 ]]; then
211219
PT_ARCH=x86_64

opt/Makefile

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ make flow_tests # run flow test
5353
COV=1 # perform coverage analysis
5454
VALGRIND|VGD=1 # test with Valgrind (implies DEBUG=1)
5555
CALLGRIND|CGD=1 # test with Callgrind (implies DEBUG=1)
56+
REDIS=addr # Address of Redis server for testing
5657

5758
make cov-upload # upload coverage data to codecov.io (requires CODECOV_TOKEN)
5859

@@ -75,6 +76,7 @@ device selection options (fetch, build, and test):
7576
CPU=1 # build for CPU
7677
GPU=1 # build for GPU
7778
CUDA=1 # build for GPU
79+
JETSON=1 # build for Nvidia Jetson
7880
endef
7981

8082
#----------------------------------------------------------------------------------------------
@@ -85,10 +87,13 @@ ifeq ($(GPU),1)
8587
ifeq ($(CPU),1)
8688
$(error CPU=1 and GPU=1 (or CUDA=1) are conflicting)
8789
endif
88-
DEPS_FLAGS=gpu
90+
DEPS_FLAGS += GPU=1
8991
DEVICE=gpu
92+
ifeq ($(JETSON),1)
93+
DEPS_FLAGS += JETSON=1
94+
endif
9095
else
91-
DEPS_FLAGS=cpu
96+
DEPS_FLAGS += CPU=1
9297
DEVICE=cpu
9398
endif
9499

@@ -194,7 +199,7 @@ setup:
194199

195200
fetch deps:
196201
@echo Fetching dependencies...
197-
$(SHOW)VERBOSE=$(_SHOW) $(ROOT)/get_deps.sh $(DEPS_FLAGS)
202+
$(SHOW)VERBOSE=$(_SHOW) $(DEPS_FLAGS) $(ROOT)/get_deps.sh
198203

199204
format:
200205
./clang-format-all.sh
@@ -243,6 +248,7 @@ flow_tests: build
243248
CLUSTER=$(CLUSTER) \
244249
GEN=$(GEN) AOF=$(AOF) SLAVES=$(SLAVES) \
245250
VALGRIND=$(VALGRIND) \
251+
REDIS=$(REDIS) \
246252
$(ROOT)/tests/flow/tests.sh
247253
$(COVERAGE_COLLECT_REPORT)
248254

@@ -257,6 +263,7 @@ test: build
257263
CLUSTER=$(CLUSTER) \
258264
GEN=$(GEN) AOF=$(AOF) SLAVES=$(SLAVES) \
259265
VALGRIND=$(VALGRIND) \
266+
REDIS=$(REDIS) \
260267
$(ROOT)/tests/flow/tests.sh
261268
$(COVERAGE_COLLECT_REPORT)
262269

opt/build/docker/Makefile

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ OSNICK ?= buster
99
REPO=redisfab
1010
STEM=$(REPO)/redisai
1111

12-
REDIS_VER:=6.0.5
12+
REDIS_VER:=6.0.9
1313

1414
ART_DIR=$(ROOT)/bin/artifacts
1515
ART_INT_DIR=/var/opt/redislabs/artifacts
@@ -23,6 +23,9 @@ BUILD_ARGS.arm32v7=WITH_ORT=0 WITH_PT=0
2323
ifeq ($(CPU),1)
2424
VAR=cpu
2525

26+
FETCH_ARGS=$(FETCH_ARGS.$(ARCH))
27+
BUILD_ARGS=$(BUILD_ARGS.$(ARCH))
28+
2629
else ifeq ($(GPU),1)
2730
VAR=gpu
2831
DOCKERFILE_STEM ?= $(ROOT)/Dockerfile.gpu
@@ -33,12 +36,20 @@ OS.centos7=centos7
3336
OS.bionic=ubuntu18.04
3437
OS.xenial=ubuntu16.04
3538

36-
else
37-
VAR=cpu
39+
FETCH_ARGS=$(FETCH_ARGS.$(ARCH))
40+
BUILD_ARGS=$(BUILD_ARGS.$(ARCH))
41+
42+
else ifeq ($(JETSON),1)
43+
VAR=jetson
44+
DOCKERFILE_STEM ?= $(ROOT)/Dockerfile.jetson
45+
46+
# FETCH_ARGS=WITH_TF=1 WITH_PT=1 WITH_TFLITE=0 WITH_ORT=0
47+
# BUILD_ARGS=WITH_TF=1 WITH_PT=1 WITH_TFLITE=0 WITH_ORT=0
48+
3849
endif
3950

40-
DOCKER_BUILD_ARGS += \
41-
DEPS_ARGS="$(FETCH_ARGS.$(ARCH))" \
42-
BUILD_ARGS="$(BUILD_ARGS.$(ARCH))"
51+
# DOCKER_BUILD_ARGS += \
52+
# DEPS_ARGS="$(FETCH_ARGS)" \
53+
# BUILD_ARGS="$(BUILD_ARGS)"
4354

4455
include $(MK)/docker

0 commit comments

Comments
 (0)