backend_build_WIP_

.circleci/config.yml

@@ -221,35 +221,25 @@ jobs:
           platform: debian
 
   platforms-build:
-    machine:
-      enabled: true
-      docker_layer_caching: true
-      resource_class: small
-      image: ubuntu-2004:202107-02
     parameters:
       lite:  # LITE value during make
         type: string
       osnick:  # OSNICK value for the base platform of the docker
         type: string
       target:  # CPU|GPU
         type: string
+    docker:
+      - image: redisfab/rmbuilder:6.2.5-x64-buster
     steps:
       - abort_for_docs
       - abort_for_noci
       - early_return_for_forked_pull_requests
+      - setup_remote_docker:
+          docker_layer_caching: true
       - checkout-all
-      - run:
-          name: Relocate docker overlay2 dir
-          command: |
-            sudo systemctl stop docker
-            sudo mkdir -p /var2/lib/docker
-            sudo mv /var/lib/docker/overlay2 /var2/lib/docker
-            sudo mkdir /var/lib/docker/overlay2
-            sudo mount --bind /var2/lib/docker/overlay2 /var/lib/docker/overlay2
-            sudo systemctl start docker
       - restore_cache:
           keys:
-          - v1.2.5-deps-{{ checksum "get_deps.sh" }}-<<parameters.osnick>>-<<parameters.target>>
+          - v1-dependencies-{{ checksum "get_deps.sh" }}-<<parameters.osnick>>-<<parameters.target>>
       - setup-automation
 
       # since we run in parallel, we need to generate docker files with different suffixes hence the DOCKER_SUFFIX
@@ -352,6 +342,45 @@ jobs:
     steps:
       - valgrind-general-steps
 
+#  build-macos:
+#    macos:
+#      xcode: 11.3.0
+#    steps:
+#      - abort_for_docs
+#      - run:
+#          name: Fix macOS Python installation
+#          command: |
+#            brew reinstall -f python2
+#      - build-steps:
+#          platform: macos
+#
+#  build-multiarch-docker:
+#    machine:
+#      enabled: true
+#      image: cimg/base:2020.01
+#    steps:
+#      - abort_for_docs
+#      - checkout-all
+#      - run:
+#          name: Checkout LFS
+#          command: |
+#            curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
+#            sudo apt-get install -y git-lfs
+#            git lfs install
+#            git lfs pull
+#      - run:
+#          name: Setup Docker client experimental features
+#          command: |
+#            sudo ./opt/readies/bin/getdocker --just-enable-exp
+#            docker version
+#      - run:
+#          name: Build
+#          command: |
+#            sudo docker login -u redisfab -p $DOCKER_REDISFAB_PWD
+#            cd opt/build/docker
+#            make build
+#            sudo make publish
+
   # internal PRs execute build-and-test either in a workflow or
   # via a github action trigger
   build-and-test-gpu:

Dockerfile.arm

@@ -0,0 +1,83 @@
+# BUILD redisfab/redisai:${VERSION}-cpu-${ARCH}-${OSNICK}
+
+ARG REDIS_VER=6.2.4
+
+# OSNICK=bionic|stretch|buster
+ARG OSNICK=buster
+
+# OS=debian:buster-slim|debian:stretch-slim|ubuntu:bionic
+ARG OS=debian:buster-slim
+
+# ARCH=arm64v8|arm32v7
+ARG ARCH=arm64v8
+
+ARG PACK=0
+ARG REDISAI_LITE=0
+ARG TEST=0
+
+#----------------------------------------------------------------------------------------------
+FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis
+FROM redisfab/xbuild:${ARCH}-${OS} AS builder
+
+ARG OSNICK
+ARG OS
+ARG ARCH
+ARG REDIS_VER
+
+RUN [ "cross-build-start" ]
+
+RUN echo "Building for ${OSNICK} (${OS}) for ${ARCH}"
+
+WORKDIR /build
+COPY --from=redis /usr/local/ /usr/local/
+
+COPY ./opt/ opt/
+COPY ./tests/flow/tests_setup/test_requirements.txt tests/flow
+
+RUN ./opt/readies/bin/getpy3
+RUN ./opt/system-setup.py
+
+ARG DEPS_ARGS=""
+COPY ./get_deps.sh .
+RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh cpu; else env $DEPS_ARGS ./get_deps.sh cpu; fi
+
+ARG BUILD_ARGS=""
+ADD ./ /build
+RUN set -e ;\
+    . ./opt/readies/bin/sourced ./profile.d ;\
+    make -C opt build $BUILD_ARGS SHOW=1
+
+ARG PACK
+ARG TEST
+
+RUN if [ "$PACK" = "1" ]; then make -C opt pack; fi
+RUN if [ "$TEST" = "1" ]; then TEST= make -C opt test $BUILD_ARGS NO_LFS=1; fi
+
+RUN [ "cross-build-end" ]
+
+#----------------------------------------------------------------------------------------------
+FROM redisfab/redis-xbuild:${REDIS_VER}-${ARCH}-${OSNICK}
+
+RUN [ "cross-build-start" ]
+
+ARG OSNICK
+ARG OS
+ARG ARCH
+ARG REDIS_VER
+ARG PACK
+
+RUN if [ ! -z $(command -v apt-get) ]; then apt-get -qq update; apt-get -q install -y libgomp1; fi
+RUN if [ ! -z $(command -v yum) ]; then yum install -y libgomp; fi
+
+ENV REDIS_MODULES /usr/lib/redis/modules
+ENV LD_LIBRARY_PATH $REDIS_MODULES
+
+RUN mkdir -p $REDIS_MODULES/
+
+COPY --from=builder /build/install-cpu/ $REDIS_MODULES/
+
+WORKDIR /data
+EXPOSE 6379
+CMD ["--loadmodule", "/usr/lib/redis/modules/redisai.so"]
+
+RUN [ "cross-build-end" ]

Dockerfile.jetson

@@ -0,0 +1,102 @@
+# BUILD redisfab/redisai:${VERSION}-jetson-${ARCH}-${OSNICK}
+
+ARG REDIS_VER=6.2.4
+
+# OSNICK=bionic|centos7|centos6
+ARG OSNICK=bionic
+
+# OS=ubuntu18.04|ubuntu16.04|centos7
+ARG OS=ubuntu18.04
+
+# ARCH=arm64v8|arm32v7
+ARG ARCH=arm64v8
+
+ARG CUDA_VER=10.2-cudnn7
+
+ARG L4T_VER=r32.4.4
+
+ARG PACK=0
+ARG REDISAI_LITE=0
+ARG TEST=0
+
+#----------------------------------------------------------------------------------------------
+FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis
+FROM redisfab/jetpack:4.4.1-arm64v8-l4t as builder
+
+ARG OSNICK
+ARG OS
+ARG ARCH
+ARG REDIS_VER
+ARG CUDA_VER
+ARG L4T_VER
+
+RUN echo "Building for ${OSNICK} (${OS}) for ${ARCH} [with Redis ${REDIS_VER}]"
+
+ENV NVIDIA_VISIBLE_DEVICES all
+ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
+
+WORKDIR /build
+COPY --from=redis /usr/local/ /usr/local/
+
+COPY ./opt/ opt/
+COPY ./tests/flow/tests_setup/test_requirements.txt tests/flow/
+
+RUN FORCE=1 ./opt/readies/bin/getpy3
+RUN ./opt/system-setup.py
+
+ARG DEPS_ARGS="GPU=1 JETSON=1 WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0"
+COPY ./get_deps.sh .
+# RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh gpu; else env $DEPS_ARGS ./get_deps.sh gpu; fi
+RUN set -e ;\
+    env $DEPS_ARGS ./get_deps.sh
+
+ARG BUILD_ARGS="GPU=1 JETSON=1 WITH_TF=1 WITH_PT=1 WITH_TFLITE=0 WITH_ORT=0"
+
+ADD ./ /build
+RUN bash -c "set -e ;\
+    . ./opt/readies/bin/sourced ./profile.d ;\
+    make -C opt build  $BUILD_ARGS SHOW=1"
+
+ARG PACK
+ARG REDISAI_LITE
+ARG TEST
+
+RUN mkdir -p bin/artifacts
+RUN set -e ;\
+    if [ "$PACK" = "1" ]; then make -C opt pack GPU=1; fi
+
+RUN set -e ;\
+    if [ "$TEST" = "1" ]; then \
+        TEST= make -C opt test GPU=1 $BUILD_ARGS NO_LFS=1 ;\
+        if [[ -d test/logs ]]; then \
+            tar -C test/logs -czf bin/artifacts/test-logs-gpu.tgz . ;\
+        fi ;\
+    fi
+
+#----------------------------------------------------------------------------------------------
+# FROM nvidia/cuda:${CUDA_VER}-runtime-${OS}
+# FROM nvidia/cuda-arm64:11.1-runtime-ubuntu18.04
+# FROM nvcr.io/nvidia/l4t-base:${L4T_VER}
+FROM redisfab/jetpack:4.4.1-arm64v8-l4t
+
+ARG OS
+ARG L4T_VER
+
+RUN if [ ! -z $(command -v apt-get) ]; then apt-get -qq update; apt-get -q install -y libgomp1; fi
+RUN if [ ! -z $(command -v yum) ]; then yum install -y libgomp; fi
+
+ENV REDIS_MODULES /usr/lib/redis/modules
+RUN mkdir -p $REDIS_MODULES/
+
+COPY --from=redis /usr/local/ /usr/local/
+COPY --from=builder /build/install-gpu/ $REDIS_MODULES/
+
+COPY --from=builder /build/bin/artifacts/ /var/opt/redislabs/artifacts
+
+# COPY --from=builder /usr/local/etc/redis/redis.conf /usr/local/etc/redis/
+
+WORKDIR /data
+EXPOSE 6379
+# CMD ["/usr/local/bin/redis-server", "/usr/local/etc/redis/redis.conf", "--loadmodule", "/usr/lib/redis/modules/redisai.so"]
+RUN rm -rf /root/.cache /usr/local/cuda/lib64/*.a /usr/local/cuda/doc /usr/local/cuda/samples
+CMD ["/usr/local/bin/redis-server", "--loadmodule", "/usr/lib/redis/modules/redisai.so"]