Dubugging #2

Makefile

@@ -1,14 +1,13 @@
 
 start:
-ifeq ($(REBUILD),1)
-	@docker-compose up --build -d
-else
 	@docker-compose up -d
-endif
 
 stop:
 	@docker-compose down
 
+build:
+	@docker-compose build
+
 test:
 	@./tests/cats-n-dogs.sh
 
@@ -31,4 +30,4 @@ setup:
 		chmod +x /usr/local/bin/docker-compose ;\
 	fi
 
-.PHONY: start stop test camera setup
+.PHONY: start stop build test camera setup

app/Dockerfile

@@ -1,15 +1,21 @@
-# FROM python:3.7.3
-FROM debian:buster
+# OSNICK=stretch|bionic|buster
+ARG OSNICK=buster
+
+#----------------------------------------------------------------------------------------------
+FROM redisfab/redisedgevision-${OSNICK}:0.2.0
+
+# This is due on the following error on ARMv8:
+# /usr/lib/aarch64-linux-gnu/libgomp.so.1: cannot allocate memory in static TLS block 
+# Something is exausting TLS, causing libgomp to fail. Preloading it as a workaround helps.
+ENV LD_PRELOAD /usr/lib/aarch64-linux-gnu/libgomp.so.1
 
 ENV DEBIAN_FRONTEND=noninteractive
-ENV RUNTIME_DEPS "python3 python3-setuptools ca-certificates curl python3-opencv"
 
-RUN set -ex ;\
-    apt-get -qq update ;\
-    apt-get -q install -y --no-install-recommends $RUNTIME_DEPS
+RUN apt-get -qq update
 
 RUN set -ex ;\
-	curl -q -s https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py ;\
+	apt-get install -y wget python3-distutils ;\
+	wget -q https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py ;\
 	python3 /tmp/get-pip.py
 
 WORKDIR /app

camera/Dockerfile.debian

@@ -1,15 +1,28 @@
-FROM debian:buster
+# OSNICK=stretch|bionic|buster
+ARG OSNICK=buster
+
+#----------------------------------------------------------------------------------------------
+FROM redisfab/redisedgevision-${OSNICK}:0.2.0
+
+# This is due on the following error on ARMv8:
+# /usr/lib/aarch64-linux-gnu/libgomp.so.1: cannot allocate memory in static TLS block 
+# Something is exausting TLS, causing libgomp to fail. Preloading it as a workaround helps.
+# ENV LD_PRELOAD /usr/lib/aarch64-linux-gnu/libgomp.so.1
+
+ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get -qq update
-RUN apt-get -q install -y python wget
+
 RUN set -x; \
+	apt-get install -y wget python3-distutils ;\
     wget -q https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py ;\
-	python /tmp/get-pip.py
-RUN apt-get -q install -y python-opencv
+	python3 /tmp/get-pip.py
+
+RUN pip install redis==3.2.1
 
 WORKDIR /usr/src/app
+
 ADD read_camera.py ./
 ADD *.jpg ./
-RUN pip install redis
 
-CMD [ "python", "./read_camera.py" ]
+CMD [ "python3", "./read_camera.py" ]

camera/read_camera.py

@@ -1,3 +1,4 @@
+
 import argparse
 import cv2
 import redis
@@ -13,7 +14,7 @@
 IMAGE_WIDTH = 640
 IMAGE_HEIGHT = 480
 
-MAX_IMAGES = 5 # 1000
+MAX_IMAGES = 1000 # 5
 
 class Webcam:
     def __init__(self, infile=0, fps=15.0):
@@ -75,7 +76,7 @@ def __len__(self):
                 'count': count,
                 'image': data.tobytes()
             }
-            _id = conn.execute_command('xadd', args.output, 'MAXLEN', '~', '1000', '*', 'count', msg['count'], 'img', msg['image'])
+            _id = conn.execute_command('xadd', args.output, 'MAXLEN', '~', str(MAX_IMAGES), '*', 'count', msg['count'], 'img', msg['image'])
             print('count: {} id: {}'.format(count, _id))
             sys.stdout.flush()
     else:

redis/Dockerfile

@@ -20,3 +20,7 @@ ENTRYPOINT ["redis-server"]
 CMD ["--loadmodule", "/usr/lib/redis/modules/redisai.so", \
 	 "--loadmodule", "/usr/lib/redis/modules/redisgears.so", \
 	 "PythonHomeDir", "/opt/redislabs/lib/modules/python3/"]
+
+# ADD redisedge.conf /etc
+
+# CMD ['/etc/redisedge.conf']

tests/cats-n-dogs.sh

@@ -3,7 +3,7 @@
 # [[ $VERBOSE == 1 ]] && set -x
 
 HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"  
-cd $HERE/..
+cd $HERE
 
 if [[ -z $DOCKER_HOST ]]; then
 	host_arg=""
@@ -13,12 +13,15 @@ fi
 
 PROJECT=catsndogs
 DOCKER_LOG=/tmp/cats-n-dogs.log
-SPEC="--no-ansi -p $PROJECT -f docker-compose.yaml -f tests/cats-n-dogs.yaml"
-[[ $REBUILD == 1 ]] && BUILD_ARG=--build
+# --compatibility 
+SPEC="--no-ansi -p $PROJECT -f cats-n-dogs.yaml"
+[[ $REBUILD == 1 ]] && BUILD_ARG="--build"
 
 rm -f $DOCKER_LOG
 
 start() {
+	[[ -z $ANIMAL ]] && export ANIMAL=cat
+
 	if [[ $VERBOSE == 1 ]]; then
 		docker-compose $SPEC up $BUILD_ARG -d
 	else
@@ -29,12 +32,19 @@ start() {
 
 stop() {
 	if [[ $VERBOSE == 1 ]]; then
+		$HERE/show-stream-stat.sh
+		$HERE/show-log.sh
 		docker-compose $SPEC down -v --remove-orphans
 	else
+		#  --rmi local
 		docker-compose $SPEC down -v --remove-orphans >> $DOCKER_LOG 2>&1
 	fi
 }
 
+count() {
+	echo $(redis-cli $host_arg xlen cats | cat)
+}
+
 build() {
 	if [[ $VERBOSE == 1 ]]; then
 		docker-compose $SPEC build
@@ -45,13 +55,23 @@ build() {
 
 show_logs() {
 	docker-compose $SPEC logs $*
+	./show-log.sh
+	./show-stream-stat.sh
 }
 
 cats_demo() {
 	echo "Testing ${ANIMAL}s ..."
 	start
-	sleep 3
-	num_cats=$(redis-cli $host_arg xlen cats | cat)
+	for ((i = 0; i < 3; i++)); do
+		sleep 5
+		[[ $VERBOSE == 1 ]] && ./show-stream-stat.sh
+		num_cats=$(count)
+		if [[ $num_cats > 0 ]]; then
+			break
+		fi
+	done
+
+	num_cats=$(count)
 	if [[ $VERBOSE == 1 ]]; then
 		echo "num_cats=$num_cats"
 		show_logs
@@ -62,20 +82,27 @@ cats_demo() {
 	stop
 }
 
+help() {
+	echo "[ANIMAL=cat|dog] [VERBOSE=0|1] [REBUILD=0|1] $0 [start|stop|build|count|logs|help]"
+}
+
 cmd=$1
 shift
 if [[ $cmd == help ]]; then
-	echo "[ANIMAL=cat|dog] [VERBOSE=0|1] [REBUILD=0|1] $0 [start|stop|build|logs|help]"
+	help
 	exit 0
 elif [[ $cmd == start ]]; then
 	start
 elif [[ $cmd == stop ]]; then
 	stop
 elif [[ $cmd == build ]]; then
 	build
+elif [[ $cmd == count ]]; then
+	echo $(count)
+	exit 0
 elif [[ $cmd == logs ]]; then
 	show_logs $*
-else
+elif [[ $cmd == "" ]]; then
 	ANIMAL=cat cats_demo
 	if [[ -z $num_cats || $num_cats == 0 ]]; then
 		echo "cats: FAIL"
@@ -90,5 +117,8 @@ else
 	fi
 
 	echo "dogs: OK"
+else
+	help
+	exit 0
 fi
 exit 0

tests/cats-n-dogs.yaml

@@ -1,11 +1,24 @@
 version: '3'
 services:
+  redis:
+    build:
+      context: ../redis
+    ports:
+      - "6379:6379"
+    environment:
+      - MAX_IMAGES
+  app:
+    build: ../app
+    depends_on:
+      - redis
+    command: ['init.py', '--url', 'redis://redis:6379']
   camera:
     build:
-      context: camera
+      context: ../camera
       dockerfile: Dockerfile.debian
     depends_on:
       - redis
     environment:
       - ANIMAL
-    command: python ./read_camera.py -u redis://redis:6379 --test
+      - MAX_IMAGES
+    command: ['python3', '/usr/src/app/read_camera.py', '-u', 'redis://redis:6379', '--test']

tests/fold.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+
+import os
+import sys
+
+def to_int(x):
+	try:
+		return int(x)
+	except:
+		if x == '':
+			return 10
+		return -1
+
+if len(sys.argv) > 1:
+	fields = sys.argv[1].split(':')
+else:
+	fields = ['']
+fields = map(lambda x: to_int(x), fields)
+row = ""
+i = 0
+n = len(fields)
+for line in sys.stdin:
+	s = line.strip()
+	m = len(s)
+	c = fields[i]
+	i = i + 1
+	if c != -1:
+		if m > c:
+			m = c
+		s = s[0:m] + ' ' * (c - m)
+		if i < n:
+			s = s + ' '
+		row = row + s
+	if i == n:
+		print(row)
+		row = ''
+		i = 0

tests/show-log.sh

@@ -6,4 +6,4 @@ else
 	host_arg="-h $(echo $DOCKER_HOST|cut -d: -f1)"
 fi
 
-redis-cli $host_arg xrange log - + | cat
+redis-cli $host_arg xrange log - + | ./fold.py 16:-:50