Add proper all-in-one make job

This change makes it possibly to fire make all / make run
without any additional prerequisites required.
It instructs Makefile to launch docker compose for thirdparty
services, as well as instructs CGW container to communicate
with newly created containers.
Also generates self-signed certs, in case if these are missing.

- Tweak docker-compose files for thirdparty services to resign
  in a cgw-dedicated network, and use hostnames where needed;
- Change CGW app default network to cgw-dedicated network;
- Tweak PGSQL scripts to create tables in CGW DB, as well
  as change ownership to them.

Signed-off-by: Oleksandr Mazur <[email protected]>

Makefile

@@ -16,9 +16,9 @@ CGW_BUILD_ENV_IMG_TAG := $(shell cat Dockerfile | sha1sum | awk '{print substr($
 
 CGW_BUILD_ENV_IMG_CONTAINER_NAME := "cgw_build_env"
 
-.PHONY: all cgw-app cgw-build-env-img cgw-img stop clean run
+.PHONY: all cgw-app cgw-build-env-img cgw-img stop clean run run_docker_services
 
-all: cgw-build-env-img cgw-img
+all: cgw-build-env-img run_docker_services run
 	@echo "uCentral CGW build app (container) done"
 
 # Executed inside build-env
@@ -62,5 +62,9 @@ clean: stop
 	@docker rmi ${CGW_BUILD_ENV_IMG_ID}:${CGW_BUILD_ENV_IMG_TAG} >/dev/null 2>&1 || true
 	@echo Done!
 
-run: stop cgw-img
+run: stop cgw-img run_docker_services
 	@./run_cgw.sh "${CGW_IMG_ID}:${CGW_IMG_TAG}" ${CGW_IMG_CONTAINER_NAME}
+
+run_docker_services:
+	@cd ./utils/docker/ && docker compose up -d
+

README.md

@@ -4,7 +4,15 @@ CGW, like OWGW, manages device (Access Points and OpenLan switches) that impleme
 The main reasoning behind a new implementation of the GW is the horizontal scalability.
 # Dependencies (runtime)
 CGW requires a set of tools and services to operate and function. Some of them are embedded into the application itself and require no external utilities,
-while others are required to be running for the CGW to operate.
+while others are required to be running for the CGW to operate. 
+
+**NOTE**: while runtime CGW depends on services like kafka, redis and PGSQL, the *make* / *make all* targets
+would build a complete out-of-the-box setup with default configs and container params: 
+- Kafka, Redis, PGSQL containers would be created and attached to default - automatically created - *docker_cgw_network* network; 
+  All three (and one additional - *init-broker-container* - needed for kafka topics initialization) will be created as part of single 
+  container project group.
+- CGW will be created as separate standalone container, attached to same *docker_cgw_network* network;
+
 ## gRPC
 CGW utilizes gRPC to communicate with other CGW instances (referred to as Shards). This functionality does not depend on some external thirdparty services.
 ## Kafka
@@ -35,6 +43,7 @@ FOREIGN KEY(infra_group_id) REFERENCES infrastructure_groups(id) ON DELETE CASCA
 ## Redis
 fast in-memory DB that CGW uses to store all needed runtime information (InfraGroup assigned CGW id, remote CGW info - IP, gRPC port etc)
 # Building
+*NOTE:* The following target builds CGW and also starts up required services with default config and params
 ```console
 $ make all
 ```

run_cgw.sh

@@ -1,40 +1,40 @@
 #!/bin/bash
 
 DEFAULT_ID=0
-DEFAULT_LOG_LEVEL="info"
+DEFAULT_LOG_LEVEL="debug"
 DEFAULT_GROUPS_CAPACITY=1000
 DEFAULT_GROUPS_THRESHOLD=50
 DEFAULT_GROUP_INFRAS_CAPACITY=2000
 
 # By default - use default subnet's SRC ip to listen to gRPC requests
 DEFAULT_GRPC_LISTENING_IP="0.0.0.0"
 DEFAULT_GRPC_LISTENING_PORT=50051
-DEFAULT_GRPC_PUBLIC_HOST="localhost"
+DEFAULT_GRPC_PUBLIC_HOST="openlan_cgw"
 DEFAULT_GRPC_PUBLIC_PORT=50051
 
 # By default - listen to all interfaces
 DEFAULT_WSS_IP="0.0.0.0"
 DEFAULT_WSS_PORT=15002
 DEFAULT_WSS_T_NUM=4
 
-DEFAULT_CERTS_PATH="/etc/ssl/certs"
+DEFAULT_CERTS_PATH="`realpath ./utils/cert_generator/certs/server/`"
 DEFAULT_WSS_CAS="cas.pem"
 DEFAULT_WSS_CERT="cert.pem"
 DEFAULT_WSS_KEY="key.pem"
 
-DEFAULT_KAFKA_HOST="localhost"
+DEFAULT_KAFKA_HOST="docker-broker-1"
 DEFAULT_KAFKA_PORT=9092
 DEFAULT_KAFKA_CONSUME_TOPIC="CnC"
 DEFAULT_KAFKA_PRODUCE_TOPIC="CnC_Res"
 
-DEFAULT_DB_HOST="localhost"
+DEFAULT_DB_HOST="docker-postgresql-1"
 DEFAULT_DB_PORT=5432
 DEFAULT_DB_NAME="cgw"
 DEFAULT_DB_USER="cgw"
 DEFAULT_DB_PASW="123"
 DEFAULT_DB_TLS="no"
 
-DEFAULT_REDIS_HOST="localhost"
+DEFAULT_REDIS_HOST="docker-redis-1"
 DEFAULT_REDIS_PORT=6379
 DEFAULT_REDIS_TLS="no"
 
@@ -94,6 +94,24 @@ if [ -z "${!CGW_REDIS_PASSWORD}" ]; then
 	export CGW_REDIS_PASSWORD="${CGW_REDIS_PASSWORD}"
 fi
 
+if [ ! -f $CGW_CERTS_PATH/$CGW_WSS_CAS ] || [ ! -f $CGW_CERTS_PATH/$CGW_WSS_CERT ] || [ ! -f $CGW_CERTS_PATH/$CGW_WSS_KEY ] ; then
+	echo "WARNING: at specified path $CGW_CERTS_PATH either CAS, CERT or KEY is missing!"
+	echo "WARNING: changing source folder for certificates to default: $DEFAULT_CERTS_PATH and generating self-signed..."
+	export CGW_CERTS_PATH="$DEFAULT_CERTS_PATH";
+	export CGW_WSS_CAS="$DEFAULT_WSS_CAS"
+	export CGW_WSS_CERT="$DEFAULT_WSS_CERT"
+	export CGW_WSS_KEY="$DEFAULT_WSS_KEY"
+	export CGW_NB_INFRA_CERTS_PATH="$DEFAULT_CERTS_PATH"
+
+	cd ./utils/cert_generator/ && \
+		./generate_certs.sh -a && \
+		./generate_certs.sh -s && \
+		cp ./certs/ca/ca.crt $DEFAULT_CERTS_PATH/cas.pem
+		cp ./certs/server/gw.crt $DEFAULT_CERTS_PATH/cert.pem && \
+		cp ./certs/server/gw.key $DEFAULT_CERTS_PATH/key.pem && \
+		echo "Generating self-signed certificates done!"
+fi
+
 echo "Starting CGW..."
 echo "CGW LOG LEVEL                     : $CGW_LOG_LEVEL"
 echo "CGW ID                            : $CGW_ID"
@@ -122,6 +140,8 @@ echo "CGW UCENTRAL AP DATAMODEL URI     : $CGW_UCENTRAL_AP_DATAMODEL_URI"
 echo "CGW UCENTRAL SWITCH DATAMODEL URI : $CGW_UCENTRAL_SWITCH_DATAMODEL_URI"
 
 docker run \
+	-p 15002:15002 \
+	-p 50051:50051 \
 	--cap-add=SYS_PTRACE --security-opt seccomp=unconfined        \
 	-v $CGW_CERTS_PATH:$CONTAINTER_CERTS_VOLUME                   \
 	-v $CGW_NB_INFRA_CERTS_PATH:$CONTAINTER_NB_INFRA_CERTS_VOLUME \
@@ -161,4 +181,4 @@ docker run \
 	-e CGW_NB_INFRA_TLS                  \
 	-e CGW_UCENTRAL_AP_DATAMODEL_URI     \
 	-e CGW_UCENTRAL_SWITCH_DATAMODEL_URI \
-	-d -t --network=host --name $2 $1 ucentral-cgw
+	-d -t --network=docker_cgw_network --name $2 $1 ucentral-cgw

utils/docker/docker-compose.yml

@@ -12,9 +12,9 @@ services:
       - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
       - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
       - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT
-      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9092,EXTERNAL://kafka_b:9094
+      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://docker-broker-1:9092,EXTERNAL://kafka_b:9094
       - KAFKA_BROKER_ID=1
-      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@127.0.0.1:9093
+      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@docker-broker-1:9093
       - ALLOW_PLAINTEXT_LISTENER=yes
       - KAFKA_CFG_NODE_ID=1
       - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
@@ -26,7 +26,8 @@ services:
       timeout: 5s
       retries: 3
       start_period: 5s
-
+    networks:
+      - cgw_network
 
   postgresql:
     image: "postgres:latest"
@@ -44,10 +45,34 @@ services:
     restart: always
     volumes:
       - ./postgresql/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh
+    networks:
+      - cgw_network
 
   redis:
     image: 'bitnami/redis:latest'
     ports:
       - "6379:6379"
     environment:
       - ALLOW_EMPTY_PASSWORD=yes
+    networks:
+      - cgw_network
+  init-broker-container:
+    image: docker.io/bitnami/kafka:latest
+    depends_on:
+      - broker
+    entrypoint: [ '/bin/sh', '-c' ]
+    command: |
+      "
+      # rather than giving sleep 15 use this
+      # to block init container to wait for Kafka broker to be ready
+      kafka-topics --bootstrap-server broker:9092 --list
+
+      # create CnC and CnC_Res topics
+      kafka-topics.sh --create --partitions 2 --bootstrap-server broker:9092 --topic CnC
+      kafka-topics.sh --create --bootstrap-server broker:9092 --partitions 2 --topic CnC_Res
+      "
+    networks:
+      - cgw_network
+
+networks:
+  cgw_network:

utils/docker/postgresql/init-db.sh

@@ -4,6 +4,10 @@ set -e
 psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL
     CREATE USER $CGW_DB_USER WITH ENCRYPTED PASSWORD '$CGW_DB_PASSWORD';
     CREATE DATABASE $CGW_DB OWNER $CGW_DB_USER;
+    \c $CGW_DB;
     CREATE TABLE infrastructure_groups ( id INT PRIMARY KEY, reserved_size INT, actual_size INT);
     CREATE TABLE infras ( mac MACADDR PRIMARY KEY, infra_group_id INT, FOREIGN KEY(infra_group_id) REFERENCES infrastructure_groups(id) ON DELETE CASCADE);
+    ALTER DATABASE $CGW_DB OWNER TO $CGW_DB_USER;
+    ALTER TABLE infrastructure_groups OWNER TO $CGW_DB_USER;
+    ALTER TABLE infras OWNER TO $CGW_DB_USER;
 EOSQL