diff --git a/README.md b/README.md index d0e017b6..7a24e56a 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,23 @@ docker build -f x86-openvino.Dockerfile -t "neuralet/smart-social-distancing:lat docker run -it -p HOST_PORT:8000 -v "$PWD/data":/repo/data -v "$PWD/config-x86-openvino.ini":/repo/config-x86-openvino.ini -v "$PWD/certificates/processor":/repo/certs -e TZ=`./timezone.sh` neuralet/smart-social-distancing:latest-x86_64_openvino ``` +##### Run on AMD node with a connected Coral USB Accelerator with Adaptive Learning Support +Currently Smart Social Distancing supports Adaptive Learning framework on X86 nodes with connected USB TPU. For more information about Adaptive Learning visit [this page](https://github.com/neuralet/neuralet/tree/master/applications/adaptive-learning) +Important notes: +1. You must install [Docker Compose](https://github.com/docker/compose) to be able to use Adaptive Learning. +2. At this time Adaptive Learning is only compatible with `mobilenet_ssd_v2` models. + +```bash +# Download model first: +./download_x86_model.sh + +# 1) Configure adaptive-learing/configs/iterdet.ini file. (you may only need to configure video VideoPath, BatchSize and Epochs) +# 2) Configure the HOST_PORT in docker-compose-adaptive-learning-usbtpu.yml +# 3) Run the Docker Compose +docker-compose -f docker-compose-adaptive-learning-usbtpu.yml up --build +``` + + ### Configurations You can read and modify the configurations in `config-*.ini` files, accordingly: diff --git a/adaptive-learning/README.md b/adaptive-learning/README.md new file mode 100644 index 00000000..770b4fd7 --- /dev/null +++ b/adaptive-learning/README.md @@ -0,0 +1,4 @@ +# Adaptive Learning of Pedestrian Detection Models + +**This module is a copy of Neuralet's Adaptive Learning Framework which helps to adapt the pedestrian detection models to each environment. For more information visit [here](https://github.com/neuralet/neuralet/tree/master/applications/adaptive-learning)** + diff --git a/adaptive-learning/__init__.py b/adaptive-learning/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/adaptive-learning/configs/__init__.py b/adaptive-learning/configs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/adaptive-learning/configs/iterdet.ini b/adaptive-learning/configs/iterdet.ini new file mode 100644 index 00000000..b6cf4bbc --- /dev/null +++ b/adaptive-learning/configs/iterdet.ini @@ -0,0 +1,31 @@ +[Teacher] +Name: iterdet +;ImageSize should be 3 numbers seperated by commas, no spaces: 300,300,3 +ImageSize: 704,576,3 +;Pedestrian class ID of the teacher detector +ClassID: 1 +;Score threshold of detector +MinScore: 0.5 +VideoPath: /repo/data/softbio_vid.mp4 +;Directory that stores the video frames and annotations +DataDir: /repo/adaptive-learning/data +ImagePath: /repo/adaptive-learning/data/images +XmlPath: /repo/adaptive-learning/data/xmls +;Maximum number of images that can be stored in hard simultaneously +MaxAllowedImage: 10000 +;Minimum number of instance in each frame to store the frame +MinDetectionPerFrame: 1 +;frequency of saving frames +SaveFrequency: 1 +Device: GPU +PostProcessing: " " +ImageFeature: " " +[Student] +BatchSize: 4 +Epochs: 20 +ValidationSplit: .17 +;how many data picks for training in each round +ExamplePerRound: 2000 +TrainingRounds: 10000 +QuantizationAware: true + diff --git a/adaptive-learning/data/.keep b/adaptive-learning/data/.keep new file mode 100644 index 00000000..e69de29b diff --git a/adaptive-learning/docker-compose.yml b/adaptive-learning/docker-compose.yml new file mode 100644 index 00000000..3414b49c --- /dev/null +++ b/adaptive-learning/docker-compose.yml @@ -0,0 +1,32 @@ +version: '2.3' +services: + teacher: + build: + context: ./teachers/ + dockerfile: iterdet-teacher.Dockerfile + stdin_open: true + tty: true + container_name: teacher-container + volumes: + - ../../:/repo + + runtime: nvidia + environment: + - NVIDIA_VISIBLE_DEVICES=0 + + student: + build: + context: ./students/ + dockerfile: ssd-mobilenet-v2-student.Dockerfile + stdin_open: true + tty: true + container_name: student-container + volumes: + - ../../:/repo + ports: + - "2029:6006" + runtime: nvidia + environment: + - NVIDIA_VISIBLE_DEVICES=0 + - PYTHONPATH=$PYTHONPATH:/models/research:/models/research/slim + - TF_FORCE_GPU_ALLOW_GROWTH=true diff --git a/adaptive-learning/download_sample_video.sh b/adaptive-learning/download_sample_video.sh new file mode 100644 index 00000000..8cc91e81 --- /dev/null +++ b/adaptive-learning/download_sample_video.sh @@ -0,0 +1 @@ +wget https://social-distancing-data.s3.amazonaws.com/softbio_vid.mp4 -O data/softbio_vid.mp4 diff --git a/adaptive-learning/students/__init__.py b/adaptive-learning/students/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/adaptive-learning/students/apply_configs.py b/adaptive-learning/students/apply_configs.py new file mode 100644 index 00000000..2ebff494 --- /dev/null +++ b/adaptive-learning/students/apply_configs.py @@ -0,0 +1,45 @@ +import argparse +import os +import sys +from pathlib import Path +import logging + +sys.path.append("../..") +from libs.config_engine import ConfigEngine +import tensorflow as tf +from google.protobuf import text_format +from object_detection.protos import pipeline_pb2 + + +def main(args): + """ + This script will change some items TensorFlow Object Detection API training config file base on adaptive learning + config.ini + """ + pipeline_config = pipeline_pb2.TrainEvalPipelineConfig() + + with tf.gfile.GFile(args.pipeline, "r") as f: + proto_str = f.read() + text_format.Merge(proto_str, pipeline_config) + config = ConfigEngine(args.config) + pipeline_config.train_config.batch_size = int(config.get_section_dict('Student')['BatchSize']) + data_dir = Path(config.get_section_dict('Teacher')['ImagePath']).parent + pipeline_config.train_input_reader.tf_record_input_reader.input_path[:] = [os.path.join(str(data_dir), + "tfrecords", + "train.record")] + pipeline_config.eval_input_reader[:][0].tf_record_input_reader.input_path[:] = [os.path.join(str(data_dir), + "tfrecords", + "val.record")] + + config_text = text_format.MessageToString(pipeline_config) + with tf.gfile.Open(args.pipeline, "wb") as f: + f.write(config_text) + logging.info("The config pipeline has been changes") + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='') + parser.add_argument('--pipeline', required=True, type=str) + parser.add_argument('--config', required=True, type=str) + args = parser.parse_args() + main(args) diff --git a/adaptive-learning/students/create_tfrecord.py b/adaptive-learning/students/create_tfrecord.py new file mode 100644 index 00000000..ce799e81 --- /dev/null +++ b/adaptive-learning/students/create_tfrecord.py @@ -0,0 +1,232 @@ +# This script is a modification of TensorFlow sample file. you can reach the file here: +# https://github.com/tensorflow/models/blob/master/research/object_detection/dataset_tools/create_pet_tf_record.py +# license of original implementation is Apache V2 +# ============================================================================== + +r"""Convert the teacher generated dataset to TFRecord for student's training process and delete corresponding images and +annotations. +See: https://www.robots.ox.ac.uk/ActiveVision/Research/Projects/2009bbenfold_headpose/project.html +Example usage: + python create_tfrecord.py \ + --data_dir $DATASET_DIR \ + --output_dir $DATASET_DIR \ + --label_map_path ./label_map.pbtxt \ + --validation_split 0.25 + --num_of_images_per_round 2500 +""" + +import glob +import hashlib +import io +import logging +import os +import random +import time + +import PIL.Image +import tensorflow as tf +from lxml import etree +from object_detection.utils import dataset_util +from object_detection.utils import label_map_util + +flags = tf.app.flags +flags.DEFINE_string('data_dir', '', 'Root directory to raw pet dataset.') +flags.DEFINE_string('output_dir', '', 'Path to directory to output TFRecords.') +flags.DEFINE_string('label_map_path', 'data/pet_label_map.pbtxt', + 'Path to label map proto') +flags.DEFINE_float('validation_split', 0.25, 'Percentage of Validation Set') +flags.DEFINE_integer('num_of_images_per_round', 100, 'number of frames for training in each round') + +FLAGS = flags.FLAGS + + +def dict_to_tf_example(data, + label_map_dict, + image_subdirectory): + """Convert XML derived dict to tf.Example proto. + Notice that this function normalizes the bounding box coordinates provided + by the raw data. + Args: + data: dict holding PASCAL XML fields for a single image (obtained by + running dataset_util.recursive_parse_xml_to_dict) + label_map_dict: A map from string label names to integers ids. + image_subdirectory: String specifying subdirectory within the + Pascal dataset directory holding the actual image data. + Returns: + example: The converted tf.Example. + Raises: + ValueError: if the image pointed to by data['filename'] is not a valid JPEG + """ + img_path = os.path.join(image_subdirectory, data['filename']) + with tf.gfile.GFile(img_path, 'rb') as fid: + encoded_jpg = fid.read() + encoded_jpg_io = io.BytesIO(encoded_jpg) + image = PIL.Image.open(encoded_jpg_io) + if image.format != 'JPEG': + raise ValueError('Image format not JPEG') + key = hashlib.sha256(encoded_jpg).hexdigest() + width = int(data['size']['width']) + height = int(data['size']['height']) + + xmins = [] + ymins = [] + xmaxs = [] + ymaxs = [] + classes = [] + classes_text = [] + truncated = [] + poses = [] + difficult_obj = [] + if 'object' in data: + for obj in data['object']: + difficult_obj.append(int(0)) + xmin = float(obj['bndbox']['xmin']) + xmax = float(obj['bndbox']['xmax']) + ymin = float(obj['bndbox']['ymin']) + ymax = float(obj['bndbox']['ymax']) + xmins.append(xmin / width) + ymins.append(ymin / height) + xmaxs.append(xmax / width) + ymaxs.append(ymax / height) + class_name = obj["name"] + classes_text.append(class_name.encode('utf8')) + classes.append(label_map_dict[class_name]) + truncated.append(int(0)) + poses.append('Unspecified'.encode('utf8')) + + feature_dict = { + 'image/height': dataset_util.int64_feature(height), + 'image/width': dataset_util.int64_feature(width), + 'image/filename': dataset_util.bytes_feature( + data['filename'].encode('utf8')), + 'image/source_id': dataset_util.bytes_feature( + data['filename'].encode('utf8')), + 'image/key/sha256': dataset_util.bytes_feature(key.encode('utf8')), + 'image/encoded': dataset_util.bytes_feature(encoded_jpg), + 'image/format': dataset_util.bytes_feature('jpeg'.encode('utf8')), + 'image/object/bbox/xmin': dataset_util.float_list_feature(xmins), + 'image/object/bbox/xmax': dataset_util.float_list_feature(xmaxs), + 'image/object/bbox/ymin': dataset_util.float_list_feature(ymins), + 'image/object/bbox/ymax': dataset_util.float_list_feature(ymaxs), + 'image/object/class/text': dataset_util.bytes_list_feature(classes_text), + 'image/object/class/label': dataset_util.int64_list_feature(classes), + 'image/object/difficult': dataset_util.int64_list_feature(difficult_obj), + 'image/object/truncated': dataset_util.int64_list_feature(truncated), + 'image/object/view': dataset_util.bytes_list_feature(poses), + } + example = tf.train.Example(features=tf.train.Features(feature=feature_dict)) + return example + + +def create_tf_record(output_filename, + label_map_dict, + annotations_dir, + image_dir, + examples): + """Creates a TFRecord file from examples. + Args: + output_filename: Path to where output file is saved. + label_map_dict: The label map dictionary. + annotations_dir: Directory where annotation files are stored. + image_dir: Directory where image files are stored. + examples: Examples to parse and save to tf record. + """ + writer = tf.python_io.TFRecordWriter(output_filename) + for idx, example in enumerate(examples): + if idx % 100 == 0: + logging.info('On image %d of %d', idx, len(examples)) + xml_path = os.path.join(annotations_dir, str(example) + '.xml') + + if not os.path.exists(xml_path): + logging.warning('Could not find %s, ignoring example.', xml_path) + continue + with tf.gfile.GFile(xml_path, 'r') as fid: + xml_str = fid.read() + xml = etree.fromstring(xml_str) + data = dataset_util.recursive_parse_xml_to_dict(xml)['annotation'] + + try: + tf_example = dict_to_tf_example( + data, + label_map_dict, + image_dir) + if tf_example: + writer.write(tf_example.SerializeToString()) + except ValueError: + logging.warning('Invalid example: %s, ignoring.', xml_path) + writer.close() + + +def select_items(directory, num_of_images, max_retry=100): + """ + select appropriate number of images from teacher's data directory and return list of selected images. + Args: + directory: directory to search + num_of_images: how many images to select + max_retry: how many times tries if the number of existing images on the directory are less than num_of_images + + Returns: + a list that contains selected images name. + """ + tries = 0 + while True: + + images_list = sorted( + list(map(lambda x: int((x.split(".")[0]).split("/")[-1]), glob.glob(directory + "/*.jpg")))) + print(directory) + if len(images_list) >= num_of_images: + # Wait a while for the data to be stored properly + time.sleep(5) + logging.info("{} images picked for training".format(num_of_images)) + return images_list[:num_of_images] + else: + if tries >= max_retry: + raise StopIteration("The maximum retry has been reached") + logging.info( + "There is not {} images on the {} directory. try number {} from {}".format(num_of_images, directory, + tries + 1, max_retry)) + tries += 1 + time.sleep(300) + + +def main(_): + data_dir = FLAGS.data_dir + validation_split = FLAGS.validation_split + label_map_dict = label_map_util.get_label_map_dict(FLAGS.label_map_path) + + logging.info('Reading from Teacher Generated Dataset.') + image_dir = os.path.join(data_dir, 'images') + annotations_dir = os.path.join(data_dir, 'xmls') + examples_list = select_items(image_dir, FLAGS.num_of_images_per_round) + + random.seed(42) + num_examples = len(examples_list) + num_train = int((1 - validation_split) * num_examples) + train_examples = examples_list[:num_train] + val_examples = examples_list[num_train:] + random.shuffle(train_examples) + logging.info('%d training and %d validation examples.', + len(train_examples), len(val_examples)) + + train_output_path = os.path.join(FLAGS.output_dir, 'train.record') + val_output_path = os.path.join(FLAGS.output_dir, 'val.record') + create_tf_record( + train_output_path, + label_map_dict, + annotations_dir, + image_dir, + train_examples) + create_tf_record( + val_output_path, + label_map_dict, + annotations_dir, + image_dir, + val_examples) + + for file_number in examples_list: + os.remove(os.path.join(image_dir, str(file_number) + ".jpg")) + os.remove(os.path.join(annotations_dir, str(file_number) + ".xml")) + + +if __name__ == '__main__': + tf.app.run() diff --git a/adaptive-learning/students/label_map.pbtxt b/adaptive-learning/students/label_map.pbtxt new file mode 100644 index 00000000..d61e619d --- /dev/null +++ b/adaptive-learning/students/label_map.pbtxt @@ -0,0 +1,4 @@ +item { + id: 1 + name: 'pedestrian' +} diff --git a/adaptive-learning/students/ssd-mobilenet-v2-student.Dockerfile b/adaptive-learning/students/ssd-mobilenet-v2-student.Dockerfile new file mode 100644 index 00000000..bd541bdd --- /dev/null +++ b/adaptive-learning/students/ssd-mobilenet-v2-student.Dockerfile @@ -0,0 +1,31 @@ +#This container will install TensorFlow Object Detection API and its dependencies in the /model/research/object_detection directory + +FROM tensorflow/tensorflow:1.15.0-gpu-py3 +VOLUME /repo +RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y git protobuf-compiler python3-tk vim wget + +RUN pip install Cython && \ + pip install contextlib2 && \ + pip install pillow && \ + pip install lxml && \ + pip install jupyter && \ + pip install matplotlib + +RUN git clone https://github.com/tensorflow/models.git && cd models && \ + git checkout 02c7112eb7ff0aed28d8d508708b3fb3a9c9c01f && cd ../ + +RUN git clone --depth 1 https://github.com/cocodataset/cocoapi.git && \ + cd cocoapi/PythonAPI && \ + make && \ + cp -r pycocotools/ /models/research/ + +RUN cd /models/research && \ + protoc object_detection/protos/*.proto --python_out=. + +RUN echo 'export PYTHONPATH=$PYTHONPATH:/models/research:/models/research/slim' >> ~/.bashrc && \ + echo 'export export TF_FORCE_GPU_ALLOW_GROWTH=true' >> ~/.bashrc && \ + source ~/.bashrc + +WORKDIR /repo/adaptive-learning/students +ENTRYPOINT ["bash", "./train_student.sh"] +CMD ["-c", "/repo/adaptive-learning/configs/iterdet.ini"] diff --git a/adaptive-learning/students/ssd_mobilenet_v2_pedestrian.config b/adaptive-learning/students/ssd_mobilenet_v2_pedestrian.config new file mode 100644 index 00000000..113938d1 --- /dev/null +++ b/adaptive-learning/students/ssd_mobilenet_v2_pedestrian.config @@ -0,0 +1,176 @@ +model { + ssd { + num_classes: 1 + image_resizer { + fixed_shape_resizer { + height: 300 + width: 300 + } + } + feature_extractor { + type: "ssd_mobilenet_v2" + depth_multiplier: 1.0 + min_depth: 16 + conv_hyperparams { + regularizer { + l2_regularizer { + weight: 3.9999999e-05 + } + } + initializer { + truncated_normal_initializer { + mean: 0.0 + stddev: 0.029999999 + } + } + activation: RELU_6 + batch_norm { + decay: 0.99970001 + center: true + scale: true + epsilon: 0.001 + train: false + } + } + } + box_coder { + faster_rcnn_box_coder { + y_scale: 10.0 + x_scale: 10.0 + height_scale: 5.0 + width_scale: 5.0 + } + } + matcher { + argmax_matcher { + matched_threshold: 0.5 + unmatched_threshold: 0.5 + ignore_thresholds: false + negatives_lower_than_unmatched: true + force_match_for_each_row: true + } + } + similarity_calculator { + iou_similarity { + } + } + box_predictor { + convolutional_box_predictor { + conv_hyperparams { + regularizer { + l2_regularizer { + weight: 3.9999999e-05 + } + } + initializer { + truncated_normal_initializer { + mean: 0.0 + stddev: 0.029999999 + } + } + activation: RELU_6 + batch_norm { + decay: 0.99970001 + center: true + scale: true + epsilon: 0.001 + train: false + } + } + min_depth: 0 + max_depth: 0 + num_layers_before_predictor: 0 + use_dropout: false + dropout_keep_probability: 0.80000001 + kernel_size: 1 + box_code_size: 4 + apply_sigmoid_to_scores: false + } + } + anchor_generator { + ssd_anchor_generator { + num_layers: 6 + min_scale: 0.2 + max_scale: 0.94999999 + aspect_ratios: 1.0 + aspect_ratios: 2.0 + aspect_ratios: 0.5 + aspect_ratios: 3.0 + aspect_ratios: 0.33329999 + } + } + post_processing { + batch_non_max_suppression { + score_threshold: 9.9999999e-09 + iou_threshold: 0.60000002 + max_detections_per_class: 30 + max_total_detections: 30 + } + score_converter: SIGMOID + } + normalize_loss_by_num_matches: true + loss { + localization_loss { + weighted_smooth_l1 { + } + } + classification_loss { + weighted_sigmoid { + } + } + hard_example_miner { + num_hard_examples: 3000 + iou_threshold: 0.99000001 + loss_type: CLASSIFICATION + max_negatives_per_positive: 3 + min_negatives_per_image: 3 + } + classification_weight: 1.0 + localization_weight: 1.0 + } + } +} +train_config { + batch_size: 2 + data_augmentation_options { + random_horizontal_flip { + } + } + data_augmentation_options { + ssd_random_crop { + } + } + optimizer { + rms_prop_optimizer { + learning_rate { + constant_learning_rate { + learning_rate: 0.0020000001 + } + } + momentum_optimizer_value: 0.89999998 + decay: 0.89999998 + epsilon: 1.0 + } + } + fine_tune_checkpoint: "/repo/adaptive-learning/data/pretrained_models/ssd_mobilenet_v2_coco_2018_03_29/model.ckpt" + num_steps: 200000 + fine_tune_checkpoint_type: "detection" +} +train_input_reader { + label_map_path: "/repo/adaptive-learning/students/label_map.pbtxt" + tf_record_input_reader { + input_path: "/repo/adaptive-learning/data/tfrecords/train.record" + } +} +eval_config { + num_examples: 8000 + eval_interval_secs: 60 +} +eval_input_reader { + label_map_path: "/repo/adaptive-learning/students/label_map.pbtxt" + shuffle: false + num_readers: 1 + tf_record_input_reader { + input_path: "/repo/adaptive-learning/data/tfrecords/val.record" + } +} diff --git a/adaptive-learning/students/ssd_mobilenet_v2_pedestrian_quant.config b/adaptive-learning/students/ssd_mobilenet_v2_pedestrian_quant.config new file mode 100644 index 00000000..031ecd5d --- /dev/null +++ b/adaptive-learning/students/ssd_mobilenet_v2_pedestrian_quant.config @@ -0,0 +1,183 @@ +model { + ssd { + num_classes: 1 + image_resizer { + fixed_shape_resizer { + height: 300 + width: 300 + } + } + feature_extractor { + type: "ssd_mobilenet_v2" + depth_multiplier: 1.0 + min_depth: 16 + conv_hyperparams { + regularizer { + l2_regularizer { + weight: 3.9999999e-05 + } + } + initializer { + truncated_normal_initializer { + mean: 0.0 + stddev: 0.029999999 + } + } + activation: RELU_6 + batch_norm { + decay: 0.99970001 + center: true + scale: true + epsilon: 0.001 + train: true + } + } + } + box_coder { + faster_rcnn_box_coder { + y_scale: 10.0 + x_scale: 10.0 + height_scale: 5.0 + width_scale: 5.0 + } + } + matcher { + argmax_matcher { + matched_threshold: 0.5 + unmatched_threshold: 0.5 + ignore_thresholds: false + negatives_lower_than_unmatched: true + force_match_for_each_row: true + } + } + similarity_calculator { + iou_similarity { + } + } + box_predictor { + convolutional_box_predictor { + conv_hyperparams { + regularizer { + l2_regularizer { + weight: 3.9999999e-05 + } + } + initializer { + truncated_normal_initializer { + mean: 0.0 + stddev: 0.029999999 + } + } + activation: RELU_6 + batch_norm { + decay: 0.99970001 + center: true + scale: true + epsilon: 0.001 + train: true + } + } + min_depth: 0 + max_depth: 0 + num_layers_before_predictor: 0 + use_dropout: false + dropout_keep_probability: 0.80000001 + kernel_size: 1 + box_code_size: 4 + apply_sigmoid_to_scores: false + } + } + anchor_generator { + ssd_anchor_generator { + num_layers: 6 + min_scale: 0.2 + max_scale: 0.94999999 + aspect_ratios: 1.0 + aspect_ratios: 2.0 + aspect_ratios: 0.5 + aspect_ratios: 3.0 + aspect_ratios: 0.33329999 + } + } + post_processing { + batch_non_max_suppression { + score_threshold: 9.9999999e-09 + iou_threshold: 0.60000002 + max_detections_per_class: 30 + max_total_detections: 30 + } + score_converter: SIGMOID + } + normalize_loss_by_num_matches: true + loss { + localization_loss { + weighted_smooth_l1 { + } + } + classification_loss { + weighted_sigmoid { + } + } + hard_example_miner { + num_hard_examples: 3000 + iou_threshold: 0.99000001 + loss_type: CLASSIFICATION + max_negatives_per_positive: 3 + min_negatives_per_image: 3 + } + classification_weight: 1.0 + localization_weight: 1.0 + } + } +} +train_config { + batch_size: 2 + data_augmentation_options { + random_horizontal_flip { + } + } + data_augmentation_options { + ssd_random_crop { + } + } + optimizer { + rms_prop_optimizer { + learning_rate { + constant_learning_rate { + learning_rate: 0.0020000001 + } + } + momentum_optimizer_value: 0.89999998 + decay: 0.89999998 + epsilon: 1.0 + } + } + fine_tune_checkpoint: "/repo/adaptive-learning/data/pretrained_models/ssd_mobilenet_v2_coco_2018_03_29/model.ckpt" + num_steps: 200000 + fine_tune_checkpoint_type: "detection" +} +train_input_reader { + label_map_path: "/repo/adaptive-learning/students/label_map.pbtxt" + tf_record_input_reader { + input_path: "/repo/adaptive-learning/data/tfrecords/train.record" + } +} +eval_config { + num_examples: 8000 + eval_interval_secs: 60 +} +eval_input_reader { + label_map_path: "/repo/adaptive-learning/students/label_map.pbtxt" + shuffle: false + num_readers: 1 + tf_record_input_reader { + input_path: "/repo/adaptive-learning/data/tfrecords/val.record" + } +} +graph_rewriter { + quantization { + delay: 15000 + weight_bits: 8 + activation_bits: 8 + } +} diff --git a/adaptive-learning/students/train_student.sh b/adaptive-learning/students/train_student.sh new file mode 100644 index 00000000..48d7e968 --- /dev/null +++ b/adaptive-learning/students/train_student.sh @@ -0,0 +1,123 @@ +#!/bin/bash + +set -e + +helpFunction() +{ + echo "" + echo "Usage: $0 -c [An adaptive learning config file]" + exit 1 # Exit script after printing help +} + +while getopts "c:h:" opt +do + # shellcheck disable=SC2220 + case "$opt" in + + c ) CONFIG_FILE="$OPTARG" ;; + h ) helpFunction ;; # Print helpFunction in case parameter is non-existent + esac +done + +# Print helpFunction in case parameters are empty +if [ -z "$CONFIG_FILE" ] +then + echo "Some or all of the parameters are empty"; + helpFunction +fi + +# Begin script in case all parameters are correct +echo "=====================================================" +echo "Adaptive Learning Process Started..." +echo "=====================================================" +sleep 5 +MODEL_DIR="/repo/adaptive-learning/data/student_model" +EXPORT_DIR=$MODEL_DIR/frozen_graph +mkdir -p $MODEL_DIR +mkdir -p $EXPORT_DIR + +Quantization_Aware=$(sed -n -e 's/^\s*QuantizationAware\s*:\s*//p' "$CONFIG_FILE") +if $Quantization_Aware +then + TRAINING_PIPELINE_FILE="/repo/adaptive-learning/students/ssd_mobilenet_v2_pedestrian_quant.config" +else + TRAINING_PIPELINE_FILE="/repo/adaptive-learning/students/ssd_mobilenet_v2_pedestrian.config" +fi + +python apply_configs.py --config "$CONFIG_FILE" --pipeline $TRAINING_PIPELINE_FILE + +# extrat config items +TRAINING_ROUNDS=$(sed -n -e 's/^\s*TrainingRounds\s*:\s*//p' "$CONFIG_FILE") +DATASET_DIR=$(sed -n -e 's/^\s*DataDir\s*:\s*//p' "$CONFIG_FILE") +BATCHSIZE=$(sed -n -e 's/^\s*BatchSize\s*:\s*//p' "$CONFIG_FILE") +EPOCHS=$(sed -n -e 's/^\s*Epochs\s*:\s*//p' "$CONFIG_FILE") +EXAMPLE_PER_ROUND=$(sed -n -e 's/^\s*ExamplePerRound\s*:\s*//p' "$CONFIG_FILE") +Validation_Split=$(sed -n -e 's/^\s*ValidationSplit\s*:\s*//p' "$CONFIG_FILE") + +NUM_TRAIN_STEPS=$(( EXAMPLE_PER_ROUND * EPOCHS / BATCHSIZE )) + +# download pretrained checkpoint if it does not exist +PRETRAINED_MODELS_DIR="/repo/adaptive-learning/data/pretrained_models" +mkdir -p $PRETRAINED_MODELS_DIR +if [ ! -f $PRETRAINED_MODELS_DIR/ssd_mobilenet_v2_coco_2018_03_29.tar.gz ] +then + echo "The Pretrained checkpoints are not exists" + echo "Start Downloading Pretrained Checkpoits" + wget http://download.tensorflow.org/models/object_detection/ssd_mobilenet_v2_coco_2018_03_29.tar.gz \ + -O $PRETRAINED_MODELS_DIR/ssd_mobilenet_v2_coco_2018_03_29.tar.gz + tar -xzvf $PRETRAINED_MODELS_DIR/ssd_mobilenet_v2_coco_2018_03_29.tar.gz -C $PRETRAINED_MODELS_DIR +fi + + +TFRECORD_DIR="/repo/adaptive-learning/data/tfrecords" +mkdir -p $TFRECORD_DIR +tensorboard --logdir $MODEL_DIR & +# TODO: add infinite loop option +for ((i=1;i<=TRAINING_ROUNDS;i++)) +do + echo "============Round $i of Training Started=================" + + echo "==================================================================================" + echo "Start Creating TFrecords ..." + echo "==================================================================================" + python create_tfrecord.py --data_dir "$DATASET_DIR" \ + --output_dir $TFRECORD_DIR \ + --label_map_path "/repo/adaptive-learning/students/label_map.pbtxt" \ + --validation_split $Validation_Split \ + --num_of_images_per_round "$EXAMPLE_PER_ROUND" + echo "==================================================================================" + echo "Start Training ..." + echo "==================================================================================" + NUM_TRAIN_STEPS_NEW=$(( NUM_TRAIN_STEPS * i )) + python /models/research/object_detection/model_main.py --pipeline_config_path=$TRAINING_PIPELINE_FILE \ + --model_dir=$MODEL_DIR \ + --num_train_steps=$NUM_TRAIN_STEPS_NEW \ + --sample_1_of_n_eval_examples=1 \ + --alsologtostderr 2>&1 | tee training_log.txt + CHECKPOINT=$(ls -t $MODEL_DIR|grep ckpt|head -n 1 | tr -dc '0-9') + TRAINED_CKPT_PREFIX=$MODEL_DIR/model.ckpt-$CHECKPOINT + + echo "==================================================================================" + echo "Start Exporting Checkpoint to Frozen Graph ..." + echo "==================================================================================" + rm -rf $EXPORT_DIR/* + if $Quantization_Aware + then + python /models/research/object_detection/export_tflite_ssd_graph.py \ + --pipeline_config_path=$TRAINING_PIPELINE_FILE \ + --trained_checkpoint_prefix="$TRAINED_CKPT_PREFIX" \ + --output_directory=$EXPORT_DIR \ + --max_detections=50 \ + --add_postprocessing_op=true + /usr/local/bin/toco --graph_def_file "$EXPORT_DIR/tflite_graph.pb" --output_format TFLITE --input_shapes 1,300,300,3 --input_arrays normalized_input_image_tensor \ + --output_arrays "TFLite_Detection_PostProcess,TFLite_Detection_PostProcess:1,TFLite_Detection_PostProcess:2,TFLite_Detection_PostProcess:3" \ + --inference_type QUANTIZED_UINT8 --mean_values 128 --std_dev_values 128 --change_concat_input_ranges false \ + --allow_custom_op --output_file "$EXPORT_DIR/detect.tflite" + + else + python /models/research/object_detection/export_inference_graph.py --input_type=image_tensor \ + --pipeline_config_path=$TRAINING_PIPELINE_FILE \ + --trained_checkpoint_prefix="$TRAINED_CKPT_PREFIX" \ + --output_directory=$EXPORT_DIR + fi +done diff --git a/adaptive-learning/teachers/__init__.py b/adaptive-learning/teachers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/adaptive-learning/teachers/iterdet-teacher.Dockerfile b/adaptive-learning/teachers/iterdet-teacher.Dockerfile new file mode 100644 index 00000000..0d2d585e --- /dev/null +++ b/adaptive-learning/teachers/iterdet-teacher.Dockerfile @@ -0,0 +1,31 @@ +ARG PYTORCH="1.5" +ARG CUDA="10.1" +ARG CUDNN="7" + +FROM pytorch/pytorch:${PYTORCH}-cuda${CUDA}-cudnn${CUDNN}-devel + +ENV TORCH_CUDA_ARCH_LIST="6.0 6.1 7.0+PTX" +ENV TORCH_NVCC_FLAGS="-Xfatbin -compress-all" +ENV CMAKE_PREFIX_PATH="$(dirname $(which conda))/../" + +RUN apt-get update && apt-get install -y git pkg-config gcc ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN conda clean --all +RUN git clone https://github.com/saic-vul/iterdet.git /iterdet +WORKDIR /iterdet +ENV FORCE_CUDA="1" +RUN pip install "git+https://github.com/cocodataset/cocoapi.git#subdirectory=PythonAPI" +RUN pip install --no-cache-dir -e . + + +VOLUME /repo +WORKDIR /repo/adaptive-learning/teachers + + +RUN pip install --upgrade pip setuptools==41.0.0 && pip install opencv-python wget scipy image lxml + + +ENTRYPOINT ["python", "teacher_main.py"] +CMD ["--config", "/repo/adaptive-learning/configs/iterdet.ini"] diff --git a/adaptive-learning/teachers/iterdet.py b/adaptive-learning/teachers/iterdet.py new file mode 100644 index 00000000..a7a053c0 --- /dev/null +++ b/adaptive-learning/teachers/iterdet.py @@ -0,0 +1,143 @@ +import copy +import os +import time +import logging +import wget +from mmdet.apis import init_detector, inference_detector +import cv2 as cv +from teacher_meta_arch import TeacherMetaArch +import torch +import numpy as np + + +class IterDet(TeacherMetaArch): + """ IterDet object detection model""" + + def __init__(self, config): + """ + IterDet class constructor + Args: + config: an adaptive learning config file + """ + super(IterDet, self).__init__(config=config) + self.detection_model = self.load_model() + self.postprocessing_method = self.config.get_section_dict('Teacher')['PostProcessing'] + # weather or not using background filtering in postprocessing step + self.background_filter = True if self.postprocessing_method == "background_filter" else False + if self.background_filter: + self.background_subtractor = cv.createBackgroundSubtractorMOG2() + self.image_features = self.config.get_section_dict('Teacher')['ImageFeature'] + + def inference(self, preprocessed_image): + """ + predict bounding boxes of a preprocessed image + Args: + preprocessed_image: a preprocessed RGB frame + + Returns: + A list of dictionaries, each item has the id, relative bounding box coordinate and prediction confidence score + of one detected instance. + """ + self.frame = preprocessed_image + output_dict = inference_detector(self.detection_model, preprocessed_image) + class_id = int(self.config.get_section_dict('Teacher')['ClassID']) + score_threshold = float(self.config.get_section_dict('Teacher')['MinScore']) + result = [] + for i, box in enumerate(output_dict[0]): # number of boxes + if box[-1] > score_threshold: + result.append({"id": str(class_id) + '-' + str(i), + "bbox": [box[0] / self.image_size[0], box[1] / self.image_size[1], + box[2] / self.image_size[0], box[3] / self.image_size[1]], + "score": box[-1]}) + + return result + + def postprocessing(self, raw_results): + """ + omit large boxes and boxes that detected as background by background subtractor. + Args: + raw_results: list of dictionaries, output of the inference method + Returns: + a filter version of raw_results + """ + post_processed_results = copy.copy(raw_results) + if self.background_filter: + self.foreground_mask = self.background_subtractor.apply(self.frame) + self.foreground_mask = cv.threshold(self.foreground_mask, 128, 255, cv.THRESH_BINARY)[1] / 255 + for bbox in raw_results: + # delete large boxes + if (bbox["bbox"][2] - bbox["bbox"][0]) * (bbox["bbox"][3] - bbox["bbox"][1]) > 0.2: + post_processed_results.remove(bbox) + continue + # delete background boxes + if self.background_filter: + x_min = max(0, int(bbox["bbox"][0] * self.image_size[0])) + y_min = max(0, int(bbox["bbox"][1] * self.image_size[1])) + x_max = min(self.image_size[0] - 1, int(bbox["bbox"][2] * self.image_size[0])) + y_max = min(self.image_size[1] - 1, int(bbox["bbox"][3] * self.image_size[1])) + bbox_mask_window = self.foreground_mask[y_min:y_max, x_min:x_max] + foreground_portion = bbox_mask_window.sum() / bbox_mask_window.size + if foreground_portion < .07: + post_processed_results.remove(bbox) + return post_processed_results + + def load_model(self): + """ + This function will load the IterDet model with its checkpoints on Crowd Human dataset. The chechpoints and configs + will be download in "data/iterdet" directory if they do not exists already. + """ + base_path = "data/iterdet" + if not os.path.exists(base_path): + os.makedirs(base_path) + config_file = os.path.join(base_path, "crowd_human_full_faster_rcnn_r50_fpn_2x.py") + if not os.path.isfile(config_file): + url = "https://raw.githubusercontent.com/saic-vul/iterdet/master/" \ + "configs/iterdet/crowd_human_full_faster_rcnn_r50_fpn_2x.py" + logging.info(f'config file does not exist under: {config_file}, downloading from {url}') + wget.download(url, config_file) + + checkpoint_file = os.path.join(base_path, "crowd_human_full_faster_rcnn_r50_fpn_2x.pth") + if not os.path.isfile(checkpoint_file): + url = "https://github.com/saic-vul/iterdet/releases/download/v2.0.0/crowd_human_full_faster_rcnn_r50_fpn_2x.pth" + logging.info(f'checkpoint file does not exist under: {checkpoint_file}, downloading from {url}') + wget.download(url, checkpoint_file) + + # build the model from a config file and a checkpoint file + device = self.config.get_section_dict('Teacher')['Device'] + if device == "GPU" and torch.cuda.is_available(): + device = "cuda:0" + else: + device = "cpu" + model = init_detector(config_file, checkpoint_file, device=device) + + return model + + def write_image(self, image, image_info): + if self.image_features == "foreground_mask": + image[..., 0] = (self.foreground_mask * 255).astype(int) + elif self.image_features == "optical_flow_magnitude": + if image_info["name"] == "0": + self.prvs = cv.cvtColor(image, cv.COLOR_BGR2GRAY) + image[..., 0] = np.zeros((self.image_size[1], self.image_size[0])) + else: + next_frame = cv.cvtColor(image, cv.COLOR_BGR2GRAY) + flow = cv.calcOpticalFlowFarneback(self.prvs, next_frame, None, 0.5, 3, 15, 3, 5, 1.2, 0) + mag, ang = cv.cartToPolar(flow[..., 0], flow[..., 1]) + image[..., 0] = cv.normalize(mag, None, 0, 255, cv.NORM_MINMAX) + self.prvs = next_frame + + elif self.image_features == "foreground_mask && optical_flow_magnitude": + if image_info["name"] == "0": + self.prvs = cv.cvtColor(image, cv.COLOR_BGR2GRAY) + image[..., 0] = self.prvs + image[..., 1] = np.zeros((self.image_size[1], self.image_size[0])) + image[..., 2] = (self.foreground_mask * 255).astype(int) + else: + next_frame = cv.cvtColor(image, cv.COLOR_BGR2GRAY) + flow = cv.calcOpticalFlowFarneback(self.prvs, next_frame, None, 0.5, 3, 15, 3, 5, 1.2, 0) + mag, ang = cv.cartToPolar(flow[..., 0], flow[..., 1]) + image[..., 0] = next_frame + image[..., 1] = cv.normalize(mag, None, 0, 255, cv.NORM_MINMAX) + image[..., 2] = (self.foreground_mask * 255).astype(int) + self.prvs = next_frame + super(IterDet, self).write_image(image, image_info) diff --git a/adaptive-learning/teachers/model_builder.py b/adaptive-learning/teachers/model_builder.py new file mode 100644 index 00000000..4358bd12 --- /dev/null +++ b/adaptive-learning/teachers/model_builder.py @@ -0,0 +1,14 @@ +from iterdet import IterDet +import logging +TEACHER_MAP = {"iterdet": IterDet} + + +def build(config): + """ build a teacher model based on config file""" + model_name = config.get_section_dict('Teacher')['Name'] + if model_name in TEACHER_MAP.keys(): + model = TEACHER_MAP[model_name] + logging.info("model loaded successfully") + else: + raise ValueError("The Teacher model name should be iterdet but {} provided".format(model_name)) + return model(config=config) diff --git a/adaptive-learning/teachers/teacher_main.py b/adaptive-learning/teachers/teacher_main.py new file mode 100644 index 00000000..0850a0b6 --- /dev/null +++ b/adaptive-learning/teachers/teacher_main.py @@ -0,0 +1,52 @@ +import sys +import time +import os +import model_builder + +sys.path.append("../..") +from libs.config_engine import ConfigEngine +import argparse +import cv2 as cv +import logging + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.INFO) + parser = argparse.ArgumentParser() + parser.add_argument('--config', required=True) + args = parser.parse_args() + config = ConfigEngine(args.config) + teacher_model = model_builder.build(config) + video_uri = config.get_section_dict("Teacher")["VideoPath"] + image_dir = config.get_section_dict("Teacher")["ImagePath"] + max_allowed_image = int(config.get_section_dict("Teacher")["MaxAllowedImage"]) + input_cap = cv.VideoCapture(video_uri) + if (input_cap.isOpened()): + print('opened video ', video_uri) + else: + print('failed to load video ', video_uri) + exit(0) + frame_num = 0 + total_infer_time = 0 + logging.info("Teacher Inference Process Started") + while input_cap.isOpened(): + number_of_existing_img = len( + [name for name in os.listdir(image_dir) if os.path.isfile(os.path.join(image_dir, name))]) + if number_of_existing_img > max_allowed_image: + logging.info("Maximum allowed image to store exceeds, Teacher will stop inference for 1 hour") + time.sleep(3600) + t_begin = time.perf_counter() + ret, cv_image = input_cap.read() + if not ret: + logging.info('Processed all frames') + break + preprocessed_image = teacher_model.preprocessing(cv_image) + raw_results = teacher_model.inference(preprocessed_image) + postprocessed_results = teacher_model.postprocessing(raw_results) + teacher_model.save_results(cv_image, postprocessed_results) + t_end = time.perf_counter() + total_infer_time += round(t_end - t_begin, 2) + if frame_num % 100 == 0: + logging.info("processed frame number {} in {} seconds".format(str(frame_num), str(round(total_infer_time, 2)))) + total_infer_time = 0 + frame_num += 1 + input_cap.release() diff --git a/adaptive-learning/teachers/teacher_meta_arch.py b/adaptive-learning/teachers/teacher_meta_arch.py new file mode 100644 index 00000000..9307308b --- /dev/null +++ b/adaptive-learning/teachers/teacher_meta_arch.py @@ -0,0 +1,119 @@ +import abc +import os +import logging +import cv2 as cv +from lxml import etree + + +class TeacherMetaArch(object): + """ + base class of teacher models + """ + def __init__(self, config): + self.config = config + self.min_detection = int(self.config.get_section_dict('Teacher')['MinDetectionPerFrame']) + self.save_frequency = int(self.config.get_section_dict('Teacher')['SaveFrequency']) + # Frames Per Second + self.fps = None + self.image_size = [int(i) for i in self.config.get_section_dict('Teacher')['ImageSize'].split(',')] + self._name = -1 + # try catch on filling image path and xml path + self.image_path = self.config.get_section_dict('Teacher')['ImagePath'] + self.xml_path = self.config.get_section_dict('Teacher')['XmlPath'] + if not os.path.exists(self.image_path): + os.makedirs(self.image_path) + if not os.path.exists(self.xml_path): + os.makedirs(self.xml_path) + logging.info("The Images and XML files will be saved under {} and {}".format(self.image_path, self.xml_path)) + + @abc.abstractmethod + def inference(self, preprocessed_image): + raise NotImplementedError + + def postprocessing(self, raw_results): + """ + override this method for custom postprocessing and filtering. + """ + return raw_results + + @property + def name(self): + self._name += 1 + return str(self._name) + + def preprocessing(self, image): + resized_image = cv.resize(image, tuple(self.image_size[:2])) + preprocessed_image = cv.cvtColor(resized_image, cv.COLOR_BGR2RGB) + return preprocessed_image + + def save_results(self, image, results): + """ + store image and teacher predicted bounding boxes based on given frequency and number of detected instances. + """ + name = self.name + if (len(results) >= self.min_detection) and (int(name) % self.save_frequency == 0): + h, w, d = image.shape + image_info = {"name": name, "w": w, "h": h, "d": d} + self.write_to_xml(results, image_info) + self.write_image(image, image_info) + + def convert_to_xml(self, bboxes, image_info): + """ + this function will create the xml tree from each frame annotation. + Args: + bboxes: list of bounding boxes + image_info: the number of frame that its xml file is creating + Returns: + an etree object + """ + annotation = etree.Element("annotation") + etree.SubElement(annotation, "filename").text = image_info["name"] + ".jpg" + size = etree.SubElement(annotation, "size") + etree.SubElement(size, "width").text = str(image_info["w"]) + etree.SubElement(size, "height").text = str(image_info["h"]) + etree.SubElement(size, "depth").text = str(image_info["d"]) + for bbox in bboxes: + obj = etree.SubElement(annotation, "object") + etree.SubElement(obj, "name").text = "pedestrian" + bounding_box = etree.SubElement(obj, "bndbox") + etree.SubElement(bounding_box, "xmin").text = bbox[0] + etree.SubElement(bounding_box, "ymin").text = bbox[1] + etree.SubElement(bounding_box, "xmax").text = bbox[2] + etree.SubElement(bounding_box, "ymax").text = bbox[3] + xml = etree.ElementTree(annotation) + return xml + + def write_to_xml(self, results, image_info): + """ + create xml annotation file from teacher prediction output + Args: + results: list of dictionary, output of postprocessing method + image_info: a dictionary contains image size and name. + """ + w = image_info["w"] + h = image_info["h"] + + bboxes = [] + for bbox in results: + x0 = int(bbox["bbox"][0] * w) + y0 = int(bbox["bbox"][1] * h) + x1 = int(bbox["bbox"][2] * w) + y1 = int(bbox["bbox"][3] * h) + x_min = max(0, x0) + y_min = max(0, y0) + x_max = min(w, x1) + y_max = min(h, y1) + bboxes.append([str(x_min), str(y_min), str(x_max), str(y_max)]) + xml = self.convert_to_xml(bboxes, image_info) + xml_file_name = os.path.join(self.xml_path, image_info["name"] + ".xml") + xml.write(xml_file_name, pretty_print=True) + + def write_image(self, image, image_info): + """ + save image frame for training purposes + Args: + image: input frame + image_info: a dictionary contains image size and name. + """ + + cv.imwrite(os.path.join(self.image_path, image_info["name"] + ".jpg"), image) diff --git a/amd64-usbtpu-adaptive-learning.Dockerfile b/amd64-usbtpu-adaptive-learning.Dockerfile new file mode 100644 index 00000000..2819dae3 --- /dev/null +++ b/amd64-usbtpu-adaptive-learning.Dockerfile @@ -0,0 +1,109 @@ +# docker can be installed on the dev board following these instructions: +# https://docs.docker.com/install/linux/docker-ce/debian/#install-using-the-repository , step 4: x86_64 / amd64 +# 1) build: docker build -f amd64-usbtpu.Dockerfile -t "neuralet/smart-social-distancing:latest-amd64" . +# 2) run: docker run -it --privileged -p HOST_PORT:8000 -v "$PWD/data":/repo/data neuralet/smart-social-distancing:latest-amd64 + +FROM amd64/debian:buster + +RUN apt-get update && apt-get install -y wget gnupg usbutils \ + && rm -rf /var/lib/apt/lists \ + && wget -qO - https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + +RUN echo "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main" | tee /etc/apt/sources.list.d/coral-edgetpu.list + +# The `python3-opencv` package isn't built with gstreamer on Ubuntu. So we need to manually build opencv. +ARG OPENCV_VERSION=4.3.0 +# http://amritamaz.net/blog/opencv-config +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + curl \ + git \ + gstreamer1.0-plugins-bad \ + gstreamer1.0-plugins-good \ + gstreamer1.0-plugins-ugly \ + gstreamer1.0-vaapi \ + libavcodec-dev \ + libavformat-dev \ + libgstreamer-plugins-base1.0-dev \ + libgstreamer1.0-dev \ + libsm6 \ + libswscale-dev \ + libxext6 \ + libxrender-dev \ + mesa-va-drivers \ + python3-dev \ + python3-numpy \ + edgetpu-compiler \ + inotify-tools \ + && rm -rf /var/lib/apt/lists/* \ + && cd /tmp/ \ + && curl -L https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.tar.gz -o opencv.tar.gz \ + && tar zxvf opencv.tar.gz && rm opencv.tar.gz \ + && cd /tmp/opencv-${OPENCV_VERSION} \ + && mkdir build \ + && cd build \ + && cmake \ + -DBUILD_opencv_python3=yes \ + -DPYTHON_EXECUTABLE=$(which python3) \ + -DCMAKE_BUILD_TYPE=RELEASE \ + -DBUILD_TESTS=OFF \ + -DBUILD_PERF_TESTS=OFF \ + -DBUILD_EXAMPLES=OFF \ + -DINSTALL_TESTS=OFF \ + -DBUILD_opencv_apps=OFF \ + -DBUILD_DOCS=OFF \ + ../ \ + && make -j$(nproc) \ + && make install \ + && cd /tmp \ + && rm -rf opencv-${OPENCV_VERSION} \ + && apt-get purge -y \ + cmake \ + git \ + libgstreamer-plugins-base1.0-dev \ + libgstreamer1.0-dev \ + libxrender-dev \ + python3-dev \ + && apt-get autoremove -y + +# https://askubuntu.com/questions/909277/avoiding-user-interaction-with-tzdata-when-installing-certbot-in-a-docker-contai +ARG DEBIAN_FRONTEND=noninteractive + +COPY api/requirements.txt / + +RUN apt-get update && apt-get install -y --no-install-recommends \ + tzdata \ + pkg-config \ + python3-dev \ + python3-numpy \ + python3-pillow \ + python3-pip \ + python3-scipy \ + python3-wget \ + python3-pytest \ + python3-requests \ + build-essential \ + libedgetpu1-std \ + procps \ + && rm -rf /var/lib/apt/lists/* \ + && python3 -m pip install --upgrade pip setuptools==41.0.0 wheel && pip install -r /requirements.txt \ + https://dl.google.com/coral/python/tflite_runtime-2.1.0.post1-cp37-cp37m-linux_x86_64.whl \ + && apt-get purge -y \ + build-essential \ + python3-dev \ + && apt-get autoremove -y + +ENV DEV_ALLOW_ALL_ORIGINS=true +ENV AWS_SHARED_CREDENTIALS_FILE=/repo/.aws/credentials +ENV AWS_CONFIG_FILE=/repo/.aws/config + +RUN cd / && apt-get update && apt-get install -y git python3-edgetpu && git clone \ + https://github.com/google-coral/project-posenet.git && sed -i 's/sudo / /g' \ + /project-posenet/install_requirements.sh && sh /project-posenet/install_requirements.sh +ENV PYTHONPATH=$PYTHONPATH:/project-posenet + +COPY . /repo +WORKDIR /repo +ENTRYPOINT ["bash", "start_services_adaptive.bash"] +CMD ["config-coral.ini"] diff --git a/api/tests/data/config-sample.ini b/api/tests/data/config-sample.ini index fa1707ed..6f6dcfa9 100644 --- a/api/tests/data/config-sample.ini +++ b/api/tests/data/config-sample.ini @@ -31,6 +31,7 @@ Tags = kitchen Name = Front Id = default Emails = +LoopVideoFile = false DistMethod = distance_method_this_is_test_sample_config [Source_1] @@ -39,6 +40,7 @@ Tags = hall Name = Backyard Id = cam2 Emails = +LoopVideoFile = true DistMethod = [PostProcessor] diff --git a/config-coral.ini b/config-coral.ini index 071ccf2b..47cd5af6 100644 --- a/config-coral.ini +++ b/config-coral.ini @@ -43,6 +43,8 @@ Id = default Emails = NotifyEveryMinutes = 0 ViolationThreshold = 60 +#wether to loop the video processing when video file reached to its final frame or not. +LoopVideoFile = False ; Distance measurement method: ; - CalibratedDistance: calculate the distance with 3-d transformed points, note that by choosing this method you should specify the inverse calibration matrix of your environment. ; - CenterPointsDistance: compare center of pedestrian boxes together diff --git a/config-jetson.ini b/config-jetson.ini index 3b235efd..899d5bc0 100644 --- a/config-jetson.ini +++ b/config-jetson.ini @@ -46,6 +46,8 @@ Id = default Emails = NotifyEveryMinutes = 0 ViolationThreshold = 60 +#wether to loop the video processing when video file reached to its final frame or not. +LoopVideoFile = False ; Distance measurement method: ; - CalibratedDistance: calculate the distance with 3-d transformed points, note that by choosing this method you should specify the inverse calibration matrix of your environment. ; - CenterPointsDistance: compare center of pedestrian boxes together diff --git a/config-x86-openvino.ini b/config-x86-openvino.ini index bb9486a0..5f9d229e 100644 --- a/config-x86-openvino.ini +++ b/config-x86-openvino.ini @@ -45,6 +45,8 @@ Id = default Emails = NotifyEveryMinutes = 0 ViolationThreshold = 60 +#wether to loop the video processing when video file reached to its final frame or not. +LoopVideoFile = False ; Distance measurement method: ; - CalibratedDistance: calculate the distance with 3-d transformed points, note that by choosing this method you should specify the inverse calibration matrix of your environment. ; - CenterPointsDistance: compare center of pedestrian boxes together diff --git a/config-x86.ini b/config-x86.ini index da93d394..5bba8ae0 100644 --- a/config-x86.ini +++ b/config-x86.ini @@ -45,6 +45,8 @@ Id = default Emails = NotifyEveryMinutes = 0 ViolationThreshold = 60 +#wether to loop the video processing when video file reached to its final frame or not. +LoopVideoFile = False ; Distance measurement method: ; - CalibratedDistance: calculate the distance with 3-d transformed points, note that by choosing this method you should specify the inverse calibration matrix of your environment. ; - CenterPointsDistance: compare center of pedestrian boxes together @@ -90,4 +92,4 @@ TimeInterval = 0.5 LogDirectory = /repo/data/processor/static/data EnableReports = no HeatmapResolution = 150,150 -WebHooksEndpoint= +WebHooksEndpoint= \ No newline at end of file diff --git a/docker-compose-adaptive-learning-usbtpu.yml b/docker-compose-adaptive-learning-usbtpu.yml new file mode 100644 index 00000000..c78b32c7 --- /dev/null +++ b/docker-compose-adaptive-learning-usbtpu.yml @@ -0,0 +1,51 @@ +version: '2.3' +services: + teacher: + build: + context: ./adaptive-learning/teachers/ + dockerfile: iterdet-teacher.Dockerfile + stdin_open: true + tty: true + container_name: teacher-container + volumes: + - ./:/repo + + runtime: nvidia + environment: + - NVIDIA_VISIBLE_DEVICES=0 + depends_on: + - smart-social-distancing + + student: + build: + context: ./adaptive-learning/students/ + dockerfile: ssd-mobilenet-v2-student.Dockerfile + stdin_open: true + tty: true + container_name: student-container + volumes: + - ./:/repo + ports: + - "6006:6006" + runtime: nvidia + environment: + - NVIDIA_VISIBLE_DEVICES=0 + - PYTHONPATH=$PYTHONPATH:/models/research:/models/research/slim + - TF_FORCE_GPU_ALLOW_GROWTH=true + depends_on: + - smart-social-distancing + + smart-social-distancing: + build: + context: ./ + dockerfile: amd64-usbtpu-adaptive-learning.Dockerfile + stdin_open: true + tty: true + container_name: smart-social-distancing + volumes: + - ./:/repo + ports: + - [HOST_PORT]:8000 + environment: + - TZ=./timezone.sh + privileged: true diff --git a/libs/config_engine.py b/libs/config_engine.py index 7fada43f..c76e7eba 100644 --- a/libs/config_engine.py +++ b/libs/config_engine.py @@ -161,6 +161,7 @@ def get_video_sources(self): else: src['should_send_email_notifications'] = False src['should_send_slack_notifications'] = False + src['loop_video_file'] = self.get_boolean(title, 'LoopVideoFile') sources.append(src) return sources except: diff --git a/libs/distancing.py b/libs/distancing.py index e53a1d02..61af5808 100755 --- a/libs/distancing.py +++ b/libs/distancing.py @@ -19,8 +19,9 @@ class Distancing: - def __init__(self, config, source, live_feed_enabled=True): + def __init__(self, config, source, loop_video_file, live_feed_enabled=True): self.config = config + self.loop_video_file = loop_video_file self.detector = None self.device = self.config.get_section_dict('Detector')['Device'] @@ -292,6 +293,7 @@ def process_video(self, video_uri): dist_threshold = float(self.config.get_section_dict("PostProcessor")["DistThreshold"]) frame_num = 0 + total_frames = input_cap.get(cv.CAP_PROP_FRAME_COUNT) start_time = time.time() last_processed_time = time.time() while input_cap.isOpened() and self.running_video: @@ -307,6 +309,10 @@ def process_video(self, video_uri): frame_num += 1 if frame_num % 100 == 1: logger.info(f'processed frame {frame_num} for {video_uri}') + if self.loop_video_file and frame_num % total_frames == 0: + input_cap.set(cv.CAP_PROP_POS_FRAMES, 0) + logger.info("reached end of video, app will process from begining") + # Save a screenshot only if the period is greater than 0, a violation is detected, and the minimum period # has occured if (self.screenshot_period > 0) and (time.time() > start_time + self.screenshot_period) and ( diff --git a/libs/engine_threading.py b/libs/engine_threading.py index f86e9ddf..1814f341 100644 --- a/libs/engine_threading.py +++ b/libs/engine_threading.py @@ -42,7 +42,7 @@ def __init__(self, config, source, live_feed_enabled=True): self.live_feed_enabled = live_feed_enabled def run(self): - self.engine = CvEngine(self.config, self.source['section'], self.live_feed_enabled) + self.engine = CvEngine(self.config, self.source['section'], self.source['loop_video_file'], self.live_feed_enabled) self.engine.process_video(self.source['url']) def stop(self): diff --git a/start_services_adaptive.bash b/start_services_adaptive.bash new file mode 100644 index 00000000..17b89694 --- /dev/null +++ b/start_services_adaptive.bash @@ -0,0 +1,17 @@ +#!/bin/bash + +CONFIG="$1" +# check if model name is correct +model=$(sed -nr "/^\[Detector\]/ { :l /^Name[ ]*=/ { s/.*=[ ]*//; p; q;}; n; b l;}" $CONFIG) + +if [ $model != "mobilenet_ssd_v2" ]; then + echo "the selected model must be 'mobilenet_ssd_v2' in adaptive learning setup but it is $model" + kill -SIGUSR1 `ps --pid $$ -oppid=` + exit 1 +fi + +bash /repo/sample_startup.bash $CONFIG & +bash /repo/update_model.bash $CONFIG & +python3 create_reports.py --config $CONFIG & +python3 run_processor_core.py --config $CONFIG & +python3 run_processor_api.py --config $CONFIG diff --git a/update_model.bash b/update_model.bash new file mode 100644 index 00000000..3f281a7c --- /dev/null +++ b/update_model.bash @@ -0,0 +1,32 @@ +#!/bin/bash +config="$1" + +MODEL_DIR="/repo/adaptive-learning/data/student_model/frozen_graph/" +sleep 20 + +# start watching files +inotifywait -e modify -m $MODEL_DIR | grep '\s\+detect.tflite$' --line-buffered | + while read dir action file; do + echo "Start Updating Model ..." + sleep 5 + edgetpu_compiler $MODEL_DIR/detect.tflite -o /repo/data/edgetpu/ + mv /repo/data/edgetpu/detect_edgetpu.tflite /repo/data/edgetpu/mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite + # restarting core + stop_response=$(curl 0.0.0.0:8000/stop-process-video) + if [ "$stop_response" = true ] ; then + echo "core stopped sucessfuly" + sleep 5 + elif [ "$stop_response" != true ] ; then + echo "failed to stopping core , ..." + fi + + + response=$(curl 0.0.0.0:8000/start-process-video) + if [ "$response" != true ] ; then + echo "restarting failed, trying again in 5 seconds!" + elif [ "$response" = true ] ; then + echo "ok video is going to be processed with updated model ..." + fi + + done +