From 243fe652465ffe3ca366d10278c913c3ed9f8f58 Mon Sep 17 00:00:00 2001 From: elvin chen Date: Fri, 7 Aug 2020 01:42:04 +0200 Subject: [PATCH] Initial code for handling GoPro metadata --- GPMF_gyro.py | 39 +++++++++ README.md | 19 ++++- camera_presets/gopro_calib.JSON | 40 +++++++++ gpmf/extract.py | 140 +++++++++++++++++++++++++++++++ gpmf/parse.py | 143 ++++++++++++++++++++++++++++++++ gyroflow.py | 6 +- 6 files changed, 383 insertions(+), 4 deletions(-) create mode 100644 GPMF_gyro.py create mode 100644 camera_presets/gopro_calib.JSON create mode 100644 gpmf/extract.py create mode 100644 gpmf/parse.py diff --git a/GPMF_gyro.py b/GPMF_gyro.py new file mode 100644 index 00000000..2390bc7c --- /dev/null +++ b/GPMF_gyro.py @@ -0,0 +1,39 @@ +# Script to extract gopro metadata into a useful format. +# Uses python-gpmf by from https://github.com/rambo/python-gpmf + +import gpmf.parse as gpmf_parse +from gpmf.extract import get_gpmf_payloads_from_file +import sys + + +class Extractor: + def __init__(self, videopath = "hero5.mp4"): + self.videopath = videopath + + payloads, parser = get_gpmf_payloads_from_file(videopath) + + for gpmf_data, timestamps in payloads: + for element, parents in gpmf_parse.recursive(gpmf_data): + try: + value = gpmf_parse.parse_value(element) + except ValueError: + value = element.data + print("{} {} > {}: {}".format( + timestamps, + ' > '.join([x.decode('ascii') for x in parents]), + element.key.decode('ascii'), + value + )) + + def get_gyro(self): + return 1 + + def get_accl(self): + return 1 + + def get_video_length(self): + return 1 + + +if __name__ == "__main__": + testing = Extractor() \ No newline at end of file diff --git a/README.md b/README.md index 376c82aa..9b411f5a 100644 --- a/README.md +++ b/README.md @@ -4,4 +4,21 @@ A program built around Python, OpenCV, and PySide2 for video stabilization using The project consists of three core parts: A utility for the generation of lens undistortion preset, a utility for stabilizing footage using gyro data, and a utility for stretching 4:3 video to 16:9 using non-linear horizontal stretching (similar to GoPro's superview). Only the last part (sorta) works as of right now. -This is very much a work in progress project, but the goal is to use the gyro data logged on drone flight controllers for stabilizing the onboard HD camera. Furthermore, the gyro data embedded in newer GoPro cameras should also be usable for stabilization purposes. \ No newline at end of file +This is very much a work in progress project, but the goal is to use the gyro data logged on drone flight controllers for stabilizing the onboard HD camera. Furthermore, the gyro data embedded in newer GoPro cameras should also be usable for stabilization purposes. + +### Status + +Working: +* Videoplayer based on OpenCV and Pyside2 +* Gyro integration using quaternions +* Non-linear stretch utility +* Basic video import/export +* Camera calibration utility with preset import/export + + +Not working (yet): +* GoPro/blackbox data import +* Symmetrical quaternion low-pass filter +* Camera rotation perspective transform +* Automatic gyro/video sync +* Stabilization UI \ No newline at end of file diff --git a/camera_presets/gopro_calib.JSON b/camera_presets/gopro_calib.JSON new file mode 100644 index 00000000..6db4234c --- /dev/null +++ b/camera_presets/gopro_calib.JSON @@ -0,0 +1,40 @@ +{ + "name": "Camera name", + "note": "", + "calibrator_version": "0.1.0 pre-alpha", + "date": "2020-08-04", + "calib_dimension": { + "w": 1920, + "h": 1440 + }, + "num_images": 5, + "use_opencv_fisheye": true, + "fisheye_params": { + "RMS_error": 1.312648728814275, + "camera_matrix": [ + [ + 853.5791051672114, + 0.0, + 973.112329623264 + ], + [ + 0.0, + 859.3536064161069, + 720.8037689142063 + ], + [ + 0.0, + 0.0, + 1.0 + ] + ], + "distortion_coeffs": [ + 0.012920277676868729, + 0.11660847198484613, + -0.11973972602821542, + 0.04344499986692209 + ] + }, + "use_opencv_standard": false, + "calib_params": {} +} \ No newline at end of file diff --git a/gpmf/extract.py b/gpmf/extract.py new file mode 100644 index 00000000..ae75a92c --- /dev/null +++ b/gpmf/extract.py @@ -0,0 +1,140 @@ +# The MIT License (MIT) +# Copyright (c) 2014 Eero af Heurlin +# https://github.com/rambo/python-gpmf + +#!/usr/bin/env python3 +import hachoir.parser +from hachoir.field import MissingField +from hachoir.field.string_field import String + + +def get_raw_content(met): + """Reads the raw bytes from the stream for this atom/field""" + if hasattr(met, 'stream'): + stream = met.stream + else: + stream = met.parent.stream + return stream.read(met.absolute_address, met.size) + + +def get_gpmf_payloads_from_file(filepath): + """Get payloads from file, returns a tuple with the payloads iterator and the parser instance""" + parser = hachoir.parser.createParser(filepath) + return (get_payloads(find_gpmd_stbl_atom(parser)), parser) + + +def get_gpmf_payloads(parser): + """Shorthand for finding the GPMF atom to be passed to get_payloads""" + return get_payloads(find_gpmd_stbl_atom(parser)) + + +def get_payloads(stbl): + """Get payloads by chunk from stbl, with timing info""" + # Locate needed subatoms + for subatom in stbl: + tag = subatom['tag'] + if tag.value == 'stsz': + stsz = subatom['stsz'] + if tag.value == 'stco': + stco = subatom['stco'] + if tag.value == 'stts': + stts = subatom['stts'] + + # Generate start and end timestamps for all chunks + timestamps = [] + for idx in range(stts['count'].value): + sample_delta = stts["sample_delta[{}]".format(idx)].value + for idx2 in range(stts["sample_count[{}]".format(idx)].value): + if idx == 0 and idx2 == 0: + sampletimes = (0, sample_delta) + else: + sampletimes = (timestamps[-1][1], timestamps[-1][1] + sample_delta) + timestamps.append(sampletimes) + + # Read chunks, yield with timing data + num_samples = stsz['count'].value + for idx in range(num_samples): + offset = stco["chunk_offset[{}]".format(idx)].value + size = stsz["sample_size[{}]".format(idx)].value + data = stbl.stream.read(offset * 8, size * 8)[1] + yield (data, timestamps[idx]) + + +def get_stream_data(stbl): + """Get raw payload bytes from stbl atom offsets""" + ret_bytes = b'' + for payload in get_payloads(stbl): + ret_bytes += payload[0] + return ret_bytes + + +def find_gpmd_stbl_atom(parser): + """Find the stbl atom""" + minf_atom = find_gpmd_minf_atom(parser) + if not minf_atom: + return None + try: + for minf_field in minf_atom: + tag = minf_field['tag'] + if tag.value != 'stbl': + continue + return minf_field['stbl'] + except MissingField: + pass + + +def find_gpmd_minf_atom(parser): + """Find minf atom for GPMF media""" + def recursive_search(atom): + try: + subtype = atom['hdlr/subtype'] + if subtype.value == 'meta': + meta_atom = atom.parent + # print(meta_atom) + for subatom in meta_atom: + tag = subatom['tag'] + if tag.value != 'minf': + continue + minf_atom = subatom['minf'] + #print(" {}".format(minf_atom)) + for minf_field in minf_atom: + tag = minf_field['tag'] + #print(" {}".format(tag)) + if tag.value != 'gmhd': + continue + if b'gpmd' in minf_field['data'].value: + return minf_atom + except MissingField: + pass + try: + for x in atom: + ret = recursive_search(x) + if ret: + return ret + except KeyError as e: + pass + return None + return recursive_search(parser) + + +def recursive_print(input): + """Recursively print hachoir parsed state""" + print(repr(input)) + if isinstance(input, String): + print(" {}".format(input.display)) + try: + for x in input: + recursive_print(x) + except KeyError as e: + pass + + +if __name__ == '__main__': + import sys + parser = hachoir.parser.createParser(sys.argv[1]) + with open(sys.argv[2], 'wb') as fp: + fp.write( + get_stream_data( + find_gpmd_stbl_atom(parser) + ) + ) diff --git a/gpmf/parse.py b/gpmf/parse.py new file mode 100644 index 00000000..d53437a9 --- /dev/null +++ b/gpmf/parse.py @@ -0,0 +1,143 @@ +# The MIT License (MIT) +# Copyright (c) 2014 Eero af Heurlin +# https://github.com/rambo/python-gpmf + +#!/usr/bin/env python3 +"""Parses the FOURCC data in GPMF stream into fields""" +import struct + +import construct +import dateutil.parser + +TYPES = construct.Enum( + construct.Byte, + int8_t=ord(b'b'), + uint8_t=ord(b'B'), + char=ord(b'c'), + int16_t=ord(b's'), + uint16_t=ord(b'S'), + int32_t=ord(b'l'), + uint32_t=ord(b'L'), + float=ord(b'f'), + double=ord(b'd'), + fourcc=ord(b'F'), + uuid=ord(b'G'), + int64_t=ord(b'j'), + uint64_t=ord(b'J'), + Q1516=ord(b'q'), + Q3132=ord(b'Q'), + utcdate=ord(b'U'), + complex=ord(b'?'), + nested=0x0, +) + +FOURCC = construct.Struct( + "key" / construct.Bytes(4), + "type" / construct.Byte, + "size" / construct.Byte, + "repeat" / construct.Int16ub, + "data" / construct.Aligned(4, construct.Bytes(construct.this.size * construct.this.repeat)) +) + + +def parse_value(element): + """Parses element value""" + type_parsed = TYPES.parse(bytes([element.type])) + #print("DEBUG: type_parsed={}, element.repeat={}, element.size={}, len(element.data): {}".format(type_parsed, element.repeat, element.size, len(element.data))) + + # Special cases + if type_parsed == 'char' and element.key == b'GPSU': + return parse_goprodate(element) + if type_parsed == 'utcdate': + return parse_goprodate(element) + + # Basic number types + struct_key = None + struct_repeat = element.repeat + if type_parsed == 'int32_t': + struct_key = 'l' + # It seems gopro is "creative" with grouped values and size vs repeat... + if element.size > 4: + struct_repeat = int(element.repeat * (element.size / 4)) + if type_parsed == 'uint32_t': + struct_key = 'L' + if element.size > 4: + struct_repeat = int(element.repeat * (element.size / 4)) + + if type_parsed == 'int16_t': + struct_key = 'h' + if element.size > 2: + struct_repeat = int(element.repeat * (element.size / 2)) + if type_parsed == 'uint16_t': + struct_key = 'H' + if element.size > 2: + struct_repeat = int(element.repeat * (element.size / 2)) + + if type_parsed == 'float': + struct_key = 'f' + if element.size > 4: + struct_repeat = int(element.repeat * (element.size / 4)) + + if not struct_key: + raise ValueError("{} does not have value parser yet".format(type_parsed)) + + struct_format = ">{}".format(''.join([struct_key for x in range(struct_repeat)])) + #print("DEBUG: struct_format={}".format(struct_format)) + try: + value_parsed = struct.unpack(struct_format, element.data) + except struct.error as e: + #print("ERROR: {}".format(e)) + #print("DEBUG: struct_format={}, data (len: {}) was: {}".format(struct_format, len(element.data), element.data)) + raise ValueError("Struct unpack failed: {}".format(e)) + + # Single value + if len(value_parsed) == 1: + return value_parsed[0] + # Grouped values + if len(value_parsed) > element.repeat: + n = int(len(value_parsed) / element.repeat) + return [value_parsed[i:i + n] for i in range(0, len(value_parsed), n)] + return list(value_parsed) + + +def parse_goprodate(element): + """Parses the gopro date string from element to Python datetime""" + goprotime = element.data.decode('UTF-8') + return dateutil.parser.parse("{}-{}-{}T{}:{}:{}Z".format( + 2000 + int(goprotime[:2]), # years + int(goprotime[2:4]), # months + int(goprotime[4:6]), # days + int(goprotime[6:8]), # hours + int(goprotime[8:10]), # minutes + float(goprotime[10:]) # seconds + )) + + +def recursive(data, parents=tuple()): + """Recursive parser returns depth-first traversing generator yielding fields and list of their parent keys""" + elements = construct.GreedyRange(FOURCC).parse(data) + for element in elements: + if element.type == 0: + subparents = parents + (element.key,) + for subyield in recursive(element.data, subparents): + yield subyield + else: + yield (element, parents) + + +if __name__ == '__main__': + import sys + from extract import get_gpmf_payloads_from_file + payloads, parser = get_gpmf_payloads_from_file(sys.argv[1]) + for gpmf_data, timestamps in payloads: + for element, parents in recursive(gpmf_data): + try: + value = parse_value(element) + except ValueError: + value = element.data + print("{} {} > {}: {}".format( + timestamps, + ' > '.join([x.decode('ascii') for x in parents]), + element.key.decode('ascii'), + value + )) diff --git a/gyroflow.py b/gyroflow.py index f163fea6..2ac22c00 100644 --- a/gyroflow.py +++ b/gyroflow.py @@ -170,7 +170,7 @@ def update_frame(self): # based on https://robonobodojo.wordpress.com/2018/07/01/automatic-image-sizing-with-pyside/ # and https://stackoverflow.com/questions/44404349/pyqt-showing-video-stream-from-opencv/44404713 class VideoPlayer(QtWidgets.QLabel): - def __init__(self, img = "cat_placeholder.jpg"): + def __init__(self, img = "placeholder.jpg"): super(VideoPlayer, self).__init__() self.setFrameStyle(QtWidgets.QFrame.StyledPanel) self.pixmap = QtGui.QPixmap(img) @@ -195,7 +195,7 @@ def __init__(self): """Widget containing videoplayer and controls """ QtWidgets.QWidget.__init__(self) - self.player = VideoPlayer("cat_placeholder.jpg") + self.player = VideoPlayer("placeholder.jpg") self.layout = QtWidgets.QVBoxLayout() self.layout.addWidget(self.player) @@ -411,7 +411,7 @@ def __init__(self): # slider for adjusting FOV self.fov_text = QtWidgets.QLabel("FOV scale ({}):".format(self.fov_scale)) self.fov_slider = QtWidgets.QSlider(QtCore.Qt.Horizontal, self) - self.fov_slider.setMinimum(10) + self.fov_slider.setMinimum(8) self.fov_slider.setValue(14) self.fov_slider.setMaximum(30) self.fov_slider.setMaximumWidth(300)