From 115756989738c982965cda09324fc55e42ce6ca9 Mon Sep 17 00:00:00 2001 From: "jenkins.onyxia" Date: Thu, 9 Jun 2022 16:49:26 +0200 Subject: [PATCH] GroundSdk release 7.2.1 --- GroundSdk.podspec | 6 +- .../ThermalPictureParser.swift | 766 ++++++++++++++++++ .../ThermalProcessing/ThermalStreamView.swift | 101 +++ 3 files changed, 870 insertions(+), 3 deletions(-) create mode 100644 GroundSdk/ThermalProcessing/ThermalPictureParser.swift create mode 100644 GroundSdk/ThermalProcessing/ThermalStreamView.swift diff --git a/GroundSdk.podspec b/GroundSdk.podspec index abead05..9d8dde3 100644 --- a/GroundSdk.podspec +++ b/GroundSdk.podspec @@ -1,17 +1,17 @@ Pod::Spec.new do |s| s.name = "GroundSdk" - s.version = "7.2.0" + s.version = "7.2.1" s.summary = "Parrot Drone SDK" s.homepage = "https://developer.parrot.com" s.license = "{ :type => 'BSD 3-Clause License', :file => 'LICENSE' }" s.author = 'Parrot Drone SAS' - s.source = { :git => 'https://github.com/Parrot-Developers/pod_groundsdk.git', :tag => "7.2.0" } + s.source = { :git => 'https://github.com/Parrot-Developers/pod_groundsdk.git', :tag => "7.2.1" } s.platform = :ios s.ios.deployment_target = '12.0' s.source_files = 'GroundSdk/**/*.{swift,h,m}' s.resources = 'GroundSdk/**/*.{vsh,fsh,txt,png}' - s.dependency 'SdkCore', '7.2.0' + s.dependency 'SdkCore', '7.2.1' s.public_header_files = ["GroundSdk/GroundSdk.h"] s.swift_version = '5' s.pod_target_xcconfig = {'SWIFT_VERSION' => '5'} diff --git a/GroundSdk/ThermalProcessing/ThermalPictureParser.swift b/GroundSdk/ThermalProcessing/ThermalPictureParser.swift new file mode 100644 index 0000000..a29ed0c --- /dev/null +++ b/GroundSdk/ThermalProcessing/ThermalPictureParser.swift @@ -0,0 +1,766 @@ +// Copyright (C) 2019 Parrot Drones SAS +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in +// the documentation and/or other materials provided with the +// distribution. +// * Neither the name of the Parrot Company nor the names +// of its contributors may be used to endorse or promote products +// derived from this software without specific prior written +// permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +// PARROT COMPANY BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +// OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +// AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +// OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +// SUCH DAMAGE. + +import Foundation + +/// Parser of thermal pictures. +/// +/// Enable parsing of thermal pictures generated by drones and creation of 'ThermalPicture' +/// instances that can be used by TPRoc for rendering. +public class ThermalPictureParser { + + /// Log tag. + static let TprocTag = ULogTag(name: "ff.tproc") + + /// Common first byte of segments markers. + private static let MARKER_COMMON: UInt8 = 0xFF + /// Start of image marker. + private static let MARKER_SOI: UInt8 = 0xD8 + /// End of image marker. + private static let MARKER_EOI: UInt8 = 0xD9 + /// Synchronization marker. + private static let MARKER_RST0: UInt8 = 0xD0 + /// Synchronization marker. + private static let MARKER_RST1: UInt8 = 0xD1 + /// Synchronization marker. + private static let MARKER_RST2: UInt8 = 0xD2 + /// Synchronization marker. + private static let MARKER_RST3: UInt8 = 0xD3 + /// Synchronization marker. + private static let MARKER_RST4: UInt8 = 0xD4 + /// Synchronization marker. + private static let MARKER_RST5: UInt8 = 0xD5 + /// Synchronization marker. + private static let MARKER_RST6: UInt8 = 0xD6 + /// Synchronization marker. + private static let MARKER_RST7: UInt8 = 0xD7 + /// Application 1 marker. + private static let MARKER_APP1: UInt8 = 0xE1 + + /// Identifier of Parrot visible data in APP1 segments. + private static let TOKEN_PARROT_VISIBLE = Data("PARV".utf8) + /// Identifier of Parrot thermal data in APP1 segments. + private static let TOKEN_PARROT_THERMAL = Data("PART".utf8) + /// Identifier of FLIR data in APP1 segments. + private static let TOKEN_FLIR = Data("FLIR".utf8) + /// Identifier of XMP data in APP1 segments. + private static let TOKEN_XMP = Data("http://ns.adobe.com/xap/1.0/\0".utf8) + + /// Read content of a thermal picture generated by a drone and create a 'ThermalPicture' object. + /// + /// - Parameter file: thermal picture file + /// - Returns: a 'ThermalPicture' instance on success, 'nil' otherwise + public static func createPicture(file: URL) -> ThermalProcPictureData? { + // jpeg data of visible image + var visibleData: Data = Data() + // TIFF data containing thermal data + var tiffData: Data = Data() + // thermal data + var thermalData: Data = Data() + // FLIR data + var flirData: Data = Data() + // parsed XMP parameters: format version and camera calibration parameters + var xmpParsedParameters: XmpParsedParameters = XmpParsedParameters() + // focal length of the lens + var focalLength: Double? + // focal plane X resolution + var focalPlaneXResolution: Double? + + // extract information from EXIF data + parseExifData(file: file, focalLength: &focalLength, focalPlaneXResolution: &focalPlaneXResolution) + + // extract visible data, thermal data, and calibratrion parameters + parseParrotData(file: file, visibleData: &visibleData, thermalData: &tiffData, flirData: &flirData, + xmpParsedParameters: &xmpParsedParameters) + + // extract thermal data from TIFF + if let imageData = TiffParser.extractImageData(tiffData: tiffData) { + thermalData.append(imageData) + } + + // format version + var formatVersionMajor: Int32 = 0 + var formatVersionMinor: Int32 = 0 + if let formatVersion = xmpParsedParameters.formatVersion { + let formatVersionNumbers = formatVersion.split(separator: ".") + if formatVersionNumbers.count >= 1 { + formatVersionMajor = Int32(formatVersionNumbers[0]) ?? 0 + } + if formatVersionNumbers.count >= 2 { + formatVersionMinor = Int32(formatVersionNumbers[1]) ?? 0 + } + } + + // use FLIR embedded image for visible image if format version is superior to 0 + if formatVersionMajor > 0 || formatVersionMinor > 0 { + let visibleDataFromFlir = FlirParser.extractVisibleImageData(flirData: flirData) + if let visibleDataFromFlir = visibleDataFromFlir { + visibleData = visibleDataFromFlir + } + } + + // decompress the visible image + var visibleWidth: Int32 = 0 + var visibleHeight: Int32 = 0 + let decompressedVisibleData: Data? = ThermalProcPictureData.decompressJpeg(from: visibleData, + width: &visibleWidth, + height: &visibleHeight) + + // thermal camera model + var thermalCameraModel = ThermalProcThermalCamera.lepton + if let droneModelId = xmpParsedParameters.modelId, + let deviceModel = DeviceModel.from(internalId: droneModelId), + case .drone(let droneModel) = deviceModel, + let cameraModel = droneModel.thermalCameraModel { + thermalCameraModel = cameraModel + } + + var thermalPicture: ThermalProcPictureData? + if let decompressedVisibleData = decompressedVisibleData, + decompressedVisibleData.endIndex > 0, + thermalData.endIndex > 0 { + // create the thermal picture + thermalPicture = ThermalProcPictureData(visibleData: decompressedVisibleData, + visibleWidth: visibleWidth, + visibleHeight: visibleHeight, + thermalData: thermalData, + formatVersionMajor: formatVersionMajor, + formatVersionMinor: formatVersionMinor, + calibR: xmpParsedParameters.calibR, + calibB: xmpParsedParameters.calibB, + calibF: xmpParsedParameters.calibF, + calibO: xmpParsedParameters.calibO, + tauWin: xmpParsedParameters.tauWin, + rWin: xmpParsedParameters.rWin, + sensorTemp: xmpParsedParameters.sensorTemp, + sensorHTemp: xmpParsedParameters.sensorHTemp, + scalingFactor: xmpParsedParameters.scalingFactor, + focalLength: focalLength ?? 0, + focalXResolution: focalPlaneXResolution ?? 0, + viewYaw: xmpParsedParameters.viewYaw ?? 0, + viewPitch: xmpParsedParameters.viewPitch ?? 0, + viewRoll: xmpParsedParameters.viewRoll ?? 0, + visibleCamYaw: xmpParsedParameters.visibleCamYaw ?? 0, + visibleCamPitch: xmpParsedParameters.visibleCamPitch ?? 0, + visibleCamRoll: xmpParsedParameters.visibleCamRoll ?? 0, + thermalCamYaw: xmpParsedParameters.thermalCamYaw ?? 0, + thermalCamPitch: xmpParsedParameters.thermalCamPitch ?? 0, + thermalCamRoll: xmpParsedParameters.thermalCamRoll ?? 0, + cameraModel: thermalCameraModel) + } else { + thermalPicture = nil + } + return thermalPicture + } + + /// Extract some camera information from EXIF data. + /// + /// - Parameters: + /// - file: thermal picture file + /// - focalLength: destination of focal length of the len + /// - focalPlaneXResolution: destination of focal plane X resolution + private static func parseExifData(file: URL, focalLength: inout Double?, focalPlaneXResolution: inout Double?) { + if let imageSource = CGImageSourceCreateWithURL(file as CFURL, nil), + let imageProperties = CGImageSourceCopyPropertiesAtIndex(imageSource, 0, nil) as Dictionary?, + let exifDict = imageProperties[kCGImagePropertyExifDictionary] { + focalLength = exifDict[kCGImagePropertyExifFocalLength] as? Double + focalPlaneXResolution = exifDict[kCGImagePropertyExifFocalPlaneXResolution] as? Double + } + } + + /// Parse Parrot specific data. + /// + /// - Parameters: + /// - file: thermal picture file + /// - visibleData: destination of visible jpeg data + /// - thermalData: destination of thermal data + /// - xmpParsedParameters: destination of xmp metadata + private static func parseParrotData(file: URL, + visibleData: inout Data, + thermalData: inout Data, + flirData: inout Data, + xmpParsedParameters: inout XmpParsedParameters) { + do { + let data = try Data(contentsOf: file, options: .mappedIfSafe) + + parseSegments(data: data, onVisible: { data in + visibleData.append(data) + }, onThermal: { data in + thermalData.append(data) + }, onFlir: { data in + flirData.append(data) + }, onXmp: { data in + let parser = XmpParser() + parser.parse(data: data, parsedParameters: &xmpParsedParameters) + }) + } catch { + ULog.w(ThermalPictureParser.TprocTag, "Failed to parse Parrot data") + } + } + + /// Parse jpeg segments. + /// + /// - Parameters: + /// - data: jpeg data + /// - onVisible: called when a segment containing visible image data is reached + /// - onThermal: called when a segment containing thermal data is reached + /// - onXmp: called when a segment containing XMP data is reached + static func parseSegments(data: Data, + onVisible: (Data) -> Void, + onThermal: (Data) -> Void, + onFlir: (Data) -> Void, + onXmp: (Data) -> Void) { + var position = 0 + while position < data.endIndex { + let marker1: UInt8 = data[position] + position += 1 + if marker1 == MARKER_COMMON { + let marker2: UInt8 = data[position] + position += 1 + if marker1 == MARKER_COMMON && marker2 != 0 { + switch marker2 { + case MARKER_SOI, MARKER_EOI, MARKER_RST0, MARKER_RST1, MARKER_RST2, MARKER_RST3, MARKER_RST4, + MARKER_RST5, MARKER_RST6, MARKER_RST7: + // segment without payload + break + case MARKER_APP1: + // APP1 segment + var payloadLen: UInt16 = data[position.. Data? { + var imageData = Data() + let data = ByteBuffer(data: tiffData) + + // Parse endianness + let endieness = data.readBytes(length: UInt(TiffParser.TIFF_HEADER_LEN)) + + if endieness == TiffParser.TIFF_HEADER_LSB { + data.isLsb = true + } else if endieness == TiffParser.TIFF_HEADER_MSB { + data.isLsb = false + } else { + ULog.w(ThermalPictureParser.TprocTag, "Tiff endianness parsing error") + return nil + } + + // Parse Image File Directories (IFD) address + var ifdAddr = data.readUInt32() + + // Parse all IDF to look for strip offset and strip length + while ifdAddr != 0 { + var stripOffset = 0 + var stripLength = 0 + data.position = UInt(ifdAddr) + + // Parse IFD + // Parse number of entries in the IFD + let entriesNumber = data.readUInt16() + for _ in 0.. data.count { + ULog.w(ThermalPictureParser.TprocTag, "Tiff IFD offset error") + return nil + } + + if stripOffset != 0 && stripLength != 0 { + // copy the image strip in the destination + let realStripOffset = tiffData.startIndex + stripOffset + imageData.append(tiffData[realStripOffset..= 64 else { + throw FlirParserError.insufficientData + } + flirData.position = 24 + recordDirectoryOffset = flirData.readUInt32() + nbRecords = flirData.readUInt32() + flirData.position += 32 + } + } + + /// Header of FLIR record. + class FlirRecordHeader { + + /// Record type value for embedded image. + static let RECORD_TYPE_EMBEDDED_IMAGE = 0xE + + /// Record type. + let recordType: UInt16 + + /// Record offset in bytes, from start of FLIR section. + let recordOffest: UInt32 + + /// Record length in bytes. + let recordLength: UInt32 + + /// Creates a FLIR record header from FLIR section data. + /// + /// - Parameter flirData: `ByteBuffer` where record header data are read + init (flirData: ByteBuffer) throws { + guard flirData.count - flirData.position >= 32 else { + throw FlirParserError.insufficientData + } + recordType = flirData.readUInt16() + flirData.position += 10 + recordOffest = flirData.readUInt32() + recordLength = flirData.readUInt32() + flirData.position += 12 + } + } + + /// Extracts visible image from FLIR section data. + /// + /// - Parameter flirData: FLIR section data to parse + /// + /// - Returns: visible image extracted from FLIR section data on success, `nil` otherwise + static func extractVisibleImageData(flirData: Data) -> Data? { + do { + let data = ByteBuffer(data: flirData) + data.isLsb = false + let header = try FlirHeader(flirData: data) + data.position = UInt(header.recordDirectoryOffset) + for _ in 0.. flirData.count { + throw FlirParserError.outOfBounds + } + return flirData[imageOffset.. Data { + let value = data[dataPosition.. UInt16 { + let value: UInt16 = data[dataPosition.. UInt32 { + let value: UInt32 = data[dataPosition.. Void)? + + /// Private thermal processing instance. + private var tproc: ThermalProcVideo? + /// Thermal processing instance. + public var thermalProc: ThermalProcVideo? { + get { + return tproc + } + set { + if tproc?.rendererIsStarted() == true { + tproc?.stopRenderer() + } + tproc = newValue + textureLoader = tproc != nil ? self : nil + } + } + + /// Thermal Camera model to used. + public var thermalCamera = ThermalProcThermalCamera.lepton { + willSet { + if newValue != thermalCamera && tproc?.rendererIsStarted() == true { + tproc?.stopRenderer() + } + } + } + + public var textureSpec = TextureSpec.sourceDimensions + + public override init(frame: CGRect) { + super.init(frame: frame) + } + + required public init?(coder aDecoder: NSCoder) { + super.init(coder: aDecoder) + } + + /// Called to render a custom GL texture. + /// + /// - Parameter frame: frame data + /// - Returns: 'true' on success, otherwise 'false' + public func loadTexture(width: Int, height: Int, frame: TextureLoaderFrame?) -> Bool { + if tproc?.rendererIsStarted() == false { + tproc?.startRenderer(thermalCamera: thermalCamera, textureWidth: Int32(width), textureHeight: Int32(height)) + } + + if let frame = frame { + tproc?.render(textureWidth: Int32(width), textureHeight: Int32(height), + frame: frame.frame, frameUserData: frame.userData, frameUserDataLength: frame.userDataLen, + mediaInfo: frame.mediaInfo) { status in + if status.hasThermalData == true && status.calibrationState != .inProgress { + self.renderStatusBlock?(status.min, status.max, status.probe) + } else { + self.renderStatusBlock?(nil, nil, nil) + } + } + } + return true + } +}