From 807f4fa23215f958232b6543f67d29e41f3e3be7 Mon Sep 17 00:00:00 2001 From: Elliot Boschwitz Date: Tue, 30 Mar 2021 16:23:01 -0700 Subject: [PATCH] Moved pre-processing step to Vision class (#25) --- Lobe_iOS.xcodeproj/project.pbxproj | 2 +- Lobe_iOS/Models/CaptureSessionManager.swift | 14 +------------- Lobe_iOS/Models/PredictionLayer.swift | 5 +++++ Lobe_iOS/PlayViewModel.swift | 6 +----- 4 files changed, 8 insertions(+), 19 deletions(-) diff --git a/Lobe_iOS.xcodeproj/project.pbxproj b/Lobe_iOS.xcodeproj/project.pbxproj index 1e6164f..a5f538d 100644 --- a/Lobe_iOS.xcodeproj/project.pbxproj +++ b/Lobe_iOS.xcodeproj/project.pbxproj @@ -89,8 +89,8 @@ 9B7B03DE2475DBF300D34020 = { isa = PBXGroup; children = ( - B5B2D938260A4D5B00BABFAB /* LobeModel.mlmodel */, 9B7B03E92475DBF300D34020 /* Lobe_iOS */, + B5B2D938260A4D5B00BABFAB /* LobeModel.mlmodel */, B513D84325809D62009D91E2 /* LobeTests */, 9B7B03E82475DBF300D34020 /* Products */, B5A9B5252557373E00FD595B /* Frameworks */, diff --git a/Lobe_iOS/Models/CaptureSessionManager.swift b/Lobe_iOS/Models/CaptureSessionManager.swift index 019348c..c29b898 100644 --- a/Lobe_iOS/Models/CaptureSessionManager.swift +++ b/Lobe_iOS/Models/CaptureSessionManager.swift @@ -174,19 +174,7 @@ extension UIImage { var isLandscape: Bool { size.width > size.height } var breadth: CGFloat { min(size.width, size.height) } var breadthSize: CGSize { .init(width: breadth, height: breadth) } - - func squared(isOpaque: Bool = false) -> UIImage? { - guard let cgImage = cgImage? - .cropping(to: .init(origin: .init(x: isLandscape ? ((size.width-size.height)/2).rounded(.down) : 0, - y: isPortrait ? ((size.height-size.width)/2).rounded(.down) : 0), - size: breadthSize)) else { return nil } - let format = imageRendererFormat - format.opaque = isOpaque - return UIGraphicsImageRenderer(size: breadthSize, format: format).image { _ in - UIImage(cgImage: cgImage, scale: 1, orientation: imageOrientation) - .draw(in: .init(origin: .zero, size: breadthSize)) - } - } + public convenience init?(pixelBuffer: CVPixelBuffer) { var cgImage: CGImage? VTCreateCGImageFromCVPixelBuffer(pixelBuffer, options: nil, imageOut: &cgImage) diff --git a/Lobe_iOS/Models/PredictionLayer.swift b/Lobe_iOS/Models/PredictionLayer.swift index 3c8f5b8..d1dc4d8 100644 --- a/Lobe_iOS/Models/PredictionLayer.swift +++ b/Lobe_iOS/Models/PredictionLayer.swift @@ -72,6 +72,11 @@ class PredictionLayer: NSObject { } onComplete(request) }) + + /// Set center cropping for the expected image size. + /// NOTE: Although center cropping is currently expected with Lobe's model version, + /// please be mindful of future changes which may affect the expected preprocessing steps. + request.imageCropAndScaleOption = .centerCrop return request } } diff --git a/Lobe_iOS/PlayViewModel.swift b/Lobe_iOS/PlayViewModel.swift index 85c5518..841b18f 100644 --- a/Lobe_iOS/PlayViewModel.swift +++ b/Lobe_iOS/PlayViewModel.swift @@ -37,11 +37,7 @@ class PlayViewModel: ObservableObject { .compactMap { $0 } // remove non-nill values .receive(on: DispatchQueue.global(qos: .userInitiated)) .sink(receiveValue: { [weak self] image in - guard let squaredImage = image.squared() else { - print("Could not create squared image in PlayViewModel.") - return - } - self?.imagePredicter.getPrediction(forImage: squaredImage) + self?.imagePredicter.getPrediction(forImage: image) }) .store(in: &disposables)