fix high CPU usage #1481

src/api-wrappers/AXUIElement.swift

@@ -21,7 +21,7 @@ func retryAxCallUntilTimeout_(_ group: DispatchGroup?, _ timeoutInSeconds: Doubl
     } catch {
         let timePassedInSeconds = Double(DispatchTime.now().uptimeNanoseconds - startTime.uptimeNanoseconds) / 1_000_000_000
         if timePassedInSeconds < timeoutInSeconds {
-            BackgroundWork.axCallsQueue.asyncAfter(deadline: .now() + .milliseconds(10)) {
+            BackgroundWork.axCallsQueue.asyncAfter(deadline: .now() + .milliseconds(250)) {
                 retryAxCallUntilTimeout_(group, timeoutInSeconds, fn, startTime)
             }
         }

src/api-wrappers/CGWindowID.swift

@@ -19,11 +19,63 @@ extension CGWindowID {
         return CGSCopySpacesForWindows(cgsMainConnectionId, CGSSpaceMask.all.rawValue, [self] as CFArray) as! [CGSSpaceID]
     }
 
-    func screenshot() -> CGImage? {
-        // CGSHWCaptureWindowList
+    // fullscreen has multiple windows
+    // e.g. Notes.app has a toolbar window and a main window
+    // We need to composite these window images
+    func fullScreenshot(_ win: Window) -> CGImage? {
+        var height: CGFloat = 0;
+        var width: CGFloat = 0;
+        var imageMap = [(CGWindow, CGImage)]()
+        var maxWidthWindowId: CGWindowID = 0
+        let screen = Spaces.spaceFrameMap.first { $0.0 == win.spaceId }!.1
+        var windowsInSpaces = Spaces.windowsInSpaces([win.spaceId]) // The returned windows are sorted from highest to lowest according to the z-index
+        var windowsToCapture: [CGWindow] = []
+        // find current app's window in the fullscreen space
+        for item in windowsInSpaces {
+            let cgWin = CGWindowListCopyWindowInfo([.excludeDesktopElements, .optionIncludingWindow], item) as! [CGWindow]
+            guard cgWin.first!.isNotMenubarOrOthers(),
+                  cgWin.first!.ownerPID() == win.application.runningApplication.processIdentifier,
+                  cgWin.first!.bounds() != nil,
+                  let bounds = CGRect(dictionaryRepresentation: cgWin.first!.bounds()!), bounds.height > 0, bounds.width > 0 else { continue }
+            windowsToCapture.append(cgWin.first!)
+        }
+        // Drawing images from lowest to highest base on the z-index
+        windowsToCapture = windowsToCapture.reversed()
+        for item in windowsToCapture {
+            let bounds = CGRect(dictionaryRepresentation: item.bounds()!)
+            if width < bounds!.width {
+                maxWidthWindowId = item.id()!
+            }
+            var windowId = item.id()!
+            let list = CGSHWCaptureWindowList(cgsMainConnectionId, &windowId, 1, [.ignoreGlobalClipShape, .nominalResolution]).takeRetainedValue() as! [CGImage]
+            imageMap.append((item, list.first!))
+        }
+        let bytesPerRow = imageMap.first { $0.0.id()! == maxWidthWindowId }!.1.bytesPerRow
+        var context = CGContext.init(data: nil,
+                width: Int(screen.frame.width),
+                height: Int(screen.frame.height),
+                bitsPerComponent: 8,
+                bytesPerRow: bytesPerRow,
+                space: imageMap.first!.1.colorSpace!,
+                bitmapInfo: imageMap.first!.1.bitmapInfo.rawValue)
+        // composite these window images
+        for item in imageMap {
+            let bounds = CGRect(dictionaryRepresentation: item.0.bounds()!)
+            // Convert the coordinate system, the origin of window is top-left, the image is bottom-left
+            // so we need to convert y-index
+            context?.draw(item.1, in: CGRect.init(x: bounds!.origin.x, y: screen.frame.height - bounds!.height - bounds!.origin.y, width: bounds!.width, height: bounds!.height))
+        }
+        return context?.makeImage()
+    }
+
+    func screenshot(_ win: Window) -> CGImage? {
         var windowId_ = self
-        let list = CGSHWCaptureWindowList(cgsMainConnectionId, &windowId_, 1, [.ignoreGlobalClipShape, .nominalResolution]).takeRetainedValue() as! [CGImage]
-        return list.first
+        if win.isFullscreen {
+            return fullScreenshot(win)
+        } else {
+            let list = CGSHWCaptureWindowList(cgsMainConnectionId, &windowId_, 1, [.ignoreGlobalClipShape, .nominalResolution]).takeRetainedValue() as! [CGImage]
+            return list.first
+        }
 
 //        // CGWindowListCreateImage
 //        return CGWindowListCreateImage(.null, .optionIncludingWindow, self, [.boundsIgnoreFraming, .bestResolution])

src/api-wrappers/HelperExtensions.swift

@@ -211,3 +211,18 @@ extension String {
         self = NSFileTypeForHFSTypeCode(fourCharCode).trimmingCharacters(in: CharacterSet(charactersIn: "'"))
     }
 }
+
+fileprivate var notificationKey: Int = 1
+
+extension NSRunningApplication {
+    var notification: Notification.Name? {
+        get {
+            return objc_getAssociatedObject(self, &notificationKey) as? Notification.Name
+        }
+        set {
+            if let value = newValue {
+                objc_setAssociatedObject(self, &notificationKey, value, .OBJC_ASSOCIATION_RETAIN_NONATOMIC)
+            }
+        }
+    }
+}

src/api-wrappers/PrivateApis.swift

@@ -122,10 +122,15 @@ func CGSCopyWindowsWithOptionsAndTags(_ cid: CGSConnectionID, _ owner: UInt32, _
 func CGSManagedDisplayGetCurrentSpace(_ cid: CGSConnectionID, _ displayUuid: CFString) -> CGSSpaceID
 
 // adds the provided windows to the provided spaces
-// * macOS 10.10+
+// * macOS 10.10-12.2
 @_silgen_name("CGSAddWindowsToSpaces")
 func CGSAddWindowsToSpaces(_ cid: CGSConnectionID, _ windows: NSArray, _ spaces: NSArray) -> Void
 
+// Move the given windows (CGWindowIDs) to the given space (CGSSpaceID)
+// * macOS 10.10+
+@_silgen_name("CGSMoveWindowsToManagedSpace")
+func CGSMoveWindowsToManagedSpace(_ cid: CGSConnectionID, _ windows: NSArray, _ space: CGSSpaceID) -> Void
+
 // remove the provided windows from the provided spaces
 // * macOS 10.10+
 @_silgen_name("CGSRemoveWindowsFromSpaces")

src/logic/Application.swift

@@ -84,8 +84,8 @@ class Application: NSObject {
     func observeNewWindows(_ group: DispatchGroup? = nil) {
         if runningApplication.isFinishedLaunching && runningApplication.activationPolicy != .prohibited {
             retryAxCallUntilTimeout(group, 5) { [weak self] in
-                guard let self = self else { return }
-                if let axWindows_ = try self.axUiElement!.windows(), axWindows_.count > 0 {
+                guard let self = self, let axWindows_ = try self.axUiElement!.windows() else { throw AxError.runtimeError }
+                if axWindows_.count > 0 {
                     // bug in macOS: sometimes the OS returns multiple duplicate windows (e.g. Mail.app starting at login)
                     let axWindows = try Array(Set(axWindows_)).compactMap {
                         if let wid = try $0.cgWindowId() {
@@ -114,14 +114,10 @@ class Application: NSObject {
                         let window = self.addWindowslessAppsIfNeeded()
                         App.app.refreshOpenUi(window)
                     }
-                    if group == nil && !self.wasLaunchedBeforeAltTab && (
-                        // workaround: opening an app while the active app is fullscreen; we wait out the space transition animation
-                        CGSSpaceGetType(cgsMainConnectionId, Spaces.currentSpaceId) == .fullscreen ||
-                            // workaround: some apps launch but have no window ready instantly. It's very unlikely an app would launch with no window
-                            // so we retry until timeout, in those rare cases (e.g. Bear.app)
-                            // we only do this for active app, to avoid wasting CPU, with the trade-off of maybe missing some windows
-                            self.runningApplication.isActive
-                    ) {
+                    // workaround: some apps launch but have no window ready instantly. It's very unlikely an app would launch with no window
+                    // so we retry until timeout, in those rare cases (e.g. Bear.app)
+                    // we only do this for active app, to avoid wasting CPU, with the trade-off of maybe missing some windows
+                    if group == nil && self.runningApplication.notification == NSWorkspace.didActivateApplicationNotification {
                         throw AxError.runtimeError
                     }
                 }
@@ -144,6 +140,42 @@ class Application: NSObject {
         return windows
     }
 
+    func getOtherSpaceWindows(_ windowsOnlyOnOtherSpaces: [CGWindowID]) -> [Window] {
+        var otherSpaceWindows: [Window] = []
+        for winId in windowsOnlyOnOtherSpaces {
+            let cgWinArray = CGWindowListCopyWindowInfo([.excludeDesktopElements, .optionIncludingWindow], winId) as! [CGWindow]
+            // get current app's windows only on other space
+            guard runningApplication.processIdentifier == cgWinArray.first!.ownerPID()
+                          && cgWinArray.first!.id() != nil
+                          && cgWinArray.first!.isNotMenubarOrOthers()
+                          && cgWinArray.first!.bounds() != nil
+                          && CGRect(dictionaryRepresentation: cgWinArray.first!.bounds()!)!.width > 100
+                          && CGRect(dictionaryRepresentation: cgWinArray.first!.bounds()!)!.height > 100
+                    else { continue }
+            guard let capture = CGWindowListCreateImage(CGRect.null, .optionIncludingWindow, cgWinArray.first!.id()!, .boundsIgnoreFraming) else { continue }
+            let win = Window(self, cgWinArray.first!)
+            Windows.appendAndUpdateFocus(win)
+            otherSpaceWindows.append(win)
+        }
+        return otherSpaceWindows
+    }
+
+    func addOtherSpaceWindows(_ windowsOnlyOnOtherSpaces: [CGWindowID]) {
+        if runningApplication.isFinishedLaunching && runningApplication.activationPolicy != .prohibited {
+            retryAxCallUntilTimeout { [weak self] in
+                guard let self = self else { return }
+                DispatchQueue.main.async { [weak self] in
+                    guard let self = self else { return }
+                    var windows = self.getOtherSpaceWindows(windowsOnlyOnOtherSpaces)
+                    if let window = self.addWindowslessAppsIfNeeded() {
+                        windows.append(contentsOf: window)
+                    }
+                    App.app.refreshOpenUi(windows)
+                }
+            }
+        }
+    }
+
     func addWindowslessAppsIfNeeded() -> [Window]? {
         if !Preferences.hideWindowlessApps &&
                runningApplication.activationPolicy == .regular &&

src/logic/Applications.swift

@@ -14,10 +14,19 @@ class Applications {
         _ = group.wait(wallTimeout: .now() + .seconds(2))
     }
 
+    static func addOtherSpaceWindows(_ windowsOnlyOnOtherSpaces: [CGWindowID]) {
+        for app in list {
+            app.wasLaunchedBeforeAltTab = true
+            guard app.runningApplication.isFinishedLaunching else { continue }
+            app.addOtherSpaceWindows(windowsOnlyOnOtherSpaces)
+        }
+    }
+
     static func initialDiscovery() {
         addInitialRunningApplications()
         addInitialRunningApplicationsWindows()
         WorkspaceEvents.observeRunningApplications()
+        WorkspaceEvents.registerFrontAppChangeNote()
     }
 
     static func addInitialRunningApplications() {
@@ -31,17 +40,18 @@ class Applications {
             let windowsOnOtherSpaces = Spaces.windowsInSpaces(otherSpaces)
             let windowsOnlyOnOtherSpaces = Array(Set(windowsOnOtherSpaces).subtracting(windowsOnCurrentSpace))
             if windowsOnlyOnOtherSpaces.count > 0 {
-                // on initial launch, we use private APIs to bring windows from other spaces into the current space, observe them, then remove them from the current space
-                CGSAddWindowsToSpaces(cgsMainConnectionId, windowsOnlyOnOtherSpaces as NSArray, [Spaces.currentSpaceId])
-                Applications.observeNewWindowsBlocking()
-                CGSRemoveWindowsFromSpaces(cgsMainConnectionId, windowsOnlyOnOtherSpaces as NSArray, [Spaces.currentSpaceId])
+                // Currently we add those window in other space without AXUIElement init
+                // We don't need to get the AXUIElement until we focus these windows.
+                // when we need to focus these windows, we use the helper window to take us to that space,
+                // then get the AXUIElement, and finally focus that window.
+                Applications.addOtherSpaceWindows(windowsOnlyOnOtherSpaces)
             }
         }
     }
 
     static func addRunningApplications(_ runningApps: [NSRunningApplication], _ wasLaunchedBeforeAltTab: Bool = false) {
         runningApps.forEach {
-            if isActualApplication($0) {
+            if isActualApplication($0, wasLaunchedBeforeAltTab) {
                 Applications.list.append(Application($0, wasLaunchedBeforeAltTab))
             }
         }
@@ -101,10 +111,37 @@ class Applications {
         }
     }
 
-    private static func isActualApplication(_ app: NSRunningApplication) -> Bool {
+    private static func isActualApplication(_ app: NSRunningApplication, _ wasLaunchedBeforeAltTab: Bool = false) -> Bool {
         // an app can start with .activationPolicy == .prohibited, then transition to != .prohibited later
         // an app can be both activationPolicy == .accessory and XPC (e.g. com.apple.dock.etci)
-        return (isNotXpc(app) || isAndroidEmulator(app)) && !app.processIdentifier.isZombie()
+        return isAnWindowApplication(app, wasLaunchedBeforeAltTab) && (isNotXpc(app) || isAndroidEmulator(app)) && !app.processIdentifier.isZombie()
+    }
+
+    private static func isAnWindowApplication(_ app: NSRunningApplication, _ wasLaunchedBeforeAltTab: Bool = false) -> Bool {
+        if (wasLaunchedBeforeAltTab) {
+            // For wasLaunchedBeforeAltTab=true, we assume that those apps are all launched, if they are programs with windows.
+            // Even if it has 0 windows at this point, axUiElement.windows() will not throw an exception. If they are programs without windows, then axUiElement.windows() will throw an exception.
+            // Here I consider there is an edge case where AltTab is starting up and this program has been loading, then it is possible that axUiElement.windows() will throw an exception.
+            // I'm not quite sure if this happens, but even if it does, then after restarting this application, AltTab captures its window without any problem. I think this happens rarely.
+            let allWindows = CGWindow.windows(.optionAll)
+            guard let winApp = (allWindows.first { app.processIdentifier == $0.ownerPID()
+                    && $0.isNotMenubarOrOthers()
+                    && $0.id() != nil
+                    && $0.bounds() != nil
+                    && CGRect(dictionaryRepresentation: $0.bounds()!)!.width > 0
+                    && CGRect(dictionaryRepresentation: $0.bounds()!)!.height > 0
+            }) else {
+                return false
+            }
+            return true
+        } else {
+            // Because we only add the application when we receive the didActivateApplicationNotification.
+            // So here is actually the handling for the case wasLaunchedBeforeAltTab=false. For applications where wasLaunchedBeforeAltTab=true, the majority of isActive is false.
+            // The reason for not using axUiElement.windows() here as a way to determine if it is a window application is that
+            // When we receive the didActivateApplicationNotification notification, the application may still be loading and axUiElement.windows() will throw an exception
+            // So we use isActive to determine if it is a window application, even if the application is not frontmost, isActive is still true at this time
+            return true;
+        }
     }
 
     private static func isNotXpc(_ app: NSRunningApplication) -> Bool {

src/logic/HelperWindow.swift

@@ -0,0 +1,18 @@
+import Cocoa
+/**
+ * we use this window to help us switch to another space
+ */
+class HelperWindow: NSWindow {
+    var canBecomeKey_ = true
+    override var canBecomeKey: Bool { canBecomeKey_ }
+    convenience init() {
+        self.init(contentRect: .zero, styleMask: [.borderless], backing: .buffered, defer: false)
+        setupWindow()
+    }
+
+    private func setupWindow() {
+        isReleasedWhenClosed = false
+        hidesOnDeactivate = false
+        title = "Helper Window"
+    }
+}

src/logic/Spaces.swift

@@ -6,14 +6,25 @@ class Spaces {
     static var visibleSpaces = [CGSSpaceID]()
     static var screenSpacesMap = [ScreenUuid: [CGSSpaceID]]()
     static var idsAndIndexes = [(CGSSpaceID, SpaceIndex)]()
+    static var spaceFrameMap = [(CGSSpaceID, NSScreen)]()
 
     static func observeSpaceChanges() {
         NSWorkspace.shared.notificationCenter.addObserver(forName: NSWorkspace.activeSpaceDidChangeNotification, object: nil, queue: nil, using: { _ in
             debugPrint("OS event", "activeSpaceDidChangeNotification")
             refreshAllIdsAndIndexes()
             updateCurrentSpace()
             // if UI was kept open during Space transition, the Spaces may be obsolete; we refresh them
-            Windows.list.forEachAsync { $0.updatesWindowSpace() }
+            Windows.list.forEachAsync {
+                $0.updatesWindowSpace()
+                // we need to set up AXUIElement for those invisiable window launched before AltTab when space changed
+                if $0.axUiElement == nil && $0.spaceId == currentSpaceId && !$0.isWindowlessApp {
+                    do {
+                        try $0.getAxUiElementAndObserveEvents()
+                    } catch {
+                        debugPrint("can not setUpMissingInfoForOtherWindows for", $0.application.runningApplication.bundleIdentifier)
+                    }
+                }
+            }
         })
         NSWorkspace.shared.notificationCenter.addObserver(forName: NSApplication.didChangeScreenParametersNotification, object: nil, queue: nil, using: { _ in
             debugPrint("OS event", "didChangeScreenParametersNotification")
@@ -47,15 +58,18 @@ class Spaces {
         idsAndIndexes.removeAll()
         screenSpacesMap.removeAll()
         visibleSpaces.removeAll()
+        spaceFrameMap.removeAll()
         var spaceIndex = SpaceIndex(1)
         (CGSCopyManagedDisplaySpaces(cgsMainConnectionId) as! [NSDictionary]).forEach { (screen: NSDictionary) in
             var display = screen["Display Identifier"] as! ScreenUuid
             if display as String == "Main", let mainUuid = NSScreen.main?.uuid() {
                 display = mainUuid
             }
+            let nsScreen = NSScreen.screens.first { $0.uuid() == display } as! NSScreen
             (screen["Spaces"] as! [NSDictionary]).forEach { (space: NSDictionary) in
                 let spaceId = space["id64"] as! CGSSpaceID
                 idsAndIndexes.append((spaceId, spaceIndex))
+                spaceFrameMap.append((spaceId, nsScreen))
                 screenSpacesMap[display, default: []].append(spaceId)
                 spaceIndex += 1
             }