From 50eaed6959a0d0001748bb8cc9495b89a34898ab Mon Sep 17 00:00:00 2001 From: metacodes Date: Thu, 28 Apr 2022 12:04:36 +0800 Subject: [PATCH] fix: fix issue that we can't grab windows in other spaces from macos 12.2 --- src/api-wrappers/CGWindowID.swift | 60 +++++++++++++++++++++-- src/api-wrappers/PrivateApis.swift | 7 ++- src/logic/Application.swift | 36 ++++++++++++++ src/logic/Applications.swift | 25 ++++++++-- src/logic/HelperWindow.swift | 18 +++++++ src/logic/Spaces.swift | 16 ++++++- src/logic/Window.swift | 76 ++++++++++++++++++++++++++++-- src/ui/App.swift | 7 +++ 8 files changed, 230 insertions(+), 15 deletions(-) create mode 100644 src/logic/HelperWindow.swift diff --git a/src/api-wrappers/CGWindowID.swift b/src/api-wrappers/CGWindowID.swift index 14faf0dc..8cfd4442 100644 --- a/src/api-wrappers/CGWindowID.swift +++ b/src/api-wrappers/CGWindowID.swift @@ -19,11 +19,63 @@ extension CGWindowID { return CGSCopySpacesForWindows(cgsMainConnectionId, CGSSpaceMask.all.rawValue, [self] as CFArray) as! [CGSSpaceID] } - func screenshot() -> CGImage? { - // CGSHWCaptureWindowList + // fullscreen has multiple windows + // e.g. Notes.app has a toolbar window and a main window + // We need to composite these window images + func fullScreenshot(_ win: Window) -> CGImage? { + var height: CGFloat = 0; + var width: CGFloat = 0; + var imageMap = [(CGWindow, CGImage)]() + var maxWidthWindowId: CGWindowID = 0 + let screen = Spaces.spaceFrameMap.first { $0.0 == win.spaceId }!.1 + var windowsInSpaces = Spaces.windowsInSpaces([win.spaceId]) // The returned windows are sorted from highest to lowest according to the z-index + var windowsToCapture: [CGWindow] = [] + // find current app's window in the fullscreen space + for item in windowsInSpaces { + let cgWin = CGWindowListCopyWindowInfo([.excludeDesktopElements, .optionIncludingWindow], item) as! [CGWindow] + guard cgWin.first!.isNotMenubarOrOthers(), + cgWin.first!.ownerPID() == win.application.runningApplication.processIdentifier, + cgWin.first!.bounds() != nil, + let bounds = CGRect(dictionaryRepresentation: cgWin.first!.bounds()!), bounds.height > 0, bounds.width > 0 else { continue } + windowsToCapture.append(cgWin.first!) + } + // Drawing images from lowest to highest base on the z-index + windowsToCapture = windowsToCapture.reversed() + for item in windowsToCapture { + let bounds = CGRect(dictionaryRepresentation: item.bounds()!) + if width < bounds!.width { + maxWidthWindowId = item.id()! + } + var windowId = item.id()! + let list = CGSHWCaptureWindowList(cgsMainConnectionId, &windowId, 1, [.ignoreGlobalClipShape, .nominalResolution]).takeRetainedValue() as! [CGImage] + imageMap.append((item, list.first!)) + } + let bytesPerRow = imageMap.first { $0.0.id()! == maxWidthWindowId }!.1.bytesPerRow + var context = CGContext.init(data: nil, + width: Int(screen.frame.width), + height: Int(screen.frame.height), + bitsPerComponent: 8, + bytesPerRow: bytesPerRow, + space: imageMap.first!.1.colorSpace!, + bitmapInfo: imageMap.first!.1.bitmapInfo.rawValue) + // composite these window images + for item in imageMap { + let bounds = CGRect(dictionaryRepresentation: item.0.bounds()!) + // Convert the coordinate system, the origin of window is top-left, the image is bottom-left + // so we need to convert y-index + context?.draw(item.1, in: CGRect.init(x: bounds!.origin.x, y: screen.frame.height - bounds!.height - bounds!.origin.y, width: bounds!.width, height: bounds!.height)) + } + return context?.makeImage() + } + + func screenshot(_ win: Window) -> CGImage? { var windowId_ = self - let list = CGSHWCaptureWindowList(cgsMainConnectionId, &windowId_, 1, [.ignoreGlobalClipShape, .nominalResolution]).takeRetainedValue() as! [CGImage] - return list.first + if win.isFullscreen { + return fullScreenshot(win) + } else { + let list = CGSHWCaptureWindowList(cgsMainConnectionId, &windowId_, 1, [.ignoreGlobalClipShape, .nominalResolution]).takeRetainedValue() as! [CGImage] + return list.first + } // // CGWindowListCreateImage // return CGWindowListCreateImage(.null, .optionIncludingWindow, self, [.boundsIgnoreFraming, .bestResolution]) diff --git a/src/api-wrappers/PrivateApis.swift b/src/api-wrappers/PrivateApis.swift index c7edbed1..fe9c40fa 100644 --- a/src/api-wrappers/PrivateApis.swift +++ b/src/api-wrappers/PrivateApis.swift @@ -122,10 +122,15 @@ func CGSCopyWindowsWithOptionsAndTags(_ cid: CGSConnectionID, _ owner: UInt32, _ func CGSManagedDisplayGetCurrentSpace(_ cid: CGSConnectionID, _ displayUuid: CFString) -> CGSSpaceID // adds the provided windows to the provided spaces -// * macOS 10.10+ +// * macOS 10.10-12.2 @_silgen_name("CGSAddWindowsToSpaces") func CGSAddWindowsToSpaces(_ cid: CGSConnectionID, _ windows: NSArray, _ spaces: NSArray) -> Void +// Move the given windows (CGWindowIDs) to the given space (CGSSpaceID) +// * macOS 10.10+ +@_silgen_name("CGSMoveWindowsToManagedSpace") +func CGSMoveWindowsToManagedSpace(_ cid: CGSConnectionID, _ windows: NSArray, _ space: CGSSpaceID) -> Void + // remove the provided windows from the provided spaces // * macOS 10.10+ @_silgen_name("CGSRemoveWindowsFromSpaces") diff --git a/src/logic/Application.swift b/src/logic/Application.swift index 85fffd3f..25486570 100644 --- a/src/logic/Application.swift +++ b/src/logic/Application.swift @@ -140,6 +140,42 @@ class Application: NSObject { return windows } + func getOtherSpaceWindows(_ windowsOnlyOnOtherSpaces: [CGWindowID]) -> [Window] { + var otherSpaceWindows: [Window] = [] + for winId in windowsOnlyOnOtherSpaces { + let cgWinArray = CGWindowListCopyWindowInfo([.excludeDesktopElements, .optionIncludingWindow], winId) as! [CGWindow] + // get current app's windows only on other space + guard runningApplication.processIdentifier == cgWinArray.first!.ownerPID() + && cgWinArray.first!.id() != nil + && cgWinArray.first!.isNotMenubarOrOthers() + && cgWinArray.first!.bounds() != nil + && CGRect(dictionaryRepresentation: cgWinArray.first!.bounds()!)!.width > 100 + && CGRect(dictionaryRepresentation: cgWinArray.first!.bounds()!)!.height > 100 + else { continue } + guard let capture = CGWindowListCreateImage(CGRect.null, .optionIncludingWindow, cgWinArray.first!.id()!, .boundsIgnoreFraming) else { continue } + let win = Window(self, cgWinArray.first!) + Windows.appendAndUpdateFocus(win) + otherSpaceWindows.append(win) + } + return otherSpaceWindows + } + + func addOtherSpaceWindows(_ windowsOnlyOnOtherSpaces: [CGWindowID]) { + if runningApplication.isFinishedLaunching && runningApplication.activationPolicy != .prohibited { + retryAxCallUntilTimeout { [weak self] in + guard let self = self else { return } + DispatchQueue.main.async { [weak self] in + guard let self = self else { return } + var windows = self.getOtherSpaceWindows(windowsOnlyOnOtherSpaces) + if let window = self.addWindowslessAppsIfNeeded() { + windows.append(contentsOf: window) + } + App.app.refreshOpenUi(windows) + } + } + } + } + func addWindowslessAppsIfNeeded() -> [Window]? { if !Preferences.hideWindowlessApps && runningApplication.activationPolicy == .regular && diff --git a/src/logic/Applications.swift b/src/logic/Applications.swift index 5d1cf1f7..ed469096 100644 --- a/src/logic/Applications.swift +++ b/src/logic/Applications.swift @@ -14,6 +14,14 @@ class Applications { _ = group.wait(wallTimeout: .now() + .seconds(2)) } + static func addOtherSpaceWindows(_ windowsOnlyOnOtherSpaces: [CGWindowID]) { + for app in list { + app.wasLaunchedBeforeAltTab = true + guard app.runningApplication.isFinishedLaunching else { continue } + app.addOtherSpaceWindows(windowsOnlyOnOtherSpaces) + } + } + static func initialDiscovery() { addInitialRunningApplications() addInitialRunningApplicationsWindows() @@ -32,10 +40,11 @@ class Applications { let windowsOnOtherSpaces = Spaces.windowsInSpaces(otherSpaces) let windowsOnlyOnOtherSpaces = Array(Set(windowsOnOtherSpaces).subtracting(windowsOnCurrentSpace)) if windowsOnlyOnOtherSpaces.count > 0 { - // on initial launch, we use private APIs to bring windows from other spaces into the current space, observe them, then remove them from the current space - CGSAddWindowsToSpaces(cgsMainConnectionId, windowsOnlyOnOtherSpaces as NSArray, [Spaces.currentSpaceId]) - Applications.observeNewWindowsBlocking() - CGSRemoveWindowsFromSpaces(cgsMainConnectionId, windowsOnlyOnOtherSpaces as NSArray, [Spaces.currentSpaceId]) + // Currently we add those window in other space without AXUIElement init + // We don't need to get the AXUIElement until we focus these windows. + // when we need to focus these windows, we use the helper window to take us to that space, + // then get the AXUIElement, and finally focus that window. + Applications.addOtherSpaceWindows(windowsOnlyOnOtherSpaces) } } } @@ -115,7 +124,13 @@ class Applications { // Here I consider there is an edge case where AltTab is starting up and this program has been loading, then it is possible that axUiElement.windows() will throw an exception. // I'm not quite sure if this happens, but even if it does, then after restarting this application, AltTab captures its window without any problem. I think this happens rarely. let allWindows = CGWindow.windows(.optionAll) - guard let winApp = (allWindows.first { app.processIdentifier == $0.ownerPID() && $0.isNotMenubarOrOthers() && $0.id() != nil && $0.title() != nil}) else { + guard let winApp = (allWindows.first { app.processIdentifier == $0.ownerPID() + && $0.isNotMenubarOrOthers() + && $0.id() != nil + && $0.bounds() != nil + && CGRect(dictionaryRepresentation: $0.bounds()!)!.width > 0 + && CGRect(dictionaryRepresentation: $0.bounds()!)!.height > 0 + }) else { return false } return true diff --git a/src/logic/HelperWindow.swift b/src/logic/HelperWindow.swift new file mode 100644 index 00000000..be791c0f --- /dev/null +++ b/src/logic/HelperWindow.swift @@ -0,0 +1,18 @@ +import Cocoa +/** + * we use this window to help us switch to another space + */ +class HelperWindow: NSWindow { + var canBecomeKey_ = true + override var canBecomeKey: Bool { canBecomeKey_ } + convenience init() { + self.init(contentRect: .zero, styleMask: [.borderless], backing: .buffered, defer: false) + setupWindow() + } + + private func setupWindow() { + isReleasedWhenClosed = false + hidesOnDeactivate = false + title = "Helper Window" + } +} diff --git a/src/logic/Spaces.swift b/src/logic/Spaces.swift index 9b616d85..9e469426 100644 --- a/src/logic/Spaces.swift +++ b/src/logic/Spaces.swift @@ -6,6 +6,7 @@ class Spaces { static var visibleSpaces = [CGSSpaceID]() static var screenSpacesMap = [ScreenUuid: [CGSSpaceID]]() static var idsAndIndexes = [(CGSSpaceID, SpaceIndex)]() + static var spaceFrameMap = [(CGSSpaceID, NSScreen)]() static func observeSpaceChanges() { NSWorkspace.shared.notificationCenter.addObserver(forName: NSWorkspace.activeSpaceDidChangeNotification, object: nil, queue: nil, using: { _ in @@ -13,7 +14,17 @@ class Spaces { refreshAllIdsAndIndexes() updateCurrentSpace() // if UI was kept open during Space transition, the Spaces may be obsolete; we refresh them - Windows.list.forEachAsync { $0.updatesWindowSpace() } + Windows.list.forEachAsync { + $0.updatesWindowSpace() + // we need to set up AXUIElement for those invisiable window launched before AltTab when space changed + if $0.axUiElement == nil && $0.spaceId == currentSpaceId && !$0.isWindowlessApp { + do { + try $0.getAxUiElementAndObserveEvents() + } catch { + debugPrint("can not setUpMissingInfoForOtherWindows for", $0.application.runningApplication.bundleIdentifier) + } + } + } }) NSWorkspace.shared.notificationCenter.addObserver(forName: NSApplication.didChangeScreenParametersNotification, object: nil, queue: nil, using: { _ in debugPrint("OS event", "didChangeScreenParametersNotification") @@ -47,15 +58,18 @@ class Spaces { idsAndIndexes.removeAll() screenSpacesMap.removeAll() visibleSpaces.removeAll() + spaceFrameMap.removeAll() var spaceIndex = SpaceIndex(1) (CGSCopyManagedDisplaySpaces(cgsMainConnectionId) as! [NSDictionary]).forEach { (screen: NSDictionary) in var display = screen["Display Identifier"] as! ScreenUuid if display as String == "Main", let mainUuid = NSScreen.main?.uuid() { display = mainUuid } + let nsScreen = NSScreen.screens.first { $0.uuid() == display } as! NSScreen (screen["Spaces"] as! [NSDictionary]).forEach { (space: NSDictionary) in let spaceId = space["id64"] as! CGSSpaceID idsAndIndexes.append((spaceId, spaceIndex)) + spaceFrameMap.append((spaceId, nsScreen)) screenSpacesMap[display, default: []].append(spaceId) spaceIndex += 1 } diff --git a/src/logic/Window.swift b/src/logic/Window.swift index 2b173515..8aa1011a 100644 --- a/src/logic/Window.swift +++ b/src/logic/Window.swift @@ -60,6 +60,44 @@ class Window { debugPrint("Adding app-window", title ?? "nil", application.runningApplication.bundleIdentifier ?? "nil") } + // we init a window when this window is not at current active space + init(_ application: Application, _ cgWindow: CGWindow) { + self.application = application + self.title = bestEffortTitle(cgWindow.title()) + self.cgWindowId = cgWindow.id()! + let bounds = CGRect(dictionaryRepresentation: cgWindow.bounds()!) + self.size = bounds!.size + self.position = bounds!.origin + updatesWindowSpace() + if CGSSpaceGetType(cgsMainConnectionId, self.spaceId) == .fullscreen { + self.isFullscreen = true + } else { + self.isFullscreen = false + } + self.isMinimized = false + if !Preferences.hideThumbnails { + refreshThumbnail() + } + application.removeWindowslessAppWindow() + debugPrint("Adding app-window", title ?? "nil", application.runningApplication.bundleIdentifier ?? "nil") + } + + func getAxUiElementAndObserveEvents() throws { + guard let windows = try self.application.axUiElement?.windows(), windows.count > 0 else { + debugPrint("try to getAxUiElementAndObserveEvents nil", self.application.runningApplication.bundleIdentifier, self.title, self.spaceId) + return + } + for win in windows { + if try cgWindowId == win.cgWindowId() { + self.axUiElement = win + self.isMinimized = try self.axUiElement.isMinimized() + self.observeEvents() + return + } + } + debugPrint("try to getAxUiElementAndObserveEvents failed", self.application.runningApplication.bundleIdentifier, self.title, self.spaceId) + } + deinit { debugPrint("Deinit window", title ?? "nil", application.runningApplication.bundleIdentifier ?? "nil") } @@ -83,13 +121,13 @@ class Window { } func refreshThumbnail() { - guard let cgImage = cgWindowId.screenshot() else { return } + guard let cgImage = cgWindowId.screenshot(self) else { return } thumbnail = NSImage(cgImage: cgImage, size: NSSize(width: cgImage.width, height: cgImage.height)) thumbnailFullSize = thumbnail!.size } func close() { - if isWindowlessApp { return } + if isWindowlessApp || self.axUiElement == nil { return } BackgroundWork.accessibilityCommandsQueue.asyncWithCap { [weak self] in guard let self = self else { return } if self.isFullscreen { @@ -102,7 +140,7 @@ class Window { } func minDemin() { - if isWindowlessApp { return } + if isWindowlessApp || self.axUiElement == nil { return } BackgroundWork.accessibilityCommandsQueue.asyncWithCap { [weak self] in guard let self = self else { return } if self.isFullscreen { @@ -119,7 +157,7 @@ class Window { } func toggleFullscreen() { - if isWindowlessApp { return } + if isWindowlessApp || self.axUiElement == nil { return } BackgroundWork.accessibilityCommandsQueue.asyncWithCap { [weak self] in guard let self = self else { return } self.axUiElement.setAttribute(kAXFullscreenAttribute, !self.isFullscreen) @@ -145,6 +183,11 @@ class Window { } } + func gotoSpace(_ spaceId: CGSSpaceID) { + CGSMoveWindowsToManagedSpace(cgsMainConnectionId, [App.app.helperWindow.windowNumber] as NSArray, spaceId) + App.app.showHelperWindow() + } + func focus() { if isWindowlessApp { if let bundleID = application.runningApplication.bundleIdentifier { @@ -152,6 +195,31 @@ class Window { } else { application.runningApplication.activate(options: .activateIgnoringOtherApps) } + } else if self.axUiElement == nil { + // the window is in other space, we can not get AXUIElement before we go to it's space + // when we want to focus the window, it means we want to go to the space and focus the window + // so we just go to that space by using helperWindow, then we get AXUIElement and focus the window + gotoSpace(self.spaceId) + do { + retryAxCallUntilTimeout { [weak self] in + guard let self = self else { return } + try self.getAxUiElementAndObserveEvents() + if self.axUiElement == nil { + // retry until we get axUiElement + throw AxError.runtimeError + } + BackgroundWork.accessibilityCommandsQueue.asyncWithCap { [weak self] in + guard let self = self else { return } + var psn = ProcessSerialNumber() + GetProcessForPID(self.application.pid, &psn) + _SLPSSetFrontProcessWithOptions(&psn, self.cgWindowId, .userGenerated) + self.makeKeyWindow(psn) + self.axUiElement.focusWindow() + } + } + } catch { + debugPrint("can not setUpMissingInfoForCurrentWindow for", self.application.runningApplication.bundleIdentifier) + } } else { // macOS bug: when switching to a System Preferences window in another space, it switches to that space, // but quickly switches back to another window in that space diff --git a/src/ui/App.swift b/src/ui/App.swift index 214cf67d..21c31fe8 100644 --- a/src/ui/App.swift +++ b/src/ui/App.swift @@ -19,6 +19,7 @@ class App: AppCenterApplication, NSApplicationDelegate { static var app: App! var thumbnailsPanel: ThumbnailsPanel! var preferencesWindow: PreferencesWindow! + var helperWindow: HelperWindow! var feedbackWindow: FeedbackWindow! var isFirstSummon = true var appIsBeingUsed = false @@ -59,6 +60,7 @@ class App: AppCenterApplication, NSApplicationDelegate { Spaces.initialDiscovery() Applications.initialDiscovery() self.preferencesWindow = PreferencesWindow() + self.helperWindow = HelperWindow() self.feedbackWindow = FeedbackWindow() KeyboardEvents.addEventHandlers() MouseEvents.observe() @@ -158,6 +160,11 @@ class App: AppCenterApplication, NSApplicationDelegate { showSecondaryWindow(preferencesWindow) } + // focus the helper window and we can go to that space + @objc func showHelperWindow() { + showSecondaryWindow(helperWindow) + } + func showSecondaryWindow(_ window: NSWindow?) { if let window = window { NSScreen.preferred().repositionPanel(window, .appleCentered)