diff --git a/README.md b/README.md index f62d63e..a266b3a 100644 --- a/README.md +++ b/README.md @@ -104,4 +104,10 @@ To build the wheel from source, run: python setup.py bdist_wheel -(Note that you may need to make sure `wheel` is installed) \ No newline at end of file +(Note that you may need to make sure `wheel` is installed) + +## Special thanks ## + +Debugging contributions: + +* [maxdule](https://github.com/maxdule) \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index c5f0e13..53242d4 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -20,6 +20,7 @@ import sys path = os.path.abspath('../..') sys.path.insert(0, path) +from lackey import __version__ try: #py3 import from unittest.mock import MagicMock @@ -78,9 +79,9 @@ def __getattr__(cls, name): # built documents. # # The short X.Y version. -version = u'0.4' +version = __version__ # The full version, including alpha/beta/rc tags. -release = u'0.4.2a1' +release = __version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/lackey/Exceptions.py b/lackey/Exceptions.py index 15bae06..fff3698 100644 --- a/lackey/Exceptions.py +++ b/lackey/Exceptions.py @@ -1,3 +1,6 @@ +""" Custom exceptions for Sikuli script """ + class FindFailed(Exception): - def __init__(self,*args,**kwargs): - Exception.__init__(self,*args,**kwargs) \ No newline at end of file + """ Exception: Unable to find the searched item """ + def __init__(self, *args, **kwargs): + Exception.__init__(self, *args, **kwargs) diff --git a/lackey/KeyCodes.py b/lackey/KeyCodes.py index 2c035d3..5b5bac6 100644 --- a/lackey/KeyCodes.py +++ b/lackey/KeyCodes.py @@ -1,68 +1,70 @@ class Button(): - LEFT = 0 - CENTER = 1 - RIGHT = 2 + LEFT = 0 + CENTER = 1 + RIGHT = 2 class Key(): - """ Key codes for PlatformManager.TypeKeys() function. Can be entered directly or concatenated with an existing string. """ - ENTER = "{ENTER}" - ESC = "{ESC}" - BACKSPACE = "{BACKSPACE}" - DELETE = "{DELETE}" - F1 = "{F1}" - F2 = "{F2}" - F3 = "{F3}" - F4 = "{F4}" - F5 = "{F5}" - F6 = "{F6}" - F7 = "{F7}" - F8 = "{F8}" - F9 = "{F9}" - F10 = "{F10}" - F11 = "{F11}" - F12 = "{F12}" - F13 = "{F13}" - F14 = "{F14}" - F15 = "{F15}" - F16 = "{F16}" - HOME = "{HOME}" - END = "{END}" - LEFT = "{LEFT}" - RIGHT = "{RIGHT}" - DOWN = "{DOWN}" - UP = "{UP}" - PAGE_DOWN = "{PAGE_DOWN}" - PAGE_UP = "{PAGE_UP}" - TAB = "{TAB}" - CAPS_LOCK = "{CAPS_LOCK}" - NUM_LOCK = "{NUM_LOCK}" - SCROLL_LOCK = "{SCROLL_LOCK}" - INSERT = "{INSERT}" - SPACE = "{SPACE}" - PRINTSCREEN = "{PRTSC}" - ALT = "{ALT}" - CMD = "{CMD}" - CONTROL = "{CTRL}" - META = "{META}" - SHIFT = "{SHIFT}" - WIN = "{WIN}" - PAUSE = "{PAUSE}" - NUM_0 = "{NUM_0}" - NUM_1 = "{NUM_1}" - NUM_2 = "{NUM_2}" - NUM_3 = "{NUM_3}" - NUM_4 = "{NUM_4}" - NUM_5 = "{NUM_5}" - NUM_6 = "{NUM_6}" - NUM_7 = "{NUM_7}" - NUM_8 = "{NUM_8}" - NUM_9 = "{NUM_9}" + """ Key codes for PlatformManager.TypeKeys() function. + + Can be entered directly or concatenated with an existing string. """ + ENTER = "{ENTER}" + ESC = "{ESC}" + BACKSPACE = "{BACKSPACE}" + DELETE = "{DELETE}" + F1 = "{F1}" + F2 = "{F2}" + F3 = "{F3}" + F4 = "{F4}" + F5 = "{F5}" + F6 = "{F6}" + F7 = "{F7}" + F8 = "{F8}" + F9 = "{F9}" + F10 = "{F10}" + F11 = "{F11}" + F12 = "{F12}" + F13 = "{F13}" + F14 = "{F14}" + F15 = "{F15}" + F16 = "{F16}" + HOME = "{HOME}" + END = "{END}" + LEFT = "{LEFT}" + RIGHT = "{RIGHT}" + DOWN = "{DOWN}" + UP = "{UP}" + PAGE_DOWN = "{PGDN}" + PAGE_UP = "{PGUP}" + TAB = "{TAB}" + CAPS_LOCK = "{CAPS_LOCK}" + NUM_LOCK = "{NUM_LOCK}" + SCROLL_LOCK = "{SCROLL_LOCK}" + INSERT = "{INSERT}" + SPACE = "{SPACE}" + PRINTSCREEN = "{PRTSC}" + ALT = "{ALT}" + CMD = "{CMD}" + CONTROL = "{CTRL}" + META = "{META}" + SHIFT = "{SHIFT}" + WIN = "{WIN}" + PAUSE = "{PAUSE}" + NUM_0 = "{NUM_0}" + NUM_1 = "{NUM_1}" + NUM_2 = "{NUM_2}" + NUM_3 = "{NUM_3}" + NUM_4 = "{NUM_4}" + NUM_5 = "{NUM_5}" + NUM_6 = "{NUM_6}" + NUM_7 = "{NUM_7}" + NUM_8 = "{NUM_8}" + NUM_9 = "{NUM_9}" class KeyModifier(): - """ Key modifiers precede either a single key [e.g., ^v] or a set of characters within parentheses [e.g., +(hello)] """ - CTRL = "^" - SHIFT = "+" - ALT = "%" - META = "@" - CMD = "@" - WIN = "@" \ No newline at end of file + """ Key modifiers precede either a single key [e.g., ^v] or a set of characters within parentheses [e.g., +(hello)] """ + CTRL = "{CTRL}" + SHIFT = "{SHIFT}" + ALT = "{ALT}" + META = "{WIN}" + CMD = "{WIN}" + WIN = "{WIN}" \ No newline at end of file diff --git a/lackey/PlatformManagerWindows.py b/lackey/PlatformManagerWindows.py index d817667..fda6ae1 100644 --- a/lackey/PlatformManagerWindows.py +++ b/lackey/PlatformManagerWindows.py @@ -1,866 +1,818 @@ +""" Platform-specific code for Windows is encapsulated in this module. """ + import os import re import time import numpy import ctypes try: - import Tkinter as tk + import Tkinter as tk except ImportError: - import tkinter as tk + import tkinter as tk from ctypes import wintypes +import keyboard +from keyboard import mouse from PIL import Image, ImageTk, ImageOps from .Settings import Debug class PlatformManagerWindows(object): - """ Abstracts Windows-specific OS-level features like mouse/keyboard control """ - def __init__(self): - #self._root = tk.Tk() - #self._root.overrideredirect(1) - #self._root.withdraw() - user32 = ctypes.WinDLL('user32', use_last_error=True) - gdi32 = ctypes.WinDLL('gdi32', use_last_error=True) - kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) - psapi = ctypes.WinDLL('psapi', use_last_error=True) - self._user32 = user32 - self._gdi32 = gdi32 - self._kernel32 = kernel32 - self._psapi = psapi - self._INPUT_MOUSE = 0 - self._INPUT_KEYBOARD = 1 - self._INPUT_HARDWARE = 2 - self._KEYEVENTF_EXTENDEDKEY = 0x0001 - self._KEYEVENTF_KEYUP = 0x0002 - self._KEYEVENTF_UNICODE = 0x0004 - self._KEYEVENTF_SCANCODE = 0x0008 - KEYEVENTF_EXTENDEDKEY = 0x0001 - KEYEVENTF_KEYUP = 0x0002 - KEYEVENTF_UNICODE = 0x0004 - KEYEVENTF_SCANCODE = 0x0008 - MAPVK_VK_TO_VSC = 0 - - self._SPECIAL_KEYCODES = { - "BACKSPACE": 0x08, - "TAB": 0x09, - "CLEAR": 0x0c, - "ENTER": 0x0d, - "SHIFT": 0x10, - "CTRL": 0x11, - "ALT": 0x12, - "PAUSE": 0x13, - "CAPS_LOCK": 0x14, - "ESC": 0x1b, - "SPACE": 0x20, - "PAGE_UP": 0x21, - "PAGE_DOWN": 0x22, - "END": 0x23, - "HOME": 0x24, - "LEFT": 0x25, - "UP": 0x26, - "RIGHT": 0x27, - "DOWN": 0x28, - "SELECT": 0x29, - "PRINT": 0x2a, - "PRINT_SCREEN": 0x2c, - "INSERT": 0x2d, - "DELETE": 0x2e, - "WIN": 0x5b, - "NUM_0": 0x60, - "NUM_1": 0x61, - "NUM_2": 0x62, - "NUM_3": 0x63, - "NUM_4": 0x64, - "NUM_5": 0x65, - "NUM_6": 0x66, - "NUM_7": 0x67, - "NUM_8": 0x68, - "NUM_9": 0x69, - "F1": 0x70, - "F2": 0x71, - "F3": 0x72, - "F4": 0x73, - "F5": 0x74, - "F6": 0x75, - "F7": 0x76, - "F8": 0x77, - "F9": 0x78, - "F10": 0x79, - "F11": 0x7a, - "F12": 0x7b, - "F13": 0x7c, - "F14": 0x7d, - "F15": 0x7e, - "F16": 0x7f, - "NUM_LOCK": 0x90, - "SCROLL_LOCK": 0x91, - "[": 0xdb, - "]": 0xdd - } - self._UPPERCASE_SPECIAL_KEYCODES = { - "+": 0xbb, - "@": 0x32, - "^": 0x36, - "%": 0x35, - "~": 0xc0, - "(": 0x39, - ")": 0x30, - "{": 0xdb, - "}": 0xdd - } - self._MODIFIER_KEYCODES = { - "+": 0x10, - "^": 0x11, - "%": 0x12, - "~": 0x0d, - "@": 0x5b - } - self._REGULAR_KEYCODES = { - "0": 0x30, - "1": 0x31, - "2": 0x32, - "3": 0x33, - "4": 0x34, - "5": 0x35, - "6": 0x36, - "7": 0x37, - "8": 0x38, - "9": 0x39, - "a": 0x41, - "b": 0x42, - "c": 0x43, - "d": 0x44, - "e": 0x45, - "f": 0x46, - "g": 0x47, - "h": 0x48, - "i": 0x49, - "j": 0x4a, - "k": 0x4b, - "l": 0x4c, - "m": 0x4d, - "n": 0x4e, - "o": 0x4f, - "p": 0x50, - "q": 0x51, - "r": 0x52, - "s": 0x53, - "t": 0x54, - "u": 0x55, - "v": 0x56, - "w": 0x57, - "x": 0x58, - "y": 0x59, - "z": 0x5A, - ";": 0xba, - "=": 0xbb, - ",": 0xbc, - "-": 0xbd, - ".": 0xbe, - "/": 0xbf, - "`": 0xc0, - "\\": 0xdc, - "'": 0xde, - " ": 0x20, - } - self._UPPERCASE_KEYCODES = { - "!": 0x31, - "#": 0x33, - "$": 0x34, - "&": 0x37, - "*": 0x38, - "A": 0x41, - "B": 0x42, - "C": 0x43, - "D": 0x44, - "E": 0x45, - "F": 0x46, - "G": 0x47, - "H": 0x48, - "I": 0x49, - "J": 0x4a, - "K": 0x4b, - "L": 0x4c, - "M": 0x4d, - "N": 0x4e, - "O": 0x4f, - "P": 0x50, - "Q": 0x51, - "R": 0x52, - "S": 0x53, - "T": 0x54, - "U": 0x55, - "V": 0x56, - "W": 0x57, - "X": 0x58, - "Y": 0x59, - "Z": 0x5A, - ":": 0xba, - "<": 0xbc, - "_": 0xbd, - ">": 0xbe, - "?": 0xbf, - "|": 0xdc, - "\"": 0xde, - } - - # __init__ internal classes - - class MOUSEINPUT(ctypes.Structure): - _fields_ = (("dx", wintypes.LONG), - ("dy", wintypes.LONG), - ("mouseData", wintypes.DWORD), - ("dwFlags", wintypes.DWORD), - ("time", wintypes.DWORD), - ("dwExtraInfo", wintypes.WPARAM)) - self._MOUSEINPUT = MOUSEINPUT - - class KEYBDINPUT(ctypes.Structure): - _fields_ = (("wVk", wintypes.WORD), - ("wScan", wintypes.WORD), - ("dwFlags", wintypes.DWORD), - ("time", wintypes.DWORD), - ("dwExtraInfo", wintypes.WPARAM)) - def __init__(self, *args, **kwds): - super(KEYBDINPUT, self).__init__(*args, **kwds) - # some programs use the scan code even if KEYEVENTF_SCANCODE - # isn't set in dwFflags, so attempt to map the correct code. - if not self.dwFlags & KEYEVENTF_UNICODE: - self.wScan = user32.MapVirtualKeyExW(self.wVk, MAPVK_VK_TO_VSC, 0) - self._KEYBDINPUT = KEYBDINPUT - - class HARDWAREINPUT(ctypes.Structure): - _fields_ = (("uMsg", wintypes.DWORD), - ("wParamL", wintypes.WORD), - ("wParamH", wintypes.WORD)) - self._HARDWAREINPUT = HARDWAREINPUT - - class INPUT(ctypes.Structure): - class _INPUT(ctypes.Union): - _fields_ = (("ki", KEYBDINPUT), - ("mi", MOUSEINPUT), - ("hi", HARDWAREINPUT)) - _anonymous_ = ("_input",) - _fields_ = (("type", wintypes.DWORD), - ("_input", _INPUT)) - self._INPUT = INPUT - - LPINPUT = ctypes.POINTER(INPUT) - user32.SendInput.errcheck = self._check_count - user32.SendInput.argtypes = (wintypes.UINT, # nInputs - LPINPUT, # pInputs - ctypes.c_int) # cbSize - - def _check_count(self, result, func, args): - """ Private function to return ctypes errors cleanly """ - if result == 0: - raise ctypes.WinError(ctypes.get_last_error()) - return args - - ## Keyboard input methods ## - - def _pressKeyCode(self, hexKeyCode): - """ Set key state to down for the specified hex key code """ - x = self._INPUT(type=self._INPUT_KEYBOARD, ki=self._KEYBDINPUT(wVk=hexKeyCode)) - self._user32.SendInput(1, ctypes.byref(x), ctypes.sizeof(x)) - def pressKey(self, text): - """ Accepts a string of keys in typeKeys format (see below). Holds down all of them. """ - - if not isinstance(text, basestring): - raise TypeError("pressKey expected text to be a string") - in_special_code = False - special_code = "" - for i in range(0, len(text)): - if text[i] == "{": - in_special_code = True - elif text[i] == "}": - in_special_code = False - if special_code in self._SPECIAL_KEYCODES.keys(): - self._pressKeyCode(self._SPECIAL_KEYCODES[special_code]) - else: - raise ValueError("Unsupported special code {{{}}}".format(special_code)) - continue - elif in_special_code: - special_code += text[i] - continue - elif text[i] in self._MODIFIER_KEYCODES.keys(): - self._pressKeyCode(self._MODIFIER_KEYCODES[text[i]]) - elif text[i] in self._REGULAR_KEYCODES.keys(): - self._pressKeyCode(self._REGULAR_KEYCODES[text[i]]) - def _releaseKeyCode(self, hexKeyCode): - """ Set key state to up for the specified hex key code """ - x = self._INPUT(type=self._INPUT_KEYBOARD, ki=self._KEYBDINPUT(wVk=hexKeyCode, dwFlags=self._KEYEVENTF_KEYUP)) - self._user32.SendInput(1, ctypes.byref(x), ctypes.sizeof(x)) - def releaseKey(self, text): - """ Accepts a string of keys in typeKeys format (see below). Releases all of them. """ - - in_special_code = False - special_code = "" - for i in range(0, len(text)): - if text[i] == "{": - in_special_code = True - elif text[i] == "}": - in_special_code = False - if special_code in self._SPECIAL_KEYCODES.keys(): - self._releaseKeyCode(self._SPECIAL_KEYCODES[special_code]) - else: - raise ValueError("Unsupported special code {{{}}}".format(special_code)) - continue - elif in_special_code: - special_code += text[i] - continue - elif text[i] in self._MODIFIER_KEYCODES.keys(): - self._releaseKeyCode(self._MODIFIER_KEYCODES[text[i]]) - elif text[i] in self._REGULAR_KEYCODES.keys(): - self._releaseKeyCode(self._REGULAR_KEYCODES[text[i]]) - def typeKeys(self, text, delay=0.1): - """ Translates a string (with modifiers) into a series of keystrokes. - - Equivalent to Microsoft's SendKeys, with the addition of "@" as a Win-key modifier. - Avoids some issues SendKeys had with applications like Citrix. - """ - in_special_code = False - special_code = "" - modifier_held = False - modifier_stuck = False - modifier_codes = [] - - for i in range(0, len(text)): - if text[i] == "{": - in_special_code = True - elif text[i] == "}": - in_special_code = False - if special_code in self._SPECIAL_KEYCODES.keys(): - self._pressKeyCode(self._SPECIAL_KEYCODES[special_code]) - self._releaseKeyCode(self._SPECIAL_KEYCODES[special_code]) - elif special_code in self._UPPERCASE_SPECIAL_KEYCODES.keys(): - self._pressKeyCode(self._SPECIAL_KEYCODES["SHIFT"]) - self._pressKeyCode(self._SPECIAL_KEYCODES[special_code]) - self._releaseKeyCode(self._SPECIAL_KEYCODES[special_code]) - self._releaseKeyCode(self._SPECIAL_KEYCODES["SHIFT"]) - else: - raise ValueError("Unrecognized special code {{{}}}".format(special_code)) - continue - elif in_special_code: - special_code += text[i] - continue - elif text[i] == "(": - modifier_stuck = True - modifier_held = False - continue - elif text[i] == ")": - modifier_stuck = False - for x in modifier_codes: - self._releaseKeyCode(x) - modifier_codes = [] - continue - elif text[i] in self._MODIFIER_KEYCODES.keys(): - modifier_codes.append(self._MODIFIER_KEYCODES[text[i]]) - self._pressKeyCode(self._MODIFIER_KEYCODES[text[i]]) - modifier_held = True - elif text[i] in self._REGULAR_KEYCODES.keys(): - self._pressKeyCode(self._REGULAR_KEYCODES[text[i]]) - self._releaseKeyCode(self._REGULAR_KEYCODES[text[i]]) - if modifier_held: - modifier_held = False - for x in modifier_codes: - self._releaseKeyCode(x) - modifier_codes = [] - elif text[i] in self._UPPERCASE_KEYCODES.keys(): - self._pressKeyCode(self._SPECIAL_KEYCODES["SHIFT"]) - self._pressKeyCode(self._UPPERCASE_KEYCODES[text[i]]) - self._releaseKeyCode(self._UPPERCASE_KEYCODES[text[i]]) - self._releaseKeyCode(self._SPECIAL_KEYCODES["SHIFT"]) - if modifier_held: - modifier_held = False - for x in modifier_codes: - self._releaseKeyCode(x) - modifier_codes = [] - if delay: - time.sleep(delay) - - if modifier_stuck or modifier_held: - for modifier in modifier_codes: - self._releaseKeyCode(modifier) - - ## Mouse input methods - - def setMousePos(self, location): - """ Accepts a tuple (x,y) and sets the mouse position accordingly """ - x, y = location - if self.isPointVisible(x, y): - self._user32.SetCursorPos(x, y) - def getMousePos(self): - """ Returns the current mouse position as a tuple (x,y) - - Relative to origin of main screen top left (0,0). May be negative. - """ - class POINT(ctypes.Structure): - _fields_ = [("x", ctypes.c_long), ("y", ctypes.c_long)] - pt = POINT() - self._user32.GetCursorPos(ctypes.byref(pt)) - return (pt.x, pt.y) - def mouseButtonDown(self, button=0): - """ Translates the button (0=LEFT, 1=MIDDLE, 2=RIGHT) and sends a mousedown to the OS """ - click_down_code = [0x0002, 0x0020, 0x0008][button] - x = self._INPUT(type=self._INPUT_MOUSE, mi=self._MOUSEINPUT(dwFlags=click_down_code)) - self._user32.SendInput(1, ctypes.byref(x), ctypes.sizeof(x)) - def mouseButtonUp(self, button=0): - """ Translates the button (0=LEFT, 1=MIDDLE, 2=RIGHT) and sends a mouseup to the OS """ - click_up_code = [0x0004, 0x0040, 0x0010][button] - x = self._INPUT(type=self._INPUT_MOUSE, mi=self._MOUSEINPUT(dwFlags=click_up_code)) - self._user32.SendInput(1, ctypes.byref(x), ctypes.sizeof(x)) - def clickMouse(self, button=0): - """ Abstracts the clicking function - - Button codes are (0=LEFT, 1=MIDDLE, 2=RIGHT) and should be provided as constants - by the Mouse class - """ - # LEFT = 0 - # MIDDLE = 1 - # RIGHT = 2 - self.mouseButtonDown(button) - self.mouseButtonUp(button) - def mouseWheel(self, direction, steps): - """ Clicks the mouse wheel the specified number of steps in the given direction - - Valid directions are 0 (for down) and 1 (for up). These should be provided - as constants by the Mouse class. - """ - MOUSEEVENTF_WHEEL = 0x0800 - if direction == 1: - wheel_moved = 120*steps - elif direction == 0: - wheel_moved = -120*steps - else: - raise ValueError("Expected direction to be 1 or 0") - x = self._INPUT(type=self._INPUT_MOUSE, mi=self._MOUSEINPUT(dwFlags=MOUSEEVENTF_WHEEL, mouseData=wheel_moved)) - self._user32.SendInput(1, ctypes.byref(x), ctypes.sizeof(x)) - - ## Screen functions - - def getBitmapFromRect(self, x, y, w, h): - """ Capture the specified area of the (virtual) screen. """ - min_x, min_y, screen_width, screen_height = self._getVirtualScreenRect() - img = self._getVirtualScreenBitmap() - # Limit the coordinates to the virtual screen - # Then offset so 0,0 is the top left corner of the image - # (Top left of virtual screen could be negative) - x1 = min(max(min_x, x), min_x+screen_width) - min_x - y1 = min(max(min_y, y), min_y+screen_height) - min_y - x2 = min(max(min_x, x+w), min_x+screen_width) - min_x - y2 = min(max(min_y, y+h), min_y+screen_height) - min_y - return numpy.array(img.crop((x1, y1, x2, y2))) - def getScreenBounds(self, screenId): - """ Returns the screen size of the specified monitor (0 being the main monitor). """ - screen_details = self.getScreenDetails() - if not isinstance(screenId, int) or screenId < -1 or screenId >= len(screen_details): - raise ValueError("Invalid screen ID") - if screenId == -1: - # -1 represents the entire virtual screen - x1, y1, x2, y2 = self._getVirtualScreenRect() - return (x1, y1, x2-x1, y2-y1) - return screen_details[screenId]["rect"] - def getScreenDetails(self): - """ Return list of attached monitors with `rect` representing each screen as positioned in virtual screen. - - List is returned in device order, with the first element (0) representing the primary monitor. - """ - monitors = self._getMonitorInfo() - primary_screen = None - screens = [] - for monitor in monitors: - # Convert screen rect to Lackey-style rect (x,y,w,h) as position in virtual screen - screen = { - "rect": ( - monitor["rect"][0], - monitor["rect"][1], - monitor["rect"][2] - monitor["rect"][0], - monitor["rect"][3] - monitor["rect"][1] - ) - } - screens.append(screen) - return screens - def isPointVisible(self, x, y): - """ Checks if a point is visible on any monitor. """ - class POINT(ctypes.Structure): - _fields_ = [("x", ctypes.c_long), ("y", ctypes.c_long)] - pt = POINT() - pt.x = x - pt.y = y - MONITOR_DEFAULTTONULL = 0 - hmon = self._user32.MonitorFromPoint(pt, MONITOR_DEFAULTTONULL) - if hmon == 0: - return False - return True - def _captureScreen(self, device_name): - """ Captures a bitmap from the given monitor device name - - Returns as a PIL Image (BGR rather than RGB, for compatibility with OpenCV) - """ - - ## Define constants/structs - class HBITMAP(ctypes.Structure): - _fields_ = [("bmType", ctypes.c_long), - ("bmWidth", ctypes.c_long), - ("bmHeight", ctypes.c_long), - ("bmWidthBytes", ctypes.c_long), - ("bmPlanes", ctypes.wintypes.WORD), - ("bmBitsPixel", ctypes.wintypes.WORD), - ("bmBits", ctypes.wintypes.LPVOID)] - class BITMAPINFOHEADER(ctypes.Structure): - _fields_ = [("biSize", ctypes.wintypes.DWORD), - ("biWidth", ctypes.c_long), - ("biHeight", ctypes.c_long), - ("biPlanes", ctypes.wintypes.WORD), - ("biBitCount", ctypes.wintypes.WORD), - ("biCompression", ctypes.wintypes.DWORD), - ("biSizeImage", ctypes.wintypes.DWORD), - ("biXPelsPerMeter", ctypes.c_long), - ("biYPelsPerMeter", ctypes.c_long), - ("biClrUsed", ctypes.wintypes.DWORD), - ("biClrImportant", ctypes.wintypes.DWORD)] - class BITMAPINFO(ctypes.Structure): - _fields_ = [("bmiHeader", BITMAPINFOHEADER), - ("bmiColors", ctypes.wintypes.DWORD*3)] - HORZRES = ctypes.c_int(8) - VERTRES = ctypes.c_int(10) - SRCCOPY = 0xCC0020 - DIB_RGB_COLORS = 0 - - ## Begin logic - hdc = self._gdi32.CreateDCA(ctypes.c_char_p(device_name), 0, 0, 0) - if hdc == 0: - raise ValueError("Empty hdc provided") - - # Get monitor specs - screen_width = self._gdi32.GetDeviceCaps(hdc, HORZRES) - screen_height = self._gdi32.GetDeviceCaps(hdc, VERTRES) - - # Create memory device context for monitor - hCaptureDC = self._gdi32.CreateCompatibleDC(hdc) - if hCaptureDC == 0: - raise WindowsError("gdi:CreateCompatibleDC failed") - - # Create bitmap compatible with monitor - hCaptureBmp = self._gdi32.CreateCompatibleBitmap(hdc, screen_width, screen_height) - if hCaptureBmp == 0: - raise WindowsError("gdi:CreateCompatibleBitmap failed") - - # Select hCaptureBmp into hCaptureDC device context - self._gdi32.SelectObject(hCaptureDC, hCaptureBmp) - - # Perform bit-block transfer from screen to device context (and thereby hCaptureBmp) - self._gdi32.BitBlt(hCaptureDC, 0, 0, screen_width, screen_height, hdc, 0, 0, SRCCOPY) - - # Capture image bits from bitmap - img_info = BITMAPINFO() - img_info.bmiHeader.biSize = ctypes.sizeof(BITMAPINFOHEADER) - img_info.bmiHeader.biWidth = screen_width - img_info.bmiHeader.biHeight = screen_height - img_info.bmiHeader.biPlanes = 1 - img_info.bmiHeader.biBitCount = 32 - img_info.bmiHeader.biCompression = 0 - img_info.bmiHeader.biClrUsed = 0 - img_info.bmiHeader.biClrImportant = 0 - - buffer_length = screen_width * 4 * screen_height - image_data = ctypes.create_string_buffer(buffer_length) - - scanlines = self._gdi32.GetDIBits(hCaptureDC, hCaptureBmp, 0, screen_height, ctypes.byref(image_data), ctypes.byref(img_info), DIB_RGB_COLORS) - if scanlines != screen_height: - raise WindowsError("gdi:GetDIBits failed") - final_image = ImageOps.flip(Image.frombuffer("RGBX", (screen_width, screen_height), image_data, "raw", "RGBX", 0, 1)) - # Destroy created device context & GDI bitmap - self._gdi32.DeleteObject(hdc) - self._gdi32.DeleteObject(hCaptureDC) - self._gdi32.DeleteObject(hCaptureBmp) - return final_image - def _getMonitorInfo(self): - """ Returns info about the attached monitors, in device order - - [0] is always the primary monitor - """ - monitors = [] - CCHDEVICENAME = 32 - def _MonitorEnumProcCallback(hMonitor, hdcMonitor, lprcMonitor, dwData): - class MONITORINFOEX(ctypes.Structure): - _fields_ = [("cbSize", ctypes.wintypes.DWORD), - ("rcMonitor", ctypes.wintypes.RECT), - ("rcWork", ctypes.wintypes.RECT), - ("dwFlags", ctypes.wintypes.DWORD), - ("szDevice", ctypes.wintypes.WCHAR*CCHDEVICENAME)] - lpmi = MONITORINFOEX() - lpmi.cbSize = ctypes.sizeof(MONITORINFOEX) - self._user32.GetMonitorInfoW(hMonitor, ctypes.byref(lpmi)) - #hdc = self._gdi32.CreateDCA(ctypes.c_char_p(lpmi.szDevice), 0, 0, 0) - monitors.append({ - "hmon": hMonitor, - #"hdc": hdc, - "rect": (lprcMonitor.contents.left, - lprcMonitor.contents.top, - lprcMonitor.contents.right, - lprcMonitor.contents.bottom), - "name": lpmi.szDevice - }) - return True - MonitorEnumProc = ctypes.WINFUNCTYPE(ctypes.c_bool, ctypes.c_ulong, ctypes.c_ulong, ctypes.POINTER(ctypes.wintypes.RECT), ctypes.c_int) - callback = MonitorEnumProc(_MonitorEnumProcCallback) - if self._user32.EnumDisplayMonitors(0,0,callback,0) == 0: - raise WindowsError("Unable to enumerate monitors") - # Clever magic to make the screen with origin of (0,0) [the primary monitor] the first in the list - monitors.sort(key=lambda x: (not (x["rect"][0] == 0 and x["rect"][1] == 0), x["name"])) # Sort by device ID - 0 is primary, 1 is next, etc. - return monitors - def _getVirtualScreenRect(self): - """ The virtual screen is the bounding box containing all monitors. - - Not all regions in the virtual screen are actually visible. The (0,0) coordinate is the top left corner of the primary - screen rather than the whole bounding box, so some regions of the virtual screen may have negative coordinates if another - screen is positioned in Windows as further to the left or above the primary screen. - - Returns the rect as (x, y, w, h) - """ - SM_XVIRTUALSCREEN = 76 # Left of virtual screen - SM_YVIRTUALSCREEN = 77 # Top of virtual screen - SM_CXVIRTUALSCREEN = 78 # Width of virtual screen - SM_CYVIRTUALSCREEN = 79 # Heigiht of virtual screen - - return (self._user32.GetSystemMetrics(SM_XVIRTUALSCREEN), \ - self._user32.GetSystemMetrics(SM_YVIRTUALSCREEN), \ - self._user32.GetSystemMetrics(SM_CXVIRTUALSCREEN), \ - self._user32.GetSystemMetrics(SM_CYVIRTUALSCREEN)) - def _getVirtualScreenBitmap(self): - """ Returns a PIL bitmap (BGR channel order) of a screenshot from all monitors arranged like the Virtual Screen """ - - # Collect information about the virtual screen & monitors - min_x, min_y, screen_width, screen_height = self._getVirtualScreenRect() - monitors = self._getMonitorInfo() - - # Initialize new black image the size of the virtual screen - virt_screen = Image.new("RGB", (screen_width, screen_height)) - - # Capture images of each of the monitors and overlay on the virtual screen - for monitor_id in range(0, len(monitors)): - img = self._captureScreen(monitors[monitor_id]["name"]) - # Capture virtscreen coordinates of monitor - x1, y1, x2, y2 = monitors[monitor_id]["rect"] - # Convert to image-local coordinates - x = x1 - min_x - y = y1 - min_y - # Paste on the virtual screen - virt_screen.paste(img, (x,y)) - return virt_screen - - ## Clipboard functions - - def getClipboard(self): - """ Uses Tkinter to fetch any text on the clipboard. - - If a Tkinter root window has already been created somewhere else, - uses that instead of creating a new one. - """ - if tk._default_root is None: - temporary_root = True - root = tk.Tk() - root.withdraw() - else: - temporary_root = False - root = tk._default_root - root.update() - to_return = str(root.clipboard_get()) - if temporary_root: - root.destroy() - return to_return - def setClipboard(self, text): - """ Uses Tkinter to set the system clipboard. - - If a Tkinter root window has already been created somewhere else, - uses that instead of creating a new one. - """ - if tk._default_root is None: - temporary_root = True - root = tk.Tk() - root.withdraw() - else: - temporary_root = False - root = tk._default_root - root.clipboard_clear() - root.clipboard_append(text) - root.update() - if temporary_root: - root.destroy() - def osCopy(self): - """ Triggers the OS "copy" keyboard shortcut """ - self.typeKeys("^c") - def osPaste(self): - """ Triggers the OS "paste" keyboard shortcut """ - self.typeKeys("^v") - - ## Window functions - - def getWindowByTitle(self, wildcard, order=0): - """ Returns a platform-specific handle for the first window that matches the provided "wildcard" regex """ - EnumWindowsProc = ctypes.WINFUNCTYPE(ctypes.c_bool, ctypes.POINTER(ctypes.c_int), ctypes.py_object) - def callback(hwnd, context): - if ctypes.windll.user32.IsWindowVisible(hwnd): - length = ctypes.windll.user32.GetWindowTextLengthW(hwnd) - buff = ctypes.create_unicode_buffer(length + 1) - ctypes.windll.user32.GetWindowTextW(hwnd, buff, length + 1) - if re.search(context["wildcard"], buff.value) != None and not context["handle"]: - if context["order"] > 0: - context["order"] -= 1 - else: - context["handle"] = hwnd - return True - data = {"wildcard": wildcard, "handle": None, "order": order} - ctypes.windll.user32.EnumWindows(EnumWindowsProc(callback), ctypes.py_object(data)) - return data["handle"] - def getWindowByPID(self, pid, order=0): - """ Returns a platform-specific handle for the first window that matches the provided PID """ - if pid <= 0: - return None - EnumWindowsProc = ctypes.WINFUNCTYPE(ctypes.c_bool, ctypes.POINTER(ctypes.c_int), ctypes.py_object) - def callback(hwnd, context): - if ctypes.windll.user32.IsWindowVisible(hwnd): - pid = ctypes.c_long() - ctypes.windll.user32.GetWindowThreadProcessId(hwnd, ctypes.byref(pid)) - if context["pid"] == int(pid.value) and not context["handle"]: - if context["order"] > 0: - context["order"] -= 1 - else: - context["handle"] = hwnd - return True - data = {"pid": pid, "handle": None, "order": order} - ctypes.windll.user32.EnumWindows(EnumWindowsProc(callback), ctypes.py_object(data)) - return data["handle"] - def getWindowRect(self, hwnd): - """ Returns a rect (x,y,w,h) for the specified window's area """ - rect = ctypes.wintypes.RECT() - if ctypes.windll.user32.GetWindowRect(hwnd, ctypes.byref(rect)): - x1 = rect.left - y1 = rect.top - x2 = rect.right - y2 = rect.bottom - return (x1, y1, x2-x1, y2-y1) - return None - def focusWindow(self, hwnd): - """ Brings specified window to the front """ - Debug.log(3, "Focusing window: " + str(hwnd)) - SW_RESTORE = 9 - if ctypes.windll.user32.IsIconic(hwnd): - ctypes.windll.user32.ShowWindow(hwnd, SW_RESTORE) - ctypes.windll.user32.SetForegroundWindow(hwnd) - def getWindowTitle(self, hwnd): - """ Gets the title for the specified window """ - length = ctypes.windll.user32.GetWindowTextLengthW(hwnd) - buff = ctypes.create_unicode_buffer(length + 1) - ctypes.windll.user32.GetWindowTextW(hwnd, buff, length + 1) - return buff.value - def getWindowPID(self, hwnd): - """ Gets the process ID that the specified window belongs to """ - pid = ctypes.c_long() - ctypes.windll.user32.GetWindowThreadProcessId(hwnd, ctypes.byref(pid)) - return int(pid.value) - def getForegroundWindow(self): - """ Returns a handle to the window in the foreground """ - return self._user32.GetForegroundWindow() - - ## Highlighting functions - - def highlight(self, rect, seconds=1): - """ Simulates a transparent rectangle over the specified ``rect`` on the screen. - - Actually takes a screenshot of the region and displays with a - rectangle border in a borderless window (due to Tkinter limitations) - - If a Tkinter root window has already been created somewhere else, - uses that instead of creating a new one. - """ - if tk._default_root is None: - Debug.log(3, "Creating new temporary Tkinter root") - temporary_root = True - root = tk.Tk() - root.withdraw() - else: - Debug.log(3, "Borrowing existing Tkinter root") - temporary_root = False - root = tk._default_root - image_to_show = self.getBitmapFromRect(*rect) - app = highlightWindow(root, rect, image_to_show) - timeout = time.time()+seconds - while time.time() < timeout: - app.update_idletasks() - app.update() - app.destroy() - if temporary_root: - root.destroy() - - ## Process functions - def isPIDValid(self, pid): - """ Checks if a PID is associated with a running process """ - ## Slightly copied wholesale from http://stackoverflow.com/questions/568271/how-to-check-if-there-exists-a-process-with-a-given-pid - ## Thanks to http://stackoverflow.com/users/1777162/ntrrgc and http://stackoverflow.com/users/234270/speedplane - class ExitCodeProcess(ctypes.Structure): - _fields_ = [ ('hProcess', ctypes.c_void_p), - ('lpExitCode', ctypes.POINTER(ctypes.c_ulong))] - SYNCHRONIZE = 0x100000 - PROCESS_QUERY_LIMITED_INFORMATION = 0x1000 - process = self._kernel32.OpenProcess(SYNCHRONIZE|PROCESS_QUERY_LIMITED_INFORMATION, 0, pid) - if not process: - return False - ec = ExitCodeProcess() - out = self._kernel32.GetExitCodeProcess(process, ctypes.byref(ec)) - if not out: - err = self._kernel32.GetLastError() - if self._kernel32.GetLastError() == 5: - # Access is denied. - logging.warning("Access is denied to get pid info.") - self._kernel32.CloseHandle(process) - return False - elif bool(ec.lpExitCode): - # There is an exit code, it quit - self._kernel32.CloseHandle(process) - return False - # No exit code, it's running. - self._kernel32.CloseHandle(process) - return True - def killProcess(self, pid): - """ Kills the process with the specified PID (if possible) """ - SYNCHRONIZE = 0x00100000L - PROCESS_TERMINATE = 0x0001 - hProcess = self._kernel32.OpenProcess(SYNCHRONIZE|PROCESS_TERMINATE, True, pid) - result = self._kernel32.TerminateProcess(hProcess, 0) - self._kernel32.CloseHandle(hProcess) - def getProcessName(self, pid): - if pid <= 0: - return "" - MAX_PATH_LEN = 2048 - proc_name = ctypes.create_string_buffer(MAX_PATH_LEN) - PROCESS_VM_READ = 0x0010 - PROCESS_QUERY_INFORMATION = 0x0400 - hProcess = self._kernel32.OpenProcess(PROCESS_VM_READ|PROCESS_QUERY_INFORMATION, 0, pid) - #self._psapi.GetProcessImageFileName.restype = ctypes.wintypes.DWORD - self._psapi.GetModuleFileNameExA(hProcess, 0, ctypes.byref(proc_name), MAX_PATH_LEN) - return os.path.basename(str(proc_name.value)) + """ Abstracts Windows-specific OS-level features like mouse/keyboard control """ + def __init__(self): + #self._root = tk.Tk() + #self._root.overrideredirect(1) + #self._root.withdraw() + user32 = ctypes.WinDLL('user32', use_last_error=True) + gdi32 = ctypes.WinDLL('gdi32', use_last_error=True) + kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) + psapi = ctypes.WinDLL('psapi', use_last_error=True) + self._user32 = user32 + self._gdi32 = gdi32 + self._kernel32 = kernel32 + self._psapi = psapi + + # Mapping to `keyboard` names + self._SPECIAL_KEYCODES = { + "BACKSPACE": "backspace", + "TAB": "tab", + "CLEAR": "clear", + "ENTER": "enter", + "SHIFT": "shift", + "CTRL": "ctrl", + "ALT": "alt", + "PAUSE": "pause", + "CAPS_LOCK": "caps lock", + "ESC": "esc", + "SPACE": "spacebar", + "PGUP": "page up", + "PGDN": "page down", + "END": "end", + "HOME": "home", + "LEFT": "left arrow", + "UP": "up arrow", + "RIGHT": "right arrow", + "DOWN": "down arrow", + "SELECT": "select", + "PRINT": "print", + "PRINT_SCREEN": "print screen", + "INSERT": "ins", + "DELETE": "del", + "WIN": "left windows", + "NUM_0": "keypad 0", + "NUM_1": "keypad 1", + "NUM_2": "keypad 2", + "NUM_3": "keypad 3", + "NUM_4": "keypad 4", + "NUM_5": "keypad 5", + "NUM_6": "keypad 6", + "NUM_7": "keypad 7", + "NUM_8": "keypad 8", + "NUM_9": "keypad 9", + "F1": "f1", + "F2": "f2", + "F3": "f3", + "F4": "f4", + "F5": "f5", + "F6": "f6", + "F7": "f7", + "F8": "f8", + "F9": "f9", + "F10": "f10", + "F11": "f11", + "F12": "f12", + "F13": "f13", + "F14": "f14", + "F15": "f15", + "F16": "f16", + "NUM_LOCK": "num lock", + "SCROLL_LOCK": "scroll lock", + } + self._REGULAR_KEYCODES = { + "0": "0", + "1": "1", + "2": "2", + "3": "3", + "4": "4", + "5": "5", + "6": "6", + "7": "7", + "8": "8", + "9": "9", + "a": "a", + "b": "b", + "c": "c", + "d": "d", + "e": "e", + "f": "f", + "g": "g", + "h": "h", + "i": "i", + "j": "j", + "k": "k", + "l": "l", + "m": "m", + "n": "n", + "o": "o", + "p": "p", + "q": "q", + "r": "r", + "s": "s", + "t": "t", + "u": "u", + "v": "v", + "w": "w", + "x": "x", + "y": "y", + "z": "z", + ";": ";", + "=": "=", + ",": ",", + "-": "-", + ".": ".", + "/": "/", + "`": "`", + "[": "[", + "\\": "\\", + "]": "]", + "'": "'", + " ": " ", + } + self._UPPERCASE_KEYCODES = { + "~": "`", + "+": "=", + ")": "0", + "!": "1", + "@": "2", + "#": "3", + "$": "4", + "%": "5", + "^": "6", + "&": "7", + "*": "8", + "(": "9", + "A": "a", + "B": "b", + "C": "c", + "D": "d", + "E": "e", + "F": "f", + "G": "g", + "H": "h", + "I": "i", + "J": "j", + "K": "k", + "L": "l", + "M": "m", + "N": "n", + "O": "o", + "P": "p", + "Q": "q", + "R": "r", + "S": "s", + "T": "t", + "U": "u", + "V": "v", + "W": "w", + "X": "x", + "Y": "y", + "Z": "z", + ":": ";", + "<": ",", + "_": "-", + ">": ".", + "?": "/", + "|": "\\", + "\"": "'", + "{": "[", + "}": "]", + } + + def _check_count(self, result, func, args): + #pylint: disable=unused-argument + """ Private function to return ctypes errors cleanly """ + if result == 0: + raise ctypes.WinError(ctypes.get_last_error()) + return args + + ## Keyboard input methods ## + def pressKey(self, text): + """ Accepts a string of keys in typeKeys format (see below). Holds down all of them. """ + + if not isinstance(text, basestring): + raise TypeError("pressKey expected text to be a string") + in_special_code = False + special_code = "" + for i in range(0, len(text)): + if text[i] == "{": + in_special_code = True + elif in_special_code and (text[i] == "}" or text[i] == " " or i == len(text)-1): + # End of special code (or it wasn't a special code after all) + in_special_code = False + print special_code + if special_code in self._SPECIAL_KEYCODES.keys(): + # Found a special code + keyboard.press(self._SPECIAL_KEYCODES[special_code]) + else: + # Wasn't a special code, just treat it as keystrokes + self.pressKey("{") + # Press the rest of the keys normally + self.pressKey(special_code) + self.pressKey(text[i]) + elif in_special_code: + special_code += text[i] + elif text[i] in self._REGULAR_KEYCODES.keys(): + keyboard.press(text[i]) + elif text[i] in self._UPPERCASE_KEYCODES.keys(): + keyboard.press(self._SPECIAL_KEYCODES["SHIFT"]) + keyboard.press(text[i]) + def releaseKey(self, text): + """ Accepts a string of keys in typeKeys format (see below). Releases all of them. """ + + in_special_code = False + special_code = "" + for i in range(0, len(text)): + if text[i] == "{": + in_special_code = True + elif in_special_code and (text[i] == "}" or text[i] == " " or i == len(text)-1): + # End of special code (or it wasn't a special code after all) + in_special_code = False + if special_code in self._SPECIAL_KEYCODES.keys(): + # Found a special code + keyboard.release(self._SPECIAL_KEYCODES[special_code]) + else: + # Wasn't a special code, just treat it as keystrokes + self.releaseKey("{") + # Release the rest of the keys normally + self.releaseKey(special_code) + self.releaseKey(text[i]) + elif in_special_code: + special_code += text[i] + elif text[i] in self._REGULAR_KEYCODES.keys(): + keyboard.release(self._REGULAR_KEYCODES[text[i]]) + elif text[i] in self._UPPERCASE_KEYCODES.keys(): + keyboard.release(self._SPECIAL_KEYCODES["SHIFT"]) + keyboard.release(self._REGULAR_KEYCODES[text[i]]) + def typeKeys(self, text, delay=0.1): + """ Translates a string into a series of keystrokes. + + Respects Sikuli special codes, like "{ENTER}". Does not + use SendKeys-like modifiers. + """ + in_special_code = False + special_code = "" + modifier_held = False + modifier_stuck = False + modifier_codes = [] + + for i in range(0, len(text)): + if text[i] == "{": + in_special_code = True + elif in_special_code and (text[i] == "}" or text[i] == " " or i == len(text)-1): + in_special_code = False + print special_code + if special_code in self._SPECIAL_KEYCODES.keys(): + # Found a special code + keyboard.press_and_release(self._SPECIAL_KEYCODES[special_code]) + else: + # Wasn't a special code, just treat it as keystrokes + keyboard.press(self._SPECIAL_KEYCODES["SHIFT"]) + keyboard.press_and_release(self._UPPERCASE_KEYCODES["{"]) + keyboard.release(self._SPECIAL_KEYCODES["SHIFT"]) + # Release the rest of the keys normally + return + self.typeKeys(special_code) + self.typeKeys(text[i]) + elif in_special_code: + special_code += text[i] + elif text[i] in self._REGULAR_KEYCODES.keys(): + keyboard.press(self._REGULAR_KEYCODES[text[i]]) + keyboard.release(self._REGULAR_KEYCODES[text[i]]) + elif text[i] in self._UPPERCASE_KEYCODES.keys(): + keyboard.press(self._SPECIAL_KEYCODES["SHIFT"]) + keyboard.press_and_release(self._UPPERCASE_KEYCODES[text[i]]) + keyboard.release(self._SPECIAL_KEYCODES["SHIFT"]) + if delay: + time.sleep(delay) + + ## Mouse input methods + + def setMousePos(self, location): + """ Accepts a tuple (x,y) and sets the mouse position accordingly """ + x, y = location + if self.isPointVisible(x, y): + mouse.move(x, y) + def getMousePos(self): + """ Returns the current mouse position as a tuple (x,y) + + Relative to origin of main screen top left (0,0). May be negative. + """ + return mouse.get_position() + def mouseButtonDown(self, button=0): + """ Translates the button (0=LEFT, 1=MIDDLE, 2=RIGHT) and sends a mousedown to the OS """ + button_code = [mouse.LEFT, mouse.MIDDLE, mouse.RIGHT][button] + mouse.press(button_code) + def mouseButtonUp(self, button=0): + """ Translates the button (0=LEFT, 1=MIDDLE, 2=RIGHT) and sends a mouseup to the OS """ + button_code = [mouse.LEFT, mouse.MIDDLE, mouse.RIGHT][button] + mouse.release(button_code) + def clickMouse(self, button=0): + """ Abstracts the clicking function + + Button codes are (0=LEFT, 1=MIDDLE, 2=RIGHT) and should be provided as constants + by the Mouse class + """ + button_code = [mouse.LEFT, mouse.MIDDLE, mouse.RIGHT][button] + mouse.click(button_code) + def mouseWheel(self, direction, steps): + """ Clicks the mouse wheel the specified number of steps in the given direction + + Valid directions are 0 (for down) and 1 (for up). These should be provided + as constants by the Mouse class. + """ + if direction == 1: + wheel_moved = steps + elif direction == 0: + wheel_moved = -1*steps + else: + raise ValueError("Expected direction to be 1 or 0") + mouse._os_mouse.queue.put(WheelEvent(wheel_moved, time.time())) + + ## Screen functions + + def getBitmapFromRect(self, x, y, w, h): + """ Capture the specified area of the (virtual) screen. """ + min_x, min_y, screen_width, screen_height = self._getVirtualScreenRect() + img = self._getVirtualScreenBitmap() + # Limit the coordinates to the virtual screen + # Then offset so 0,0 is the top left corner of the image + # (Top left of virtual screen could be negative) + x1 = min(max(min_x, x), min_x+screen_width) - min_x + y1 = min(max(min_y, y), min_y+screen_height) - min_y + x2 = min(max(min_x, x+w), min_x+screen_width) - min_x + y2 = min(max(min_y, y+h), min_y+screen_height) - min_y + return numpy.array(img.crop((x1, y1, x2, y2))) + def getScreenBounds(self, screenId): + """ Returns the screen size of the specified monitor (0 being the main monitor). """ + screen_details = self.getScreenDetails() + if not isinstance(screenId, int) or screenId < -1 or screenId >= len(screen_details): + raise ValueError("Invalid screen ID") + if screenId == -1: + # -1 represents the entire virtual screen + x1, y1, x2, y2 = self._getVirtualScreenRect() + return (x1, y1, x2-x1, y2-y1) + return screen_details[screenId]["rect"] + def getScreenDetails(self): + """ Return list of attached monitors + + For each monitor (as dict), ``monitor["rect"]`` represents the screen as positioned + in virtual screen. List is returned in device order, with the first element (0) + representing the primary monitor. + """ + monitors = self._getMonitorInfo() + primary_screen = None + screens = [] + for monitor in monitors: + # Convert screen rect to Lackey-style rect (x,y,w,h) as position in virtual screen + screen = { + "rect": ( + monitor["rect"][0], + monitor["rect"][1], + monitor["rect"][2] - monitor["rect"][0], + monitor["rect"][3] - monitor["rect"][1] + ) + } + screens.append(screen) + return screens + def isPointVisible(self, x, y): + """ Checks if a point is visible on any monitor. """ + class POINT(ctypes.Structure): + _fields_ = [("x", ctypes.c_long), ("y", ctypes.c_long)] + pt = POINT() + pt.x = x + pt.y = y + MONITOR_DEFAULTTONULL = 0 + hmon = self._user32.MonitorFromPoint(pt, MONITOR_DEFAULTTONULL) + if hmon == 0: + return False + return True + def _captureScreen(self, device_name): + """ Captures a bitmap from the given monitor device name + + Returns as a PIL Image (BGR rather than RGB, for compatibility with OpenCV) + """ + + ## Define constants/structs + class HBITMAP(ctypes.Structure): + _fields_ = [("bmType", ctypes.c_long), + ("bmWidth", ctypes.c_long), + ("bmHeight", ctypes.c_long), + ("bmWidthBytes", ctypes.c_long), + ("bmPlanes", ctypes.wintypes.WORD), + ("bmBitsPixel", ctypes.wintypes.WORD), + ("bmBits", ctypes.wintypes.LPVOID)] + class BITMAPINFOHEADER(ctypes.Structure): + _fields_ = [("biSize", ctypes.wintypes.DWORD), + ("biWidth", ctypes.c_long), + ("biHeight", ctypes.c_long), + ("biPlanes", ctypes.wintypes.WORD), + ("biBitCount", ctypes.wintypes.WORD), + ("biCompression", ctypes.wintypes.DWORD), + ("biSizeImage", ctypes.wintypes.DWORD), + ("biXPelsPerMeter", ctypes.c_long), + ("biYPelsPerMeter", ctypes.c_long), + ("biClrUsed", ctypes.wintypes.DWORD), + ("biClrImportant", ctypes.wintypes.DWORD)] + class BITMAPINFO(ctypes.Structure): + _fields_ = [("bmiHeader", BITMAPINFOHEADER), + ("bmiColors", ctypes.wintypes.DWORD*3)] + HORZRES = ctypes.c_int(8) + VERTRES = ctypes.c_int(10) + SRCCOPY = 0xCC0020 + DIB_RGB_COLORS = 0 + + ## Begin logic + hdc = self._gdi32.CreateDCA(ctypes.c_char_p(device_name), 0, 0, 0) + if hdc == 0: + raise ValueError("Empty hdc provided") + + # Get monitor specs + screen_width = self._gdi32.GetDeviceCaps(hdc, HORZRES) + screen_height = self._gdi32.GetDeviceCaps(hdc, VERTRES) + + # Create memory device context for monitor + hCaptureDC = self._gdi32.CreateCompatibleDC(hdc) + if hCaptureDC == 0: + raise WindowsError("gdi:CreateCompatibleDC failed") + + # Create bitmap compatible with monitor + hCaptureBmp = self._gdi32.CreateCompatibleBitmap(hdc, screen_width, screen_height) + if hCaptureBmp == 0: + raise WindowsError("gdi:CreateCompatibleBitmap failed") + + # Select hCaptureBmp into hCaptureDC device context + self._gdi32.SelectObject(hCaptureDC, hCaptureBmp) + + # Perform bit-block transfer from screen to device context (and thereby hCaptureBmp) + self._gdi32.BitBlt(hCaptureDC, 0, 0, screen_width, screen_height, hdc, 0, 0, SRCCOPY) + + # Capture image bits from bitmap + img_info = BITMAPINFO() + img_info.bmiHeader.biSize = ctypes.sizeof(BITMAPINFOHEADER) + img_info.bmiHeader.biWidth = screen_width + img_info.bmiHeader.biHeight = screen_height + img_info.bmiHeader.biPlanes = 1 + img_info.bmiHeader.biBitCount = 32 + img_info.bmiHeader.biCompression = 0 + img_info.bmiHeader.biClrUsed = 0 + img_info.bmiHeader.biClrImportant = 0 + + buffer_length = screen_width * 4 * screen_height + image_data = ctypes.create_string_buffer(buffer_length) + + scanlines = self._gdi32.GetDIBits( + hCaptureDC, + hCaptureBmp, + 0, + screen_height, + ctypes.byref(image_data), + ctypes.byref(img_info), + DIB_RGB_COLORS) + if scanlines != screen_height: + raise WindowsError("gdi:GetDIBits failed") + final_image = ImageOps.flip( + Image.frombuffer( + "RGBX", + (screen_width, screen_height), + image_data, + "raw", + "RGBX", + 0, + 1)) + # Destroy created device context & GDI bitmap + self._gdi32.DeleteObject(hdc) + self._gdi32.DeleteObject(hCaptureDC) + self._gdi32.DeleteObject(hCaptureBmp) + return final_image + def _getMonitorInfo(self): + """ Returns info about the attached monitors, in device order + + [0] is always the primary monitor + """ + monitors = [] + CCHDEVICENAME = 32 + def _MonitorEnumProcCallback(hMonitor, hdcMonitor, lprcMonitor, dwData): + class MONITORINFOEX(ctypes.Structure): + _fields_ = [("cbSize", ctypes.wintypes.DWORD), + ("rcMonitor", ctypes.wintypes.RECT), + ("rcWork", ctypes.wintypes.RECT), + ("dwFlags", ctypes.wintypes.DWORD), + ("szDevice", ctypes.wintypes.WCHAR*CCHDEVICENAME)] + lpmi = MONITORINFOEX() + lpmi.cbSize = ctypes.sizeof(MONITORINFOEX) + self._user32.GetMonitorInfoW(hMonitor, ctypes.byref(lpmi)) + #hdc = self._gdi32.CreateDCA(ctypes.c_char_p(lpmi.szDevice), 0, 0, 0) + monitors.append({ + "hmon": hMonitor, + #"hdc": hdc, + "rect": (lprcMonitor.contents.left, + lprcMonitor.contents.top, + lprcMonitor.contents.right, + lprcMonitor.contents.bottom), + "name": lpmi.szDevice + }) + return True + MonitorEnumProc = ctypes.WINFUNCTYPE( + ctypes.c_bool, + ctypes.c_ulong, + ctypes.c_ulong, + ctypes.POINTER(ctypes.wintypes.RECT), + ctypes.c_int) + callback = MonitorEnumProc(_MonitorEnumProcCallback) + if self._user32.EnumDisplayMonitors(0, 0, callback, 0) == 0: + raise WindowsError("Unable to enumerate monitors") + # Clever magic to make the screen with origin of (0,0) [the primary monitor] + # the first in the list + # Sort by device ID - 0 is primary, 1 is next, etc. + monitors.sort(key=lambda x: (not (x["rect"][0] == 0 and x["rect"][1] == 0), x["name"])) + + return monitors + def _getVirtualScreenRect(self): + """ The virtual screen is the bounding box containing all monitors. + + Not all regions in the virtual screen are actually visible. The (0,0) coordinate + is the top left corner of the primary screen rather than the whole bounding box, so + some regions of the virtual screen may have negative coordinates if another screen + is positioned in Windows as further to the left or above the primary screen. + + Returns the rect as (x, y, w, h) + """ + SM_XVIRTUALSCREEN = 76 # Left of virtual screen + SM_YVIRTUALSCREEN = 77 # Top of virtual screen + SM_CXVIRTUALSCREEN = 78 # Width of virtual screen + SM_CYVIRTUALSCREEN = 79 # Heigiht of virtual screen + + return (self._user32.GetSystemMetrics(SM_XVIRTUALSCREEN), \ + self._user32.GetSystemMetrics(SM_YVIRTUALSCREEN), \ + self._user32.GetSystemMetrics(SM_CXVIRTUALSCREEN), \ + self._user32.GetSystemMetrics(SM_CYVIRTUALSCREEN)) + def _getVirtualScreenBitmap(self): + """ Returns a PIL bitmap (BGR channel order) of all monitors + + Arranged like the Virtual Screen + """ + + # Collect information about the virtual screen & monitors + min_x, min_y, screen_width, screen_height = self._getVirtualScreenRect() + monitors = self._getMonitorInfo() + + # Initialize new black image the size of the virtual screen + virt_screen = Image.new("RGB", (screen_width, screen_height)) + + # Capture images of each of the monitors and overlay on the virtual screen + for monitor_id in range(0, len(monitors)): + img = self._captureScreen(monitors[monitor_id]["name"]) + # Capture virtscreen coordinates of monitor + x1, y1, x2, y2 = monitors[monitor_id]["rect"] + # Convert to image-local coordinates + x = x1 - min_x + y = y1 - min_y + # Paste on the virtual screen + virt_screen.paste(img, (x, y)) + return virt_screen + + ## Clipboard functions + + def getClipboard(self): + """ Uses Tkinter to fetch any text on the clipboard. + + If a Tkinter root window has already been created somewhere else, + uses that instead of creating a new one. + """ + if tk._default_root is None: + temporary_root = True + root = tk.Tk() + root.withdraw() + else: + temporary_root = False + root = tk._default_root + root.update() + to_return = str(root.clipboard_get()) + if temporary_root: + root.destroy() + return to_return + def setClipboard(self, text): + """ Uses Tkinter to set the system clipboard. + + If a Tkinter root window has already been created somewhere else, + uses that instead of creating a new one. + """ + if tk._default_root is None: + temporary_root = True + root = tk.Tk() + root.withdraw() + else: + temporary_root = False + root = tk._default_root + root.clipboard_clear() + root.clipboard_append(text) + root.update() + if temporary_root: + root.destroy() + def osCopy(self): + """ Triggers the OS "copy" keyboard shortcut """ + self.pressKey("{CTRL}") + self.typeKeys("c") + self.releaseKey("{CTRL}") + def osPaste(self): + """ Triggers the OS "paste" keyboard shortcut """ + self.pressKey("{CTRL}") + self.typeKeys("v") + self.releaseKey("{CTRL}") + + ## Window functions + + def getWindowByTitle(self, wildcard, order=0): + """ Returns a handle for the first window that matches the provided "wildcard" regex """ + EnumWindowsProc = ctypes.WINFUNCTYPE( + ctypes.c_bool, + ctypes.POINTER(ctypes.c_int), + ctypes.py_object) + def callback(hwnd, context): + if ctypes.windll.user32.IsWindowVisible(hwnd): + length = ctypes.windll.user32.GetWindowTextLengthW(hwnd) + buff = ctypes.create_unicode_buffer(length + 1) + ctypes.windll.user32.GetWindowTextW(hwnd, buff, length + 1) + if re.search(context["wildcard"], buff.value) != None and not context["handle"]: + if context["order"] > 0: + context["order"] -= 1 + else: + context["handle"] = hwnd + return True + data = {"wildcard": wildcard, "handle": None, "order": order} + ctypes.windll.user32.EnumWindows(EnumWindowsProc(callback), ctypes.py_object(data)) + return data["handle"] + def getWindowByPID(self, pid, order=0): + """ Returns a handle for the first window that matches the provided PID """ + if pid <= 0: + return None + EnumWindowsProc = ctypes.WINFUNCTYPE( + ctypes.c_bool, + ctypes.POINTER(ctypes.c_int), + ctypes.py_object) + def callback(hwnd, context): + if ctypes.windll.user32.IsWindowVisible(hwnd): + pid = ctypes.c_long() + ctypes.windll.user32.GetWindowThreadProcessId(hwnd, ctypes.byref(pid)) + if context["pid"] == int(pid.value) and not context["handle"]: + if context["order"] > 0: + context["order"] -= 1 + else: + context["handle"] = hwnd + return True + data = {"pid": pid, "handle": None, "order": order} + ctypes.windll.user32.EnumWindows(EnumWindowsProc(callback), ctypes.py_object(data)) + return data["handle"] + def getWindowRect(self, hwnd): + """ Returns a rect (x,y,w,h) for the specified window's area """ + rect = ctypes.wintypes.RECT() + if ctypes.windll.user32.GetWindowRect(hwnd, ctypes.byref(rect)): + x1 = rect.left + y1 = rect.top + x2 = rect.right + y2 = rect.bottom + return (x1, y1, x2-x1, y2-y1) + return None + def focusWindow(self, hwnd): + """ Brings specified window to the front """ + Debug.log(3, "Focusing window: " + str(hwnd)) + SW_RESTORE = 9 + if ctypes.windll.user32.IsIconic(hwnd): + ctypes.windll.user32.ShowWindow(hwnd, SW_RESTORE) + ctypes.windll.user32.SetForegroundWindow(hwnd) + def getWindowTitle(self, hwnd): + """ Gets the title for the specified window """ + length = ctypes.windll.user32.GetWindowTextLengthW(hwnd) + buff = ctypes.create_unicode_buffer(length + 1) + ctypes.windll.user32.GetWindowTextW(hwnd, buff, length + 1) + return buff.value + def getWindowPID(self, hwnd): + """ Gets the process ID that the specified window belongs to """ + pid = ctypes.c_long() + ctypes.windll.user32.GetWindowThreadProcessId(hwnd, ctypes.byref(pid)) + return int(pid.value) + def getForegroundWindow(self): + """ Returns a handle to the window in the foreground """ + return self._user32.GetForegroundWindow() + + ## Highlighting functions + + def highlight(self, rect, seconds=1): + """ Simulates a transparent rectangle over the specified ``rect`` on the screen. + + Actually takes a screenshot of the region and displays with a + rectangle border in a borderless window (due to Tkinter limitations) + + If a Tkinter root window has already been created somewhere else, + uses that instead of creating a new one. + """ + if tk._default_root is None: + Debug.log(3, "Creating new temporary Tkinter root") + temporary_root = True + root = tk.Tk() + root.withdraw() + else: + Debug.log(3, "Borrowing existing Tkinter root") + temporary_root = False + root = tk._default_root + image_to_show = self.getBitmapFromRect(*rect) + app = highlightWindow(root, rect, image_to_show) + timeout = time.time()+seconds + while time.time() < timeout: + app.update_idletasks() + app.update() + app.destroy() + if temporary_root: + root.destroy() + + ## Process functions + def isPIDValid(self, pid): + """ Checks if a PID is associated with a running process """ + ## Slightly copied wholesale from http://stackoverflow.com/questions/568271/how-to-check-if-there-exists-a-process-with-a-given-pid + ## Thanks to http://stackoverflow.com/users/1777162/ntrrgc and http://stackoverflow.com/users/234270/speedplane + class ExitCodeProcess(ctypes.Structure): + _fields_ = [('hProcess', ctypes.c_void_p), + ('lpExitCode', ctypes.POINTER(ctypes.c_ulong))] + SYNCHRONIZE = 0x100000 + PROCESS_QUERY_LIMITED_INFORMATION = 0x1000 + process = self._kernel32.OpenProcess(SYNCHRONIZE|PROCESS_QUERY_LIMITED_INFORMATION, 0, pid) + if not process: + return False + ec = ExitCodeProcess() + out = self._kernel32.GetExitCodeProcess(process, ctypes.byref(ec)) + if not out: + err = self._kernel32.GetLastError() + if self._kernel32.GetLastError() == 5: + # Access is denied. + logging.warning("Access is denied to get pid info.") + self._kernel32.CloseHandle(process) + return False + elif bool(ec.lpExitCode): + # There is an exit code, it quit + self._kernel32.CloseHandle(process) + return False + # No exit code, it's running. + self._kernel32.CloseHandle(process) + return True + def killProcess(self, pid): + """ Kills the process with the specified PID (if possible) """ + SYNCHRONIZE = 0x00100000L + PROCESS_TERMINATE = 0x0001 + hProcess = self._kernel32.OpenProcess(SYNCHRONIZE|PROCESS_TERMINATE, True, pid) + result = self._kernel32.TerminateProcess(hProcess, 0) + self._kernel32.CloseHandle(hProcess) + def getProcessName(self, pid): + if pid <= 0: + return "" + MAX_PATH_LEN = 2048 + proc_name = ctypes.create_string_buffer(MAX_PATH_LEN) + PROCESS_VM_READ = 0x0010 + PROCESS_QUERY_INFORMATION = 0x0400 + hProcess = self._kernel32.OpenProcess(PROCESS_VM_READ|PROCESS_QUERY_INFORMATION, 0, pid) + #self._psapi.GetProcessImageFileName.restype = ctypes.wintypes.DWORD + self._psapi.GetModuleFileNameExA(hProcess, 0, ctypes.byref(proc_name), MAX_PATH_LEN) + return os.path.basename(str(proc_name.value)) ## Helper class for highlighting class highlightWindow(tk.Toplevel): - def __init__(self, root, rect, screen_cap): - """ Accepts rect as (x,y,w,h) """ - self.root = root - tk.Toplevel.__init__(self, self.root, bg="red", bd=0) - - ## Set toplevel geometry, remove borders, and push to the front - self.geometry("{2}x{3}+{0}+{1}".format(*rect)) - self.overrideredirect(1) - self.attributes("-topmost", True) - - ## Create canvas and fill it with the provided image. Then draw rectangle outline - self.canvas = tk.Canvas(self, width=rect[2], height=rect[3], bd=0, bg="blue", highlightthickness=0) - self.tk_image = ImageTk.PhotoImage(Image.fromarray(screen_cap[...,[2,1,0]])) - self.canvas.create_image(0,0,image=self.tk_image,anchor=tk.NW) - self.canvas.create_rectangle(2,2,rect[2]-2,rect[3]-2, outline="red", width=4)#, outline="red", fill="") - self.canvas.pack(fill=tk.BOTH, expand=tk.YES) - - ## Lift to front if necessary and refresh. - self.lift() - self.update() \ No newline at end of file + def __init__(self, root, rect, screen_cap): + """ Accepts rect as (x,y,w,h) """ + self.root = root + tk.Toplevel.__init__(self, self.root, bg="red", bd=0) + + ## Set toplevel geometry, remove borders, and push to the front + self.geometry("{2}x{3}+{0}+{1}".format(*rect)) + self.overrideredirect(1) + self.attributes("-topmost", True) + + ## Create canvas and fill it with the provided image. Then draw rectangle outline + self.canvas = tk.Canvas( + self, + width=rect[2], + height=rect[3], + bd=0, + bg="blue", + highlightthickness=0) + self.tk_image = ImageTk.PhotoImage(Image.fromarray(screen_cap[..., [2, 1, 0]])) + self.canvas.create_image(0, 0, image=self.tk_image, anchor=tk.NW) + self.canvas.create_rectangle( + 2, + 2, + rect[2]-2, + rect[3]-2, + outline="red", + width=4) + self.canvas.pack(fill=tk.BOTH, expand=tk.YES) + + ## Lift to front if necessary and refresh. + self.lift() + self.update() diff --git a/lackey/RegionMatching.py b/lackey/RegionMatching.py index 2db6e7c..073f4b3 100644 --- a/lackey/RegionMatching.py +++ b/lackey/RegionMatching.py @@ -14,1337 +14,1440 @@ from .PlatformManagerWindows import PlatformManagerWindows from .Exceptions import FindFailed from .Settings import Settings, Debug -from .TemplateMatchers import PyramidTemplateMatcher as TemplateMatcher +from .TemplateMatchers import PyramidTemplateMatcher as TemplateMatcher if platform.system() == "Windows": - PlatformManager = PlatformManagerWindows() # No other input managers built yet + PlatformManager = PlatformManagerWindows() # No other input managers built yet else: - if not (os.environ.get('READTHEDOCS') == 'True'): # Avoid throwing an error if it's just being imported for documentation purposes - raise NotImplementedError("Lackey is currently only compatible with Windows.") - + # Avoid throwing an error if it's just being imported for documentation purposes + if not os.environ.get('READTHEDOCS') == 'True': + raise NotImplementedError("Lackey is currently only compatible with Windows.") + class Pattern(object): - """ Defines a pattern based on a bitmap, similarity, and target offset """ - def __init__(self, path): - ## Loop through image paths to find the image - found = False - for image_path in [Settings.BundlePath] + Settings.ImagePaths: - full_path = os.path.join(image_path, path) - if os.path.exists(full_path): - # Image file not found - found = True - break - ## Check if path is valid - if not found: - raise OSError("File not found: {}".format(path)) - self.path = full_path - self.similarity = 0.7 - self.offset = Location(0,0) - - def similar(self, similarity): - """ Returns a new Pattern with the specified similarity threshold """ - pattern = Pattern(self.path) - pattern.similarity = similarity - return pattern - - def exact(self): - """ Returns a new Pattern with a similarity threshold of 1.0 """ - pattern = Pattern(self.path) - pattern.similarity = 1.0 - return pattern - - def targetOffset(self, dx, dy): - """ Returns a new Pattern with the given target offset """ - pattern = Pattern(self.path) - pattern.similarity = self.similarity - pattern.offset = Location(dx, dy) - return pattern - - def getFilename(self): - """ Returns the path to this Pattern's bitmap """ - return self.path - - def getTargetOffset(self): - """ Returns the target offset as a Location(dx, dy) """ - return self.offset - - def debugPreview(self, title="Debug"): - """ Loads and displays the image at ``Pattern.path`` """ - haystack = cv2.imread(self.path) - cv2.imshow(title, haystack) - cv2.waitKey(0) - cv2.destroyAllWindows() + """ Defines a pattern based on a bitmap, similarity, and target offset """ + def __init__(self, path): + ## Loop through image paths to find the image + found = False + for image_path in [Settings.BundlePath, os.getcwd()] + Settings.ImagePaths: + full_path = os.path.join(image_path, path) + if os.path.exists(full_path): + # Image file not found + found = True + break + ## Check if path is valid + if not found: + raise OSError("File not found: {}".format(path)) + self.path = full_path + self.similarity = Settings.MinSimilarity + self.offset = Location(0, 0) + + def similar(self, similarity): + """ Returns a new Pattern with the specified similarity threshold """ + pattern = Pattern(self.path) + pattern.similarity = similarity + return pattern + + def exact(self): + """ Returns a new Pattern with a similarity threshold of 1.0 """ + pattern = Pattern(self.path) + pattern.similarity = 1.0 + return pattern + + def targetOffset(self, dx, dy): + """ Returns a new Pattern with the given target offset """ + pattern = Pattern(self.path) + pattern.similarity = self.similarity + pattern.offset = Location(dx, dy) + return pattern + + def getFilename(self): + """ Returns the path to this Pattern's bitmap """ + return self.path + + def getTargetOffset(self): + """ Returns the target offset as a Location(dx, dy) """ + return self.offset + + def debugPreview(self, title="Debug"): + """ Loads and displays the image at ``Pattern.path`` """ + haystack = cv2.imread(self.path) + cv2.imshow(title, haystack) + cv2.waitKey(0) + cv2.destroyAllWindows() class Region(object): - def __init__(self, *args): - if len(args) == 4: - x,y,w,h = args - elif len(args) == 1: - if isinstance(args[0], Region): - x,y,w,h = args[0].getTuple() - elif isinstance(args[0], tuple): - x,y,w,h = args[0] - else: - raise TypeError("Unrecognized argument for Region()") - else: - raise TypeError("Unrecognized argument(s) for Region()") - self.setROI(x, y, w, h) - self._lastMatch = None - self._lastMatches = [] - self._lastMatchTime = 0 - self.autoWaitTimeout = 3.0 - self._defaultScanRate = 1/Settings.WaitScanRate # Converts searches per second to actual second interval - self._defaultMouseSpeed = Settings.MoveMouseDelay - self._defaultTypeSpeed = 0.05 - self._raster = (0,0) - - def setX(self, x): - """ Set the x-coordinate of the upper left-hand corner """ - self.x = int(x) - def setY(self, y): - """ Set the y-coordinate of the upper left-hand corner """ - self.y = int(y) - def setW(self, w): - """ Set the width of the region """ - self.w = int(w) - def setH(self, h): - """ Set the height of the region """ - self.h = int(h) - - def getX(self): - """ Get the x-coordinate of the upper left-hand corner """ - return self.x - def getY(self): - """ Get the y-coordinate of the upper left-hand corner """ - return self.y - def getW(self): - """ Get the width of the region """ - return self.w - def getH(self): - """ Get the height of the region """ - return self.h - - def moveTo(self, location): - """ Change the upper left-hand corner to a new ``Location`` (without changing width or height) """ - if not location or not isinstance(location, Location): - raise ValueError("moveTo expected a Location object") - self.x = location.x - self.y = location.y - return self - - def setROI(self, *args): - """ Set Region of Interest (same as Region.setRect()) """ - if len(args) == 4: - x,y,w,h = args - elif len(args) == 1: - if isinstance(args[0], Region): - x,y,w,h = args[0].getTuple() - elif isinstance(args[0], tuple): - x,y,w,h = args[0] - else: - raise TypeError("Unrecognized argument for Region()") - else: - raise TypeError("Unrecognized argument(s) for Region()") - self.setX(x) - self.setY(y) - self.setW(w) - self.setH(h) - setRect = setROI - - def morphTo(self, region): - """ Change shape of this region to match the given ``Region`` object """ - if not region or not isinstance(region, Region): - raise TypeError("morphTo expected a Region object") - self.setROI(region.x, region.y, region.w, region.h) - return self - - def getCenter(self): - """ Return the ``Location`` of the center of this region """ - return Location(self.x+(self.w/2), self.y+(self.h/2)) - def getTopLeft(self): - """ Return the ``Location`` of the top left corner of this region """ - return Location(self.x, self.y) - def getTopRight(self): - """ Return the ``Location`` of the top right corner of this region """ - return Location(self.x+self.w, self.y) - def getBottomLeft(self): - """ Return the ``Location`` of the bottom left corner of this region """ - return Location(self.x, self.y+self.h) - def getBottomRight(self): - """ Return the ``Location`` of the bottom right corner of this region """ - return Location(self.x+self.w, self.y+self.h) - - def getScreen(self): - """ Return an instance of the ``Screen`` object representing the screen this region is inside. - - Checks the top left corner of this region (if it touches multiple screens) is inside. Returns None if the region isn't positioned in any screen. """ - screens = PlatformManager.getScreenDetails() - for screen in screens: - s_x, s_y, s_w, s_h = screen["rect"] - if (self.x >= s_x) and (self.x < s_x + s_w) and (self.y >= s_y) and (self.y < s_y + s_h): - # Top left corner is inside screen region - return Screen(screens.index(screen)) - return None # Could not find matching screen - - def getLastMatch(self): - """ Returns the last successful ``Match`` returned by ``find()``, ``exists()``, etc. """ - return self._lastMatch - def getLastMatches(self): - """ Returns the last successful set of ``Match`` objects returned by ``findAll()`` """ - return self._lastMatches - def getTime(self): - """ Returns the elapsed time in milliseconds to find the last match """ - return self._lastMatchTime - - def setAutoWaitTimeout(self, seconds): - """ Specify the time to wait for an image to appear on the screen """ - self.autoWaitTimeout = float(seconds) - def getAutoWaitTimeout(self): - """ Returns the time to wait for an image to appear on the screen """ - return self.autoWaitTimeout - def setWaitScanRate(self, seconds): - """Set this Region's scan rate: A find op should repeat the search for the given Visual rate times per second until found or the maximum waiting time is reached.""" - if seconds == 0: - seconds = 3 - self._defaultScanRate = 1/seconds - def getWaitScanRate(self): - """ Get the current scan rate """ - return 1/self._defaultScanRate - - def offset(self, location, dy=0): - """ Returns a new ``Region`` of the same width and height, but offset from this one by ``location`` """ - if not isinstance(location, Location): - # Assume variables passed were dx,dy - location = Location(location, dy) - r = Region(self.x+location.x, self.y+location.y, self.w, self.h).clipRegionToScreen() - if r is None: - raise FindFailed("Specified region is not visible on any screen") - return None - return r - def grow(self, width, height=None): - """ Expands the region by ``width`` on both sides and ``height`` on the top and bottom. - - If only one value is provided, expands the region by that amount on all sides (equivalent to ``nearby()``). - """ - if height is None: - return self.nearby(width) - else: - return Region(self.x-width, self.y-height, self.w+(2*width), self.h+(2*height)).clipRegionToScreen() - def inside(self): - """ Returns the same object. Included for Sikuli compatibility. """ - return self - def nearby(self, expand=None): - """ Returns a new Region that includes the nearby neighbourhood of the the current region. - - The new region is defined by extending the current region's dimensions in - all directions by range number of pixels. The center of the new region remains the - same. - """ - return Region(self.x-expand, self.y-expand, self.w+(2*expand), self.h+(2*expand)).clipRegionToScreen() - def above(self, expand=None): - """ Returns a new Region that is defined above the current region's top border with a height of range number of pixels. - - So it does not include the current region. If range is omitted, it reaches to the top of the screen. - The new region has the same width and x-position as the current region. - """ - if expand == None: - x = self.x - y = 0 - w = self.w - h = self.y - else: - x = self.x - y = self.y - expand - w = self.w - h = expand - return Region(x, y, w, h).clipRegionToScreen() - def below(self, expand=None): - """ Returns a new Region that is defined below the current region's bottom border with a height of range number of pixels. - - So it does not include the current region. If range is omitted, it reaches to the bottom of the screen. - The new region has the same width and x-position as the current region. - """ - if expand == None: - x = self.x - y = self.y+self.h - w = self.w - h = self.getScreen().getBounds()[3] - y # Screen height - else: - x = self.x - y = self.y + self.h - w = self.w - h = expand - return Region(x, y, w, h).clipRegionToScreen() - def left(self, expand=None): - """ Returns a new Region that is defined left of the current region's left border with a width of range number of pixels. - - So it does not include the current region. If range is omitted, it reaches to the left border of the screen. - The new region has the same height and y-position as the current region. - """ - if expand == None: - x = 0 - y = self.y - w = self.x - h = self.h - else: - x = self.x-expand - y = self.y - w = expand - h = self.h - return Region(x, y, w, h).clipRegionToScreen() - def right(self, expand=None): - """ Returns a new Region that is defined right of the current region's right border with a width of range number of pixels. - - So it does not include the current region. If range is omitted, it reaches to the right border of the screen. - The new region has the same height and y-position as the current region. - """ - if expand == None: - x = self.x+self.w - y = self.y - w = self.getScreen().getBounds()[2] - x - h = self.h - else: - x = self.x+self.w - y = self.y - w = expand - h = self.h - return Region(x, y, w, h).clipRegionToScreen() - - def getBitmap(self): - """ Captures screen area of this region, at least the part that is on the screen - - Returns image as numpy array - """ - return PlatformManager.getBitmapFromRect(self.x, self.y, self.w, self.h) - def debugPreview(self, title="Debug"): - """ Displays the region in a preview window. - - If the region is a Match, circles the target area. If the region is larger than half the - primary screen in either dimension, scales it down to half size. - """ - region = self - haystack = self.getBitmap() - if isinstance(region, Match): - cv2.circle(haystack, (region.getTarget().x - self.x, region.getTarget().y - self.y), 5, 255) - if haystack.shape[0] > (Screen(0).getBounds()[2]/2) or haystack.shape[1] > (Screen(0).getBounds()[3]/2): - # Image is bigger than half the screen; scale it down - haystack = cv2.resize(haystack, (0,0), fx=0.5, fy=0.5) - cv2.imshow(title, haystack) - cv2.waitKey(0) - cv2.destroyAllWindows() - def highlight(self, seconds=1): - """ Temporarily using ``debugPreview()`` to show the region instead of highlighting it - - Probably requires transparent GUI creation/manipulation. TODO - """ - PlatformManager.highlight((self.getX(), self.getY(), self.getW(), self.getH()), seconds) - - def find(self, pattern): - """ Searches for an image pattern in the given region - - Throws ``FindFailed`` exception if the image could not be found. - Sikuli supports OCR search with a text parameter. This does not (yet). - """ - match = self.exists(pattern) - if match is None: - path = pattern.path if isinstance(pattern, Pattern) else pattern - raise FindFailed("Could not find pattern '{}'".format(path)) - return None - return match - def findAll(self, pattern): - """ Searches for an image pattern in the given region - - Returns ``Match`` object if ``pattern`` exists, empty array otherwise (does not throw exception) - Sikuli supports OCR search with a text parameter. This does not (yet). - """ - find_time = time.time() - r = self.clipRegionToScreen() - if r is None: - raise FindFailed("Region outside all visible screens") - return None - seconds = self.autoWaitTimeout - if isinstance(pattern, int): - time.sleep(pattern) - return - if not pattern: - time.sleep(seconds) - if not isinstance(pattern, Pattern): - if not isinstance(pattern, basestring): - raise TypeError("find expected a string [image path] or Pattern object") - pattern = Pattern(pattern) - needle = cv2.imread(pattern.path) - if needle is None: - raise ValueError("Unable to load image '{}'".format(pattern.path)) - needle_height, needle_width, needle_channels = needle.shape - positions = [] - timeout = time.time() + seconds - - # Check TemplateMatcher for valid matches - matches = [] - while time.time() < timeout and len(matches) == 0: - matcher = TemplateMatcher(r.getBitmap()) - matches = matcher.findAllMatches(needle,pattern.similarity) - time.sleep(self._defaultScanRate) - - if len(matches) == 0: - Debug.info("Couldn't find '{}' with enough similarity.".format(pattern.path)) - return iter([]) - - # Matches found! Turn them into Match objects - lastMatches = [] - for match in matches: - position, confidence = match - x, y = position - lastMatches.append(Match(confidence, pattern.offset, ((x+self.x, y+self.y), (needle_width, needle_height)))) - self._lastMatches = iter(lastMatches) - Debug.info("Found match(es) for pattern '{}' at similarity ({})".format(pattern.path, pattern.similarity)) - self._lastMatchTime = (time.time() - find_time) * 1000 # Capture find time in milliseconds - return self._lastMatches - - def wait(self, pattern, seconds=3): - """ Searches for an image pattern in the given region, given a specified timeout period - - Functionally identical to find() - Sikuli supports OCR search with a text parameter. This does not (yet). - """ - if seconds: - timeout = time.time() + seconds - else: - timeout = time.time() - while True: - match = self.exists(pattern) - if match: - return match - if time.time() >= timeout: - break - path = pattern.path if isinstance(pattern, Pattern) else pattern - raise FindFailed("Could not find pattern '{}'".format(path)) - return None - def waitVanish(self, pattern, seconds=None): - """ Waits until the specified pattern is not visible on screen. - - If ``seconds`` pass and the pattern is still visible, raises FindFailed exception. - Sikuli supports OCR search with a text parameter. This does not (yet). - """ - r = self.clipRegionToScreen() - if r is None: - raise FindFailed("Region outside all visible screens") - return None - if seconds is None: - seconds = self.autoWaitTimeout - if isinstance(pattern, int): - time.sleep(pattern) - return - if not pattern: - time.sleep(seconds) - if not isinstance(pattern, Pattern): - if not isinstance(pattern, basestring): - raise TypeError("find expected a string [image path] or Pattern object") - pattern = Pattern(pattern) - - needle = cv2.imread(pattern.path) - match = True - timeout = time.time() + seconds - - while match and time.time() < timeout: - matcher = TemplateMatcher(r.getBitmap()) - match = matcher.findBestMatch(needle, pattern.similarity) # When needle disappears, matcher returns None - time.sleep(self._defaultScanRate) - if match: - return False - #self._findFailedHandler(FindFailed("Pattern '{}' did not vanish".format(pattern.path))) - def exists(self, pattern, seconds=None): - """ Searches for an image pattern in the given region - - Returns Match if pattern exists, None otherwise (does not throw exception) - Sikuli supports OCR search with a text parameter. This does not (yet). - """ - find_time = time.time() - r = self.clipRegionToScreen() - if r is None: - raise FindFailed("Region outside all visible screens") - return None - if seconds is None: - seconds = self.autoWaitTimeout - if isinstance(pattern, int): - # Actually just a "wait" statement - time.sleep(pattern) - return - if not pattern: - time.sleep(seconds) - if not isinstance(pattern, Pattern): - if not isinstance(pattern, basestring): - raise TypeError("find expected a string [image path] or Pattern object") - pattern = Pattern(pattern) - needle = cv2.imread(pattern.path) - if needle is None: - raise ValueError("Unable to load image '{}'".format(pattern.path)) - needle_height, needle_width, needle_channels = needle.shape - match = None - timeout = time.time() + seconds - - # Consult TemplateMatcher to find needle - while not match and time.time() < timeout: - matcher = TemplateMatcher(r.getBitmap()) - match = matcher.findBestMatch(needle,pattern.similarity) - time.sleep(self._defaultScanRate) - - if match is None: - Debug.info("Couldn't find '{}' with enough similarity.".format(pattern.path)) - return None - - # Translate local position into global screen position - position, confidence = match - position = (position[0] + self.x, position[1] + self.y) - self._lastMatch = Match(confidence, pattern.offset, (position, (needle_width, needle_height))) - #self._lastMatch.debug_preview() - Debug.info("Found match for pattern '{}' at ({},{}) with confidence ({}). Target at ({},{})".format(pattern.path, self._lastMatch.getX(), self._lastMatch.getY(), self._lastMatch.getScore(), self._lastMatch.getTarget().x, self._lastMatch.getTarget().y)) - self._lastMatchTime = (time.time() - find_time) * 1000 # Capture find time in milliseconds - return self._lastMatch - - def click(self, target, modifiers=""): - """ Moves the cursor to the target location and clicks the default mouse button. """ - target_location = None - mouse = Mouse() - if isinstance(target, Pattern): - target_location = self.find(target).getTarget() - elif isinstance(target, basestring): - target_location = self.find(target).getTarget() - elif isinstance(target, Match): - target_location = target.getTarget() - elif isinstance(target, Region): - target_location = target.getCenter() - elif isinstance(target, Location): - target_location = target - elif target is None and isinstance(self._lastMatch, Match): - target_location = self._lastMatch.getTarget() - else: - raise TypeError("click expected Pattern, String, Match, Region, or Location object") - if modifiers != "": - PlatformManager.pressKey(modifiers) - - mouse.moveSpeed(target_location, self._defaultMouseSpeed) - time.sleep(0.1) # For responsiveness - if Settings.ClickDelay > 0: - time.sleep(min(1.0, Settings.ClickDelay)) - Settings.ClickDelay = 0.0 - mouse.click() - time.sleep(0.1) - - if modifiers != 0: - PlatformManager.releaseKey(modifiers) - Debug.history("Clicked at {}".format(target_location)) - def doubleClick(self, target, modifiers=""): - """ Moves the cursor to the target location and double-clicks the default mouse button. """ - target_location = None - mouse = Mouse() - if isinstance(target, Pattern): - target_location = self.find(target).getTarget() - elif isinstance(target, basestring): - target_location = self.find(target).getTarget() - elif isinstance(target, Match): - target_location = target.getTarget() - elif isinstance(target, Region): - target_location = target.getCenter() - elif isinstance(target, Location): - target_location = target - else: - raise TypeError("doubleClick expected Pattern, String, Match, Region, or Location object") - if modifiers != "": - PlatformManager.pressKey(modifiers) - - mouse.moveSpeed(target_location, self._defaultMouseSpeed) - time.sleep(0.1) - if Settings.ClickDelay > 0: - time.sleep(min(1.0, Settings.ClickDelay)) - Settings.ClickDelay = 0.0 - mouse.click() - time.sleep(0.1) - if Settings.ClickDelay > 0: - time.sleep(min(1.0, Settings.ClickDelay)) - Settings.ClickDelay = 0.0 - mouse.click() - time.sleep(0.1) - - if modifiers != 0: - PlatformManager.releaseKey(modifiers) - def rightClick(self, target, modifiers=""): - """ Moves the cursor to the target location and clicks the right mouse button. """ - target_location = None - mouse = Mouse() - if isinstance(target, Pattern): - target_location = self.find(target).getTarget() - elif isinstance(target, basestring): - target_location = self.find(target).getTarget() - elif isinstance(target, Match): - target_location = target.getTarget() - elif isinstance(target, Region): - target_location = target.getCenter() - elif isinstance(target, Location): - target_location = target - else: - raise TypeError("rightClick expected Pattern, String, Match, Region, or Location object") - - if modifiers != "": - PlatformManager.pressKey(modifiers) - - mouse.moveSpeed(target_location, self._defaultMouseSpeed) - time.sleep(0.1) - if Settings.ClickDelay > 0: - time.sleep(min(1.0, Settings.ClickDelay)) - Settings.ClickDelay = 0.0 - mouse.click(Mouse.RIGHT) - time.sleep(0.1) - - if modifiers != "": - PlatformManager.releaseKey(modifiers) - - def hover(self, target): - """ Moves the cursor to the target location """ - target_location = None - mouse = Mouse() - if isinstance(target, Pattern): - target_location = self.find(target).getTarget() - elif isinstance(target, basestring): - target_location = self.find(target).getTarget() - elif isinstance(target, Match): - target_location = target.getTarget() - elif isinstance(target, Region): - target_location = target.getCenter() - elif isinstance(target, Location): - target_location = target - else: - raise TypeError("hover expected Pattern, String, Match, Region, or Location object") - - mouse.moveSpeed(target_location, self._defaultMouseSpeed) - def drag(self, dragFrom): - """ Moves the cursor to the target location and clicks the mouse in preparation to drag a screen element """ - dragFromLocation = None - mouse = Mouse() - if isinstance(dragFrom, Pattern): - dragFromLocation = self.find(dragFrom).getTarget() - elif isinstance(dragFrom, basestring): - dragFromLocation = self.find(dragFrom).getTarget() - elif isinstance(dragFrom, Match): - dragFromLocation = dragFrom.getTarget() - elif isinstance(dragFrom, Region): - dragFromLocation = dragFrom.getCenter() - elif isinstance(dragFrom, Location): - dragFromLocation = dragFrom - else: - raise TypeError("drag expected dragFrom to be Pattern, String, Match, Region, or Location object") - mouse.moveSpeed(dragFromLocation, self._defaultMouseSpeed) - time.sleep(Settings.DelayBeforeMouseDown) - mouse.buttonDown() - Debug.history("Began drag at {}".format(dragFromLocation)) - def dropAt(self, dragTo, delay=None): - """ Moves the cursor to the target location, waits ``delay`` seconds, and releases the mouse button """ - mouse = Mouse() - if isinstance(dragTo, Pattern): - dragToLocation = self.find(dragTo).getTarget() - elif isinstance(dragTo, basestring): - dragToLocation = self.find(dragTo).getTarget() - elif isinstance(dragTo, Match): - dragToLocation = dragTo.getTarget() - elif isinstance(dragTo, Region): - dragToLocation = dragTo.getCenter() - elif isinstance(dragTo, Location): - dragToLocation = dragTo - else: - raise TypeError("dragDrop expected dragTo to be Pattern, String, Match, Region, or Location object") - - mouse.moveSpeed(dragToLocation, self._defaultMouseSpeed) - time.sleep(delay if delay is not None else Settings.DelayBeforeDrop) - mouse.buttonUp() - Debug.history("Ended drag at {}".format(dragToLocation)) - def dragDrop(self, dragFrom, dragTo, modifiers=""): - """ Holds down the mouse button on ``dragFrom``, moves the mouse to ``dragTo``, and releases the mouse button - - ``modifiers`` may be a typeKeys() compatible string. The specified keys will be held during the drag-drop operation. - """ - if modifiers != "": - PlatformManager.pressKey(modifiers) - - self.drag(dragFrom) - time.sleep(Settings.DelayBeforeDrag) - self.dropAt(dragTo) - - if modifiers != "": - PlatformManager.releaseKey(modifiers) - - def type(self, *args): - """ Usage: type([PSMRL], text, [modifiers]) - - If a pattern is specified, the pattern is clicked first. Doesn't support text paths. - - This implementation varies slightly from Sikuli by allowing a SendKeys variant format. - The following special characters are available as modifiers: - - * ``^`` - Ctrl - * ``+`` - Shift - * ``%`` - Alt - * ``@`` - Win/Meta/Cmd - * ``~`` - Enter/Return - - They can be used to modify a single following character. ``^c`` will type Ctrl+C. - If you need to modify multiple characters, use parentheses: ``+(abc)`` will hold down - Shift and type "ABC". - - To enter these characters as literals, enclose them in brackets: ``{@}`` - """ - pattern = None - text = None - modifiers = None - if len(args) == 1 and isinstance(args[0], basestring): - # Is a string (or Key) to type - text = args[0] - elif len(args) == 2: - if not isinstance(args[0], basestring) and isinstance(args[1], basestring): - pattern = args[0] - text = args[1] - else: - text = args[0] - modifiers = args[1] - elif len(args) == 3 and not isinstance(args[0], basestring): - pattern = args[0] - text = args[1] - modifiers = args[2] - else: - raise TypeError("type method expected ([PSMRL], text, [modifiers])") - - if pattern: - self.click(pattern) - - # Patch some Sikuli special codes - text = text.replace("{PGDN}", "{PAGE_DOWN}") - text = text.replace("{PGUP}", "{PAGE_UP}") - - Debug.history("Typing '{}'".format(text)) - kb = Keyboard() - if modifiers: - kb.keyDown(modifiers) - if Settings.TypeDelay > 0: - typeSpeed = min(1.0, Settings.TypeDelay) - Settings.TypeDelay = 0.0 - else: - typeSpeed = self._defaultTypeSpeed - kb.type(text, typeSpeed) - if modifiers: - kb.keyUp(modifiers) - time.sleep(0.2) - def paste(self, *args): - """ Usage: paste([PSMRL], text) - - If a pattern is specified, the pattern is clicked first. Doesn't support text paths. - ``text`` is pasted as is using the OS paste shortcut (Ctrl+V for Windows/Linux, Cmd+V for OS X). Note that `paste()` does NOT use special formatting like `type()`. - """ - target = None - text = "" - if len(args) == 1 and isinstance(args[0], basestring): - text = args[0] - elif len(args) == 2 and isinstance(args[1], basestring): - self.click(target) - text = args[1] - else: - raise TypeError("paste method expected [PSMRL], text") - - PlatformManager.setClipboard(text) - # Triggers OS paste for foreground window - PlatformManager.osPaste() - time.sleep(0.2) - def getClipboard(self): - """ Returns the contents of the clipboard (can be used to pull outside text into the application) """ - return PlatformManager.getClipboard() - def text(self): - """ OCR method. Todo. """ - raise NotImplementedError("OCR not yet supported") - - def mouseDown(self, button): - """ Low-level mouse actions. Todo """ - return PlatformManager.mouseButtonDown(button) - def mouseUp(self, button): - """ Low-level mouse actions """ - return PlatformManager.mouseButtonUp(button) - def mouseMove(self, PSRML, dy=0): - """ Low-level mouse actions """ - if isinstance(PSRML, int): - # Assume called as mouseMove(dx, dy) - offset = Location(PSRML, dy) - PSRML = Mouse().getPos().offset(offset) - Mouse().moveSpeed(PSRML) - def wheel(self, PSRML, direction, steps): - """ Clicks the wheel the specified number of ticks """ - self.mouseMove(PSRML) - Mouse().wheel(direction, steps) - def keyDown(self, keys): - """ Concatenate multiple keys to press them all down. """ - return Keyboard().keyDown(keys) - def keyUp(self, keys): - """ Concatenate multiple keys to up them all. """ - return Keyboard().keyUp(keys) - - def isRegionValid(self): - """ Returns false if the whole region is outside any screen, otherwise true """ - screens = PlatformManager.getScreenDetails() - for screen in screens: - s_x, s_y, s_w, s_h = screen["rect"] - if (self.x+self.w < s_x or s_x+s_w < self.x or self.y+self.h < s_y or s_y+s_h < self.y): - # Rects overlap - return False - return True - - def clipRegionToScreen(self): - """ Returns the part of the region that is visible on a screen (or the screen with the smallest ID, if the region is visible on multiple screens). - - Returns None if the region is outside the screen. - """ - if not self.isRegionValid(): - return None - screens = PlatformManager.getScreenDetails() - containing_screen = None - for screen in screens: - s_x, s_y, s_w, s_h = screen["rect"] - if (self.x >= s_x and self.x+self.w <= s_x+s_w and self.y >= s_y and self.y+self.h <= s_y+s_h): - # Region completely inside screen - return self - elif (self.x+self.w < s_x or s_x+s_w < self.x or self.y+self.h < s_y or s_y+s_h < self.y): - # Region completely outside screen - continue - else: - # Region partially inside screen - x = max(self.x, s_x) - y = max(self.y, s_y) - w = min(self.w, s_w) - h = min(self.h, s_h) - return Region(x,y,w,h) - return None - - - # Partitioning constants - NORTH = 202 # Upper half - NORTH_WEST = 300 # Left third in upper third - NORTH_MID = 301 # Middle third in upper third - NORTH_EAST = 302 # Right third in upper third - SOUTH = 212 # Lower half - SOUTH_WEST = 320 # Left third in lower third - SOUTH_MID = 321 # Middle third in lower third - SOUTH_EAST = 322 # Right third in lower third - EAST = 220 # Right half - EAST_MID = 310 # Middle third in right third - WEST = 221 # Left half - WEST_MID = 312 # Middle third in left third - MID_THIRD = 311 # Middle third in middle third - TT = 200 # Top left quarter - RR = 201 # Top right quarter - BB = 211 # Bottom right quarter - LL = 210 # Bottom left quarter - - MID_VERTICAL = "MID_VERT" # Half of width vertically centered - MID_HORIZONTAL = "MID_HORZ" # Half of height horizontally centered - MID_BIG = "MID_HALF" # Half of width/half of height centered - - def setRaster(self, rows, columns): - """ Sets the raster for the region, allowing sections to be indexed by row/column """ - rows = int(rows) - columns = int(columns) - if rows <= 0 or columns <= 0: - return self - self._raster = (rows,columns) - return self.getCell(0,0) - def getRow(self, row, numberRows=None): - """ Returns the specified row of the region (if the raster is set) - - If numberRows is provided, uses that instead of the raster - """ - row = int(row) - if self._raster[0] == 0 or self._raster[1] == 0: - return self - if numberRows is None or numberRows < 1 or numberRows > 9: - numberRows = self._raster[0] - rowHeight = self.h / numberRows - if row < 0: - # If row is negative, count backwards from the end - row = numberRows - row - if row < 0: - # Bad row index, return last row - return Region(self.x, self.y+self.h-rowHeight, self.w, rowHeight) - elif row > numberRows: - # Bad row index, return first row - return Region(self.x, self.y, self.w, rowHeight) - return Region(self.x, self.y + (row * rowHeight), self.w, rowHeight) - def getCol(self, column, numberColumns=None): - """ Returns the specified column of the region (if the raster is set) - - If numberColumns is provided, uses that instead of the raster - """ - column = int(column) - if self._raster[0] == 0 or self._raster[1] == 0: - return self - if numberColumns is None or numberColumns < 1 or numberColumns > 9: - numberColumns = self._raster[1] - columnWidth = self.w / numberColumns - if column < 0: - # If column is negative, count backwards from the end - column = numberColumns - column - if column < 0: - # Bad column index, return last column - return Region(self.x+self.w-columnWidth, self.y, columnWidth, self.h) - elif column > numberColumns: - # Bad column index, return first column - return Region(self.x, self.y, columnWidth, self.h) - return Region(self.x + (column * columnWidth), self.y, columnWidth, self.h) - def getCell(self, row, column): - """ Returns the specified cell (if a raster is set for the region) """ - row = int(row) - column = int(column) - if self._raster[0] == 0 or self._raster[1] == 0: - return self - rowHeight = self.h / self._raster[0] - columnWidth = self.h / self._raster[1] - if column < 0: - # If column is negative, count backwards from the end - column = self._raster[1] - column - if column < 0: - # Bad column index, return last column - column = self._raster[1] - elif column > self._raster[1]: - # Bad column index, return first column - column = 0 - if row < 0: - # If row is negative, count backwards from the end - row = self._raster[0] - row - if row < 0: - # Bad row index, return last row - row = self._raster[0] - elif row > self._raster[0]: - # Bad row index, return first row - row = 0 - return Region(self.x+(column*columnWidth), self.y+(row*rowHeight), columnWidth, rowHeight) - def get(self, part): - """ Returns a section of the region as a new region - - Accepts partitioning constants, e.g. Region.NORTH, Region.NORTH_WEST, etc. - Also accepts an int 200-999: - * First digit: Raster (*n* rows by *n* columns) - * Second digit: Row index (if equal to raster, gets the whole row) - * Third digit: Column index (if equal to raster, gets the whole column) - - Region.get(522) will use a raster of 5 rows and 5 columns and return - the cell in the middle. - - Region.get(525) will use a raster of 5 rows and 5 columns and return the row in the middle. - """ - if part == self.MID_VERTICAL: - return Region(self.x+(self.w/4), y, self.w/2, self.h) - elif part == self.MID_HORIZONTAL: - return Region(self.x,self.y+(self.h/4), self.w, self.h/2) - elif part == self.MID_BIG: - return Region(self.x+(self.w/4),self.y+(self.h/4), self.w/2, self.h/2) - elif isinstance(part, int) and part >= 200 and part <= 999: - raster, row, column = str(part) - self.setRaster(raster,raster) - if row == raster and column == raster: - return self - elif row == raster: - return self.getCol(column) - elif column == raster: - return self.getRow(row) - else: - return self.getCell(row,column) - else: - return self - def setRows(self, rows): - """ Sets the number of rows in the raster (if columns have not been initialized, set to 1 as well) """ - self._raster[0] = rows - if self._raster[1] == 0: - self._raster[1] = 1 - def setCols(self, columns): - """ Sets the number of columns in the raster (if rows have not been initialized, set to 1 as well) """ - self._raster[1] = columns - if self._raster[0] == 0: - self._raster[0] = 1 - def isRasterValid(self): - return self.getCols() > 0 and self.getRows() > 0 - def getRows(self): - return self._raster[0] - def getCols(self): - return self._raster[1] - def getRowH(self): - if self._raster[0] == 0: - return 0 - return self.h / self._raster[0] - def getColW(self): - if self._raster[1] == 0: - return 0 - return self.w / self._raster[1] + def __init__(self, *args): + if len(args) == 4: + x, y, w, h = args + elif len(args) == 1: + if isinstance(args[0], Region): + x, y, w, h = args[0].getTuple() + elif isinstance(args[0], tuple): + x, y, w, h = args[0] + else: + raise TypeError("Unrecognized argument for Region()") + else: + raise TypeError("Unrecognized argument(s) for Region()") + self.setROI(x, y, w, h) + self._lastMatch = None + self._lastMatches = [] + self._lastMatchTime = 0 + self.autoWaitTimeout = 3.0 + # Converts searches per second to actual second interval + self._defaultScanRate = 1/Settings.WaitScanRate + self._defaultMouseSpeed = Settings.MoveMouseDelay + self._defaultTypeSpeed = 0.05 + self._raster = (0, 0) + + def setX(self, x): + """ Set the x-coordinate of the upper left-hand corner """ + self.x = int(x) + def setY(self, y): + """ Set the y-coordinate of the upper left-hand corner """ + self.y = int(y) + def setW(self, w): + """ Set the width of the region """ + self.w = int(w) + def setH(self, h): + """ Set the height of the region """ + self.h = int(h) + + def getX(self): + """ Get the x-coordinate of the upper left-hand corner """ + return self.x + def getY(self): + """ Get the y-coordinate of the upper left-hand corner """ + return self.y + def getW(self): + """ Get the width of the region """ + return self.w + def getH(self): + """ Get the height of the region """ + return self.h + + def moveTo(self, location): + """ Change the upper left-hand corner to a new ``Location`` + + Doesn't change width or height + """ + if not location or not isinstance(location, Location): + raise ValueError("moveTo expected a Location object") + self.x = location.x + self.y = location.y + return self + + def setROI(self, *args): + """ Set Region of Interest (same as Region.setRect()) """ + if len(args) == 4: + x, y, w, h = args + elif len(args) == 1: + if isinstance(args[0], Region): + x, y, w, h = args[0].getTuple() + elif isinstance(args[0], tuple): + x, y, w, h = args[0] + else: + raise TypeError("Unrecognized argument for Region()") + else: + raise TypeError("Unrecognized argument(s) for Region()") + self.setX(x) + self.setY(y) + self.setW(w) + self.setH(h) + setRect = setROI + + def morphTo(self, region): + """ Change shape of this region to match the given ``Region`` object """ + if not region or not isinstance(region, Region): + raise TypeError("morphTo expected a Region object") + self.setROI(region.x, region.y, region.w, region.h) + return self + + def getCenter(self): + """ Return the ``Location`` of the center of this region """ + return Location(self.x+(self.w/2), self.y+(self.h/2)) + def getTopLeft(self): + """ Return the ``Location`` of the top left corner of this region """ + return Location(self.x, self.y) + def getTopRight(self): + """ Return the ``Location`` of the top right corner of this region """ + return Location(self.x+self.w, self.y) + def getBottomLeft(self): + """ Return the ``Location`` of the bottom left corner of this region """ + return Location(self.x, self.y+self.h) + def getBottomRight(self): + """ Return the ``Location`` of the bottom right corner of this region """ + return Location(self.x+self.w, self.y+self.h) + + def getScreen(self): + """ Return an instance of the ``Screen`` object this region is inside. + + Checks the top left corner of this region (if it touches multiple screens) is inside. + Returns None if the region isn't positioned in any screen. + """ + screens = PlatformManager.getScreenDetails() + for screen in screens: + s_x, s_y, s_w, s_h = screen["rect"] + if (self.x >= s_x) and (self.x < s_x + s_w) and (self.y >= s_y) and (self.y < s_y + s_h): + # Top left corner is inside screen region + return Screen(screens.index(screen)) + return None # Could not find matching screen + + def getLastMatch(self): + """ Returns the last successful ``Match`` returned by ``find()``, ``exists()``, etc. """ + return self._lastMatch + def getLastMatches(self): + """ Returns the last successful set of ``Match`` objects returned by ``findAll()`` """ + return self._lastMatches + def getTime(self): + """ Returns the elapsed time in milliseconds to find the last match """ + return self._lastMatchTime + + def setAutoWaitTimeout(self, seconds): + """ Specify the time to wait for an image to appear on the screen """ + self.autoWaitTimeout = float(seconds) + def getAutoWaitTimeout(self): + """ Returns the time to wait for an image to appear on the screen """ + return self.autoWaitTimeout + def setWaitScanRate(self, seconds): + """Set this Region's scan rate + + A find op should repeat the search for the given Visual rate times per second until + found or the maximum waiting time is reached. + """ + if seconds == 0: + seconds = 3 + self._defaultScanRate = 1/seconds + def getWaitScanRate(self): + """ Get the current scan rate """ + return 1/self._defaultScanRate + + def offset(self, location, dy=0): + """ Returns a new ``Region`` offset from this one by ``location`` + + Width and height remain the same + """ + if not isinstance(location, Location): + # Assume variables passed were dx,dy + location = Location(location, dy) + r = Region(self.x+location.x, self.y+location.y, self.w, self.h).clipRegionToScreen() + if r is None: + raise FindFailed("Specified region is not visible on any screen") + return None + return r + def grow(self, width, height=None): + """ Expands the region by ``width`` on both sides and ``height`` on the top and bottom. + + If only one value is provided, expands the region by that amount on all sides. + Equivalent to ``nearby()``. + """ + if height is None: + return self.nearby(width) + else: + return Region( + self.x-width, + self.y-height, + self.w+(2*width), + self.h+(2*height)).clipRegionToScreen() + def inside(self): + """ Returns the same object. Included for Sikuli compatibility. """ + return self + def nearby(self, expand=50): + """ Returns a new Region that includes the nearby neighbourhood of the the current region. + + The new region is defined by extending the current region's dimensions + all directions by range number of pixels. The center of the new region remains the + same. + """ + return Region( + self.x-expand, + self.y-expand, + self.w+(2*expand), + self.h+(2*expand)).clipRegionToScreen() + def above(self, expand=None): + """ Returns a new Region above the current region with a height of ``expand`` pixels. + + Does not include the current region. If range is omitted, it reaches to the top of the + screen. The new region has the same width and x-position as the current region. + """ + if expand == None: + x = self.x + y = 0 + w = self.w + h = self.y + else: + x = self.x + y = self.y - expand + w = self.w + h = expand + return Region(x, y, w, h).clipRegionToScreen() + def below(self, expand=None): + """ Returns a new Region below the current region with a height of ``expand`` pixels. + + Does not include the current region. If range is omitted, it reaches to the bottom + of the screen. The new region has the same width and x-position as the current region. + """ + if expand == None: + x = self.x + y = self.y+self.h + w = self.w + h = self.getScreen().getBounds()[3] - y # Screen height + else: + x = self.x + y = self.y + self.h + w = self.w + h = expand + return Region(x, y, w, h).clipRegionToScreen() + def left(self, expand=None): + """ Returns a new Region left of the current region with a width of ``expand`` pixels. + + Does not include the current region. If range is omitted, it reaches to the left border + of the screen. The new region has the same height and y-position as the current region. + """ + if expand == None: + x = 0 + y = self.y + w = self.x + h = self.h + else: + x = self.x-expand + y = self.y + w = expand + h = self.h + return Region(x, y, w, h).clipRegionToScreen() + def right(self, expand=None): + """ Returns a new Region right of the current region with a width of ``expand`` pixels. + + Does not include the current region. If range is omitted, it reaches to the right border + of the screen. The new region has the same height and y-position as the current region. + """ + if expand == None: + x = self.x+self.w + y = self.y + w = self.getScreen().getBounds()[2] - x + h = self.h + else: + x = self.x+self.w + y = self.y + w = expand + h = self.h + return Region(x, y, w, h).clipRegionToScreen() + + def getBitmap(self): + """ Captures screen area of this region, at least the part that is on the screen + + Returns image as numpy array + """ + return PlatformManager.getBitmapFromRect(self.x, self.y, self.w, self.h) + def debugPreview(self, title="Debug"): + """ Displays the region in a preview window. + + If the region is a Match, circles the target area. If the region is larger than half the + primary screen in either dimension, scales it down to half size. + """ + region = self + haystack = self.getBitmap() + if isinstance(region, Match): + cv2.circle( + haystack, + (region.getTarget().x - self.x, region.getTarget().y - self.y), + 5, + 255) + if haystack.shape[0] > (Screen(0).getBounds()[2]/2) or haystack.shape[1] > (Screen(0).getBounds()[3]/2): + # Image is bigger than half the screen; scale it down + haystack = cv2.resize(haystack, (0, 0), fx=0.5, fy=0.5) + cv2.imshow(title, haystack) + cv2.waitKey(0) + cv2.destroyAllWindows() + def highlight(self, seconds=1): + """ Temporarily using ``debugPreview()`` to show the region instead of highlighting it + + Probably requires transparent GUI creation/manipulation. TODO + """ + PlatformManager.highlight((self.getX(), self.getY(), self.getW(), self.getH()), seconds) + + def find(self, pattern): + """ Searches for an image pattern in the given region + + Throws ``FindFailed`` exception if the image could not be found. + Sikuli supports OCR search with a text parameter. This does not (yet). + """ + match = self.exists(pattern) + if match is None: + path = pattern.path if isinstance(pattern, Pattern) else pattern + raise FindFailed("Could not find pattern '{}'".format(path)) + return None + return match + def findAll(self, pattern): + """ Searches for an image pattern in the given region + + Returns ``Match`` object if ``pattern`` exists, empty array otherwise (does not + throw exception). Sikuli supports OCR search with a text parameter. This does not (yet). + """ + find_time = time.time() + r = self.clipRegionToScreen() + if r is None: + raise FindFailed("Region outside all visible screens") + return None + seconds = self.autoWaitTimeout + if isinstance(pattern, int): + time.sleep(pattern) + return + if not pattern: + time.sleep(seconds) + if not isinstance(pattern, Pattern): + if not isinstance(pattern, basestring): + raise TypeError("find expected a string [image path] or Pattern object") + pattern = Pattern(pattern) + needle = cv2.imread(pattern.path) + if needle is None: + raise ValueError("Unable to load image '{}'".format(pattern.path)) + needle_height, needle_width, needle_channels = needle.shape + positions = [] + timeout = time.time() + seconds + + # Check TemplateMatcher for valid matches + matches = [] + while time.time() < timeout and len(matches) == 0: + matcher = TemplateMatcher(r.getBitmap()) + matches = matcher.findAllMatches(needle, pattern.similarity) + time.sleep(self._defaultScanRate) + + if len(matches) == 0: + Debug.info("Couldn't find '{}' with enough similarity.".format(pattern.path)) + return iter([]) + + # Matches found! Turn them into Match objects + lastMatches = [] + for match in matches: + position, confidence = match + x, y = position + lastMatches.append( + Match( + confidence, + pattern.offset, + ((x+self.x, y+self.y), (needle_width, needle_height)))) + self._lastMatches = iter(lastMatches) + Debug.info("Found match(es) for pattern '{}' at similarity ({})".format(pattern.path, pattern.similarity)) + self._lastMatchTime = (time.time() - find_time) * 1000 # Capture find time in milliseconds + return self._lastMatches + + def wait(self, pattern, seconds=3): + """ Searches for an image pattern in the given region, given a specified timeout period + + Functionally identical to find() + Sikuli supports OCR search with a text parameter. This does not (yet). + """ + if seconds: + timeout = time.time() + seconds + else: + timeout = time.time() + while True: + match = self.exists(pattern) + if match: + return match + if time.time() >= timeout: + break + path = pattern.path if isinstance(pattern, Pattern) else pattern + raise FindFailed("Could not find pattern '{}'".format(path)) + return None + def waitVanish(self, pattern, seconds=None): + """ Waits until the specified pattern is not visible on screen. + + If ``seconds`` pass and the pattern is still visible, raises FindFailed exception. + Sikuli supports OCR search with a text parameter. This does not (yet). + """ + r = self.clipRegionToScreen() + if r is None: + raise FindFailed("Region outside all visible screens") + return None + if seconds is None: + seconds = self.autoWaitTimeout + if isinstance(pattern, int): + time.sleep(pattern) + return + if not pattern: + time.sleep(seconds) + if not isinstance(pattern, Pattern): + if not isinstance(pattern, basestring): + raise TypeError("find expected a string [image path] or Pattern object") + pattern = Pattern(pattern) + + needle = cv2.imread(pattern.path) + match = True + timeout = time.time() + seconds + + while match and time.time() < timeout: + matcher = TemplateMatcher(r.getBitmap()) + # When needle disappears, matcher returns None + match = matcher.findBestMatch(needle, pattern.similarity) + time.sleep(self._defaultScanRate) + if match: + return False + #self._findFailedHandler(FindFailed("Pattern '{}' did not vanish".format(pattern.path))) + def exists(self, pattern, seconds=None): + """ Searches for an image pattern in the given region + + Returns Match if pattern exists, None otherwise (does not throw exception) + Sikuli supports OCR search with a text parameter. This does not (yet). + """ + find_time = time.time() + r = self.clipRegionToScreen() + if r is None: + raise FindFailed("Region outside all visible screens") + return None + if seconds is None: + seconds = self.autoWaitTimeout + if isinstance(pattern, int): + # Actually just a "wait" statement + time.sleep(pattern) + return + if not pattern: + time.sleep(seconds) + if not isinstance(pattern, Pattern): + if not isinstance(pattern, basestring): + raise TypeError("find expected a string [image path] or Pattern object") + pattern = Pattern(pattern) + needle = cv2.imread(pattern.path) + if needle is None: + raise ValueError("Unable to load image '{}'".format(pattern.path)) + needle_height, needle_width, needle_channels = needle.shape + match = None + timeout = time.time() + seconds + + # Consult TemplateMatcher to find needle + while not match and time.time() < timeout: + matcher = TemplateMatcher(r.getBitmap()) + match = matcher.findBestMatch(needle, pattern.similarity) + time.sleep(self._defaultScanRate) + + if match is None: + Debug.info("Couldn't find '{}' with enough similarity.".format(pattern.path)) + return None + + # Translate local position into global screen position + position, confidence = match + position = (position[0] + self.x, position[1] + self.y) + self._lastMatch = Match( + confidence, + pattern.offset, + (position, (needle_width, needle_height))) + #self._lastMatch.debug_preview() + Debug.info("Found match for pattern '{}' at ({},{}) with confidence ({}). Target at ({},{})".format( + pattern.path, + self._lastMatch.getX(), + self._lastMatch.getY(), + self._lastMatch.getScore(), + self._lastMatch.getTarget().x, + self._lastMatch.getTarget().y)) + self._lastMatchTime = (time.time() - find_time) * 1000 # Capture find time in milliseconds + return self._lastMatch + + def click(self, target, modifiers=""): + """ Moves the cursor to the target location and clicks the default mouse button. """ + target_location = None + mouse = Mouse() + if isinstance(target, Pattern): + target_location = self.find(target).getTarget() + elif isinstance(target, basestring): + target_location = self.find(target).getTarget() + elif isinstance(target, Match): + target_location = target.getTarget() + elif isinstance(target, Region): + target_location = target.getCenter() + elif isinstance(target, Location): + target_location = target + elif target is None and isinstance(self._lastMatch, Match): + target_location = self._lastMatch.getTarget() + else: + raise TypeError("click expected Pattern, String, Match, Region, or Location object") + if modifiers != "": + PlatformManager.pressKey(modifiers) + + mouse.moveSpeed(target_location, self._defaultMouseSpeed) + time.sleep(0.1) # For responsiveness + if Settings.ClickDelay > 0: + time.sleep(min(1.0, Settings.ClickDelay)) + Settings.ClickDelay = 0.0 + mouse.click() + time.sleep(0.1) + + if modifiers != 0: + PlatformManager.releaseKey(modifiers) + Debug.history("Clicked at {}".format(target_location)) + def doubleClick(self, target, modifiers=""): + """ Moves the cursor to the target location and double-clicks the default mouse button. """ + target_location = None + mouse = Mouse() + if isinstance(target, Pattern): + target_location = self.find(target).getTarget() + elif isinstance(target, basestring): + target_location = self.find(target).getTarget() + elif isinstance(target, Match): + target_location = target.getTarget() + elif isinstance(target, Region): + target_location = target.getCenter() + elif isinstance(target, Location): + target_location = target + else: + raise TypeError("doubleClick expected Pattern, String, Match, Region, or Location object") + if modifiers != "": + PlatformManager.pressKey(modifiers) + + mouse.moveSpeed(target_location, self._defaultMouseSpeed) + time.sleep(0.1) + if Settings.ClickDelay > 0: + time.sleep(min(1.0, Settings.ClickDelay)) + Settings.ClickDelay = 0.0 + mouse.click() + time.sleep(0.1) + if Settings.ClickDelay > 0: + time.sleep(min(1.0, Settings.ClickDelay)) + Settings.ClickDelay = 0.0 + mouse.click() + time.sleep(0.1) + + if modifiers != 0: + PlatformManager.releaseKey(modifiers) + def rightClick(self, target, modifiers=""): + """ Moves the cursor to the target location and clicks the right mouse button. """ + target_location = None + mouse = Mouse() + if isinstance(target, Pattern): + target_location = self.find(target).getTarget() + elif isinstance(target, basestring): + target_location = self.find(target).getTarget() + elif isinstance(target, Match): + target_location = target.getTarget() + elif isinstance(target, Region): + target_location = target.getCenter() + elif isinstance(target, Location): + target_location = target + else: + raise TypeError("rightClick expected Pattern, String, Match, Region, or Location object") + + if modifiers != "": + PlatformManager.pressKey(modifiers) + + mouse.moveSpeed(target_location, self._defaultMouseSpeed) + time.sleep(0.1) + if Settings.ClickDelay > 0: + time.sleep(min(1.0, Settings.ClickDelay)) + Settings.ClickDelay = 0.0 + mouse.click(Mouse.RIGHT) + time.sleep(0.1) + + if modifiers != "": + PlatformManager.releaseKey(modifiers) + + def hover(self, target): + """ Moves the cursor to the target location """ + target_location = None + mouse = Mouse() + if isinstance(target, Pattern): + target_location = self.find(target).getTarget() + elif isinstance(target, basestring): + target_location = self.find(target).getTarget() + elif isinstance(target, Match): + target_location = target.getTarget() + elif isinstance(target, Region): + target_location = target.getCenter() + elif isinstance(target, Location): + target_location = target + else: + raise TypeError("hover expected Pattern, String, Match, Region, or Location object") + + mouse.moveSpeed(target_location, self._defaultMouseSpeed) + def drag(self, dragFrom): + """ Starts a dragDrop operation. + + Moves the cursor to the target location and clicks the mouse in preparation to drag + a screen element """ + dragFromLocation = None + mouse = Mouse() + if isinstance(dragFrom, Pattern): + dragFromLocation = self.find(dragFrom).getTarget() + elif isinstance(dragFrom, basestring): + dragFromLocation = self.find(dragFrom).getTarget() + elif isinstance(dragFrom, Match): + dragFromLocation = dragFrom.getTarget() + elif isinstance(dragFrom, Region): + dragFromLocation = dragFrom.getCenter() + elif isinstance(dragFrom, Location): + dragFromLocation = dragFrom + else: + raise TypeError("drag expected dragFrom to be Pattern, String, Match, Region, or Location object") + mouse.moveSpeed(dragFromLocation, self._defaultMouseSpeed) + time.sleep(Settings.DelayBeforeMouseDown) + mouse.buttonDown() + Debug.history("Began drag at {}".format(dragFromLocation)) + def dropAt(self, dragTo, delay=None): + """ Completes a dragDrop operation + + Moves the cursor to the target location, waits ``delay`` seconds, and releases the mouse + button """ + mouse = Mouse() + if isinstance(dragTo, Pattern): + dragToLocation = self.find(dragTo).getTarget() + elif isinstance(dragTo, basestring): + dragToLocation = self.find(dragTo).getTarget() + elif isinstance(dragTo, Match): + dragToLocation = dragTo.getTarget() + elif isinstance(dragTo, Region): + dragToLocation = dragTo.getCenter() + elif isinstance(dragTo, Location): + dragToLocation = dragTo + else: + raise TypeError("dragDrop expected dragTo to be Pattern, String, Match, Region, or Location object") + + mouse.moveSpeed(dragToLocation, self._defaultMouseSpeed) + time.sleep(delay if delay is not None else Settings.DelayBeforeDrop) + mouse.buttonUp() + Debug.history("Ended drag at {}".format(dragToLocation)) + def dragDrop(self, dragFrom, dragTo, modifiers=""): + """ Performs a dragDrop operation. + + Holds down the mouse button on ``dragFrom``, moves the mouse to ``dragTo``, and releases + the mouse button. + + ``modifiers`` may be a typeKeys() compatible string. The specified keys will be held + during the drag-drop operation. + """ + if modifiers != "": + PlatformManager.pressKey(modifiers) + + self.drag(dragFrom) + time.sleep(Settings.DelayBeforeDrag) + self.dropAt(dragTo) + + if modifiers != "": + PlatformManager.releaseKey(modifiers) + + def type(self, *args): + """ Usage: type([PSMRL], text, [modifiers]) + + If a pattern is specified, the pattern is clicked first. Doesn't support text paths. + + This implementation varies slightly from Sikuli by allowing a SendKeys variant format. + The following special characters are available as modifiers: + + * ``^`` - Ctrl + * ``+`` - Shift + * ``%`` - Alt + * ``@`` - Win/Meta/Cmd + * ``~`` - Enter/Return + + They can be used to modify a single following character. ``^c`` will type Ctrl+C. + If you need to modify multiple characters, use parentheses: ``+(abc)`` will hold down + Shift and type "ABC". + + To enter these characters as literals, enclose them in brackets: ``{@}`` + """ + pattern = None + text = None + modifiers = None + if len(args) == 1 and isinstance(args[0], basestring): + # Is a string (or Key) to type + text = args[0] + elif len(args) == 2: + if not isinstance(args[0], basestring) and isinstance(args[1], basestring): + pattern = args[0] + text = args[1] + else: + text = args[0] + modifiers = args[1] + elif len(args) == 3 and not isinstance(args[0], basestring): + pattern = args[0] + text = args[1] + modifiers = args[2] + else: + raise TypeError("type method expected ([PSMRL], text, [modifiers])") + + if pattern: + self.click(pattern) + + Debug.history("Typing '{}' with modifiers '{}'".format(text, modifiers)) + kb = Keyboard() + if modifiers: + kb.keyDown(modifiers) + if Settings.TypeDelay > 0: + typeSpeed = min(1.0, Settings.TypeDelay) + Settings.TypeDelay = 0.0 + else: + typeSpeed = self._defaultTypeSpeed + kb.type(text, typeSpeed) + if modifiers: + kb.keyUp(modifiers) + time.sleep(0.2) + def paste(self, *args): + """ Usage: paste([PSMRL], text) + + If a pattern is specified, the pattern is clicked first. Doesn't support text paths. + ``text`` is pasted as is using the OS paste shortcut (Ctrl+V for Windows/Linux, Cmd+V + for OS X). Note that `paste()` does NOT use special formatting like `type()`. + """ + target = None + text = "" + if len(args) == 1 and isinstance(args[0], basestring): + text = args[0] + elif len(args) == 2 and isinstance(args[1], basestring): + self.click(target) + text = args[1] + else: + raise TypeError("paste method expected [PSMRL], text") + + PlatformManager.setClipboard(text) + # Triggers OS paste for foreground window + PlatformManager.osPaste() + time.sleep(0.2) + def getClipboard(self): + """ Returns the contents of the clipboard + + Can be used to pull outside text into the application. """ + return PlatformManager.getClipboard() + def text(self): + """ OCR method. Todo. """ + raise NotImplementedError("OCR not yet supported") + + def mouseDown(self, button): + """ Low-level mouse actions. """ + return PlatformManager.mouseButtonDown(button) + def mouseUp(self, button): + """ Low-level mouse actions """ + return PlatformManager.mouseButtonUp(button) + def mouseMove(self, PSRML, dy=0): + """ Low-level mouse actions """ + if isinstance(PSRML, int): + # Assume called as mouseMove(dx, dy) + offset = Location(PSRML, dy) + PSRML = Mouse().getPos().offset(offset) + Mouse().moveSpeed(PSRML) + def wheel(self, PSRML, direction, steps): + """ Clicks the wheel the specified number of ticks """ + self.mouseMove(PSRML) + Mouse().wheel(direction, steps) + def keyDown(self, keys): + """ Concatenate multiple keys to press them all down. """ + return Keyboard().keyDown(keys) + def keyUp(self, keys): + """ Concatenate multiple keys to up them all. """ + return Keyboard().keyUp(keys) + + def isRegionValid(self): + """ Returns false if the whole region is outside any screen, otherwise true """ + screens = PlatformManager.getScreenDetails() + for screen in screens: + s_x, s_y, s_w, s_h = screen["rect"] + if (self.x+self.w < s_x or s_x+s_w < self.x or self.y+self.h < s_y or s_y+s_h < self.y): + # Rects overlap + return False + return True + + def clipRegionToScreen(self): + """ Returns the part of the region that is visible on a screen + + If the region is visible on multiple screens, returns the screen with the smallest ID. + Returns None if the region is outside the screen. + """ + if not self.isRegionValid(): + return None + screens = PlatformManager.getScreenDetails() + containing_screen = None + for screen in screens: + s_x, s_y, s_w, s_h = screen["rect"] + if self.x >= s_x and self.x+self.w <= s_x+s_w and self.y >= s_y and self.y+self.h <= s_y+s_h: + # Region completely inside screen + return self + elif self.x+self.w < s_x or s_x+s_w < self.x or self.y+self.h < s_y or s_y+s_h < self.y: + # Region completely outside screen + continue + else: + # Region partially inside screen + x = max(self.x, s_x) + y = max(self.y, s_y) + w = min(self.w, s_w) + h = min(self.h, s_h) + return Region(x, y, w, h) + return None + + + # Partitioning constants + NORTH = 202 # Upper half + NORTH_WEST = 300 # Left third in upper third + NORTH_MID = 301 # Middle third in upper third + NORTH_EAST = 302 # Right third in upper third + SOUTH = 212 # Lower half + SOUTH_WEST = 320 # Left third in lower third + SOUTH_MID = 321 # Middle third in lower third + SOUTH_EAST = 322 # Right third in lower third + EAST = 220 # Right half + EAST_MID = 310 # Middle third in right third + WEST = 221 # Left half + WEST_MID = 312 # Middle third in left third + MID_THIRD = 311 # Middle third in middle third + TT = 200 # Top left quarter + RR = 201 # Top right quarter + BB = 211 # Bottom right quarter + LL = 210 # Bottom left quarter + + MID_VERTICAL = "MID_VERT" # Half of width vertically centered + MID_HORIZONTAL = "MID_HORZ" # Half of height horizontally centered + MID_BIG = "MID_HALF" # Half of width/half of height centered + + def setRaster(self, rows, columns): + """ Sets the raster for the region, allowing sections to be indexed by row/column """ + rows = int(rows) + columns = int(columns) + if rows <= 0 or columns <= 0: + return self + self._raster = (rows, columns) + return self.getCell(0, 0) + def getRow(self, row, numberRows=None): + """ Returns the specified row of the region (if the raster is set) + + If numberRows is provided, uses that instead of the raster + """ + row = int(row) + if self._raster[0] == 0 or self._raster[1] == 0: + return self + if numberRows is None or numberRows < 1 or numberRows > 9: + numberRows = self._raster[0] + rowHeight = self.h / numberRows + if row < 0: + # If row is negative, count backwards from the end + row = numberRows - row + if row < 0: + # Bad row index, return last row + return Region(self.x, self.y+self.h-rowHeight, self.w, rowHeight) + elif row > numberRows: + # Bad row index, return first row + return Region(self.x, self.y, self.w, rowHeight) + return Region(self.x, self.y + (row * rowHeight), self.w, rowHeight) + def getCol(self, column, numberColumns=None): + """ Returns the specified column of the region (if the raster is set) + + If numberColumns is provided, uses that instead of the raster + """ + column = int(column) + if self._raster[0] == 0 or self._raster[1] == 0: + return self + if numberColumns is None or numberColumns < 1 or numberColumns > 9: + numberColumns = self._raster[1] + columnWidth = self.w / numberColumns + if column < 0: + # If column is negative, count backwards from the end + column = numberColumns - column + if column < 0: + # Bad column index, return last column + return Region(self.x+self.w-columnWidth, self.y, columnWidth, self.h) + elif column > numberColumns: + # Bad column index, return first column + return Region(self.x, self.y, columnWidth, self.h) + return Region(self.x + (column * columnWidth), self.y, columnWidth, self.h) + def getCell(self, row, column): + """ Returns the specified cell (if a raster is set for the region) """ + row = int(row) + column = int(column) + if self._raster[0] == 0 or self._raster[1] == 0: + return self + rowHeight = self.h / self._raster[0] + columnWidth = self.h / self._raster[1] + if column < 0: + # If column is negative, count backwards from the end + column = self._raster[1] - column + if column < 0: + # Bad column index, return last column + column = self._raster[1] + elif column > self._raster[1]: + # Bad column index, return first column + column = 0 + if row < 0: + # If row is negative, count backwards from the end + row = self._raster[0] - row + if row < 0: + # Bad row index, return last row + row = self._raster[0] + elif row > self._raster[0]: + # Bad row index, return first row + row = 0 + return Region(self.x+(column*columnWidth), self.y+(row*rowHeight), columnWidth, rowHeight) + def get(self, part): + """ Returns a section of the region as a new region + + Accepts partitioning constants, e.g. Region.NORTH, Region.NORTH_WEST, etc. + Also accepts an int 200-999: + * First digit: Raster (*n* rows by *n* columns) + * Second digit: Row index (if equal to raster, gets the whole row) + * Third digit: Column index (if equal to raster, gets the whole column) + + Region.get(522) will use a raster of 5 rows and 5 columns and return + the cell in the middle. + + Region.get(525) will use a raster of 5 rows and 5 columns and return the row in the middle. + """ + if part == self.MID_VERTICAL: + return Region(self.x+(self.w/4), y, self.w/2, self.h) + elif part == self.MID_HORIZONTAL: + return Region(self.x, self.y+(self.h/4), self.w, self.h/2) + elif part == self.MID_BIG: + return Region(self.x+(self.w/4), self.y+(self.h/4), self.w/2, self.h/2) + elif isinstance(part, int) and part >= 200 and part <= 999: + raster, row, column = str(part) + self.setRaster(raster, raster) + if row == raster and column == raster: + return self + elif row == raster: + return self.getCol(column) + elif column == raster: + return self.getRow(row) + else: + return self.getCell(row,column) + else: + return self + def setRows(self, rows): + """ Sets the number of rows in the raster (if columns have not been initialized, set to 1 as well) """ + self._raster[0] = rows + if self._raster[1] == 0: + self._raster[1] = 1 + def setCols(self, columns): + """ Sets the number of columns in the raster (if rows have not been initialized, set to 1 as well) """ + self._raster[1] = columns + if self._raster[0] == 0: + self._raster[0] = 1 + def isRasterValid(self): + return self.getCols() > 0 and self.getRows() > 0 + def getRows(self): + return self._raster[0] + def getCols(self): + return self._raster[1] + def getRowH(self): + if self._raster[0] == 0: + return 0 + return self.h / self._raster[0] + def getColW(self): + if self._raster[1] == 0: + return 0 + return self.w / self._raster[1] class Match(Region): - """ Extended Region object with additional data on click target, match score """ - def __init__(self, score, target, rect): - super(Match, self).__init__(rect[0][0], rect[0][1], rect[1][0], rect[1][1]) - self._score = float(score) - if not target or not isinstance(target, Location): - raise TypeError("Match expected target to be a Location object") - self._target = target + """ Extended Region object with additional data on click target, match score """ + def __init__(self, score, target, rect): + super(Match, self).__init__(rect[0][0], rect[0][1], rect[1][0], rect[1][1]) + self._score = float(score) + if not target or not isinstance(target, Location): + raise TypeError("Match expected target to be a Location object") + self._target = target - def getScore(self): - """ Returns confidence score of the match """ - return self._score + def getScore(self): + """ Returns confidence score of the match """ + return self._score - def getTarget(self): - """ Returns the location of the match click target (center by default, but may be offset) """ - return self.getCenter().offset(self._target.x, self._target.y) + def getTarget(self): + """ Returns the location of the match click target (center by default, but may be offset) """ + return self.getCenter().offset(self._target.x, self._target.y) - def __repr__(self): - return "Match[{},{} {}x{}] score={.2f}, target={}".format(self.x, self.y, self.w, self.h, self._score, self._target.getTuple()) + def __repr__(self): + return "Match[{},{} {}x{}] score={:2f}, target={}".format(self.x, self.y, self.w, self.h, self._score, self._target.getTuple()) class Screen(Region): - """ Individual screen objects can be created for each monitor in a multi-monitor system. - - Screens are indexed according to the system order. 0 is the primary monitor (display 1), 1 is the next monitor, etc. - - Lackey also makes it possible to search all screens as a single "virtual screen," arranged - according to the system's settings. Screen(-1) returns this virtual screen. Note that the - larger your search region is, the slower your search will be, so it's best practice to adjust - your region to the particular area of the screen where you know your target will be. - - Note that Sikuli is inconsistent in identifying screens. In Windows, Sikuli identifies the - first hardware monitor as Screen(0) rather than the actual primary monitor. However, on OS X - it follows the latter convention. We've opted to make Screen(0) the actual primary monitor - (wherever the Start Menu/System Menu Bar is) across the board. - """ - def __init__(self, screenId=None): - """ Defaults to the main screen. """ - if not isinstance(screenId, int) or screenId < -1 or screenId >= len(PlatformManager.getScreenDetails()): - screenId = 0 - self._screenId = screenId - x, y, w, h = self.getBounds() - super(Screen, self).__init__(x, y, w, h) - def getNumberScreens(self): - """ Get the number of screens in a multi-monitor environment at the time the script is running """ - return len(PlatformManager.getScreenDetails()) - def getBounds(self): - """ Returns bounds of screen as (x, y, w, h) """ - return PlatformManager.getScreenBounds(self._screenId) - def capture(self, *args): #x=None, y=None, w=None, h=None): - """ Captures the region as an image and saves to a temporary file (specified by TMPDIR, TEMP, or TMP environmental variable) """ - if len(args) == 0: - # Capture screen region - region = self - elif isinstance(args[0], Region): - # Capture specified region - region = args[0] - elif isinstance(args[0], tuple): - # Capture region defined by specified tuple - region = Region(*args[0]) - elif isinstance(args[0], basestring): - # Interactive mode - raise NotImplementedError("Interactive capture mode not defined") - elif isinstance(args[0], int): - # Capture region defined by provided x,y,w,h - region = Region(*args) - bitmap = region.getBitmap() - tfile, tpath = tempfile.mkstemp(".png") - cv2.imwrite(tpath, bitmap) - return tpath - def selectRegion(self, text=""): - """ Not yet implemented """ - raise NotImplementedError() + """ Individual screen objects can be created for each monitor in a multi-monitor system. + + Screens are indexed according to the system order. 0 is the primary monitor (display 1), 1 is the next monitor, etc. + + Lackey also makes it possible to search all screens as a single "virtual screen," arranged + according to the system's settings. Screen(-1) returns this virtual screen. Note that the + larger your search region is, the slower your search will be, so it's best practice to adjust + your region to the particular area of the screen where you know your target will be. + + Note that Sikuli is inconsistent in identifying screens. In Windows, Sikuli identifies the + first hardware monitor as Screen(0) rather than the actual primary monitor. However, on OS X + it follows the latter convention. We've opted to make Screen(0) the actual primary monitor + (wherever the Start Menu/System Menu Bar is) across the board. + """ + def __init__(self, screenId=None): + """ Defaults to the main screen. """ + if not isinstance(screenId, int) or screenId < -1 or screenId >= len(PlatformManager.getScreenDetails()): + screenId = 0 + self._screenId = screenId + x, y, w, h = self.getBounds() + super(Screen, self).__init__(x, y, w, h) + def getNumberScreens(self): + """ Get the number of screens in a multi-monitor environment at the time the script is running """ + return len(PlatformManager.getScreenDetails()) + def getBounds(self): + """ Returns bounds of screen as (x, y, w, h) """ + return PlatformManager.getScreenBounds(self._screenId) + def capture(self, *args): #x=None, y=None, w=None, h=None): + """ Captures the region as an image and saves to a temporary file (specified by TMPDIR, TEMP, or TMP environmental variable) """ + if len(args) == 0: + # Capture screen region + region = self + elif isinstance(args[0], Region): + # Capture specified region + region = args[0] + elif isinstance(args[0], tuple): + # Capture region defined by specified tuple + region = Region(*args[0]) + elif isinstance(args[0], basestring): + # Interactive mode + raise NotImplementedError("Interactive capture mode not defined") + elif isinstance(args[0], int): + # Capture region defined by provided x,y,w,h + region = Region(*args) + bitmap = region.getBitmap() + tfile, tpath = tempfile.mkstemp(".png") + cv2.imwrite(tpath, bitmap) + return tpath + def selectRegion(self, text=""): + """ Not yet implemented """ + raise NotImplementedError() class Location(object): - """ Basic 2D point object """ - def __init__(self, x, y): - self.setLocation(x, y) - - def getX(self): - """ Returns the X-component of the location """ - return self.x - def getY(self): - """ Returns the Y-component of the location """ - return self.y - - def setLocation(self, x, y): - """Set the location of this object to the specified coordinates.""" - self.x = int(x) - self.y = int(y) - - def offset(self, dx, dy): - """Get a new location which is dx and dy pixels away horizontally and vertically from the current location.""" - return Location(self.x+dx, self.y+dy) - - def above(self, dy): - """Get a new location which is dy pixels vertically above the current location.""" - return Location(self.x, self.y-dy) - def below(self, dy): - """Get a new location which is dy pixels vertically below the current location.""" - return Location(self.x, self.y+dy) - def left(self, dx): - """Get a new location which is dx pixels horizontally to the left of the current location.""" - return Location(self.x-dx, self.y) - def right(self, dx): - """Get a new location which is dx pixels horizontally to the right of the current location.""" - return Location(self.x+dx, self.y) - - def getTuple(self): - """ Returns coordinates as a tuple (for some PlatformManager methods) """ - return (self.x, self.y) - - def __repr__(self): - return "(Location object at ({},{}))".format(self.x, self.y) + """ Basic 2D point object """ + def __init__(self, x, y): + self.setLocation(x, y) + + def getX(self): + """ Returns the X-component of the location """ + return self.x + def getY(self): + """ Returns the Y-component of the location """ + return self.y + + def setLocation(self, x, y): + """Set the location of this object to the specified coordinates.""" + self.x = int(x) + self.y = int(y) + + def offset(self, dx, dy): + """Get a new location which is dx and dy pixels away horizontally and vertically from the current location.""" + return Location(self.x+dx, self.y+dy) + + def above(self, dy): + """Get a new location dy pixels vertically above the current location.""" + return Location(self.x, self.y-dy) + def below(self, dy): + """Get a new location dy pixels vertically below the current location.""" + return Location(self.x, self.y+dy) + def left(self, dx): + """Get a new location dx pixels horizontally to the left of the current location.""" + return Location(self.x-dx, self.y) + def right(self, dx): + """Get a new location dx pixels horizontally to the right of the current location.""" + return Location(self.x+dx, self.y) + + def getTuple(self): + """ Returns coordinates as a tuple (for some PlatformManager methods) """ + return (self.x, self.y) + + def __repr__(self): + return "(Location object at ({},{}))".format(self.x, self.y) class Mouse(object): - """ Mid-level mouse routines. Interfaces with ``PlatformManager`` """ - def __init__(self): - self._defaultScanRate = 0.01 - - # Class constants - WHEEL_DOWN = 0 - WHEEL_UP = 1 - LEFT = 0 - MIDDLE = 1 - RIGHT = 2 - - def move(self, location): - """ Moves cursor to specified ``Location`` """ - - # Check if move point is outside a monitor rectangle - if not PlatformManager.isPointVisible(location.x, location.y): - # move point is outside a monitor rectangle. Snap to closest edge of primary monitor. - s_x, s_y, s_w, s_h = Screen(0).getBounds() - location.x = min(max(location.x, s_x), s_x+s_w) - location.y = min(max(location.y, s_y), s_y+s_h) - - PlatformManager.setMousePos(location.getTuple()) - - def getPos(self): - """ Gets ``Location`` of cursor """ - x, y = PlatformManager.getMousePos() - return Location(x, y) - - def moveSpeed(self, location, seconds=0.3): - """ Moves cursor to specified ``Location`` over ``seconds``. - - If ``seconds`` is 0, moves the cursor immediately. Used for smooth - somewhat-human-like motion. - """ - if seconds == 0: - # If the mouse isn't on the main screen, snap to point automatically instead of trying to track a path back - self.move(location) - return - - # Check if move point is outside a monitor rectangle - if not PlatformManager.isPointVisible(location.x, location.y): - # move point is outside a monitor rectangle. Snap to closest edge of primary monitor. - s_x, s_y, s_w, s_h = Screen(0).getBounds() - location.x = min(max(location.x, s_x), s_x+s_w) - location.y = min(max(location.y, s_y), s_y+s_h) - - frames = int(seconds / self._defaultScanRate) - while frames > 0: - mouse_pos = self.getPos() - deltax = int(round(float(location.x - mouse_pos.x) / frames)) - deltay = int(round(float(location.y - mouse_pos.y) / frames)) - self.move(Location(mouse_pos.x + deltax, mouse_pos.y + deltay)) - frames -= 1 - time.sleep(self._defaultScanRate) - - def click(self, button=0): - """ Clicks the specified mouse button. - - Use Mouse.LEFT, Mouse.MIDDLE, Mouse.RIGHT - """ - PlatformManager.clickMouse(button) - def buttonDown(self, button=0): - """ Holds down the specified mouse button. - - Use Mouse.LEFT, Mouse.MIDDLE, Mouse.RIGHT - """ - PlatformManager.mouseButtonDown(button) - def buttonUp(self, button=0): - """ Releases the specified mouse button. - - Use Mouse.LEFT, Mouse.MIDDLE, Mouse.RIGHT - """ - PlatformManager.mouseButtonUp(button) - def wheel(self, direction, steps): - """ Clicks the wheel the specified number of steps in the given direction. - - Use Mouse.WHEEL_DOWN, Mouse.WHEEL_UP - """ - return PlatformManager.mouseWheel(direction, steps) + """ Mid-level mouse routines. Interfaces with ``PlatformManager`` """ + def __init__(self): + self._defaultScanRate = 0.01 + + # Class constants + WHEEL_DOWN = 0 + WHEEL_UP = 1 + LEFT = 0 + MIDDLE = 1 + RIGHT = 2 + + def move(self, location): + """ Moves cursor to specified ``Location`` """ + + # Check if move point is outside a monitor rectangle + if not PlatformManager.isPointVisible(location.x, location.y): + # move point is outside a monitor rectangle. Snap to closest edge of primary monitor. + s_x, s_y, s_w, s_h = Screen(0).getBounds() + location.x = min(max(location.x, s_x), s_x+s_w) + location.y = min(max(location.y, s_y), s_y+s_h) + + PlatformManager.setMousePos(location.getTuple()) + + def getPos(self): + """ Gets ``Location`` of cursor """ + x, y = PlatformManager.getMousePos() + return Location(x, y) + + def moveSpeed(self, location, seconds=0.3): + """ Moves cursor to specified ``Location`` over ``seconds``. + + If ``seconds`` is 0, moves the cursor immediately. Used for smooth + somewhat-human-like motion. + """ + if seconds == 0: + # If the mouse isn't on the main screen, snap to point automatically instead of + # trying to track a path back + self.move(location) + return + + # Check if move point is outside a monitor rectangle + if not PlatformManager.isPointVisible(location.x, location.y): + # move point is outside a monitor rectangle. Snap to closest edge of primary monitor. + s_x, s_y, s_w, s_h = Screen(0).getBounds() + location.x = min(max(location.x, s_x), s_x+s_w) + location.y = min(max(location.y, s_y), s_y+s_h) + + frames = int(seconds / self._defaultScanRate) + while frames > 0: + mouse_pos = self.getPos() + deltax = int(round(float(location.x - mouse_pos.x) / frames)) + deltay = int(round(float(location.y - mouse_pos.y) / frames)) + self.move(Location(mouse_pos.x + deltax, mouse_pos.y + deltay)) + frames -= 1 + time.sleep(self._defaultScanRate) + + def click(self, button=0): + """ Clicks the specified mouse button. + + Use Mouse.LEFT, Mouse.MIDDLE, Mouse.RIGHT + """ + PlatformManager.clickMouse(button) + def buttonDown(self, button=0): + """ Holds down the specified mouse button. + + Use Mouse.LEFT, Mouse.MIDDLE, Mouse.RIGHT + """ + PlatformManager.mouseButtonDown(button) + def buttonUp(self, button=0): + """ Releases the specified mouse button. + + Use Mouse.LEFT, Mouse.MIDDLE, Mouse.RIGHT + """ + PlatformManager.mouseButtonUp(button) + def wheel(self, direction, steps): + """ Clicks the wheel the specified number of steps in the given direction. + + Use Mouse.WHEEL_DOWN, Mouse.WHEEL_UP + """ + return PlatformManager.mouseWheel(direction, steps) class Keyboard(object): - """ Mid-level keyboard routines. Interfaces with ``PlatformManager`` """ - def __init__(self): - pass - - def keyDown(self, keys): - """ Holds down the specified keys """ - return PlatformManager.pressKey(keys) - def keyUp(self, keys): - """ Releases the specified keys """ - return PlatformManager.releaseKey(keys) - def type(self, text, delay=0.1): - """ Types ``text`` with ``delay`` seconds between keypresses """ - return PlatformManager.typeKeys(text, delay) + """ Mid-level keyboard routines. Interfaces with ``PlatformManager`` """ + def __init__(self): + pass + + def keyDown(self, keys): + """ Holds down the specified keys """ + return PlatformManager.pressKey(keys) + def keyUp(self, keys): + """ Releases the specified keys """ + return PlatformManager.releaseKey(keys) + def type(self, text, delay=0.1): + """ Types ``text`` with ``delay`` seconds between keypresses """ + return PlatformManager.typeKeys(text, delay) class App(object): - """ Allows apps to be selected by title, PID, or by starting an - application directly. Can address individual windows tied to an - app. - - For more information, see [Sikuli's App documentation](http://sikulix-2014.readthedocs.io/en/latest/appclass.html#App) - """ - def __init__(self, identifier=None): - self._pid = None - self._search = "" - self._params = "" - self._process = None - self._defaultScanRate = 0.1 - self.proc = None - - # Replace class methods with instance methods - self.focus = self._focus_instance - self.close = self._close_instance - self.open = self._open_instance - - # Process `identifier` - if isinstance(identifier, int): - # `identifier` is a PID - Debug.log(3, "Creating App by PID ({})".format(identifier)) - self._pid = identifier - elif isinstance(identifier, basestring): - # `identifier` is either part of a window title - # or a command line to execute. Sikuli is ambiguous - # on this point: it treats the string as a window title - # unless explicitly told to open() the application. - Debug.log(3, "Creating App by string ({})".format(identifier)) - self._search = identifier - self._pid = PlatformManager.getWindowPID(PlatformManager.getWindowByTitle(re.escape(self._search))) - else: - self._pid = -1 # Unrecognized identifier, setting to empty app - - self._pid = self.getPID() # Confirm PID is an active process (sets to -1 otherwise) - - @classmethod - def pause(cls, waitTime): - time.sleep(waitTime) - - @classmethod - def focus(cls, appName): - """ Searches for exact text, case insensitive, anywhere in the window title. Brings the matching window to the foreground. - - As a class method, accessible as `App.focus(appName)`. As an instance method, accessible as `App(appName).focus()`. - """ - app = cls(appName) - return app.focus() - - def _focus_instance(self): - """ For instances of App, the ``focus()`` classmethod is replaced with this instance method. """ - if self._search: - Debug.log(3, "Focusing app with title like ({})".format(self._search)) - PlatformManager.focusWindow(PlatformManager.getWindowByTitle(re.escape(self._search))) - elif self._pid: - Debug.log(3, "Focusing app with pid ({})".format(self._pid)) - PlatformManager.focusWindow(PlatformManager.getWindowByPID(self._pid)) - return self - - @classmethod - def close(cls, appName): - """ Closes the process associated with the specified app. - - As a class method, accessible as `App.class(appName)`. As an instance method, accessible as `App(appName).close()`. - """ - return cls(appName).close() - def _close_instance(self): - if self._process: - self._process.terminate() - else: - PlatformManager.killProcess(self.getPID()) - - @classmethod - def open(self, executable): - """ Runs the specified command and returns an App linked to the generated PID. - - As a class method, accessible as `App.open(executable_path)`. As an instance method, accessible as `App(executable_path).open()`. - - If run as an instance method, runs the specified command and links the instance - to the generated PID. - """ - return App(executable).open() - def _open_instance(self, waitTime=0): - args = [self._search] - if self._params != "": - args.append(self._params) - self._process = subprocess.Popen(args, shell=False) - self._pid = self._process.pid - time.sleep(waitTime) - return self - - @classmethod - def focusedWindow(cls): - """ Returns a Region corresponding to whatever window is in the foreground """ - x,y,w,h = PlatformManager.getWindowRect(PlatformManager.getForegroundWindow()) - return Region(x,y,w,h) - - def getWindow(self): - """ Returns the title of the main window of the app. - """ - if self._process: - # self._search is a path, not a window title - return PlatformManager.getWindowTitle(PlatformManager.getWindowByPID(self.getPID())) - elif self._search: - # self._search is (part of) a window title - return PlatformManager.getWindowTitle(PlatformManager.getWindowByTitle(re.escape(self._search))) - elif self._pid > 0: - # Search by PID - return PlatformManager.getWindowTitle(PlatformManager.getWindowByPID(self.getPID())) - else: - return "" - def getName(self): - """ Returns the short name of the app as shown in the process list """ - return PlatformManager.getProcessName(self.getPID()) - def getPID(self): - """ Returns the PID for the associated app (or -1, if no app is associated or the app is not running) """ - if self._pid is not None: - if not PlatformManager.isPIDValid(self._pid): - self._pid = -1 - return self._pid - return -1 - - def hasWindow(self): - """ Returns True if the process has a window associated, False otherwise """ - return PlatformManager.getWindowByPID(self.getPID()) is not None - - def window(self, windowNum=0): - """ Returns the region corresponding to the specified window of the app. - - Defaults to the first window found for the corresponding PID. - """ - if self._pid == -1: - raise FindFailed("Window not found for app \"{}\"".format(self._search)) - x,y,w,h = PlatformManager.getWindowRect(PlatformManager.getWindowByPID(self._pid, windowNum)) - return Region(x,y,w,h).clipRegionToScreen() - - def setUsing(self, params): - self._params = params - - def __repr__(self): - """ Returns a string representation of the app """ - return "[{pid}:{executable} ({windowtitle})] {searchtext}".format(pid=self._pid, executable=self.getName(), windowtitle=self.getWindow(), searchtext=self._search) - - def isRunning(self, waitTime=0): - """ If PID isn't set yet, checks if there is a window with the specified title. """ - waitUntil = time.time() + waitTime - while True: - if self.getPID() > 0: - return True - else: - self._pid = PlatformManager.getWindowPID(PlatformManager.getWindowByTitle(re.escape(self._search))) - - # Check if we've waited long enough - if time.time() > waitUntil: - break - else: - time.sleep(self._defaultScanRate) - return self.getPID() > 0 + """ Allows apps to be selected by title, PID, or by starting an + application directly. Can address individual windows tied to an + app. + + For more information, see [Sikuli's App documentation](http://sikulix-2014.readthedocs.io/en/latest/appclass.html#App) + """ + def __init__(self, identifier=None): + self._pid = None + self._search = identifier + self._title = "" + self._exec = "" + self._params = "" + self._process = None + self._defaultScanRate = 0.1 + self.proc = None + + # Replace class methods with instance methods + self.focus = self._focus_instance + self.close = self._close_instance + self.open = self._open_instance + + # Process `identifier` + if isinstance(identifier, int): + # `identifier` is a PID + Debug.log(3, "Creating App by PID ({})".format(identifier)) + self._pid = identifier + elif isinstance(identifier, basestring): + # `identifier` is either part of a window title + # or a command line to execute. If it starts with a "+", + # launch it immediately. Otherwise, store it until open() is called. + Debug.log(3, "Creating App by string ({})".format(identifier)) + launchNow = False + if identifier.startswith("+"): + # Should launch immediately - strip the `+` sign and continue + launchNow = True + identifier = identifier[1:] + # Check if `identifier` is an executable commmand + # Possible formats: + # Case 1: notepad.exe C:\sample.txt + # Case 2: "C:\Program Files\someprogram.exe" -flag + + # Extract hypothetical executable name + if identifier.startswith('"'): + executable = identifier[1:].split('"')[0] + params = identifier[len(executable)+2:].split(" ") if len(identifier) > len(executable) + 2 else [] + else: + executable = identifier.split(" ")[0] + params = identifier[len(executable)+1:].split(" ") if len(identifier) > len(executable) + 1 else [] + + # Check if hypothetical executable exists + if self._which(executable) is not None: + # Found the referenced executable + self._exec = executable + self._params = params + # If the command was keyed to execute immediately, do so. + if launchNow: + self.open() + else: + # No executable found - treat as a title instead. Try to capture window. + self._title = identifier + self.open + else: + self._pid = -1 # Unrecognized identifier, setting to empty app + + self._pid = self.getPID() # Confirm PID is an active process (sets to -1 otherwise) + + def _which(self, program): + """ Private method to check if an executable exists + + Shamelessly stolen from http://stackoverflow.com/questions/377017/test-if-executable-exists-in-python + """ + def is_exe(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + fpath, fname = os.path.split(program) + if fpath: + if is_exe(program): + return program + else: + for path in os.environ["PATH"].split(os.pathsep): + path = path.strip('"') + exe_file = os.path.join(path, program) + if is_exe(exe_file): + return exe_file + return None + + @classmethod + def pause(cls, waitTime): + time.sleep(waitTime) + + @classmethod + def focus(cls, appName): + """ Searches for exact text, case insensitive, anywhere in the window title. + + Brings the matching window to the foreground. + + As a class method, accessible as `App.focus(appName)`. As an instance method, + accessible as `App(appName).focus()`. + """ + app = cls(appName) + return app.focus() + def _focus_instance(self): + """ In instances, the ``focus()`` classmethod is replaced with this instance method. """ + if self._title: + Debug.log(3, "Focusing app with title like ({})".format(self._title)) + PlatformManager.focusWindow(PlatformManager.getWindowByTitle(re.escape(self._title))) + if self.getPID() == -1: + self.open() + elif self._pid and self._pid != -1: + Debug.log(3, "Focusing app with pid ({})".format(self._pid)) + PlatformManager.focusWindow(PlatformManager.getWindowByPID(self._pid)) + return self + + @classmethod + def close(cls, appName): + """ Closes the process associated with the specified app. + + As a class method, accessible as `App.class(appName)`. + As an instance method, accessible as `App(appName).close()`. + """ + return cls(appName).close() + def _close_instance(self): + if self._process: + self._process.terminate() + elif self.getPID() != -1: + PlatformManager.killProcess(self.getPID()) + + @classmethod + def open(self, executable): + """ Runs the specified command and returns an App linked to the generated PID. + + As a class method, accessible as `App.open(executable_path)`. + As an instance method, accessible as `App(executable_path).open()`. + """ + return App(executable).open() + def _open_instance(self, waitTime=0): + if self._exec != "": + # Open from an executable + parameters + self._process = subprocess.Popen([self._exec] + self._params, shell=False) + self._pid = self._process.pid + elif self._title != "": + # Capture an existing window that matches self._title + self._pid = PlatformManager.getWindowPID( + PlatformManager.getWindowByTitle( + re.escape(self._title))) + time.sleep(waitTime) + return self + + @classmethod + def focusedWindow(cls): + """ Returns a Region corresponding to whatever window is in the foreground """ + x, y, w, h = PlatformManager.getWindowRect(PlatformManager.getForegroundWindow()) + return Region(x,y,w,h) + + def getWindow(self): + """ Returns the title of the main window of the currently open app. + + Returns an empty string if no match could be found. + """ + if self.getPID() != -1: + return PlatformManager.getWindowTitle(PlatformManager.getWindowByPID(self.getPID())) + else: + return "" + def getName(self): + """ Returns the short name of the app as shown in the process list """ + return PlatformManager.getProcessName(self.getPID()) + def getPID(self): + """ Returns the PID for the associated app (or -1, if no app is associated or the app is not running) """ + if self._pid is not None: + if not PlatformManager.isPIDValid(self._pid): + self._pid = -1 + return self._pid + return -1 + + def hasWindow(self): + """ Returns True if the process has a window associated, False otherwise """ + return PlatformManager.getWindowByPID(self.getPID()) is not None + + def window(self, windowNum=0): + """ Returns the region corresponding to the specified window of the app. + + Defaults to the first window found for the corresponding PID. + """ + if self._pid == -1: + raise FindFailed("Window not found for app \"{}\"".format(self)) + x,y,w,h = PlatformManager.getWindowRect(PlatformManager.getWindowByPID(self._pid, windowNum)) + return Region(x,y,w,h).clipRegionToScreen() + + def setUsing(self, params): + self._params = params.split(" ") + + def __repr__(self): + """ Returns a string representation of the app """ + return "[{pid}:{executable} ({windowtitle})] {searchtext}".format(pid=self._pid, executable=self.getName(), windowtitle=self.getWindow(), searchtext=self._search) + + def isRunning(self, waitTime=0): + """ If PID isn't set yet, checks if there is a window with the specified title. """ + waitUntil = time.time() + waitTime + while True: + if self.getPID() > 0: + return True + else: + self._pid = PlatformManager.getWindowPID(PlatformManager.getWindowByTitle(re.escape(self._title))) + + # Check if we've waited long enough + if time.time() > waitUntil: + break + else: + time.sleep(self._defaultScanRate) + return self.getPID() > 0 diff --git a/lackey/Settings.py b/lackey/Settings.py index 88fc14a..9a7434d 100644 --- a/lackey/Settings.py +++ b/lackey/Settings.py @@ -1,160 +1,167 @@ +""" Defines Settings and Debug objects """ import datetime import os +import __main__ class DebugMaster(object): - """ Used to create the global Debug object """ - _log_file = None - _debug_level = 0 - _logger = None - _logger_no_prefix = False - _logger_methods = { - "user": None, - "info": None, - "action": None, - "error": None, - "debug": None - } - def user(self, message): - """ Creates a user log (if user logging is turned on) + """ Used to create the global Debug object """ + _log_file = None + _debug_level = 0 + _logger = None + _logger_no_prefix = False + _logger_methods = { + "user": None, + "info": None, + "action": None, + "error": None, + "debug": None + } + def user(self, message): + """ Creates a user log (if user logging is turned on) - Uses the log path defined by ``Debug.setUserLogFile()``. If no log file is - defined, sends to STDOUT + Uses the log path defined by ``Debug.setUserLogFile()``. If no log file is + defined, sends to STDOUT - Note: Does *not* use Java string formatting like Sikuli. - Format your message with Python ``basestring.format()`` instead. - """ - if Settings.UserLogs: - self._write_log(Settings.UserLogPrefix, Settings.UserLogTime, message) - def history(self, message): - """ Records an Action-level log message + Note: Does *not* use Java string formatting like Sikuli. + Format your message with Python ``basestring.format()`` instead. + """ + if Settings.UserLogs: + self._write_log(Settings.UserLogPrefix, Settings.UserLogTime, message) + def history(self, message): + """ Records an Action-level log message - Uses the log path defined by ``Debug.setUserLogFile()``. If no log file is - defined, sends to STDOUT - """ - if Settings.ActionLogs: - self._write_log("action", Settings.LogTime, message) - def error(self, message): - """ Records an Error-level log message + Uses the log path defined by ``Debug.setUserLogFile()``. If no log file is + defined, sends to STDOUT + """ + if Settings.ActionLogs: + self._write_log("action", Settings.LogTime, message) + def error(self, message): + """ Records an Error-level log message - Uses the log path defined by ``Debug.setUserLogFile()``. If no log file is - defined, sends to STDOUT - """ - if Settings.ErrorLogs: - self._write_log("error", Settings.LogTime, message) - def info(self, message): - """ Records an Info-level log message + Uses the log path defined by ``Debug.setUserLogFile()``. If no log file is + defined, sends to STDOUT + """ + if Settings.ErrorLogs: + self._write_log("error", Settings.LogTime, message) + def info(self, message): + """ Records an Info-level log message - Uses the log path defined by ``Debug.setUserLogFile()``. If no log file is - defined, sends to STDOUT - """ - if Settings.InfoLogs: - self._write_log("info", Settings.LogTime, message) - def on(self, level): - """ Turns on all debugging messages up to the specified level + Uses the log path defined by ``Debug.setUserLogFile()``. If no log file is + defined, sends to STDOUT + """ + if Settings.InfoLogs: + self._write_log("info", Settings.LogTime, message) + def on(self, level): + """ Turns on all debugging messages up to the specified level - 0 = None; 1 = User; - """ - if isinstance(level, int) and level >= 0 and level <= 3: - self._debug_level = level - def off(self): - """ Turns off all debugging messages """ - self._debug_level = 0 - def log(self, level, message): - """ Records a Debug-level log message - - Uses the log path defined by ``Debug.setUserLogFile()``. If no log file is - defined, sends to STDOUT - """ - if level <= self._debug_level: - self._write_log("debug", Settings.LogTime, message) + 0 = None; 1 = User; + """ + if isinstance(level, int) and level >= 0 and level <= 3: + self._debug_level = level + def off(self): + """ Turns off all debugging messages """ + self._debug_level = 0 + def log(self, level, message): + """ Records a Debug-level log message + Uses the log path defined by ``Debug.setUserLogFile()``. If no log file is + defined, sends to STDOUT + """ + if level <= self._debug_level: + self._write_log("debug", Settings.LogTime, message) - def setLogger(self, logger_obj): - """ Sets log handler to ``logger_obj`` """ - self._logger = logger_obj - def setLoggerNoPrefix(self, logger_obj): - """ Sets log handler to ``logger_obj`` """ - self._logger = logger_obj - self._logger_no_prefix = True - def setLoggerAll(self, mthd): - """ Sends all messages to ``logger.[mthd]()`` for handling """ - for key in _logger_methods.keys(): - _logger_methods[key] = mthd - def setLoggerUser(self, mthd): - """ Sends user messages to ``logger.[mthd]()`` for handling """ - _logger_methods["user"] = mthd - def setLoggerInfo(self, mthd): - """ Sends info messages to ``logger.[mthd]()`` for handling """ - _logger_methods["info"] = mthd - def setLoggerAction(self, mthd): - """ Sends action messages to ``logger.[mthd]()`` for handling """ - _logger_methods["action"] = mthd - def setLoggerError(self, mthd): - """ Sends error messages to ``logger.[mthd]()`` for handling """ - _logger_methods["error"] = mthd - def setLoggerDebug(self, mthd): - """ Sends debug messages to ``logger.[mthd]()`` for handling """ - _logger_methods["debug"] = mthd - def setLogFile(self, filepath): - """ Defines the file to which output log messages should be sent """ - parsed_path = os.abspath(filepath) - # Checks if the provided log filename is in a real directory, and that - # the filename itself is not a directory. - if os.path.isdir(os.path.dirname(parsed_path)) and not os.path.isdir(parsed_path): - self._log_file = parsed_path - else: - raise FileNotFoundError(filepath) - def _write_log(self, log_type, log_time, message): - """ Private method to abstract log writing for different types of logs """ - timestamp = " %Y-%m-%d %H:%M:%S".format(datetime.datetime.now()) - log_entry = "[{}{}] {}".format(log_type, timestamp if log_time else "", message) - if self._logger and callable(getattr(self._logger, self._logger_methods[log_type], None)): - # Check for log handler (sends message only if _logger_no_prefix is True) - getattr(self._logger, self._logger_methods[log_type], None)(message if self._logger_no_prefix else log_entry) - elif self._log_file: - # Otherwise write to file, if a file has been specified - with open(self._log_file) as logfile: - logfile.write(log_entry) - else: - # Otherwise, print to STDOUT - print log_entry + def setLogger(self, logger_obj): + """ Sets log handler to ``logger_obj`` """ + self._logger = logger_obj + def setLoggerNoPrefix(self, logger_obj): + """ Sets log handler to ``logger_obj`` """ + self._logger = logger_obj + self._logger_no_prefix = True + def setLoggerAll(self, mthd): + """ Sends all messages to ``logger.[mthd]()`` for handling """ + for key in self._logger_methods: + self._logger_methods[key] = mthd + def setLoggerUser(self, mthd): + """ Sends user messages to ``logger.[mthd]()`` for handling """ + self._logger_methods["user"] = mthd + def setLoggerInfo(self, mthd): + """ Sends info messages to ``logger.[mthd]()`` for handling """ + self._logger_methods["info"] = mthd + def setLoggerAction(self, mthd): + """ Sends action messages to ``logger.[mthd]()`` for handling """ + self._logger_methods["action"] = mthd + def setLoggerError(self, mthd): + """ Sends error messages to ``logger.[mthd]()`` for handling """ + self._logger_methods["error"] = mthd + def setLoggerDebug(self, mthd): + """ Sends debug messages to ``logger.[mthd]()`` for handling """ + self._logger_methods["debug"] = mthd + def setLogFile(self, filepath): + """ Defines the file to which output log messages should be sent """ + parsed_path = os.path.abspath(filepath) + # Checks if the provided log filename is in a real directory, and that + # the filename itself is not a directory. + if os.path.isdir(os.path.dirname(parsed_path)) and not os.path.isdir(parsed_path): + self._log_file = parsed_path + else: + raise IOError("File not found: " + filepath) + def _write_log(self, log_type, log_time, message): + """ Private method to abstract log writing for different types of logs """ + timestamp = datetime.datetime.now().strftime(" %Y-%m-%d %H:%M:%S") + log_entry = "[{}{}] {}".format(log_type, timestamp if log_time else "", message) + if self._logger and callable(getattr(self._logger, self._logger_methods[log_type], None)): + # Check for log handler (sends message only if _logger_no_prefix is True) + getattr( + self._logger, + self._logger_methods[log_type], + None + )(message if self._logger_no_prefix else log_entry) + elif self._log_file: + # Otherwise write to file, if a file has been specified + with open(self._log_file) as logfile: + logfile.write(log_entry) + else: + # Otherwise, print to STDOUT + print log_entry class SettingsMaster(object): - """ Global settings that Lackey refers to by default """ - ## Logging Settings - ActionLogs = True # Message prefix: [log] - InfoLogs = True # Message prefix: [info] - DebugLogs = False # Message prefix: [debug] - LogTime = False - ### User Logging - UserLogs = True - UserLogPrefix = "user" - UserLogTime = True + """ Global settings that Lackey refers to by default """ + ## Logging Settings + ActionLogs = True # Message prefix: [log] + InfoLogs = True # Message prefix: [info] + DebugLogs = False # Message prefix: [debug] + ErrorLogs = False # Message prefix: [error] + LogTime = False + ### User Logging + UserLogs = True + UserLogPrefix = "user" + UserLogTime = True - ## Region Settings - MinSimilarity = 0.7 - SlowMotionDelay = 3 # Extra duration of slowed-down visual effects - WaitScanRate = 3 # Searches per second - ObserveScanRate = 3 # Searches per second (observers) - OberveMinChangedPixels = 50 # Threshold to trigger onChange() (not implemented yet) + ## Region Settings + MinSimilarity = 0.7 + SlowMotionDelay = 3 # Extra duration of slowed-down visual effects + WaitScanRate = 3 # Searches per second + ObserveScanRate = 3 # Searches per second (observers) + OberveMinChangedPixels = 50 # Threshold to trigger onChange() (not implemented yet) - ## Keyboard/Mouse Settings - MoveMouseDelay = 0.3 # Time to take moving mouse to target location - DelayBeforeMouseDown = 0.3 - DelayBeforeDrag = 0.3 - DelayBeforeDrop = 0.3 - ClickDelay = 0.0 # Resets to 0 after next click - TypeDelay = 0.0 # Resets to 0 after next keypress + ## Keyboard/Mouse Settings + MoveMouseDelay = 0.3 # Time to take moving mouse to target location + DelayBeforeMouseDown = 0.3 + DelayBeforeDrag = 0.3 + DelayBeforeDrop = 0.3 + ClickDelay = 0.0 # Resets to 0 after next click + TypeDelay = 0.0 # Resets to 0 after next keypress - ## Action Settings - ShowActions = False + ## Action Settings + ShowActions = False - ## File Settings - BundlePath = os.path.abspath(os.getcwd()) # Path to Sikuli project - ImagePaths = [] - OcrDataPath = None + ## File Settings + # Path to Sikuli project - might not be current directory + BundlePath = os.path.dirname(os.path.abspath(os.path.join(os.getcwd(), __main__.__file__))) + ImagePaths = [] + OcrDataPath = None - ## Popup settings - PopupLocation = None + ## Popup settings + PopupLocation = None Debug = DebugMaster() -Settings = SettingsMaster() \ No newline at end of file +Settings = SettingsMaster() diff --git a/lackey/SikuliGui.py b/lackey/SikuliGui.py index 8e85efb..ea9ac3d 100644 --- a/lackey/SikuliGui.py +++ b/lackey/SikuliGui.py @@ -1,143 +1,203 @@ +""" Gui helper classes for Lackey + +""" + import Tkinter as tk import ttk -from .Settings import Settings, Debug +from .Settings import Settings class PopupInput(tk.Frame): - def __init__(self, parent, msg, default, title, hidden, text_variable): - tk.Frame.__init__(self, parent) - self.parent = parent - self.parent.protocol("WM_DELETE_WINDOW", self.cancel_function) - self.parent.bind('', self.ok_function) - self.parent.title(title) - self.input_text = text_variable - if Settings.PopupLocation: - self.geometry("+{}+{}".format(Settings.PopupLocation.x, Settings.PopupLocation.y)) - self.msg = tk.Message(self.parent, text=msg) - self.msg.grid(row=0, sticky="NSEW", padx=10, pady=10) - self.input_entry = tk.Entry(self.parent, width=50, textvariable=self.input_text) - if hidden: - self.input_entry.config(show="*") - self.input_entry.grid(row=1, sticky="EW", padx=10) - self.button_frame = tk.Frame(self.parent) - self.button_frame.grid(row=2, sticky="E") - self.cancel = tk.Button(self.button_frame, text="Cancel", command=self.cancel_function, width=10) - self.cancel.grid(row=0, column=0, padx=10, pady=10) - self.ok = tk.Button(self.button_frame, text="Ok", command=self.ok_function, width=10) - self.ok.grid(row=0, column=1, padx=10, pady=10) - self.input_entry.focus_set() - - def cancel_function(self): - self.input_text.set("") - self.parent.destroy() - def ok_function(self, event=None): - self.parent.destroy() + """ A basic popup dialog with a text input """ + def __init__(self, parent, msg, title, hidden, text_variable): + tk.Frame.__init__(self, parent) + self.parent = parent + self.parent.protocol("WM_DELETE_WINDOW", self.cancel_function) + self.parent.bind('', self.ok_function) + self.parent.title(title) + self.input_text = text_variable + if Settings.PopupLocation: + self.parent.geometry("+{}+{}".format( + Settings.PopupLocation.x, + Settings.PopupLocation.y)) + self.msg = tk.Message(self.parent, text=msg) + self.msg.grid(row=0, sticky="NSEW", padx=10, pady=10) + self.input_entry = tk.Entry(self.parent, width=50, textvariable=self.input_text) + if hidden: + self.input_entry.config(show="*") + self.input_entry.grid(row=1, sticky="EW", padx=10) + self.button_frame = tk.Frame(self.parent) + self.button_frame.grid(row=2, sticky="E") + self.cancel = tk.Button( + self.button_frame, + text="Cancel", + command=self.cancel_function, + width=10) + self.cancel.grid(row=0, column=0, padx=10, pady=10) + self.ok_button = tk.Button( + self.button_frame, + text="Ok", + command=self.ok_function, + width=10) + self.ok_button.grid(row=0, column=1, padx=10, pady=10) + self.input_entry.focus_set() + + def cancel_function(self): + """ Handler for cancel button """ + self.input_text.set("") + self.parent.destroy() + def ok_function(self, event=None): + """ Handler for ok button """ + #pylint: disable=unused-argument + self.parent.destroy() class PopupList(tk.Frame): - def __init__(self, parent, msg, title, options, default, text_variable): - tk.Frame.__init__(self, parent) - self.parent = parent - self.parent.protocol("WM_DELETE_WINDOW", self.cancel_function) - self.parent.bind('', self.ok_function) - self.parent.title(title) - self.input_text = text_variable - self.input_text.set(default) - if Settings.PopupLocation: - self.geometry("+{}+{}".format(Settings.PopupLocation.x, Settings.PopupLocation.y)) - self.msg = tk.Message(self.parent, text=msg) - self.msg.grid(row=0, sticky="NSEW", padx=10, pady=10) - self.input_list = ttk.Combobox(self.parent, textvariable=self.input_text, state="readonly", values=options) - #self.input_list.activate(options.index(default)) - self.input_list.grid(row=1, sticky="EW", padx=10) - self.button_frame = tk.Frame(self.parent) - self.button_frame.grid(row=2, sticky="E") - self.cancel = tk.Button(self.button_frame, text="Cancel", command=self.cancel_function, width=10) - self.cancel.grid(row=0, column=0, padx=10, pady=10) - self.ok = tk.Button(self.button_frame, text="Ok", command=self.ok_function, width=10) - self.ok.grid(row=0, column=1, padx=10, pady=10) - self.input_list.focus_set() - - def cancel_function(self): - self.input_text.set("") - self.parent.destroy() - def ok_function(self, event=None): - #self.input_text.set(self.input_list.get(self.input_list.cur_selection()[0])) - self.parent.destroy() + """ A basic popup dialog with a list dropdown """ + def __init__(self, parent, msg, title, options, default, text_variable): + tk.Frame.__init__(self, parent) + self.parent = parent + self.parent.protocol("WM_DELETE_WINDOW", self.cancel_function) + self.parent.bind('', self.ok_function) + self.parent.title(title) + self.input_text = text_variable + self.input_text.set(default) + if Settings.PopupLocation: + self.parent.geometry("+{}+{}".format( + Settings.PopupLocation.x, + Settings.PopupLocation.y)) + self.msg = tk.Message(self.parent, text=msg) + self.msg.grid(row=0, sticky="NSEW", padx=10, pady=10) + self.input_list = ttk.Combobox( + self.parent, + textvariable=self.input_text, + state="readonly", + values=options) + #self.input_list.activate(options.index(default)) + self.input_list.grid(row=1, sticky="EW", padx=10) + self.button_frame = tk.Frame(self.parent) + self.button_frame.grid(row=2, sticky="E") + self.cancel = tk.Button( + self.button_frame, + text="Cancel", + command=self.cancel_function, + width=10) + self.cancel.grid(row=0, column=0, padx=10, pady=10) + self.ok_button = tk.Button( + self.button_frame, + text="Ok", + command=self.ok_function, + width=10) + self.ok_button.grid(row=0, column=1, padx=10, pady=10) + self.input_list.focus_set() + + def cancel_function(self): + """ Handler for cancel button """ + self.input_text.set("") + self.parent.destroy() + def ok_function(self, event=None): + """ Handler for ok button """ + #pylint: disable=unused-argument + #self.input_text.set(self.input_list.get(self.input_list.cur_selection()[0])) + self.parent.destroy() class PopupTextarea(tk.Frame): - def __init__(self, parent, message, title, lines, width, text, input_text): - tk.Frame.__init__(self, parent) - self.parent = parent - self.parent.protocol("WM_DELETE_WINDOW", self.cancel_function) - #self.parent.bind('', self.ok_function) - self.parent.title(title) - self.input_text = input_text - if Settings.PopupLocation: - self.geometry("+{}+{}".format(Settings.PopupLocation.x, Settings.PopupLocation.y)) - - self.input_entry = TextExtension(self.parent, textvariable=self.input_text, width=width, height=lines) - self.input_entry.grid(row=1, sticky="EW", padx=10, pady=10) - - self.msg = tk.Message(self.parent, text=message) - self.msg.grid(row=0, sticky="NSEW", padx=10) - - self.button_frame = tk.Frame(self.parent) - self.button_frame.grid(row=2, sticky="E") - self.cancel = tk.Button(self.button_frame, text="Cancel", command=self.cancel_function, width=10) - self.cancel.grid(row=0, column=0, padx=10, pady=10) - self.ok = tk.Button(self.button_frame, text="Ok", command=self.ok_function, width=10) - self.ok.grid(row=0, column=1, padx=10, pady=10) - self.input_entry._text_widget.focus_set() - def cancel_function(self): - self.input_text.set("") - self.parent.destroy() - def ok_function(self, event=None): - self.parent.destroy() + """ A basic popup dialog with a textarea input """ + def __init__(self, parent, message, title, lines, width, input_text): + tk.Frame.__init__(self, parent) + self.parent = parent + self.parent.protocol("WM_DELETE_WINDOW", self.cancel_function) + #self.parent.bind('', self.ok_function) + self.parent.title(title) + self.input_text = input_text + if Settings.PopupLocation: + self.parent.geometry("+{}+{}".format( + Settings.PopupLocation.x, + Settings.PopupLocation.y)) + + self.input_entry = TextExtension( + self.parent, + textvariable=self.input_text, + width=width, + height=lines) + self.input_entry.grid(row=1, sticky="EW", padx=10, pady=10) + + self.msg = tk.Message(self.parent, text=message) + self.msg.grid(row=0, sticky="NSEW", padx=10) + + self.button_frame = tk.Frame(self.parent) + self.button_frame.grid(row=2, sticky="E") + self.cancel = tk.Button( + self.button_frame, + text="Cancel", + command=self.cancel_function, + width=10) + self.cancel.grid(row=0, column=0, padx=10, pady=10) + self.ok_button = tk.Button( + self.button_frame, + text="Ok", + command=self.ok_function, + width=10) + self.ok_button.grid(row=0, column=1, padx=10, pady=10) + self.input_entry.focus() + def cancel_function(self): + """ Handler for cancel button """ + self.input_text.set("") + self.parent.destroy() + def ok_function(self, event=None): + """ Handler for ok button """ + #pylint: disable=unused-argument + self.parent.destroy() class TextExtension(tk.Frame): - """Extends Frame. Intended as a container for a Text field. Better related data handling - and has Y scrollbar.""" + """Extends Frame. Intended as a container for a Text field. Better related data handling + and has Y scrollbar.""" + + def __init__(self, master, textvariable=None, *args, **kwargs): - def __init__(self, master, textvariable=None, *args, **kwargs): + tk.Frame.__init__(self, master) + # Init GUI - tk.Frame.__init__(self, master) - # Init GUI + self._y_scrollbar = tk.Scrollbar(self, orient=tk.VERTICAL) - self._y_scrollbar = tk.Scrollbar(self, orient=tk.VERTICAL) + self._text_widget = tk.Text(self, yscrollcommand=self._y_scrollbar.set, *args, **kwargs) + self._text_widget.pack(side=tk.LEFT, fill=tk.BOTH, expand=1) - self._text_widget = tk.Text(self, yscrollcommand=self._y_scrollbar.set, *args, **kwargs) - self._text_widget.pack(side=tk.LEFT, fill=tk.BOTH, expand=1) + self._y_scrollbar.config(command=self._text_widget.yview) + self._y_scrollbar.pack(side=tk.RIGHT, fill=tk.Y) - self._y_scrollbar.config(command=self._text_widget.yview) - self._y_scrollbar.pack(side=tk.RIGHT, fill=tk.Y) + if textvariable is not None: + if not isinstance(textvariable, tk.Variable): + raise TypeError("tkinter.Variable type expected, {} given.".format( + type(textvariable))) + self._text_variable = textvariable + self.var_modified() + self._text_trace = self._text_widget.bind('<>', self.text_modified) + self._var_trace = textvariable.trace("w", self.var_modified) - if textvariable is not None: - if not (isinstance(textvariable, tk.Variable)): - raise TypeError("tkinter.Variable type expected, " + str(type(textvariable)) + " given.".format(type(textvariable))) - self._text_variable = textvariable - self.var_modified() - self._text_trace = self._text_widget.bind('<>', self.text_modified) - self._var_trace = textvariable.trace("w", self.var_modified) + def text_modified(self): + #pylint: disable=unused-argument + if self._text_variable is not None: + self._text_variable.trace_vdelete("w", self._var_trace) + self._text_variable.set(self._text_widget.get(1.0, tk.END)) + self._var_trace = self._text_variable.trace("w", self.var_modified) + self._text_widget.edit_modified(False) - def text_modified(self, *args): - if self._text_variable is not None: - self._text_variable.trace_vdelete("w", self._var_trace) - self._text_variable.set(self._text_widget.get(1.0, tk.END)) - self._var_trace = self._text_variable.trace("w", self.var_modified) - self._text_widget.edit_modified(False) + def var_modified(self): + #pylint: disable=unused-argument + self.set_text(self._text_variable.get()) + self._text_widget.edit_modified(False) - def var_modified(self, *args): - self.set_text(self._text_variable.get()) - self._text_widget.edit_modified(False) + def unhook(self): + if self._text_variable is not None: + self._text_variable.trace_vdelete("w", self._var_trace) - def unhook(self): - if self._text_variable is not None: - self._text_variable.trace_vdelete("w", self._var_trace) + def clear(self): + self._text_widget.delete(1.0, tk.END) - def clear(self): - self._text_widget.delete(1.0, tk.END) + def set_text(self, _value): + self.clear() + if _value is not None: + self._text_widget.insert(tk.END, _value) - def set_text(self, _value): - self.clear() - if (_value is not None): - self._text_widget.insert(tk.END, _value) \ No newline at end of file + def focus(self): + self._text_widget.focus_set() diff --git a/lackey/TemplateMatchers.py b/lackey/TemplateMatchers.py index 20358fa..afc30ea 100644 --- a/lackey/TemplateMatchers.py +++ b/lackey/TemplateMatchers.py @@ -6,272 +6,324 @@ from .Settings import Debug class NaiveTemplateMatcher(object): - def __init__(self, haystack): - self.haystack = haystack - - def findBestMatch(self, needle, similarity): - """ Find the best match for ``needle`` that has a similarity better than or equal to ``similarity``. - - Returns a tuple of ``(position, confidence)`` if a match is found, or ``None`` otherwise. - """ - method = cv2.TM_CCOEFF_NORMED - position = None - - match = cv2.matchTemplate(self.haystack,needle,method) - min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(match) - if method == cv2.TM_SQDIFF_NORMED or method == cv2.TM_SQDIFF: - confidence = min_val - best_loc = min_loc - if min_val <= 1-similarity: - # Confidence checks out - position = min_loc - else: - confidence = max_val - best_loc = max_loc - if max_val >= similarity: - # Confidence checks out - position = max_loc - - if not position: - return None - - return (position, confidence) - - def findAllMatches(self, needle, similarity): - """ Find all matches for ``needle`` that has a similarity better than or equal to ``similarity``. - - Returns an array of tuples ``(position, confidence)`` if match(es) is/are found, or an empty array otherwise. - """ - positions = [] - method = cv2.TM_CCOEFF_NORMED - - match = cv2.matchTemplate(self.haystack,self.needle,method) - - indices = (-match).argpartition(100, axis=None)[:100] # Review the 100 top matches - unraveled_indices = numpy.array(numpy.unravel_index(indices, match.shape)).T - for location in unraveled_indices: - y, x = location - confidence = match[y][x] - if method == cv2.TM_SQDIFF_NORMED or method == cv2.TM_SQDIFF: - if confidence <= 1-similarity: - positions.append(((x,y),confidence)) - else: - if confidence >= similarity: - positions.append(((x,y),confidence)) - - positions.sort(key=lambda x: (x[0][1], x[0][0])) - return positions + """ Python wrapper for OpenCV's TemplateMatcher + + Does not try to optimize speed + """ + def __init__(self, haystack): + self.haystack = haystack + + def findBestMatch(self, needle, similarity): + """ Find the best match for ``needle`` that has a similarity better than or equal to ``similarity``. + + Returns a tuple of ``(position, confidence)`` if a match is found, or ``None`` otherwise. + + *Developer's Note - Despite the name, this method actually returns the **first** result + with enough similarity, not the **best** result.* + """ + method = cv2.TM_CCOEFF_NORMED + position = None + + match = cv2.matchTemplate(self.haystack, needle, method) + min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(match) + if method == cv2.TM_SQDIFF_NORMED or method == cv2.TM_SQDIFF: + confidence = min_val + if min_val <= 1-similarity: + # Confidence checks out + position = min_loc + else: + confidence = max_val + if max_val >= similarity: + # Confidence checks out + position = max_loc + + if not position: + return None + + return (position, confidence) + + def findAllMatches(self, needle, similarity): + """ Find all matches for ``needle`` with confidence better than or equal to ``similarity``. + + Returns an array of tuples ``(position, confidence)`` if match(es) is/are found, + or an empty array otherwise. + """ + positions = [] + method = cv2.TM_CCOEFF_NORMED + + match = cv2.matchTemplate(self.haystack, self.needle, method) + + indices = (-match).argpartition(100, axis=None)[:100] # Review the 100 top matches + unraveled_indices = numpy.array(numpy.unravel_index(indices, match.shape)).T + for location in unraveled_indices: + y, x = location + confidence = match[y][x] + if method == cv2.TM_SQDIFF_NORMED or method == cv2.TM_SQDIFF: + if confidence <= 1-similarity: + positions.append(((x, y), confidence)) + else: + if confidence >= similarity: + positions.append(((x, y), confidence)) + + positions.sort(key=lambda x: (x[0][1], x[0][0])) + return positions class PyramidTemplateMatcher(object): - def __init__(self, haystack): - self.haystack = cv2.cvtColor(haystack, cv2.COLOR_BGR2GRAY) # Convert to grayscale - self._iterations = 3 # Number of times to downsample - - def findBestMatch(self, needle, similarity): - """ Finds the best match using a search pyramid to improve efficiency - - Pyramid implementation unashamedly stolen from https://github.com/stb-tester/stb-tester - """ - needle = cv2.cvtColor(needle, cv2.COLOR_BGR2GRAY) # Convert to grayscale - - levels = 3 - needlePyr = self._build_pyramid(needle, levels) - # Needle will be smaller than haystack, so may not be able to create ``levels`` smaller versions - # of itself. If not, create only as many levels for haystack as we could for needle. - haystackPyr = self._build_pyramid(self.haystack, min(levels, len(needlePyr))) - roi_mask = None - method = cv2.TM_CCOEFF_NORMED - - - - # Run through each level in the pyramid, refining found ROIs - for level in range(len(haystackPyr)): - # Populate the heatmap with ones or zeroes depending on the appropriate method - lvl_haystack = haystackPyr[level] - lvl_needle = needlePyr[level] - matches_heatmap = ((numpy.ones if method == cv2.TM_SQDIFF_NORMED else numpy.zeros)((lvl_haystack.shape[0] - lvl_needle.shape[0] + 1,lvl_haystack.shape[1] - lvl_needle.shape[1] + 1),dtype=numpy.float32)) - - # Scale up region of interest for the next level in the pyramid - # (if it's been set and is a valid size) - if roi_mask is not None: - if any(x < 3 for x in roi_mask.shape): - roi_mask = None - else: - roi_mask = cv2.pyrUp(roi_mask) - - # If roi_mask is set, only search the best candidates in haystack - # for the needle: - - if roi_mask is None: - # Initialize mask to the whole image - rois = [(0, 0, matches_heatmap.shape[1], matches_heatmap.shape[0])] - else: - # Depending on version of OpenCV, findContours returns either a three-tuple - # or a two-tuple. Unsure why the install is different (possibly linked to - # OS version). - try: - im2, contours, hierarchy = cv2.findContours( roi_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) - except ValueError: - contours, hierarchy = cv2.findContours( roi_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) - # Expand contour rect by 1px on all sides with some tuple magic - rois = [tuple(sum(y) for y in zip(cv2.boundingRect(x), (-1,-1,2,2))) for x in contours] - - - for roi in rois: - # Add needle dimensions to roi - x, y, w, h = roi - roi = (x, y, w+lvl_needle.shape[1]-1, h+lvl_needle.shape[0]-1) - roi_slice = (slice(roi[1], roi[1]+roi[3]), slice(roi[0], roi[0]+roi[2])) # numpy 2D slice - r_slice = (slice(y, y+h), slice(x, x+w)) # numpy 2D slice - - # Search the region of interest for needle (and update heatmap) - matches_heatmap[r_slice] = cv2.matchTemplate(lvl_haystack[roi_slice],lvl_needle,method) - - min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(matches_heatmap) - # Reduce similarity to allow for scaling distortion (unless we are on the original image) - pyr_similarity = max(0, similarity - (0.2 if level < len(haystackPyr)-1 else 0)) - position = None - confidence = None - # Check for a match - if method == cv2.TM_SQDIFF_NORMED: - confidence = min_val - best_loc = min_loc - if min_val <= 1-pyr_similarity: - # Confidence checks out - position = min_loc - else: - confidence = max_val - best_loc = max_loc - if max_val >= pyr_similarity: - # Confidence checks out - position = max_loc - - if not position: - break - - # Find the best regions of interest - ret, roi_mask = cv2.threshold( - matches_heatmap, # Source image - ((1-pyr_similarity) if method == cv2.TM_SQDIFF_NORMED else pyr_similarity), # Confidence threshold - 255, # Max value - (cv2.THRESH_BINARY_INV if method == cv2.TM_SQDIFF_NORMED else cv2.THRESH_BINARY)) # Thresholding style - roi_mask = roi_mask.astype(numpy.uint8) - - # Whew! Let's see if there's a match after all that. - - if not position: - Debug.log(3, "Best match: {} at {}".format(max_val, max_loc)) - return None - - # There was a match! - return (position, confidence) - - def findAllMatches(self, needle, similarity): - """ Finds all matches above ``similarity`` using a search pyramid to improve efficiency - - Pyramid implementation unashamedly stolen from https://github.com/stb-tester/stb-tester - """ - needle = cv2.cvtColor(needle, cv2.COLOR_BGR2GRAY) # Convert to grayscale - - levels = 3 - needlePyr = self._build_pyramid(needle, levels) - # Needle will be smaller than haystack, so may not be able to create ``levels`` smaller versions - # of itself. If not, create only as many levels for haystack as we could for needle. - haystackPyr = self._build_pyramid(self.haystack, min(levels, len(needlePyr))) - roi_mask = None - method = cv2.TM_CCOEFF_NORMED - positions = [] - - # Run through each level in the pyramid, refining found ROIs - for level in range(len(haystackPyr)): - # Populate the heatmap with ones or zeroes depending on the appropriate method - lvl_haystack = haystackPyr[level] - lvl_needle = needlePyr[level] - matches_heatmap = ((numpy.ones if method == cv2.TM_SQDIFF_NORMED else numpy.zeros)((lvl_haystack.shape[0] - lvl_needle.shape[0] + 1,lvl_haystack.shape[1] - lvl_needle.shape[1] + 1),dtype=numpy.float32)) - - # Scale up region of interest for the next level in the pyramid - # (if it's been set and is a valid size) - if roi_mask is not None: - if any(x < 3 for x in roi_mask.shape): - roi_mask = None - else: - roi_mask = cv2.pyrUp(roi_mask) - - # If roi_mask is set, only search the best candidates in haystack - # for the needle: - - if roi_mask is None: - # Initialize mask to the whole image - rois = [(0, 0, matches_heatmap.shape[1], matches_heatmap.shape[0])] - else: - # Depending on version of OpenCV, findContours returns either a three-tuple - # or a two-tuple. Unsure why the install is different (possibly linked to - # OS version). - try: - im2, contours, hierarchy = cv2.findContours( roi_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) - except ValueError: - contours, hierarchy = cv2.findContours( roi_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) - # Expand contour rect by 1px on all sides with some tuple magic - rois = [tuple(sum(y) for y in zip(cv2.boundingRect(x), (-10,-10,20,20))) for x in contours] - - - for roi in rois: - # Add needle dimensions to roi - x, y, w, h = roi - # Contour coordinates may be negative, which will mess up the slices. - # Snap to zero if they are. - x = max(0,x) - y = max(0,y) - - roi = (x, y, w+lvl_needle.shape[1]-1, h+lvl_needle.shape[0]-1) - roi_slice = (slice(roi[1], roi[1]+roi[3]), slice(roi[0], roi[0]+roi[2])) # numpy 2D slice - - r_slice = (slice(y, y+h), slice(x, x+w)) # numpy 2D slice - - # Search the region of interest for needle (and update heatmap) - matches_heatmap[r_slice] = cv2.matchTemplate(lvl_haystack[roi_slice],lvl_needle,method) - - min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(matches_heatmap) - # Reduce similarity to allow for scaling distortion (unless we are on the original image) - Debug.log(3, "Best match: {} at {}".format(max_val, max_loc)) - pyr_similarity = max(0, similarity - (0.2 if level < len(haystackPyr)-1 else 0)) - positions = [] - confidence = None - - # Check for a match - indices = (-matches_heatmap).argpartition(100, axis=None)[:100] # Review the 100 top matches - unraveled_indices = numpy.array(numpy.unravel_index(indices, matches_heatmap.shape)).T - for location in unraveled_indices: - y, x = location - confidence = matches_heatmap[y][x] - if method == cv2.TM_SQDIFF_NORMED or method == cv2.TM_SQDIFF: - if confidence <= 1-pyr_similarity: - positions.append(((x,y),confidence)) - else: - if confidence >= pyr_similarity: - positions.append(((x,y),confidence)) - - if not len(positions): - break - - # Find the best regions of interest - ret, roi_mask = cv2.threshold( - matches_heatmap, # Source image - ((1-pyr_similarity) if method == cv2.TM_SQDIFF_NORMED else pyr_similarity), # Confidence threshold - 255, # Max value - (cv2.THRESH_BINARY_INV if method == cv2.TM_SQDIFF_NORMED else cv2.THRESH_BINARY)) # Thresholding style - roi_mask = roi_mask.astype(numpy.uint8) - - # Whew! Let's see if there's a match after all that. - positions.sort(key=lambda x: (x[0][1], x[0][0])) - print(len(positions)) - return positions - - def _build_pyramid(self, image, levels): - """ Returns a list of reduced-size images, from smallest to original size """ - pyramid = [image] - for l in range(levels-1): - if any(x < 20 for x in pyramid[-1].shape[:2]): - break - pyramid.append(cv2.pyrDown(pyramid[-1])) - return list(reversed(pyramid)) \ No newline at end of file + """ Python wrapper for OpenCV's TemplateMatcher + + Uses a pyramid model to optimize matching speed + """ + def __init__(self, haystack): + self.haystack = cv2.cvtColor(haystack, cv2.COLOR_BGR2GRAY) # Convert to grayscale + self._iterations = 3 # Number of times to downsample + + def findBestMatch(self, needle, similarity): + """ Finds the best match using a search pyramid to improve efficiency + + Pyramid implementation unashamedly stolen from https://github.com/stb-tester/stb-tester + + *Developer's Note - Despite the name, this method actually returns the **first** result + with enough similarity, not the **best** result.* + """ + needle = cv2.cvtColor(needle, cv2.COLOR_BGR2GRAY) # Convert to grayscale + + levels = 3 + needle_pyramid = self._build_pyramid(needle, levels) + # Needle will be smaller than haystack, so may not be able to create + # ``levels`` smaller versions of itself. If not, create only as many + # levels for ``haystack`` as we could for ``needle``. + haystack_pyramid = self._build_pyramid(self.haystack, min(levels, len(needle_pyramid))) + roi_mask = None + method = cv2.TM_CCOEFF_NORMED + + + + # Run through each level in the pyramid, refining found ROIs + for level in range(len(haystack_pyramid)): + # Populate the heatmap with ones or zeroes depending on the appropriate method + lvl_haystack = haystack_pyramid[level] + lvl_needle = needle_pyramid[level] + if (lvl_needle.shape[0] > lvl_haystack.shape[0]) or (lvl_needle.shape[1] > lvl_haystack.shape[1]): + raise ValueError("Image to find is larger than search area") + matches_heatmap = ( + (numpy.ones if method == cv2.TM_SQDIFF_NORMED else numpy.zeros)( + (lvl_haystack.shape[0] - lvl_needle.shape[0] + 1, lvl_haystack.shape[1] - lvl_needle.shape[1] + 1), + dtype=numpy.float32)) + + # Scale up region of interest for the next level in the pyramid + # (if it's been set and is a valid size) + if roi_mask is not None: + if any(x < 3 for x in roi_mask.shape): + roi_mask = None + else: + roi_mask = cv2.pyrUp(roi_mask) + + # If roi_mask is set, only search the best candidates in haystack + # for the needle: + + if roi_mask is None: + # Initialize mask to the whole image + rois = [(0, 0, matches_heatmap.shape[1], matches_heatmap.shape[0])] + else: + # Depending on version of OpenCV, findContours returns either a three-tuple + # or a two-tuple. Unsure why the install is different (possibly linked to + # OS version). + try: + _, contours, _ = cv2.findContours( + roi_mask, + cv2.RETR_EXTERNAL, + cv2.CHAIN_APPROX_NONE) + except ValueError: + contours, _ = cv2.findContours( + roi_mask, + cv2.RETR_EXTERNAL, + cv2.CHAIN_APPROX_NONE) + # Expand contour rect by 1px on all sides with some tuple magic + rois = [tuple(sum(y) for y in zip(cv2.boundingRect(x), (-1, -1, 2, 2))) for x in contours] + + + for roi in rois: + # Add needle dimensions to roi + x, y, w, h = roi + roi = (x, y, w+lvl_needle.shape[1]-1, h+lvl_needle.shape[0]-1) + # numpy 2D slice + roi_slice = (slice(roi[1], roi[1]+roi[3]), slice(roi[0], roi[0]+roi[2])) + # numpy 2D slice + r_slice = (slice(y, y+h), slice(x, x+w)) + + # Search the region of interest for needle (and update heatmap) + matches_heatmap[r_slice] = cv2.matchTemplate( + lvl_haystack[roi_slice], + lvl_needle, + method) + + min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(matches_heatmap) + # Reduce similarity to allow for scaling distortion + # (unless we are on the original image) + pyr_similarity = max(0, similarity - (0.2 if level < len(haystack_pyramid)-1 else 0)) + position = None + confidence = None + # Check for a match + if method == cv2.TM_SQDIFF_NORMED: + confidence = min_val + if min_val <= 1-pyr_similarity: + # Confidence checks out + position = min_loc + else: + confidence = max_val + if max_val >= pyr_similarity: + # Confidence checks out + position = max_loc + + if not position: + break + + # Find the best regions of interest + _, roi_mask = cv2.threshold( + # Source image + matches_heatmap, + # Confidence threshold + ((1-pyr_similarity) if method == cv2.TM_SQDIFF_NORMED else pyr_similarity), + # Max value + 255, + # Thresholding style + (cv2.THRESH_BINARY_INV if method == cv2.TM_SQDIFF_NORMED else cv2.THRESH_BINARY)) + roi_mask = roi_mask.astype(numpy.uint8) + + # Whew! Let's see if there's a match after all that. + + if not position: + Debug.log(3, "Best match: {} at {}".format(max_val, max_loc)) + return None + + # There was a match! + return (position, confidence) + + def findAllMatches(self, needle, similarity): + """ Finds all matches above ``similarity`` using a search pyramid to improve efficiency + + Pyramid implementation unashamedly stolen from https://github.com/stb-tester/stb-tester + """ + needle = cv2.cvtColor(needle, cv2.COLOR_BGR2GRAY) # Convert to grayscale + + levels = 3 + needle_pyramid = self._build_pyramid(needle, levels) + # Needle will be smaller than haystack, so may not be able to create ``levels`` smaller + # versions of itself. If not, create only as many levels for haystack as we could for + # needle. + haystack_pyramid = self._build_pyramid(self.haystack, min(levels, len(needle_pyramid))) + roi_mask = None + method = cv2.TM_CCOEFF_NORMED + positions = [] + + # Run through each level in the pyramid, refining found ROIs + for level in range(len(haystack_pyramid)): + # Populate the heatmap with ones or zeroes depending on the appropriate method + lvl_haystack = haystack_pyramid[level] + lvl_needle = needle_pyramid[level] + matches_heatmap = ( + (numpy.ones if method == cv2.TM_SQDIFF_NORMED else numpy.zeros)( + (lvl_haystack.shape[0] - lvl_needle.shape[0] + 1, lvl_haystack.shape[1] - lvl_needle.shape[1] + 1), + dtype=numpy.float32)) + + # Scale up region of interest for the next level in the pyramid + # (if it's been set and is a valid size) + if roi_mask is not None: + if any(x < 3 for x in roi_mask.shape): + roi_mask = None + else: + roi_mask = cv2.pyrUp(roi_mask) + + # If roi_mask is set, only search the best candidates in haystack + # for the needle: + + if roi_mask is None: + # Initialize mask to the whole image + rois = [(0, 0, matches_heatmap.shape[1], matches_heatmap.shape[0])] + else: + # Depending on version of OpenCV, findContours returns either a three-tuple + # or a two-tuple. Unsure why the install is different (possibly linked to + # OS version). + try: + _, contours, _ = cv2.findContours( + roi_mask, + cv2.RETR_EXTERNAL, + cv2.CHAIN_APPROX_NONE) + except ValueError: + contours, _ = cv2.findContours( + roi_mask, + cv2.RETR_EXTERNAL, + cv2.CHAIN_APPROX_NONE) + # Expand contour rect by 1px on all sides with some tuple magic + rois = [tuple(sum(y) for y in zip(cv2.boundingRect(x), (-10, -10, 20, 20))) for x in contours] + + + for roi in rois: + # Add needle dimensions to roi + x, y, w, h = roi + # Contour coordinates may be negative, which will mess up the slices. + # Snap to zero if they are. + x = max(0, x) + y = max(0, y) + + roi = (x, y, w+lvl_needle.shape[1]-1, h+lvl_needle.shape[0]-1) + # numpy 2D slice + roi_slice = (slice(roi[1], roi[1]+roi[3]), slice(roi[0], roi[0]+roi[2])) + + r_slice = (slice(y, y+h), slice(x, x+w)) # numpy 2D slice + + # Search the region of interest for needle (and update heatmap) + matches_heatmap[r_slice] = cv2.matchTemplate( + lvl_haystack[roi_slice], + lvl_needle, + method) + min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(matches_heatmap) + # Reduce similarity to allow for scaling distortion + # (unless we are on the original image) + Debug.log(3, "Best match: {} at {}".format(max_val, max_loc)) + pyr_similarity = max(0, similarity - (0.2 if level < len(haystack_pyramid)-1 else 0)) + positions = [] + confidence = None + + # Check for a match + # Review the 100 top matches + indices = (-matches_heatmap).argpartition(100, axis=None)[:100] + unraveled_indices = numpy.array(numpy.unravel_index(indices, matches_heatmap.shape)).T + for location in unraveled_indices: + y, x = location + confidence = matches_heatmap[y][x] + if method == cv2.TM_SQDIFF_NORMED or method == cv2.TM_SQDIFF: + if confidence <= 1-pyr_similarity: + positions.append(((x, y), confidence)) + else: + if confidence >= pyr_similarity: + positions.append(((x, y), confidence)) + + if not len(positions): + break + + # Find the best regions of interest + _, roi_mask = cv2.threshold( + # Source image + matches_heatmap, + # Confidence threshold + ((1-pyr_similarity) if method == cv2.TM_SQDIFF_NORMED else pyr_similarity), + # Max value + 255, + # Thresholding style + (cv2.THRESH_BINARY_INV if method == cv2.TM_SQDIFF_NORMED else cv2.THRESH_BINARY)) + roi_mask = roi_mask.astype(numpy.uint8) + + # Whew! Let's see if there's a match after all that. + positions.sort(key=lambda x: (x[0][1], x[0][0])) + print(len(positions)) + return positions + + def _build_pyramid(self, image, levels): + """ Returns a list of reduced-size images, from smallest to original size """ + pyramid = [image] + for l in range(levels-1): + if any(x < 20 for x in pyramid[-1].shape[:2]): + break + pyramid.append(cv2.pyrDown(pyramid[-1])) + return list(reversed(pyramid)) diff --git a/lackey/__init__.py b/lackey/__init__.py index b07567b..8a827b5 100644 --- a/lackey/__init__.py +++ b/lackey/__init__.py @@ -1,11 +1,17 @@ +""" Sikuli script implementation for Python + +See https://github.com/glitchassassin/lackey +""" + from zipfile import ZipFile import Tkinter as tk import tkFileDialog import tkMessageBox import platform -import requests +import sys import time import os +import requests ## Lackey sub-files @@ -14,7 +20,8 @@ from .RegionMatching import Pattern, Region, Match, Screen, Location, Mouse, Keyboard, App from .Exceptions import FindFailed from .Settings import Debug, Settings, DebugMaster, SettingsMaster -import SikuliGui +from . import SikuliGui +from ._version import __version__ VALID_PLATFORMS = ["Windows"] @@ -25,76 +32,83 @@ _type = type _input = input +_exit = exit #_zip = zip ## Sikuli Convenience Functions def sleep(seconds): - """ Convenience function. Pauses script for `seconds`. """ - time.sleep(seconds) + """ Convenience function. Pauses script for `seconds`. """ + time.sleep(seconds) def exit(value): - """ Convenience function. Exits with code `value`. """ - sys.exit(value) + """ Convenience function. Exits with code `value`. """ + sys.exit(value) def setShowActions(value): - """ Convenience function. Sets "show actions" setting (True or False) """ - Settings.ShowActions = bool(value) + """ Convenience function. Sets "show actions" setting (True or False) """ + Settings.ShowActions = bool(value) def getBundlePath(): - """ Convenience function. Returns the path of the \*.sikuli bundle. """ - return Settings.BundlePath + """ Convenience function. Returns the path of the \\*.sikuli bundle. """ + return Settings.BundlePath def getBundleFolder(): - """ Convenience function. Same as `getBundlePath()` plus the OS default path separator. """ - return getBundlePath() + os.path.sep + """ Convenience function. Same as `getBundlePath()` plus the OS default path separator. """ + return getBundlePath() + os.path.sep def setBundlePath(path): - """ Convenience function. Changes the path of the \*.sikuli bundle. """ - if os.path.exists(path): - Settings.BundlePath = path - else: - raise FileNotFoundError(path) + """ Convenience function. Changes the path of the \\*.sikuli bundle. """ + if os.path.exists(path): + Settings.BundlePath = path + else: + raise OSError("File not found: " + path) def getImagePath(): - """ Convenience function. Returns a list of paths to search for images. """ - return [getBundlePath()] + Settings.ImagePaths + """ Convenience function. Returns a list of paths to search for images. """ + return [getBundlePath()] + Settings.ImagePaths def addImagePath(new_path): - """ Convenience function. Adds a path to the list of paths to search for images. Can be a URL (but must be accessible). """ - if os.path.exists(new_path): - Settings.ImagePaths.append(new_path) - elif "http://" in new_path or "https://" in new_path: - request = requests.get(new_path) - if request.status_code < 400: - # Path exists - Settings.ImagePaths.append(new_path) - else: - raise FileNotFoundError("Unable to connect to", new_path) - else: - raise FileNotFoundError(new_path) + """ Convenience function. Adds a path to the list of paths to search for images. + + Can be a URL (but must be accessible). """ + if os.path.exists(new_path): + Settings.ImagePaths.append(new_path) + elif "http://" in new_path or "https://" in new_path: + request = requests.get(new_path) + if request.status_code < 400: + # Path exists + Settings.ImagePaths.append(new_path) + else: + raise OSError("Unable to connect to " + new_path) + else: + raise OSError("File not found: " + new_path) def addHTTPImagePath(new_path): - """ Convenience function. Same as `addImagePath()`. """ - addImagePath(new_path) + """ Convenience function. Same as `addImagePath()`. """ + addImagePath(new_path) def getParentPath(): - """ Convenience function. Returns the parent folder of the \*.sikuli bundle. """ - return os.path.dirname(Settings.BundlePath) + """ Convenience function. Returns the parent folder of the \\*.sikuli bundle. """ + return os.path.dirname(Settings.BundlePath) def getParentFolder(): - """ Convenience function. Same as `getParentPath()` plus the OS default path separator. """ - return getParentPath() + os.path.sep + """ Convenience function. Same as `getParentPath()` plus the OS default path separator. """ + return getParentPath() + os.path.sep def makePath(*args): - """ Convenience function. Returns a path from a series of path components. Same as `os.path.join`. """ - return os.path.join(*args) + """ Convenience function. Returns a path from a series of path components. + + Same as `os.path.join`. """ + return os.path.join(*args) def makeFolder(*args): - """ Convenience function. Same as `makePath()` plus the OS default path separator. """ - return makePath(*args) + os.path.sep + """ Convenience function. Same as `makePath()` plus the OS default path separator. """ + return makePath(*args) + os.path.sep ## Sikuli implements the unzip() file, below. Included here to avoid breaking old ## scripts. ``zipfile()`` is coded here, but not included in Sikuli, so I've ## commented it out for the time being. Note that ``zip`` is a reserved keyword ## in Python. -def unzip(fromFile, toFolder): - """ Convenience function. Extracts files from the zip file `fromFile` into the folder `toFolder`. """ - with ZipFile(os.path.abspath(fromFile), 'r') as to_unzip: - to_unzip.extractall(os.path.abspath(toFolder)) +def unzip(from_file, to_folder): + """ Convenience function. + + Extracts files from the zip file `fromFile` into the folder `toFolder`. """ + with ZipFile(os.path.abspath(from_file), 'r') as to_unzip: + to_unzip.extractall(os.path.abspath(to_folder)) #def zipfile(fromFolder, toFile): # with ZipFile(toFile, 'w') as to_zip: # for root, dirs, files in os.walk(fromFolder): @@ -105,81 +119,91 @@ def unzip(fromFile, toFolder): ## Popup/input dialogs def popat(*args): - """ Convenience function. Sets the popup location (currently not used). """ - if len(args) == 2 and isinstance(args[0], int) and isinstance(args[1], int): - # popat(x,y) - Settings.PopupLocation = Location(args[0],args[1]) - elif len(args) == 1 and isinstance(args[0], Location): - # popat(location) - Settings.PopupLocation = args[0] - elif len(args) == 1 and isinstance(args[0], Region): - Settings.PopupLocation = args[0].getCenter() - elif len(args) == 0: - Settings.PopupLocation = SCREEN.getCenter() - else: - raise TypeError("Unrecognized parameter(s) for popat") + """ Convenience function. Sets the popup location (currently not used). """ + if len(args) == 2 and isinstance(args[0], int) and isinstance(args[1], int): + # popat(x,y) + Settings.PopupLocation = Location(args[0], args[1]) + elif len(args) == 1 and isinstance(args[0], Location): + # popat(location) + Settings.PopupLocation = args[0] + elif len(args) == 1 and isinstance(args[0], Region): + Settings.PopupLocation = args[0].getCenter() + elif len(args) == 0: + Settings.PopupLocation = SCREEN.getCenter() + else: + raise TypeError("Unrecognized parameter(s) for popat") def popup(text, title="Lackey Info"): - """ Creates an info dialog with the specified text. """ - root = tk.Tk() - root.withdraw() - tkMessageBox.showinfo(title, text) + """ Creates an info dialog with the specified text. """ + root = tk.Tk() + root.withdraw() + tkMessageBox.showinfo(title, text) def popError(text, title="Lackey Error"): - """ Creates an error dialog with the specified text. """ - root = tk.Tk() - root.withdraw() - tkMessageBox.showerror(title, text) + """ Creates an error dialog with the specified text. """ + root = tk.Tk() + root.withdraw() + tkMessageBox.showerror(title, text) def popAsk(text, title="Lackey Decision"): - """ Creates a yes-no dialog with the specified text. """ - root = tk.Tk() - root.withdraw() - return tkMessageBox.askyesno(title, text) + """ Creates a yes-no dialog with the specified text. """ + root = tk.Tk() + root.withdraw() + return tkMessageBox.askyesno(title, text) # Be aware this overwrites the Python input() command-line function. def input(msg="", default="", title="Lackey Input", hidden=False): - """ Creates an input dialog with the specified message and default text. If `hidden`, creates a password dialog instead. Returns the entered value. """ - root = tk.Tk() - input_text = tk.StringVar() - input_text.set(default) - dialog = SikuliGui.PopupInput(root, msg, default, title, hidden, input_text) - root.focus_force() - root.mainloop() - return str(input_text.get()) + """ Creates an input dialog with the specified message and default text. + + If `hidden`, creates a password dialog instead. Returns the entered value. """ + root = tk.Tk() + input_text = tk.StringVar() + input_text.set(default) + SikuliGui.PopupInput(root, msg, title, hidden, input_text) + root.focus_force() + root.mainloop() + return str(input_text.get()) def inputText(message="", title="Lackey Input", lines=9, width=20, text=""): - """ Creates a textarea dialog with the specified message and default text. Returns the entered value. """ - root = tk.Tk() - input_text = tk.StringVar() - input_text.set(text) - dialog = SikuliGui.PopupTextarea(root, message, title, lines, width, text, input_text) - root.focus_force() - root.mainloop() - return str(input_text.get()) -def select(message="", title="Lackey Input", options=[], default=None): - """ Creates a dropdown selection dialog with the specified message and options. `default` must be one of the options. Returns the selected value. """ - if len(options) == 0: - return "" - if default is None: - default = options[0] - if default not in options: - raise ValueError("<> not in options[]") - root = tk.Tk() - input_text = tk.StringVar() - input_text.set(text) - dialog = SikuliGui.PopupList(root, message, title, options, default, input_text) - root.focus_force() - root.mainloop() - return str(input_text.get()) + """ Creates a textarea dialog with the specified message and default text. + + Returns the entered value. """ + root = tk.Tk() + input_text = tk.StringVar() + input_text.set(text) + SikuliGui.PopupTextarea(root, message, title, lines, width, input_text) + root.focus_force() + root.mainloop() + return str(input_text.get()) +def select(message="", title="Lackey Input", options=None, default=None): + """ Creates a dropdown selection dialog with the specified message and options + + `default` must be one of the options. + + Returns the selected value. """ + if options is None or len(options) == 0: + return "" + if default is None: + default = options[0] + if default not in options: + raise ValueError("<> not in options[]") + root = tk.Tk() + input_text = tk.StringVar() + input_text.set(message) + SikuliGui.PopupList(root, message, title, options, default, input_text) + root.focus_force() + root.mainloop() + return str(input_text.get()) def popFile(title="Lackey Open File"): - """ Creates a file selection dialog with the specified message and options. Returns the selected file. """ - root = tk.Tk() - root.withdraw() - return str(tkFileDialog.askopenfilename()) + """ Creates a file selection dialog with the specified message and options. + + Returns the selected file. """ + root = tk.Tk() + root.withdraw() + return str(tkFileDialog.askopenfilename(title=title)) # If this is a valid platform, set up initial Screen object. Otherwise, might be ReadTheDocs if platform.system() in VALID_PLATFORMS: - SCREEN = Screen(0) - for prop in dir(SCREEN): - if callable(getattr(SCREEN, prop, None)) and prop[0] != "_": - # Property is a method, and is not private. Dump it into the global namespace. - globals()[prop] = getattr(SCREEN, prop, None) - \ No newline at end of file + SCREEN = Screen(0) + for prop in dir(SCREEN): + if callable(getattr(SCREEN, prop, None)) and prop[0] != "_": + # Property is a method, and is not private. Dump it into the global namespace. + globals()[prop] = getattr(SCREEN, prop, None) + \ No newline at end of file diff --git a/lackey/_version.py b/lackey/_version.py new file mode 100644 index 0000000..1b1ad04 --- /dev/null +++ b/lackey/_version.py @@ -0,0 +1,5 @@ +""" Version information for Lackey module + +""" + +__version__ = "0.5.0" diff --git a/requirements.txt b/requirements.txt index c1c9d44..8c8c01a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,9 @@ ####### requirements.txt ####### # ###### Requirements without Version Specifiers ###### +requests numpy pillow opencv-python -wheel \ No newline at end of file +wheel +keyboard \ No newline at end of file diff --git a/setup.py b/setup.py index c475a25..8347bad 100644 --- a/setup.py +++ b/setup.py @@ -1,35 +1,45 @@ +""" Setup script for Lackey + +""" + from setuptools import setup, find_packages from setuptools.dist import Distribution +from lackey import __version__ class BinaryDistribution(Distribution): + """ Custom class for platform-specific modules + + """ def is_pure(self): return False setup( - name="Lackey", - version="0.4.2a1", - description="A Sikuli script implementation in Python", - long_description="Lackey is an implementation of Sikuli script, using image recognition to control complex and non-OS-standard business applications. Potential applications include automating tedious workflows, routine user interface testing, etc.", - url="https://github.com/glitchassassin/lackey", - author="Jon Winsley", - author_email="jon.winsley@gmail.com", - license="MIT", - classifiers=[ - "Development Status :: 3 - Alpha", - "Environment :: Console", - "Intended Audience :: Developers", - "Intended Audience :: Information Technology", - "Intended Audience :: System Administrators", - "License :: OSI Approved :: MIT License", - "Operating System :: Microsoft :: Windows", - "Operating System :: Microsoft :: Windows", - "Topic :: Software Development :: Testing", - "Topic :: Utilities", - "Topic :: Desktop Environment" - ], - keywords="automation testing sikuli", - packages=find_packages(exclude=['docs', 'tests']), - install_requires=['requests', 'pillow', 'numpy', 'opencv-python'], - include_package_data=True, - distclass=BinaryDistribution -) \ No newline at end of file + name="Lackey", + description="A Sikuli script implementation in Python", + long_description="""Lackey is an implementation of Sikuli script, using image recognition + to control complex and non-OS-standard business applications. Potential applications include + automating tedious workflows, routine user interface testing, etc.""", + url="https://github.com/glitchassassin/lackey", + author="Jon Winsley", + author_email="jon.winsley@gmail.com", + license="MIT", + version=__version__, + classifiers=[ + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: MIT License", + "Operating System :: Microsoft :: Windows", + "Operating System :: Microsoft :: Windows", + "Topic :: Software Development :: Testing", + "Topic :: Utilities", + "Topic :: Desktop Environment" + ], + keywords="automation testing sikuli", + packages=find_packages(exclude=['docs', 'tests']), + install_requires=['requests', 'pillow', 'numpy', 'opencv-python', 'keyboard'], + include_package_data=True, + distclass=BinaryDistribution +) diff --git a/tests/test_cases.py b/tests/test_cases.py index 35787e7..f966468 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -4,7 +4,7 @@ import time import sys import os -sys.path.insert(0, os.path.abspath('..')) +#sys.path.insert(0, os.path.abspath('..')) import lackey class TestLocationMethods(unittest.TestCase): @@ -87,7 +87,7 @@ def test_keys(self): self.kb.keyDown("{SHIFT}") self.kb.keyUp("{CTRL}") self.kb.keyUp("{SHIFT}") - self.kb.type("%{CTRL}") + self.kb.type("{CTRL}", lackey.Key.ALT) # Really this should check to make sure these keys have all been released, but # I'm not sure how to make that work without continuously monitoring the keyboard # (which is the usual scenario). Ah well... if your computer is acting weird after @@ -278,30 +278,41 @@ def assertHasMethod(self, cls, mthd, args=0): self.assertEqual(len(inspect.getargspec(getattr(cls, mthd))[0]), args) class TestAppMethods(unittest.TestCase): - def setUp(self): + def test_getters(self): if sys.platform.startswith("win"): - self.app = lackey.App("notepad.exe") - self.app.setUsing("test_cases.py") - self.app.open() + app = lackey.App("notepad.exe test_cases.py") + #app.setUsing("test_cases.py") + app.open() time.sleep(1) else: raise NotImplementedError("Platforms supported include: Windows") - self.app.focus() - def tearDown(self): - if sys.platform.startswith("win"): - self.app.close() - time.sleep(1) + app.focus() - def test_getters(self): - self.assertEqual(self.app.getName(), "notepad.exe") - self.assertTrue(self.app.isRunning()) - self.assertEqual(self.app.getWindow(), "test_cases.py - Notepad") - self.assertNotEqual(self.app.getPID(), -1) - region = self.app.window() + self.assertEqual(app.getName(), "notepad.exe") + self.assertTrue(app.isRunning()) + self.assertEqual(app.getWindow(), "test_cases.py - Notepad") + self.assertNotEqual(app.getPID(), -1) + region = app.window() self.assertIsInstance(region, lackey.Region) self.assertGreater(region.getW(), 0) self.assertGreater(region.getH(), 0) + if sys.platform.startswith("win"): + app.close() + time.sleep(1) + + def test_launchers(self): + app = lackey.App("notepad.exe") + app.setUsing("test_cases.py") + app.open() + time.sleep(1) + self.assertEqual(app.getName(), "notepad.exe") + self.assertTrue(app.isRunning()) + self.assertEqual(app.getWindow(), "test_cases.py - Notepad") + self.assertNotEqual(app.getPID(), -1) + app.close() + time.sleep(1) + class TestScreenMethods(unittest.TestCase): def setUp(self): self.primaryScreen = lackey.Screen(0) @@ -328,15 +339,15 @@ def testTypeCopyPaste(self): raise NotImplementedError("Platforms supported include: Windows") r = app.window() - r.type("This is a +test") # Type should translate "+" into shift modifier for capital first letters - r.type("^a") # Select all - r.type("^c") # Copy + r.type("This is a Test") # Type should translate "+" into shift modifier for capital first letters + r.type("a", lackey.Key.CONTROL) # Select all + r.type("c", lackey.Key.CONTROL) # Copy self.assertEqual(r.getClipboard(), "This is a Test") r.type("{DELETE}") # Clear the selected text - r.paste("This, on the other hand, is a +broken +record.") # Paste should ignore special characters and insert the string as is - r.type("^a") # Select all - r.type("^c") # Copy - self.assertEqual(r.getClipboard(), "This, on the other hand, is a +broken +record.") + r.paste("This, on the other hand, is a {SHIFT}broken {SHIFT}record.") # Paste should ignore special characters and insert the string as is + r.type("a", lackey.Key.CONTROL) # Select all + r.type("c", lackey.Key.CONTROL) # Copy + self.assertEqual(r.getClipboard(), "This, on the other hand, is a {SHIFT}broken {SHIFT}record.") if sys.platform.startswith("win"): app.close() @@ -352,10 +363,10 @@ def testOpenApp(self): r.type("This is a test") r.rightClick(lackey.Pattern("test_text.png").similar(0.6)) r.click("select_all.png") - r.type("^c") # Copy + r.type("c", lackey.Key.CONTROL) # Copy self.assertEqual(r.getClipboard(), "This is a test") r.type("{DELETE}") - r.type("%{F4}") + r.type("{F4}", lackey.Key.ALT) def testDragDrop(self): """ This relies on two specific icons on the desktop. @@ -365,7 +376,7 @@ def testDragDrop(self): r = lackey.Screen(0) r.dragDrop("test_file_txt.png", "notepad.png") self.assertTrue(r.exists("test_file_text.png")) - r.type("%{F4}") + r.type("{F4}", lackey.Key.ALT) class TestRegionFeatures(unittest.TestCase): def setUp(self):