diff --git a/pupil_src/launchables/eye.py b/pupil_src/launchables/eye.py index 69903c274a..3b09caf8db 100644 --- a/pupil_src/launchables/eye.py +++ b/pupil_src/launchables/eye.py @@ -126,6 +126,7 @@ def eye( from pyglui import ui, graph, cygl from pyglui.cygl.utils import draw_points, RGBA, draw_polyline from pyglui.cygl.utils import Named_Texture + import gl_utils from gl_utils import basic_gl_setup, adjust_gl_view, clear_gl_screen from gl_utils import make_coord_system_pixel_based from gl_utils import make_coord_system_norm_based @@ -175,7 +176,7 @@ def interrupt_handler(sig, frame): icon_bar_width = 50 window_size = None - hdpi_factor = 1.0 + content_scale = 1.0 # g_pool holds variables for this process g_pool = SimpleNamespace() @@ -213,7 +214,7 @@ def get_timestamp(): default_capture_name = "UVC_Source" default_capture_settings = { "preferred_names": preferred_names, - "frame_size": (320, 240), + "frame_size": (192, 192), "frame_rate": 120, } @@ -231,25 +232,95 @@ def get_timestamp(): ("Roi", {}), ] + def consume_events_and_render_buffer(): + glfw.glfwMakeContextCurrent(main_window) + clear_gl_screen() + + glViewport(0, 0, *g_pool.camera_render_size) + for p in g_pool.plugins: + p.gl_display() + + glViewport(0, 0, *window_size) + # render graphs + fps_graph.draw() + cpu_graph.draw() + + # render GUI + try: + clipboard = glfw.glfwGetClipboardString(main_window).decode() + except AttributeError: # clipboard is None, might happen on startup + clipboard = "" + g_pool.gui.update_clipboard(clipboard) + user_input = g_pool.gui.update() + if user_input.clipboard != clipboard: + # only write to clipboard if content changed + glfw.glfwSetClipboardString(main_window, user_input.clipboard.encode()) + + for button, action, mods in user_input.buttons: + x, y = glfw.glfwGetCursorPos(main_window) + pos = glfw.window_coordinate_to_framebuffer_coordinate( + main_window, x, y, cached_scale=None + ) + pos = normalize(pos, g_pool.camera_render_size) + if g_pool.flip: + pos = 1 - pos[0], 1 - pos[1] + # Position in img pixels + pos = denormalize(pos, g_pool.capture.frame_size) + + for plugin in g_pool.plugins: + if plugin.on_click(pos, button, action): + break + + for key, scancode, action, mods in user_input.keys: + for plugin in g_pool.plugins: + if plugin.on_key(key, scancode, action, mods): + break + + for char_ in user_input.chars: + for plugin in g_pool.plugins: + if plugin.on_char(char_): + break + + # update screen + glfw.glfwSwapBuffers(main_window) + # Callback functions def on_resize(window, w, h): nonlocal window_size - nonlocal hdpi_factor + nonlocal content_scale + + # Always clear buffers on resize to make sure that there are no overlapping + # artifacts from previous frames. + gl_utils.glClear(gl_utils.GL_COLOR_BUFFER_BIT) + gl_utils.glClearColor(0, 0, 0, 1) active_window = glfw.glfwGetCurrentContext() glfw.glfwMakeContextCurrent(window) - hdpi_factor = glfw.getHDPIFactor(window) - g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor + content_scale = glfw.get_content_scale(window) + framebuffer_scale = glfw.get_framebuffer_scale(window) + g_pool.gui.scale = content_scale window_size = w, h g_pool.camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() for g in g_pool.graphs: - g.scale = hdpi_factor + g.scale = content_scale g.adjust_window_size(w, h) adjust_gl_view(w, h) glfw.glfwMakeContextCurrent(active_window) + # Minimum window size required, otherwise parts of the UI can cause openGL + # issues with permanent effects. Depends on the content scale, which can + # potentially be dynamically modified, so we re-adjust the size limits every + # time here. + min_size = int(2 * icon_bar_width * g_pool.gui.scale / framebuffer_scale) + glfw.glfwSetWindowSizeLimits( + window, min_size, min_size, glfw.GLFW_DONT_CARE, glfw.GLFW_DONT_CARE + ) + + # Needed, to update the window buffer while resizing + consume_events_and_render_buffer() + def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) @@ -263,7 +334,9 @@ def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): - x, y = x * hdpi_factor, y * hdpi_factor + x, y = glfw.window_coordinate_to_framebuffer_coordinate( + window, x, y, cached_scale=None + ) g_pool.gui.update_mouse(x, y) pos = x, y @@ -321,11 +394,17 @@ def toggle_general_settings(collapsed): # Initialize glfw glfw.glfwInit() + glfw.glfwWindowHint(glfw.GLFW_SCALE_TO_MONITOR, glfw.GLFW_TRUE) if hide_ui: glfw.glfwWindowHint(glfw.GLFW_VISIBLE, 0) # hide window title = "Pupil Capture - eye {}".format(eye_id) - width, height = session_settings.get("window_size", (640 + icon_bar_width, 480)) + # Pupil Cam1 uses 4:3 resolutions. Pupil Cam2 and Cam3 use 1:1 resolutions. + # As all Pupil Core and VR/AR add-ons are shipped with Pupil Cam2 and Cam3 + # cameras, we adjust the default eye window size to a 1:1 content aspect ratio. + # The size of 500 was chosen s.t. the menu still fits. + default_window_size = 500 + icon_bar_width, 500 + width, height = session_settings.get("window_size", default_window_size) main_window = glfw.glfwCreateWindow(width, height, title, None, None) window_pos = session_settings.get("window_position", window_position_default) @@ -333,11 +412,6 @@ def toggle_general_settings(collapsed): glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() - # UI callback functions - def set_scale(new_scale): - g_pool.gui_user_scale = new_scale - on_resize(main_window, *glfw.glfwGetFramebufferSize(main_window)) - # gl_state settings basic_gl_setup() g_pool.image_tex = Named_Texture() @@ -345,7 +419,6 @@ def set_scale(new_scale): # setup GUI g_pool.gui = ui.UI() - g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0) g_pool.menubar = ui.Scrolling_Menu( "Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left" ) @@ -356,22 +429,30 @@ def set_scale(new_scale): g_pool.gui.append(g_pool.iconbar) general_settings = ui.Growing_Menu("General", header_pos="headline") - general_settings.append( - ui.Selector( - "gui_user_scale", - g_pool, - setter=set_scale, - selection=[0.8, 0.9, 1.0, 1.1, 1.2], - label="Interface Size", - ) - ) def set_window_size(): + # Get current capture frame size f_width, f_height = g_pool.capture.frame_size - f_width *= 2 - f_height *= 2 - f_width += int(icon_bar_width * g_pool.gui.scale) - glfw.glfwSetWindowSize(main_window, f_width, f_height) + # Eye camera resolutions are too small to be used as default window sizes. + # We use double their size instead. + frame_scale_factor = 2 + f_width *= frame_scale_factor + f_height *= frame_scale_factor + + # Get current display scale factor + content_scale = glfw.get_content_scale(main_window) + framebuffer_scale = glfw.get_framebuffer_scale(main_window) + display_scale_factor = content_scale / framebuffer_scale + + # Scale the capture frame size by display scale factor + f_width *= display_scale_factor + f_height *= display_scale_factor + + # Increas the width to account for the added scaled icon bar width + f_width += icon_bar_width * display_scale_factor + + # Set the newly calculated size (scaled capture frame size + scaled icon bar width) + glfw.glfwSetWindowSize(main_window, int(f_width), int(f_height)) general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append(ui.Switch("flip", g_pool, label="Flip image display")) @@ -403,7 +484,6 @@ def set_window_size(): ) icon.tooltip = "General Settings" g_pool.iconbar.append(icon) - toggle_general_settings(False) plugins_to_load = session_settings.get("loaded_plugins", default_plugins) if overwrite_cap_settings: @@ -420,6 +500,8 @@ def set_window_size(): g_pool.plugin_by_name[default_capture_name], default_capture_settings ) + toggle_general_settings(True) + g_pool.writer = None # Register callbacks main_window @@ -434,6 +516,11 @@ def set_window_size(): # load last gui configuration g_pool.gui.configuration = session_settings.get("ui_config", {}) + # If previously selected plugin was not loaded this time, we will have an + # expanded menubar without any menu selected. We need to ensure the menubar is + # collapsed in this case. + if all(submenu.collapsed for submenu in g_pool.menubar.elements): + g_pool.menubar.collapsed = True # set up performance graphs pid = os.getpid() @@ -603,56 +690,7 @@ def window_should_update(): # GL drawing if window_should_update(): if is_window_visible(main_window): - glfw.glfwMakeContextCurrent(main_window) - clear_gl_screen() - - glViewport(0, 0, *g_pool.camera_render_size) - for p in g_pool.plugins: - p.gl_display() - - glViewport(0, 0, *window_size) - # render graphs - fps_graph.draw() - cpu_graph.draw() - - # render GUI - try: - clipboard = glfw.glfwGetClipboardString(main_window).decode() - except AttributeError: # clipboard is None, might happen on startup - clipboard = "" - g_pool.gui.update_clipboard(clipboard) - user_input = g_pool.gui.update() - if user_input.clipboard != clipboard: - # only write to clipboard if content changed - glfw.glfwSetClipboardString( - main_window, user_input.clipboard.encode() - ) - - for button, action, mods in user_input.buttons: - x, y = glfw.glfwGetCursorPos(main_window) - pos = x * hdpi_factor, y * hdpi_factor - pos = normalize(pos, g_pool.camera_render_size) - if g_pool.flip: - pos = 1 - pos[0], 1 - pos[1] - # Position in img pixels - pos = denormalize(pos, g_pool.capture.frame_size) - - for plugin in g_pool.plugins: - if plugin.on_click(pos, button, action): - break - - for key, scancode, action, mods in user_input.keys: - for plugin in g_pool.plugins: - if plugin.on_key(key, scancode, action, mods): - break - - for char_ in user_input.chars: - for plugin in g_pool.plugins: - if plugin.on_char(char_): - break - - # update screen - glfw.glfwSwapBuffers(main_window) + consume_events_and_render_buffer() glfw.glfwPollEvents() # END while running @@ -665,7 +703,6 @@ def window_should_update(): session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() # save session persistent settings - session_settings["gui_scale"] = g_pool.gui_user_scale session_settings["flip"] = g_pool.flip session_settings["display_mode"] = g_pool.display_mode session_settings["ui_config"] = g_pool.gui.configuration @@ -676,7 +713,15 @@ def window_should_update(): session_settings["window_position"] = glfw.glfwGetWindowPos(main_window) session_window_size = glfw.glfwGetWindowSize(main_window) if 0 not in session_window_size: - session_settings["window_size"] = session_window_size + f_width, f_height = session_window_size + if platform.system() in ("Windows", "Linux"): + # Store unscaled window size as the operating system will scale the + # windows appropriately during launch on Windows and Linux. + f_width, f_height = ( + f_width / content_scale, + f_height / content_scale, + ) + session_settings["window_size"] = int(f_width), int(f_height) session_settings.close() diff --git a/pupil_src/launchables/player.py b/pupil_src/launchables/player.py index a33d33af30..90f88c14bf 100644 --- a/pupil_src/launchables/player.py +++ b/pupil_src/launchables/player.py @@ -132,7 +132,7 @@ def player( ) assert VersionFormat(pyglui_version) >= VersionFormat( - "1.27" + "1.28" ), "pyglui out of date, please upgrade to newest version" process_was_interrupted = False @@ -183,14 +183,64 @@ def interrupt_handler(sig, frame): plugins = system_plugins + user_plugins + def consume_events_and_render_buffer(): + gl_utils.glViewport(0, 0, *g_pool.camera_render_size) + g_pool.capture.gl_display() + for p in g_pool.plugins: + p.gl_display() + + gl_utils.glViewport(0, 0, *window_size) + + try: + clipboard = glfw.glfwGetClipboardString(main_window).decode() + except AttributeError: # clipbaord is None, might happen on startup + clipboard = "" + g_pool.gui.update_clipboard(clipboard) + user_input = g_pool.gui.update() + if user_input.clipboard and user_input.clipboard != clipboard: + # only write to clipboard if content changed + glfw.glfwSetClipboardString(main_window, user_input.clipboard.encode()) + + for b in user_input.buttons: + button, action, mods = b + x, y = glfw.glfwGetCursorPos(main_window) + pos = glfw.window_coordinate_to_framebuffer_coordinate( + main_window, x, y, cached_scale=None + ) + pos = normalize(pos, g_pool.camera_render_size) + pos = denormalize(pos, g_pool.capture.frame_size) + + for plugin in g_pool.plugins: + if plugin.on_click(pos, button, action): + break + + for key, scancode, action, mods in user_input.keys: + for plugin in g_pool.plugins: + if plugin.on_key(key, scancode, action, mods): + break + + for char_ in user_input.chars: + for plugin in g_pool.plugins: + if plugin.on_char(char_): + break + + glfw.glfwSwapBuffers(main_window) + # Callback functions def on_resize(window, w, h): nonlocal window_size - nonlocal hdpi_factor + nonlocal content_scale if w == 0 or h == 0: return - hdpi_factor = glfw.getHDPIFactor(window) - g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor + + # Always clear buffers on resize to make sure that there are no overlapping + # artifacts from previous frames. + gl_utils.glClear(gl_utils.GL_COLOR_BUFFER_BIT) + gl_utils.glClearColor(0, 0, 0, 1) + + content_scale = glfw.get_content_scale(window) + framebuffer_scale = glfw.get_framebuffer_scale(window) + g_pool.gui.scale = content_scale window_size = w, h g_pool.camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) @@ -198,6 +248,18 @@ def on_resize(window, w, h): for p in g_pool.plugins: p.on_window_resize(window, *g_pool.camera_render_size) + # Minimum window size required, otherwise parts of the UI can cause openGL + # issues with permanent effects. Depends on the content scale, which can + # potentially be dynamically modified, so we re-adjust the size limits every + # time here. + min_size = int(2 * icon_bar_width * g_pool.gui.scale / framebuffer_scale) + glfw.glfwSetWindowSizeLimits( + window, min_size, min_size, glfw.GLFW_DONT_CARE, glfw.GLFW_DONT_CARE + ) + + # Needed, to update the window buffer while resizing + consume_events_and_render_buffer() + def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) @@ -208,7 +270,9 @@ def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): - x, y = x * hdpi_factor, y * hdpi_factor + x, y = glfw.window_coordinate_to_framebuffer_coordinate( + window, x, y, cached_scale=None + ) g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, g_pool.camera_render_size) @@ -256,7 +320,7 @@ def get_dt(): icon_bar_width = 50 window_size = None - hdpi_factor = 1.0 + content_scale = 1.0 # create container for globally scoped vars g_pool = SimpleNamespace() @@ -296,22 +360,13 @@ def get_dt(): window_name = f"Pupil Player: {meta_info.recording_name} - {rec_dir}" glfw.glfwInit() + glfw.glfwWindowHint(glfw.GLFW_SCALE_TO_MONITOR, glfw.GLFW_TRUE) main_window = glfw.glfwCreateWindow(width, height, window_name, None, None) glfw.glfwSetWindowPos(main_window, window_pos[0], window_pos[1]) glfw.glfwMakeContextCurrent(main_window) cygl.utils.init() g_pool.main_window = main_window - def set_scale(new_scale): - g_pool.gui_user_scale = new_scale - window_size = ( - g_pool.camera_render_size[0] - + int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), - glfw.glfwGetFramebufferSize(main_window)[1], - ) - logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor) - glfw.glfwSetWindowSize(main_window, *window_size) - g_pool.version = app_version g_pool.timestamps = g_pool.capture.timestamps g_pool.get_timestamp = lambda: 0.0 @@ -391,7 +446,6 @@ def toggle_general_settings(collapsed): general_settings.collapsed = collapsed g_pool.gui = ui.UI() - g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0) g_pool.menubar = ui.Scrolling_Menu( "Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos="left" ) @@ -411,21 +465,26 @@ def toggle_general_settings(collapsed): g_pool.timelines.append(vert_constr) def set_window_size(): + # Get current capture frame size f_width, f_height = g_pool.capture.frame_size - f_width += int(icon_bar_width * g_pool.gui.scale) - glfw.glfwSetWindowSize(main_window, f_width, f_height) + + # Get current display scale factor + content_scale = glfw.get_content_scale(main_window) + framebuffer_scale = glfw.get_framebuffer_scale(main_window) + display_scale_factor = content_scale / framebuffer_scale + + # Scale the capture frame size by display scale factor + f_width *= display_scale_factor + f_height *= display_scale_factor + + # Increas the width to account for the added scaled icon bar width + f_width += icon_bar_width * display_scale_factor + + # Set the newly calculated size (scaled capture frame size + scaled icon bar width) + glfw.glfwSetWindowSize(main_window, int(f_width), int(f_height)) general_settings = ui.Growing_Menu("General", header_pos="headline") general_settings.append(ui.Button("Reset window size", set_window_size)) - general_settings.append( - ui.Selector( - "gui_user_scale", - g_pool, - setter=set_scale, - selection=[0.8, 0.9, 1.0, 1.1, 1.2] + list(np.arange(1.5, 5.1, 0.5)), - label="Interface Size", - ) - ) general_settings.append( ui.Info_Text(f"Minimum Player Version: {meta_info.min_player_version}") ) @@ -548,6 +607,11 @@ def set_window_size(): toggle_general_settings(True) g_pool.gui.configuration = session_settings.get("ui_config", {}) + # If previously selected plugin was not loaded this time, we will have an + # expanded menubar without any menu selected. We need to ensure the menubar is + # collapsed in this case. + if all(submenu.collapsed for submenu in g_pool.menubar.elements): + g_pool.menubar.collapsed = True # gl_state settings gl_utils.basic_gl_setup() @@ -637,7 +701,9 @@ def handle_notifications(n): for b in user_input.buttons: button, action, mods = b x, y = glfw.glfwGetCursorPos(main_window) - pos = x * hdpi_factor, y * hdpi_factor + pos = glfw.window_coordinate_to_framebuffer_coordinate( + main_window, x, y, cached_scale=None + ) pos = normalize(pos, g_pool.camera_render_size) pos = denormalize(pos, g_pool.capture.frame_size) @@ -664,14 +730,19 @@ def handle_notifications(n): session_settings[ "min_calibration_confidence" ] = g_pool.min_calibration_confidence - session_settings["gui_scale"] = g_pool.gui_user_scale session_settings["ui_config"] = g_pool.gui.configuration session_settings["window_position"] = glfw.glfwGetWindowPos(main_window) session_settings["version"] = str(g_pool.version) session_window_size = glfw.glfwGetWindowSize(main_window) if 0 not in session_window_size: - session_settings["window_size"] = session_window_size + f_width, f_height = session_window_size + if platform.system() in ("Windows", "Linux"): + f_width, f_height = ( + f_width / content_scale, + f_height / content_scale, + ) + session_settings["window_size"] = int(f_width), int(f_height) session_settings.close() @@ -768,6 +839,7 @@ def on_drop(window, count, paths): window_pos = session_settings.get("window_position", window_position_default) glfw.glfwInit() + glfw.glfwWindowHint(glfw.GLFW_SCALE_TO_MONITOR, glfw.GLFW_TRUE) glfw.glfwWindowHint(glfw.GLFW_RESIZABLE, 0) window = glfw.glfwCreateWindow(w, h, "Pupil Player") glfw.glfwWindowHint(glfw.GLFW_RESIZABLE, 1) @@ -787,10 +859,10 @@ def on_drop(window, count, paths): # text = "Please supply a Pupil recording directory as first arg when calling Pupil Player." def display_string(string, font_size, center_y): - x = w / 2 * hdpi_factor - y = center_y * hdpi_factor + x = w / 2 * content_scale + y = center_y * content_scale - glfont.set_size(font_size * hdpi_factor) + glfont.set_size(font_size * content_scale) glfont.set_blur(10.5) glfont.set_color_float((0.0, 0.0, 0.0, 1.0)) @@ -803,7 +875,7 @@ def display_string(string, font_size, center_y): while not glfw.glfwWindowShouldClose(window) and not process_was_interrupted: fb_size = glfw.glfwGetFramebufferSize(window) - hdpi_factor = glfw.getHDPIFactor(window) + content_scale = glfw.get_content_scale(window) gl_utils.adjust_gl_view(*fb_size) if rec_dir: diff --git a/pupil_src/launchables/world.py b/pupil_src/launchables/world.py index 018df0295a..da465533b0 100644 --- a/pupil_src/launchables/world.py +++ b/pupil_src/launchables/world.py @@ -208,7 +208,7 @@ def interrupt_handler(sig, frame): icon_bar_width = 50 window_size = None camera_render_size = None - hdpi_factor = 1.0 + content_scale = 1.0 # g_pool holds variables for this process they are accessible to all plugins g_pool = SimpleNamespace() @@ -314,8 +314,8 @@ def get_timestamp(): ("Log_Display", {}), ("Dummy_Gaze_Mapper", {}), ("Display_Recent_Gaze", {}), - # Calibration choreography plugin is added bellow by calling - # patch_world_session_settings_with_choreography_plugin + # Calibration choreography plugin is added below by calling + # patch_loaded_plugins_with_choreography_plugin ("Recorder", {}), ("NetworkApiPlugin", {}), ("Fixation_Detector", {}), @@ -325,15 +325,63 @@ def get_timestamp(): ("System_Graphs", {}), ] + def consume_events_and_render_buffer(): + gl_utils.glViewport(0, 0, *camera_render_size) + for p in g_pool.plugins: + p.gl_display() + + gl_utils.glViewport(0, 0, *window_size) + try: + clipboard = glfw.glfwGetClipboardString(main_window).decode() + except AttributeError: # clipboard is None, might happen on startup + clipboard = "" + g_pool.gui.update_clipboard(clipboard) + user_input = g_pool.gui.update() + if user_input.clipboard != clipboard: + # only write to clipboard if content changed + glfw.glfwSetClipboardString(main_window, user_input.clipboard.encode()) + + for button, action, mods in user_input.buttons: + x, y = glfw.glfwGetCursorPos(main_window) + pos = glfw.window_coordinate_to_framebuffer_coordinate( + main_window, x, y, cached_scale=None + ) + pos = normalize(pos, camera_render_size) + # Position in img pixels + pos = denormalize(pos, g_pool.capture.frame_size) + + for plugin in g_pool.plugins: + if plugin.on_click(pos, button, action): + break + + for key, scancode, action, mods in user_input.keys: + for plugin in g_pool.plugins: + if plugin.on_key(key, scancode, action, mods): + break + + for char_ in user_input.chars: + for plugin in g_pool.plugins: + if plugin.on_char(char_): + break + + glfw.glfwSwapBuffers(main_window) + # Callback functions def on_resize(window, w, h): nonlocal window_size nonlocal camera_render_size - nonlocal hdpi_factor + nonlocal content_scale if w == 0 or h == 0: return - hdpi_factor = glfw.getHDPIFactor(window) - g_pool.gui.scale = g_pool.gui_user_scale * hdpi_factor + + # Always clear buffers on resize to make sure that there are no overlapping + # artifacts from previous frames. + gl_utils.glClear(gl_utils.GL_COLOR_BUFFER_BIT) + gl_utils.glClearColor(0, 0, 0, 1) + + content_scale = glfw.get_content_scale(window) + framebuffer_scale = glfw.get_framebuffer_scale(window) + g_pool.gui.scale = content_scale window_size = w, h camera_render_size = w - int(icon_bar_width * g_pool.gui.scale), h g_pool.gui.update_window(*window_size) @@ -342,6 +390,18 @@ def on_resize(window, w, h): for p in g_pool.plugins: p.on_window_resize(window, *camera_render_size) + # Minimum window size required, otherwise parts of the UI can cause openGL + # issues with permanent effects. Depends on the content scale, which can + # potentially be dynamically modified, so we re-adjust the size limits every + # time here. + min_size = int(2 * icon_bar_width * g_pool.gui.scale / framebuffer_scale) + glfw.glfwSetWindowSizeLimits( + window, min_size, min_size, glfw.GLFW_DONT_CARE, glfw.GLFW_DONT_CARE + ) + + # Needed, to update the window buffer while resizing + consume_events_and_render_buffer() + def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) @@ -352,7 +412,9 @@ def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): - x, y = x * hdpi_factor, y * hdpi_factor + x, y = glfw.window_coordinate_to_framebuffer_coordinate( + window, x, y, cached_scale=None + ) g_pool.gui.update_mouse(x, y) pos = x, y pos = normalize(pos, camera_render_size) @@ -448,6 +510,7 @@ def handle_notifications(noti): # window and gl setup glfw.glfwInit() + glfw.glfwWindowHint(glfw.GLFW_SCALE_TO_MONITOR, glfw.GLFW_TRUE) if hide_ui: glfw.glfwWindowHint(glfw.GLFW_VISIBLE, 0) # hide window main_window = glfw.glfwCreateWindow(width, height, "Pupil Capture - World") @@ -457,16 +520,6 @@ def handle_notifications(noti): cygl.utils.init() g_pool.main_window = main_window - def set_scale(new_scale): - g_pool.gui_user_scale = new_scale - window_size = ( - camera_render_size[0] - + int(icon_bar_width * g_pool.gui_user_scale * hdpi_factor), - glfw.glfwGetFramebufferSize(main_window)[1], - ) - logger.warning(icon_bar_width * g_pool.gui_user_scale * hdpi_factor) - glfw.glfwSetWindowSize(main_window, *window_size) - def reset_restart(): logger.warning("Resetting all settings and restarting Capture.") glfw.glfwSetWindowShouldClose(main_window, True) @@ -484,7 +537,6 @@ def toggle_general_settings(collapsed): # setup GUI g_pool.gui = ui.UI() - g_pool.gui_user_scale = session_settings.get("gui_scale", 1.0) g_pool.menubar = ui.Scrolling_Menu( "Settings", pos=(-400, 0), size=(-icon_bar_width, 0), header_pos="left" ) @@ -497,21 +549,25 @@ def toggle_general_settings(collapsed): g_pool.gui.append(g_pool.quickbar) general_settings = ui.Growing_Menu("General", header_pos="headline") - general_settings.append( - ui.Selector( - "gui_user_scale", - g_pool, - setter=set_scale, - selection=[0.6, 0.8, 1.0, 1.2, 1.4], - label="Interface size", - ) - ) def set_window_size(): + # Get current capture frame size f_width, f_height = g_pool.capture.frame_size - f_width += int(icon_bar_width * g_pool.gui.scale) - glfw.glfwSetWindowSize(main_window, f_width, f_height) - on_resize(main_window, f_width, f_height) + + # Get current display scale factor + content_scale = glfw.get_content_scale(main_window) + framebuffer_scale = glfw.get_framebuffer_scale(main_window) + display_scale_factor = content_scale / framebuffer_scale + + # Scale the capture frame size by display scale factor + f_width *= display_scale_factor + f_height *= display_scale_factor + + # Increas the width to account for the added scaled icon bar width + f_width += icon_bar_width * display_scale_factor + + # Set the newly calculated size (scaled capture frame size + scaled icon bar width) + glfw.glfwSetWindowSize(main_window, int(f_width), int(f_height)) general_settings.append(ui.Button("Reset window size", set_window_size)) general_settings.append( @@ -608,6 +664,11 @@ def set_window_size(): # now that we have a proper window we can load the last gui configuration g_pool.gui.configuration = session_settings.get("ui_config", {}) + # If previously selected plugin was not loaded this time, we will have an + # expanded menubar without any menu selected. We need to ensure the menubar is + # collapsed in this case. + if all(submenu.collapsed for submenu in g_pool.menubar.elements): + g_pool.menubar.collapsed = True # create a timer to control window update frequency window_update_timer = timer(1 / 60) @@ -698,7 +759,9 @@ def window_should_update(): for button, action, mods in user_input.buttons: x, y = glfw.glfwGetCursorPos(main_window) - pos = x * hdpi_factor, y * hdpi_factor + pos = glfw.window_coordinate_to_framebuffer_coordinate( + main_window, x, y, cached_scale=None + ) pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, g_pool.capture.frame_size) @@ -720,7 +783,6 @@ def window_should_update(): glfw.glfwSwapBuffers(main_window) session_settings["loaded_plugins"] = g_pool.plugins.get_initializers() - session_settings["gui_scale"] = g_pool.gui_user_scale session_settings["ui_config"] = g_pool.gui.configuration session_settings["version"] = str(g_pool.version) session_settings["eye0_process_alive"] = eye_procs_alive[0].value @@ -736,7 +798,13 @@ def window_should_update(): session_settings["window_position"] = glfw.glfwGetWindowPos(main_window) session_window_size = glfw.glfwGetWindowSize(main_window) if 0 not in session_window_size: - session_settings["window_size"] = session_window_size + f_width, f_height = session_window_size + if platform.system() in ("Windows", "Linux"): + f_width, f_height = ( + f_width / content_scale, + f_height / content_scale, + ) + session_settings["window_size"] = int(f_width), int(f_height) session_settings.close() diff --git a/pupil_src/shared_modules/accuracy_visualizer.py b/pupil_src/shared_modules/accuracy_visualizer.py index e1f305119a..fc2c957465 100644 --- a/pupil_src/shared_modules/accuracy_visualizer.py +++ b/pupil_src/shared_modules/accuracy_visualizer.py @@ -335,10 +335,31 @@ def recalculate(self): outlier_threshold=self.outlier_threshold, succession_threshold=self.succession_threshold, ) - logger.info("Angular accuracy: {}. Used {} of {} samples.".format(*results[0])) - logger.info("Angular precision: {}. Used {} of {} samples.".format(*results[1])) - self.accuracy = results[0].result - self.precision = results[1].result + + accuracy = results[0].result + if np.isnan(accuracy): + self.accuracy = None + logger.warning( + "Not enough data available for angular accuracy calculation." + ) + else: + self.accuracy = accuracy + logger.info( + "Angular accuracy: {}. Used {} of {} samples.".format(*results[0]) + ) + + precision = results[1].result + if np.isnan(precision): + self.precision = None + logger.warning( + "Not enough data available for angular precision calculation." + ) + else: + self.precision = precision + logger.info( + "Angular precision: {}. Used {} of {} samples.".format(*results[1]) + ) + self.error_lines = results[2] ref_locations = [loc["norm_pos"] for loc in self.recent_input.ref_list] diff --git a/pupil_src/shared_modules/annotations.py b/pupil_src/shared_modules/annotations.py index 1e2772d443..44311ac47f 100644 --- a/pupil_src/shared_modules/annotations.py +++ b/pupil_src/shared_modules/annotations.py @@ -20,7 +20,7 @@ import file_methods as fm import player_methods as pm import zmq_tools -from plugin import Analysis_Plugin_Base, Plugin +from plugin import Plugin logger = logging.getLogger(__name__) @@ -222,7 +222,7 @@ def recent_events(self, events): events["annotation"] = recent_annotation_data -class Annotation_Player(AnnotationPlugin, Analysis_Plugin_Base): +class Annotation_Player(AnnotationPlugin, Plugin): """ Pupil Player plugin to view, edit, and add annotations. """ diff --git a/pupil_src/shared_modules/audio_playback.py b/pupil_src/shared_modules/audio_playback.py index 6a6066ecaf..85545a9758 100644 --- a/pupil_src/shared_modules/audio_playback.py +++ b/pupil_src/shared_modules/audio_playback.py @@ -404,6 +404,8 @@ def fill_audio_queue(self): self.audio_bytes_fifo.append((audio_buffer, audio_playback_time)) def draw_audio(self, width, height, scale): + if self.audio_viz_data is None: + return with gl_utils.Coord_System(*self.xlim, *self.ylim): pyglui_utils.draw_bars_buffer(self.audio_viz_data, color=viz_color) diff --git a/pupil_src/shared_modules/batch_exporter.py b/pupil_src/shared_modules/batch_exporter.py index 2f4264731f..2441ef999b 100644 --- a/pupil_src/shared_modules/batch_exporter.py +++ b/pupil_src/shared_modules/batch_exporter.py @@ -28,7 +28,7 @@ pupil_base_dir = os.path.abspath(__file__).rsplit("pupil_src", 1)[0] sys.path.append(os.path.join(pupil_base_dir, "pupil_src", "shared_modules")) -from plugin import Analysis_Plugin_Base, System_Plugin_Base +from plugin import Plugin, System_Plugin_Base from exporter import export as export_function from player_methods import is_pupil_rec_dir @@ -199,7 +199,7 @@ def cleanup(self): ) -class Batch_Exporter(Analysis_Plugin_Base): +class Batch_Exporter(Plugin): """The Batch_Exporter searches for available recordings and exports them to a common location""" icon_chr = chr(0xEC05) diff --git a/pupil_src/shared_modules/blink_detection.py b/pupil_src/shared_modules/blink_detection.py index 37e04a9dd3..ec85c3e6fa 100644 --- a/pupil_src/shared_modules/blink_detection.py +++ b/pupil_src/shared_modules/blink_detection.py @@ -27,7 +27,7 @@ import gl_utils import player_methods as pm from observable import Observable -from plugin import Analysis_Plugin_Base +from plugin import Plugin logger = logging.getLogger(__name__) @@ -37,7 +37,7 @@ threshold_color = cygl_utils.RGBA(0.9961, 0.8438, 0.3984, 0.8) -class Blink_Detection(Analysis_Plugin_Base): +class Blink_Detection(Plugin): """ This plugin implements a blink detection algorithm, based on sudden drops in the pupil detection confidence. diff --git a/pupil_src/shared_modules/calibration_choreography/controller/gui_window.py b/pupil_src/shared_modules/calibration_choreography/controller/gui_window.py index 682765d68d..3c79a29b71 100644 --- a/pupil_src/shared_modules/calibration_choreography/controller/gui_window.py +++ b/pupil_src/shared_modules/calibration_choreography/controller/gui_window.py @@ -28,9 +28,9 @@ def unsafe_handle(self): return self.__gl_handle @property - def hdpi_factor(self) -> float: + def content_scale(self) -> float: if self.__gl_handle is not None: - return glfw.getHDPIFactor(self.__gl_handle) + return glfw.get_content_scale(self.__gl_handle) else: return 1.0 diff --git a/pupil_src/shared_modules/calibration_choreography/controller/marker_window_controller.py b/pupil_src/shared_modules/calibration_choreography/controller/marker_window_controller.py index 93fc98bf99..a37aeb62c1 100644 --- a/pupil_src/shared_modules/calibration_choreography/controller/marker_window_controller.py +++ b/pupil_src/shared_modules/calibration_choreography/controller/marker_window_controller.py @@ -356,7 +356,7 @@ def __draw_status_text(self, clicks_needed: int): @property def __marker_radius(self) -> float: - return self.marker_scale * self.__window.hdpi_factor + return self.marker_scale * self.__window.content_scale def __marker_position_on_screen(self, marker_position) -> T.Tuple[float, float]: padding = 90 * self.__marker_radius diff --git a/pupil_src/shared_modules/fixation_detector.py b/pupil_src/shared_modules/fixation_detector.py index a00f843f83..81f95f74a1 100644 --- a/pupil_src/shared_modules/fixation_detector.py +++ b/pupil_src/shared_modules/fixation_detector.py @@ -49,7 +49,7 @@ from observable import Observable import player_methods as pm from methods import denormalize -from plugin import Analysis_Plugin_Base +from plugin import Plugin logger = logging.getLogger(__name__) @@ -59,7 +59,7 @@ class FixationDetectionMethod(enum.Enum): GAZE_3D = "3d gaze" -class Fixation_Detector_Base(Analysis_Plugin_Base): +class Fixation_Detector_Base(Plugin): icon_chr = chr(0xEC03) icon_font = "pupil_icons" @@ -689,6 +689,7 @@ def __init__(self, g_pool, max_dispersion=3.0, min_duration=300, **kwargs): self.min_duration = min_duration self.max_dispersion = max_dispersion self.id_counter = 0 + self.recent_fixation = None def recent_events(self, events): events["fixations"] = [] diff --git a/pupil_src/shared_modules/gaze_mapping/gazer_2d.py b/pupil_src/shared_modules/gaze_mapping/gazer_2d.py index 45dafd9f25..ed8b30e82b 100644 --- a/pupil_src/shared_modules/gaze_mapping/gazer_2d.py +++ b/pupil_src/shared_modules/gaze_mapping/gazer_2d.py @@ -45,11 +45,17 @@ def is_fitted(self) -> bool: return self._is_fitted def set_params(self, **params): + if params == {}: + return for key, value in params.items(): setattr(self._regressor, key, np.asarray(value)) self._is_fitted = True def get_params(self): + has_coef = hasattr(self._regressor, "coef_") + has_intercept = hasattr(self._regressor, "intercept_") + if not has_coef or not has_intercept: + return {} return { "coef_": self._regressor.coef_.tolist(), "intercept_": self._regressor.intercept_.tolist(), diff --git a/pupil_src/shared_modules/gaze_producer/gaze_producer_base.py b/pupil_src/shared_modules/gaze_producer/gaze_producer_base.py index 362ce50867..4123d3e154 100644 --- a/pupil_src/shared_modules/gaze_producer/gaze_producer_base.py +++ b/pupil_src/shared_modules/gaze_producer/gaze_producer_base.py @@ -14,10 +14,10 @@ import data_changed from observable import Observable import player_methods as pm -from plugin import Producer_Plugin_Base +from plugin import System_Plugin_Base -class GazeProducerBase(Observable, Producer_Plugin_Base): +class GazeProducerBase(Observable, System_Plugin_Base): uniqueness = "by_base_class" order = 0.02 icon_chr = chr(0xEC14) diff --git a/pupil_src/shared_modules/gaze_producer/ui/calibration_menu.py b/pupil_src/shared_modules/gaze_producer/ui/calibration_menu.py index 8c6354ab5e..be3631e295 100644 --- a/pupil_src/shared_modules/gaze_producer/ui/calibration_menu.py +++ b/pupil_src/shared_modules/gaze_producer/ui/calibration_menu.py @@ -36,6 +36,9 @@ def __init__(self, calibration_storage, calibration_controller, index_range_as_s self.menu.collapsed = True + self._ui_button_duplicate = None + self._ui_button_delete = None + calibration_controller.add_observer( "on_calibration_computed", self._on_calibration_computed ) @@ -73,6 +76,16 @@ def _render_ui_normally(self, calibration, menu): ] ) + def _create_duplicate_button(self): + # Save a reference to the created duplicate button + self._ui_button_duplicate = super()._create_duplicate_button() + return self._ui_button_duplicate + + def _create_delete_button(self): + # Save a reference to the created delete button + self._ui_button_delete = super()._create_delete_button() + return self._ui_button_delete + def _create_name_input(self, calibration): return ui.Text_Input( "name", calibration, label="Name", setter=self._on_name_change @@ -158,29 +171,25 @@ def _info_text_for_online_calibration(self, calibration): ) def _on_click_duplicate_button(self): - if not self._calibration_controller.is_from_same_recording(self.current_item): - logger.error("Cannot duplicate calibrations from other recordings!") - return - - if not self.current_item.is_offline_calibration: - logger.error("Cannot duplicate pre-recorded calibrations!") - return - - super()._on_click_duplicate_button() + if self.__check_duplicate_button_click_is_allowed( + should_log_reason_as_error=True + ): + super()._on_click_duplicate_button() def _on_click_delete(self): - if self.current_item is None: - return - - if not self._calibration_controller.is_from_same_recording(self.current_item): - logger.error("Cannot delete calibrations from other recordings!") - return - - if not self.current_item.is_offline_calibration: - logger.error("Cannot delete pre-recorded calibrations!") - return - - super()._on_click_delete() + if self.__check_delete_button_click_is_allowed(should_log_reason_as_error=True): + super()._on_click_delete() + + def _on_change_current_item(self, item): + super()._on_change_current_item(item) + if self._ui_button_duplicate: + self._ui_button_duplicate.read_only = not self.__check_duplicate_button_click_is_allowed( + should_log_reason_as_error=False + ) + if self._ui_button_delete: + self._ui_button_delete.read_only = not self.__check_delete_button_click_is_allowed( + should_log_reason_as_error=False + ) def _on_name_change(self, new_name): self._calibration_storage.rename(self.current_item, new_name) @@ -204,3 +213,34 @@ def _on_calibration_computed(self, calibration): def _on_calculation_could_not_be_started(self): self.render() + + def __check_duplicate_button_click_is_allowed( + self, should_log_reason_as_error: bool + ): + if not self._calibration_controller.is_from_same_recording(self.current_item): + if should_log_reason_as_error: + logger.error("Cannot duplicate calibrations from other recordings!") + return False + + if not self.current_item.is_offline_calibration: + if should_log_reason_as_error: + logger.error("Cannot duplicate pre-recorded calibrations!") + return False + + return True + + def __check_delete_button_click_is_allowed(self, should_log_reason_as_error: bool): + if self.current_item is None: + return False + + if not self._calibration_controller.is_from_same_recording(self.current_item): + if should_log_reason_as_error: + logger.error("Cannot delete calibrations from other recordings!") + return False + + if not self.current_item.is_offline_calibration: + if should_log_reason_as_error: + logger.error("Cannot delete pre-recorded calibrations!") + return False + + return True diff --git a/pupil_src/shared_modules/gaze_producer/ui/storage_edit_menu.py b/pupil_src/shared_modules/gaze_producer/ui/storage_edit_menu.py index 12b8e3284e..94f0a9ef93 100644 --- a/pupil_src/shared_modules/gaze_producer/ui/storage_edit_menu.py +++ b/pupil_src/shared_modules/gaze_producer/ui/storage_edit_menu.py @@ -27,6 +27,7 @@ class StorageEditMenu(plugin_ui.SelectAndRefreshMenu, abc.ABC): new_button_label = "New" duplicate_button_label = "Duplicate Current Configuration" + delete_button_label = "Delete" def __init__(self, storage): super().__init__() @@ -73,9 +74,12 @@ def _create_duplicate_button(self): label=self.duplicate_button_label, function=self._on_click_duplicate_button ) + def _create_delete_button(self): + return ui.Button(label=self.delete_button_label, function=self._on_click_delete) + def render_item(self, item, menu): self._render_custom_ui(item, menu) - menu.append(ui.Button(label="Delete", function=self._on_click_delete)) + menu.append(self._create_delete_button()) def _on_click_new_button(self): new_item = self._new_item() diff --git a/pupil_src/shared_modules/gaze_producer/worker/map_gaze.py b/pupil_src/shared_modules/gaze_producer/worker/map_gaze.py index 2c17309389..5183cf8ab4 100644 --- a/pupil_src/shared_modules/gaze_producer/worker/map_gaze.py +++ b/pupil_src/shared_modules/gaze_producer/worker/map_gaze.py @@ -67,11 +67,16 @@ def _map_gaze( first_ts = pupil_pos_in_mapping_range[0]["timestamp"] last_ts = pupil_pos_in_mapping_range[-1]["timestamp"] ts_span = last_ts - first_ts + curr_ts = first_ts for gaze_datum in gazer.map_pupil_to_gaze(pupil_pos_in_mapping_range): _apply_manual_correction(gaze_datum, manual_correction_x, manual_correction_y) - curr_ts = gaze_datum["timestamp"] + # gazer.map_pupil_to_gaze does not yield gaze with monotonic timestamps. + # Binocular pupil matches are delayed internally. To avoid non-monotonic + # progress updates, we use the largest timestamp that has been returned up to + # the current point in time. + curr_ts = max(curr_ts, gaze_datum["timestamp"]) shared_memory.progress = (curr_ts - first_ts) / ts_span result = (curr_ts, fm.Serialized_Dict(gaze_datum)) diff --git a/pupil_src/shared_modules/glfw.py b/pupil_src/shared_modules/glfw.py index 860da64766..80da04953a 100644 --- a/pupil_src/shared_modules/glfw.py +++ b/pupil_src/shared_modules/glfw.py @@ -102,6 +102,10 @@ GLFW_VERSION_REVISION = 1 __version__ = GLFW_VERSION_MAJOR, GLFW_VERSION_MINOR, GLFW_VERSION_REVISION +GLFW_FALSE = 0 +GLFW_TRUE = 1 +GLFW_DONT_CARE = -1 + # --- Input handling definitions ---------------------------------------------- GLFW_RELEASE = 0 GLFW_PRESS = 1 @@ -323,6 +327,10 @@ GLFW_OPENGL_FORWARD_COMPAT = 0x00022006 GLFW_OPENGL_DEBUG_CONTEXT = 0x00022007 GLFW_OPENGL_PROFILE = 0x00022008 +# GLFW_CONTEXT_RELEASE_BEHAVIOR = 0x00022009 +# GLFW_CONTEXT_NO_ERROR = 0x0002200A +# GLFW_CONTEXT_CREATION_API = 0x0002200B +GLFW_SCALE_TO_MONITOR = 0x0002200C # --- GLFW_OPENGL_API = 0x00030001 @@ -439,6 +447,7 @@ class GLFWmonitor(Structure): # glfwGetWindowPos = _glfw.glfwGetWindowPos glfwSetWindowPos = _glfw.glfwSetWindowPos # glfwGetWindowSize = _glfw.glfwGetWindowSize +glfwSetWindowSizeLimits = _glfw.glfwSetWindowSizeLimits glfwSetWindowSize = _glfw.glfwSetWindowSize # glfwGetFramebufferSize = _glfw.glfwGetFramebufferSize glfwIconifyWindow = _glfw.glfwIconifyWindow @@ -666,6 +675,12 @@ def glfwGetVideoMode(monitor): ) +def glfwGetWindowContentScale(window): + xscale, yscale = c_float(0), c_float(0) + _glfw.glfwGetWindowContentScale(window, byref(xscale), byref(yscale)) + return xscale.value, yscale.value + + def GetGammaRamp(monitor): _glfw.glfwGetGammaRamp.restype = POINTER(GLFWgammaramp) c_gamma = _glfw.glfwGetGammaRamp(monitor).contents @@ -729,8 +744,20 @@ def {callback}(window, callback = None): exec(__callback__("Drop")) -def getHDPIFactor(window): +def get_content_scale(window) -> float: + return glfwGetWindowContentScale(window)[0] + + +def get_framebuffer_scale(window) -> float: + window_width = glfwGetWindowSize(window)[0] + framebuffer_width = glfwGetFramebufferSize(window)[0] + try: - return float(glfwGetFramebufferSize(window)[0] / glfwGetWindowSize(window)[0]) + return float(framebuffer_width / window_width) except ZeroDivisionError: return 1.0 + + +def window_coordinate_to_framebuffer_coordinate(window, x, y, cached_scale=None): + scale = cached_scale or get_framebuffer_scale(window) + return x * scale, y * scale diff --git a/pupil_src/shared_modules/head_pose_tracker/ui/gl_window.py b/pupil_src/shared_modules/head_pose_tracker/ui/gl_window.py index 3ae993928f..cabe90ab9b 100644 --- a/pupil_src/shared_modules/head_pose_tracker/ui/gl_window.py +++ b/pupil_src/shared_modules/head_pose_tracker/ui/gl_window.py @@ -62,6 +62,7 @@ def _init_trackball(): def _glfw_init(self): glfw.glfwInit() + glfw.glfwWindowHint(glfw.GLFW_SCALE_TO_MONITOR, glfw.GLFW_TRUE) window = glfw.glfwCreateWindow( title="Head Pose Tracker Visualizer", share=glfw.glfwGetCurrentContext() ) diff --git a/pupil_src/shared_modules/log_display.py b/pupil_src/shared_modules/log_display.py index 8c9b145064..f7d278bee2 100644 --- a/pupil_src/shared_modules/log_display.py +++ b/pupil_src/shared_modules/log_display.py @@ -86,7 +86,7 @@ def on_log(self, record): self.alpha = min(self.alpha, 6.0) def on_window_resize(self, window, w, h): - self.window_scale = glfw.getHDPIFactor(window) + self.window_scale = glfw.get_content_scale(window) self.glfont.set_size(32 * self.window_scale) self.window_size = w, h self.tex.resize(*self.window_size) diff --git a/pupil_src/shared_modules/plugin.py b/pupil_src/shared_modules/plugin.py index 0d2746e737..02e4e5391a 100644 --- a/pupil_src/shared_modules/plugin.py +++ b/pupil_src/shared_modules/plugin.py @@ -503,22 +503,5 @@ def import_runtime_plugins(plugin_dir): return runtime_plugins -# Base plugin definitons -class Visualizer_Plugin_Base(Plugin): - pass - - -class Analysis_Plugin_Base(Plugin): - pass - - -class Producer_Plugin_Base(Plugin): - pass - - class System_Plugin_Base(Plugin): pass - - -class Experimental_Plugin_Base(Plugin): - pass diff --git a/pupil_src/shared_modules/plugin_manager.py b/pupil_src/shared_modules/plugin_manager.py index 5be642a8d7..514871a5a0 100644 --- a/pupil_src/shared_modules/plugin_manager.py +++ b/pupil_src/shared_modules/plugin_manager.py @@ -9,7 +9,7 @@ ---------------------------------------------------------------------------~(*) """ -from plugin import System_Plugin_Base, Producer_Plugin_Base +from plugin import System_Plugin_Base from pyglui import ui from calibration_choreography import CalibrationChoreographyPlugin from gaze_mapping.gazer_base import GazerBase @@ -28,7 +28,6 @@ def __init__(self, g_pool): Base_Source, CalibrationChoreographyPlugin, GazerBase, - Producer_Plugin_Base, ) self.user_plugins = [ p diff --git a/pupil_src/shared_modules/pupil_producers.py b/pupil_src/shared_modules/pupil_producers.py index 7841b124cc..678b67ddb9 100644 --- a/pupil_src/shared_modules/pupil_producers.py +++ b/pupil_src/shared_modules/pupil_producers.py @@ -26,7 +26,7 @@ import pyglui.cygl.utils as cygl_utils import zmq_tools from observable import Observable -from plugin import Producer_Plugin_Base +from plugin import System_Plugin_Base from pyglui import ui from pyglui.pyfontstash import fontstash as fs from video_capture.utils import VideoSet @@ -41,7 +41,7 @@ DATA_KEY_DIAMETER = "diameter_3d" -class Pupil_Producer_Base(Observable, Producer_Plugin_Base): +class Pupil_Producer_Base(Observable, System_Plugin_Base): uniqueness = "by_base_class" order = 0.01 icon_chr = chr(0xEC12) diff --git a/pupil_src/shared_modules/raw_data_exporter.py b/pupil_src/shared_modules/raw_data_exporter.py index 3e1c6b3d40..88e2fe6f18 100644 --- a/pupil_src/shared_modules/raw_data_exporter.py +++ b/pupil_src/shared_modules/raw_data_exporter.py @@ -19,13 +19,13 @@ import csv_utils import player_methods as pm -from plugin import Analysis_Plugin_Base +from plugin import Plugin # logging logger = logging.getLogger(__name__) -class Raw_Data_Exporter(Analysis_Plugin_Base): +class Raw_Data_Exporter(Plugin): """ pupil_positions.csv keys: diff --git a/pupil_src/shared_modules/service_ui.py b/pupil_src/shared_modules/service_ui.py index 37a38edd6d..c1c9a29c12 100644 --- a/pupil_src/shared_modules/service_ui.py +++ b/pupil_src/shared_modules/service_ui.py @@ -49,6 +49,7 @@ def __init__( self.texture = np.zeros((1, 1, 3), dtype=np.uint8) + 128 glfw.glfwInit() + glfw.glfwWindowHint(glfw.GLFW_SCALE_TO_MONITOR, glfw.GLFW_TRUE) if g_pool.hide_ui: glfw.glfwWindowHint(glfw.GLFW_VISIBLE, 0) # hide window main_window = glfw.glfwCreateWindow(*window_size, "Pupil Service") @@ -58,7 +59,6 @@ def __init__( g_pool.main_window = main_window g_pool.gui = ui.UI() - g_pool.gui_user_scale = gui_scale g_pool.menubar = ui.Scrolling_Menu( "Settings", pos=(0, 0), size=(0, 0), header_pos="headline" ) @@ -66,12 +66,20 @@ def __init__( # Callback functions def on_resize(window, w, h): + # Always clear buffers on resize to make sure that there are no overlapping + # artifacts from previous frames. + gl_utils.glClear(gl_utils.GL_COLOR_BUFFER_BIT) + gl_utils.glClearColor(0, 0, 0, 1) + self.window_size = w, h - self.hdpi_factor = glfw.getHDPIFactor(window) - g_pool.gui.scale = g_pool.gui_user_scale * self.hdpi_factor + self.content_scale = glfw.get_content_scale(window) + g_pool.gui.scale = self.content_scale g_pool.gui.update_window(w, h) g_pool.gui.collect_menus() + # Needed, to update the window buffer while resizing + self.update_ui() + def on_window_key(window, key, scancode, action, mods): g_pool.gui.update_key(key, scancode, action, mods) @@ -82,18 +90,29 @@ def on_window_mouse_button(window, button, action, mods): g_pool.gui.update_button(button, action, mods) def on_pos(window, x, y): - x, y = x * self.hdpi_factor, y * self.hdpi_factor + x, y = glfw.window_coordinate_to_framebuffer_coordinate( + window, x, y, cached_scale=None + ) g_pool.gui.update_mouse(x, y) def on_scroll(window, x, y): g_pool.gui.update_scroll(x, y * scroll_factor) - def set_scale(new_scale): - g_pool.gui_user_scale = new_scale - on_resize(main_window, *self.window_size) - def set_window_size(): - glfw.glfwSetWindowSize(main_window, *window_size_default) + # Get default window size + f_width, f_height = window_size_default + + # Get current display scale factor + content_scale = glfw.get_content_scale(main_window) + framebuffer_scale = glfw.get_framebuffer_scale(main_window) + display_scale_factor = content_scale / framebuffer_scale + + # Scale the capture frame size by display scale factor + f_width *= display_scale_factor + f_height *= display_scale_factor + + # Set the newly calculated size (scaled capture frame size + scaled icon bar width) + glfw.glfwSetWindowSize(main_window, int(f_width), int(f_height)) def reset_restart(): logger.warning("Resetting all settings and restarting Capture.") @@ -101,16 +120,6 @@ def reset_restart(): self.notify_all({"subject": "clear_settings_process.should_start"}) self.notify_all({"subject": "service_process.should_start", "delay": 2.0}) - g_pool.menubar.append( - ui.Selector( - "gui_user_scale", - g_pool, - setter=set_scale, - selection=[0.6, 0.8, 1.0, 1.2, 1.4], - label="Interface size", - ) - ) - g_pool.menubar.append(ui.Button("Reset window size", set_window_size)) pupil_remote_addr = "{}:{}".format( @@ -214,13 +223,19 @@ def cleanup(self): def get_init_dict(self): sess = { "window_position": glfw.glfwGetWindowPos(self.g_pool.main_window), - "gui_scale": self.g_pool.gui_user_scale, "ui_config": self.g_pool.gui.configuration, } session_window_size = glfw.glfwGetWindowSize(self.g_pool.main_window) if 0 not in session_window_size: - sess["window_size"] = session_window_size + f_width, f_height = session_window_size + if platform.system() in ("Windows", "Linux"): + content_scale = glfw.get_content_scale(self.g_pool.main_window) + f_width, f_height = ( + f_width / content_scale, + f_height / content_scale, + ) + sess["window_size"] = int(f_width), int(f_height) return sess diff --git a/pupil_src/shared_modules/surface_tracker/surface_tracker_offline.py b/pupil_src/shared_modules/surface_tracker/surface_tracker_offline.py index 4b4d90b9bf..d5f1571084 100644 --- a/pupil_src/shared_modules/surface_tracker/surface_tracker_offline.py +++ b/pupil_src/shared_modules/surface_tracker/surface_tracker_offline.py @@ -26,7 +26,7 @@ import file_methods import gl_utils from observable import Observable -from plugin import Analysis_Plugin_Base +from plugin import Plugin from . import background_tasks, offline_utils from .cache import Cache @@ -51,7 +51,7 @@ mp_context = multiprocessing.get_context() -class Surface_Tracker_Offline(Observable, Surface_Tracker, Analysis_Plugin_Base): +class Surface_Tracker_Offline(Observable, Surface_Tracker, Plugin): """ The Surface_Tracker_Offline does marker based AOI tracking in a recording. All marker and surface detections are calculated in the background and cached to reduce diff --git a/pupil_src/shared_modules/system_graphs.py b/pupil_src/shared_modules/system_graphs.py index 955deb99c8..face887390 100644 --- a/pupil_src/shared_modules/system_graphs.py +++ b/pupil_src/shared_modules/system_graphs.py @@ -73,12 +73,12 @@ def init_ui(self): def on_window_resize(self, window, *args): fb_size = glfw.glfwGetFramebufferSize(window) - hdpi_factor = glfw.getHDPIFactor(window) + content_scale = glfw.get_content_scale(window) - self.cpu_graph.scale = hdpi_factor - self.fps_graph.scale = hdpi_factor - self.conf0_graph.scale = hdpi_factor - self.conf1_graph.scale = hdpi_factor + self.cpu_graph.scale = content_scale + self.fps_graph.scale = content_scale + self.conf0_graph.scale = content_scale + self.conf1_graph.scale = content_scale self.cpu_graph.adjust_window_size(*fb_size) self.fps_graph.adjust_window_size(*fb_size) diff --git a/pupil_src/shared_modules/video_export/plugin_base/video_exporter.py b/pupil_src/shared_modules/video_export/plugin_base/video_exporter.py index 6646254103..7ac44b12df 100644 --- a/pupil_src/shared_modules/video_export/plugin_base/video_exporter.py +++ b/pupil_src/shared_modules/video_export/plugin_base/video_exporter.py @@ -14,10 +14,10 @@ from pyglui import ui from task_manager import TaskManager -from plugin import Analysis_Plugin_Base +from plugin import Plugin -class VideoExporter(TaskManager, Analysis_Plugin_Base, abc.ABC): +class VideoExporter(TaskManager, Plugin, abc.ABC): """ Base for video exporting plugins. Every time the user hits "export", the method export_data gets called diff --git a/pupil_src/shared_modules/video_overlay/ui/interactions.py b/pupil_src/shared_modules/video_overlay/ui/interactions.py index 89d70064b0..d6bdc8163f 100644 --- a/pupil_src/shared_modules/video_overlay/ui/interactions.py +++ b/pupil_src/shared_modules/video_overlay/ui/interactions.py @@ -9,7 +9,7 @@ ---------------------------------------------------------------------------~(*) """ -from glfw import getHDPIFactor, glfwGetCurrentContext, glfwGetCursorPos, GLFW_PRESS +from glfw import get_content_scale, glfwGetCurrentContext, glfwGetCursorPos, GLFW_PRESS from methods import normalize, denormalize @@ -60,9 +60,9 @@ def _effective_overlay_frame_size(self): def current_mouse_pos(window, camera_render_size, frame_size): - hdpi_fac = getHDPIFactor(window) + content_scale = get_content_scale(window) x, y = glfwGetCursorPos(glfwGetCurrentContext()) - pos = x * hdpi_fac, y * hdpi_fac + pos = x * content_scale, y * content_scale pos = normalize(pos, camera_render_size) # Position in img pixels pos = denormalize(pos, frame_size) diff --git a/pupil_src/shared_modules/vis_circle.py b/pupil_src/shared_modules/vis_circle.py index b2fc1071d6..975237231d 100644 --- a/pupil_src/shared_modules/vis_circle.py +++ b/pupil_src/shared_modules/vis_circle.py @@ -10,13 +10,13 @@ """ from player_methods import transparent_circle -from plugin import Visualizer_Plugin_Base +from plugin import Plugin from pyglui import ui from methods import denormalize -class Vis_Circle(Visualizer_Plugin_Base): +class Vis_Circle(Plugin): uniqueness = "not_unique" icon_chr = chr(0xE061) icon_font = "pupil_icons" diff --git a/pupil_src/shared_modules/vis_cross.py b/pupil_src/shared_modules/vis_cross.py index 1d001420df..ee1f850e90 100644 --- a/pupil_src/shared_modules/vis_cross.py +++ b/pupil_src/shared_modules/vis_cross.py @@ -9,7 +9,7 @@ ---------------------------------------------------------------------------~(*) """ -from plugin import Visualizer_Plugin_Base +from plugin import Plugin import numpy as np import cv2 @@ -17,7 +17,7 @@ from methods import denormalize -class Vis_Cross(Visualizer_Plugin_Base): +class Vis_Cross(Plugin): uniqueness = "not_unique" icon_chr = chr(0xEC13) icon_font = "pupil_icons" diff --git a/pupil_src/shared_modules/vis_fixation.py b/pupil_src/shared_modules/vis_fixation.py index 3b0b1af995..0044566f8e 100644 --- a/pupil_src/shared_modules/vis_fixation.py +++ b/pupil_src/shared_modules/vis_fixation.py @@ -10,14 +10,14 @@ """ from player_methods import transparent_circle -from plugin import Visualizer_Plugin_Base +from plugin import Plugin import numpy as np import cv2 from methods import denormalize from pyglui import ui -class Vis_Fixation(Visualizer_Plugin_Base): +class Vis_Fixation(Plugin): uniqueness = "not_unique" icon_chr = chr(0xEC03) icon_font = "pupil_icons" diff --git a/pupil_src/shared_modules/vis_light_points.py b/pupil_src/shared_modules/vis_light_points.py index b45ba325d0..25cff8ac2d 100644 --- a/pupil_src/shared_modules/vis_light_points.py +++ b/pupil_src/shared_modules/vis_light_points.py @@ -10,7 +10,7 @@ """ import cv2 -from plugin import Visualizer_Plugin_Base +from plugin import Plugin import numpy as np from pyglui import ui @@ -20,7 +20,7 @@ logger = logging.getLogger(__name__) -class Vis_Light_Points(Visualizer_Plugin_Base): +class Vis_Light_Points(Plugin): """docstring show gaze dots at light dots on numpy. diff --git a/pupil_src/shared_modules/vis_polyline.py b/pupil_src/shared_modules/vis_polyline.py index f2af24e42f..6fb666aa8c 100644 --- a/pupil_src/shared_modules/vis_polyline.py +++ b/pupil_src/shared_modules/vis_polyline.py @@ -9,7 +9,7 @@ ---------------------------------------------------------------------------~(*) """ -from plugin import Visualizer_Plugin_Base +from plugin import Plugin import numpy as np import cv2 @@ -23,7 +23,7 @@ from scan_path.utils import np_denormalize -class Vis_Polyline(Visualizer_Plugin_Base, Observable): +class Vis_Polyline(Plugin, Observable): order = 0.9 uniqueness = "not_unique" icon_chr = chr(0xE922) diff --git a/pupil_src/shared_modules/vis_watermark.py b/pupil_src/shared_modules/vis_watermark.py index 952faff2e6..56609309cd 100644 --- a/pupil_src/shared_modules/vis_watermark.py +++ b/pupil_src/shared_modules/vis_watermark.py @@ -10,7 +10,7 @@ """ from player_methods import transparent_image_overlay -from plugin import Visualizer_Plugin_Base +from plugin import Plugin import numpy as np import cv2 from glob import glob @@ -24,7 +24,7 @@ logger = logging.getLogger(__name__) -class Vis_Watermark(Visualizer_Plugin_Base): +class Vis_Watermark(Plugin): uniqueness = "not_unique" icon_chr = chr(0xEC04) icon_font = "pupil_icons" diff --git a/pupil_src/shared_modules/visualizer.py b/pupil_src/shared_modules/visualizer.py index a7bd1d6685..ad0ad0c4d0 100644 --- a/pupil_src/shared_modules/visualizer.py +++ b/pupil_src/shared_modules/visualizer.py @@ -165,6 +165,7 @@ def open_window(self): # get glfw started if self.run_independently: glfwInit() + glfwWindowHint(GLFW_SCALE_TO_MONITOR, GLFW_TRUE) self.window = glfwCreateWindow( self.window_size[0], self.window_size[1], self.name, None ) @@ -227,8 +228,9 @@ def on_window_mouse_button(self, window, button, action, mods): self.input["button"] = None def on_pos(self, window, x, y): - hdpi_factor = getHDPIFactor(window) - x, y = x * hdpi_factor, y * hdpi_factor + x, y = window_coordinate_to_framebuffer_coordinate( + window, x, y, cached_scale=None + ) # self.gui.update_mouse(x,y) if self.input["button"] == GLFW_MOUSE_BUTTON_RIGHT: old_x, old_y = self.input["mouse"] diff --git a/pupil_src/shared_modules/zmq_tools.py b/pupil_src/shared_modules/zmq_tools.py index f49f551caa..f9a93ccf7a 100644 --- a/pupil_src/shared_modules/zmq_tools.py +++ b/pupil_src/shared_modules/zmq_tools.py @@ -42,6 +42,8 @@ def emit(self, record): try: self.socket.send(record_dict) except TypeError: + # stringify message in case it is not a string yet + record_dict["msg"] = str(record_dict["msg"]) # stringify `exc_info` since it includes unserializable objects if record_dict["exc_info"]: # do not convert if it is None record_dict["exc_info"] = str(record_dict["exc_info"]) @@ -153,7 +155,7 @@ def send(self, payload, deprecated=()): the contents of the iterable in '__raw_data__' require exposing the pyhton memoryview interface. """ - assert deprecated is (), "Depracted use of send()" + assert deprecated == (), "Depracted use of send()" assert "topic" in payload, "`topic` field required in {}".format(payload) if "__raw_data__" not in payload: