[PATCH] ui/cocoa: Support hardware cursor interface

[Elliot Nunn]

Implement dpy_cursor_define() and dpy_mouse_set() on macOS.

The main benefit is from dpy_cursor_define: in absolute pointing mode, the
host can redraw the cursor on the guest's behalf much faster than the guest
can itself.

To provide the programmatic movement expected from a hardware cursor,
dpy_mouse_set is also implemented.

Tricky cases are handled:
- dpy_mouse_set() avoids rounded window corners.
- The sometimes-delay between warping the cursor and an affected mouse-move
  event is accounted for.
- Cursor bitmaps are nearest-neighbor scaled to Retina size.

Signed-off-by: Elliot Nunn <elliot@nunn.io>
---
 ui/cocoa.m | 263 ++++++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 240 insertions(+), 23 deletions(-)

diff --git a/ui/cocoa.m b/ui/cocoa.m
index 5a8bd5dd84..f9d54448e4 100644
--- a/ui/cocoa.m
+++ b/ui/cocoa.m
@@ -85,12 +85,20 @@ static void cocoa_switch(DisplayChangeListener *dcl,
 
 static void cocoa_refresh(DisplayChangeListener *dcl);
 
+static void cocoa_mouse_set(DisplayChangeListener *dcl,
+                            int x, int y, int on);
+
+static void cocoa_cursor_define(DisplayChangeListener *dcl,
+                                QEMUCursor *c);
+
 static NSWindow *normalWindow;
 static const DisplayChangeListenerOps dcl_ops = {
     .dpy_name          = "cocoa",
     .dpy_gfx_update = cocoa_update,
     .dpy_gfx_switch = cocoa_switch,
     .dpy_refresh = cocoa_refresh,
+    .dpy_mouse_set = cocoa_mouse_set,
+    .dpy_cursor_define = cocoa_cursor_define,
 };
 static DisplayChangeListener dcl = {
     .ops = &dcl_ops,
@@ -313,6 +321,13 @@ @interface QemuCocoaView : NSView
     BOOL isFullscreen;
     BOOL isAbsoluteEnabled;
     CFMachPortRef eventsTap;
+    NSCursor *guestCursor;
+    BOOL cursorHiddenByMe;
+    BOOL guestCursorVis;
+    int guestCursorX, guestCursorY;
+    int lastWarpX, lastWarpY;
+    int warpDeltaX, warpDeltaY;
+    BOOL ignoreNextMouseMove;
 }
 - (void) switchSurface:(pixman_image_t *)image;
 - (void) grabMouse;
@@ -323,6 +338,10 @@ - (void) handleMonitorInput:(NSEvent *)event;
 - (bool) handleEvent:(NSEvent *)event;
 - (bool) handleEventLocked:(NSEvent *)event;
 - (void) setAbsoluteEnabled:(BOOL)tIsAbsoluteEnabled;
+- (void) cursorDefine:(NSCursor *)cursor;
+- (void) mouseSetX:(int)x Y:(int)y on:(int)on;
+- (void) setCursorAppearance;
+- (void) setCursorPosition;
 /* The state surrounding mouse grabbing is potentially confusing.
  * isAbsoluteEnabled tracks qemu_input_is_absolute() [ie "is the emulated
  *   pointing device an absolute-position one?"], but is only updated on
@@ -432,22 +451,6 @@ - (CGPoint) screenLocationOfEvent:(NSEvent *)ev
     }
 }
 
-- (void) hideCursor
-{
-    if (!cursor_hide) {
-        return;
-    }
-    [NSCursor hide];
-}
-
-- (void) unhideCursor
-{
-    if (!cursor_hide) {
-        return;
-    }
-    [NSCursor unhide];
-}
-
 - (void) drawRect:(NSRect) rect
 {
     COCOA_DEBUG("QemuCocoaView: drawRect\n");
@@ -635,6 +638,8 @@ - (void) switchSurface:(pixman_image_t *)image
         screen.height = h;
         [self setContentDimensions];
         [self setFrame:NSMakeRect(cx, cy, cw, ch)];
+        [self setCursorAppearance];
+        [self setCursorPosition];
     }
 
     // update screenBuffer
@@ -681,6 +686,7 @@ - (void) toggleFullScreen:(id)sender
             styleMask:NSWindowStyleMaskBorderless
             backing:NSBackingStoreBuffered
             defer:NO];
+        [fullScreenWindow disableCursorRects];
         [fullScreenWindow setAcceptsMouseMovedEvents: YES];
         [fullScreenWindow setHasShadow:NO];
         [fullScreenWindow setBackgroundColor: [NSColor blackColor]];
@@ -812,6 +818,7 @@ - (bool) handleEventLocked:(NSEvent *)event
     int buttons = 0;
     int keycode = 0;
     bool mouse_event = false;
+    bool mousemoved_event = false;
     // Location of event in virtual screen coordinates
     NSPoint p = [self screenLocationOfEvent:event];
     NSUInteger modifiers = [event modifierFlags];
@@ -1023,6 +1030,7 @@ - (bool) handleEventLocked:(NSEvent *)event
                 }
             }
             mouse_event = true;
+            mousemoved_event = true;
             break;
         case NSEventTypeLeftMouseDown:
             buttons |= MOUSE_EVENT_LBUTTON;
@@ -1039,14 +1047,17 @@ - (bool) handleEventLocked:(NSEvent *)event
         case NSEventTypeLeftMouseDragged:
             buttons |= MOUSE_EVENT_LBUTTON;
             mouse_event = true;
+            mousemoved_event = true;
             break;
         case NSEventTypeRightMouseDragged:
             buttons |= MOUSE_EVENT_RBUTTON;
             mouse_event = true;
+            mousemoved_event = true;
             break;
         case NSEventTypeOtherMouseDragged:
             buttons |= MOUSE_EVENT_MBUTTON;
             mouse_event = true;
+            mousemoved_event = true;
             break;
         case NSEventTypeLeftMouseUp:
             mouse_event = true;
@@ -1121,7 +1132,12 @@ - (bool) handleEventLocked:(NSEvent *)event
             qemu_input_update_buttons(dcl.con, bmap, last_buttons, buttons);
             last_buttons = buttons;
         }
-        if (isMouseGrabbed) {
+
+        if (!isMouseGrabbed) {
+            return false;
+        }
+
+        if (mousemoved_event) {
             if (isAbsoluteEnabled) {
                 /* Note that the origin for Cocoa mouse coords is bottom left, 
not top left.
                  * The check on screenContainsPoint is to avoid sending out of 
range values for
@@ -1132,11 +1148,38 @@ - (bool) handleEventLocked:(NSEvent *)event
                     qemu_input_queue_abs(dcl.con, INPUT_AXIS_Y, screen.height 
- p.y, 0, screen.height);
                 }
             } else {
-                qemu_input_queue_rel(dcl.con, INPUT_AXIS_X, (int)[event 
deltaX]);
-                qemu_input_queue_rel(dcl.con, INPUT_AXIS_Y, (int)[event 
deltaY]);
+                if (ignoreNextMouseMove) {
+                    // Discard the first mouse-move event after a grab, because
+                    // it includes the warp delta from an unknown initial 
position.
+                    ignoreNextMouseMove = NO;
+                    warpDeltaX = warpDeltaY = 0;
+                } else {
+                    // Correct subsequent events to remove the known warp 
delta.
+                    // The warp delta is sometimes late to be reported, so 
never
+                    // allow the delta compensation to alter the direction.
+                    int dX = (int)[event deltaX];
+                    int dY = (int)[event deltaY];
+
+                    if (dX == 0 || (dX ^ (dX - warpDeltaX)) < 0) { // Flipped 
sign?
+                        warpDeltaX -= dX; // Save excess correction for later
+                        dX = 0;
+                    } else {
+                        dX -= warpDeltaX; // Apply entire correction
+                        warpDeltaX = 0;
+                    }
+
+                    if (dY == 0 || (dY ^ (dY - warpDeltaY)) < 0) {
+                        warpDeltaY -= dY;
+                        dY = 0;
+                    } else {
+                        dY -= warpDeltaY;
+                        warpDeltaY = 0;
+                    }
+
+                    qemu_input_queue_rel(dcl.con, INPUT_AXIS_X, dX);
+                    qemu_input_queue_rel(dcl.con, INPUT_AXIS_Y, dY);
+                }
             }
-        } else {
-            return false;
         }
         qemu_input_event_sync();
     }
@@ -1153,9 +1196,15 @@ - (void) grabMouse
         else
             [normalWindow setTitle:@"QEMU - (Press ctrl + alt + g to release 
Mouse)"];
     }
-    [self hideCursor];
     CGAssociateMouseAndMouseCursorPosition(isAbsoluteEnabled);
     isMouseGrabbed = TRUE; // while isMouseGrabbed = TRUE, QemuCocoaApp sends 
all events to [cocoaView handleEvent:]
+    [self setCursorAppearance];
+    [self setCursorPosition];
+
+    // We took over and warped the mouse, so ignore the next mouse-move
+    if (!isAbsoluteEnabled) {
+        ignoreNextMouseMove = YES;
+    }
 }
 
 - (void) ungrabMouse
@@ -1168,9 +1217,14 @@ - (void) ungrabMouse
         else
             [normalWindow setTitle:@"QEMU"];
     }
-    [self unhideCursor];
     CGAssociateMouseAndMouseCursorPosition(TRUE);
     isMouseGrabbed = FALSE;
+    [self setCursorAppearance];
+
+    if (!isAbsoluteEnabled) {
+        ignoreNextMouseMove = NO;
+        warpDeltaX = warpDeltaY = 0;
+    }
 }
 
 - (void) setAbsoluteEnabled:(BOOL)tIsAbsoluteEnabled {
@@ -1179,6 +1233,116 @@ - (void) setAbsoluteEnabled:(BOOL)tIsAbsoluteEnabled {
         CGAssociateMouseAndMouseCursorPosition(isAbsoluteEnabled);
     }
 }
+
+// Indirectly called by dpy_cursor_define() in the virtual GPU
+- (void) cursorDefine:(NSCursor *)cursor {
+    guestCursor = cursor;
+    [self setCursorAppearance];
+}
+
+// Indirectly called by dpy_mouse_set() in the virtual GPU
+- (void) mouseSetX:(int)x Y:(int)y on:(int)on {
+    if (!on != !guestCursorVis) {
+        guestCursorVis = on;
+        [self setCursorAppearance];
+    }
+
+    if (on && (x != guestCursorX || y != guestCursorY)) {
+        guestCursorX = x;
+        guestCursorY = y;
+        [self setCursorPosition];
+    }
+}
+
+// Change the cursor image to the default, the guest cursor bitmap or hidden.
+// Said to be an expensive operation on macOS Monterey, so use sparingly.
+- (void) setCursorAppearance {
+    NSCursor *cursor = NULL; // NULL means hidden
+
+    if (!isMouseGrabbed) {
+        cursor = [NSCursor arrowCursor];
+    } else if (!guestCursor && !cursor_hide) {
+        cursor = [NSCursor arrowCursor];
+    } else if (guestCursorVis && guestCursor) {
+        cursor = guestCursor;
+    } else {
+        cursor = NULL;
+    }
+
+    if (cursor != NULL) {
+        [cursor set];
+
+        if (cursorHiddenByMe) {
+            [NSCursor unhide];
+            cursorHiddenByMe = NO;
+        }
+    } else {
+        if (!cursorHiddenByMe) {
+            [NSCursor hide];
+            cursorHiddenByMe = YES;
+        }
+    }
+}
+
+// Move the cursor within the virtual screen
+- (void) setCursorPosition {
+    // Ignore the guest's request if the cursor belongs to Cocoa
+    if (!isMouseGrabbed || isAbsoluteEnabled) {
+        return;
+    }
+
+    // Get guest screen rect in Cocoa coordinates (bottom-left origin).
+    NSRect virtualScreen = [[self window] convertRectToScreen:[self frame]];
+
+    // Convert to top-left origin.
+    NSInteger hostScreenH = [NSScreen screens][0].frame.size.height;
+    int scrX = virtualScreen.origin.x;
+    int scrY = hostScreenH - virtualScreen.origin.y - 
virtualScreen.size.height;
+    int scrW = virtualScreen.size.width;
+    int scrH = virtualScreen.size.height;
+
+    int cursX = scrX + guestCursorX;
+    int cursY = scrY + guestCursorY;
+
+    // Clip to edges
+    cursX = MIN(MAX(scrX, cursX), scrX + scrW - 1);
+    cursY = MIN(MAX(scrY, cursY), scrY + scrH - 1);
+
+    // Move diagonally towards the center to avoid rounded window corners.
+    // Limit the number of hit-tests and discard failed attempts.
+    int betterX = cursX, betterY = cursY;
+    for (int i=0; i<16; i++) {
+        if ([NSWindow windowNumberAtPoint:NSMakePoint(betterX, hostScreenH - 
betterY)
+            belowWindowWithWindowNumber:0] == self.window.windowNumber) {
+            cursX = betterX;
+            cursY = betterY;
+            break;
+        };
+
+        if (betterX < scrX + scrW/2) {
+            betterX++;
+        } else {
+            betterX--;
+        }
+
+        if (betterY < scrY + scrH/2) {
+            betterY++;
+        } else {
+            betterY--;
+        }
+    }
+
+    // Subtract this warp delta from the next NSEventTypeMouseMoved.
+    // These are in down-is-positive coords, same as NSEvent deltaX/deltaY.
+    warpDeltaX += cursX - lastWarpX;
+    warpDeltaY += cursY - lastWarpY;
+
+    CGWarpMouseCursorPosition(NSMakePoint(cursX, cursY));
+
+    lastWarpX = cursX;
+    lastWarpY = cursY;
+}
+
 - (BOOL) isMouseGrabbed {return isMouseGrabbed;}
 - (BOOL) isAbsoluteEnabled {return isAbsoluteEnabled;}
 - (float) cdx {return cdx;}
@@ -1251,6 +1415,7 @@ - (id) init
             error_report("(cocoa) can't create window");
             exit(1);
         }
+        [normalWindow disableCursorRects];
         [normalWindow setAcceptsMouseMovedEvents:YES];
         [normalWindow setTitle:@"QEMU"];
         [normalWindow setContentView:cocoaView];
@@ -2123,6 +2288,58 @@ static void cocoa_display_init(DisplayState *ds, 
DisplayOptions *opts)
     qemu_clipboard_peer_register(&cbpeer);
 }
 
+static void cocoa_mouse_set(DisplayChangeListener *dcl, int x, int y, int on) {
+    dispatch_async(dispatch_get_main_queue(), ^{
+        [cocoaView mouseSetX:x Y:y on:on];
+    });
+}
+
+// Convert QEMUCursor to NSCursor, then call cursorDefine
+static void cocoa_cursor_define(DisplayChangeListener *dcl, QEMUCursor 
*cursor) {
+    CFDataRef cfdata = CFDataCreate(
+        /*allocator*/ NULL,
+        /*bytes*/ (void *)cursor->data,
+        /*length*/ sizeof(uint32_t) * cursor->width * cursor->height);
+
+    CGDataProviderRef dataprovider = CGDataProviderCreateWithCFData(cfdata);
+
+    CGImageRef cgimage = CGImageCreate(
+        cursor->width, cursor->height,
+        /*bitsPerComponent*/ 8,
+        /*bitsPerPixel*/ 32,
+        /*bytesPerRow*/ sizeof(uint32_t) * cursor->width,
+        /*colorspace*/ CGColorSpaceCreateWithName(kCGColorSpaceSRGB),
+        /*bitmapInfo*/ kCGBitmapByteOrder32Host | kCGImageAlphaLast,
+        /*provider*/ dataprovider,
+        /*decode*/ NULL,
+        /*shouldInterpolate*/ FALSE,
+        /*intent*/ kCGRenderingIntentDefault);
+
+    NSImage *unscaled = [[NSImage alloc] initWithCGImage:cgimage 
size:NSZeroSize];
+
+    CFRelease(cfdata);
+    CGDataProviderRelease(dataprovider);
+    CGImageRelease(cgimage);
+
+    // Nearest-neighbor scale to the possibly "Retina" cursor size
+    NSImage *scaled = [NSImage
+        imageWithSize:NSMakeSize(cursor->width, cursor->height)
+        flipped:NO
+        drawingHandler:^BOOL(NSRect dest) {
+            [NSGraphicsContext currentContext].imageInterpolation = 
NSImageInterpolationNone;
+            [unscaled drawInRect:dest];
+            return YES;
+        }];
+
+    NSCursor *nscursor = [[NSCursor alloc]
+        initWithImage:scaled
+        hotSpot:NSMakePoint(cursor->hot_x, cursor->hot_y)];
+
+    dispatch_async(dispatch_get_main_queue(), ^{
+        [cocoaView cursorDefine:nscursor];
+    });
+}
+
 static QemuDisplay qemu_display_cocoa = {
     .type       = DISPLAY_TYPE_COCOA,
     .init       = cocoa_display_init,