Re: [PATCH] ui/cocoa: Support hardware cursor interface

[Elliot Nunn]

Resending this patch now that 7.1 is released (well done).

> On 4 Aug 2022, at 2:27 pm, Elliot Nunn <elliot@nunn.io> wrote:
> 
> Implement dpy_cursor_define() and dpy_mouse_set() on macOS.
> 
> The main benefit is from dpy_cursor_define: in absolute pointing mode, the
> host can redraw the cursor on the guest's behalf much faster than the guest
> can itself.
> 
> To provide the programmatic movement expected from a hardware cursor,
> dpy_mouse_set is also implemented.
> 
> Tricky cases are handled:
> - dpy_mouse_set() avoids rounded window corners.
> - The sometimes-delay between warping the cursor and an affected mouse-move
>  event is accounted for.
> - Cursor bitmaps are nearest-neighbor scaled to Retina size.
> 
> Signed-off-by: Elliot Nunn <elliot@nunn.io>
> ---
> ui/cocoa.m | 263 ++++++++++++++++++++++++++++++++++++++++++++++++-----
> 1 file changed, 240 insertions(+), 23 deletions(-)
> 
> diff --git a/ui/cocoa.m b/ui/cocoa.m
> index 5a8bd5dd84..f9d54448e4 100644
> --- a/ui/cocoa.m
> +++ b/ui/cocoa.m
> @@ -85,12 +85,20 @@ static void cocoa_switch(DisplayChangeListener *dcl,
> 
> static void cocoa_refresh(DisplayChangeListener *dcl);
> 
> +static void cocoa_mouse_set(DisplayChangeListener *dcl,
> +                            int x, int y, int on);
> +
> +static void cocoa_cursor_define(DisplayChangeListener *dcl,
> +                                QEMUCursor *c);
> +
> static NSWindow *normalWindow;
> static const DisplayChangeListenerOps dcl_ops = {
>     .dpy_name          = "cocoa",
>     .dpy_gfx_update = cocoa_update,
>     .dpy_gfx_switch = cocoa_switch,
>     .dpy_refresh = cocoa_refresh,
> +    .dpy_mouse_set = cocoa_mouse_set,
> +    .dpy_cursor_define = cocoa_cursor_define,
> };
> static DisplayChangeListener dcl = {
>     .ops = &dcl_ops,
> @@ -313,6 +321,13 @@ @interface QemuCocoaView : NSView
>     BOOL isFullscreen;
>     BOOL isAbsoluteEnabled;
>     CFMachPortRef eventsTap;
> +    NSCursor *guestCursor;
> +    BOOL cursorHiddenByMe;
> +    BOOL guestCursorVis;
> +    int guestCursorX, guestCursorY;
> +    int lastWarpX, lastWarpY;
> +    int warpDeltaX, warpDeltaY;
> +    BOOL ignoreNextMouseMove;
> }
> - (void) switchSurface:(pixman_image_t *)image;
> - (void) grabMouse;
> @@ -323,6 +338,10 @@ - (void) handleMonitorInput:(NSEvent *)event;
> - (bool) handleEvent:(NSEvent *)event;
> - (bool) handleEventLocked:(NSEvent *)event;
> - (void) setAbsoluteEnabled:(BOOL)tIsAbsoluteEnabled;
> +- (void) cursorDefine:(NSCursor *)cursor;
> +- (void) mouseSetX:(int)x Y:(int)y on:(int)on;
> +- (void) setCursorAppearance;
> +- (void) setCursorPosition;
> /* The state surrounding mouse grabbing is potentially confusing.
>  * isAbsoluteEnabled tracks qemu_input_is_absolute() [ie "is the emulated
>  *   pointing device an absolute-position one?"], but is only updated on
> @@ -432,22 +451,6 @@ - (CGPoint) screenLocationOfEvent:(NSEvent *)ev
>     }
> }
> 
> -- (void) hideCursor
> -{
> -    if (!cursor_hide) {
> -        return;
> -    }
> -    [NSCursor hide];
> -}
> -
> -- (void) unhideCursor
> -{
> -    if (!cursor_hide) {
> -        return;
> -    }
> -    [NSCursor unhide];
> -}
> -
> - (void) drawRect:(NSRect) rect
> {
>     COCOA_DEBUG("QemuCocoaView: drawRect\n");
> @@ -635,6 +638,8 @@ - (void) switchSurface:(pixman_image_t *)image
>         screen.height = h;
>         [self setContentDimensions];
>         [self setFrame:NSMakeRect(cx, cy, cw, ch)];
> +        [self setCursorAppearance];
> +        [self setCursorPosition];
>     }
> 
>     // update screenBuffer
> @@ -681,6 +686,7 @@ - (void) toggleFullScreen:(id)sender
>             styleMask:NSWindowStyleMaskBorderless
>             backing:NSBackingStoreBuffered
>             defer:NO];
> +        [fullScreenWindow disableCursorRects];
>         [fullScreenWindow setAcceptsMouseMovedEvents: YES];
>         [fullScreenWindow setHasShadow:NO];
>         [fullScreenWindow setBackgroundColor: [NSColor blackColor]];
> @@ -812,6 +818,7 @@ - (bool) handleEventLocked:(NSEvent *)event
>     int buttons = 0;
>     int keycode = 0;
>     bool mouse_event = false;
> +    bool mousemoved_event = false;
>     // Location of event in virtual screen coordinates
>     NSPoint p = [self screenLocationOfEvent:event];
>     NSUInteger modifiers = [event modifierFlags];
> @@ -1023,6 +1030,7 @@ - (bool) handleEventLocked:(NSEvent *)event
>                 }
>             }
>             mouse_event = true;
> +            mousemoved_event = true;
>             break;
>         case NSEventTypeLeftMouseDown:
>             buttons |= MOUSE_EVENT_LBUTTON;
> @@ -1039,14 +1047,17 @@ - (bool) handleEventLocked:(NSEvent *)event
>         case NSEventTypeLeftMouseDragged:
>             buttons |= MOUSE_EVENT_LBUTTON;
>             mouse_event = true;
> +            mousemoved_event = true;
>             break;
>         case NSEventTypeRightMouseDragged:
>             buttons |= MOUSE_EVENT_RBUTTON;
>             mouse_event = true;
> +            mousemoved_event = true;
>             break;
>         case NSEventTypeOtherMouseDragged:
>             buttons |= MOUSE_EVENT_MBUTTON;
>             mouse_event = true;
> +            mousemoved_event = true;
>             break;
>         case NSEventTypeLeftMouseUp:
>             mouse_event = true;
> @@ -1121,7 +1132,12 @@ - (bool) handleEventLocked:(NSEvent *)event
>             qemu_input_update_buttons(dcl.con, bmap, last_buttons, buttons);
>             last_buttons = buttons;
>         }
> -        if (isMouseGrabbed) {
> +
> +        if (!isMouseGrabbed) {
> +            return false;
> +        }
> +
> +        if (mousemoved_event) {
>             if (isAbsoluteEnabled) {
>                 /* Note that the origin for Cocoa mouse coords is bottom 
> left, not top left.
>                  * The check on screenContainsPoint is to avoid sending out 
> of range values for
> @@ -1132,11 +1148,38 @@ - (bool) handleEventLocked:(NSEvent *)event
>                     qemu_input_queue_abs(dcl.con, INPUT_AXIS_Y, screen.height 
> - p.y, 0, screen.height);
>                 }
>             } else {
> -                qemu_input_queue_rel(dcl.con, INPUT_AXIS_X, (int)[event 
> deltaX]);
> -                qemu_input_queue_rel(dcl.con, INPUT_AXIS_Y, (int)[event 
> deltaY]);
> +                if (ignoreNextMouseMove) {
> +                    // Discard the first mouse-move event after a grab, 
> because
> +                    // it includes the warp delta from an unknown initial 
> position.
> +                    ignoreNextMouseMove = NO;
> +                    warpDeltaX = warpDeltaY = 0;
> +                } else {
> +                    // Correct subsequent events to remove the known warp 
> delta.
> +                    // The warp delta is sometimes late to be reported, so 
> never
> +                    // allow the delta compensation to alter the direction.
> +                    int dX = (int)[event deltaX];
> +                    int dY = (int)[event deltaY];
> +
> +                    if (dX == 0 || (dX ^ (dX - warpDeltaX)) < 0) { // 
> Flipped sign?
> +                        warpDeltaX -= dX; // Save excess correction for later
> +                        dX = 0;
> +                    } else {
> +                        dX -= warpDeltaX; // Apply entire correction
> +                        warpDeltaX = 0;
> +                    }
> +
> +                    if (dY == 0 || (dY ^ (dY - warpDeltaY)) < 0) {
> +                        warpDeltaY -= dY;
> +                        dY = 0;
> +                    } else {
> +                        dY -= warpDeltaY;
> +                        warpDeltaY = 0;
> +                    }
> +
> +                    qemu_input_queue_rel(dcl.con, INPUT_AXIS_X, dX);
> +                    qemu_input_queue_rel(dcl.con, INPUT_AXIS_Y, dY);
> +                }
>             }
> -        } else {
> -            return false;
>         }
>         qemu_input_event_sync();
>     }
> @@ -1153,9 +1196,15 @@ - (void) grabMouse
>         else
>             [normalWindow setTitle:@"QEMU - (Press ctrl + alt + g to release 
> Mouse)"];
>     }
> -    [self hideCursor];
>     CGAssociateMouseAndMouseCursorPosition(isAbsoluteEnabled);
>     isMouseGrabbed = TRUE; // while isMouseGrabbed = TRUE, QemuCocoaApp sends 
> all events to [cocoaView handleEvent:]
> +    [self setCursorAppearance];
> +    [self setCursorPosition];
> +
> +    // We took over and warped the mouse, so ignore the next mouse-move
> +    if (!isAbsoluteEnabled) {
> +        ignoreNextMouseMove = YES;
> +    }
> }
> 
> - (void) ungrabMouse
> @@ -1168,9 +1217,14 @@ - (void) ungrabMouse
>         else
>             [normalWindow setTitle:@"QEMU"];
>     }
> -    [self unhideCursor];
>     CGAssociateMouseAndMouseCursorPosition(TRUE);
>     isMouseGrabbed = FALSE;
> +    [self setCursorAppearance];
> +
> +    if (!isAbsoluteEnabled) {
> +        ignoreNextMouseMove = NO;
> +        warpDeltaX = warpDeltaY = 0;
> +    }
> }
> 
> - (void) setAbsoluteEnabled:(BOOL)tIsAbsoluteEnabled {
> @@ -1179,6 +1233,116 @@ - (void) setAbsoluteEnabled:(BOOL)tIsAbsoluteEnabled {
>         CGAssociateMouseAndMouseCursorPosition(isAbsoluteEnabled);
>     }
> }
> +
> +// Indirectly called by dpy_cursor_define() in the virtual GPU
> +- (void) cursorDefine:(NSCursor *)cursor {
> +    guestCursor = cursor;
> +    [self setCursorAppearance];
> +}
> +
> +// Indirectly called by dpy_mouse_set() in the virtual GPU
> +- (void) mouseSetX:(int)x Y:(int)y on:(int)on {
> +    if (!on != !guestCursorVis) {
> +        guestCursorVis = on;
> +        [self setCursorAppearance];
> +    }
> +
> +    if (on && (x != guestCursorX || y != guestCursorY)) {
> +        guestCursorX = x;
> +        guestCursorY = y;
> +        [self setCursorPosition];
> +    }
> +}
> +
> +// Change the cursor image to the default, the guest cursor bitmap or hidden.
> +// Said to be an expensive operation on macOS Monterey, so use sparingly.
> +- (void) setCursorAppearance {
> +    NSCursor *cursor = NULL; // NULL means hidden
> +
> +    if (!isMouseGrabbed) {
> +        cursor = [NSCursor arrowCursor];
> +    } else if (!guestCursor && !cursor_hide) {
> +        cursor = [NSCursor arrowCursor];
> +    } else if (guestCursorVis && guestCursor) {
> +        cursor = guestCursor;
> +    } else {
> +        cursor = NULL;
> +    }
> +
> +    if (cursor != NULL) {
> +        [cursor set];
> +
> +        if (cursorHiddenByMe) {
> +            [NSCursor unhide];
> +            cursorHiddenByMe = NO;
> +        }
> +    } else {
> +        if (!cursorHiddenByMe) {
> +            [NSCursor hide];
> +            cursorHiddenByMe = YES;
> +        }
> +    }
> +}
> +
> +// Move the cursor within the virtual screen
> +- (void) setCursorPosition {
> +    // Ignore the guest's request if the cursor belongs to Cocoa
> +    if (!isMouseGrabbed || isAbsoluteEnabled) {
> +        return;
> +    }
> +
> +    // Get guest screen rect in Cocoa coordinates (bottom-left origin).
> +    NSRect virtualScreen = [[self window] convertRectToScreen:[self frame]];
> +
> +    // Convert to top-left origin.
> +    NSInteger hostScreenH = [NSScreen screens][0].frame.size.height;
> +    int scrX = virtualScreen.origin.x;
> +    int scrY = hostScreenH - virtualScreen.origin.y - 
> virtualScreen.size.height;
> +    int scrW = virtualScreen.size.width;
> +    int scrH = virtualScreen.size.height;
> +
> +    int cursX = scrX + guestCursorX;
> +    int cursY = scrY + guestCursorY;
> +
> +    // Clip to edges
> +    cursX = MIN(MAX(scrX, cursX), scrX + scrW - 1);
> +    cursY = MIN(MAX(scrY, cursY), scrY + scrH - 1);
> +
> +    // Move diagonally towards the center to avoid rounded window corners.
> +    // Limit the number of hit-tests and discard failed attempts.
> +    int betterX = cursX, betterY = cursY;
> +    for (int i=0; i<16; i++) {
> +        if ([NSWindow windowNumberAtPoint:NSMakePoint(betterX, hostScreenH - 
> betterY)
> +            belowWindowWithWindowNumber:0] == self.window.windowNumber) {
> +            cursX = betterX;
> +            cursY = betterY;
> +            break;
> +        };
> +
> +        if (betterX < scrX + scrW/2) {
> +            betterX++;
> +        } else {
> +            betterX--;
> +        }
> +
> +        if (betterY < scrY + scrH/2) {
> +            betterY++;
> +        } else {
> +            betterY--;
> +        }
> +    }
> +
> +    // Subtract this warp delta from the next NSEventTypeMouseMoved.
> +    // These are in down-is-positive coords, same as NSEvent deltaX/deltaY.
> +    warpDeltaX += cursX - lastWarpX;
> +    warpDeltaY += cursY - lastWarpY;
> +
> +    CGWarpMouseCursorPosition(NSMakePoint(cursX, cursY));
> +
> +    lastWarpX = cursX;
> +    lastWarpY = cursY;
> +}
> +
> - (BOOL) isMouseGrabbed {return isMouseGrabbed;}
> - (BOOL) isAbsoluteEnabled {return isAbsoluteEnabled;}
> - (float) cdx {return cdx;}
> @@ -1251,6 +1415,7 @@ - (id) init
>             error_report("(cocoa) can't create window");
>             exit(1);
>         }
> +        [normalWindow disableCursorRects];
>         [normalWindow setAcceptsMouseMovedEvents:YES];
>         [normalWindow setTitle:@"QEMU"];
>         [normalWindow setContentView:cocoaView];
> @@ -2123,6 +2288,58 @@ static void cocoa_display_init(DisplayState *ds, 
> DisplayOptions *opts)
>     qemu_clipboard_peer_register(&cbpeer);
> }
> 
> +static void cocoa_mouse_set(DisplayChangeListener *dcl, int x, int y, int 
> on) {
> +    dispatch_async(dispatch_get_main_queue(), ^{
> +        [cocoaView mouseSetX:x Y:y on:on];
> +    });
> +}
> +
> +// Convert QEMUCursor to NSCursor, then call cursorDefine
> +static void cocoa_cursor_define(DisplayChangeListener *dcl, QEMUCursor 
> *cursor) {
> +    CFDataRef cfdata = CFDataCreate(
> +        /*allocator*/ NULL,
> +        /*bytes*/ (void *)cursor->data,
> +        /*length*/ sizeof(uint32_t) * cursor->width * cursor->height);
> +
> +    CGDataProviderRef dataprovider = CGDataProviderCreateWithCFData(cfdata);
> +
> +    CGImageRef cgimage = CGImageCreate(
> +        cursor->width, cursor->height,
> +        /*bitsPerComponent*/ 8,
> +        /*bitsPerPixel*/ 32,
> +        /*bytesPerRow*/ sizeof(uint32_t) * cursor->width,
> +        /*colorspace*/ CGColorSpaceCreateWithName(kCGColorSpaceSRGB),
> +        /*bitmapInfo*/ kCGBitmapByteOrder32Host | kCGImageAlphaLast,
> +        /*provider*/ dataprovider,
> +        /*decode*/ NULL,
> +        /*shouldInterpolate*/ FALSE,
> +        /*intent*/ kCGRenderingIntentDefault);
> +
> +    NSImage *unscaled = [[NSImage alloc] initWithCGImage:cgimage 
> size:NSZeroSize];
> +
> +    CFRelease(cfdata);
> +    CGDataProviderRelease(dataprovider);
> +    CGImageRelease(cgimage);
> +
> +    // Nearest-neighbor scale to the possibly "Retina" cursor size
> +    NSImage *scaled = [NSImage
> +        imageWithSize:NSMakeSize(cursor->width, cursor->height)
> +        flipped:NO
> +        drawingHandler:^BOOL(NSRect dest) {
> +            [NSGraphicsContext currentContext].imageInterpolation = 
> NSImageInterpolationNone;
> +            [unscaled drawInRect:dest];
> +            return YES;
> +        }];
> +
> +    NSCursor *nscursor = [[NSCursor alloc]
> +        initWithImage:scaled
> +        hotSpot:NSMakePoint(cursor->hot_x, cursor->hot_y)];
> +
> +    dispatch_async(dispatch_get_main_queue(), ^{
> +        [cocoaView cursorDefine:nscursor];
> +    });
> +}
> +
> static QemuDisplay qemu_display_cocoa = {
>     .type       = DISPLAY_TYPE_COCOA,
>     .init       = cocoa_display_init,