Unable to post synthetic mouse events to windows outside the application

I'm working on a toy Swift implementation of Teamviewer where users can collaborate. To achieve this, I'm creating a secondary, remotely controlled cursor on macOS using Cocoa. My goal is to allow this secondary cursor to manipulate windows and post mouse events below it. I've managed to create the cursor and successfully made it move and animate within the window.

However, I'm struggling with enabling mouse events to be fired by this secondary cursor. When I try to post synthetic mouse events, it doesn't seem to have any effect. Here's the relevant portion of my code:

    func click(at point: CGPoint) {
        guard let mouseDown = CGEvent(mouseEventSource: nil, mouseType: .leftMouseDown, mouseCursorPosition: point, mouseButton: .left),
              let mouseUp = CGEvent(mouseEventSource: nil, mouseType: .leftMouseUp, mouseCursorPosition: point, mouseButton: .left) else {
            return
        }
        mouseDown.post(tap: .cgSessionEventTap)
        mouseUp.post(tap: .cgSessionEventTap)
    }

I have enabled the Accessibility features, tried posting to specific PIDs, tried posting events twice in a row (to ensure it's not a focus issue), replaced .cgSessionEventTap with .cghidEventTap, all to no avail.

Here's the full file if you'd like more context:


import Cocoa
import Foundation

class CursorView: NSView {
    let image: NSImage

    init(image: NSImage) {
        self.image = image
        super.init(frame: .zero)
    }

    required init?(coder: NSCoder) {
        fatalError("init(coder:) has not been implemented")
    }

    override func draw(_ dirtyRect: NSRect) {
        super.draw(dirtyRect)
        image.draw(in: dirtyRect)
    }
}

@NSApplicationMain
class AppDelegate: NSObject, NSApplicationDelegate {

    var window: NSWindow!
    var userCursorView: CursorView?
    var remoteCursorView: CursorView?
    var timer: Timer?

    var destination: CGPoint = .zero
    var t: CGFloat = 0
    let duration: TimeInterval = 2
    let clickProbability: CGFloat = 0.01
    func applicationDidFinishLaunching(_ aNotification: Notification) {
        let screenRect = NSScreen.main!.frame
        window = NSWindow(contentRect: screenRect,
                          styleMask: .borderless,
                          backing: .buffered,
                          defer: false)
        window.level = NSWindow.Level(rawValue: Int(CGWindowLevelForKey(.maximumWindow)))
        window.backgroundColor = NSColor.clear
        window.ignoresMouseEvents = true

        let maxHeight: CGFloat = 70.0
        if let userImage = NSImage(named: "userCursorImage") {
            let aspectRatio = userImage.size.width / userImage.size.height
            let newWidth = aspectRatio * maxHeight
            userCursorView = CursorView(image: userImage)
            userCursorView!.frame.size = NSSize(width: newWidth, height: maxHeight)
            window.contentView?.addSubview(userCursorView!)
        }

        if let remoteImage = NSImage(named: "remoteCursorImage") {
            let aspectRatio = remoteImage.size.width / remoteImage.size.height
            let newWidth = aspectRatio * maxHeight
            remoteCursorView = CursorView(image: remoteImage)
            remoteCursorView!.frame.size = NSSize(width: newWidth, height: maxHeight)
            window.contentView?.addSubview(remoteCursorView!)
            // Initialize remote cursor position and destination
            remoteCursorView!.frame.origin = randomPointWithinScreen()
            destination = randomPointWithinScreen()
        }

        window.makeKeyAndOrderFront(nil)
        window.orderFrontRegardless()

        NSCursor.hide()

        NSEvent.addGlobalMonitorForEvents(matching: [.mouseMoved, .leftMouseDragged, .rightMouseDragged]) { [weak self] event in
            self?.updateCursorPosition(with: event)
        }

        // Move the remote cursor every 0.01 second
        timer = Timer.scheduledTimer(withTimeInterval: 0.01, repeats: true) { [weak self] _ in
            self?.moveRemoteCursor()
        }
        
        // Exit the app when pressing the escape key
        NSEvent.addGlobalMonitorForEvents(matching: .keyDown) { event in
            if event.keyCode == 53 {
                NSApplication.shared.terminate(self)
            }
        }
    }

    func updateCursorPosition(with event: NSEvent) {
        var newLocation = event.locationInWindow
        newLocation.y -= userCursorView!.frame.size.height
        userCursorView?.frame.origin = newLocation
    }

    func moveRemoteCursor() {
        if remoteCursorView!.frame.origin.distance(to: destination) < 1 || t >= 1 {
            destination = randomPointWithinScreen()
            t = 0
            let windowPoint = remoteCursorView!.frame.origin
            let screenPoint = window.convertToScreen(NSRect(origin: windowPoint, size: .zero)).origin
            let screenHeight = NSScreen.main?.frame.height ?? 0
            let cgScreenPoint = CGPoint(x: screenPoint.x, y: screenHeight - screenPoint.y)
            click(at: cgScreenPoint)
        } else {
            let newPosition = cubicBezier(t: t, start: remoteCursorView!.frame.origin, end: destination)
            remoteCursorView?.frame.origin = newPosition
            t += CGFloat(0.01 / duration)
        }
    }

    func click(at point: CGPoint) {
        guard let mouseDown = CGEvent(mouseEventSource: nil, mouseType: .leftMouseDown, mouseCursorPosition: point, mouseButton: .left),
              let mouseUp = CGEvent(mouseEventSource: nil, mouseType: .leftMouseUp, mouseCursorPosition: point, mouseButton: .left) else {
            return
        }
        // Post the events to the session event tap
        mouseDown.post(tap: .cgSessionEventTap)
        mouseUp.post(tap: .cgSessionEventTap)
    }
    
    func randomPointWithinScreen() -> CGPoint {
        guard let screen = NSScreen.main else { return .zero }
        let randomX = CGFloat.random(in: 0...screen.frame.width / 2)
        let randomY = CGFloat.random(in: 100...screen.frame.height)
        return CGPoint(x: randomX, y: randomY)
    }

    func cubicBezier(t: CGFloat, start: CGPoint, end: CGPoint) -> CGPoint {
        let control1 = CGPoint(x: 2 * start.x / 3 + end.x / 3, y: start.y)
        let control2 = CGPoint(x: start.x / 3 + 2 * end.x / 3, y: end.y)
        let x = pow(1 - t, 3) * start.x + 3 * pow(1 - t, 2) * t * control1.x + 3 * (1 - t) * pow(t, 2) * control2.x + pow(t, 3) * end.x
        let y = pow(1 - t, 3) * start.y + 3 * pow(1 - t, 2) * t * control1.y + 3 * (1 - t) * pow(t, 2) * control2.y + pow(t, 3) * end.y
                return CGPoint(x: x, y: y)
    }

    func applicationWillTerminate(_ aNotification: Notification) {
        // Show the system cursor when the application is about to terminate
        NSCursor.unhide()
    }
}

extension CGPoint {
    func distance(to point: CGPoint) -> CGFloat {
        return hypot(point.x - x, point.y - y)
    }
}

Replies

You don’t need the Accessibility privilege to post mouse events. There’s a separate privilege, Post Event. You can test for that privilege using CGPreflightPostEventAccess and request it using CGRequestPostEventAccess. And on the good news front, this is actually compatible with the App Sandbox.

IMPORTANT While that’s true, and cool, I strongly recommend that disable the App Sandbox while bringing up this feature. Once you get it working, you can then decide whether to re-enable the App Sandbox for your final product.

As to why your event posting isn’t working, I don’t know for sure. Here’s some code that I use for this:

func start(_ setStatus: @escaping (String) -> Void) {
    precondition(self.poster == nil)

    os_log(.debug, log: self.log, "will create poster")
    let poster = Timer.scheduledTimer(withTimeInterval: 5.0, repeats: true) { _ in
        os_log(.debug, log: self.log, "will post A down")
        let down = CGEvent(keyboardEventSource: nil, virtualKey: CGKeyCode(kVK_ANSI_A), keyDown: true)!
        down.post(tap: .cgSessionEventTap)
        Timer.scheduledTimer(withTimeInterval: 0.1, repeats: false) { _ in
            os_log(.debug, log: self.log, "will post A up")
            let up = CGEvent(keyboardEventSource: nil, virtualKey: CGKeyCode(kVK_ANSI_A), keyDown: false)!
            up.post(tap: .cgSessionEventTap)
        }
    }
    self.poster = poster
    os_log(.debug, log: self.log, "did create poster")
}

It posts keyboard events but the process is the same.

Share and Enjoy

Quinn “The Eskimo!” @ Developer Technical Support @ Apple
let myEmail = "eskimo" + "1" + "@" + "apple.com"