| 1 | import AppKit |
| 2 | import ApplicationServices |
| 3 | import ImageIO |
| 4 | import ScreenCaptureKit |
| 5 | import UniformTypeIdentifiers |
| 6 | |
| 7 | func output(_ value: Any) { |
| 8 | let data = try! JSONSerialization.data(withJSONObject: value, options: [.sortedKeys]) |
| 9 | print(String(data: data, encoding: .utf8)!) |
| 10 | } |
| 11 | |
| 12 | func fail(_ message: String) -> Never { |
| 13 | output(["error": message]) |
| 14 | exit(1) |
| 15 | } |
| 16 | |
| 17 | func attribute(_ element: AXUIElement, _ name: String) -> CFTypeRef? { |
| 18 | var value: CFTypeRef? |
| 19 | return AXUIElementCopyAttributeValue(element, name as CFString, &value) == .success ? value : nil |
| 20 | } |
| 21 | |
| 22 | func tree(_ element: AXUIElement, path: [Int], remaining: inout Int) -> [String: Any] { |
| 23 | remaining -= 1 |
| 24 | var pid: pid_t = 0 |
| 25 | AXUIElementGetPid(element, &pid) |
| 26 | var result: [String: Any] = ["path": path, "target": ["pid": pid, "path": path, "role": attribute(element, "AXRole") as? String ?? "", "name": attribute(element, "AXTitle") as? String ?? ""]] |
| 27 | for name in ["AXRole", "AXSubrole", "AXTitle", "AXDescription", "AXValue", "AXEnabled", "AXFocused"] { |
| 28 | if let value = attribute(element, name), value is String || value is NSNumber { |
| 29 | result[name] = value |
| 30 | } |
| 31 | } |
| 32 | for name in ["AXPosition", "AXSize"] { |
| 33 | if let value = attribute(element, name), CFGetTypeID(value) == AXValueGetTypeID() { |
| 34 | let axValue = value as! AXValue |
| 35 | if AXValueGetType(axValue) == .cgPoint { |
| 36 | var point = CGPoint.zero |
| 37 | AXValueGetValue(axValue, .cgPoint, &point) |
| 38 | result[name] = [point.x, point.y] |
| 39 | } else if AXValueGetType(axValue) == .cgSize { |
| 40 | var size = CGSize.zero |
| 41 | AXValueGetValue(axValue, .cgSize, &size) |
| 42 | result[name] = [size.width, size.height] |
| 43 | } |
| 44 | } |
| 45 | } |
| 46 | var actions: CFArray? |
| 47 | if AXUIElementCopyActionNames(element, &actions) == .success { result["actions"] = actions } |
| 48 | var settable = DarwinBoolean(false) |
| 49 | if AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &settable) == .success, settable.boolValue { |
| 50 | result["actions"] = (result["actions"] as? [String] ?? []) + ["set_value"] |
| 51 | } |
| 52 | if path.count < 16, let children = attribute(element, "AXChildren") as? [AXUIElement] { |
| 53 | var nodes: [[String: Any]] = [] |
| 54 | for (index, child) in children.enumerated() where remaining > 0 { |
| 55 | nodes.append(tree(child, path: path + [index], remaining: &remaining)) |
| 56 | } |
| 57 | result["children"] = nodes |
| 58 | if nodes.count < children.count { result["truncated"] = true } |
| 59 | } |
| 60 | return result |
| 61 | } |
| 62 | |
| 63 | func typeText(_ text: String, _ eventSource: CGEventSource) { |
| 64 | let units = Array(text.utf16) |
| 65 | var start = 0 |
| 66 | while start < units.count { |
| 67 | var end = min(start + 20, units.count) |
| 68 | if end < units.count && (0xD800...0xDBFF).contains(units[end - 1]) { end -= 1 } |
| 69 | let chunk = Array(units[start..<end]) |
| 70 | for down in [true, false] { |
| 71 | let event = CGEvent(keyboardEventSource: eventSource, virtualKey: 0, keyDown: down) |
| 72 | event?.flags = [] |
| 73 | event?.keyboardSetUnicodeString(stringLength: chunk.count, unicodeString: chunk) |
| 74 | event?.post(tap: .cghidEventTap) |
| 75 | usleep(10000) |
| 76 | } |
| 77 | start = end |
| 78 | } |
| 79 | usleep(50000) |
| 80 | } |
| 81 | |
| 82 | let arguments = Array(CommandLine.arguments.dropFirst()) |
| 83 | guard let command = arguments.first else { fail("Use status, request-permissions, screenshot, tree, press, click, key, or type.") } |
| 84 | guard let eventSource = CGEventSource(stateID: .privateState) else { fail("Could not create a private input event source.") } |
| 85 | switch command { |
| 86 | case "status", "request-permissions": |
| 87 | if command == "request-permissions" { |
| 88 | _ = AXIsProcessTrustedWithOptions([kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] as CFDictionary) |
| 89 | _ = CGRequestScreenCaptureAccess() |
| 90 | _ = CGRequestPostEventAccess() |
| 91 | } |
| 92 | output(["accessibility": AXIsProcessTrusted(), "screenCapture": CGPreflightScreenCaptureAccess(), "postEvent": CGPreflightPostEventAccess()]) |
| 93 | case "screenshot": |
| 94 | guard arguments.count == 2 else { fail("Provide an output PNG path.") } |
| 95 | let image: CGImage |
| 96 | do { |
| 97 | let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true) |
| 98 | guard let display = content.displays.first(where: { $0.displayID == CGMainDisplayID() }) else { fail("No active display.") } |
| 99 | let configuration = SCStreamConfiguration() |
| 100 | configuration.width = display.width |
| 101 | configuration.height = display.height |
| 102 | configuration.showsCursor = false |
| 103 | image = try await SCScreenshotManager.captureImage(contentFilter: SCContentFilter(display: display, excludingWindows: []), configuration: configuration) |
| 104 | } catch { fail("Screen capture failed: \(error.localizedDescription)") } |
| 105 | guard let destination = CGImageDestinationCreateWithURL(URL(fileURLWithPath: arguments[1]) as CFURL, UTType.png.identifier as CFString, 1, nil) |
| 106 | else { fail("Could not create the output PNG.") } |
| 107 | CGImageDestinationAddImage(destination, image, nil) |
| 108 | guard CGImageDestinationFinalize(destination) else { fail("Could not write the output PNG.") } |
| 109 | let bounds = CGDisplayBounds(CGMainDisplayID()) |
| 110 | output(["width": image.width, "height": image.height, "path": arguments[1], |
| 111 | "bounds": [bounds.origin.x, bounds.origin.y, bounds.width, bounds.height], |
| 112 | "scale": [Double(image.width) / bounds.width, Double(image.height) / bounds.height]]) |
| 113 | case "tree", "press": |
| 114 | guard AXIsProcessTrusted() else { fail("Accessibility permission is required.") } |
| 115 | let app: AXUIElement |
| 116 | if arguments.count > 1 { |
| 117 | guard let pid = Int32(arguments[1]), pid > 0 else { fail("Provide a positive application PID.") } |
| 118 | app = AXUIElementCreateApplication(pid) |
| 119 | } else { |
| 120 | guard let front = NSWorkspace.shared.frontmostApplication else { fail("No active application.") } |
| 121 | app = AXUIElementCreateApplication(front.processIdentifier) |
| 122 | } |
| 123 | if command == "tree" { |
| 124 | var remaining = 5000 |
| 125 | output(tree(app, path: [], remaining: &remaining)) |
| 126 | } else { |
| 127 | guard arguments.count == 3 else { fail("Provide a PID and a dot-separated AX child path.") } |
| 128 | var element = app |
| 129 | for item in arguments[2].split(separator: ".") { |
| 130 | guard let index = Int(item), let children = attribute(element, "AXChildren") as? [AXUIElement], children.indices.contains(index) |
| 131 | else { fail("The AX path no longer exists; read the tree again.") } |
| 132 | element = children[index] |
| 133 | } |
| 134 | let result = AXUIElementPerformAction(element, kAXPressAction as CFString) |
| 135 | guard result == .success else { fail("AXPress failed: \(result.rawValue)") } |
| 136 | output(["pressed": true]) |
| 137 | } |
| 138 | case "action": |
| 139 | guard AXIsProcessTrusted() else { fail("Accessibility permission is required.") } |
| 140 | guard let request = try? JSONSerialization.jsonObject(with: FileHandle.standardInput.readDataToEndOfFile()) as? [String: Any], |
| 141 | let target = request["target"] as? [String: Any], let pid = target["pid"] as? Int32, |
| 142 | let path = target["path"] as? [Int], path.count <= 16, |
| 143 | let operation = request["operation"] as? String else { fail("Use a target returned by computer_ui.") } |
| 144 | var element = AXUIElementCreateApplication(pid) |
| 145 | for index in path { |
| 146 | guard let children = attribute(element, "AXChildren") as? [AXUIElement], children.indices.contains(index) |
| 147 | else { fail("The UI element changed. Read a fresh tree.") } |
| 148 | element = children[index] |
| 149 | } |
| 150 | guard (attribute(element, "AXRole") as? String ?? "") == (target["role"] as? String ?? ""), |
| 151 | (attribute(element, "AXTitle") as? String ?? "") == (target["name"] as? String ?? "") |
| 152 | else { fail("The UI element changed. Read a fresh tree.") } |
| 153 | let result: AXError |
| 154 | if operation == "set_value" { |
| 155 | guard let value = request["value"] as? String else { fail("Provide the new value.") } |
| 156 | result = AXUIElementSetAttributeValue(element, kAXValueAttribute as CFString, value as CFString) |
| 157 | } else if operation == "focus" { |
| 158 | result = AXUIElementSetAttributeValue(element, kAXFocusedAttribute as CFString, kCFBooleanTrue) |
| 159 | } else { |
| 160 | result = AXUIElementPerformAction(element, operation as CFString) |
| 161 | } |
| 162 | guard result == .success else { fail("The application refused the UI action. Read a fresh tree.") } |
| 163 | output(["completed": true]) |
| 164 | case "input": |
| 165 | guard AXIsProcessTrusted() else { fail("Accessibility permission is required.") } |
| 166 | guard let request = try? JSONSerialization.jsonObject(with: FileHandle.standardInput.readDataToEndOfFile()) as? [String: Any], |
| 167 | let actions = request["actions"] as? [[String: Any]], (1...100).contains(actions.count) |
| 168 | else { fail("Send between 1 and 100 desktop actions.") } |
| 169 | let bounds = CGDisplayBounds(CGMainDisplayID()) |
| 170 | let pixelWidth = Double(CGDisplayPixelsWide(CGMainDisplayID())) |
| 171 | let pixelHeight = Double(CGDisplayPixelsHigh(CGMainDisplayID())) |
| 172 | func point(_ action: [String: Any], _ x: String = "x", _ y: String = "y") -> CGPoint { |
| 173 | guard let px = action[x] as? Double, let py = action[y] as? Double, |
| 174 | px >= 0, py >= 0, px < pixelWidth, py < pixelHeight else { fail("Choose coordinates inside the latest screenshot.") } |
| 175 | return CGPoint(x: bounds.minX + px * bounds.width / pixelWidth, y: bounds.minY + py * bounds.height / pixelHeight) |
| 176 | } |
| 177 | func mouse(_ type: CGEventType, _ at: CGPoint, _ button: CGMouseButton, _ count: Int64 = 1) { |
| 178 | let event = CGEvent(mouseEventSource: eventSource, mouseType: type, mouseCursorPosition: at, mouseButton: button) |
| 179 | event?.setIntegerValueField(.mouseEventClickState, value: count) |
| 180 | event?.flags = [] |
| 181 | event?.post(tap: .cghidEventTap) |
| 182 | usleep(10000) |
| 183 | } |
| 184 | let keys: [String: UInt16] = ["a":0,"s":1,"d":2,"f":3,"h":4,"g":5,"z":6,"x":7,"c":8,"v":9,"b":11,"q":12,"w":13,"e":14,"r":15,"y":16,"t":17,"1":18,"2":19,"3":20,"4":21,"6":22,"5":23,"=":24,"9":25,"7":26,"-":27,"8":28,"0":29, |
| 185 | "]":30,"o":31,"u":32,"[":33,"i":34,"p":35,"Enter":36,"l":37,"j":38,"'":39,"k":40,";":41,"\\":42,",":43,"/":44,"n":45,"m":46,".":47,"Tab":48,"Space":49,"`":50,"Backspace":51,"Escape":53,"Home":115,"PageUp":116,"Delete":117,"End":119,"PageDown":121,"Left":123,"Right":124,"Down":125,"Up":126,"F1":122,"F2":120,"F3":99,"F4":118,"F5":96,"F6":97,"F7":98,"F8":100,"F9":101,"F10":109,"F11":103,"F12":111] |
| 186 | for action in actions { |
| 187 | guard let kind = action["type"] as? String else { fail("Choose a desktop action.") } |
| 188 | switch kind { |
| 189 | case "click", "move", "drag": |
| 190 | let at = point(action) |
| 191 | let button: CGMouseButton = action["button"] as? String == "right" ? .right : (action["button"] as? String == "middle" ? .center : .left) |
| 192 | let down: CGEventType = button == .left ? .leftMouseDown : (button == .right ? .rightMouseDown : .otherMouseDown) |
| 193 | let up: CGEventType = button == .left ? .leftMouseUp : (button == .right ? .rightMouseUp : .otherMouseUp) |
| 194 | mouse(.mouseMoved, at, button) |
| 195 | if kind == "click" { |
| 196 | for count in 1...max(1,min(action["count"] as? Int ?? 1,3)) { mouse(down,at,button,Int64(count)); mouse(up,at,button,Int64(count)) } |
| 197 | } else if kind == "drag" { |
| 198 | let end = point(action,"to_x","to_y") |
| 199 | mouse(down,at,button) |
| 200 | for step in 1...20 { |
| 201 | let p = CGPoint(x:at.x+(end.x-at.x)*Double(step)/20,y:at.y+(end.y-at.y)*Double(step)/20) |
| 202 | mouse(button == .left ? .leftMouseDragged : (button == .right ? .rightMouseDragged : .otherMouseDragged),p,button) |
| 203 | usleep(UInt32(max(1,min(action["duration_ms"] as? Int ?? 400,5000)))*50) |
| 204 | } |
| 205 | mouse(up,end,button) |
| 206 | } |
| 207 | case "scroll": |
| 208 | mouse(.mouseMoved,point(action),.left) |
| 209 | let amount = Int32(max(1,min(action["amount"] as? Int ?? 3,100))) |
| 210 | let direction = action["direction"] as? String ?? "down" |
| 211 | let vertical: Int32 = direction == "up" ? amount : (direction == "down" ? -amount : 0) |
| 212 | let horizontal: Int32 = direction == "left" ? amount : (direction == "right" ? -amount : 0) |
| 213 | CGEvent(scrollWheelEvent2Source:eventSource,units:.line,wheelCount:2,wheel1:vertical,wheel2:horizontal,wheel3:0)?.post(tap:.cghidEventTap) |
| 214 | case "key": |
| 215 | guard let chord = action["keys"] as? [String], (1...8).contains(chord.count) else { fail("Send between 1 and 8 key names.") } |
| 216 | var flags: CGEventFlags = [] |
| 217 | var codes: [UInt16] = [] |
| 218 | for key in chord { |
| 219 | switch key { |
| 220 | case "Meta": flags.insert(.maskCommand) |
| 221 | case "Control": flags.insert(.maskControl) |
| 222 | case "Alt": flags.insert(.maskAlternate) |
| 223 | case "Shift": flags.insert(.maskShift) |
| 224 | default: |
| 225 | guard let code = keys[key] ?? keys[key.lowercased()] else { fail("Choose a supported key name.") } |
| 226 | codes.append(code) |
| 227 | if key.count == 1 && key != key.lowercased() { flags.insert(.maskShift) } |
| 228 | } |
| 229 | } |
| 230 | for down in [true,false] { |
| 231 | for code in down ? codes : Array(codes.reversed()) { |
| 232 | let event = CGEvent(keyboardEventSource:eventSource,virtualKey:code,keyDown:down) |
| 233 | event?.flags = down ? flags : [] |
| 234 | event?.post(tap:.cghidEventTap) |
| 235 | usleep(10000) |
| 236 | } |
| 237 | } |
| 238 | case "text": |
| 239 | guard let text = action["text"] as? String, text.utf8.count <= 16384 else { fail("Type at most 16 KiB at a time.") } |
| 240 | typeText(text, eventSource) |
| 241 | case "wait": usleep(UInt32(max(1,min(action["duration_ms"] as? Int ?? 1,5000)))*1000) |
| 242 | default: fail("Choose a supported desktop action.") |
| 243 | } |
| 244 | } |
| 245 | output(["completed":actions.count]) |
| 246 | case "click": |
| 247 | guard AXIsProcessTrusted() else { fail("Accessibility permission is required.") } |
| 248 | guard arguments.count == 3, let x = Double(arguments[1]), let y = Double(arguments[2]) else { fail("Provide x and y coordinates.") } |
| 249 | let point = CGPoint(x: x, y: y) |
| 250 | for type in [CGEventType.mouseMoved, .leftMouseDown, .leftMouseUp] { |
| 251 | let event = CGEvent(mouseEventSource: eventSource, mouseType: type, mouseCursorPosition: point, mouseButton: .left) |
| 252 | event?.flags = [] |
| 253 | event?.post(tap: .cghidEventTap) |
| 254 | usleep(10000) |
| 255 | } |
| 256 | output(["clicked": [x, y]]) |
| 257 | case "key": |
| 258 | guard AXIsProcessTrusted() else { fail("Accessibility permission is required.") } |
| 259 | guard arguments.count >= 2, let code = UInt16(arguments[1]) else { fail("Provide a macOS virtual key code, optionally followed by command, shift, option, or control.") } |
| 260 | var flags: CGEventFlags = [] |
| 261 | for modifier in arguments.dropFirst(2) { |
| 262 | switch modifier { |
| 263 | case "command": flags.insert(.maskCommand) |
| 264 | case "shift": flags.insert(.maskShift) |
| 265 | case "option": flags.insert(.maskAlternate) |
| 266 | case "control": flags.insert(.maskControl) |
| 267 | default: fail("Unknown key modifier.") |
| 268 | } |
| 269 | } |
| 270 | for down in [true, false] { |
| 271 | let event = CGEvent(keyboardEventSource: eventSource, virtualKey: code, keyDown: down) |
| 272 | event?.flags = down ? flags : [] |
| 273 | event?.post(tap: .cghidEventTap) |
| 274 | usleep(10000) |
| 275 | } |
| 276 | output(["key": code]) |
| 277 | case "type": |
| 278 | guard AXIsProcessTrusted() else { fail("Accessibility permission is required.") } |
| 279 | guard arguments.count == 2 else { fail("Provide text, or a dash to read UTF-8 from stdin.") } |
| 280 | let text = arguments[1] == "-" ? String(data: FileHandle.standardInput.readDataToEndOfFile(), encoding: .utf8)! : arguments[1] |
| 281 | typeText(text, eventSource) |
| 282 | output(["typed": true]) |
| 283 | default: |
| 284 | fail("Unknown command.") |
| 285 | } |