1import AppKit
2import ApplicationServices
3import ImageIO
4import ScreenCaptureKit
5import UniformTypeIdentifiers
6
7func output(_ value: Any) {
8 let data = try! JSONSerialization.data(withJSONObject: value, options: [.sortedKeys])
9 print(String(data: data, encoding: .utf8)!)
10}
11
12func fail(_ message: String) -> Never {
13 output(["error": message])
14 exit(1)
15}
16
17func attribute(_ element: AXUIElement, _ name: String) -> CFTypeRef? {
18 var value: CFTypeRef?
19 return AXUIElementCopyAttributeValue(element, name as CFString, &value) == .success ? value : nil
20}
21
22func tree(_ element: AXUIElement, path: [Int], remaining: inout Int) -> [String: Any] {
23 remaining -= 1
24 var pid: pid_t = 0
25 AXUIElementGetPid(element, &pid)
26 var result: [String: Any] = ["path": path, "target": ["pid": pid, "path": path, "role": attribute(element, "AXRole") as? String ?? "", "name": attribute(element, "AXTitle") as? String ?? ""]]
27 for name in ["AXRole", "AXSubrole", "AXTitle", "AXDescription", "AXValue", "AXEnabled", "AXFocused"] {
28 if let value = attribute(element, name), value is String || value is NSNumber {
29 result[name] = value
30 }
31 }
32 for name in ["AXPosition", "AXSize"] {
33 if let value = attribute(element, name), CFGetTypeID(value) == AXValueGetTypeID() {
34 let axValue = value as! AXValue
35 if AXValueGetType(axValue) == .cgPoint {
36 var point = CGPoint.zero
37 AXValueGetValue(axValue, .cgPoint, &point)
38 result[name] = [point.x, point.y]
39 } else if AXValueGetType(axValue) == .cgSize {
40 var size = CGSize.zero
41 AXValueGetValue(axValue, .cgSize, &size)
42 result[name] = [size.width, size.height]
43 }
44 }
45 }
46 var actions: CFArray?
47 if AXUIElementCopyActionNames(element, &actions) == .success { result["actions"] = actions }
48 var settable = DarwinBoolean(false)
49 if AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &settable) == .success, settable.boolValue {
50 result["actions"] = (result["actions"] as? [String] ?? []) + ["set_value"]
51 }
52 if path.count < 16, let children = attribute(element, "AXChildren") as? [AXUIElement] {
53 var nodes: [[String: Any]] = []
54 for (index, child) in children.enumerated() where remaining > 0 {
55 nodes.append(tree(child, path: path + [index], remaining: &remaining))
56 }
57 result["children"] = nodes
58 if nodes.count < children.count { result["truncated"] = true }
59 }
60 return result
61}
62
63func typeText(_ text: String, _ eventSource: CGEventSource) {
64 let units = Array(text.utf16)
65 var start = 0
66 while start < units.count {
67 var end = min(start + 20, units.count)
68 if end < units.count && (0xD800...0xDBFF).contains(units[end - 1]) { end -= 1 }
69 let chunk = Array(units[start..<end])
70 for down in [true, false] {
71 let event = CGEvent(keyboardEventSource: eventSource, virtualKey: 0, keyDown: down)
72 event?.flags = []
73 event?.keyboardSetUnicodeString(stringLength: chunk.count, unicodeString: chunk)
74 event?.post(tap: .cghidEventTap)
75 usleep(10000)
76 }
77 start = end
78 }
79 usleep(50000)
80}
81
82let arguments = Array(CommandLine.arguments.dropFirst())
83guard let command = arguments.first else { fail("Use status, request-permissions, screenshot, tree, press, click, key, or type.") }
84guard let eventSource = CGEventSource(stateID: .privateState) else { fail("Could not create a private input event source.") }
85switch command {
86case "status", "request-permissions":
87 if command == "request-permissions" {
88 _ = AXIsProcessTrustedWithOptions([kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] as CFDictionary)
89 _ = CGRequestScreenCaptureAccess()
90 _ = CGRequestPostEventAccess()
91 }
92 output(["accessibility": AXIsProcessTrusted(), "screenCapture": CGPreflightScreenCaptureAccess(), "postEvent": CGPreflightPostEventAccess()])
93case "screenshot":
94 guard arguments.count == 2 else { fail("Provide an output PNG path.") }
95 let image: CGImage
96 do {
97 let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true)
98 guard let display = content.displays.first(where: { $0.displayID == CGMainDisplayID() }) else { fail("No active display.") }
99 let configuration = SCStreamConfiguration()
100 configuration.width = display.width
101 configuration.height = display.height
102 configuration.showsCursor = false
103 image = try await SCScreenshotManager.captureImage(contentFilter: SCContentFilter(display: display, excludingWindows: []), configuration: configuration)
104 } catch { fail("Screen capture failed: \(error.localizedDescription)") }
105 guard let destination = CGImageDestinationCreateWithURL(URL(fileURLWithPath: arguments[1]) as CFURL, UTType.png.identifier as CFString, 1, nil)
106 else { fail("Could not create the output PNG.") }
107 CGImageDestinationAddImage(destination, image, nil)
108 guard CGImageDestinationFinalize(destination) else { fail("Could not write the output PNG.") }
109 let bounds = CGDisplayBounds(CGMainDisplayID())
110 output(["width": image.width, "height": image.height, "path": arguments[1],
111 "bounds": [bounds.origin.x, bounds.origin.y, bounds.width, bounds.height],
112 "scale": [Double(image.width) / bounds.width, Double(image.height) / bounds.height]])
113case "tree", "press":
114 guard AXIsProcessTrusted() else { fail("Accessibility permission is required.") }
115 let app: AXUIElement
116 if arguments.count > 1 {
117 guard let pid = Int32(arguments[1]), pid > 0 else { fail("Provide a positive application PID.") }
118 app = AXUIElementCreateApplication(pid)
119 } else {
120 guard let front = NSWorkspace.shared.frontmostApplication else { fail("No active application.") }
121 app = AXUIElementCreateApplication(front.processIdentifier)
122 }
123 if command == "tree" {
124 var remaining = 5000
125 output(tree(app, path: [], remaining: &remaining))
126 } else {
127 guard arguments.count == 3 else { fail("Provide a PID and a dot-separated AX child path.") }
128 var element = app
129 for item in arguments[2].split(separator: ".") {
130 guard let index = Int(item), let children = attribute(element, "AXChildren") as? [AXUIElement], children.indices.contains(index)
131 else { fail("The AX path no longer exists; read the tree again.") }
132 element = children[index]
133 }
134 let result = AXUIElementPerformAction(element, kAXPressAction as CFString)
135 guard result == .success else { fail("AXPress failed: \(result.rawValue)") }
136 output(["pressed": true])
137 }
138case "action":
139 guard AXIsProcessTrusted() else { fail("Accessibility permission is required.") }
140 guard let request = try? JSONSerialization.jsonObject(with: FileHandle.standardInput.readDataToEndOfFile()) as? [String: Any],
141 let target = request["target"] as? [String: Any], let pid = target["pid"] as? Int32,
142 let path = target["path"] as? [Int], path.count <= 16,
143 let operation = request["operation"] as? String else { fail("Use a target returned by computer_ui.") }
144 var element = AXUIElementCreateApplication(pid)
145 for index in path {
146 guard let children = attribute(element, "AXChildren") as? [AXUIElement], children.indices.contains(index)
147 else { fail("The UI element changed. Read a fresh tree.") }
148 element = children[index]
149 }
150 guard (attribute(element, "AXRole") as? String ?? "") == (target["role"] as? String ?? ""),
151 (attribute(element, "AXTitle") as? String ?? "") == (target["name"] as? String ?? "")
152 else { fail("The UI element changed. Read a fresh tree.") }
153 let result: AXError
154 if operation == "set_value" {
155 guard let value = request["value"] as? String else { fail("Provide the new value.") }
156 result = AXUIElementSetAttributeValue(element, kAXValueAttribute as CFString, value as CFString)
157 } else if operation == "focus" {
158 result = AXUIElementSetAttributeValue(element, kAXFocusedAttribute as CFString, kCFBooleanTrue)
159 } else {
160 result = AXUIElementPerformAction(element, operation as CFString)
161 }
162 guard result == .success else { fail("The application refused the UI action. Read a fresh tree.") }
163 output(["completed": true])
164case "input":
165 guard AXIsProcessTrusted() else { fail("Accessibility permission is required.") }
166 guard let request = try? JSONSerialization.jsonObject(with: FileHandle.standardInput.readDataToEndOfFile()) as? [String: Any],
167 let actions = request["actions"] as? [[String: Any]], (1...100).contains(actions.count)
168 else { fail("Send between 1 and 100 desktop actions.") }
169 let bounds = CGDisplayBounds(CGMainDisplayID())
170 let pixelWidth = Double(CGDisplayPixelsWide(CGMainDisplayID()))
171 let pixelHeight = Double(CGDisplayPixelsHigh(CGMainDisplayID()))
172 func point(_ action: [String: Any], _ x: String = "x", _ y: String = "y") -> CGPoint {
173 guard let px = action[x] as? Double, let py = action[y] as? Double,
174 px >= 0, py >= 0, px < pixelWidth, py < pixelHeight else { fail("Choose coordinates inside the latest screenshot.") }
175 return CGPoint(x: bounds.minX + px * bounds.width / pixelWidth, y: bounds.minY + py * bounds.height / pixelHeight)
176 }
177 func mouse(_ type: CGEventType, _ at: CGPoint, _ button: CGMouseButton, _ count: Int64 = 1) {
178 let event = CGEvent(mouseEventSource: eventSource, mouseType: type, mouseCursorPosition: at, mouseButton: button)
179 event?.setIntegerValueField(.mouseEventClickState, value: count)
180 event?.flags = []
181 event?.post(tap: .cghidEventTap)
182 usleep(10000)
183 }
184 let keys: [String: UInt16] = ["a":0,"s":1,"d":2,"f":3,"h":4,"g":5,"z":6,"x":7,"c":8,"v":9,"b":11,"q":12,"w":13,"e":14,"r":15,"y":16,"t":17,"1":18,"2":19,"3":20,"4":21,"6":22,"5":23,"=":24,"9":25,"7":26,"-":27,"8":28,"0":29,
185 "]":30,"o":31,"u":32,"[":33,"i":34,"p":35,"Enter":36,"l":37,"j":38,"'":39,"k":40,";":41,"\\":42,",":43,"/":44,"n":45,"m":46,".":47,"Tab":48,"Space":49,"`":50,"Backspace":51,"Escape":53,"Home":115,"PageUp":116,"Delete":117,"End":119,"PageDown":121,"Left":123,"Right":124,"Down":125,"Up":126,"F1":122,"F2":120,"F3":99,"F4":118,"F5":96,"F6":97,"F7":98,"F8":100,"F9":101,"F10":109,"F11":103,"F12":111]
186 for action in actions {
187 guard let kind = action["type"] as? String else { fail("Choose a desktop action.") }
188 switch kind {
189 case "click", "move", "drag":
190 let at = point(action)
191 let button: CGMouseButton = action["button"] as? String == "right" ? .right : (action["button"] as? String == "middle" ? .center : .left)
192 let down: CGEventType = button == .left ? .leftMouseDown : (button == .right ? .rightMouseDown : .otherMouseDown)
193 let up: CGEventType = button == .left ? .leftMouseUp : (button == .right ? .rightMouseUp : .otherMouseUp)
194 mouse(.mouseMoved, at, button)
195 if kind == "click" {
196 for count in 1...max(1,min(action["count"] as? Int ?? 1,3)) { mouse(down,at,button,Int64(count)); mouse(up,at,button,Int64(count)) }
197 } else if kind == "drag" {
198 let end = point(action,"to_x","to_y")
199 mouse(down,at,button)
200 for step in 1...20 {
201 let p = CGPoint(x:at.x+(end.x-at.x)*Double(step)/20,y:at.y+(end.y-at.y)*Double(step)/20)
202 mouse(button == .left ? .leftMouseDragged : (button == .right ? .rightMouseDragged : .otherMouseDragged),p,button)
203 usleep(UInt32(max(1,min(action["duration_ms"] as? Int ?? 400,5000)))*50)
204 }
205 mouse(up,end,button)
206 }
207 case "scroll":
208 mouse(.mouseMoved,point(action),.left)
209 let amount = Int32(max(1,min(action["amount"] as? Int ?? 3,100)))
210 let direction = action["direction"] as? String ?? "down"
211 let vertical: Int32 = direction == "up" ? amount : (direction == "down" ? -amount : 0)
212 let horizontal: Int32 = direction == "left" ? amount : (direction == "right" ? -amount : 0)
213 CGEvent(scrollWheelEvent2Source:eventSource,units:.line,wheelCount:2,wheel1:vertical,wheel2:horizontal,wheel3:0)?.post(tap:.cghidEventTap)
214 case "key":
215 guard let chord = action["keys"] as? [String], (1...8).contains(chord.count) else { fail("Send between 1 and 8 key names.") }
216 var flags: CGEventFlags = []
217 var codes: [UInt16] = []
218 for key in chord {
219 switch key {
220 case "Meta": flags.insert(.maskCommand)
221 case "Control": flags.insert(.maskControl)
222 case "Alt": flags.insert(.maskAlternate)
223 case "Shift": flags.insert(.maskShift)
224 default:
225 guard let code = keys[key] ?? keys[key.lowercased()] else { fail("Choose a supported key name.") }
226 codes.append(code)
227 if key.count == 1 && key != key.lowercased() { flags.insert(.maskShift) }
228 }
229 }
230 for down in [true,false] {
231 for code in down ? codes : Array(codes.reversed()) {
232 let event = CGEvent(keyboardEventSource:eventSource,virtualKey:code,keyDown:down)
233 event?.flags = down ? flags : []
234 event?.post(tap:.cghidEventTap)
235 usleep(10000)
236 }
237 }
238 case "text":
239 guard let text = action["text"] as? String, text.utf8.count <= 16384 else { fail("Type at most 16 KiB at a time.") }
240 typeText(text, eventSource)
241 case "wait": usleep(UInt32(max(1,min(action["duration_ms"] as? Int ?? 1,5000)))*1000)
242 default: fail("Choose a supported desktop action.")
243 }
244 }
245 output(["completed":actions.count])
246case "click":
247 guard AXIsProcessTrusted() else { fail("Accessibility permission is required.") }
248 guard arguments.count == 3, let x = Double(arguments[1]), let y = Double(arguments[2]) else { fail("Provide x and y coordinates.") }
249 let point = CGPoint(x: x, y: y)
250 for type in [CGEventType.mouseMoved, .leftMouseDown, .leftMouseUp] {
251 let event = CGEvent(mouseEventSource: eventSource, mouseType: type, mouseCursorPosition: point, mouseButton: .left)
252 event?.flags = []
253 event?.post(tap: .cghidEventTap)
254 usleep(10000)
255 }
256 output(["clicked": [x, y]])
257case "key":
258 guard AXIsProcessTrusted() else { fail("Accessibility permission is required.") }
259 guard arguments.count >= 2, let code = UInt16(arguments[1]) else { fail("Provide a macOS virtual key code, optionally followed by command, shift, option, or control.") }
260 var flags: CGEventFlags = []
261 for modifier in arguments.dropFirst(2) {
262 switch modifier {
263 case "command": flags.insert(.maskCommand)
264 case "shift": flags.insert(.maskShift)
265 case "option": flags.insert(.maskAlternate)
266 case "control": flags.insert(.maskControl)
267 default: fail("Unknown key modifier.")
268 }
269 }
270 for down in [true, false] {
271 let event = CGEvent(keyboardEventSource: eventSource, virtualKey: code, keyDown: down)
272 event?.flags = down ? flags : []
273 event?.post(tap: .cghidEventTap)
274 usleep(10000)
275 }
276 output(["key": code])
277case "type":
278 guard AXIsProcessTrusted() else { fail("Accessibility permission is required.") }
279 guard arguments.count == 2 else { fail("Provide text, or a dash to read UTF-8 from stdin.") }
280 let text = arguments[1] == "-" ? String(data: FileHandle.standardInput.readDataToEndOfFile(), encoding: .utf8)! : arguments[1]
281 typeText(text, eventSource)
282 output(["typed": true])
283default:
284 fail("Unknown command.")
285}