authoravatarclover caruso<git@paperclover.net>2026-10-19 11:26:41-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-19 22:58:56-07:00
logb383ce36c80b1570d513606e5c304576c3da3232
treedbe6e205fdef2b2894ce7d9b7d923a91ea956864
parent3d2698e42b763a8b291c546360efcf2512995fa2
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

test(ios): drive menus, alerts and list rows from SNOWBOUND_SCRIPT

activate: now picks a menu's item, a list's row and an alert's button, reaches a button within a row as `Row/Button`, prefers the frontmost of two matches, and keeps running after the page that started it goes. Its search no longer doubles at each level of the view tree, which took minutes on an iPad. Assisted-by: claude-opus-5.5

1 files changed, 54 insertions(+), 12 deletions(-)

apps/ios/Snowbound/Script.swift+54-12
...@@ -20,8 +20,8 @@ extension CanvasView {...@@ -20,8 +20,8 @@ extension CanvasView {
20 /// Documents/tree.txt, and `shot:a` the window to Documents/a.png, as a device has no20 /// Documents/tree.txt, and `shot:a` the window to Documents/a.png, as a device has no
21 /// screenshot command; `snap:a` only logs, for a simulator screenshot with the keyboard;21 /// screenshot command; `snap:a` only logs, for a simulator screenshot with the keyboard;
22 /// `activate:Label` activates the element of any window so named, as VoiceOver would,22 /// `activate:Label` activates the element of any window so named, as VoiceOver would,
23 /// opening a button's menu, and `audit` logs whether the format bar and tag grid name every23 /// opening a button's menu or picking a menu's item or a list's row (`Row/Button` for a
24 /// button.24 /// button in a row), and `audit` logs whether the format bar and tag grid name every button.
25 func runScript() {25 func runScript() {
26 // Only the first page shown runs it, not every page opened afterwards.26 // Only the first page shown runs it, not every page opened afterwards.
27 guard !scriptedPage, let script = ProcessInfo.processInfo.environment["SNOWBOUND_SCRIPT"] else { return }27 guard !scriptedPage, let script = ProcessInfo.processInfo.environment["SNOWBOUND_SCRIPT"] else { return }
...@@ -35,9 +35,8 @@ extension CanvasView {...@@ -35,9 +35,8 @@ extension CanvasView {
35 var delay = 1.035 var delay = 1.0
36 for step in script.split(separator: "|") {36 for step in script.split(separator: "|") {
37 if step.hasPrefix("wait:") { delay += Double(step.dropFirst(5)) ?? 0 }37 if step.hasPrefix("wait:") { delay += Double(step.dropFirst(5)) ?? 0 }
38 DispatchQueue.main.asyncAfter(deadline: .now() + delay) { [weak self] in38 // Held, so that steps after the page goes, as Delete Page takes it, still run.
39 self?.perform(String(step))39 DispatchQueue.main.asyncAfter(deadline: .now() + delay) { [self] in perform(String(step)) }
40 }
41 delay += 0.440 delay += 0.4
42 }41 }
43 }42 }
...@@ -106,15 +105,45 @@ extension CanvasView {...@@ -106,15 +105,45 @@ extension CanvasView {
106 case "done": _ = resignFirstResponder()105 case "done": _ = resignFirstResponder()
107 case "wait", "snap": break106 case "wait", "snap": break
108 case "activate":107 case "activate":
109 guard let found = (window?.windowScene?.windows ?? []).reversed().lazy108 // An alert's button, which takes no accessibility activation outside VoiceOver.
110 .compactMap({ Self.element(argument, in: $0) }).first109 let alert = Self.windows.lazy.compactMap { window in
111 else {110 sequence(first: window.rootViewController, next: { $0?.presentedViewController }).compactMap { $0 }.last
111 }.compactMap { $0 as? UIAlertController }.first
112 if let alert, let action = alert.actions.first(where: { $0.title == argument }),
113 let handler = action.value(forKey: "handler")
114 {
115 alert.dismiss(animated: true) {
116 unsafeBitCast(handler as AnyObject, to: (@convention(block) (UIAlertAction) -> Void).self)(action)
117 }
118 break
119 }
120 // `Row/Button` names a button within a row.
121 let names = argument.split(separator: "/").map(String.init)
122 var found: NSObject? = Self.windows.reversed().lazy.compactMap { Self.element(names[0], in: $0) }.first
123 for name in names.dropFirst() {
124 found = found.flatMap { Self.children($0).lazy.compactMap { Self.element(name, in: $0) }.first }
125 }
126 guard let found else {
112 NSLog("script activate: no \(argument)")127 NSLog("script activate: no \(argument)")
113 break128 break
114 }129 }
115 if let button = found as? UIButton, button.showsMenuAsPrimaryAction, let menu = button.contextMenuInteraction {130 let chain = (found as? UIView).map { Array(sequence(first: $0, next: \.superview)) } ?? []
131 let named = { (name: String) in chain.first { NSStringFromClass(type(of: $0)) == name } }
132 if let button = found as? UIControl, button.showsMenuAsPrimaryAction, let menu = button.contextMenuInteraction {
116 let middle = CGPoint(x: button.bounds.midX, y: button.bounds.midY)133 let middle = CGPoint(x: button.bounds.midX, y: button.bounds.midY)
117 menu.perform(Selector(("_presentMenuAtLocation:")), with: NSValue(cgPoint: middle))134 menu.perform(Selector(("_presentMenuAtLocation:")), with: NSValue(cgPoint: middle))
135 } else if let cell = chain.lazy.compactMap({ $0 as? UICollectionViewCell }).first,
136 let list = chain.lazy.compactMap({ $0 as? UICollectionView }).first, let index = list.indexPath(for: cell)
137 {
138 // A menu's item or a list's row, as a tap picks it.
139 if let items = named("_UIContextMenuListView"), let menu = named("_UIContextMenuView"),
140 let item = items.perform(Selector(("elementAtIndexPath:")), with: index)?.takeUnretainedValue()
141 {
142 menu.perform(Selector(("_handleSelectionForElement:")), with: item)
143 } else {
144 list.selectItem(at: index, animated: false, scrollPosition: [])
145 list.delegate?.collectionView?(list, didSelectItemAt: index)
146 }
118 } else if !found.accessibilityActivate(), let control = found as? UIControl {147 } else if !found.accessibilityActivate(), let control = found as? UIControl {
119 control.sendActions(for: .primaryActionTriggered)148 control.sendActions(for: .primaryActionTriggered)
120 }149 }
...@@ -156,12 +185,25 @@ extension CanvasView {...@@ -156,12 +185,25 @@ extension CanvasView {
156}185}
157186
158extension CanvasView {187extension CanvasView {
159 /// The first view or accessibility element under `root` VoiceOver names `label`.188 /// The scene's windows, menus' and alerts' among them.
189 fileprivate static var windows: [UIWindow] {
190 UIApplication.shared.connectedScenes.compactMap { $0 as? UIWindowScene }.flatMap(\.windows)
191 }
192
193 /// The frontmost view or accessibility element under `root` VoiceOver names `label`.
160 fileprivate static func element(_ label: String, in root: NSObject) -> NSObject? {194 fileprivate static func element(_ label: String, in root: NSObject) -> NSObject? {
161 if let view = root as? UIView, view.isHidden { return nil }195 if let view = root as? UIView, view.isHidden { return nil }
162 if root.isAccessibilityElement, root.accessibilityLabel == label { return root }196 if root.isAccessibilityElement, root.accessibilityLabel == label { return root }
163 let children = ((root as? UIView)?.subviews ?? []) + (root.accessibilityElements as? [NSObject] ?? [])197 // A loop, as a lazy `first` evaluates each match twice, doubling at every level.
164 return children.lazy.compactMap { element(label, in: $0) }.first198 for child in children(root) {
199 if let found = element(label, in: child) { return found }
200 }
201 return nil
202 }
203
204 /// `root`'s subviews, frontmost first, then its accessibility elements.
205 fileprivate static func children(_ root: NSObject) -> [NSObject] {
206 ((root as? UIView)?.subviews ?? []).reversed() + (root.accessibilityElements as? [NSObject] ?? [])
165 }207 }
166208
167 /// Logs `audit ok`, or each button of the format bar and tag grid VoiceOver can't name.209 /// Logs `audit ok`, or each button of the format bar and tag grid VoiceOver can't name.