authoravatarclover caruso<git@paperclover.net>2026-10-19 11:26:41-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-19 22:58:56-07:00
logb383ce36c80b1570d513606e5c304576c3da3232
treedbe6e205fdef2b2894ce7d9b7d923a91ea956864
parent3d2698e42b763a8b291c546360efcf2512995fa2
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

test(ios): drive menus, alerts and list rows from SNOWBOUND_SCRIPT

activate: now picks a menu's item, a list's row and an alert's button, reaches a button within a row as `Row/Button`, prefers the frontmost of two matches, and keeps running after the page that started it goes. Its search no longer doubles at each level of the view tree, which took minutes on an iPad. Assisted-by: claude-opus-5.5

1 files changed, 54 insertions(+), 12 deletions(-)

apps/ios/Snowbound/Script.swift+54-12
......@@ -20,8 +20,8 @@ extension CanvasView {
2020 /// Documents/tree.txt, and `shot:a` the window to Documents/a.png, as a device has no
2121 /// screenshot command; `snap:a` only logs, for a simulator screenshot with the keyboard;
2222 /// `activate:Label` activates the element of any window so named, as VoiceOver would,
23 /// opening a button's menu, and `audit` logs whether the format bar and tag grid name every
24 /// button.
23 /// opening a button's menu or picking a menu's item or a list's row (`Row/Button` for a
24 /// button in a row), and `audit` logs whether the format bar and tag grid name every button.
2525 func runScript() {
2626 // Only the first page shown runs it, not every page opened afterwards.
2727 guard !scriptedPage, let script = ProcessInfo.processInfo.environment["SNOWBOUND_SCRIPT"] else { return }
......@@ -35,9 +35,8 @@ extension CanvasView {
3535 var delay = 1.0
3636 for step in script.split(separator: "|") {
3737 if step.hasPrefix("wait:") { delay += Double(step.dropFirst(5)) ?? 0 }
38 DispatchQueue.main.asyncAfter(deadline: .now() + delay) { [weak self] in
39 self?.perform(String(step))
40 }
38 // Held, so that steps after the page goes, as Delete Page takes it, still run.
39 DispatchQueue.main.asyncAfter(deadline: .now() + delay) { [self] in perform(String(step)) }
4140 delay += 0.4
4241 }
4342 }
......@@ -106,15 +105,45 @@ extension CanvasView {
106105 case "done": _ = resignFirstResponder()
107106 case "wait", "snap": break
108107 case "activate":
109 guard let found = (window?.windowScene?.windows ?? []).reversed().lazy
110 .compactMap({ Self.element(argument, in: $0) }).first
111 else {
108 // An alert's button, which takes no accessibility activation outside VoiceOver.
109 let alert = Self.windows.lazy.compactMap { window in
110 sequence(first: window.rootViewController, next: { $0?.presentedViewController }).compactMap { $0 }.last
111 }.compactMap { $0 as? UIAlertController }.first
112 if let alert, let action = alert.actions.first(where: { $0.title == argument }),
113 let handler = action.value(forKey: "handler")
114 {
115 alert.dismiss(animated: true) {
116 unsafeBitCast(handler as AnyObject, to: (@convention(block) (UIAlertAction) -> Void).self)(action)
117 }
118 break
119 }
120 // `Row/Button` names a button within a row.
121 let names = argument.split(separator: "/").map(String.init)
122 var found: NSObject? = Self.windows.reversed().lazy.compactMap { Self.element(names[0], in: $0) }.first
123 for name in names.dropFirst() {
124 found = found.flatMap { Self.children($0).lazy.compactMap { Self.element(name, in: $0) }.first }
125 }
126 guard let found else {
112127 NSLog("script activate: no \(argument)")
113128 break
114129 }
115 if let button = found as? UIButton, button.showsMenuAsPrimaryAction, let menu = button.contextMenuInteraction {
130 let chain = (found as? UIView).map { Array(sequence(first: $0, next: \.superview)) } ?? []
131 let named = { (name: String) in chain.first { NSStringFromClass(type(of: $0)) == name } }
132 if let button = found as? UIControl, button.showsMenuAsPrimaryAction, let menu = button.contextMenuInteraction {
116133 let middle = CGPoint(x: button.bounds.midX, y: button.bounds.midY)
117134 menu.perform(Selector(("_presentMenuAtLocation:")), with: NSValue(cgPoint: middle))
135 } else if let cell = chain.lazy.compactMap({ $0 as? UICollectionViewCell }).first,
136 let list = chain.lazy.compactMap({ $0 as? UICollectionView }).first, let index = list.indexPath(for: cell)
137 {
138 // A menu's item or a list's row, as a tap picks it.
139 if let items = named("_UIContextMenuListView"), let menu = named("_UIContextMenuView"),
140 let item = items.perform(Selector(("elementAtIndexPath:")), with: index)?.takeUnretainedValue()
141 {
142 menu.perform(Selector(("_handleSelectionForElement:")), with: item)
143 } else {
144 list.selectItem(at: index, animated: false, scrollPosition: [])
145 list.delegate?.collectionView?(list, didSelectItemAt: index)
146 }
118147 } else if !found.accessibilityActivate(), let control = found as? UIControl {
119148 control.sendActions(for: .primaryActionTriggered)
120149 }
......@@ -156,12 +185,25 @@ extension CanvasView {
156185}
157186
158187extension CanvasView {
159 /// The first view or accessibility element under `root` VoiceOver names `label`.
188 /// The scene's windows, menus' and alerts' among them.
189 fileprivate static var windows: [UIWindow] {
190 UIApplication.shared.connectedScenes.compactMap { $0 as? UIWindowScene }.flatMap(\.windows)
191 }
192
193 /// The frontmost view or accessibility element under `root` VoiceOver names `label`.
160194 fileprivate static func element(_ label: String, in root: NSObject) -> NSObject? {
161195 if let view = root as? UIView, view.isHidden { return nil }
162196 if root.isAccessibilityElement, root.accessibilityLabel == label { return root }
163 let children = ((root as? UIView)?.subviews ?? []) + (root.accessibilityElements as? [NSObject] ?? [])
164 return children.lazy.compactMap { element(label, in: $0) }.first
197 // A loop, as a lazy `first` evaluates each match twice, doubling at every level.
198 for child in children(root) {
199 if let found = element(label, in: child) { return found }
200 }
201 return nil
202 }
203
204 /// `root`'s subviews, frontmost first, then its accessibility elements.
205 fileprivate static func children(_ root: NSObject) -> [NSObject] {
206 ((root as? UIView)?.subviews ?? []).reversed() + (root.accessibilityElements as? [NSObject] ?? [])
165207 }
166208
167209 /// Logs `audit ok`, or each button of the format bar and tag grid VoiceOver can't name.