Flutter Engine
accessibility_bridge.mm
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #import "flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h"
6 
7 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterEngine_Internal.h"
8 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController_Internal.h"
9 #import "flutter/shell/platform/darwin/ios/framework/Source/accessibility_text_entry.h"
10 #import "flutter/shell/platform/darwin/ios/platform_view_ios.h"
11 
12 #pragma GCC diagnostic error "-Wundeclared-selector"
13 
15 
16 namespace flutter {
17 namespace {
18 
19 constexpr int32_t kSemanticObjectIdInvalid = -1;
20 
21 class DefaultIosDelegate : public AccessibilityBridge::IosDelegate {
22  public:
23  bool IsFlutterViewControllerPresentingModalViewController(
24  FlutterViewController* view_controller) override {
25  if (view_controller) {
26  return view_controller.isPresentingViewController;
27  } else {
28  return false;
29  }
30  }
31 
32  void PostAccessibilityNotification(UIAccessibilityNotifications notification,
33  id argument) override {
34  UIAccessibilityPostNotification(notification, argument);
35  }
36 };
37 } // namespace
38 
40  FlutterViewController* view_controller,
42  std::shared_ptr<FlutterPlatformViewsController> platform_views_controller,
43  std::unique_ptr<IosDelegate> ios_delegate)
44  : view_controller_(view_controller),
45  platform_view_(platform_view),
46  platform_views_controller_(platform_views_controller),
47  last_focused_semantics_object_id_(kSemanticObjectIdInvalid),
48  objects_([[NSMutableDictionary alloc] init]),
49  previous_route_id_(0),
50  previous_routes_({}),
51  ios_delegate_(ios_delegate ? std::move(ios_delegate)
52  : std::make_unique<DefaultIosDelegate>()),
53  weak_factory_(this) {
54  accessibility_channel_.reset([[FlutterBasicMessageChannel alloc]
55  initWithName:@"flutter/accessibility"
56  binaryMessenger:platform_view->GetOwnerViewController().get().engine.binaryMessenger
57  codec:[FlutterStandardMessageCodec sharedInstance]]);
58  [accessibility_channel_.get() setMessageHandler:^(id message, FlutterReply reply) {
59  HandleEvent((NSDictionary*)message);
60  }];
61 }
62 
64  [accessibility_channel_.get() setMessageHandler:nil];
65  clearState();
66  view_controller_.view.accessibilityElements = nil;
67 }
68 
69 UIView<UITextInput>* AccessibilityBridge::textInputView() {
70  return [[platform_view_->GetOwnerViewController().get().engine textInputPlugin] textInputView];
71 }
72 
74  last_focused_semantics_object_id_ = id;
75 }
76 
78  if (last_focused_semantics_object_id_ == id) {
79  last_focused_semantics_object_id_ = kSemanticObjectIdInvalid;
80  }
81 }
82 
85  BOOL layoutChanged = NO;
86  BOOL scrollOccured = NO;
87  BOOL needsAnnouncement = NO;
88  for (const auto& entry : actions) {
89  const flutter::CustomAccessibilityAction& action = entry.second;
90  actions_[action.id] = action;
91  }
92  for (const auto& entry : nodes) {
93  const flutter::SemanticsNode& node = entry.second;
94  SemanticsObject* object = GetOrCreateObject(node.id, nodes);
95  layoutChanged = layoutChanged || [object nodeWillCauseLayoutChange:&node];
96  scrollOccured = scrollOccured || [object nodeWillCauseScroll:&node];
97  needsAnnouncement = [object nodeShouldTriggerAnnouncement:&node];
98  [object setSemanticsNode:&node];
99  NSUInteger newChildCount = node.childrenInTraversalOrder.size();
100  NSMutableArray* newChildren =
101  [[[NSMutableArray alloc] initWithCapacity:newChildCount] autorelease];
102  for (NSUInteger i = 0; i < newChildCount; ++i) {
103  SemanticsObject* child = GetOrCreateObject(node.childrenInTraversalOrder[i], nodes);
104  [newChildren addObject:child];
105  }
106  object.children = newChildren;
107  if (node.customAccessibilityActions.size() > 0) {
108  NSMutableArray<FlutterCustomAccessibilityAction*>* accessibilityCustomActions =
109  [[[NSMutableArray alloc] init] autorelease];
110  for (int32_t action_id : node.customAccessibilityActions) {
111  flutter::CustomAccessibilityAction& action = actions_[action_id];
112  if (action.overrideId != -1) {
113  // iOS does not support overriding standard actions, so we ignore any
114  // custom actions that have an override id provided.
115  continue;
116  }
117  NSString* label = @(action.label.data());
118  SEL selector = @selector(onCustomAccessibilityAction:);
119  FlutterCustomAccessibilityAction* customAction =
120  [[[FlutterCustomAccessibilityAction alloc] initWithName:label
121  target:object
122  selector:selector] autorelease];
123  customAction.uid = action_id;
124  [accessibilityCustomActions addObject:customAction];
125  }
126  object.accessibilityCustomActions = accessibilityCustomActions;
127  }
128 
129  if (needsAnnouncement) {
130  // Try to be more polite - iOS 11+ supports
131  // UIAccessibilitySpeechAttributeQueueAnnouncement which should avoid
132  // interrupting system notifications or other elements.
133  // Expectation: roughly match the behavior of polite announcements on
134  // Android.
135  NSString* announcement =
136  [[[NSString alloc] initWithUTF8String:object.node.label.c_str()] autorelease];
137  if (@available(iOS 11.0, *)) {
138  UIAccessibilityPostNotification(
139  UIAccessibilityAnnouncementNotification,
140  [[[NSAttributedString alloc]
141  initWithString:announcement
142  attributes:@{
143  UIAccessibilitySpeechAttributeQueueAnnouncement : @YES
144  }] autorelease]);
145  } else {
146  UIAccessibilityPostNotification(UIAccessibilityAnnouncementNotification, announcement);
147  }
148  }
149  }
150 
151  SemanticsObject* root = objects_.get()[@(kRootNodeId)];
152 
153  bool routeChanged = false;
154  SemanticsObject* lastAdded = nil;
155 
156  if (root) {
157  if (!view_controller_.view.accessibilityElements) {
158  view_controller_.view.accessibilityElements = @[ [root accessibilityContainer] ];
159  }
160  NSMutableArray<SemanticsObject*>* newRoutes = [[[NSMutableArray alloc] init] autorelease];
161  [root collectRoutes:newRoutes];
162  // Finds the last route that is not in the previous routes.
163  for (SemanticsObject* route in newRoutes) {
164  if (std::find(previous_routes_.begin(), previous_routes_.end(), [route uid]) ==
165  previous_routes_.end()) {
166  lastAdded = route;
167  }
168  }
169  // If all the routes are in the previous route, get the last route.
170  if (lastAdded == nil && [newRoutes count] > 0) {
171  int index = [newRoutes count] - 1;
172  lastAdded = [newRoutes objectAtIndex:index];
173  }
174  // There are two cases if lastAdded != nil
175  // 1. lastAdded is not in previous routes. In this case,
176  // [lastAdded uid] != previous_route_id_
177  // 2. All new routes are in previous routes and
178  // lastAdded = newRoutes.last.
179  // In the first case, we need to announce new route. In the second case,
180  // we need to announce if one list is shorter than the other.
181  if (lastAdded != nil &&
182  ([lastAdded uid] != previous_route_id_ || [newRoutes count] != previous_routes_.size())) {
183  previous_route_id_ = [lastAdded uid];
184  routeChanged = true;
185  }
186  previous_routes_.clear();
187  for (SemanticsObject* route in newRoutes) {
188  previous_routes_.push_back([route uid]);
189  }
190  } else {
191  view_controller_.view.accessibilityElements = nil;
192  }
193 
194  NSMutableArray<NSNumber*>* doomed_uids = [NSMutableArray arrayWithArray:[objects_ allKeys]];
195  if (root) {
196  VisitObjectsRecursivelyAndRemove(root, doomed_uids);
197  }
198  [objects_ removeObjectsForKeys:doomed_uids];
199 
200  for (SemanticsObject* object in [objects_ allValues]) {
201  [object accessibilityBridgeDidFinishUpdate];
202  }
203 
204  if (!ios_delegate_->IsFlutterViewControllerPresentingModalViewController(view_controller_)) {
205  layoutChanged = layoutChanged || [doomed_uids count] > 0;
206 
207  if (routeChanged) {
208  NSString* routeName = [lastAdded routeName];
209  ios_delegate_->PostAccessibilityNotification(UIAccessibilityScreenChangedNotification,
210  routeName);
211  }
212 
213  if (layoutChanged) {
214  ios_delegate_->PostAccessibilityNotification(
215  UIAccessibilityLayoutChangedNotification,
216  FindNextFocusableIfNecessary().nativeAccessibility);
217  } else if (scrollOccured) {
218  // TODO(chunhtai): figure out what string to use for notification. At this
219  // point, it is guarantee the previous focused object is still in the tree
220  // so that we don't need to worry about focus lost. (e.g. "Screen 0 of 3")
221  ios_delegate_->PostAccessibilityNotification(
222  UIAccessibilityPageScrolledNotification,
223  FindNextFocusableIfNecessary().nativeAccessibility);
224  }
225  }
226 }
227 
229  platform_view_->DispatchSemanticsAction(uid, action, {});
230 }
231 
235  platform_view_->DispatchSemanticsAction(uid, action, std::move(args));
236 }
237 
239  SemanticsObject* newObject,
240  NSMutableDictionary<NSNumber*, SemanticsObject*>* objects) {
241  // `newObject` should represent the same id as `oldObject`.
242  assert(oldObject.node.id == newObject.uid);
243  NSNumber* nodeId = @(oldObject.node.id);
244  NSUInteger positionInChildlist = [oldObject.parent.children indexOfObject:oldObject];
245  [objects removeObjectForKey:nodeId];
246  [oldObject.parent replaceChildAtIndex:positionInChildlist withChild:newObject];
247  objects[nodeId] = newObject;
248 }
249 
254  // Text fields are backed by objects that implement UITextInput.
255  return [[[TextInputSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id] autorelease];
258  return [[[FlutterSwitchSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id] autorelease];
260  return [[[FlutterScrollableSemanticsObject alloc] initWithBridge:weak_ptr
261  uid:node.id] autorelease];
262  } else if (node.IsPlatformViewNode()) {
264  initWithBridge:weak_ptr
265  uid:node.id
266  platformView:weak_ptr->GetPlatformViewsController()->GetPlatformViewByID(
267  node.platformViewId)] autorelease];
268  } else {
269  return [[[FlutterSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id] autorelease];
270  }
271 }
272 
273 static bool DidFlagChange(const flutter::SemanticsNode& oldNode,
274  const flutter::SemanticsNode& newNode,
276  return oldNode.HasFlag(flag) != newNode.HasFlag(flag);
277 }
278 
279 SemanticsObject* AccessibilityBridge::GetOrCreateObject(int32_t uid,
281  SemanticsObject* object = objects_.get()[@(uid)];
282  if (!object) {
283  object = CreateObject(updates[uid], GetWeakPtr());
284  objects_.get()[@(uid)] = object;
285  } else {
286  // Existing node case
287  auto nodeEntry = updates.find(object.node.id);
288  if (nodeEntry != updates.end()) {
289  // There's an update for this node
290  flutter::SemanticsNode node = nodeEntry->second;
291  if (DidFlagChange(object.node, node, flutter::SemanticsFlags::kIsTextField) ||
296  // The node changed its type. In this case, we cannot reuse the existing
297  // SemanticsObject implementation. Instead, we replace it with a new
298  // instance.
299  SemanticsObject* newSemanticsObject = CreateObject(node, GetWeakPtr());
300  ReplaceSemanticsObject(object, newSemanticsObject, objects_.get());
301  object = newSemanticsObject;
302  }
303  }
304  }
305  return object;
306 }
307 
308 void AccessibilityBridge::VisitObjectsRecursivelyAndRemove(SemanticsObject* object,
309  NSMutableArray<NSNumber*>* doomed_uids) {
310  [doomed_uids removeObject:@(object.uid)];
311  for (SemanticsObject* child in [object children])
312  VisitObjectsRecursivelyAndRemove(child, doomed_uids);
313 }
314 
315 SemanticsObject* AccessibilityBridge::FindNextFocusableIfNecessary() {
316  // This property will be -1 if the focus is outside of the flutter
317  // application. In this case, we should not refocus anything.
318  if (last_focused_semantics_object_id_ == kSemanticObjectIdInvalid) {
319  return nil;
320  }
321 
322  // Tries to refocus the previous focused semantics object to avoid random jumps.
323  return FindFirstFocusable([objects_.get() objectForKey:@(last_focused_semantics_object_id_)]);
324 }
325 
326 SemanticsObject* AccessibilityBridge::FindFirstFocusable(SemanticsObject* parent) {
327  SemanticsObject* currentObject = parent ?: objects_.get()[@(kRootNodeId)];
328  ;
329  if (!currentObject) {
330  return nil;
331  }
332 
333  if (currentObject.isAccessibilityElement) {
334  return currentObject;
335  }
336 
337  for (SemanticsObject* child in [currentObject children]) {
338  SemanticsObject* candidate = FindFirstFocusable(child);
339  if (candidate) {
340  return candidate;
341  }
342  }
343  return nil;
344 }
345 
346 void AccessibilityBridge::HandleEvent(NSDictionary<NSString*, id>* annotatedEvent) {
347  NSString* type = annotatedEvent[@"type"];
348  if ([type isEqualToString:@"announce"]) {
349  NSString* message = annotatedEvent[@"data"][@"message"];
350  ios_delegate_->PostAccessibilityNotification(UIAccessibilityAnnouncementNotification, message);
351  }
352 }
353 
355  return weak_factory_.GetWeakPtr();
356 }
357 
359  [objects_ removeAllObjects];
360  previous_route_id_ = 0;
361  previous_routes_.clear();
362 }
363 
364 } // namespace flutter
G_BEGIN_DECLS FlValue * args
static bool DidFlagChange(const flutter::SemanticsNode &oldNode, const flutter::SemanticsNode &newNode, SemanticsFlags flag)
bool IsPlatformViewNode() const
void DispatchSemanticsAction(int32_t id, SemanticsAction action, fml::MallocMapping args)
Used by embedders to dispatch an accessibility action to a running isolate hosted by the engine...
std::unique_ptr< flutter::PlatformViewIOS > platform_view
void DispatchSemanticsAction(int32_t id, flutter::SemanticsAction action) override
std::vector< int32_t > customAccessibilityActions
void AccessibilityObjectDidBecomeFocused(int32_t id) override
static constexpr int32_t kRootNodeId
The ID of the root node in the accessibility tree. In Flutter,.
FlutterTextInputPlugin * textInputPlugin
void reset(NST object=nil)
std::unordered_map< int32_t, SemanticsNode > SemanticsNodeUpdates
NSArray< SemanticsObject * > * children
std::vector< int32_t > childrenInTraversalOrder
bool HasFlag(SemanticsFlags flag) const
fml::WeakPtr< AccessibilityBridge > GetWeakPtr()
AccessibilityBridge(std::unique_ptr< AccessibilityBridgeDelegate > delegate)
Creates a new instance of a accessibility bridge.
static void ReplaceSemanticsObject(SemanticsObject *oldObject, SemanticsObject *newObject, NSMutableDictionary< NSNumber *, SemanticsObject *> *objects)
GdkEventType type
Definition: fl_view.cc:80
uint32_t * target
void AccessibilityObjectDidLoseFocus(int32_t id) override
SemanticsAction action
SemanticsObject * parent
FlutterSemanticsFlag flag
NS_ASSUME_NONNULL_BEGIN typedef void(^ FlutterReply)(id _Nullable reply)
int BOOL
Definition: windows_types.h:37
void UpdateSemantics(flutter::SemanticsNodeUpdates nodes, flutter::CustomAccessibilityActionUpdates actions)
A Mapping like NonOwnedMapping, but uses Free as its release proc.
Definition: mapping.h:144
static SemanticsObject * CreateObject(const flutter::SemanticsNode &node, fml::WeakPtr< AccessibilityBridge > weak_ptr)
flutter::SemanticsNode node
int32_t id
std::unordered_map< int32_t, CustomAccessibilityAction > CustomAccessibilityActionUpdates
UIView< UITextInput > * textInputView() override
fml::WeakPtr< FlutterViewController > GetOwnerViewController() const