1 // Copyright 2017 The Chromium Embedded Framework Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be found
3 // in the LICENSE file.
4
5 #include "libcef/browser/osr/osr_accessibility_util.h"
6
7 #include <algorithm>
8 #include <string>
9 #include <utility>
10
11 #include "base/json/string_escape.h"
12 #include "base/stl_util.h"
13 #include "base/strings/string_number_conversions.h"
14 #include "base/strings/stringprintf.h"
15 #include "content/public/browser/ax_event_notification_details.h"
16 #include "ui/accessibility/ax_enum_util.h"
17 #include "ui/accessibility/ax_enums.mojom.h"
18 #include "ui/accessibility/ax_text_utils.h"
19 #include "ui/accessibility/ax_tree_update.h"
20 #include "ui/gfx/geometry/transform.h"
21
22 namespace {
23 using ui::ToString;
24
25 template <typename T>
26 CefRefPtr<CefListValue> ToCefValue(const std::vector<T>& vecData);
27
28 template <>
ToCefValue(const std::vector<int> & vecData)29 CefRefPtr<CefListValue> ToCefValue<int>(const std::vector<int>& vecData) {
30 CefRefPtr<CefListValue> value = CefListValue::Create();
31 for (size_t i = 0; i < vecData.size(); i++)
32 value->SetInt(i, vecData[i]);
33
34 return value;
35 }
36
37 // Helper function for AXNodeData::ToCefValue - Converts AXState attributes to
38 // CefListValue.
ToCefValue(uint32_t state)39 CefRefPtr<CefListValue> ToCefValue(uint32_t state) {
40 CefRefPtr<CefListValue> value = CefListValue::Create();
41
42 int index = 0;
43 // Iterate and find which states are set.
44 for (unsigned i = static_cast<unsigned>(ax::mojom::Role::kMinValue) + 1;
45 i <= static_cast<unsigned>(ax::mojom::Role::kMaxValue); i++) {
46 if (state & (1 << i))
47 value->SetString(index++, ToString(static_cast<ax::mojom::State>(i)));
48 }
49 return value;
50 }
51
52 // Helper function for AXNodeData::ToCefValue - converts GfxRect to
53 // CefDictionaryValue.
ToCefValue(const gfx::RectF & bounds)54 CefRefPtr<CefDictionaryValue> ToCefValue(const gfx::RectF& bounds) {
55 CefRefPtr<CefDictionaryValue> value = CefDictionaryValue::Create();
56 value->SetDouble("x", bounds.x());
57 value->SetDouble("y", bounds.y());
58 value->SetDouble("width", bounds.width());
59 value->SetDouble("height", bounds.height());
60 return value;
61 }
62
63 // Helper Functor for adding AxNodeData::attributes to AXNodeData::ToCefValue.
64 struct PopulateAxNodeAttributes {
65 CefRefPtr<CefDictionaryValue> attributes;
66
PopulateAxNodeAttributes__anon881cfd350111::PopulateAxNodeAttributes67 explicit PopulateAxNodeAttributes(CefRefPtr<CefDictionaryValue> attrs)
68 : attributes(attrs) {}
69
70 // Int Attributes
operator ()__anon881cfd350111::PopulateAxNodeAttributes71 void operator()(const std::pair<ax::mojom::IntAttribute, int32_t> attr) {
72 if (attr.first == ax::mojom::IntAttribute::kNone)
73 return;
74
75 switch (attr.first) {
76 case ax::mojom::IntAttribute::kNone:
77 break;
78 case ax::mojom::IntAttribute::kScrollX:
79 case ax::mojom::IntAttribute::kScrollXMin:
80 case ax::mojom::IntAttribute::kScrollXMax:
81 case ax::mojom::IntAttribute::kScrollY:
82 case ax::mojom::IntAttribute::kScrollYMin:
83 case ax::mojom::IntAttribute::kScrollYMax:
84 case ax::mojom::IntAttribute::kHasPopup:
85 case ax::mojom::IntAttribute::kHierarchicalLevel:
86 case ax::mojom::IntAttribute::kTextSelStart:
87 case ax::mojom::IntAttribute::kTextSelEnd:
88 case ax::mojom::IntAttribute::kAriaColumnCount:
89 case ax::mojom::IntAttribute::kAriaCellColumnIndex:
90 case ax::mojom::IntAttribute::kAriaRowCount:
91 case ax::mojom::IntAttribute::kAriaCellRowIndex:
92 case ax::mojom::IntAttribute::kTableRowCount:
93 case ax::mojom::IntAttribute::kTableColumnCount:
94 case ax::mojom::IntAttribute::kTableCellColumnIndex:
95 case ax::mojom::IntAttribute::kTableCellRowIndex:
96 case ax::mojom::IntAttribute::kTableCellColumnSpan:
97 case ax::mojom::IntAttribute::kTableCellRowSpan:
98 case ax::mojom::IntAttribute::kTableColumnHeaderId:
99 case ax::mojom::IntAttribute::kTableColumnIndex:
100 case ax::mojom::IntAttribute::kTableHeaderId:
101 case ax::mojom::IntAttribute::kTableRowHeaderId:
102 case ax::mojom::IntAttribute::kTableRowIndex:
103 case ax::mojom::IntAttribute::kActivedescendantId:
104 case ax::mojom::IntAttribute::kInPageLinkTargetId:
105 case ax::mojom::IntAttribute::kErrormessageId:
106 case ax::mojom::IntAttribute::kDOMNodeId:
107 case ax::mojom::IntAttribute::kDropeffect:
108 case ax::mojom::IntAttribute::kMemberOfId:
109 case ax::mojom::IntAttribute::kNextFocusId:
110 case ax::mojom::IntAttribute::kNextOnLineId:
111 case ax::mojom::IntAttribute::kPreviousFocusId:
112 case ax::mojom::IntAttribute::kPreviousOnLineId:
113 case ax::mojom::IntAttribute::kSetSize:
114 case ax::mojom::IntAttribute::kPosInSet:
115 case ax::mojom::IntAttribute::kPopupForId:
116 attributes->SetInt(ToString(attr.first), attr.second);
117 break;
118 case ax::mojom::IntAttribute::kDefaultActionVerb:
119 attributes->SetString(
120 ToString(attr.first),
121 ui::ToString(
122 static_cast<ax::mojom::DefaultActionVerb>(attr.second)));
123 break;
124 case ax::mojom::IntAttribute::kInvalidState: {
125 auto state = static_cast<ax::mojom::InvalidState>(attr.second);
126 if (ax::mojom::InvalidState::kNone != state) {
127 attributes->SetString(ToString(attr.first), ToString(state));
128 }
129 } break;
130 case ax::mojom::IntAttribute::kCheckedState: {
131 auto state = static_cast<ax::mojom::CheckedState>(attr.second);
132 if (ax::mojom::CheckedState::kNone != state) {
133 attributes->SetString(ToString(attr.first), ToString(state));
134 }
135 } break;
136 case ax::mojom::IntAttribute::kRestriction:
137 attributes->SetString(
138 ToString(attr.first),
139 ToString(static_cast<ax::mojom::Restriction>(attr.second)));
140 break;
141 case ax::mojom::IntAttribute::kListStyle: {
142 auto state = static_cast<ax::mojom::ListStyle>(attr.second);
143 if (ax::mojom::ListStyle::kNone != state) {
144 attributes->SetString(ToString(attr.first), ToString(state));
145 }
146 } break;
147 case ax::mojom::IntAttribute::kSortDirection: {
148 auto state = static_cast<ax::mojom::SortDirection>(attr.second);
149 if (ax::mojom::SortDirection::kNone != state) {
150 attributes->SetString(ToString(attr.first), ToString(state));
151 }
152 } break;
153 case ax::mojom::IntAttribute::kTextAlign: {
154 auto state = static_cast<ax::mojom::TextAlign>(attr.second);
155 if (ax::mojom::TextAlign::kNone != state) {
156 attributes->SetString(ToString(attr.first), ToString(state));
157 }
158 } break;
159 case ax::mojom::IntAttribute::kNameFrom:
160 attributes->SetString(
161 ToString(attr.first),
162 ToString(static_cast<ax::mojom::NameFrom>(attr.second)));
163 break;
164 case ax::mojom::IntAttribute::kColorValue:
165 case ax::mojom::IntAttribute::kBackgroundColor:
166 case ax::mojom::IntAttribute::kColor:
167 attributes->SetString(ToString(attr.first),
168 base::StringPrintf("0x%X", attr.second));
169 break;
170 case ax::mojom::IntAttribute::kDescriptionFrom:
171 attributes->SetString(
172 ToString(attr.first),
173 ToString(static_cast<ax::mojom::DescriptionFrom>(attr.second)));
174 break;
175 case ax::mojom::IntAttribute::kAriaCurrentState: {
176 auto state = static_cast<ax::mojom::AriaCurrentState>(attr.second);
177 if (ax::mojom::AriaCurrentState::kNone != state) {
178 attributes->SetString(ToString(attr.first), ToString(state));
179 }
180 } break;
181 case ax::mojom::IntAttribute::kTextDirection: {
182 auto state = static_cast<ax::mojom::WritingDirection>(attr.second);
183 if (ax::mojom::WritingDirection::kNone != state) {
184 attributes->SetString(ToString(attr.first), ToString(state));
185 }
186 } break;
187 case ax::mojom::IntAttribute::kTextPosition: {
188 auto state = static_cast<ax::mojom::TextPosition>(attr.second);
189 if (ax::mojom::TextPosition::kNone != state) {
190 attributes->SetString(ToString(attr.first), ToString(state));
191 }
192 } break;
193 case ax::mojom::IntAttribute::kTextStyle: {
194 static ax::mojom::TextStyle textStyleArr[] = {
195 ax::mojom::TextStyle::kBold, ax::mojom::TextStyle::kItalic,
196 ax::mojom::TextStyle::kUnderline,
197 ax::mojom::TextStyle::kLineThrough,
198 ax::mojom::TextStyle::kOverline};
199
200 CefRefPtr<CefListValue> list = CefListValue::Create();
201 int index = 0;
202 // Iterate and find which states are set.
203 for (unsigned i = 0; i < base::size(textStyleArr); i++) {
204 if (attr.second & static_cast<int>(textStyleArr[i]))
205 list->SetString(index++, ToString(textStyleArr[i]));
206 }
207 attributes->SetList(ToString(attr.first), list);
208 } break;
209 case ax::mojom::IntAttribute::kTextOverlineStyle:
210 case ax::mojom::IntAttribute::kTextStrikethroughStyle:
211 case ax::mojom::IntAttribute::kTextUnderlineStyle: {
212 auto state = static_cast<ax::mojom::TextDecorationStyle>(attr.second);
213 if (ax::mojom::TextDecorationStyle::kNone != state) {
214 attributes->SetString(ToString(attr.first), ToString(state));
215 }
216 } break;
217 case ax::mojom::IntAttribute::kAriaCellColumnSpan:
218 case ax::mojom::IntAttribute::kAriaCellRowSpan:
219 case ax::mojom::IntAttribute::kImageAnnotationStatus: {
220 // TODO(cef): Implement support for Image Annotation Status,
221 // kAriaCellColumnSpan and kAriaCellRowSpan
222 } break;
223 }
224 }
225
226 // Set Bool Attributes.
operator ()__anon881cfd350111::PopulateAxNodeAttributes227 void operator()(const std::pair<ax::mojom::BoolAttribute, bool> attr) {
228 if (attr.first != ax::mojom::BoolAttribute::kNone)
229 attributes->SetBool(ToString(attr.first), attr.second);
230 }
231 // Set String Attributes.
operator ()__anon881cfd350111::PopulateAxNodeAttributes232 void operator()(
233 const std::pair<ax::mojom::StringAttribute, std::string>& attr) {
234 if (attr.first != ax::mojom::StringAttribute::kNone)
235 attributes->SetString(ToString(attr.first), attr.second);
236 }
237 // Set Float attributes.
operator ()__anon881cfd350111::PopulateAxNodeAttributes238 void operator()(const std::pair<ax::mojom::FloatAttribute, float>& attr) {
239 if (attr.first != ax::mojom::FloatAttribute::kNone)
240 attributes->SetDouble(ToString(attr.first), attr.second);
241 }
242
243 // Set Int list attributes.
operator ()__anon881cfd350111::PopulateAxNodeAttributes244 void operator()(const std::pair<ax::mojom::IntListAttribute,
245 std::vector<int32_t>>& attr) {
246 if (attr.first != ax::mojom::IntListAttribute::kNone) {
247 CefRefPtr<CefListValue> list;
248
249 if (ax::mojom::IntListAttribute::kMarkerTypes == attr.first) {
250 list = CefListValue::Create();
251 int index = 0;
252 for (size_t i = 0; i < attr.second.size(); ++i) {
253 auto type = static_cast<ax::mojom::MarkerType>(attr.second[i]);
254
255 if (type == ax::mojom::MarkerType::kNone)
256 continue;
257
258 static ax::mojom::MarkerType marktypeArr[] = {
259 ax::mojom::MarkerType::kSpelling, ax::mojom::MarkerType::kGrammar,
260 ax::mojom::MarkerType::kTextMatch};
261
262 // Iterate and find which markers are set.
263 for (unsigned j = 0; j < base::size(marktypeArr); j++) {
264 if (attr.second[i] & static_cast<int>(marktypeArr[j]))
265 list->SetString(index++, ToString(marktypeArr[j]));
266 }
267 }
268 } else {
269 list = ToCefValue(attr.second);
270 }
271 attributes->SetList(ToString(attr.first), list);
272 }
273 }
274 };
275
276 // Converts AXNodeData to CefDictionaryValue(like AXNodeData::ToString).
ToCefValue(const ui::AXNodeData & node)277 CefRefPtr<CefDictionaryValue> ToCefValue(const ui::AXNodeData& node) {
278 CefRefPtr<CefDictionaryValue> value = CefDictionaryValue::Create();
279
280 if (node.id != -1)
281 value->SetInt("id", node.id);
282
283 value->SetString("role", ToString(node.role));
284 value->SetList("state", ToCefValue(node.state));
285
286 if (node.relative_bounds.offset_container_id != -1) {
287 value->SetInt("offset_container_id",
288 node.relative_bounds.offset_container_id);
289 }
290
291 value->SetDictionary("location", ToCefValue(node.relative_bounds.bounds));
292
293 // Transform matrix is private, so we set the string that Clients can parse
294 // and use if needed.
295 if (node.relative_bounds.transform &&
296 !node.relative_bounds.transform->IsIdentity()) {
297 value->SetString("transform", node.relative_bounds.transform->ToString());
298 }
299
300 if (!node.child_ids.empty()) {
301 value->SetList("child_ids", ToCefValue(node.child_ids));
302 }
303
304 CefRefPtr<CefListValue> actions_strings;
305 size_t actions_idx = 0;
306 for (int action_index = static_cast<int>(ax::mojom::Action::kMinValue) + 1;
307 action_index <= static_cast<int>(ax::mojom::Action::kMaxValue);
308 ++action_index) {
309 auto action = static_cast<ax::mojom::Action>(action_index);
310 if (node.HasAction(action)) {
311 if (!actions_strings)
312 actions_strings = CefListValue::Create();
313 actions_strings->SetString(actions_idx++, ToString(action));
314 }
315 }
316 if (actions_strings)
317 value->SetList("actions", actions_strings);
318
319 CefRefPtr<CefDictionaryValue> attributes = CefDictionaryValue::Create();
320 PopulateAxNodeAttributes func(attributes);
321
322 // Poupulate Int Attributes.
323 std::for_each(node.int_attributes.begin(), node.int_attributes.end(), func);
324
325 // Poupulate String Attributes.
326 std::for_each(node.string_attributes.begin(), node.string_attributes.end(),
327 func);
328
329 // Poupulate Float Attributes.
330 std::for_each(node.float_attributes.begin(), node.float_attributes.end(),
331 func);
332
333 // Poupulate Bool Attributes.
334 std::for_each(node.bool_attributes.begin(), node.bool_attributes.end(), func);
335
336 // Populate int list attributes.
337 std::for_each(node.intlist_attributes.begin(), node.intlist_attributes.end(),
338 func);
339
340 value->SetDictionary("attributes", attributes);
341
342 return value;
343 }
344
345 // Converts AXTreeData to CefDictionaryValue(like AXTreeData::ToString).
ToCefValue(const ui::AXTreeData & treeData)346 CefRefPtr<CefDictionaryValue> ToCefValue(const ui::AXTreeData& treeData) {
347 CefRefPtr<CefDictionaryValue> value = CefDictionaryValue::Create();
348
349 if (!treeData.tree_id.ToString().empty())
350 value->SetString("tree_id", treeData.tree_id.ToString());
351
352 if (!treeData.parent_tree_id.ToString().empty())
353 value->SetString("parent_tree_id", treeData.parent_tree_id.ToString());
354
355 if (!treeData.focused_tree_id.ToString().empty())
356 value->SetString("focused_tree_id", treeData.focused_tree_id.ToString());
357
358 if (!treeData.doctype.empty())
359 value->SetString("doctype", treeData.doctype);
360
361 value->SetBool("loaded", treeData.loaded);
362
363 if (treeData.loading_progress != 0.0)
364 value->SetDouble("loading_progress", treeData.loading_progress);
365
366 if (!treeData.mimetype.empty())
367 value->SetString("mimetype", treeData.mimetype);
368 if (!treeData.url.empty())
369 value->SetString("url", treeData.url);
370 if (!treeData.title.empty())
371 value->SetString("title", treeData.title);
372
373 if (treeData.sel_anchor_object_id != -1) {
374 value->SetInt("sel_anchor_object_id", treeData.sel_anchor_object_id);
375 value->SetInt("sel_anchor_offset", treeData.sel_anchor_offset);
376 value->SetString("sel_anchor_affinity",
377 ToString(treeData.sel_anchor_affinity));
378 }
379 if (treeData.sel_focus_object_id != -1) {
380 value->SetInt("sel_focus_object_id", treeData.sel_anchor_object_id);
381 value->SetInt("sel_focus_offset", treeData.sel_anchor_offset);
382 value->SetString("sel_focus_affinity",
383 ToString(treeData.sel_anchor_affinity));
384 }
385
386 if (treeData.focus_id != -1)
387 value->SetInt("focus_id", treeData.focus_id);
388
389 return value;
390 }
391
392 // Converts AXTreeUpdate to CefDictionaryValue(like AXTreeUpdate::ToString).
ToCefValue(const ui::AXTreeUpdate & update)393 CefRefPtr<CefDictionaryValue> ToCefValue(const ui::AXTreeUpdate& update) {
394 CefRefPtr<CefDictionaryValue> value = CefDictionaryValue::Create();
395
396 if (update.has_tree_data) {
397 value->SetBool("has_tree_data", true);
398 value->SetDictionary("tree_data", ToCefValue(update.tree_data));
399 }
400
401 if (update.node_id_to_clear != 0)
402 value->SetInt("node_id_to_clear", update.node_id_to_clear);
403
404 if (update.root_id != 0)
405 value->SetInt("root_id", update.root_id);
406
407 value->SetList("nodes", ToCefValue(update.nodes));
408
409 return value;
410 }
411
412 // Converts AXEvent to CefDictionaryValue.
ToCefValue(const ui::AXEvent & event)413 CefRefPtr<CefDictionaryValue> ToCefValue(const ui::AXEvent& event) {
414 CefRefPtr<CefDictionaryValue> value = CefDictionaryValue::Create();
415
416 if (event.event_type != ax::mojom::Event::kNone)
417 value->SetString("event_type", ToString(event.event_type));
418
419 if (event.id != -1)
420 value->SetInt("id", event.id);
421
422 if (event.event_from != ax::mojom::EventFrom::kNone)
423 value->SetString("event_from", ToString(event.event_from));
424
425 if (event.action_request_id != -1)
426 value->SetInt("action_request_id", event.action_request_id);
427
428 return value;
429 }
430
431 // Convert AXEventNotificationDetails to CefDictionaryValue.
ToCefValue(const content::AXEventNotificationDetails & eventData)432 CefRefPtr<CefDictionaryValue> ToCefValue(
433 const content::AXEventNotificationDetails& eventData) {
434 CefRefPtr<CefDictionaryValue> value = CefDictionaryValue::Create();
435
436 if (!eventData.ax_tree_id.ToString().empty())
437 value->SetString("ax_tree_id", eventData.ax_tree_id.ToString());
438
439 if (eventData.updates.size() > 0) {
440 CefRefPtr<CefListValue> updates = CefListValue::Create();
441 updates->SetSize(eventData.updates.size());
442 size_t i = 0;
443 for (const auto& update : eventData.updates) {
444 updates->SetDictionary(i++, ToCefValue(update));
445 }
446 value->SetList("updates", updates);
447 }
448
449 if (eventData.events.size() > 0) {
450 CefRefPtr<CefListValue> events = CefListValue::Create();
451 events->SetSize(eventData.events.size());
452 size_t i = 0;
453 for (const auto& event : eventData.events) {
454 events->SetDictionary(i++, ToCefValue(event));
455 }
456 value->SetList("events", events);
457 }
458
459 return value;
460 }
461
462 // Convert AXRelativeBounds to CefDictionaryValue. Similar to
463 // AXRelativeBounds::ToString. See that for more details
ToCefValue(const ui::AXRelativeBounds & location)464 CefRefPtr<CefDictionaryValue> ToCefValue(const ui::AXRelativeBounds& location) {
465 CefRefPtr<CefDictionaryValue> value = CefDictionaryValue::Create();
466
467 if (location.offset_container_id != -1)
468 value->SetInt("offset_container_id", location.offset_container_id);
469
470 value->SetDictionary("bounds", ToCefValue(location.bounds));
471
472 // Transform matrix is private, so we set the string that Clients can parse
473 // and use if needed.
474 if (location.transform && !location.transform->IsIdentity())
475 value->SetString("transform", location.transform->ToString());
476
477 return value;
478 }
479
480 // Convert AXLocationChangeNotificationDetails to CefDictionaryValue.
ToCefValue(const content::AXLocationChangeNotificationDetails & locData)481 CefRefPtr<CefDictionaryValue> ToCefValue(
482 const content::AXLocationChangeNotificationDetails& locData) {
483 CefRefPtr<CefDictionaryValue> value = CefDictionaryValue::Create();
484
485 if (locData.id != -1)
486 value->SetInt("id", locData.id);
487
488 if (!locData.ax_tree_id.ToString().empty())
489 value->SetString("ax_tree_id", locData.ax_tree_id.ToString());
490
491 value->SetDictionary("new_location", ToCefValue(locData.new_location));
492
493 return value;
494 }
495
496 template <typename T>
ToCefValue(const std::vector<T> & vecData)497 CefRefPtr<CefListValue> ToCefValue(const std::vector<T>& vecData) {
498 CefRefPtr<CefListValue> value = CefListValue::Create();
499
500 for (size_t i = 0; i < vecData.size(); i++)
501 value->SetDictionary(i, ToCefValue(vecData[i]));
502
503 return value;
504 }
505
506 } // namespace
507
508 namespace osr_accessibility_util {
509
ParseAccessibilityEventData(const content::AXEventNotificationDetails & data)510 CefRefPtr<CefValue> ParseAccessibilityEventData(
511 const content::AXEventNotificationDetails& data) {
512 CefRefPtr<CefValue> value = CefValue::Create();
513 value->SetDictionary(ToCefValue(data));
514 return value;
515 }
516
ParseAccessibilityLocationData(const std::vector<content::AXLocationChangeNotificationDetails> & data)517 CefRefPtr<CefValue> ParseAccessibilityLocationData(
518 const std::vector<content::AXLocationChangeNotificationDetails>& data) {
519 CefRefPtr<CefValue> value = CefValue::Create();
520 value->SetList(ToCefValue(data));
521 return value;
522 }
523
524 } // namespace osr_accessibility_util
525