2 * Copyright (c) 2021 Samsung Electronics Co., Ltd.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
19 #include <dali/internal/accessibility/bridge/bridge-accessible.h>
24 //comment out 2 lines below to get more logs
26 #define LOG() _LoggerEmpty()
28 using namespace Dali::Accessibility;
30 #define GET_NAVIGABLE_AT_POINT_MAX_RECURSION_DEPTH 10000
32 BridgeAccessible::BridgeAccessible()
36 void BridgeAccessible::RegisterInterfaces()
38 DBus::DBusInterfaceDescription desc{AtspiDbusInterfaceAccessible};
39 AddGetPropertyToInterface(desc, "ChildCount", &BridgeAccessible::GetChildCount);
40 AddGetPropertyToInterface(desc, "Name", &BridgeAccessible::GetName);
41 AddGetPropertyToInterface(desc, "Description", &BridgeAccessible::GetDescription);
42 AddGetPropertyToInterface(desc, "Parent", &BridgeAccessible::GetParent);
43 AddFunctionToInterface(desc, "GetRole", &BridgeAccessible::GetRole);
44 AddFunctionToInterface(desc, "GetRoleName", &BridgeAccessible::GetRoleName);
45 AddFunctionToInterface(desc, "GetLocalizedRoleName", &BridgeAccessible::GetLocalizedRoleName);
46 AddFunctionToInterface(desc, "GetState", &BridgeAccessible::GetStates);
47 AddFunctionToInterface(desc, "GetAttributes", &BridgeAccessible::GetAttributes);
48 AddFunctionToInterface(desc, "GetInterfaces", &BridgeAccessible::GetInterfaces);
49 AddFunctionToInterface(desc, "GetChildAtIndex", &BridgeAccessible::GetChildAtIndex);
50 AddFunctionToInterface(desc, "GetChildren", &BridgeAccessible::GetChildren);
51 AddFunctionToInterface(desc, "GetIndexInParent", &BridgeAccessible::GetIndexInParent);
52 AddFunctionToInterface(desc, "GetNavigableAtPoint", &BridgeAccessible::GetNavigableAtPoint);
53 AddFunctionToInterface(desc, "GetNeighbor", &BridgeAccessible::GetNeighbor);
54 AddFunctionToInterface(desc, "GetDefaultLabelInfo", &BridgeAccessible::GetDefaultLabelInfo);
55 AddFunctionToInterface(desc, "DoGesture", &BridgeAccessible::DoGesture);
56 AddFunctionToInterface(desc, "GetReadingMaterial", &BridgeAccessible::GetReadingMaterial);
57 AddFunctionToInterface(desc, "GetRelationSet", &BridgeAccessible::GetRelationSet);
58 dbusServer.addInterface("/", desc, true);
61 static bool AcceptObjectCheckRole(Component* obj)
65 switch(obj->GetRole())
67 case Role::APPLICATION:
69 case Role::SCROLL_PANE:
70 case Role::SPLIT_PANE:
77 case Role::REDUNDANT_OBJECT:
78 case Role::COLOR_CHOOSER:
79 case Role::TREE_TABLE:
80 case Role::PAGE_TAB_LIST:
82 case Role::SPIN_BUTTON:
83 case Role::INPUT_METHOD_WINDOW:
86 case Role::NOTIFICATION:
87 case Role::DATE_EDITOR:
101 static bool AcceptObjectCheckRelations(Component* obj)
103 auto r = obj->GetRelationSet();
105 for(const auto& it : r)
106 if(it.relationType == RelationType::CONTROLLED_BY)
112 static Component* GetScrollableParent(Accessible* obj)
116 obj = obj->GetParent();
117 auto comp = dynamic_cast<Component*>(obj);
118 if(comp && comp->IsScrollable())
124 static bool ObjectIsItem(Component* obj)
128 auto role = obj->GetRole();
129 return role == Role::LIST_ITEM || role == Role::MENU_ITEM;
132 static bool ObjectIsCollapsed(Component* obj)
136 const auto states = obj->GetStates();
137 return states[State::EXPANDABLE] && !states[State::EXPANDED];
140 static bool OobjectIsZeroSize(Component* obj)
144 auto extents = obj->GetExtents(CoordType::WINDOW);
145 return extents.height == 0 || extents.width == 0;
148 static bool AcceptObject(Component* obj)
152 const auto states = obj->GetStates();
153 if(!states[State::VISIBLE])
155 if(!AcceptObjectCheckRole(obj))
157 if(!AcceptObjectCheckRelations(obj))
159 if(!states[State::HIGHLIGHTABLE])
162 if(GetScrollableParent(obj) != nullptr)
164 auto parent = dynamic_cast<Component*>(obj->GetParent());
168 return !ObjectIsItem(obj) || !ObjectIsCollapsed(parent);
173 if(OobjectIsZeroSize(obj))
177 if(!states[State::SHOWING])
185 static bool AcceptObject(Accessible* obj)
187 auto c = dynamic_cast<Component*>(obj);
188 return AcceptObject(c);
191 static std::string objDump(Component* obj)
195 std::ostringstream o;
196 auto e = obj->GetExtents(CoordType::SCREEN);
197 o << "name: " << obj->GetName() << " extent: (" << e.x << ", "
198 << e.y << "), [" << e.width << ", " << e.height << "]";
202 Component* BridgeAccessible::GetObjectInRelation(Accessible* obj, RelationType ralationType)
206 for(auto& relation : obj->GetRelationSet())
208 if(relation.relationType == ralationType)
210 for(auto& address : relation.targets)
212 auto component = dynamic_cast<Component*>(Find(address));
221 static std::string makeIndent(unsigned int maxRecursionDepth)
223 return std::string(GET_NAVIGABLE_AT_POINT_MAX_RECURSION_DEPTH - maxRecursionDepth, ' ');
226 Component* BridgeAccessible::CalculateNavigableAccessibleAtPoint(Accessible* root, Point p, CoordType cType, unsigned int maxRecursionDepth)
228 if(!root || maxRecursionDepth == 0)
230 auto root_component = dynamic_cast<Component*>(root);
231 LOG() << "CalculateNavigableAccessibleAtPoint: checking: " << makeIndent(maxRecursionDepth) << objDump(root_component);
233 if(root_component && !root_component->Contains(p, cType))
236 auto children = root->GetChildren();
237 for(auto childIt = children.rbegin(); childIt != children.rend(); childIt++)
239 //check recursively all children first
240 auto result = CalculateNavigableAccessibleAtPoint(*childIt, p, cType, maxRecursionDepth - 1);
246 //Found a candidate, all its children are already checked
247 auto controledBy = GetObjectInRelation(root_component, RelationType::CONTROLLED_BY);
249 controledBy = root_component;
251 if(controledBy->IsProxy() || AcceptObject(controledBy))
253 LOG() << "CalculateNavigableAccessibleAtPoint: found: " << makeIndent(maxRecursionDepth) << objDump(root_component);
260 BridgeAccessible::ReadingMaterialType BridgeAccessible::GetReadingMaterial()
262 auto self = FindSelf();
263 auto attributes = self->GetAttributes();
264 auto name = self->GetName();
265 std::string labeledByName = "";
266 std::string textIfceName = "";
267 auto role = static_cast<uint32_t>(self->GetRole());
268 auto states = self->GetStates();
269 auto localizedName = self->GetLocalizedRoleName();
270 auto childCount = static_cast<int32_t>(self->GetChildCount());
272 double currentValue = 0.0;
273 double minimumIncrement = 0.0;
274 double maximumValue = 0.0;
275 double minimumValue = 0.0;
277 auto* value = dynamic_cast<Dali::Accessibility::Value*>(self);
280 currentValue = value->GetCurrent();
281 minimumIncrement = value->GetMinimumIncrement();
282 maximumValue = value->GetMaximum();
283 minimumValue = value->GetMinimum();
286 auto description = self->GetDescription();
287 auto indexInParent = static_cast<int32_t>(self->GetIndexInParent());
288 bool isSelectedInParent = false;
289 bool hasCheckBoxChild = false;
290 int32_t firstSelectedChildIndex = -1;
291 int32_t selectedChildCount = 0;
293 for(auto i = 0u; i < static_cast<size_t>(childCount); ++i)
295 auto q = self->GetChildAtIndex(i);
296 auto s = q->GetStates();
297 if(s[State::SELECTABLE])
299 if(s[State::SELECTED])
301 ++selectedChildCount;
302 if(firstSelectedChildIndex < 0)
303 firstSelectedChildIndex = static_cast<int32_t>(i);
306 if(q->GetRole() == Role::CHECK_BOX)
307 hasCheckBoxChild = true;
310 int32_t listChildrenCount = 0;
311 Accessible* parent = self->GetParent();
312 auto parentStateSet = parent ? parent->GetStates() : States{};
313 auto parentChildCount = parent ? static_cast<int32_t>(parent->GetChildCount()) : 0;
314 auto parentRole = static_cast<uint32_t>(parent ? parent->GetRole() : Role{});
315 Accessible* describedByObject = nullptr;
335 firstSelectedChildIndex,
344 void BridgeAccessible::SuppressScreenReader(bool suppress)
346 suppressScreenReader = suppress;
349 DBus::ValueOrError<bool> BridgeAccessible::DoGesture(Dali::Accessibility::Gesture type, int32_t xBeg, int32_t yBeg, int32_t xEnd, int32_t yEnd, Dali::Accessibility::GestureState state, uint32_t eventTime)
351 return FindSelf()->DoGesture(Dali::Accessibility::GestureInfo{type, xBeg, xEnd, yBeg, yEnd, state, eventTime});
354 DBus::ValueOrError<Accessible*, uint8_t, Accessible*> BridgeAccessible::GetNavigableAtPoint(int32_t x, int32_t y, uint32_t coordType)
356 Accessible* deputy = nullptr;
357 auto accessible = FindSelf();
358 auto cType = static_cast<CoordType>(coordType);
359 LOG() << "GetNavigableAtPoint: " << x << ", " << y << " type: " << coordType;
360 auto component = CalculateNavigableAccessibleAtPoint(accessible, {x, y}, cType, GET_NAVIGABLE_AT_POINT_MAX_RECURSION_DEPTH);
361 bool recurse = false;
364 recurse = component->IsProxy();
367 return {component, recurse, deputy};
370 static bool CheckChainEndWithAttribute(Accessible* obj, unsigned char forward)
374 auto attrs = obj->GetAttributes();
375 for(auto& attr : attrs)
377 if(attr.first == "relation_chain_end")
379 if((attr.second == "prev,end" && forward == 0) || (attr.second == "next,end" && forward == 1) || attr.second == "prev,next,end")
388 static Accessible* DeputyOfProxyInParentGet(Accessible* obj)
393 Accessible* BridgeAccessible::GetCurrentlyHighlighted()
395 //TODO: add currently highlighted object
399 std::vector<Accessible*> BridgeAccessible::ValidChildrenGet(const std::vector<Accessible*>& children, Accessible* start, Accessible* root)
404 static bool DeputyIs(Accessible* obj)
410 static Accessible* ProxyInParentGet(Accessible* obj)
414 auto children = obj->GetChildren();
415 for(auto& child : children)
423 static bool ObjectRoleIsAcceptableWhenNavigatingNextPrev(Accessible* obj)
427 auto role = obj->GetRole();
428 return role != Role::POPUP_MENU && role != Role::DIALOG;
432 struct CycleDetection
434 CycleDetection(const T value)
436 currentSearchSize(1),
440 bool check(const T value)
446 currentSearchSize <<= 1;
447 if(currentSearchSize == 0)
448 return true; // UNDEFINED BEHAVIOR
449 counter = currentSearchSize;
455 unsigned int currentSearchSize;
456 unsigned int counter;
459 static Accessible* FindNonDefunctChild(const std::vector<Accessible*>& children, unsigned int currentIndex, unsigned char forward)
461 unsigned int childrenCount = children.size();
462 for(; currentIndex < childrenCount; forward ? ++currentIndex : --currentIndex)
464 Accessible* n = children[currentIndex];
465 if(n && !n->GetStates()[State::DEFUNCT])
471 static Accessible* DirectionalDepthFirstSearchTryNonDefunctChild(Accessible* node, const std::vector<Accessible*>& children, unsigned char forward)
475 auto childrenCount = children.size();
476 if(childrenCount > 0)
478 const bool isShowing = GetScrollableParent(node) == nullptr ? node->GetStates()[State::SHOWING] : true;
481 return FindNonDefunctChild(children, forward ? 0 : childrenCount - 1, forward);
487 Accessible* BridgeAccessible::GetNextNonDefunctSibling(Accessible* obj, Accessible* start, Accessible* root, unsigned char forward)
491 auto parent = obj->GetParent();
495 auto children = ValidChildrenGet(parent->GetChildren(), start, root);
497 unsigned int children_count = children.size();
498 if(children_count == 0)
502 unsigned int current = 0;
503 for(; current < children_count && children[current] != obj; ++current)
505 if(current >= children_count)
509 forward ? ++current : --current;
510 auto ret = FindNonDefunctChild(children, current, forward);
514 Accessible* BridgeAccessible::DirectionalDepthFirstSearchTryNonDefunctSibling(bool& all_children_visited, Accessible* node, Accessible* start, Accessible* root, unsigned char forward)
518 Accessible* sibling = GetNextNonDefunctSibling(node, start, root, forward);
522 all_children_visited = false;
526 node = node->GetParent();
527 if(node == nullptr || node == root)
530 // in backward traversing stop the walk up on parent
537 Accessible* BridgeAccessible::CalculateNeighbor(Accessible* root, Accessible* start, unsigned char forward, BridgeAccessible::GetNeighborSearchMode search_mode)
539 if(start && CheckChainEndWithAttribute(start, forward))
541 if(root && root->GetStates()[State::DEFUNCT])
543 if(start && start->GetStates()[State::DEFUNCT])
549 if(search_mode == BridgeAccessible::GetNeighborSearchMode::recurseToOutside)
551 // This only works if we navigate backward, and it is not possible to
552 // find in embedded process. In this case the deputy should be used */
553 return DeputyOfProxyInParentGet(start);
556 Accessible* node = start ? start : root;
560 // initialization of all-children-visited flag for start node - we assume
561 // that when we begin at start node and we navigate backward, then all children
562 // are visited, so navigation will ignore start's children and go to
563 // previous sibling available.
564 // Regarding condtion (start != root):
565 // The last object can be found only if all_children_visited is false.
566 // The start is same with root, when looking for the last object.
567 bool all_children_visited = (start != root) && (search_mode != BridgeAccessible::GetNeighborSearchMode::recurseFromRoot && !forward);
568 // true, if starting element should be ignored. this is only used in rare case of
569 // recursive search failing to find an object.
570 // consider tree, where element A on bus BUS_A has child B on bus BUS_B. when going "next" from
571 // element A algorithm has to descend into BUS_B and search element B and its children. this is done
572 // by returning to our caller object B with special flag set (meaning - continue the search from B on bus BUS_B).
573 // if next object will be found there (on BUS_B), then search ends. but if not, then our caller will find it out
574 // and will call us again with object A and flag search_mode set to NEIGHBOR_SEARCH_MODE_CONTINUE_AFTER_FAILED_RECURSING.
575 // this flag means, that object A was already checked previously and we should skip it and its children.
576 bool force_next = (search_mode == BridgeAccessible::GetNeighborSearchMode::continueAfterFailedRecursion);
578 CycleDetection<Accessible*> cycleDetection(node);
581 if(node->GetStates()[State::DEFUNCT])
584 // always accept proxy object from different world
585 if(!force_next && node->IsProxy())
588 auto children = node->GetChildren();
589 children = ValidChildrenGet(children, start, root);
593 // 2. parent after all children in backward traversing
594 // 3. Nodes with roles: ATSPI_ROLE_PAGE_TAB, ATSPI_ROLE_POPUP_MENU and ATSPI_ROLE_DIALOG, only when looking for first or last element.
595 // Objects with those roles shouldnt be reachable, when navigating next / prev.
596 bool all_children_visited_or_moving_forward = (children.size() == 0 || forward || all_children_visited);
597 if(!force_next && node != start && all_children_visited_or_moving_forward && AcceptObject(node))
599 if(start == NULL || ObjectRoleIsAcceptableWhenNavigatingNextPrev(node))
603 Accessible* next_related_in_direction = !force_next ? GetObjectInRelation(node, forward ? RelationType::FLOWS_TO : RelationType::FLOWS_FROM) : nullptr;
604 // force_next means that the search_mode is NEIGHBOR_SEARCH_MODE_CONTINUE_AFTER_FAILED_RECURSING
605 // in this case the node is elm_layout which is parent of proxy object.
606 // There is an access object working for the proxy object, and the access
607 // object could have relation information. This relation information should
608 // be checked first before using the elm_layout as a node.
609 if(force_next && forward)
611 auto deputy = DeputyOfProxyInParentGet(node);
612 next_related_in_direction =
613 GetObjectInRelation(deputy, RelationType::FLOWS_TO);
616 if(next_related_in_direction && start && start->GetStates()[State::DEFUNCT])
618 next_related_in_direction = NULL;
621 unsigned char want_cycle_detection = 0;
622 if(next_related_in_direction)
624 // Check next_related_in_direction is deputy object
628 // If the prev object is deputy, then go to inside of its proxy first
629 if(DeputyIs(next_related_in_direction))
631 parent = next_related_in_direction->GetParent();
632 next_related_in_direction = ProxyInParentGet(parent);
637 // If current object is deputy, and it has relation next object,
638 // then do not use the relation next object, and use proxy first
641 parent = node->GetParent();
642 next_related_in_direction = ProxyInParentGet(parent);
645 node = next_related_in_direction;
646 want_cycle_detection = 1;
650 auto child = !force_next && !all_children_visited ? DirectionalDepthFirstSearchTryNonDefunctChild(node, children, forward) : nullptr;
653 want_cycle_detection = 1;
657 if(!force_next && node == root)
659 all_children_visited = true;
660 child = DirectionalDepthFirstSearchTryNonDefunctSibling(all_children_visited, node, start, root, forward);
665 if(want_cycle_detection && cycleDetection.check(node))
673 DBus::ValueOrError<Accessible*, uint8_t> BridgeAccessible::GetNeighbor(std::string rootPath, int32_t direction, int32_t search_mode)
675 auto start = FindSelf();
676 rootPath = StripPrefix(rootPath);
677 auto root = !rootPath.empty() ? Find(rootPath) : nullptr;
678 auto accessible = CalculateNeighbor(root, start, direction == 1, static_cast<GetNeighborSearchMode>(search_mode));
679 unsigned char recurse = 0;
682 recurse = accessible->IsProxy();
684 return {accessible, recurse};
687 Accessible* BridgeAccessible::GetParent()
689 // NOTE: currently bridge supports single application root element.
690 // only element set as application root might return nullptr from GetParent
691 // if you want more, then you need to change setApplicationRoot to
692 // add/remove ApplicationRoot and make roots a vector.
693 auto p = FindSelf()->GetParent();
697 DBus::ValueOrError<std::vector<Accessible*>> BridgeAccessible::GetChildren()
699 return FindSelf()->GetChildren();
701 std::string BridgeAccessible::GetDescription()
703 return FindSelf()->GetDescription();
705 DBus::ValueOrError<uint32_t> BridgeAccessible::GetRole()
707 return static_cast<unsigned int>(FindSelf()->GetRole());
709 DBus::ValueOrError<std::string> BridgeAccessible::GetRoleName()
711 return FindSelf()->GetRoleName();
713 DBus::ValueOrError<std::string> BridgeAccessible::GetLocalizedRoleName()
715 return FindSelf()->GetLocalizedRoleName();
717 DBus::ValueOrError<int32_t> BridgeAccessible::GetIndexInParent()
719 return FindSelf()->GetIndexInParent();
721 DBus::ValueOrError<std::array<uint32_t, 2>> BridgeAccessible::GetStates()
723 return FindSelf()->GetStates().GetRawData();
725 DBus::ValueOrError<std::unordered_map<std::string, std::string>> BridgeAccessible::GetAttributes()
727 std::unordered_map<std::string, std::string> attributes = FindSelf()->GetAttributes();
728 if(suppressScreenReader)
730 attributes.insert({"suppress-screen-reader", "true"});
735 DBus::ValueOrError<std::vector<std::string>> BridgeAccessible::GetInterfaces()
737 return FindSelf()->GetInterfaces();
739 int BridgeAccessible::GetChildCount()
741 return FindSelf()->GetChildCount();
743 DBus::ValueOrError<Accessible*> BridgeAccessible::GetChildAtIndex(int index)
746 throw std::domain_error{"negative index (" + std::to_string(index) + ")"};
747 return FindSelf()->GetChildAtIndex(static_cast<size_t>(index));
750 std::string BridgeAccessible::GetName()
752 return FindSelf()->GetName();
755 DBus::ValueOrError<Accessible*, uint32_t, std::unordered_map<std::string, std::string>> BridgeAccessible::GetDefaultLabelInfo()
757 auto defaultLabel = FindSelf()->GetDefaultLabel();
758 return {defaultLabel, static_cast<uint32_t>(defaultLabel->GetRole()), defaultLabel->GetAttributes()};
761 DBus::ValueOrError<std::vector<BridgeAccessible::Relation>> BridgeAccessible::GetRelationSet()
763 auto relations = FindSelf()->GetRelationSet();
764 std::vector<BridgeAccessible::Relation> ret;
766 for(auto& it : relations)
767 ret.emplace_back(Relation{static_cast<uint32_t>(it.relationType), it.targets});