2 * Copyright (c) 2021 Samsung Electronics Co., Ltd.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
19 #include <dali/internal/accessibility/bridge/bridge-accessible.h>
24 //comment out 2 lines below to get more logs
26 #define LOG() _LoggerEmpty()
28 using namespace Dali::Accessibility;
30 #define GET_NAVIGABLE_AT_POINT_MAX_RECURSION_DEPTH 10000
32 BridgeAccessible::BridgeAccessible()
36 void BridgeAccessible::RegisterInterfaces()
38 DBus::DBusInterfaceDescription desc{AtspiDbusInterfaceAccessible};
39 AddGetPropertyToInterface(desc, "ChildCount", &BridgeAccessible::GetChildCount);
40 AddGetPropertyToInterface(desc, "Name", &BridgeAccessible::GetName);
41 AddGetPropertyToInterface(desc, "Description", &BridgeAccessible::GetDescription);
42 AddGetPropertyToInterface(desc, "Parent", &BridgeAccessible::GetParent);
43 AddFunctionToInterface(desc, "GetRole", &BridgeAccessible::GetRole);
44 AddFunctionToInterface(desc, "GetRoleName", &BridgeAccessible::GetRoleName);
45 AddFunctionToInterface(desc, "GetLocalizedRoleName", &BridgeAccessible::GetLocalizedRoleName);
46 AddFunctionToInterface(desc, "GetState", &BridgeAccessible::GetStates);
47 AddFunctionToInterface(desc, "GetAttributes", &BridgeAccessible::GetAttributes);
48 AddFunctionToInterface(desc, "GetInterfaces", &BridgeAccessible::GetInterfaces);
49 AddFunctionToInterface(desc, "GetChildAtIndex", &BridgeAccessible::GetChildAtIndex);
50 AddFunctionToInterface(desc, "GetChildren", &BridgeAccessible::GetChildren);
51 AddFunctionToInterface(desc, "GetIndexInParent", &BridgeAccessible::GetIndexInParent);
52 AddFunctionToInterface(desc, "GetNavigableAtPoint", &BridgeAccessible::GetNavigableAtPoint);
53 AddFunctionToInterface(desc, "GetNeighbor", &BridgeAccessible::GetNeighbor);
54 AddFunctionToInterface(desc, "GetDefaultLabelInfo", &BridgeAccessible::GetDefaultLabelInfo);
55 AddFunctionToInterface(desc, "DoGesture", &BridgeAccessible::DoGesture);
56 AddFunctionToInterface(desc, "GetReadingMaterial", &BridgeAccessible::GetReadingMaterial);
57 AddFunctionToInterface(desc, "GetRelationSet", &BridgeAccessible::GetRelationSet);
58 dbusServer.addInterface("/", desc, true);
61 static bool AcceptObjectCheckRole(Component* obj)
65 switch(obj->GetRole())
67 case Role::APPLICATION:
69 case Role::SCROLL_PANE:
70 case Role::SPLIT_PANE:
77 case Role::REDUNDANT_OBJECT:
78 case Role::COLOR_CHOOSER:
79 case Role::TREE_TABLE:
80 case Role::PAGE_TAB_LIST:
82 case Role::SPIN_BUTTON:
83 case Role::INPUT_METHOD_WINDOW:
86 case Role::NOTIFICATION:
87 case Role::DATE_EDITOR:
101 static bool AcceptObjectCheckRelations(Component* obj)
103 auto r = obj->GetRelationSet();
105 for(const auto& it : r)
106 if(it.relationType == RelationType::CONTROLLED_BY)
112 static Component* GetScrollableParent(Accessible* obj)
116 obj = obj->GetParent();
117 auto comp = dynamic_cast<Component*>(obj);
118 if(comp && comp->IsScrollable())
124 static bool ObjectIsItem(Component* obj)
128 auto role = obj->GetRole();
129 return role == Role::LIST_ITEM || role == Role::MENU_ITEM;
132 static bool ObjectIsCollapsed(Component* obj)
136 const auto states = obj->GetStates();
137 return states[State::EXPANDABLE] && !states[State::EXPANDED];
140 static bool OobjectIsZeroSize(Component* obj)
144 auto extents = obj->GetExtents(CoordType::WINDOW);
145 return extents.height == 0 || extents.width == 0;
148 static bool AcceptObject(Component* obj)
152 const auto states = obj->GetStates();
153 if(!states[State::VISIBLE])
155 if(!AcceptObjectCheckRole(obj))
157 if(!AcceptObjectCheckRelations(obj))
159 if(!states[State::HIGHLIGHTABLE])
162 if(GetScrollableParent(obj) != nullptr)
164 auto parent = dynamic_cast<Component*>(obj->GetParent());
168 return !ObjectIsItem(obj) || !ObjectIsCollapsed(parent);
173 if(OobjectIsZeroSize(obj))
177 if(!states[State::SHOWING])
185 static bool AcceptObject(Accessible* obj)
187 auto c = dynamic_cast<Component*>(obj);
188 return AcceptObject(c);
191 static std::string objDump(Component* obj)
195 std::ostringstream o;
196 auto e = obj->GetExtents(CoordType::SCREEN);
197 o << "name: " << obj->GetName() << " extent: (" << e.x << ", "
198 << e.y << "), [" << e.width << ", " << e.height << "]";
202 Component* BridgeAccessible::GetObjectInRelation(Accessible* obj, RelationType ralationType)
206 for(auto& relation : obj->GetRelationSet())
208 if(relation.relationType == ralationType)
210 for(auto& address : relation.targets)
212 auto component = dynamic_cast<Component*>(Find(address));
221 static std::string makeIndent(unsigned int maxRecursionDepth)
223 return std::string(GET_NAVIGABLE_AT_POINT_MAX_RECURSION_DEPTH - maxRecursionDepth, ' ');
226 Component* BridgeAccessible::CalculateNavigableAccessibleAtPoint(Accessible* root, Point p, CoordType cType, unsigned int maxRecursionDepth)
228 if(!root || maxRecursionDepth == 0)
230 auto root_component = dynamic_cast<Component*>(root);
231 LOG() << "CalculateNavigableAccessibleAtPoint: checking: " << makeIndent(maxRecursionDepth) << objDump(root_component);
233 if(root_component && !root_component->Contains(p, cType))
236 auto children = root->GetChildren();
237 for(auto childIt = children.rbegin(); childIt != children.rend(); childIt++)
239 //check recursively all children first
240 auto result = CalculateNavigableAccessibleAtPoint(*childIt, p, cType, maxRecursionDepth - 1);
246 //Found a candidate, all its children are already checked
247 auto controledBy = GetObjectInRelation(root_component, RelationType::CONTROLLED_BY);
249 controledBy = root_component;
251 if(controledBy->IsProxy() || AcceptObject(controledBy))
253 LOG() << "CalculateNavigableAccessibleAtPoint: found: " << makeIndent(maxRecursionDepth) << objDump(root_component);
260 BridgeAccessible::ReadingMaterialType BridgeAccessible::GetReadingMaterial()
262 auto self = FindSelf();
263 auto attributes = self->GetAttributes();
264 auto name = self->GetName();
265 std::string labeledByName = "";
266 std::string textIfceName = "";
267 auto role = static_cast<uint32_t>(self->GetRole());
268 auto states = self->GetStates();
269 auto localizedName = self->GetLocalizedRoleName();
270 auto childCount = static_cast<int32_t>(self->GetChildCount());
272 double currentValue = 0.0;
273 double minimumIncrement = 0.0;
274 double maximumValue = 0.0;
275 double minimumValue = 0.0;
277 auto* value = dynamic_cast<Dali::Accessibility::Value*>(self);
280 currentValue = value->GetCurrent();
281 minimumIncrement = value->GetMinimumIncrement();
282 maximumValue = value->GetMaximum();
283 minimumValue = value->GetMinimum();
286 auto description = self->GetDescription();
287 auto indexInParent = static_cast<int32_t>(self->GetIndexInParent());
288 bool isSelectedInParent = false;
289 bool hasCheckBoxChild = false;
290 int32_t firstSelectedChildIndex = -1;
291 int32_t selectedChildCount = 0;
293 for(auto i = 0u; i < static_cast<size_t>(childCount); ++i)
295 auto q = self->GetChildAtIndex(i);
296 auto s = q->GetStates();
297 if(s[State::SELECTABLE])
299 if(s[State::SELECTED])
301 ++selectedChildCount;
302 if(firstSelectedChildIndex < 0)
303 firstSelectedChildIndex = static_cast<int32_t>(i);
306 if(q->GetRole() == Role::CHECK_BOX)
307 hasCheckBoxChild = true;
310 int32_t listChildrenCount = 0;
311 Accessible* parent = self->GetParent();
312 auto parentStateSet = parent ? parent->GetStates() : States{};
313 auto parentChildCount = parent ? static_cast<int32_t>(parent->GetChildCount()) : 0;
314 auto parentRole = static_cast<uint32_t>(parent ? parent->GetRole() : Role{});
315 Accessible* describedByObject = nullptr;
335 firstSelectedChildIndex,
344 DBus::ValueOrError<bool> BridgeAccessible::DoGesture(Dali::Accessibility::Gesture type, int32_t xBeg, int32_t yBeg, int32_t xEnd, int32_t yEnd, Dali::Accessibility::GestureState state, uint32_t eventTime)
346 return FindSelf()->DoGesture(Dali::Accessibility::GestureInfo{type, xBeg, xEnd, yBeg, yEnd, state, eventTime});
349 DBus::ValueOrError<Accessible*, uint8_t, Accessible*> BridgeAccessible::GetNavigableAtPoint(int32_t x, int32_t y, uint32_t coordType)
351 Accessible* deputy = nullptr;
352 auto accessible = FindSelf();
353 auto cType = static_cast<CoordType>(coordType);
354 LOG() << "GetNavigableAtPoint: " << x << ", " << y << " type: " << coordType;
355 auto component = CalculateNavigableAccessibleAtPoint(accessible, {x, y}, cType, GET_NAVIGABLE_AT_POINT_MAX_RECURSION_DEPTH);
356 bool recurse = false;
359 recurse = component->IsProxy();
362 return {component, recurse, deputy};
365 static bool CheckChainEndWithAttribute(Accessible* obj, unsigned char forward)
369 auto attrs = obj->GetAttributes();
370 for(auto& attr : attrs)
372 if(attr.first == "relation_chain_end")
374 if((attr.second == "prev,end" && forward == 0) || (attr.second == "next,end" && forward == 1) || attr.second == "prev,next,end")
383 static Accessible* DeputyOfProxyInParentGet(Accessible* obj)
388 Accessible* BridgeAccessible::GetCurrentlyHighlighted()
390 //TODO: add currently highlighted object
394 std::vector<Accessible*> BridgeAccessible::ValidChildrenGet(const std::vector<Accessible*>& children, Accessible* start, Accessible* root)
399 static bool DeputyIs(Accessible* obj)
405 static Accessible* ProxyInParentGet(Accessible* obj)
409 auto children = obj->GetChildren();
410 for(auto& child : children)
418 static bool ObjectRoleIsAcceptableWhenNavigatingNextPrev(Accessible* obj)
422 auto role = obj->GetRole();
423 return role != Role::POPUP_MENU && role != Role::DIALOG;
427 struct CycleDetection
429 CycleDetection(const T value)
431 currentSearchSize(1),
435 bool check(const T value)
441 currentSearchSize <<= 1;
442 if(currentSearchSize == 0)
443 return true; // UNDEFINED BEHAVIOR
444 counter = currentSearchSize;
450 unsigned int currentSearchSize;
451 unsigned int counter;
454 static Accessible* FindNonDefunctChild(const std::vector<Accessible*>& children, unsigned int currentIndex, unsigned char forward)
456 unsigned int childrenCount = children.size();
457 for(; currentIndex < childrenCount; forward ? ++currentIndex : --currentIndex)
459 Accessible* n = children[currentIndex];
460 if(n && !n->GetStates()[State::DEFUNCT])
466 static Accessible* DirectionalDepthFirstSearchTryNonDefunctChild(Accessible* node, const std::vector<Accessible*>& children, unsigned char forward)
470 auto childrenCount = children.size();
471 if(childrenCount > 0)
473 const bool isShowing = GetScrollableParent(node) == nullptr ? node->GetStates()[State::SHOWING] : true;
476 return FindNonDefunctChild(children, forward ? 0 : childrenCount - 1, forward);
482 Accessible* BridgeAccessible::GetNextNonDefunctSibling(Accessible* obj, Accessible* start, Accessible* root, unsigned char forward)
486 auto parent = obj->GetParent();
490 auto children = ValidChildrenGet(parent->GetChildren(), start, root);
492 unsigned int children_count = children.size();
493 if(children_count == 0)
497 unsigned int current = 0;
498 for(; current < children_count && children[current] != obj; ++current)
500 if(current >= children_count)
504 forward ? ++current : --current;
505 auto ret = FindNonDefunctChild(children, current, forward);
509 Accessible* BridgeAccessible::DirectionalDepthFirstSearchTryNonDefunctSibling(bool& all_children_visited, Accessible* node, Accessible* start, Accessible* root, unsigned char forward)
513 Accessible* sibling = GetNextNonDefunctSibling(node, start, root, forward);
517 all_children_visited = false;
521 node = node->GetParent();
522 if(node == nullptr || node == root)
525 // in backward traversing stop the walk up on parent
532 Accessible* BridgeAccessible::CalculateNeighbor(Accessible* root, Accessible* start, unsigned char forward, BridgeAccessible::GetNeighborSearchMode search_mode)
534 if(start && CheckChainEndWithAttribute(start, forward))
536 if(root && root->GetStates()[State::DEFUNCT])
538 if(start && start->GetStates()[State::DEFUNCT])
544 if(search_mode == BridgeAccessible::GetNeighborSearchMode::recurseToOutside)
546 // This only works if we navigate backward, and it is not possible to
547 // find in embedded process. In this case the deputy should be used */
548 return DeputyOfProxyInParentGet(start);
551 Accessible* node = start ? start : root;
555 // initialization of all-children-visited flag for start node - we assume
556 // that when we begin at start node and we navigate backward, then all children
557 // are visited, so navigation will ignore start's children and go to
558 // previous sibling available.
559 // Regarding condtion (start != root):
560 // The last object can be found only if all_children_visited is false.
561 // The start is same with root, when looking for the last object.
562 bool all_children_visited = (start != root) && (search_mode != BridgeAccessible::GetNeighborSearchMode::recurseFromRoot && !forward);
563 // true, if starting element should be ignored. this is only used in rare case of
564 // recursive search failing to find an object.
565 // consider tree, where element A on bus BUS_A has child B on bus BUS_B. when going "next" from
566 // element A algorithm has to descend into BUS_B and search element B and its children. this is done
567 // by returning to our caller object B with special flag set (meaning - continue the search from B on bus BUS_B).
568 // if next object will be found there (on BUS_B), then search ends. but if not, then our caller will find it out
569 // and will call us again with object A and flag search_mode set to NEIGHBOR_SEARCH_MODE_CONTINUE_AFTER_FAILED_RECURSING.
570 // this flag means, that object A was already checked previously and we should skip it and its children.
571 bool force_next = (search_mode == BridgeAccessible::GetNeighborSearchMode::continueAfterFailedRecursion);
573 CycleDetection<Accessible*> cycleDetection(node);
576 if(node->GetStates()[State::DEFUNCT])
579 // always accept proxy object from different world
580 if(!force_next && node->IsProxy())
583 auto children = node->GetChildren();
584 children = ValidChildrenGet(children, start, root);
588 // 2. parent after all children in backward traversing
589 // 3. Nodes with roles: ATSPI_ROLE_PAGE_TAB, ATSPI_ROLE_POPUP_MENU and ATSPI_ROLE_DIALOG, only when looking for first or last element.
590 // Objects with those roles shouldnt be reachable, when navigating next / prev.
591 bool all_children_visited_or_moving_forward = (children.size() == 0 || forward || all_children_visited);
592 if(!force_next && node != start && all_children_visited_or_moving_forward && AcceptObject(node))
594 if(start == NULL || ObjectRoleIsAcceptableWhenNavigatingNextPrev(node))
598 Accessible* next_related_in_direction = !force_next ? GetObjectInRelation(node, forward ? RelationType::FLOWS_TO : RelationType::FLOWS_FROM) : nullptr;
599 // force_next means that the search_mode is NEIGHBOR_SEARCH_MODE_CONTINUE_AFTER_FAILED_RECURSING
600 // in this case the node is elm_layout which is parent of proxy object.
601 // There is an access object working for the proxy object, and the access
602 // object could have relation information. This relation information should
603 // be checked first before using the elm_layout as a node.
604 if(force_next && forward)
606 auto deputy = DeputyOfProxyInParentGet(node);
607 next_related_in_direction =
608 GetObjectInRelation(deputy, RelationType::FLOWS_TO);
611 if(next_related_in_direction && start && start->GetStates()[State::DEFUNCT])
613 next_related_in_direction = NULL;
616 unsigned char want_cycle_detection = 0;
617 if(next_related_in_direction)
619 // Check next_related_in_direction is deputy object
623 // If the prev object is deputy, then go to inside of its proxy first
624 if(DeputyIs(next_related_in_direction))
626 parent = next_related_in_direction->GetParent();
627 next_related_in_direction = ProxyInParentGet(parent);
632 // If current object is deputy, and it has relation next object,
633 // then do not use the relation next object, and use proxy first
636 parent = node->GetParent();
637 next_related_in_direction = ProxyInParentGet(parent);
640 node = next_related_in_direction;
641 want_cycle_detection = 1;
645 auto child = !force_next && !all_children_visited ? DirectionalDepthFirstSearchTryNonDefunctChild(node, children, forward) : nullptr;
648 want_cycle_detection = 1;
652 if(!force_next && node == root)
654 all_children_visited = true;
655 child = DirectionalDepthFirstSearchTryNonDefunctSibling(all_children_visited, node, start, root, forward);
660 if(want_cycle_detection && cycleDetection.check(node))
668 DBus::ValueOrError<Accessible*, uint8_t> BridgeAccessible::GetNeighbor(std::string rootPath, int32_t direction, int32_t search_mode)
670 auto start = FindSelf();
671 rootPath = StripPrefix(rootPath);
672 auto root = !rootPath.empty() ? Find(rootPath) : nullptr;
673 auto accessible = CalculateNeighbor(root, start, direction == 1, static_cast<GetNeighborSearchMode>(search_mode));
674 unsigned char recurse = 0;
677 recurse = accessible->IsProxy();
679 return {accessible, recurse};
682 Accessible* BridgeAccessible::GetParent()
684 // NOTE: currently bridge supports single application root element.
685 // only element set as application root might return nullptr from GetParent
686 // if you want more, then you need to change setApplicationRoot to
687 // add/remove ApplicationRoot and make roots a vector.
688 auto p = FindSelf()->GetParent();
692 DBus::ValueOrError<std::vector<Accessible*>> BridgeAccessible::GetChildren()
694 return FindSelf()->GetChildren();
696 std::string BridgeAccessible::GetDescription()
698 return FindSelf()->GetDescription();
700 DBus::ValueOrError<uint32_t> BridgeAccessible::GetRole()
702 return static_cast<unsigned int>(FindSelf()->GetRole());
704 DBus::ValueOrError<std::string> BridgeAccessible::GetRoleName()
706 return FindSelf()->GetRoleName();
708 DBus::ValueOrError<std::string> BridgeAccessible::GetLocalizedRoleName()
710 return FindSelf()->GetLocalizedRoleName();
712 DBus::ValueOrError<int32_t> BridgeAccessible::GetIndexInParent()
714 return FindSelf()->GetIndexInParent();
716 DBus::ValueOrError<std::array<uint32_t, 2>> BridgeAccessible::GetStates()
718 return FindSelf()->GetStates().GetRawData();
720 DBus::ValueOrError<std::unordered_map<std::string, std::string>> BridgeAccessible::GetAttributes()
722 return FindSelf()->GetAttributes();
724 DBus::ValueOrError<std::vector<std::string>> BridgeAccessible::GetInterfaces()
726 return FindSelf()->GetInterfaces();
728 int BridgeAccessible::GetChildCount()
730 return FindSelf()->GetChildCount();
732 DBus::ValueOrError<Accessible*> BridgeAccessible::GetChildAtIndex(int index)
735 throw std::domain_error{"negative index (" + std::to_string(index) + ")"};
736 return FindSelf()->GetChildAtIndex(static_cast<size_t>(index));
739 std::string BridgeAccessible::GetName()
741 return FindSelf()->GetName();
744 DBus::ValueOrError<Accessible*, uint32_t, std::unordered_map<std::string, std::string>> BridgeAccessible::GetDefaultLabelInfo()
746 auto defaultLabel = FindSelf()->GetDefaultLabel();
747 return {defaultLabel, static_cast<uint32_t>(defaultLabel->GetRole()), defaultLabel->GetAttributes()};
750 DBus::ValueOrError<std::vector<BridgeAccessible::Relation>> BridgeAccessible::GetRelationSet()
752 auto relations = FindSelf()->GetRelationSet();
753 std::vector<BridgeAccessible::Relation> ret;
755 for(auto& it : relations)
756 ret.emplace_back(Relation{static_cast<uint32_t>(it.relationType), it.targets});