2 * Copyright (c) 2021 Samsung Electronics Co., Ltd.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
19 #include <dali/internal/accessibility/bridge/bridge-accessible.h>
24 //comment out 2 lines below to get more logs
26 #define LOG() _LoggerEmpty()
28 using namespace Dali::Accessibility;
30 #define GET_NAVIGABLE_AT_POINT_MAX_RECURSION_DEPTH 10000
32 BridgeAccessible::BridgeAccessible()
36 void BridgeAccessible::RegisterInterfaces()
38 DBus::DBusInterfaceDescription desc{AtspiDbusInterfaceAccessible};
39 AddGetPropertyToInterface(desc, "ChildCount", &BridgeAccessible::GetChildCount);
40 AddGetPropertyToInterface(desc, "Name", &BridgeAccessible::GetName);
41 AddGetPropertyToInterface(desc, "Description", &BridgeAccessible::GetDescription);
42 AddGetPropertyToInterface(desc, "Parent", &BridgeAccessible::GetParent);
43 AddFunctionToInterface(desc, "GetRole", &BridgeAccessible::GetRole);
44 AddFunctionToInterface(desc, "GetRoleName", &BridgeAccessible::GetRoleName);
45 AddFunctionToInterface(desc, "GetLocalizedRoleName", &BridgeAccessible::GetLocalizedRoleName);
46 AddFunctionToInterface(desc, "GetState", &BridgeAccessible::GetStates);
47 AddFunctionToInterface(desc, "GetAttributes", &BridgeAccessible::GetAttributes);
48 AddFunctionToInterface(desc, "GetInterfaces", &BridgeAccessible::GetInterfaces);
49 AddFunctionToInterface(desc, "GetChildAtIndex", &BridgeAccessible::GetChildAtIndex);
50 AddFunctionToInterface(desc, "GetChildren", &BridgeAccessible::GetChildren);
51 AddFunctionToInterface(desc, "GetIndexInParent", &BridgeAccessible::GetIndexInParent);
52 AddFunctionToInterface(desc, "GetNavigableAtPoint", &BridgeAccessible::GetNavigableAtPoint);
53 AddFunctionToInterface(desc, "GetNeighbor", &BridgeAccessible::GetNeighbor);
54 AddFunctionToInterface(desc, "GetDefaultLabelInfo", &BridgeAccessible::GetDefaultLabelInfo);
55 AddFunctionToInterface(desc, "DoGesture", &BridgeAccessible::DoGesture);
56 AddFunctionToInterface(desc, "GetReadingMaterial", &BridgeAccessible::GetReadingMaterial);
57 AddFunctionToInterface(desc, "GetRelationSet", &BridgeAccessible::GetRelationSet);
58 dbusServer.addInterface("/", desc, true);
61 static bool AcceptObjectCheckRelations(Component* obj)
63 auto r = obj->GetRelationSet();
65 for(const auto& it : r)
66 if(it.relationType == RelationType::CONTROLLED_BY)
72 static Component* GetScrollableParent(Accessible* obj)
76 obj = obj->GetParent();
77 auto comp = dynamic_cast<Component*>(obj);
78 if(comp && comp->IsScrollable())
84 static bool ObjectIsItem(Component* obj)
88 auto role = obj->GetRole();
89 return role == Role::LIST_ITEM || role == Role::MENU_ITEM;
92 static bool ObjectIsCollapsed(Component* obj)
96 const auto states = obj->GetStates();
97 return states[State::EXPANDABLE] && !states[State::EXPANDED];
100 static bool OobjectIsZeroSize(Component* obj)
104 auto extents = obj->GetExtents(CoordType::WINDOW);
105 return extents.height == 0 || extents.width == 0;
108 static bool AcceptObject(Component* obj)
112 const auto states = obj->GetStates();
113 if(!states[State::VISIBLE])
115 if(!AcceptObjectCheckRelations(obj))
117 if(!states[State::HIGHLIGHTABLE])
120 if(GetScrollableParent(obj) != nullptr)
122 auto parent = dynamic_cast<Component*>(obj->GetParent());
126 return !ObjectIsItem(obj) || !ObjectIsCollapsed(parent);
131 if(OobjectIsZeroSize(obj))
135 if(!states[State::SHOWING])
143 static bool AcceptObject(Accessible* obj)
145 auto c = dynamic_cast<Component*>(obj);
146 return AcceptObject(c);
149 static std::string objDump(Component* obj)
153 std::ostringstream o;
154 auto e = obj->GetExtents(CoordType::SCREEN);
155 o << "name: " << obj->GetName() << " extent: (" << e.x << ", "
156 << e.y << "), [" << e.width << ", " << e.height << "]";
160 Component* BridgeAccessible::GetObjectInRelation(Accessible* obj, RelationType ralationType)
164 for(auto& relation : obj->GetRelationSet())
166 if(relation.relationType == ralationType)
168 for(auto& address : relation.targets)
170 auto component = dynamic_cast<Component*>(Find(address));
179 static std::string makeIndent(unsigned int maxRecursionDepth)
181 return std::string(GET_NAVIGABLE_AT_POINT_MAX_RECURSION_DEPTH - maxRecursionDepth, ' ');
184 Component* BridgeAccessible::CalculateNavigableAccessibleAtPoint(Accessible* root, Point p, CoordType cType, unsigned int maxRecursionDepth)
186 if(!root || maxRecursionDepth == 0)
188 auto root_component = dynamic_cast<Component*>(root);
189 LOG() << "CalculateNavigableAccessibleAtPoint: checking: " << makeIndent(maxRecursionDepth) << objDump(root_component);
191 if(root_component && !root_component->Contains(p, cType))
194 auto children = root->GetChildren();
195 for(auto childIt = children.rbegin(); childIt != children.rend(); childIt++)
197 //check recursively all children first
198 auto result = CalculateNavigableAccessibleAtPoint(*childIt, p, cType, maxRecursionDepth - 1);
204 //Found a candidate, all its children are already checked
205 auto controledBy = GetObjectInRelation(root_component, RelationType::CONTROLLED_BY);
207 controledBy = root_component;
209 if(controledBy->IsProxy() || AcceptObject(controledBy))
211 LOG() << "CalculateNavigableAccessibleAtPoint: found: " << makeIndent(maxRecursionDepth) << objDump(root_component);
218 BridgeAccessible::ReadingMaterialType BridgeAccessible::GetReadingMaterial()
220 auto self = FindSelf();
221 auto attributes = self->GetAttributes();
222 auto name = self->GetName();
223 std::string labeledByName = "";
224 std::string textIfceName = "";
225 auto role = static_cast<uint32_t>(self->GetRole());
226 auto states = self->GetStates();
227 auto localizedName = self->GetLocalizedRoleName();
228 auto childCount = static_cast<int32_t>(self->GetChildCount());
230 double currentValue = 0.0;
231 double minimumIncrement = 0.0;
232 double maximumValue = 0.0;
233 double minimumValue = 0.0;
235 auto* value = dynamic_cast<Dali::Accessibility::Value*>(self);
238 currentValue = value->GetCurrent();
239 minimumIncrement = value->GetMinimumIncrement();
240 maximumValue = value->GetMaximum();
241 minimumValue = value->GetMinimum();
244 auto description = self->GetDescription();
245 auto indexInParent = static_cast<int32_t>(self->GetIndexInParent());
246 bool isSelectedInParent = false;
247 bool hasCheckBoxChild = false;
248 int32_t firstSelectedChildIndex = -1;
249 int32_t selectedChildCount = 0;
251 for(auto i = 0u; i < static_cast<size_t>(childCount); ++i)
253 auto q = self->GetChildAtIndex(i);
254 auto s = q->GetStates();
255 if(s[State::SELECTABLE])
257 if(s[State::SELECTED])
259 ++selectedChildCount;
260 if(firstSelectedChildIndex < 0)
261 firstSelectedChildIndex = static_cast<int32_t>(i);
264 if(q->GetRole() == Role::CHECK_BOX)
265 hasCheckBoxChild = true;
268 int32_t listChildrenCount = 0;
269 Accessible* parent = self->GetParent();
270 auto parentStateSet = parent ? parent->GetStates() : States{};
271 auto parentChildCount = parent ? static_cast<int32_t>(parent->GetChildCount()) : 0;
272 auto parentRole = static_cast<uint32_t>(parent ? parent->GetRole() : Role{});
273 Accessible* describedByObject = nullptr;
293 firstSelectedChildIndex,
302 void BridgeAccessible::SuppressScreenReader(bool suppress)
304 suppressScreenReader = suppress;
307 DBus::ValueOrError<bool> BridgeAccessible::DoGesture(Dali::Accessibility::Gesture type, int32_t xBeg, int32_t yBeg, int32_t xEnd, int32_t yEnd, Dali::Accessibility::GestureState state, uint32_t eventTime)
309 return FindSelf()->DoGesture(Dali::Accessibility::GestureInfo{type, xBeg, xEnd, yBeg, yEnd, state, eventTime});
312 DBus::ValueOrError<Accessible*, uint8_t, Accessible*> BridgeAccessible::GetNavigableAtPoint(int32_t x, int32_t y, uint32_t coordType)
314 Accessible* deputy = nullptr;
315 auto accessible = FindSelf();
316 auto cType = static_cast<CoordType>(coordType);
317 LOG() << "GetNavigableAtPoint: " << x << ", " << y << " type: " << coordType;
318 auto component = CalculateNavigableAccessibleAtPoint(accessible, {x, y}, cType, GET_NAVIGABLE_AT_POINT_MAX_RECURSION_DEPTH);
319 bool recurse = false;
322 recurse = component->IsProxy();
325 return {component, recurse, deputy};
328 static bool CheckChainEndWithAttribute(Accessible* obj, unsigned char forward)
332 auto attrs = obj->GetAttributes();
333 for(auto& attr : attrs)
335 if(attr.first == "relation_chain_end")
337 if((attr.second == "prev,end" && forward == 0) || (attr.second == "next,end" && forward == 1) || attr.second == "prev,next,end")
346 static Accessible* DeputyOfProxyInParentGet(Accessible* obj)
351 Accessible* BridgeAccessible::GetCurrentlyHighlighted()
353 //TODO: add currently highlighted object
357 std::vector<Accessible*> BridgeAccessible::ValidChildrenGet(const std::vector<Accessible*>& children, Accessible* start, Accessible* root)
362 static bool DeputyIs(Accessible* obj)
368 static Accessible* ProxyInParentGet(Accessible* obj)
372 auto children = obj->GetChildren();
373 for(auto& child : children)
381 static bool ObjectRoleIsAcceptableWhenNavigatingNextPrev(Accessible* obj)
385 auto role = obj->GetRole();
386 return role != Role::POPUP_MENU && role != Role::DIALOG;
390 struct CycleDetection
392 CycleDetection(const T value)
394 currentSearchSize(1),
398 bool check(const T value)
404 currentSearchSize <<= 1;
405 if(currentSearchSize == 0)
406 return true; // UNDEFINED BEHAVIOR
407 counter = currentSearchSize;
413 unsigned int currentSearchSize;
414 unsigned int counter;
417 static Accessible* FindNonDefunctChild(const std::vector<Accessible*>& children, unsigned int currentIndex, unsigned char forward)
419 unsigned int childrenCount = children.size();
420 for(; currentIndex < childrenCount; forward ? ++currentIndex : --currentIndex)
422 Accessible* n = children[currentIndex];
423 if(n && !n->GetStates()[State::DEFUNCT])
429 static Accessible* DirectionalDepthFirstSearchTryNonDefunctChild(Accessible* node, const std::vector<Accessible*>& children, unsigned char forward)
433 auto childrenCount = children.size();
434 if(childrenCount > 0)
436 const bool isShowing = GetScrollableParent(node) == nullptr ? node->GetStates()[State::SHOWING] : true;
439 return FindNonDefunctChild(children, forward ? 0 : childrenCount - 1, forward);
445 Accessible* BridgeAccessible::GetNextNonDefunctSibling(Accessible* obj, Accessible* start, Accessible* root, unsigned char forward)
449 auto parent = obj->GetParent();
453 auto children = ValidChildrenGet(parent->GetChildren(), start, root);
455 unsigned int children_count = children.size();
456 if(children_count == 0)
460 unsigned int current = 0;
461 for(; current < children_count && children[current] != obj; ++current)
463 if(current >= children_count)
467 forward ? ++current : --current;
468 auto ret = FindNonDefunctChild(children, current, forward);
472 Accessible* BridgeAccessible::DirectionalDepthFirstSearchTryNonDefunctSibling(bool& all_children_visited, Accessible* node, Accessible* start, Accessible* root, unsigned char forward)
476 Accessible* sibling = GetNextNonDefunctSibling(node, start, root, forward);
480 all_children_visited = false;
484 node = node->GetParent();
485 if(node == nullptr || node == root)
488 // in backward traversing stop the walk up on parent
495 Accessible* BridgeAccessible::CalculateNeighbor(Accessible* root, Accessible* start, unsigned char forward, BridgeAccessible::GetNeighborSearchMode search_mode)
497 if(start && CheckChainEndWithAttribute(start, forward))
499 if(root && root->GetStates()[State::DEFUNCT])
501 if(start && start->GetStates()[State::DEFUNCT])
507 if(search_mode == BridgeAccessible::GetNeighborSearchMode::recurseToOutside)
509 // This only works if we navigate backward, and it is not possible to
510 // find in embedded process. In this case the deputy should be used */
511 return DeputyOfProxyInParentGet(start);
514 Accessible* node = start ? start : root;
518 // initialization of all-children-visited flag for start node - we assume
519 // that when we begin at start node and we navigate backward, then all children
520 // are visited, so navigation will ignore start's children and go to
521 // previous sibling available.
522 // Regarding condtion (start != root):
523 // The last object can be found only if all_children_visited is false.
524 // The start is same with root, when looking for the last object.
525 bool all_children_visited = (start != root) && (search_mode != BridgeAccessible::GetNeighborSearchMode::recurseFromRoot && !forward);
526 // true, if starting element should be ignored. this is only used in rare case of
527 // recursive search failing to find an object.
528 // consider tree, where element A on bus BUS_A has child B on bus BUS_B. when going "next" from
529 // element A algorithm has to descend into BUS_B and search element B and its children. this is done
530 // by returning to our caller object B with special flag set (meaning - continue the search from B on bus BUS_B).
531 // if next object will be found there (on BUS_B), then search ends. but if not, then our caller will find it out
532 // and will call us again with object A and flag search_mode set to NEIGHBOR_SEARCH_MODE_CONTINUE_AFTER_FAILED_RECURSING.
533 // this flag means, that object A was already checked previously and we should skip it and its children.
534 bool force_next = (search_mode == BridgeAccessible::GetNeighborSearchMode::continueAfterFailedRecursion);
536 CycleDetection<Accessible*> cycleDetection(node);
539 if(node->GetStates()[State::DEFUNCT])
542 // always accept proxy object from different world
543 if(!force_next && node->IsProxy())
546 auto children = node->GetChildren();
547 children = ValidChildrenGet(children, start, root);
551 // 2. parent after all children in backward traversing
552 // 3. Nodes with roles: ATSPI_ROLE_PAGE_TAB, ATSPI_ROLE_POPUP_MENU and ATSPI_ROLE_DIALOG, only when looking for first or last element.
553 // Objects with those roles shouldnt be reachable, when navigating next / prev.
554 bool all_children_visited_or_moving_forward = (children.size() == 0 || forward || all_children_visited);
555 if(!force_next && node != start && all_children_visited_or_moving_forward && AcceptObject(node))
557 if(start == NULL || ObjectRoleIsAcceptableWhenNavigatingNextPrev(node))
561 Accessible* next_related_in_direction = !force_next ? GetObjectInRelation(node, forward ? RelationType::FLOWS_TO : RelationType::FLOWS_FROM) : nullptr;
562 // force_next means that the search_mode is NEIGHBOR_SEARCH_MODE_CONTINUE_AFTER_FAILED_RECURSING
563 // in this case the node is elm_layout which is parent of proxy object.
564 // There is an access object working for the proxy object, and the access
565 // object could have relation information. This relation information should
566 // be checked first before using the elm_layout as a node.
567 if(force_next && forward)
569 auto deputy = DeputyOfProxyInParentGet(node);
570 next_related_in_direction =
571 GetObjectInRelation(deputy, RelationType::FLOWS_TO);
574 if(next_related_in_direction && start && start->GetStates()[State::DEFUNCT])
576 next_related_in_direction = NULL;
579 unsigned char want_cycle_detection = 0;
580 if(next_related_in_direction)
582 // Check next_related_in_direction is deputy object
586 // If the prev object is deputy, then go to inside of its proxy first
587 if(DeputyIs(next_related_in_direction))
589 parent = next_related_in_direction->GetParent();
590 next_related_in_direction = ProxyInParentGet(parent);
595 // If current object is deputy, and it has relation next object,
596 // then do not use the relation next object, and use proxy first
599 parent = node->GetParent();
600 next_related_in_direction = ProxyInParentGet(parent);
603 node = next_related_in_direction;
604 want_cycle_detection = 1;
608 auto child = !force_next && !all_children_visited ? DirectionalDepthFirstSearchTryNonDefunctChild(node, children, forward) : nullptr;
611 want_cycle_detection = 1;
615 if(!force_next && node == root)
617 all_children_visited = true;
618 child = DirectionalDepthFirstSearchTryNonDefunctSibling(all_children_visited, node, start, root, forward);
623 if(want_cycle_detection && cycleDetection.check(node))
631 DBus::ValueOrError<Accessible*, uint8_t> BridgeAccessible::GetNeighbor(std::string rootPath, int32_t direction, int32_t search_mode)
633 auto start = FindSelf();
634 rootPath = StripPrefix(rootPath);
635 auto root = !rootPath.empty() ? Find(rootPath) : nullptr;
636 auto accessible = CalculateNeighbor(root, start, direction == 1, static_cast<GetNeighborSearchMode>(search_mode));
637 unsigned char recurse = 0;
640 recurse = accessible->IsProxy();
642 return {accessible, recurse};
645 Accessible* BridgeAccessible::GetParent()
647 // NOTE: currently bridge supports single application root element.
648 // only element set as application root might return nullptr from GetParent
649 // if you want more, then you need to change setApplicationRoot to
650 // add/remove ApplicationRoot and make roots a vector.
651 auto p = FindSelf()->GetParent();
655 DBus::ValueOrError<std::vector<Accessible*>> BridgeAccessible::GetChildren()
657 return FindSelf()->GetChildren();
659 std::string BridgeAccessible::GetDescription()
661 return FindSelf()->GetDescription();
663 DBus::ValueOrError<uint32_t> BridgeAccessible::GetRole()
665 return static_cast<unsigned int>(FindSelf()->GetRole());
667 DBus::ValueOrError<std::string> BridgeAccessible::GetRoleName()
669 return FindSelf()->GetRoleName();
671 DBus::ValueOrError<std::string> BridgeAccessible::GetLocalizedRoleName()
673 return FindSelf()->GetLocalizedRoleName();
675 DBus::ValueOrError<int32_t> BridgeAccessible::GetIndexInParent()
677 return FindSelf()->GetIndexInParent();
679 DBus::ValueOrError<std::array<uint32_t, 2>> BridgeAccessible::GetStates()
681 return FindSelf()->GetStates().GetRawData();
683 DBus::ValueOrError<std::unordered_map<std::string, std::string>> BridgeAccessible::GetAttributes()
685 std::unordered_map<std::string, std::string> attributes = FindSelf()->GetAttributes();
686 if(suppressScreenReader)
688 attributes.insert({"suppress-screen-reader", "true"});
693 DBus::ValueOrError<std::vector<std::string>> BridgeAccessible::GetInterfaces()
695 return FindSelf()->GetInterfaces();
697 int BridgeAccessible::GetChildCount()
699 return FindSelf()->GetChildCount();
701 DBus::ValueOrError<Accessible*> BridgeAccessible::GetChildAtIndex(int index)
704 throw std::domain_error{"negative index (" + std::to_string(index) + ")"};
705 return FindSelf()->GetChildAtIndex(static_cast<size_t>(index));
708 std::string BridgeAccessible::GetName()
710 return FindSelf()->GetName();
713 DBus::ValueOrError<Accessible*, uint32_t, std::unordered_map<std::string, std::string>> BridgeAccessible::GetDefaultLabelInfo()
715 auto defaultLabel = FindSelf()->GetDefaultLabel();
716 return {defaultLabel, static_cast<uint32_t>(defaultLabel->GetRole()), defaultLabel->GetAttributes()};
719 DBus::ValueOrError<std::vector<BridgeAccessible::Relation>> BridgeAccessible::GetRelationSet()
721 auto relations = FindSelf()->GetRelationSet();
722 std::vector<BridgeAccessible::Relation> ret;
724 for(auto& it : relations)
725 ret.emplace_back(Relation{static_cast<uint32_t>(it.relationType), it.targets});