2 * Copyright (c) 2019 Samsung Electronics Co., Ltd.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
19 #include <dali/internal/accessibility/bridge/bridge-accessible.h>
24 //comment out 2 lines below to get more logs
26 #define LOG() _LoggerEmpty()
28 using namespace Dali::Accessibility;
30 #define GET_NAVIGABLE_AT_POINT_MAX_RECURSION_DEPTH 10000
32 BridgeAccessible::BridgeAccessible()
36 void BridgeAccessible::RegisterInterfaces()
38 DBus::DBusInterfaceDescription desc{AtspiDbusInterfaceAccessible};
39 AddGetPropertyToInterface( desc, "ChildCount", &BridgeAccessible::GetChildCount );
40 AddGetPropertyToInterface( desc, "Name", &BridgeAccessible::GetName );
41 AddGetPropertyToInterface( desc, "Description", &BridgeAccessible::GetDescription );
42 AddGetPropertyToInterface( desc, "Parent", &BridgeAccessible::GetParent );
43 AddFunctionToInterface( desc, "GetRole", &BridgeAccessible::GetRole );
44 AddFunctionToInterface( desc, "GetRoleName", &BridgeAccessible::GetRoleName );
45 AddFunctionToInterface( desc, "GetLocalizedRoleName", &BridgeAccessible::GetLocalizedRoleName );
46 AddFunctionToInterface( desc, "GetState", &BridgeAccessible::GetStates );
47 AddFunctionToInterface( desc, "GetAttributes", &BridgeAccessible::GetAttributes );
48 AddFunctionToInterface( desc, "GetInterfaces", &BridgeAccessible::GetInterfaces );
49 AddFunctionToInterface( desc, "GetChildAtIndex", &BridgeAccessible::GetChildAtIndex );
50 AddFunctionToInterface( desc, "GetChildren", &BridgeAccessible::GetChildren );
51 AddFunctionToInterface( desc, "GetIndexInParent", &BridgeAccessible::GetIndexInParent );
52 AddFunctionToInterface( desc, "GetNavigableAtPoint", &BridgeAccessible::GetNavigableAtPoint );
53 AddFunctionToInterface( desc, "GetNeighbor", &BridgeAccessible::GetNeighbor );
54 AddFunctionToInterface( desc, "GetDefaultLabelInfo", &BridgeAccessible::GetDefaultLabelInfo );
55 AddFunctionToInterface( desc, "DoGesture", &BridgeAccessible::DoGesture );
56 AddFunctionToInterface( desc, "GetReadingMaterial", &BridgeAccessible::GetReadingMaterial );
57 AddFunctionToInterface( desc, "GetRelationSet", &BridgeAccessible::GetRelationSet );
58 dbusServer.addInterface( "/", desc, true );
61 static bool AcceptObjectCheckRole( Component* obj )
65 switch( obj->GetRole() )
67 case Role::APPLICATION:
69 case Role::SCROLL_PANE:
70 case Role::SPLIT_PANE:
77 case Role::REDUNDANT_OBJECT:
78 case Role::COLOR_CHOOSER:
79 case Role::TREE_TABLE:
80 case Role::PAGE_TAB_LIST:
82 case Role::SPIN_BUTTON:
83 case Role::INPUT_METHOD_WINDOW:
86 case Role::NOTIFICATION:
87 case Role::DATE_EDITOR:
101 static bool AcceptObjectCheckRelations( Component* obj)
103 auto r = obj->GetRelationSet();
105 for (const auto& it : r)
106 if (it.relationType == RelationType::CONTROLLED_BY)
112 static Component* GetScrollableParent( Accessible* obj )
116 obj = obj->GetParent();
117 auto comp = dynamic_cast< Component* >( obj );
118 if( comp && comp->IsScrollable() )
124 static bool ObjectIsItem( Component* obj )
128 auto role = obj->GetRole();
129 return role == Role::LIST_ITEM || role == Role::MENU_ITEM;
132 static bool ObjectIsCollapsed( Component* obj )
136 const auto states = obj->GetStates();
137 return states[State::EXPANDABLE] && !states[State::EXPANDED];
140 static bool OobjectIsZeroSize( Component* obj )
144 auto extents = obj->GetExtents( CoordType::WINDOW );
145 return extents.height == 0 || extents.width == 0;
148 static bool AcceptObject( Component* obj )
152 const auto states = obj->GetStates();
153 if( !states[State::VISIBLE] )
155 if( !AcceptObjectCheckRole( obj ) )
157 if ( !AcceptObjectCheckRelations( obj ) )
159 // if (CALL(get_object_in_relation_by_type, obj, ATSPI_RELATION_CONTROLLED_BY) != NULL) return 0;
160 if ( !AcceptObjectCheckRelations( obj ) )
162 if( !states[State::HIGHLIGHTABLE] )
165 if( GetScrollableParent( obj ) != nullptr )
167 auto parent = dynamic_cast< Component* >( obj->GetParent() );
171 return !ObjectIsItem( obj ) || !ObjectIsCollapsed( parent );
176 if( OobjectIsZeroSize( obj ) )
180 if( !states[State::SHOWING] )
188 static bool AcceptObject( Accessible* obj )
190 auto c = dynamic_cast< Component* >( obj );
191 return AcceptObject( c );
194 static std::string objDump( Component* obj )
198 std::ostringstream o;
199 auto e = obj->GetExtents( CoordType::SCREEN );
200 o << "name: " << obj->GetName() << " extent: (" << e.x << ", "
201 << e.y << "), [" << e.width << ", " << e.height << "]";
205 Component * BridgeAccessible::GetObjectInRelation( Accessible * obj, RelationType ralationType )
209 for ( auto &relation : obj->GetRelationSet() )
211 if ( relation.relationType == ralationType )
213 for ( auto &address : relation.targets )
215 auto component = dynamic_cast<Component*>( Find( address ) );
224 static std::string makeIndent( unsigned int maxRecursionDepth )
226 return std::string( GET_NAVIGABLE_AT_POINT_MAX_RECURSION_DEPTH - maxRecursionDepth, ' ' );
229 Component* BridgeAccessible::CalculateNavigableAccessibleAtPoint( Accessible* root, Point p, CoordType cType, unsigned int maxRecursionDepth )
231 if( !root || maxRecursionDepth == 0 )
233 auto root_component = dynamic_cast< Component* >( root );
234 LOG() << "CalculateNavigableAccessibleAtPoint: checking: " << makeIndent(maxRecursionDepth) << objDump(root_component);
236 if( root_component && !root_component->Contains( p, cType ) )
239 auto children = root->GetChildren();
240 for( auto childIt = children.rbegin(); childIt != children.rend(); childIt++ )
242 //check recursively all children first
243 auto result = CalculateNavigableAccessibleAtPoint( *childIt, p, cType, maxRecursionDepth - 1 );
249 //Found a candidate, all its children are already checked
250 auto controledBy = GetObjectInRelation( root_component, RelationType::CONTROLLED_BY );
252 controledBy = root_component;
254 if ( controledBy->IsProxy() || AcceptObject( controledBy ) )
256 LOG() << "CalculateNavigableAccessibleAtPoint: found: " << makeIndent(maxRecursionDepth) << objDump( root_component );
263 BridgeAccessible::ReadingMaterialType BridgeAccessible::GetReadingMaterial()
265 auto self = FindSelf();
266 auto attributes = self->GetAttributes();
267 auto name = self->GetName();
268 std::string labeledByName = "";
269 std::string textIfceName = "";
270 auto role = static_cast< uint32_t >( self->GetRole() );
271 auto states = self->GetStates();
272 auto localizedName = self->GetLocalizedRoleName();
273 auto childCount = static_cast< int32_t >( self->GetChildCount() );
275 double currentValue = 0.0;
276 double minimumIncrement = 0.0;
277 double maximumValue = 0.0;
278 double minimumValue = 0.0;
280 auto description = self->GetDescription();
281 auto indexInParent = static_cast< int32_t >( self->GetIndexInParent() );
282 bool isSelectedInParent = false;
283 bool hasCheckBoxChild = false;
284 int32_t firstSelectedChildIndex = 0;
285 int32_t selectedChildCount = 0;
287 for( auto i = 0u; i < static_cast< size_t >( childCount ); ++i )
289 auto q = self->GetChildAtIndex( i );
290 auto s = q->GetStates();
291 if( s[State::SELECTABLE] )
293 ++selectedChildCount;
294 if( s[State::SELECTED] )
296 if( firstSelectedChildIndex < 0 )
297 firstSelectedChildIndex = static_cast< int32_t >( i );
300 if( q->GetRole() == Role::CHECK_BOX )
301 hasCheckBoxChild = true;
304 int32_t listChildrenCount = 0;
305 Accessible* parent = self->GetParent();
306 auto parentStateSet = parent ? parent->GetStates() : States{};
307 auto parentChildCount = parent ? static_cast< int32_t >( parent->GetChildCount() ) : 0;
308 auto parentRole = static_cast< uint32_t >( parent ? parent->GetRole() : Role{} );
309 Accessible* describedByObject = nullptr;
329 firstSelectedChildIndex,
338 DBus::ValueOrError< bool > BridgeAccessible::DoGesture( Dali::Accessibility::Gesture type, int32_t xBeg, int32_t xEnd, int32_t yBeg, int32_t yEnd, Dali::Accessibility::GestureState state, uint32_t eventTime )
340 return FindSelf()->DoGesture( Dali::Accessibility::GestureInfo {type, xBeg, xEnd, yBeg, yEnd, state, eventTime});
343 DBus::ValueOrError< Accessible*, uint8_t, Accessible* > BridgeAccessible::GetNavigableAtPoint( int32_t x, int32_t y, uint32_t coordType )
345 Accessible* deputy = nullptr;
346 auto accessible = FindSelf();
347 auto cType = static_cast< CoordType >( coordType );
348 LOG() << "GetNavigableAtPoint: " << x << ", " << y << " type: " << coordType;
349 auto component = CalculateNavigableAccessibleAtPoint( accessible, {x, y}, cType, GET_NAVIGABLE_AT_POINT_MAX_RECURSION_DEPTH );
350 bool recurse = false;
353 const auto states = component->GetStates();
354 if( states[State::MODAL] )
361 recurse = component->IsProxy();
364 return {component, recurse, deputy};
367 static bool CheckChainEndWithAttribute( Accessible* obj, unsigned char forward )
371 auto attrs = obj->GetAttributes();
372 for( auto& attr : attrs )
374 if( attr.first == "relation_chain_end" )
376 if( ( attr.second == "prev,end" && forward == 0 ) || ( attr.second == "next,end" && forward == 1 ) || attr.second == "prev,next,end" )
385 static Accessible* DeputyOfProxyInParentGet( Accessible* obj )
390 // Accessible *deputy = nullptr;
391 // auto children = obj->GetChildren();
392 // unsigned int index = 0;
393 // for (auto child : children) {
394 // if (child->IsProxy()) {
396 // //WRN("Proxy does not have deputy object");
399 // deputy = children[index - 1];
407 Accessible* BridgeAccessible::GetCurrentlyHighlighted()
409 //TODO: add currently highlighted object
413 std::vector< Accessible* > BridgeAccessible::ValidChildrenGet( const std::vector< Accessible* >& children, Accessible* start, Accessible* root )
415 /* condition to find first(last) object regardless of scrollable parent.
416 looping navigation does not care scrollable parent.
417 1. currently highlighted object exists
418 2. both start and root are same */
420 /* TODO: add code, we need a scrollable implementation first
421 Accessible *current = GetCurrentlyHighlighted();
422 if (current && start == root) return children;
423 if(children.size() == 0) return {};
425 Eo *child = children[0];
429 Evas_Coord x = 0, y = 0, w = 0, h = 0;
430 Evas_Coord sx = 0, sy = 0, sw = 0, sh = 0;
432 if (_new_scrollable_parent_viewport_geometry_get(child, start,
435 Eina_List *l, *l_next;
436 EINA_LIST_FOREACH_SAFE(children, l, l_next, child)
439 elm_interface_atspi_component_extents_get(EINA_FALSE,
441 if (w == 0 || h == 0 ||
442 !ELM_RECTS_INTERSECT(x, y, w, h, sx, sy, sw, sh))
443 children = eina_list_remove_list(children, l);
451 static bool DeputyIs( Accessible* obj )
457 static Accessible* ProxyInParentGet( Accessible* obj )
461 auto children = obj->GetChildren();
462 for( auto& child : children )
464 if( child->IsProxy() )
470 static bool ObjectRoleIsAcceptableWhenNavigatingNextPrev( Accessible* obj )
474 auto role = obj->GetRole();
475 return role != Role::POPUP_MENU && role != Role::DIALOG;
479 struct CycleDetection
481 CycleDetection( const T value ) : key( value ), currentSearchSize( 1 ), counter( 1 ) {}
482 bool check( const T value )
488 currentSearchSize <<= 1;
489 if( currentSearchSize == 0 )
490 return true; // UNDEFINED BEHAVIOR
491 counter = currentSearchSize;
497 unsigned int currentSearchSize;
498 unsigned int counter;
501 static Accessible* FindNonDefunctChild( const std::vector< Accessible* >& children, unsigned int currentIndex, unsigned char forward )
503 unsigned int childrenCount = children.size();
504 for( ; currentIndex < childrenCount; forward ? ++currentIndex : --currentIndex )
506 Accessible* n = children[currentIndex];
507 if( n && !n->GetStates()[State::DEFUNCT] )
513 static Accessible* DirectionalDepthFirstSearchTryNonDefunctChild( Accessible* node, const std::vector< Accessible* >& children, unsigned char forward )
517 auto childrenCount = children.size();
518 if( childrenCount > 0 )
520 const bool isShowing = GetScrollableParent( node ) == nullptr ? node->GetStates()[State::SHOWING] : true;
523 return FindNonDefunctChild( children, forward ? 0 : childrenCount - 1, forward );
529 Accessible* BridgeAccessible::GetNextNonDefunctSibling( Accessible* obj, Accessible* start, Accessible* root, unsigned char forward )
533 auto parent = obj->GetParent();
537 auto children = ValidChildrenGet( parent->GetChildren(), start, root );
539 unsigned int children_count = children.size();
540 if( children_count == 0 )
544 unsigned int current = 0;
545 for( ; current < children_count && children[current] != obj; ++current )
547 if( current >= children_count )
551 forward ? ++current : --current;
552 auto ret = FindNonDefunctChild( children, current, forward );
556 Accessible* BridgeAccessible::DirectionalDepthFirstSearchTryNonDefunctSibling( bool& all_children_visited, Accessible* node, Accessible* start, Accessible* root, unsigned char forward )
560 Accessible* sibling = GetNextNonDefunctSibling( node, start, root, forward );
564 all_children_visited = false;
568 node = node->GetParent();
569 if( node == nullptr || node == root )
572 // in backward traversing stop the walk up on parent
579 Accessible* BridgeAccessible::CalculateNeighbor( Accessible* root, Accessible* start, unsigned char forward, BridgeAccessible::GetNeighborSearchMode search_mode )
581 if( start && CheckChainEndWithAttribute( start, forward ) )
583 if( root && root->GetStates()[State::DEFUNCT] )
585 if( start && start->GetStates()[State::DEFUNCT] )
591 if( search_mode == BridgeAccessible::GetNeighborSearchMode::recurseToOutside )
593 /* This only works if we navigate backward, and it is not possible to
594 find in embedded process. In this case the deputy should be used */
595 return DeputyOfProxyInParentGet( start );
598 Accessible* node = start ? start : root;
602 // initialization of all-children-visited flag for start node - we assume
603 // that when we begin at start node and we navigate backward, then all children
604 // are visited, so navigation will ignore start's children and go to
605 // previous sibling available.
606 /* Regarding condtion (start != root):
607 The last object can be found only if all_children_visited is false.
608 The start is same with root, when looking for the last object. */
609 bool all_children_visited = ( start != root ) && ( search_mode != BridgeAccessible::GetNeighborSearchMode::recurseFromRoot && !forward );
610 // true, if starting element should be ignored. this is only used in rare case of
611 // recursive search failing to find an object.
612 // consider tree, where element A on bus BUS_A has child B on bus BUS_B. when going "next" from
613 // element A algorithm has to descend into BUS_B and search element B and its children. this is done
614 // by returning to our caller object B with special flag set (meaning - continue the search from B on bus BUS_B).
615 // if next object will be found there (on BUS_B), then search ends. but if not, then our caller will find it out
616 // and will call us again with object A and flag search_mode set to NEIGHBOR_SEARCH_MODE_CONTINUE_AFTER_FAILED_RECURSING.
617 // this flag means, that object A was already checked previously and we should skip it and its children.
618 bool force_next = ( search_mode == BridgeAccessible::GetNeighborSearchMode::continueAfterFailedRecursion );
620 CycleDetection< Accessible* > cycleDetection( node );
623 if( node->GetStates()[State::DEFUNCT] )
626 // always accept proxy object from different world
627 if( !force_next && node->IsProxy() )
630 auto children = node->GetChildren();
631 children = ValidChildrenGet( children, start, root );
635 // 2. parent after all children in backward traversing
636 // 3. Nodes with roles: ATSPI_ROLE_PAGE_TAB, ATSPI_ROLE_POPUP_MENU and ATSPI_ROLE_DIALOG, only when looking for first or last element.
637 // Objects with those roles shouldnt be reachable, when navigating next / prev.
638 bool all_children_visited_or_moving_forward = ( children.size() == 0 || forward || all_children_visited );
639 if( !force_next && node != start && all_children_visited_or_moving_forward && AcceptObject( node ) )
641 if( start == NULL || ObjectRoleIsAcceptableWhenNavigatingNextPrev( node ) )
645 Accessible* next_related_in_direction = !force_next ? GetObjectInRelation( node, forward ? RelationType::FLOWS_TO : RelationType::FLOWS_FROM ) : nullptr;
646 /* force_next means that the search_mode is NEIGHBOR_SEARCH_MODE_CONTINUE_AFTER_FAILED_RECURSING
647 in this case the node is elm_layout which is parent of proxy object.
648 There is an access object working for the proxy object, and the access
649 object could have relation information. This relation information should
650 be checked first before using the elm_layout as a node. */
651 if( force_next && forward )
653 auto deputy = DeputyOfProxyInParentGet( node );
654 next_related_in_direction =
655 GetObjectInRelation( deputy, forward ? RelationType::FLOWS_TO : RelationType::FLOWS_FROM );
658 if( next_related_in_direction && start->GetStates()[State::DEFUNCT] )
659 next_related_in_direction = NULL;
660 unsigned char want_cycle_detection = 0;
661 if( next_related_in_direction )
663 /* Check next_related_in_direction is deputy object */
667 /* If the prev object is deputy, then go to inside of its proxy first */
668 if( DeputyIs( next_related_in_direction ) )
670 parent = next_related_in_direction->GetParent();
671 next_related_in_direction = ProxyInParentGet( parent );
676 /* If current object is deputy, and it has relation next object,
677 then do not use the relation next object, and use proxy first */
678 if( DeputyIs( node ) )
680 parent = node->GetParent();
681 next_related_in_direction = ProxyInParentGet( parent );
684 node = next_related_in_direction;
685 want_cycle_detection = 1;
689 auto child = !force_next && !all_children_visited ? DirectionalDepthFirstSearchTryNonDefunctChild( node, children, forward ) : nullptr;
692 want_cycle_detection = 1;
696 if( !force_next && node == root )
698 all_children_visited = true;
699 child = DirectionalDepthFirstSearchTryNonDefunctSibling( all_children_visited, node, start, root, forward );
704 if( want_cycle_detection && cycleDetection.check( node ) )
712 DBus::ValueOrError< Accessible*, uint8_t > BridgeAccessible::GetNeighbor( std::string rootPath, int32_t direction, int32_t search_mode )
714 auto start = FindSelf();
715 rootPath = StripPrefix( rootPath );
716 auto root = !rootPath.empty() ? Find( rootPath ) : nullptr;
717 auto accessible = CalculateNeighbor( root, start, direction == 1, static_cast< GetNeighborSearchMode >( search_mode ) );
718 unsigned char recurse = 0;
721 recurse = accessible->IsProxy();
723 return {accessible, recurse};
726 Accessible* BridgeAccessible::GetParent()
728 // NOTE: currently bridge supports single application root element.
729 // only element set as application root might return nullptr from GetParent
730 // if you want more, then you need to change setApplicationRoot to
731 // add/remove ApplicationRoot and make roots a vector.
732 auto p = FindSelf()->GetParent();
736 DBus::ValueOrError< std::vector< Accessible* > > BridgeAccessible::GetChildren()
738 return FindSelf()->GetChildren();
740 std::string BridgeAccessible::GetDescription()
742 return FindSelf()->GetDescription();
744 DBus::ValueOrError< uint32_t > BridgeAccessible::GetRole()
746 return static_cast< unsigned int >( FindSelf()->GetRole() );
748 DBus::ValueOrError< std::string > BridgeAccessible::GetRoleName()
750 return FindSelf()->GetRoleName();
752 DBus::ValueOrError< std::string > BridgeAccessible::GetLocalizedRoleName()
754 return FindSelf()->GetLocalizedRoleName();
756 DBus::ValueOrError< int32_t > BridgeAccessible::GetIndexInParent()
758 return FindSelf()->GetIndexInParent();
760 DBus::ValueOrError< std::array< uint32_t, 2 > > BridgeAccessible::GetStates()
762 return FindSelf()->GetStates().GetRawData();
764 DBus::ValueOrError< std::unordered_map< std::string, std::string > > BridgeAccessible::GetAttributes()
766 return FindSelf()->GetAttributes();
768 DBus::ValueOrError< std::vector< std::string > > BridgeAccessible::GetInterfaces()
770 return FindSelf()->GetInterfaces();
772 int BridgeAccessible::GetChildCount()
774 return FindSelf()->GetChildCount();
776 DBus::ValueOrError< Accessible* > BridgeAccessible::GetChildAtIndex( int index )
779 throw std::domain_error{"negative index (" + std::to_string( index ) + ")"};
780 return FindSelf()->GetChildAtIndex( static_cast< size_t >( index ) );
783 std::string BridgeAccessible::GetName()
785 return FindSelf()->GetName();
788 DBus::ValueOrError< Accessible*, uint32_t , std::unordered_map< std::string, std::string > > BridgeAccessible::GetDefaultLabelInfo()
790 auto defaultLabel = FindSelf()->GetDefaultLabel();
791 return {defaultLabel, static_cast< uint32_t >( defaultLabel->GetRole() ) , defaultLabel->GetAttributes()};
794 DBus::ValueOrError<std::vector< BridgeAccessible::Relation >> BridgeAccessible::GetRelationSet()
796 auto relations = FindSelf()->GetRelationSet();
797 std::vector< BridgeAccessible::Relation > ret;
799 for (auto &it : relations)
800 ret.emplace_back(Relation{static_cast<uint32_t>(it.relationType), it.targets});