2 * Copyright (c) 2019 Samsung Electronics Co., Ltd.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
19 #include <dali/internal/accessibility/bridge/bridge-accessible.h>
24 //comment out 2 lines below to get more logs
26 #define LOG() _LoggerEmpty()
28 using namespace Dali::Accessibility;
30 #define GET_NAVIGABLE_AT_POINT_MAX_RECURSION_DEPTH 10000
32 BridgeAccessible::BridgeAccessible()
36 void BridgeAccessible::RegisterInterfaces()
38 DBus::DBusInterfaceDescription desc{AtspiDbusInterfaceAccessible};
39 AddGetPropertyToInterface( desc, "ChildCount", &BridgeAccessible::GetChildCount );
40 AddGetPropertyToInterface( desc, "Name", &BridgeAccessible::GetName );
41 AddGetPropertyToInterface( desc, "Description", &BridgeAccessible::GetDescription );
42 AddGetPropertyToInterface( desc, "Parent", &BridgeAccessible::GetParent );
43 AddFunctionToInterface( desc, "GetRole", &BridgeAccessible::GetRole );
44 AddFunctionToInterface( desc, "GetRoleName", &BridgeAccessible::GetRoleName );
45 AddFunctionToInterface( desc, "GetLocalizedRoleName", &BridgeAccessible::GetLocalizedRoleName );
46 AddFunctionToInterface( desc, "GetState", &BridgeAccessible::GetStates );
47 AddFunctionToInterface( desc, "GetAttributes", &BridgeAccessible::GetAttributes );
48 AddFunctionToInterface( desc, "GetInterfaces", &BridgeAccessible::GetInterfaces );
49 AddFunctionToInterface( desc, "GetChildAtIndex", &BridgeAccessible::GetChildAtIndex );
50 AddFunctionToInterface( desc, "GetChildren", &BridgeAccessible::GetChildren );
51 AddFunctionToInterface( desc, "GetIndexInParent", &BridgeAccessible::GetIndexInParent );
52 AddFunctionToInterface( desc, "GetNavigableAtPoint", &BridgeAccessible::GetNavigableAtPoint );
53 AddFunctionToInterface( desc, "GetNeighbor", &BridgeAccessible::GetNeighbor );
54 AddFunctionToInterface( desc, "GetDefaultLabelInfo", &BridgeAccessible::GetDefaultLabelInfo );
55 AddFunctionToInterface( desc, "DoGesture", &BridgeAccessible::DoGesture );
56 AddFunctionToInterface( desc, "GetReadingMaterial", &BridgeAccessible::GetReadingMaterial );
57 AddFunctionToInterface( desc, "GetRelationSet", &BridgeAccessible::GetRelationSet );
58 dbusServer.addInterface( "/", desc, true );
61 static bool AcceptObjectCheckRole( Component* obj )
65 switch( obj->GetRole() )
67 case Role::APPLICATION:
69 case Role::SCROLL_PANE:
70 case Role::SPLIT_PANE:
77 case Role::REDUNDANT_OBJECT:
78 case Role::COLOR_CHOOSER:
79 case Role::TREE_TABLE:
80 case Role::PAGE_TAB_LIST:
82 case Role::SPIN_BUTTON:
83 case Role::INPUT_METHOD_WINDOW:
86 case Role::NOTIFICATION:
87 case Role::DATE_EDITOR:
101 static bool AcceptObjectCheckRelations( Component* obj)
103 auto r = obj->GetRelationSet();
105 for (const auto& it : r)
106 if (it.relationType == RelationType::CONTROLLED_BY)
112 static Component* GetScrollableParent( Accessible* obj )
116 obj = obj->GetParent();
117 auto comp = dynamic_cast< Component* >( obj );
118 if( comp && comp->IsScrollable() )
124 static bool ObjectIsItem( Component* obj )
128 auto role = obj->GetRole();
129 return role == Role::LIST_ITEM || role == Role::MENU_ITEM;
132 static bool ObjectIsCollapsed( Component* obj )
136 const auto states = obj->GetStates();
137 return states[State::EXPANDABLE] && !states[State::EXPANDED];
140 static bool OobjectIsZeroSize( Component* obj )
144 auto extents = obj->GetExtents( CoordType::WINDOW );
145 return extents.height == 0 || extents.width == 0;
148 static bool AcceptObject( Component* obj )
152 const auto states = obj->GetStates();
153 if( !states[State::VISIBLE] )
155 if( !AcceptObjectCheckRole( obj ) )
157 if ( !AcceptObjectCheckRelations( obj ) )
159 // if (CALL(get_object_in_relation_by_type, obj, ATSPI_RELATION_CONTROLLED_BY) != NULL) return 0;
160 if ( !AcceptObjectCheckRelations( obj ) )
162 if( !states[State::HIGHLIGHTABLE] )
165 if( GetScrollableParent( obj ) != nullptr )
167 auto parent = dynamic_cast< Component* >( obj->GetParent() );
171 return !ObjectIsItem( obj ) || !ObjectIsCollapsed( parent );
176 if( OobjectIsZeroSize( obj ) )
180 if( !states[State::SHOWING] )
188 static bool AcceptObject( Accessible* obj )
190 auto c = dynamic_cast< Component* >( obj );
191 return AcceptObject( c );
194 static std::string objDump( Component* obj )
198 std::ostringstream o;
199 auto e = obj->GetExtents( CoordType::SCREEN );
200 o << "name: " << obj->GetName() << " extent: (" << e.x << ", "
201 << e.y << "), [" << e.width << ", " << e.height << "]";
205 Component * BridgeAccessible::GetObjectInRelation( Accessible * obj, RelationType ralationType )
209 for ( auto &relation : obj->GetRelationSet() )
211 if ( relation.relationType == ralationType )
213 for ( auto &address : relation.targets )
215 auto component = dynamic_cast<Component*>( Find( address ) );
224 static std::string makeIndent( unsigned int maxRecursionDepth )
226 return std::string( GET_NAVIGABLE_AT_POINT_MAX_RECURSION_DEPTH - maxRecursionDepth, ' ' );
229 Component* BridgeAccessible::CalculateNavigableAccessibleAtPoint( Accessible* root, Point p, CoordType cType, unsigned int maxRecursionDepth )
231 if( !root || maxRecursionDepth == 0 )
233 auto root_component = dynamic_cast< Component* >( root );
234 LOG() << "CalculateNavigableAccessibleAtPoint: checking: " << makeIndent(maxRecursionDepth) << objDump(root_component);
236 if( root_component && !root_component->Contains( p, cType ) )
239 auto children = root->GetChildren();
240 for( auto childIt = children.rbegin(); childIt != children.rend(); childIt++ )
242 //check recursively all children first
243 auto result = CalculateNavigableAccessibleAtPoint( *childIt, p, cType, maxRecursionDepth - 1 );
249 //Found a candidate, all its children are already checked
250 auto controledBy = GetObjectInRelation( root_component, RelationType::CONTROLLED_BY );
252 controledBy = root_component;
254 if ( controledBy->IsProxy() || AcceptObject( controledBy ) )
256 LOG() << "CalculateNavigableAccessibleAtPoint: found: " << makeIndent(maxRecursionDepth) << objDump( root_component );
263 BridgeAccessible::ReadingMaterialType BridgeAccessible::GetReadingMaterial()
265 auto self = FindSelf();
266 auto attributes = self->GetAttributes();
267 auto name = self->GetName();
268 std::string labeledByName = "";
269 std::string textIfceName = "";
270 auto role = static_cast< uint32_t >( self->GetRole() );
271 auto states = self->GetStates();
272 auto localizedName = self->GetLocalizedRoleName();
273 auto childCount = static_cast< int32_t >( self->GetChildCount() );
275 double currentValue = 0.0;
276 double minimumIncrement = 0.0;
277 double maximumValue = 0.0;
278 double minimumValue = 0.0;
280 auto *value = dynamic_cast<Dali::Accessibility::Value *>(self);
283 currentValue = value->GetCurrent();
284 minimumIncrement = value->GetMinimumIncrement();
285 maximumValue = value->GetMaximum();
286 minimumValue = value->GetMinimum();
289 auto description = self->GetDescription();
290 auto indexInParent = static_cast< int32_t >( self->GetIndexInParent() );
291 bool isSelectedInParent = false;
292 bool hasCheckBoxChild = false;
293 int32_t firstSelectedChildIndex = 0;
294 int32_t selectedChildCount = 0;
296 for( auto i = 0u; i < static_cast< size_t >( childCount ); ++i )
298 auto q = self->GetChildAtIndex( i );
299 auto s = q->GetStates();
300 if( s[State::SELECTABLE] )
302 ++selectedChildCount;
303 if( s[State::SELECTED] )
305 if( firstSelectedChildIndex < 0 )
306 firstSelectedChildIndex = static_cast< int32_t >( i );
309 if( q->GetRole() == Role::CHECK_BOX )
310 hasCheckBoxChild = true;
313 int32_t listChildrenCount = 0;
314 Accessible* parent = self->GetParent();
315 auto parentStateSet = parent ? parent->GetStates() : States{};
316 auto parentChildCount = parent ? static_cast< int32_t >( parent->GetChildCount() ) : 0;
317 auto parentRole = static_cast< uint32_t >( parent ? parent->GetRole() : Role{} );
318 Accessible* describedByObject = nullptr;
338 firstSelectedChildIndex,
347 DBus::ValueOrError< bool > BridgeAccessible::DoGesture( Dali::Accessibility::Gesture type, int32_t xBeg, int32_t xEnd, int32_t yBeg, int32_t yEnd, Dali::Accessibility::GestureState state, uint32_t eventTime )
349 return FindSelf()->DoGesture( Dali::Accessibility::GestureInfo {type, xBeg, xEnd, yBeg, yEnd, state, eventTime});
352 DBus::ValueOrError< Accessible*, uint8_t, Accessible* > BridgeAccessible::GetNavigableAtPoint( int32_t x, int32_t y, uint32_t coordType )
354 Accessible* deputy = nullptr;
355 auto accessible = FindSelf();
356 auto cType = static_cast< CoordType >( coordType );
357 LOG() << "GetNavigableAtPoint: " << x << ", " << y << " type: " << coordType;
358 auto component = CalculateNavigableAccessibleAtPoint( accessible, {x, y}, cType, GET_NAVIGABLE_AT_POINT_MAX_RECURSION_DEPTH );
359 bool recurse = false;
362 const auto states = component->GetStates();
363 if( states[State::MODAL] )
370 recurse = component->IsProxy();
373 return {component, recurse, deputy};
376 static bool CheckChainEndWithAttribute( Accessible* obj, unsigned char forward )
380 auto attrs = obj->GetAttributes();
381 for( auto& attr : attrs )
383 if( attr.first == "relation_chain_end" )
385 if( ( attr.second == "prev,end" && forward == 0 ) || ( attr.second == "next,end" && forward == 1 ) || attr.second == "prev,next,end" )
394 static Accessible* DeputyOfProxyInParentGet( Accessible* obj )
399 Accessible* BridgeAccessible::GetCurrentlyHighlighted()
401 //TODO: add currently highlighted object
405 std::vector< Accessible* > BridgeAccessible::ValidChildrenGet( const std::vector< Accessible* >& children, Accessible* start, Accessible* root )
410 static bool DeputyIs( Accessible* obj )
416 static Accessible* ProxyInParentGet( Accessible* obj )
420 auto children = obj->GetChildren();
421 for( auto& child : children )
423 if( child->IsProxy() )
429 static bool ObjectRoleIsAcceptableWhenNavigatingNextPrev( Accessible* obj )
433 auto role = obj->GetRole();
434 return role != Role::POPUP_MENU && role != Role::DIALOG;
438 struct CycleDetection
440 CycleDetection( const T value ) : key( value ), currentSearchSize( 1 ), counter( 1 ) {}
441 bool check( const T value )
447 currentSearchSize <<= 1;
448 if( currentSearchSize == 0 )
449 return true; // UNDEFINED BEHAVIOR
450 counter = currentSearchSize;
456 unsigned int currentSearchSize;
457 unsigned int counter;
460 static Accessible* FindNonDefunctChild( const std::vector< Accessible* >& children, unsigned int currentIndex, unsigned char forward )
462 unsigned int childrenCount = children.size();
463 for( ; currentIndex < childrenCount; forward ? ++currentIndex : --currentIndex )
465 Accessible* n = children[currentIndex];
466 if( n && !n->GetStates()[State::DEFUNCT] )
472 static Accessible* DirectionalDepthFirstSearchTryNonDefunctChild( Accessible* node, const std::vector< Accessible* >& children, unsigned char forward )
476 auto childrenCount = children.size();
477 if( childrenCount > 0 )
479 const bool isShowing = GetScrollableParent( node ) == nullptr ? node->GetStates()[State::SHOWING] : true;
482 return FindNonDefunctChild( children, forward ? 0 : childrenCount - 1, forward );
488 Accessible* BridgeAccessible::GetNextNonDefunctSibling( Accessible* obj, Accessible* start, Accessible* root, unsigned char forward )
492 auto parent = obj->GetParent();
496 auto children = ValidChildrenGet( parent->GetChildren(), start, root );
498 unsigned int children_count = children.size();
499 if( children_count == 0 )
503 unsigned int current = 0;
504 for( ; current < children_count && children[current] != obj; ++current )
506 if( current >= children_count )
510 forward ? ++current : --current;
511 auto ret = FindNonDefunctChild( children, current, forward );
515 Accessible* BridgeAccessible::DirectionalDepthFirstSearchTryNonDefunctSibling( bool& all_children_visited, Accessible* node, Accessible* start, Accessible* root, unsigned char forward )
519 Accessible* sibling = GetNextNonDefunctSibling( node, start, root, forward );
523 all_children_visited = false;
527 node = node->GetParent();
528 if( node == nullptr || node == root )
531 // in backward traversing stop the walk up on parent
538 Accessible* BridgeAccessible::CalculateNeighbor( Accessible* root, Accessible* start, unsigned char forward, BridgeAccessible::GetNeighborSearchMode search_mode )
540 if( start && CheckChainEndWithAttribute( start, forward ) )
542 if( root && root->GetStates()[State::DEFUNCT] )
544 if( start && start->GetStates()[State::DEFUNCT] )
550 if( search_mode == BridgeAccessible::GetNeighborSearchMode::recurseToOutside )
552 // This only works if we navigate backward, and it is not possible to
553 // find in embedded process. In this case the deputy should be used */
554 return DeputyOfProxyInParentGet( start );
557 Accessible* node = start ? start : root;
561 // initialization of all-children-visited flag for start node - we assume
562 // that when we begin at start node and we navigate backward, then all children
563 // are visited, so navigation will ignore start's children and go to
564 // previous sibling available.
565 // Regarding condtion (start != root):
566 // The last object can be found only if all_children_visited is false.
567 // The start is same with root, when looking for the last object.
568 bool all_children_visited = ( start != root ) && ( search_mode != BridgeAccessible::GetNeighborSearchMode::recurseFromRoot && !forward );
569 // true, if starting element should be ignored. this is only used in rare case of
570 // recursive search failing to find an object.
571 // consider tree, where element A on bus BUS_A has child B on bus BUS_B. when going "next" from
572 // element A algorithm has to descend into BUS_B and search element B and its children. this is done
573 // by returning to our caller object B with special flag set (meaning - continue the search from B on bus BUS_B).
574 // if next object will be found there (on BUS_B), then search ends. but if not, then our caller will find it out
575 // and will call us again with object A and flag search_mode set to NEIGHBOR_SEARCH_MODE_CONTINUE_AFTER_FAILED_RECURSING.
576 // this flag means, that object A was already checked previously and we should skip it and its children.
577 bool force_next = ( search_mode == BridgeAccessible::GetNeighborSearchMode::continueAfterFailedRecursion );
579 CycleDetection< Accessible* > cycleDetection( node );
582 if( node->GetStates()[State::DEFUNCT] )
585 // always accept proxy object from different world
586 if( !force_next && node->IsProxy() )
589 auto children = node->GetChildren();
590 children = ValidChildrenGet( children, start, root );
594 // 2. parent after all children in backward traversing
595 // 3. Nodes with roles: ATSPI_ROLE_PAGE_TAB, ATSPI_ROLE_POPUP_MENU and ATSPI_ROLE_DIALOG, only when looking for first or last element.
596 // Objects with those roles shouldnt be reachable, when navigating next / prev.
597 bool all_children_visited_or_moving_forward = ( children.size() == 0 || forward || all_children_visited );
598 if( !force_next && node != start && all_children_visited_or_moving_forward && AcceptObject( node ) )
600 if( start == NULL || ObjectRoleIsAcceptableWhenNavigatingNextPrev( node ) )
604 Accessible* next_related_in_direction = !force_next ? GetObjectInRelation( node, forward ? RelationType::FLOWS_TO : RelationType::FLOWS_FROM ) : nullptr;
605 // force_next means that the search_mode is NEIGHBOR_SEARCH_MODE_CONTINUE_AFTER_FAILED_RECURSING
606 // in this case the node is elm_layout which is parent of proxy object.
607 // There is an access object working for the proxy object, and the access
608 // object could have relation information. This relation information should
609 // be checked first before using the elm_layout as a node.
610 if( force_next && forward )
612 auto deputy = DeputyOfProxyInParentGet( node );
613 next_related_in_direction =
614 GetObjectInRelation( deputy, RelationType::FLOWS_TO );
617 if( next_related_in_direction && start && start->GetStates()[State::DEFUNCT] )
619 next_related_in_direction = NULL;
622 unsigned char want_cycle_detection = 0;
623 if( next_related_in_direction )
625 // Check next_related_in_direction is deputy object
629 // If the prev object is deputy, then go to inside of its proxy first
630 if( DeputyIs( next_related_in_direction ) )
632 parent = next_related_in_direction->GetParent();
633 next_related_in_direction = ProxyInParentGet( parent );
638 // If current object is deputy, and it has relation next object,
639 // then do not use the relation next object, and use proxy first
640 if( DeputyIs( node ) )
642 parent = node->GetParent();
643 next_related_in_direction = ProxyInParentGet( parent );
646 node = next_related_in_direction;
647 want_cycle_detection = 1;
651 auto child = !force_next && !all_children_visited ? DirectionalDepthFirstSearchTryNonDefunctChild( node, children, forward ) : nullptr;
654 want_cycle_detection = 1;
658 if( !force_next && node == root )
660 all_children_visited = true;
661 child = DirectionalDepthFirstSearchTryNonDefunctSibling( all_children_visited, node, start, root, forward );
666 if( want_cycle_detection && cycleDetection.check( node ) )
674 DBus::ValueOrError< Accessible*, uint8_t > BridgeAccessible::GetNeighbor( std::string rootPath, int32_t direction, int32_t search_mode )
676 auto start = FindSelf();
677 rootPath = StripPrefix( rootPath );
678 auto root = !rootPath.empty() ? Find( rootPath ) : nullptr;
679 auto accessible = CalculateNeighbor( root, start, direction == 1, static_cast< GetNeighborSearchMode >( search_mode ) );
680 unsigned char recurse = 0;
683 recurse = accessible->IsProxy();
685 return {accessible, recurse};
688 Accessible* BridgeAccessible::GetParent()
690 // NOTE: currently bridge supports single application root element.
691 // only element set as application root might return nullptr from GetParent
692 // if you want more, then you need to change setApplicationRoot to
693 // add/remove ApplicationRoot and make roots a vector.
694 auto p = FindSelf()->GetParent();
698 DBus::ValueOrError< std::vector< Accessible* > > BridgeAccessible::GetChildren()
700 return FindSelf()->GetChildren();
702 std::string BridgeAccessible::GetDescription()
704 return FindSelf()->GetDescription();
706 DBus::ValueOrError< uint32_t > BridgeAccessible::GetRole()
708 return static_cast< unsigned int >( FindSelf()->GetRole() );
710 DBus::ValueOrError< std::string > BridgeAccessible::GetRoleName()
712 return FindSelf()->GetRoleName();
714 DBus::ValueOrError< std::string > BridgeAccessible::GetLocalizedRoleName()
716 return FindSelf()->GetLocalizedRoleName();
718 DBus::ValueOrError< int32_t > BridgeAccessible::GetIndexInParent()
720 return FindSelf()->GetIndexInParent();
722 DBus::ValueOrError< std::array< uint32_t, 2 > > BridgeAccessible::GetStates()
724 return FindSelf()->GetStates().GetRawData();
726 DBus::ValueOrError< std::unordered_map< std::string, std::string > > BridgeAccessible::GetAttributes()
728 return FindSelf()->GetAttributes();
730 DBus::ValueOrError< std::vector< std::string > > BridgeAccessible::GetInterfaces()
732 return FindSelf()->GetInterfaces();
734 int BridgeAccessible::GetChildCount()
736 return FindSelf()->GetChildCount();
738 DBus::ValueOrError< Accessible* > BridgeAccessible::GetChildAtIndex( int index )
741 throw std::domain_error{"negative index (" + std::to_string( index ) + ")"};
742 return FindSelf()->GetChildAtIndex( static_cast< size_t >( index ) );
745 std::string BridgeAccessible::GetName()
747 return FindSelf()->GetName();
750 DBus::ValueOrError< Accessible*, uint32_t , std::unordered_map< std::string, std::string > > BridgeAccessible::GetDefaultLabelInfo()
752 auto defaultLabel = FindSelf()->GetDefaultLabel();
753 return {defaultLabel, static_cast< uint32_t >( defaultLabel->GetRole() ) , defaultLabel->GetAttributes()};
756 DBus::ValueOrError<std::vector< BridgeAccessible::Relation >> BridgeAccessible::GetRelationSet()
758 auto relations = FindSelf()->GetRelationSet();
759 std::vector< BridgeAccessible::Relation > ret;
761 for (auto &it : relations)
762 ret.emplace_back(Relation{static_cast<uint32_t>(it.relationType), it.targets});