org.tizen.aurum-bootstrap/src/Commands/FindElementsCommand.cc

   1 /*
   2  * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  *  Licensed under the Apache License, Version 2.0 (the "License");
   5  *  you may not use this file except in compliance with the License.
   6  *  You may obtain a copy of the License at
   7  *
   8  *               http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  *  Unless required by applicable law or agreed to in writing, software
  11  *  distributed under the License is distributed on an "AS IS" BASIS,
  12  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *  See the License for the specific language governing permissions and
  14  *  limitations under the License.
  15  *
  16  */
  17
  18 #include "bootstrap.h"
  19 #include "FindElementsCommand.h"
  20 #include "UiObject.h"
  21 #include "UiDevice.h"
  22 #include "UiSelector.h"
  23 #include "Sel.h"
  24 #include "ISearchable.h"
  25 #ifdef MQTT_ENABLED
  26 #include "SaObject.h"
  27 #endif
  28 #include "Until.h"
  29
  30 FindElementsCommand::FindElementsCommand(const ::aurum::ReqFindElements *request,
  31                                        ::aurum::RspFindElements *response,
  32                                        int timeout)
  33     : mRequest{request}, mResponse{response}, mTimeout{timeout}
  34 {
  35     mObjMap = ObjectMapper::getInstance();
  36 }
  37 std::shared_ptr<ISearchable> FindElementsCommand::getSearchableTop(void)
  38 {
  39     std::shared_ptr<ISearchable> searchableObj{nullptr};
  40
  41     if (mRequest->_elementid_case() != 0) {
  42         searchableObj = mObjMap->getElement(mRequest->elementid());
  43     }
  44     if (!searchableObj) searchableObj = UiDevice::getInstance();
  45
  46     return searchableObj;
  47 }
  48
  49 std::vector<std::shared_ptr<UiSelector>> FindElementsCommand::getSelectors(void)
  50 {
  51     auto sel = std::make_shared<UiSelector>();
  52
  53     if(mRequest->_elementid_case())        sel->id(mRequest->elementid());
  54     if(mRequest->_automationid_case())     sel->automationid(mRequest->automationid());
  55     if(mRequest->_textfield_case())        sel->text(mRequest->textfield());
  56     if(mRequest->_widgettype_case())       sel->type(mRequest->widgettype());
  57     if(mRequest->_widgetstyle_case())      sel->style(mRequest->widgetstyle());
  58     if(mRequest->_ischecked_case())        sel->isChecked(mRequest->ischecked());
  59     if(mRequest->_ischeckable_case())      sel->isCheckable(mRequest->ischeckable());
  60     if(mRequest->_isclickable_case())      sel->isClickable(mRequest->isclickable());
  61     if(mRequest->_isenabled_case())        sel->isEnabled(mRequest->isenabled());
  62     if(mRequest->_isfocused_case())        sel->isFocused(mRequest->isfocused());
  63     if(mRequest->_isfocusable_case())      sel->isFocusable(mRequest->isfocusable());
  64     if(mRequest->_isscrollable_case())     sel->isScrollable(mRequest->isscrollable());
  65     if(mRequest->_isselected_case())       sel->isSelected(mRequest->isselected());
  66     if(mRequest->_isshowing_case())        sel->isShowing(mRequest->isshowing());
  67     if(mRequest->_isactive_case())         sel->isActive(mRequest->isactive());
  68     if(mRequest->_mindepth_case())         sel->minDepth(mRequest->mindepth());
  69     if(mRequest->_maxdepth_case())         sel->maxDepth(mRequest->maxdepth());
  70     if(mRequest->_packagename_case())      sel->pkg(mRequest->packagename());
  71     if(mRequest->_textpartialmatch_case()) sel->textPartialMatch(mRequest->textpartialmatch());
  72     if(mRequest->_xpath_case())            sel->xpath(mRequest->xpath());
  73     if(mRequest->_ocrtext_case())          sel->ocrText(mRequest->ocrtext());
  74     if(mRequest->_geometry_case())         sel->geometry(Rect<int>{mRequest->geometry().x(), mRequest->geometry().y(), mRequest->geometry().x() + mRequest->geometry().width(), mRequest->geometry().y() + mRequest->geometry().height()}, false);
  75     if(mRequest->_isvisible_case())        sel->isVisible(mRequest->isvisible());
  76     if(mRequest->_isselectable_case())     sel->isSelectable(mRequest->isselectable());
  77     if(mRequest->_ishighlightable_case())  sel->isHighlightable(mRequest->ishighlightable());
  78
  79     return std::vector<std::shared_ptr<UiSelector>>{sel};
  80 }
  81
  82 ::grpc::Status FindElementsCommand::execute()
  83 {
  84     auto searchableObj = getSearchableTop();
  85     auto selectors     = getSelectors();
  86     std::shared_ptr<UiDevice> mDevice = UiDevice::getInstance();
  87
  88 #ifdef MQTT_ENABLED
  89     if (mDevice->getExternalAppLaunched())
  90     {
  91         mDevice->RequestScreenAnalyze();
  92
  93         std::vector<std::shared_ptr<SaObject>> founds = {};
  94
  95         for ( auto &sel : selectors ) {
  96             auto ret = mDevice->getSAWatcher()->findSaObjects(sel);
  97             std::move(std::begin(ret), std::end(ret), std::back_inserter(founds));
  98         }
  99         if (founds.size() > 0) {
 100             for (auto& found : founds) {
 101                 SaObject *obj = found.get();
 102                 ::aurum::Element *elm = mResponse->add_elements();
 103                 elm->set_elementid(obj->getId());
 104
 105                 ::aurum::Rect *rect = elm->mutable_geometry();
 106                 const Rect<int> &size = obj->getScreenBoundingBox();
 107                 rect->set_x(size.mTopLeft.x);
 108                 rect->set_y(size.mTopLeft.y);
 109                 rect->set_width(size.width());
 110                 rect->set_height(size.height());
 111
 112                 elm->set_widgettype(obj->getType());
 113
 114                 elm->set_text(obj->getOcrText());
 115                 elm->set_toolkit("External");
 116
 117                 elm->set_isclickable(obj->isClickable());
 118                 elm->set_isfocused(obj->isFocused());
 119                 elm->set_isfocusable(obj->isFocusable());
 120                 elm->set_isactive(obj->isActive());
 121                 elm->set_isshowing(true);
 122                 elm->set_isvisible(true);
 123             }
 124                 mResponse->set_status(::aurum::RspStatus::OK);
 125         }
 126         else {
 127             mResponse->set_status(::aurum::RspStatus::ERROR);
 128         }
 129     }
 130     else
 131 #endif
 132     {
 133         if (mDevice->getWithScreenAnalyzer())
 134         {
 135             LOGI("Find object with SA information");
 136             mDevice->RequestScreenAnalyze();
 137         }
 138
 139         std::vector<std::shared_ptr<UiObject>> founds = {};
 140
 141         auto waiter = new Waiter(searchableObj.get(), nullptr, mTimeout);
 142
 143         for ( auto &sel : selectors ) {
 144             auto ret = waiter->waitFor(Until::findObjects(sel));
 145             std::move(std::begin(ret), std::end(ret), std::back_inserter(founds));
 146         }
 147
 148         if (waiter) delete waiter;
 149
 150         if (founds.size() > 0) {
 151             for (auto& found : founds) {
 152                 UiObject *obj = found.get();
 153                 obj->refresh();
 154                 if (mObjMap->getElement(obj->getId()) == nullptr)
 155                     mObjMap->addElement(std::move(found));
 156
 157                 LOGI("found object : %p elementId:%s", obj, obj->getId().c_str());
 158
 159                 ::aurum::Element *elm = mResponse->add_elements();
 160                 elm->set_elementid(obj->getId());
 161                 elm->set_package(obj->getApplicationPackage());
 162
 163                 ::aurum::Rect *rect = elm->mutable_geometry();
 164                 const Rect<int> &size = obj->getScreenBoundingBox();
 165                 rect->set_x(size.mTopLeft.x);
 166                 rect->set_y(size.mTopLeft.y);
 167                 rect->set_width(size.width());
 168                 rect->set_height(size.height());
 169
 170                 ::aurum::Rect *windowRect = elm->mutable_windowrelativegeometry();
 171                 const Rect<int> &windowRelativeSize = obj->getWindowBoundingBox();
 172                 windowRect->set_x(windowRelativeSize.mTopLeft.x);
 173                 windowRect->set_y(windowRelativeSize.mTopLeft.y);
 174                 windowRect->set_width(windowRelativeSize.width());
 175                 windowRect->set_height(windowRelativeSize.height());
 176
 177                 elm->set_widgettype(obj->getType());
 178                 elm->set_widgetstyle(obj->getElementStyle());
 179
 180                 elm->set_text(obj->getText());
 181                 elm->set_xpath(obj->getXPath());
 182                 elm->set_ocrtext(obj->getOcrText());
 183                 elm->set_automationid(obj->getAutomationId());
 184                 elm->set_package(obj->getApplicationPackage());
 185                 elm->set_role(obj->getRole());
 186
 187                 elm->set_ischecked(obj->isChecked());
 188                 elm->set_ischeckable(obj->isCheckable());
 189                 elm->set_isclickable(obj->isClickable());
 190                 elm->set_isenabled(obj->isEnabled());
 191                 elm->set_isfocused(obj->isFocused());
 192                 elm->set_isfocusable(obj->isFocusable());
 193                 elm->set_isscrollable(obj->isScrollable());
 194                 elm->set_isselected(obj->isSelected());
 195                 elm->set_isshowing(obj->isShowing());
 196                 elm->set_isactive(obj->isActive());
 197                 elm->set_isvisible(obj->isVisible());
 198                 elm->set_isselectable(obj->isSelectable());
 199                 elm->set_ishighlightable(obj->isHighlightable());
 200
 201                 elm->set_minvalue(obj->getMinValue());
 202                 elm->set_maxvalue(obj->getMaxValue());
 203                 elm->set_value(obj->getValue());
 204                 elm->set_increment(obj->getIncrement());
 205             }
 206             mResponse->set_status(::aurum::RspStatus::OK);
 207         } else {
 208             mResponse->set_status(::aurum::RspStatus::ERROR);
 209         }
 210     }
 211
 212     return grpc::Status::OK;
 213 }