From: Woochan Lee Date: Wed, 25 Oct 2023 11:04:47 +0000 (+0900) Subject: Delete Screen Analyzer client code X-Git-Tag: accepted/tizen/unified/20231101.174502~1 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=refs%2Fchanges%2F68%2F300468%2F2;p=platform%2Fcore%2Fuifw%2Faurum.git Delete Screen Analyzer client code Screen Analyzer server is not working anymore since 2023/05 So currently aurum client codes has become useless Developed on the client code and the diff is growing so delete all related codes Change-Id: I8b33c9ded88abfbeff3c6a0420488c2a72e3c07f --- diff --git a/docs/protocol/aurum.proto b/docs/protocol/aurum.proto index d929d4d..b83bea5 100644 --- a/docs/protocol/aurum.proto +++ b/docs/protocol/aurum.proto @@ -34,7 +34,6 @@ service Bootstrap { rpc setFocus(ReqSetFocus) returns (RspSetFocus) {} rpc findElements(ReqFindElements) returns (RspFindElements) {} rpc getActiveAppToolkitName(ReqGetActiveAppToolkitName) returns (RspGetActiveAppToolkitName) {} - rpc enableScreenAnalyzer(ReqEnableScreenAnalyzer) returns (RspEnableScreenAnalyzer) {} rpc getTextMinBoundingRect(ReqGetTextMinBoundingRect) returns (RspGetTextMinBoundingRect) {} rpc setTimeout(ReqSetTimeout) returns (RspSetTimeout) {} rpc setXMLSync(ReqSetXMLSync) returns (RspSetXMLSync) {} @@ -631,15 +630,6 @@ message RspGetActiveAppToolkitName { string toolkitName = 2; } -message ReqEnableScreenAnalyzer { - bool enable = 1; - string serverIp = 2; -} - -message RspEnableScreenAnalyzer { - RspStatus status = 1; -} - message ReqGetTextMinBoundingRect { string elementId = 1; } diff --git a/libaurum/inc/Aurum.h b/libaurum/inc/Aurum.h index d6b2d5b..9abe627 100644 --- a/libaurum/inc/Aurum.h +++ b/libaurum/inc/Aurum.h @@ -62,9 +62,5 @@ #include "A11yEvent.h" #include "A11yEventHandler.h" #include "Runnables.h" -#ifdef MQTT_ENABLED -#include "ScreenAnalyzerWatcher.h" -#include "SaObject.h" -#endif #endif diff --git a/libaurum/inc/SaObject.h b/libaurum/inc/SaObject.h deleted file mode 100644 index 422f264..0000000 --- a/libaurum/inc/SaObject.h +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#ifndef _SA_OBJECT_H_ -#define _SA_OBJECT_H_ - -#include "config.h" -#include - -namespace Aurum { - -/** - * @class SaObject - * - * @ingroup aurum - * - * @brief A SaObject is a result of Screen Analyzer. - */ -class SaObject : public IObject, public std::enable_shared_from_this { -public: - - /** - * @brief SaObject constructor with initial values. - * - * @since_tizen 7.0 - */ - SaObject(std::string id, std::string type, Rect geometry, std::string ocrText, std::vector states); - - /** - * @brief SaObject destructor. - * - * @since_tizen 7.0 - */ - ~SaObject(); - - /** - * @copydoc UiObject::getId() - */ - std::string getId() const override; - - /** - * @copydoc UiObject::getType() - */ - std::string getType() const override; - - /** - * @copydoc UiObject::getScreenBoundingBox() - */ - const Rect getScreenBoundingBox() const override; - - std::string getOcrText() const override; - - /** - * @copydoc UiObject::isFocusable() - */ - bool isFocusable() const override; - - /** - * @copydoc UiObject::isFocused() - */ - bool isFocused() const override; - - /** - * @copydoc UiObject::isClickable() - */ - bool isClickable() const override; - - /** - * @copydoc UiObject::isActive() - */ - bool isActive() const override; - - /** - * @copydoc UiObject::isShowing() - */ - bool isShowing() const override; - -private: - std::string mId; - std::string mElementType; - Rect mScreenBoundingBox; - std::string mOcrText; - bool mIsFocusable; - bool mIsFocused; - bool mIsClickable; - bool mIsActive; -}; - -} - -#endif diff --git a/libaurum/inc/ScreenAnalyzer/ScreenAnalyzerWatcher.h b/libaurum/inc/ScreenAnalyzer/ScreenAnalyzerWatcher.h deleted file mode 100644 index 4941833..0000000 --- a/libaurum/inc/ScreenAnalyzer/ScreenAnalyzerWatcher.h +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#ifndef _SCREEN_ANALYZER_WATCHER_H_ -#define _SCREEN_ANALYZER_WATCHER_H_ - -#include - -namespace Aurum { - -/** - * @class ScreenAnalyzerWatcher - * - * @ingroup aurum - * - * @brief ScreenAnalyzerWatcher communicates with Screen Analyzer Server via MQTT Protocol(mosquitto). - * It requests screen analyzer to server and parses the result that can be used in aurum. - */ -class ScreenAnalyzerWatcher { -public: - - /** - * @brief ScreenAnalyzerWatcher constructor. - * - * @since_tizen 7.0 - */ - ScreenAnalyzerWatcher(); - - /** - * @brief ScreenAnalyzerWatcher destructor. - * - * @since_tizen 7.0 - */ - ~ScreenAnalyzerWatcher(); - - /** - * @brief When message(analyze result) is received from the broker. - * parse the data and create SaObject. - * - * @param[in] mosq mosquitto instance - * @param[in] obj user data provided in mosquitto_new - * @param[in] msg the message data - * - * @since_tizen 7.0 - */ - static void onMessage(struct mosquitto *mosq, void *obj, const struct mosquitto_message *msg); - - /** - * @brief Publish current screen image to the broker. - * capture current screen and publish a message on a given topic. - * - * @since_tizen 7.0 - */ - void PublishData(); - - /** - * @brief Publish current screen image to the broker. - * here capture current screen and publish a message on a given topic. - * - * @since_tizen 7.0 - */ - std::vector> GetSaObjects(); - - /** - * @copydoc PartialMatch::checkCriteria() - */ - bool checkCriteria(const std::string textA, const std::string textB, const bool textPartialMatch); - - /** - * @copydoc PartialMatch::checkCriteria() - */ - bool checkCriteria(const bool boolA, const bool boolB); - - /** - * @copydoc PartialMatch::checkCriteria() - */ - bool checkCriteria(const std::shared_ptr selector, const std::shared_ptr node); - - /** - * @brief find object from screen analyzer result. - * - * @param[in] selector @UiSelector - - * - * @return SaObject vector if found, else nullptr - * - * @since_tizen 7.0 - */ - std::vector> findSaObjects(const std::shared_ptr selector); - - /** - * @brief find object from screen analyzer result. - * - * @param[in] selector @UiSelector - - * - * @return SaObject if found, else nullptr - * - * @since_tizen 7.0 - */ - std::shared_ptr findSaObject(const std::shared_ptr selector); - - /** - * @brief Get focused app name. - * - * @param[in] selector @UiSelector - * - * @return SaObject if found, else nullptr - * - * @since_tizen 7.0 - */ - std::string GetFocusedAppId(); - - /** - * @brief Set server IP address. - * - * @param[in] ip std::string - * - * @since_tizen 7.0 - */ - void SetServerIp(std::string ip); - -private: - struct mosquitto *mMosq; - static std::string mPkgName; - static std::vector> mSaObjects; - static bool mLoadDone; - std::string mIp; -}; - -} - -#endif diff --git a/libaurum/inc/UiDevice.h b/libaurum/inc/UiDevice.h index 27b234b..e9d519c 100644 --- a/libaurum/inc/UiDevice.h +++ b/libaurum/inc/UiDevice.h @@ -35,11 +35,6 @@ #include #include -#ifdef MQTT_ENABLED -#include "SaObject.h" -#include "ScreenAnalyzerWatcher.h" -#endif - namespace Aurum { /** @@ -405,32 +400,6 @@ public: */ std::vector> getWindowRoot() const override; -#ifdef MQTT_ENABLED - /** - * @brief Gets screen analyzer Object vector. - * - * @return SaObject ptr vector - * - * @since_tizen 7.0 - */ - std::vector> getSaObject(); - - /** - * @brief Gets ScreenAnalyzerWatcher instance. - * - * @return ScreenAnalyzerWatcher ptr - * - * @since_tizen 7.0 - */ - std::shared_ptr getSAWatcher(); -#endif - /** - * @brief Requests current screen analyze. - * - * @since_tizen 7.0 - */ - void RequestScreenAnalyze(); - /** * @brief Gets external application launched or not. * @@ -441,24 +410,6 @@ public: bool getExternalAppLaunched(); /** - * @brief Sets whether to use screen analyzer for Tizen apps. - * - * @param[in] withScreenAnalyzer boolean value - * - * @since_tizen 7.0 - */ - void setWithScreenAnalyzer(bool withScreenAnalyzer); - - /** - * @brief Gets whether to use screen analyzer for Tizen apps. - * - * @return true screen analyzer use for Tizen apps, otherwise false - * - * @since_tizen 7.0 - */ - bool getWithScreenAnalyzer(); - - /** * @brief Gets angle of root window. * * @return window angle int @@ -510,10 +461,6 @@ public: private: IDevice *mDeviceImpl; const Waiter *mWaiter; - bool mIsWithSA; -#ifdef MQTT_ENABLED - static std::shared_ptr mSAWatcher; -#endif static std::once_flag mOnceFlag; }; diff --git a/libaurum/meson.build b/libaurum/meson.build index 7e31175..96dfe18 100644 --- a/libaurum/meson.build +++ b/libaurum/meson.build @@ -34,13 +34,6 @@ libaurum_install_inc = [ './inc/Aurum.h', ] -if get_option('mqtt_enabled') == true - libaurum_install_inc += [ - './inc/SaObject.h', - './inc/ScreenAnalyzer/ScreenAnalyzerWatcher.h', - ] -endif - libaurum_inc = [ include_directories('./inc'), include_directories('./inc/Accessibility'), @@ -52,12 +45,6 @@ libaurum_inc = [ root_inc, ] -if get_option('mqtt_enabled') == true - libaurum_inc += [ - include_directories('./inc/ScreenAnalyzer'), - ] -endif - libaurum_src = [] subdir('src') @@ -78,14 +65,6 @@ if get_option('tizen') == true ] endif -if get_option('mqtt_enabled') == true - libaurum_dep += [ - dependency('libmosquitto'), - dependency('jsoncpp'), - dependency('capi-appfw-app-manager'), - ] -endif - libaurum_lib = library('aurum', libaurum_src, dependencies: libaurum_dep, include_directories: libaurum_inc, diff --git a/libaurum/src/PartialMatch.cc b/libaurum/src/PartialMatch.cc index f5b8e92..a18726b 100644 --- a/libaurum/src/PartialMatch.cc +++ b/libaurum/src/PartialMatch.cc @@ -65,34 +65,6 @@ bool PartialMatch::checkCriteria(const std::shared_ptr selector, const std::shared_ptr node, std::shared_ptr device) { -#ifdef MQTT_ENABLED - if (selector->mMatchOcrText) - { - if (!device->getWithScreenAnalyzer() || !node->isShowing() || !(node->getText().size() == 0)) return false; - //FIXME: getSaObjectHasText(); - auto saObjs = device->getSAWatcher()->GetSaObjects(); - - node->updateExtents(); - auto objBoundingBox = node->getScreenBoundingBox(); - for (auto saObj : saObjs) { - if (saObj->getOcrText().size() != 0) { - if (objBoundingBox.isInRect(saObj->getScreenBoundingBox().midPoint()) && - ((objBoundingBox.width() * objBoundingBox.height() * 1.5) > (saObj->getScreenBoundingBox().width() * saObj->getScreenBoundingBox().height())) && - ((objBoundingBox.width() * objBoundingBox.height() < (saObj->getScreenBoundingBox().width() * saObj->getScreenBoundingBox().height() * 1.5)))) - { - LOGI("Text Set %s ", saObj->getOcrText().c_str()); - node->setOcrText(saObj->getOcrText()); - } - } - } - - if (node->getOcrText().size() == 0) return false; - if (checkCriteria(selector->mOcrText, node->getOcrText(), 1)) return false; - - LOGI("node ocr = %s, selector ocr = %s",node->getOcrText().c_str(), selector->mOcrText.c_str()); - } -#endif - if (selector->mMatchText || selector->mMatchTextPartialMatch) { node->updateName(); if (selector->mMatchText && checkCriteria(selector->mText, node->getText(), 0)) return false; diff --git a/libaurum/src/SaObject.cc b/libaurum/src/SaObject.cc deleted file mode 100644 index ade4f32..0000000 --- a/libaurum/src/SaObject.cc +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#include "Aurum.h" - -#include -#include -#include -#include -#include -#include - -#include "ScreenAnalyzerWatcher.h" - -using namespace Aurum; - -SaObject::SaObject(std::string id, std::string type, Rect geometry, std::string ocrText, std::vector states) - : mId(id), mElementType(type), mScreenBoundingBox(geometry), mOcrText(ocrText) -{ - LOGI("SaObject Constructor"); - mIsFocusable = false; - mIsFocused = false; - mIsClickable = false; - mIsActive = false; - for (auto state : states) - { - if (state == "clickable") mIsClickable = true; - else if (state == "focused") mIsFocused = true; - else if (state == "focusable") mIsFocusable = true; - else if (state == "active") mIsActive = true; - } -} - -SaObject::~SaObject() -{ - LOGI("SaObject destructor"); -} - -std::string SaObject::getId() const -{ - return mId; -} - -std::string SaObject::getType() const -{ - return mElementType; -} - -const Rect SaObject::getScreenBoundingBox() const -{ - return mScreenBoundingBox; -} - -std::string SaObject::getOcrText() const -{ - return mOcrText; -} - -bool SaObject::isFocusable() const -{ - return mIsFocusable; -} - -bool SaObject::isFocused() const -{ - return mIsFocused; -} - -bool SaObject::isClickable() const -{ - return mIsClickable; -} - -bool SaObject::isActive() const -{ - return mIsActive; -} - -bool SaObject::isShowing() const -{ - return true; -} \ No newline at end of file diff --git a/libaurum/src/ScreenAnalyzer/ScreenAnalyzerWatcher.cc b/libaurum/src/ScreenAnalyzer/ScreenAnalyzerWatcher.cc deleted file mode 100644 index be0a93c..0000000 --- a/libaurum/src/ScreenAnalyzer/ScreenAnalyzerWatcher.cc +++ /dev/null @@ -1,386 +0,0 @@ -/* - * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#include "Aurum.h" - -#include "ScreenAnalyzerWatcher.h" -#include -#include -#include -#include "SaObject.h" -#include -#include -#include -#include -#include -#include -#include - -using namespace Aurum; - -//Initialize static member variables -std::vector> ScreenAnalyzerWatcher::mSaObjects; -bool ScreenAnalyzerWatcher::mLoadDone; -std::string ScreenAnalyzerWatcher::mPkgName; - -void on_connect(struct mosquitto *mosq, void *obj, int reason_code) -{ - LOGI("mosquitto on connect"); - - if(reason_code != 0){ - LOGE("mosquitto on connect error : %d", reason_code); - mosquitto_disconnect(mosq); - } - int rc; - rc = mosquitto_subscribe(mosq, NULL, "screen_analyzer/json_aurum", 1); - if(rc != MOSQ_ERR_SUCCESS){ - LOGE("server on connect subscribe fail"); - mosquitto_disconnect(mosq); - } - -} - -void on_publish(struct mosquitto *mosq, void *obj, int mid) -{ - LOGI("mosquitto on publish"); -} - -void on_subscribe(struct mosquitto *mosq, void *obj, int mid, int qos_count, const int *granted_qos) -{ - LOGI("server on subscribe qos : %d", qos_count); -} - -ScreenAnalyzerWatcher::ScreenAnalyzerWatcher() -{ - LOGI("Screen Analyzer constructor"); - - mosquitto_lib_init(); - LOGI("mosquitto lib init"); - mMosq = mosquitto_new(NULL, true, NULL); - if(mMosq == NULL){ - LOGE("mosquitto new fail"); - return; - } - - mosquitto_connect_callback_set(mMosq, on_connect); - mosquitto_publish_callback_set(mMosq, on_publish); - mosquitto_subscribe_callback_set(mMosq, on_subscribe); - mosquitto_message_callback_set(mMosq, ScreenAnalyzerWatcher::onMessage); - - //FIXME : When the class is created, there is an issue if We do not try to connect the server, - // so I put the AIM team IP to try to connect. - mIp = "10.113.16.21"; //This is AIM default server user can change this using SetServerIp() - - int rc; - rc = mosquitto_connect(mMosq, mIp.c_str(), 1883, 60); - if(rc != MOSQ_ERR_SUCCESS){ - mosquitto_destroy(mMosq); - LOGE("mosquitto connect fail"); - return; - } - rc = mosquitto_loop_start(mMosq); - if(rc != MOSQ_ERR_SUCCESS){ - mosquitto_destroy(mMosq); - LOGE("mosquitto loop fail"); - return; - } -} - -ScreenAnalyzerWatcher::~ScreenAnalyzerWatcher() -{ - LOGI("Screen Analyzer destructor"); - - if (mMosq) - { - mosquitto_disconnect(mMosq); - mosquitto_loop_stop(mMosq, false); - mosquitto_lib_cleanup(); - mMosq = nullptr; - } -} - -void ScreenAnalyzerWatcher::onMessage(struct mosquitto *mosq, void *obj, const struct mosquitto_message *msg) -{ - std::string txt((char *)msg->payload); - - mSaObjects.clear(); - - Json::Reader reader; - Json::Value root; - bool ret = reader.parse(txt, root); - if (!ret) LOGE("mosquitto parse fail"); - - const Json::Value objs = root["objects"]; - LOGI("objects count = %d", root["objectCount"].asInt()); - - std::string id; - std::string type; - Rect geometry; - std::string ocrText; - std::vector states{}; - - std::shared_ptr mDevice = UiDevice::getInstance(); - //Add dummy for window - id = "9999"; - type = "window"; - geometry = {0, 0, mDevice->getScreenSize().width, mDevice->getScreenSize().height}; - states.push_back("active"); - ocrText = mPkgName; - mSaObjects.push_back(std::make_shared(id, type, geometry, ocrText, states)); - - for (unsigned int idx = 0; idx < objs.size(); ++idx) - { - states.clear(); - id = objs[idx]["id"].asString(); - LOGI("objs[%d] ID : %s", idx,id.c_str()); - - type = objs[idx]["type"].asString(); - LOGI("objs[%d] TYPE : %s", idx, objs[idx]["type"].asString().c_str()); - - const Json::Value geo = objs[idx]["geometry"]; - geometry = { geo[0].asInt(), geo[1].asInt(), geo[2].asInt(), geo[3].asInt() }; - LOGI("objs[%d] GEOMETRY : %d %d %d %d", idx, geo[0].asInt(), geo[1].asInt(), geo[2].asInt(), geo[3].asInt()); - - ocrText = objs[idx]["ocrText"].asString(); - LOGI("objs[%d] OCRTEXT : %s", idx, objs[idx]["ocrText"].asString().c_str()); - - const Json::Value sta = objs[idx]["states"]; - for (unsigned int sidx = 0; sidx < sta.size(); ++sidx) { - LOGI("objs[%d] STATE: %s", idx, sta[sidx].asString().c_str()); - states.push_back(sta[sidx].asString()); - } - - mSaObjects.push_back(std::make_shared(id, type, geometry, ocrText, states)); - } - - mLoadDone = true; -} - -void ScreenAnalyzerWatcher::PublishData() -{ - LOGI("Prepare capture data for data publish"); - mLoadDone = false; - void *ptr = NULL; - unsigned char *src = NULL; - unsigned char *dst = NULL; - - efl_util_screenshot_h screenshot = NULL; - tbm_surface_h tbm_surface = NULL; - tbm_surface_info_s info; - std::shared_ptr mDevice = UiDevice::getInstance(); - const int WIDTH = mDevice->getScreenSize().width; - const int HEIGHT = mDevice->getScreenSize().height; - - screenshot = efl_util_screenshot_initialize(WIDTH, HEIGHT); - - if (screenshot) - { - tbm_surface = efl_util_screenshot_take_tbm_surface(screenshot); - if (tbm_surface) - { - tbm_surface_map(tbm_surface, TBM_SURF_OPTION_READ, &info); - - ptr = malloc( WIDTH * HEIGHT * 4 ); - src = (unsigned char *)info.planes[0].ptr; - dst = (unsigned char *)ptr; - - memcpy(dst, src, WIDTH * HEIGHT * 4); - } - else - { - efl_util_screenshot_deinitialize(screenshot); - return; - } - - int payloadlen = WIDTH * HEIGHT * 4; - - LOGI("mosquitto publish start"); - int rc = mosquitto_publish(mMosq, NULL, "screen_analyzer/image_aurum", payloadlen , ptr, 2, false); - if(rc != MOSQ_ERR_SUCCESS) { - LOGE("mosquitto publish fail"); - } - - tbm_surface_destroy(tbm_surface); - efl_util_screenshot_deinitialize(screenshot); - } - else - { - LOGE("Screen shot fail"); - return; - } - - while(!mLoadDone) { - std::this_thread::sleep_for(std::chrono::milliseconds{10}); - } - - if (tbm_surface) { - free(ptr); - tbm_surface_unmap(tbm_surface); - tbm_surface_destroy(tbm_surface); - } -} - -std::vector> ScreenAnalyzerWatcher::GetSaObjects() -{ - return mSaObjects; -} - -bool ScreenAnalyzerWatcher::checkCriteria(const std::string textA, const std::string textB, const bool textPartialMatch) -{ - if (textB.empty()) return true; - - bool rst; - if (textPartialMatch) { - if (textB.find(textA) != std::string::npos) rst = false; - else rst = true; - } - else { - if (!textA.compare(textB)) rst = false; - else rst = true; - } - - return rst; -} - -bool ScreenAnalyzerWatcher::checkCriteria(const bool boolA, const bool boolB) -{ - return boolA != boolB; -} - -bool ScreenAnalyzerWatcher::checkCriteria(const std::shared_ptr selector, - const std::shared_ptr node) -{ - if (selector->mMatchText || selector->mMatchTextPartialMatch) { - if (selector->mMatchText && checkCriteria(selector->mText, node->getOcrText(), 0)) return false; - if (selector->mMatchTextPartialMatch && checkCriteria(selector->mTextPartialMatch, node->getOcrText(), 1)) return false; - } - if (selector->mMatchId) { - if (checkCriteria(selector->mId, node->getId(), 0)) return false; - } - if (selector->mMatchType) { - if (selector->mMatchType && checkCriteria(selector->mType, node->getType(), 0)) return false; - } - if (selector->mMatchClickable && checkCriteria(selector->mIsclickable, node->isClickable())) return false; - if (selector->mMatchFocused && checkCriteria(selector->mIsfocused, node->isFocused())) return false; - if (selector->mMatchFocusable && checkCriteria(selector->mIsfocusable, node->isFocusable())) return false; - if (selector->mMatchActive && checkCriteria(selector->mIsactive, node->isActive())) return false; - if (selector->mMatchShowing && checkCriteria(selector->mIsshowing, node->isShowing())) return false; - - return true; -} - -std::vector> ScreenAnalyzerWatcher::findSaObjects(const std::shared_ptr selector) -{ - std::vector> rets{}; - - LOGI("FindSaObject start"); - for (auto saObj : mSaObjects) { - if (checkCriteria(selector, saObj)) { - LOGI("saObj(%s) pushed", saObj->getId().c_str()); - rets.push_back(saObj); - } - } - - return rets; -} - -std::shared_ptr ScreenAnalyzerWatcher::findSaObject(const std::shared_ptr selector) -{ - LOGI("FindSaObject start"); - for (auto saObj : mSaObjects) { - if (checkCriteria(selector, saObj)) { - LOGI("saObj(%s) pushed", saObj->getId().c_str()); - return saObj; - } - } - - return nullptr; -} - -std::string ScreenAnalyzerWatcher::GetFocusedAppId() -{ - int return_val = APP_MANAGER_ERROR_NONE; - - app_context_h app_context; - char *app_id = NULL; - - return_val = app_manager_get_focused_app_context(&app_context); - if (return_val != APP_MANAGER_ERROR_NONE) { - LOGE("failed to get app-context"); - return NULL; - } - - return_val = app_context_get_app_id(app_context, &app_id); - if (return_val != APP_MANAGER_ERROR_NONE) { - LOGE("failed to get app_id"); - app_context_destroy(app_context); - return NULL; - } - - LOGI("focused app_id = %s", app_id); - - return_val = app_context_destroy(app_context); - if (return_val != APP_MANAGER_ERROR_NONE) { - LOGE("failed to destroy app-context"); - return NULL; - } - - app_context = NULL; - - return app_id; -} - -void ScreenAnalyzerWatcher::SetServerIp(std::string ip) -{ - LOGI("SA Server Initialize"); - mIp = ip; - - if (mMosq) - { - mosquitto_disconnect(mMosq); - mosquitto_loop_stop(mMosq, false); - mosquitto_lib_cleanup(); - mMosq = nullptr; - } - - mosquitto_lib_init(); - LOGI("mosquitto lib init"); - mMosq = mosquitto_new(NULL, true, NULL); - if(mMosq == NULL){ - LOGE("mosquitto new fail"); - return; - } - - mosquitto_connect_callback_set(mMosq, on_connect); - mosquitto_publish_callback_set(mMosq, on_publish); - mosquitto_subscribe_callback_set(mMosq, on_subscribe); - mosquitto_message_callback_set(mMosq, ScreenAnalyzerWatcher::onMessage); - - int rc; - rc = mosquitto_connect(mMosq, mIp.c_str(), 1883, 60); - if (rc != MOSQ_ERR_SUCCESS){ - mosquitto_destroy(mMosq); - LOGE("mosquitto connect fail"); - return; - } - rc = mosquitto_loop_start(mMosq); - if (rc != MOSQ_ERR_SUCCESS){ - mosquitto_destroy(mMosq); - LOGE("mosquitto loop fail"); - return; - } -} diff --git a/libaurum/src/ScreenAnalyzer/meson.build b/libaurum/src/ScreenAnalyzer/meson.build deleted file mode 100644 index 5047b43..0000000 --- a/libaurum/src/ScreenAnalyzer/meson.build +++ /dev/null @@ -1,3 +0,0 @@ -libaurum_src += [ - files('ScreenAnalyzerWatcher.cc') -] diff --git a/libaurum/src/UiDevice.cc b/libaurum/src/UiDevice.cc index 6286d56..60f9ab2 100644 --- a/libaurum/src/UiDevice.cc +++ b/libaurum/src/UiDevice.cc @@ -37,21 +37,12 @@ using namespace AurumInternal; std::once_flag UiDevice::mOnceFlag; -#ifdef MQTT_ENABLED -std::shared_ptr UiDevice::mSAWatcher; -#endif - UiDevice::UiDevice() : UiDevice(nullptr) {} UiDevice::UiDevice(IDevice *impl) : mDeviceImpl(impl), mWaiter(new Waiter{this}) { LOGI("UiDevice constructor"); - mIsWithSA = false; -#ifdef MQTT_ENABLED - mSAWatcher = std::make_shared(); -#endif - LOGI("UiDevice constructor finish"); } UiDevice::~UiDevice() @@ -294,41 +285,12 @@ const Size2D UiDevice::getScreenSize() return mDeviceImpl->getScreenSize(); } -#ifdef MQTT_ENABLED -std::vector> UiDevice::getSaObject() -{ - return mSAWatcher->GetSaObjects(); -} - -std::shared_ptr UiDevice::getSAWatcher() -{ - return mSAWatcher; -} -#endif - -void UiDevice::RequestScreenAnalyze() -{ -#ifdef MQTT_ENABLED - mSAWatcher->PublishData(); -#endif -} - bool UiDevice::getExternalAppLaunched() { auto ret = this->getWindowRoot(); return (ret.size() > 0) ? false : true; } -void UiDevice::setWithScreenAnalyzer(bool withScreenAnalyzer) -{ - mIsWithSA = withScreenAnalyzer; -} - -bool UiDevice::getWithScreenAnalyzer() -{ - return mIsWithSA; -} - bool UiDevice::registerCallback(const A11yEvent type, EventHandler cb, void *data) const { return AccessibleWatcher::getInstance()->registerCallback(type, cb, data); diff --git a/libaurum/src/meson.build b/libaurum/src/meson.build index caae22d..17595be 100644 --- a/libaurum/src/meson.build +++ b/libaurum/src/meson.build @@ -13,15 +13,6 @@ libaurum_src += [ files('AurumXML.cc'), ] -if get_option('mqtt_enabled') == true - libaurum_src += [ - files('SaObject.cc'), - ] -endif - subdir('Accessibility') -if get_option('mqtt_enabled') == true - subdir('ScreenAnalyzer') -endif subdir('Impl') subdir('Runnable') diff --git a/meson.build b/meson.build index e6af72e..1022f82 100644 --- a/meson.build +++ b/meson.build @@ -17,10 +17,6 @@ if get_option('tizen_gcov') == true config_h.set10('TIZEN_GCOV', true) endif -if get_option('mqtt_enabled') == true - config_h.set10('MQTT_ENABLED', true) -endif - root_inc = include_directories('./') subdir('pugixml') diff --git a/meson_options.txt b/meson_options.txt index ec44da5..726ed1d 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -26,9 +26,3 @@ option('enable_documentation', value: false, description: 'enable generating documentation by using doxygen' ) - -option('mqtt_enabled', - type: 'boolean', - value: false, - description: 'enable mqtt protocol(mosquitto) for Screen Analyzer' -) diff --git a/org.tizen.aurum-bootstrap/inc/AurumServiceImpl.h b/org.tizen.aurum-bootstrap/inc/AurumServiceImpl.h index 458f4b7..dc7a6ce 100644 --- a/org.tizen.aurum-bootstrap/inc/AurumServiceImpl.h +++ b/org.tizen.aurum-bootstrap/inc/AurumServiceImpl.h @@ -120,9 +120,6 @@ public: ::grpc::Status getActiveAppToolkitName(::grpc::ServerContext *context, const ::aurum::ReqGetActiveAppToolkitName *request, ::aurum::RspGetActiveAppToolkitName *response) override; - ::grpc::Status enableScreenAnalyzer(::grpc::ServerContext *context, - const ::aurum::ReqEnableScreenAnalyzer *request, - ::aurum::RspEnableScreenAnalyzer *response) override; ::grpc::Status getTextMinBoundingRect(::grpc::ServerContext *context, const ::aurum::ReqGetTextMinBoundingRect *request, ::aurum::RspGetTextMinBoundingRect *response) override; diff --git a/org.tizen.aurum-bootstrap/inc/Commands/Commands.h b/org.tizen.aurum-bootstrap/inc/Commands/Commands.h index 9ecdb4c..a98c138 100644 --- a/org.tizen.aurum-bootstrap/inc/Commands/Commands.h +++ b/org.tizen.aurum-bootstrap/inc/Commands/Commands.h @@ -43,7 +43,6 @@ #include "Commands/ActionAndWaitEventCommand.h" #include "Commands/SetFocusCommand.h" #include "Commands/GetActiveAppToolkitNameCommand.h" -#include "Commands/EnableScreenAnalyzerCommand.h" #include "Commands/GetTextMinBoundingRectCommand.h" #include "Commands/SetXMLSyncCommand.h" #include "Commands/GetAngleCommand.h" diff --git a/org.tizen.aurum-bootstrap/inc/Commands/EnableScreenAnalyzerCommand.h b/org.tizen.aurum-bootstrap/inc/Commands/EnableScreenAnalyzerCommand.h deleted file mode 100644 index 105d8b9..0000000 --- a/org.tizen.aurum-bootstrap/inc/Commands/EnableScreenAnalyzerCommand.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#include "bootstrap.h" - -class EnableScreenAnalyzerCommand : public Command { -private: - const ::aurum::ReqEnableScreenAnalyzer *mRequest; - ::aurum::RspEnableScreenAnalyzer *mResponse; - -public: - EnableScreenAnalyzerCommand(const ::aurum::ReqEnableScreenAnalyzer *request, - ::aurum::RspEnableScreenAnalyzer*response); - ::grpc::Status execute() override; -}; diff --git a/org.tizen.aurum-bootstrap/meson.build b/org.tizen.aurum-bootstrap/meson.build index 5eed3ed..eb3876b 100644 --- a/org.tizen.aurum-bootstrap/meson.build +++ b/org.tizen.aurum-bootstrap/meson.build @@ -46,7 +46,6 @@ bootstrap_svr_src += [ files('src/Commands/ActionAndWaitEventCommand.cc'), files('src/Commands/SetFocusCommand.cc'), files('src/Commands/GetActiveAppToolkitNameCommand.cc'), - files('src/Commands/EnableScreenAnalyzerCommand.cc'), files('src/Commands/GetTextMinBoundingRectCommand.cc'), files('src/Commands/SetXMLSyncCommand.cc'), files('src/Commands/GetAngleCommand.cc'), diff --git a/org.tizen.aurum-bootstrap/src/AurumServiceImpl.cc b/org.tizen.aurum-bootstrap/src/AurumServiceImpl.cc index 3c9978b..ad9aa13 100644 --- a/org.tizen.aurum-bootstrap/src/AurumServiceImpl.cc +++ b/org.tizen.aurum-bootstrap/src/AurumServiceImpl.cc @@ -270,14 +270,6 @@ aurumServiceImpl::~aurumServiceImpl() return execute(cmd.get(), true); } -::grpc::Status aurumServiceImpl::enableScreenAnalyzer(::grpc::ServerContext *context, - const ::aurum::ReqEnableScreenAnalyzer *request, - ::aurum::RspEnableScreenAnalyzer *response) -{ - std::unique_ptr cmd = std::make_unique(request, response); - return execute(cmd.get(), true); -} - ::grpc::Status aurumServiceImpl::getTextMinBoundingRect(::grpc::ServerContext *context, const ::aurum::ReqGetTextMinBoundingRect *request, ::aurum::RspGetTextMinBoundingRect *response) diff --git a/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc index 4ed405d..6d9b4ab 100644 --- a/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc +++ b/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc @@ -97,68 +97,15 @@ void DumpObjectTreeCommand::traverse(::aurum::Element *root, std::shared_ptr mDevice = UiDevice::getInstance(); -#ifdef MQTT_ENABLED - if (mDevice->getExternalAppLaunched()) - { - mDevice->RequestScreenAnalyze(); - - auto objs = mDevice->getSAWatcher()->GetSaObjects(); - ::aurum::Element *root; - int idx = 0; - for (auto obj : objs) { - if (!idx) { - root = mResponse->add_roots(); - root->set_elementid(obj->getId()); - root->set_widgettype(obj->getType()); - root->set_text(obj->getOcrText()); - root->set_isclickable(obj->isClickable()); - root->set_isfocused(obj->isFocused()); - root->set_isfocusable(obj->isFocusable()); - root->set_isactive(obj->isActive()); - root->set_isshowing(true); - root->set_isvisible(true); - ::aurum::Rect *rect = root->mutable_geometry(); - const Rect &size = obj->getScreenBoundingBox(); - rect->set_x(size.mTopLeft.x); - rect->set_y(size.mTopLeft.y); - rect->set_width(size.width()); - rect->set_height(size.height()); - } - else { - ::aurum::Element *elm = root->add_child(); - elm->set_elementid(obj->getId()); - elm->set_widgettype(obj->getType()); - elm->set_text(obj->getOcrText()); - elm->set_isclickable(obj->isClickable()); - elm->set_isfocused(obj->isFocused()); - elm->set_isfocusable(obj->isFocusable()); - elm->set_isactive(obj->isActive()); - elm->set_isshowing(true); - elm->set_isvisible(true); - - ::aurum::Rect *rect = elm->mutable_geometry(); - const Rect &size = obj->getScreenBoundingBox(); - rect->set_x(size.mTopLeft.x); - rect->set_y(size.mTopLeft.y); - rect->set_width(size.width()); - rect->set_height(size.height()); - } - - idx++; - } - } - else -#endif - { - LOGI("elementid : %s", mRequest->elementid().c_str()); - if (mRequest->elementid().length()) { - auto obj = mObjMap->getElement(mRequest->elementid()); - if (!obj) return grpc::Status::OK;; - - auto node = obj->getDescendant(); - ::aurum::Element *root = mResponse->add_roots(); - traverse(root, node, 0); - } + + LOGI("elementid : %s", mRequest->elementid().c_str()); + if (mRequest->elementid().length()) { + auto obj = mObjMap->getElement(mRequest->elementid()); + if (!obj) return grpc::Status::OK; + + auto node = obj->getDescendant(); + ::aurum::Element *root = mResponse->add_roots(); + traverse(root, node, 0); } return grpc::Status::OK; diff --git a/org.tizen.aurum-bootstrap/src/Commands/EnableScreenAnalyzerCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/EnableScreenAnalyzerCommand.cc deleted file mode 100644 index 04c8425..0000000 --- a/org.tizen.aurum-bootstrap/src/Commands/EnableScreenAnalyzerCommand.cc +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#include "EnableScreenAnalyzerCommand.h" -#include "UiDevice.h" -#include "config.h" - -EnableScreenAnalyzerCommand::EnableScreenAnalyzerCommand(const ::aurum::ReqEnableScreenAnalyzer *request, - ::aurum::RspEnableScreenAnalyzer *response) - : mRequest{request}, mResponse{response} -{ -} - -::grpc::Status EnableScreenAnalyzerCommand::execute() -{ - LOGI("EnableScreenAnalyzer --------------- "); - - LOGI("Screen Analyzer enabled (%d) Server IP : (%s)", mRequest->enable(), mRequest->serverip().c_str()); - std::shared_ptr mDevice = UiDevice::getInstance(); - mDevice->setWithScreenAnalyzer(mRequest->enable()); -#ifdef MQTT_ENABLED - if (mRequest->serverip().size() > 0) - mDevice->getSAWatcher()->SetServerIp(mRequest->serverip()); -#endif - - return grpc::Status::OK; -} diff --git a/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc index 9567d93..d147096 100644 --- a/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc +++ b/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc @@ -22,9 +22,6 @@ #include "Sel.h" #include "ISearchable.h" #include "config.h" -#ifdef MQTT_ENABLED -#include "SaObject.h" -#endif #include "Until.h" FindElementCommand::FindElementCommand(const ::aurum::ReqFindElement *request, @@ -87,118 +84,72 @@ std::shared_ptr FindElementCommand::getSelector(void) auto selector = getSelector(); std::shared_ptr mDevice = UiDevice::getInstance(); -#ifdef MQTT_ENABLED - if (mDevice->getExternalAppLaunched()) - { - mDevice->RequestScreenAnalyze(); - - LOGI("Search Object start"); - - auto found = mDevice->getSAWatcher()->findSaObject(selector); - if (found != nullptr) { - SaObject *obj = found.get(); - ::aurum::Element *elm = mResponse->mutable_element(); - elm->set_elementid(obj->getId()); - - ::aurum::Rect *rect = elm->mutable_geometry(); - const Rect &size = obj->getScreenBoundingBox(); - rect->set_x(size.mTopLeft.x); - rect->set_y(size.mTopLeft.y); - rect->set_width(size.width()); - rect->set_height(size.height()); - - elm->set_widgettype(obj->getType()); - - elm->set_text(obj->getOcrText()); - - elm->set_isclickable(obj->isClickable()); - elm->set_isfocused(obj->isFocused()); - elm->set_isfocusable(obj->isFocusable()); - elm->set_isactive(obj->isActive()); - elm->set_isshowing(true); - elm->set_isvisible(true); - - mResponse->set_status(::aurum::RspStatus::OK); - } - else - mResponse->set_status(::aurum::RspStatus::ERROR); - } - else -#endif - { - if (mDevice->getWithScreenAnalyzer()) - { - LOGI("Find object with SA information"); - mDevice->RequestScreenAnalyze(); - } - - auto waiter = new Waiter(searchableObj.get(), nullptr, mTimeout); - auto found = waiter->waitFor(Until::findObject(selector)); - - if (waiter) delete waiter; - - if (found != nullptr) { - UiObject *obj = found.get(); - obj->refresh(); - if (mObjMap->getElement(obj->getId()) == nullptr) - mObjMap->addElement(std::move(found)); - - LOGI("found object : %p elementId:%s", obj, obj->getId().c_str()); - - ::aurum::Element *elm = mResponse->mutable_element(); - elm->set_elementid(obj->getId()); - elm->set_package(obj->getApplicationPackage()); - - ::aurum::Rect *rect = elm->mutable_geometry(); - const Rect &size = obj->getScreenBoundingBox(); - rect->set_x(size.mTopLeft.x); - rect->set_y(size.mTopLeft.y); - rect->set_width(size.width()); - rect->set_height(size.height()); - - ::aurum::Rect *windowRect = elm->mutable_windowrelativegeometry(); - const Rect &windowRelativeSize = obj->getWindowBoundingBox(); - windowRect->set_x(windowRelativeSize.mTopLeft.x); - windowRect->set_y(windowRelativeSize.mTopLeft.y); - windowRect->set_width(windowRelativeSize.width()); - windowRect->set_height(windowRelativeSize.height()); - - elm->set_widgettype(obj->getType()); - elm->set_widgetstyle(obj->getElementStyle()); - - elm->set_text(obj->getText()); - elm->set_xpath(obj->getXPath()); - elm->set_ocrtext(obj->getOcrText()); - elm->set_automationid(obj->getAutomationId()); - elm->set_package(obj->getApplicationPackage()); - elm->set_role(obj->getRole()); - - elm->set_ischecked(obj->isChecked()); - elm->set_ischeckable(obj->isCheckable()); - elm->set_isclickable(obj->isClickable()); - elm->set_isenabled(obj->isEnabled()); - elm->set_isfocused(obj->isFocused()); - elm->set_isfocusable(obj->isFocusable()); - elm->set_isscrollable(obj->isScrollable()); - elm->set_isselected(obj->isSelected()); - elm->set_isshowing(obj->isShowing()); - elm->set_isactive(obj->isActive()); - elm->set_isvisible(obj->isVisible()); - elm->set_isselectable(obj->isSelectable()); - elm->set_ishighlightable(obj->isHighlightable()); - - elm->set_minvalue(obj->getMinValue()); - elm->set_maxvalue(obj->getMaxValue()); - elm->set_value(obj->getValue()); - elm->set_increment(obj->getIncrement()); - - elm->set_windowangle(obj->getWindowAngle()); - elm->set_targetangle(obj->getTargetAngle()); - - mResponse->set_status(::aurum::RspStatus::OK); - } else { - mResponse->set_status(::aurum::RspStatus::ERROR); - } + auto waiter = new Waiter(searchableObj.get(), nullptr, mTimeout); + auto found = waiter->waitFor(Until::findObject(selector)); + + if (waiter) delete waiter; + + if (found != nullptr) { + UiObject *obj = found.get(); + obj->refresh(); + if (mObjMap->getElement(obj->getId()) == nullptr) + mObjMap->addElement(std::move(found)); + + LOGI("found object : %p elementId:%s", obj, obj->getId().c_str()); + + ::aurum::Element *elm = mResponse->mutable_element(); + elm->set_elementid(obj->getId()); + elm->set_package(obj->getApplicationPackage()); + + ::aurum::Rect *rect = elm->mutable_geometry(); + const Rect &size = obj->getScreenBoundingBox(); + rect->set_x(size.mTopLeft.x); + rect->set_y(size.mTopLeft.y); + rect->set_width(size.width()); + rect->set_height(size.height()); + + ::aurum::Rect *windowRect = elm->mutable_windowrelativegeometry(); + const Rect &windowRelativeSize = obj->getWindowBoundingBox(); + windowRect->set_x(windowRelativeSize.mTopLeft.x); + windowRect->set_y(windowRelativeSize.mTopLeft.y); + windowRect->set_width(windowRelativeSize.width()); + windowRect->set_height(windowRelativeSize.height()); + + elm->set_widgettype(obj->getType()); + elm->set_widgetstyle(obj->getElementStyle()); + + elm->set_text(obj->getText()); + elm->set_xpath(obj->getXPath()); + elm->set_ocrtext(obj->getOcrText()); + elm->set_automationid(obj->getAutomationId()); + elm->set_package(obj->getApplicationPackage()); + elm->set_role(obj->getRole()); + + elm->set_ischecked(obj->isChecked()); + elm->set_ischeckable(obj->isCheckable()); + elm->set_isclickable(obj->isClickable()); + elm->set_isenabled(obj->isEnabled()); + elm->set_isfocused(obj->isFocused()); + elm->set_isfocusable(obj->isFocusable()); + elm->set_isscrollable(obj->isScrollable()); + elm->set_isselected(obj->isSelected()); + elm->set_isshowing(obj->isShowing()); + elm->set_isactive(obj->isActive()); + elm->set_isvisible(obj->isVisible()); + elm->set_isselectable(obj->isSelectable()); + elm->set_ishighlightable(obj->isHighlightable()); + + elm->set_minvalue(obj->getMinValue()); + elm->set_maxvalue(obj->getMaxValue()); + elm->set_value(obj->getValue()); + elm->set_increment(obj->getIncrement()); + + elm->set_windowangle(obj->getWindowAngle()); + elm->set_targetangle(obj->getTargetAngle()); + + mResponse->set_status(::aurum::RspStatus::OK); + } else { + mResponse->set_status(::aurum::RspStatus::ERROR); } return grpc::Status::OK; diff --git a/org.tizen.aurum-bootstrap/src/Commands/FindElementsCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/FindElementsCommand.cc index 711be13..cd113f1 100644 --- a/org.tizen.aurum-bootstrap/src/Commands/FindElementsCommand.cc +++ b/org.tizen.aurum-bootstrap/src/Commands/FindElementsCommand.cc @@ -22,9 +22,6 @@ #include "Sel.h" #include "ISearchable.h" #include "config.h" -#ifdef MQTT_ENABLED -#include "SaObject.h" -#endif #include "Until.h" FindElementsCommand::FindElementsCommand(const ::aurum::ReqFindElements *request, @@ -87,131 +84,79 @@ std::vector> FindElementsCommand::getSelectors(void) auto selectors = getSelectors(); std::shared_ptr mDevice = UiDevice::getInstance(); -#ifdef MQTT_ENABLED - if (mDevice->getExternalAppLaunched()) - { - mDevice->RequestScreenAnalyze(); + std::vector> founds = {}; - std::vector> founds = {}; + auto waiter = new Waiter(searchableObj.get(), nullptr, mTimeout); - for ( auto &sel : selectors ) { - auto ret = mDevice->getSAWatcher()->findSaObjects(sel); - std::move(std::begin(ret), std::end(ret), std::back_inserter(founds)); - } - if (founds.size() > 0) { - for (auto& found : founds) { - SaObject *obj = found.get(); - ::aurum::Element *elm = mResponse->add_elements(); - elm->set_elementid(obj->getId()); - - ::aurum::Rect *rect = elm->mutable_geometry(); - const Rect &size = obj->getScreenBoundingBox(); - rect->set_x(size.mTopLeft.x); - rect->set_y(size.mTopLeft.y); - rect->set_width(size.width()); - rect->set_height(size.height()); - - elm->set_widgettype(obj->getType()); - - elm->set_text(obj->getOcrText()); - elm->set_toolkit("External"); - - elm->set_isclickable(obj->isClickable()); - elm->set_isfocused(obj->isFocused()); - elm->set_isfocusable(obj->isFocusable()); - elm->set_isactive(obj->isActive()); - elm->set_isshowing(true); - elm->set_isvisible(true); - } - mResponse->set_status(::aurum::RspStatus::OK); - } - else { - mResponse->set_status(::aurum::RspStatus::ERROR); - } + for (auto &sel : selectors) { + auto ret = waiter->waitFor(Until::findObjects(sel)); + std::move(std::begin(ret), std::end(ret), std::back_inserter(founds)); } - else -#endif - { - if (mDevice->getWithScreenAnalyzer()) - { - LOGI("Find object with SA information"); - mDevice->RequestScreenAnalyze(); - } - - std::vector> founds = {}; - - auto waiter = new Waiter(searchableObj.get(), nullptr, mTimeout); - - for ( auto &sel : selectors ) { - auto ret = waiter->waitFor(Until::findObjects(sel)); - std::move(std::begin(ret), std::end(ret), std::back_inserter(founds)); - } - if (waiter) delete waiter; - - if (founds.size() > 0) { - for (auto& found : founds) { - UiObject *obj = found.get(); - obj->refresh(); - if (mObjMap->getElement(obj->getId()) == nullptr) - mObjMap->addElement(std::move(found)); - - LOGI("found object : %p elementId:%s", obj, obj->getId().c_str()); - - ::aurum::Element *elm = mResponse->add_elements(); - elm->set_elementid(obj->getId()); - elm->set_package(obj->getApplicationPackage()); - - ::aurum::Rect *rect = elm->mutable_geometry(); - const Rect &size = obj->getScreenBoundingBox(); - rect->set_x(size.mTopLeft.x); - rect->set_y(size.mTopLeft.y); - rect->set_width(size.width()); - rect->set_height(size.height()); - - ::aurum::Rect *windowRect = elm->mutable_windowrelativegeometry(); - const Rect &windowRelativeSize = obj->getWindowBoundingBox(); - windowRect->set_x(windowRelativeSize.mTopLeft.x); - windowRect->set_y(windowRelativeSize.mTopLeft.y); - windowRect->set_width(windowRelativeSize.width()); - windowRect->set_height(windowRelativeSize.height()); - - elm->set_widgettype(obj->getType()); - elm->set_widgetstyle(obj->getElementStyle()); - - elm->set_text(obj->getText()); - elm->set_xpath(obj->getXPath()); - elm->set_ocrtext(obj->getOcrText()); - elm->set_automationid(obj->getAutomationId()); - elm->set_package(obj->getApplicationPackage()); - elm->set_role(obj->getRole()); - - elm->set_ischecked(obj->isChecked()); - elm->set_ischeckable(obj->isCheckable()); - elm->set_isclickable(obj->isClickable()); - elm->set_isenabled(obj->isEnabled()); - elm->set_isfocused(obj->isFocused()); - elm->set_isfocusable(obj->isFocusable()); - elm->set_isscrollable(obj->isScrollable()); - elm->set_isselected(obj->isSelected()); - elm->set_isshowing(obj->isShowing()); - elm->set_isactive(obj->isActive()); - elm->set_isvisible(obj->isVisible()); - elm->set_isselectable(obj->isSelectable()); - elm->set_ishighlightable(obj->isHighlightable()); - - elm->set_minvalue(obj->getMinValue()); - elm->set_maxvalue(obj->getMaxValue()); - elm->set_value(obj->getValue()); - elm->set_increment(obj->getIncrement()); - - elm->set_windowangle(obj->getWindowAngle()); - elm->set_targetangle(obj->getTargetAngle()); - } - mResponse->set_status(::aurum::RspStatus::OK); - } else { - mResponse->set_status(::aurum::RspStatus::ERROR); + if (waiter) delete waiter; + + if (founds.size() > 0) { + for (auto &found : founds) { + UiObject *obj = found.get(); + obj->refresh(); + if (mObjMap->getElement(obj->getId()) == nullptr) + mObjMap->addElement(std::move(found)); + + LOGI("found object : %p elementId:%s", obj, obj->getId().c_str()); + + ::aurum::Element *elm = mResponse->add_elements(); + elm->set_elementid(obj->getId()); + elm->set_package(obj->getApplicationPackage()); + + ::aurum::Rect *rect = elm->mutable_geometry(); + const Rect &size = obj->getScreenBoundingBox(); + rect->set_x(size.mTopLeft.x); + rect->set_y(size.mTopLeft.y); + rect->set_width(size.width()); + rect->set_height(size.height()); + + ::aurum::Rect *windowRect = elm->mutable_windowrelativegeometry(); + const Rect &windowRelativeSize = obj->getWindowBoundingBox(); + windowRect->set_x(windowRelativeSize.mTopLeft.x); + windowRect->set_y(windowRelativeSize.mTopLeft.y); + windowRect->set_width(windowRelativeSize.width()); + windowRect->set_height(windowRelativeSize.height()); + + elm->set_widgettype(obj->getType()); + elm->set_widgetstyle(obj->getElementStyle()); + + elm->set_text(obj->getText()); + elm->set_xpath(obj->getXPath()); + elm->set_ocrtext(obj->getOcrText()); + elm->set_automationid(obj->getAutomationId()); + elm->set_package(obj->getApplicationPackage()); + elm->set_role(obj->getRole()); + + elm->set_ischecked(obj->isChecked()); + elm->set_ischeckable(obj->isCheckable()); + elm->set_isclickable(obj->isClickable()); + elm->set_isenabled(obj->isEnabled()); + elm->set_isfocused(obj->isFocused()); + elm->set_isfocusable(obj->isFocusable()); + elm->set_isscrollable(obj->isScrollable()); + elm->set_isselected(obj->isSelected()); + elm->set_isshowing(obj->isShowing()); + elm->set_isactive(obj->isActive()); + elm->set_isvisible(obj->isVisible()); + elm->set_isselectable(obj->isSelectable()); + elm->set_ishighlightable(obj->isHighlightable()); + + elm->set_minvalue(obj->getMinValue()); + elm->set_maxvalue(obj->getMaxValue()); + elm->set_value(obj->getValue()); + elm->set_increment(obj->getIncrement()); + + elm->set_windowangle(obj->getWindowAngle()); + elm->set_targetangle(obj->getTargetAngle()); } + mResponse->set_status(::aurum::RspStatus::OK); + } else { + mResponse->set_status(::aurum::RspStatus::ERROR); } return grpc::Status::OK; diff --git a/org.tizen.aurum-bootstrap/src/Commands/GetSizeCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/GetSizeCommand.cc index 19033f1..62850f2 100644 --- a/org.tizen.aurum-bootstrap/src/Commands/GetSizeCommand.cc +++ b/org.tizen.aurum-bootstrap/src/Commands/GetSizeCommand.cc @@ -22,10 +22,6 @@ #include "Sel.h" #include "ISearchable.h" #include "config.h" -#ifdef MQTT_ENABLED -#include "SaObject.h" -#endif - GetSizeCommand::GetSizeCommand(const ::aurum::ReqGetSize *request, ::aurum::RspGetSize *response) @@ -39,56 +35,25 @@ GetSizeCommand::GetSizeCommand(const ::aurum::ReqGetSize *request, std::shared_ptr mDevice = UiDevice::getInstance(); -#ifdef MQTT_ENABLED - if (mDevice->getExternalAppLaunched()) - { - mDevice->RequestScreenAnalyze(); - - std::vector> founds = {}; - - auto tempSel = std::make_shared(); - tempSel->id(mRequest->elementid()); - auto selectors = std::vector>{tempSel}; - - for ( auto &sel : selectors ) { - auto ret = mDevice->getSAWatcher()->findSaObjects(sel); - std::move(std::begin(ret), std::end(ret), std::back_inserter(founds)); - } - if (founds.size() > 0) { - for (auto& found : founds) { - SaObject *obj = found.get(); - ::aurum::Rect *rect = mResponse->mutable_size(); - const Rect &size = obj->getScreenBoundingBox(); - rect->set_x(size.mTopLeft.x); - rect->set_y(size.mTopLeft.y); - rect->set_width(size.width()); - rect->set_height(size.height()); - } - } - } - else -#endif - { - ::aurum::ReqGetSize_CoordType type = mRequest->type(); - ObjectMapper *mObjMap = ObjectMapper::getInstance(); - std::shared_ptr obj = mObjMap->getElement(mRequest->elementid()); - if (obj) { - obj->updateExtents(); - ::aurum::Rect *rect = mResponse->mutable_size(); - if (type == ::aurum::ReqGetSize_CoordType::ReqGetSize_CoordType_SCREEN) { - const Rect &size = obj->getScreenBoundingBox(); - rect->set_x(size.mTopLeft.x); - rect->set_y(size.mTopLeft.y); - rect->set_width(size.width()); - rect->set_height(size.height()); - } - else { - const Rect &windowRelativeSize = obj->getWindowBoundingBox(); - rect->set_x(windowRelativeSize.mTopLeft.x); - rect->set_y(windowRelativeSize.mTopLeft.y); - rect->set_width(windowRelativeSize.width()); - rect->set_height(windowRelativeSize.height()); - } + ::aurum::ReqGetSize_CoordType type = mRequest->type(); + ObjectMapper *mObjMap = ObjectMapper::getInstance(); + std::shared_ptr obj = mObjMap->getElement(mRequest->elementid()); + if (obj) { + obj->updateExtents(); + ::aurum::Rect *rect = mResponse->mutable_size(); + if (type == + ::aurum::ReqGetSize_CoordType::ReqGetSize_CoordType_SCREEN) { + const Rect &size = obj->getScreenBoundingBox(); + rect->set_x(size.mTopLeft.x); + rect->set_y(size.mTopLeft.y); + rect->set_width(size.width()); + rect->set_height(size.height()); + } else { + const Rect &windowRelativeSize = obj->getWindowBoundingBox(); + rect->set_x(windowRelativeSize.mTopLeft.x); + rect->set_y(windowRelativeSize.mTopLeft.y); + rect->set_width(windowRelativeSize.width()); + rect->set_height(windowRelativeSize.height()); } } diff --git a/packaging/aurum.spec b/packaging/aurum.spec index 1a54fae..90efb54 100644 --- a/packaging/aurum.spec +++ b/packaging/aurum.spec @@ -31,11 +31,6 @@ BuildRequires: pkgconfig(capi-system-system-settings) BuildRequires: pkgconfig(capi-base-utils-i18n) BuildRequires: pkgconfig(vconf) -%if "%{mqtt}" == "1" -BuildRequires: pkgconfig(libmosquitto) -BuildRequires: pkgconfig(jsoncpp) -%endif - %if 0%{?gendoc:1} BuildRequires: doxygen %endif @@ -91,19 +86,12 @@ export LDFLAGS+="-Wl,-z,noexecstack" %define TIZEN_GEN_DOC false %endif -%if "%{mqtt}" == "1" -%define MQTT_ENABLED true -%else -%define MQTT_ENABLED false -%endif - meson \ --prefix /usr \ --libdir %{_libdir} \ -Dcpp_std=c++17 \ -Dtizen=true \ -Denable_documentation=%{TIZEN_GEN_DOC} \ - -Dmqtt_enabled=%{MQTT_ENABLED} \ -Dtzapp_path=%{TZ_SYS_RO_APP} \ -Dtzpackage_path=%{TZ_SYS_RO_PACKAGES} \ gbsbuild 2>&1 | sed \ diff --git a/protocol/aurum.proto b/protocol/aurum.proto index d929d4d..b83bea5 100644 --- a/protocol/aurum.proto +++ b/protocol/aurum.proto @@ -34,7 +34,6 @@ service Bootstrap { rpc setFocus(ReqSetFocus) returns (RspSetFocus) {} rpc findElements(ReqFindElements) returns (RspFindElements) {} rpc getActiveAppToolkitName(ReqGetActiveAppToolkitName) returns (RspGetActiveAppToolkitName) {} - rpc enableScreenAnalyzer(ReqEnableScreenAnalyzer) returns (RspEnableScreenAnalyzer) {} rpc getTextMinBoundingRect(ReqGetTextMinBoundingRect) returns (RspGetTextMinBoundingRect) {} rpc setTimeout(ReqSetTimeout) returns (RspSetTimeout) {} rpc setXMLSync(ReqSetXMLSync) returns (RspSetXMLSync) {} @@ -631,15 +630,6 @@ message RspGetActiveAppToolkitName { string toolkitName = 2; } -message ReqEnableScreenAnalyzer { - bool enable = 1; - string serverIp = 2; -} - -message RspEnableScreenAnalyzer { - RspStatus status = 1; -} - message ReqGetTextMinBoundingRect { string elementId = 1; } diff --git a/ui_automation/python/tv/sampleWithUtils.py b/ui_automation/python/tv/sampleWithUtils.py index 9e30a0c..83b760b 100644 --- a/ui_automation/python/tv/sampleWithUtils.py +++ b/ui_automation/python/tv/sampleWithUtils.py @@ -10,7 +10,6 @@ def run(): """ stub = getStub() - stub.enableScreenAnalyzer(ReqEnableScreenAnalyzer(enable=True)) response = stub.findElements(ReqFindElements()) print(response)