rpc setFocus(ReqSetFocus) returns (RspSetFocus) {}
rpc findElements(ReqFindElements) returns (RspFindElements) {}
rpc getActiveAppToolkitName(ReqGetActiveAppToolkitName) returns (RspGetActiveAppToolkitName) {}
- rpc enableScreenAnalyzer(ReqEnableScreenAnalyzer) returns (RspEnableScreenAnalyzer) {}
rpc getTextMinBoundingRect(ReqGetTextMinBoundingRect) returns (RspGetTextMinBoundingRect) {}
rpc setTimeout(ReqSetTimeout) returns (RspSetTimeout) {}
rpc setXMLSync(ReqSetXMLSync) returns (RspSetXMLSync) {}
string toolkitName = 2;
}
-message ReqEnableScreenAnalyzer {
- bool enable = 1;
- string serverIp = 2;
-}
-
-message RspEnableScreenAnalyzer {
- RspStatus status = 1;
-}
-
message ReqGetTextMinBoundingRect {
string elementId = 1;
}
#include "A11yEvent.h"
#include "A11yEventHandler.h"
#include "Runnables.h"
-#ifdef MQTT_ENABLED
-#include "ScreenAnalyzerWatcher.h"
-#include "SaObject.h"
-#endif
#endif
+++ /dev/null
-/*
- * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef _SA_OBJECT_H_
-#define _SA_OBJECT_H_
-
-#include "config.h"
-#include <vector>
-
-namespace Aurum {
-
-/**
- * @class SaObject
- *
- * @ingroup aurum
- *
- * @brief A SaObject is a result of Screen Analyzer.
- */
-class SaObject : public IObject, public std::enable_shared_from_this<SaObject> {
-public:
-
- /**
- * @brief SaObject constructor with initial values.
- *
- * @since_tizen 7.0
- */
- SaObject(std::string id, std::string type, Rect<int> geometry, std::string ocrText, std::vector<std::string> states);
-
- /**
- * @brief SaObject destructor.
- *
- * @since_tizen 7.0
- */
- ~SaObject();
-
- /**
- * @copydoc UiObject::getId()
- */
- std::string getId() const override;
-
- /**
- * @copydoc UiObject::getType()
- */
- std::string getType() const override;
-
- /**
- * @copydoc UiObject::getScreenBoundingBox()
- */
- const Rect<int> getScreenBoundingBox() const override;
-
- std::string getOcrText() const override;
-
- /**
- * @copydoc UiObject::isFocusable()
- */
- bool isFocusable() const override;
-
- /**
- * @copydoc UiObject::isFocused()
- */
- bool isFocused() const override;
-
- /**
- * @copydoc UiObject::isClickable()
- */
- bool isClickable() const override;
-
- /**
- * @copydoc UiObject::isActive()
- */
- bool isActive() const override;
-
- /**
- * @copydoc UiObject::isShowing()
- */
- bool isShowing() const override;
-
-private:
- std::string mId;
- std::string mElementType;
- Rect<int> mScreenBoundingBox;
- std::string mOcrText;
- bool mIsFocusable;
- bool mIsFocused;
- bool mIsClickable;
- bool mIsActive;
-};
-
-}
-
-#endif
+++ /dev/null
-/*
- * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef _SCREEN_ANALYZER_WATCHER_H_
-#define _SCREEN_ANALYZER_WATCHER_H_
-
-#include <mosquitto.h>
-
-namespace Aurum {
-
-/**
- * @class ScreenAnalyzerWatcher
- *
- * @ingroup aurum
- *
- * @brief ScreenAnalyzerWatcher communicates with Screen Analyzer Server via MQTT Protocol(mosquitto).
- * It requests screen analyzer to server and parses the result that can be used in aurum.
- */
-class ScreenAnalyzerWatcher {
-public:
-
- /**
- * @brief ScreenAnalyzerWatcher constructor.
- *
- * @since_tizen 7.0
- */
- ScreenAnalyzerWatcher();
-
- /**
- * @brief ScreenAnalyzerWatcher destructor.
- *
- * @since_tizen 7.0
- */
- ~ScreenAnalyzerWatcher();
-
- /**
- * @brief When message(analyze result) is received from the broker.
- * parse the data and create SaObject.
- *
- * @param[in] mosq mosquitto instance
- * @param[in] obj user data provided in mosquitto_new
- * @param[in] msg the message data
- *
- * @since_tizen 7.0
- */
- static void onMessage(struct mosquitto *mosq, void *obj, const struct mosquitto_message *msg);
-
- /**
- * @brief Publish current screen image to the broker.
- * capture current screen and publish a message on a given topic.
- *
- * @since_tizen 7.0
- */
- void PublishData();
-
- /**
- * @brief Publish current screen image to the broker.
- * here capture current screen and publish a message on a given topic.
- *
- * @since_tizen 7.0
- */
- std::vector<std::shared_ptr<Aurum::SaObject>> GetSaObjects();
-
- /**
- * @copydoc PartialMatch::checkCriteria()
- */
- bool checkCriteria(const std::string textA, const std::string textB, const bool textPartialMatch);
-
- /**
- * @copydoc PartialMatch::checkCriteria()
- */
- bool checkCriteria(const bool boolA, const bool boolB);
-
- /**
- * @copydoc PartialMatch::checkCriteria()
- */
- bool checkCriteria(const std::shared_ptr<UiSelector> selector, const std::shared_ptr<SaObject> node);
-
- /**
- * @brief find object from screen analyzer result.
- *
- * @param[in] selector @UiSelector
-
- *
- * @return SaObject vector if found, else nullptr
- *
- * @since_tizen 7.0
- */
- std::vector<std::shared_ptr<Aurum::SaObject>> findSaObjects(const std::shared_ptr<UiSelector> selector);
-
- /**
- * @brief find object from screen analyzer result.
- *
- * @param[in] selector @UiSelector
-
- *
- * @return SaObject if found, else nullptr
- *
- * @since_tizen 7.0
- */
- std::shared_ptr<Aurum::SaObject> findSaObject(const std::shared_ptr<UiSelector> selector);
-
- /**
- * @brief Get focused app name.
- *
- * @param[in] selector @UiSelector
- *
- * @return SaObject if found, else nullptr
- *
- * @since_tizen 7.0
- */
- std::string GetFocusedAppId();
-
- /**
- * @brief Set server IP address.
- *
- * @param[in] ip std::string
- *
- * @since_tizen 7.0
- */
- void SetServerIp(std::string ip);
-
-private:
- struct mosquitto *mMosq;
- static std::string mPkgName;
- static std::vector<std::shared_ptr<SaObject>> mSaObjects;
- static bool mLoadDone;
- std::string mIp;
-};
-
-}
-
-#endif
#include <functional>
#include <string>
-#ifdef MQTT_ENABLED
-#include "SaObject.h"
-#include "ScreenAnalyzerWatcher.h"
-#endif
-
namespace Aurum {
/**
*/
std::vector<std::shared_ptr<AccessibleNode>> getWindowRoot() const override;
-#ifdef MQTT_ENABLED
- /**
- * @brief Gets screen analyzer Object vector.
- *
- * @return SaObject ptr vector
- *
- * @since_tizen 7.0
- */
- std::vector<std::shared_ptr<SaObject>> getSaObject();
-
- /**
- * @brief Gets ScreenAnalyzerWatcher instance.
- *
- * @return ScreenAnalyzerWatcher ptr
- *
- * @since_tizen 7.0
- */
- std::shared_ptr<ScreenAnalyzerWatcher> getSAWatcher();
-#endif
- /**
- * @brief Requests current screen analyze.
- *
- * @since_tizen 7.0
- */
- void RequestScreenAnalyze();
-
/**
* @brief Gets external application launched or not.
*
bool getExternalAppLaunched();
/**
- * @brief Sets whether to use screen analyzer for Tizen apps.
- *
- * @param[in] withScreenAnalyzer boolean value
- *
- * @since_tizen 7.0
- */
- void setWithScreenAnalyzer(bool withScreenAnalyzer);
-
- /**
- * @brief Gets whether to use screen analyzer for Tizen apps.
- *
- * @return true screen analyzer use for Tizen apps, otherwise false
- *
- * @since_tizen 7.0
- */
- bool getWithScreenAnalyzer();
-
- /**
* @brief Gets angle of root window.
*
* @return window angle int
private:
IDevice *mDeviceImpl;
const Waiter *mWaiter;
- bool mIsWithSA;
-#ifdef MQTT_ENABLED
- static std::shared_ptr<ScreenAnalyzerWatcher> mSAWatcher;
-#endif
static std::once_flag mOnceFlag;
};
'./inc/Aurum.h',
]
-if get_option('mqtt_enabled') == true
- libaurum_install_inc += [
- './inc/SaObject.h',
- './inc/ScreenAnalyzer/ScreenAnalyzerWatcher.h',
- ]
-endif
-
libaurum_inc = [
include_directories('./inc'),
include_directories('./inc/Accessibility'),
root_inc,
]
-if get_option('mqtt_enabled') == true
- libaurum_inc += [
- include_directories('./inc/ScreenAnalyzer'),
- ]
-endif
-
libaurum_src = []
subdir('src')
]
endif
-if get_option('mqtt_enabled') == true
- libaurum_dep += [
- dependency('libmosquitto'),
- dependency('jsoncpp'),
- dependency('capi-appfw-app-manager'),
- ]
-endif
-
libaurum_lib = library('aurum', libaurum_src,
dependencies: libaurum_dep,
include_directories: libaurum_inc,
const std::shared_ptr<AccessibleNode> node,
std::shared_ptr<UiDevice> device)
{
-#ifdef MQTT_ENABLED
- if (selector->mMatchOcrText)
- {
- if (!device->getWithScreenAnalyzer() || !node->isShowing() || !(node->getText().size() == 0)) return false;
- //FIXME: getSaObjectHasText();
- auto saObjs = device->getSAWatcher()->GetSaObjects();
-
- node->updateExtents();
- auto objBoundingBox = node->getScreenBoundingBox();
- for (auto saObj : saObjs) {
- if (saObj->getOcrText().size() != 0) {
- if (objBoundingBox.isInRect(saObj->getScreenBoundingBox().midPoint()) &&
- ((objBoundingBox.width() * objBoundingBox.height() * 1.5) > (saObj->getScreenBoundingBox().width() * saObj->getScreenBoundingBox().height())) &&
- ((objBoundingBox.width() * objBoundingBox.height() < (saObj->getScreenBoundingBox().width() * saObj->getScreenBoundingBox().height() * 1.5))))
- {
- LOGI("Text Set %s ", saObj->getOcrText().c_str());
- node->setOcrText(saObj->getOcrText());
- }
- }
- }
-
- if (node->getOcrText().size() == 0) return false;
- if (checkCriteria(selector->mOcrText, node->getOcrText(), 1)) return false;
-
- LOGI("node ocr = %s, selector ocr = %s",node->getOcrText().c_str(), selector->mOcrText.c_str());
- }
-#endif
-
if (selector->mMatchText || selector->mMatchTextPartialMatch) {
node->updateName();
if (selector->mMatchText && checkCriteria(selector->mText, node->getText(), 0)) return false;
+++ /dev/null
-/*
- * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "Aurum.h"
-
-#include <unistd.h>
-#include <utility>
-#include <vector>
-#include <chrono>
-#include <algorithm>
-#include <iostream>
-
-#include "ScreenAnalyzerWatcher.h"
-
-using namespace Aurum;
-
-SaObject::SaObject(std::string id, std::string type, Rect<int> geometry, std::string ocrText, std::vector<std::string> states)
- : mId(id), mElementType(type), mScreenBoundingBox(geometry), mOcrText(ocrText)
-{
- LOGI("SaObject Constructor");
- mIsFocusable = false;
- mIsFocused = false;
- mIsClickable = false;
- mIsActive = false;
- for (auto state : states)
- {
- if (state == "clickable") mIsClickable = true;
- else if (state == "focused") mIsFocused = true;
- else if (state == "focusable") mIsFocusable = true;
- else if (state == "active") mIsActive = true;
- }
-}
-
-SaObject::~SaObject()
-{
- LOGI("SaObject destructor");
-}
-
-std::string SaObject::getId() const
-{
- return mId;
-}
-
-std::string SaObject::getType() const
-{
- return mElementType;
-}
-
-const Rect<int> SaObject::getScreenBoundingBox() const
-{
- return mScreenBoundingBox;
-}
-
-std::string SaObject::getOcrText() const
-{
- return mOcrText;
-}
-
-bool SaObject::isFocusable() const
-{
- return mIsFocusable;
-}
-
-bool SaObject::isFocused() const
-{
- return mIsFocused;
-}
-
-bool SaObject::isClickable() const
-{
- return mIsClickable;
-}
-
-bool SaObject::isActive() const
-{
- return mIsActive;
-}
-
-bool SaObject::isShowing() const
-{
- return true;
-}
\ No newline at end of file
+++ /dev/null
-/*
- * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "Aurum.h"
-
-#include "ScreenAnalyzerWatcher.h"
-#include <fstream>
-#include <json/json.h>
-#include <sstream>
-#include "SaObject.h"
-#include <time.h>
-#include <thread>
-#include <app_manager_extension.h>
-#include <tdm_helper.h>
-#include <tbm_surface.h>
-#include <system_info.h>
-#include <efl_util.h>
-
-using namespace Aurum;
-
-//Initialize static member variables
-std::vector<std::shared_ptr<SaObject>> ScreenAnalyzerWatcher::mSaObjects;
-bool ScreenAnalyzerWatcher::mLoadDone;
-std::string ScreenAnalyzerWatcher::mPkgName;
-
-void on_connect(struct mosquitto *mosq, void *obj, int reason_code)
-{
- LOGI("mosquitto on connect");
-
- if(reason_code != 0){
- LOGE("mosquitto on connect error : %d", reason_code);
- mosquitto_disconnect(mosq);
- }
- int rc;
- rc = mosquitto_subscribe(mosq, NULL, "screen_analyzer/json_aurum", 1);
- if(rc != MOSQ_ERR_SUCCESS){
- LOGE("server on connect subscribe fail");
- mosquitto_disconnect(mosq);
- }
-
-}
-
-void on_publish(struct mosquitto *mosq, void *obj, int mid)
-{
- LOGI("mosquitto on publish");
-}
-
-void on_subscribe(struct mosquitto *mosq, void *obj, int mid, int qos_count, const int *granted_qos)
-{
- LOGI("server on subscribe qos : %d", qos_count);
-}
-
-ScreenAnalyzerWatcher::ScreenAnalyzerWatcher()
-{
- LOGI("Screen Analyzer constructor");
-
- mosquitto_lib_init();
- LOGI("mosquitto lib init");
- mMosq = mosquitto_new(NULL, true, NULL);
- if(mMosq == NULL){
- LOGE("mosquitto new fail");
- return;
- }
-
- mosquitto_connect_callback_set(mMosq, on_connect);
- mosquitto_publish_callback_set(mMosq, on_publish);
- mosquitto_subscribe_callback_set(mMosq, on_subscribe);
- mosquitto_message_callback_set(mMosq, ScreenAnalyzerWatcher::onMessage);
-
- //FIXME : When the class is created, there is an issue if We do not try to connect the server,
- // so I put the AIM team IP to try to connect.
- mIp = "10.113.16.21"; //This is AIM default server user can change this using SetServerIp()
-
- int rc;
- rc = mosquitto_connect(mMosq, mIp.c_str(), 1883, 60);
- if(rc != MOSQ_ERR_SUCCESS){
- mosquitto_destroy(mMosq);
- LOGE("mosquitto connect fail");
- return;
- }
- rc = mosquitto_loop_start(mMosq);
- if(rc != MOSQ_ERR_SUCCESS){
- mosquitto_destroy(mMosq);
- LOGE("mosquitto loop fail");
- return;
- }
-}
-
-ScreenAnalyzerWatcher::~ScreenAnalyzerWatcher()
-{
- LOGI("Screen Analyzer destructor");
-
- if (mMosq)
- {
- mosquitto_disconnect(mMosq);
- mosquitto_loop_stop(mMosq, false);
- mosquitto_lib_cleanup();
- mMosq = nullptr;
- }
-}
-
-void ScreenAnalyzerWatcher::onMessage(struct mosquitto *mosq, void *obj, const struct mosquitto_message *msg)
-{
- std::string txt((char *)msg->payload);
-
- mSaObjects.clear();
-
- Json::Reader reader;
- Json::Value root;
- bool ret = reader.parse(txt, root);
- if (!ret) LOGE("mosquitto parse fail");
-
- const Json::Value objs = root["objects"];
- LOGI("objects count = %d", root["objectCount"].asInt());
-
- std::string id;
- std::string type;
- Rect<int> geometry;
- std::string ocrText;
- std::vector<std::string> states{};
-
- std::shared_ptr<UiDevice> mDevice = UiDevice::getInstance();
- //Add dummy for window
- id = "9999";
- type = "window";
- geometry = {0, 0, mDevice->getScreenSize().width, mDevice->getScreenSize().height};
- states.push_back("active");
- ocrText = mPkgName;
- mSaObjects.push_back(std::make_shared<Aurum::SaObject>(id, type, geometry, ocrText, states));
-
- for (unsigned int idx = 0; idx < objs.size(); ++idx)
- {
- states.clear();
- id = objs[idx]["id"].asString();
- LOGI("objs[%d] ID : %s", idx,id.c_str());
-
- type = objs[idx]["type"].asString();
- LOGI("objs[%d] TYPE : %s", idx, objs[idx]["type"].asString().c_str());
-
- const Json::Value geo = objs[idx]["geometry"];
- geometry = { geo[0].asInt(), geo[1].asInt(), geo[2].asInt(), geo[3].asInt() };
- LOGI("objs[%d] GEOMETRY : %d %d %d %d", idx, geo[0].asInt(), geo[1].asInt(), geo[2].asInt(), geo[3].asInt());
-
- ocrText = objs[idx]["ocrText"].asString();
- LOGI("objs[%d] OCRTEXT : %s", idx, objs[idx]["ocrText"].asString().c_str());
-
- const Json::Value sta = objs[idx]["states"];
- for (unsigned int sidx = 0; sidx < sta.size(); ++sidx) {
- LOGI("objs[%d] STATE: %s", idx, sta[sidx].asString().c_str());
- states.push_back(sta[sidx].asString());
- }
-
- mSaObjects.push_back(std::make_shared<Aurum::SaObject>(id, type, geometry, ocrText, states));
- }
-
- mLoadDone = true;
-}
-
-void ScreenAnalyzerWatcher::PublishData()
-{
- LOGI("Prepare capture data for data publish");
- mLoadDone = false;
- void *ptr = NULL;
- unsigned char *src = NULL;
- unsigned char *dst = NULL;
-
- efl_util_screenshot_h screenshot = NULL;
- tbm_surface_h tbm_surface = NULL;
- tbm_surface_info_s info;
- std::shared_ptr<UiDevice> mDevice = UiDevice::getInstance();
- const int WIDTH = mDevice->getScreenSize().width;
- const int HEIGHT = mDevice->getScreenSize().height;
-
- screenshot = efl_util_screenshot_initialize(WIDTH, HEIGHT);
-
- if (screenshot)
- {
- tbm_surface = efl_util_screenshot_take_tbm_surface(screenshot);
- if (tbm_surface)
- {
- tbm_surface_map(tbm_surface, TBM_SURF_OPTION_READ, &info);
-
- ptr = malloc( WIDTH * HEIGHT * 4 );
- src = (unsigned char *)info.planes[0].ptr;
- dst = (unsigned char *)ptr;
-
- memcpy(dst, src, WIDTH * HEIGHT * 4);
- }
- else
- {
- efl_util_screenshot_deinitialize(screenshot);
- return;
- }
-
- int payloadlen = WIDTH * HEIGHT * 4;
-
- LOGI("mosquitto publish start");
- int rc = mosquitto_publish(mMosq, NULL, "screen_analyzer/image_aurum", payloadlen , ptr, 2, false);
- if(rc != MOSQ_ERR_SUCCESS) {
- LOGE("mosquitto publish fail");
- }
-
- tbm_surface_destroy(tbm_surface);
- efl_util_screenshot_deinitialize(screenshot);
- }
- else
- {
- LOGE("Screen shot fail");
- return;
- }
-
- while(!mLoadDone) {
- std::this_thread::sleep_for(std::chrono::milliseconds{10});
- }
-
- if (tbm_surface) {
- free(ptr);
- tbm_surface_unmap(tbm_surface);
- tbm_surface_destroy(tbm_surface);
- }
-}
-
-std::vector<std::shared_ptr<Aurum::SaObject>> ScreenAnalyzerWatcher::GetSaObjects()
-{
- return mSaObjects;
-}
-
-bool ScreenAnalyzerWatcher::checkCriteria(const std::string textA, const std::string textB, const bool textPartialMatch)
-{
- if (textB.empty()) return true;
-
- bool rst;
- if (textPartialMatch) {
- if (textB.find(textA) != std::string::npos) rst = false;
- else rst = true;
- }
- else {
- if (!textA.compare(textB)) rst = false;
- else rst = true;
- }
-
- return rst;
-}
-
-bool ScreenAnalyzerWatcher::checkCriteria(const bool boolA, const bool boolB)
-{
- return boolA != boolB;
-}
-
-bool ScreenAnalyzerWatcher::checkCriteria(const std::shared_ptr<UiSelector> selector,
- const std::shared_ptr<Aurum::SaObject> node)
-{
- if (selector->mMatchText || selector->mMatchTextPartialMatch) {
- if (selector->mMatchText && checkCriteria(selector->mText, node->getOcrText(), 0)) return false;
- if (selector->mMatchTextPartialMatch && checkCriteria(selector->mTextPartialMatch, node->getOcrText(), 1)) return false;
- }
- if (selector->mMatchId) {
- if (checkCriteria(selector->mId, node->getId(), 0)) return false;
- }
- if (selector->mMatchType) {
- if (selector->mMatchType && checkCriteria(selector->mType, node->getType(), 0)) return false;
- }
- if (selector->mMatchClickable && checkCriteria(selector->mIsclickable, node->isClickable())) return false;
- if (selector->mMatchFocused && checkCriteria(selector->mIsfocused, node->isFocused())) return false;
- if (selector->mMatchFocusable && checkCriteria(selector->mIsfocusable, node->isFocusable())) return false;
- if (selector->mMatchActive && checkCriteria(selector->mIsactive, node->isActive())) return false;
- if (selector->mMatchShowing && checkCriteria(selector->mIsshowing, node->isShowing())) return false;
-
- return true;
-}
-
-std::vector<std::shared_ptr<Aurum::SaObject>> ScreenAnalyzerWatcher::findSaObjects(const std::shared_ptr<UiSelector> selector)
-{
- std::vector<std::shared_ptr<Aurum::SaObject>> rets{};
-
- LOGI("FindSaObject start");
- for (auto saObj : mSaObjects) {
- if (checkCriteria(selector, saObj)) {
- LOGI("saObj(%s) pushed", saObj->getId().c_str());
- rets.push_back(saObj);
- }
- }
-
- return rets;
-}
-
-std::shared_ptr<Aurum::SaObject> ScreenAnalyzerWatcher::findSaObject(const std::shared_ptr<UiSelector> selector)
-{
- LOGI("FindSaObject start");
- for (auto saObj : mSaObjects) {
- if (checkCriteria(selector, saObj)) {
- LOGI("saObj(%s) pushed", saObj->getId().c_str());
- return saObj;
- }
- }
-
- return nullptr;
-}
-
-std::string ScreenAnalyzerWatcher::GetFocusedAppId()
-{
- int return_val = APP_MANAGER_ERROR_NONE;
-
- app_context_h app_context;
- char *app_id = NULL;
-
- return_val = app_manager_get_focused_app_context(&app_context);
- if (return_val != APP_MANAGER_ERROR_NONE) {
- LOGE("failed to get app-context");
- return NULL;
- }
-
- return_val = app_context_get_app_id(app_context, &app_id);
- if (return_val != APP_MANAGER_ERROR_NONE) {
- LOGE("failed to get app_id");
- app_context_destroy(app_context);
- return NULL;
- }
-
- LOGI("focused app_id = %s", app_id);
-
- return_val = app_context_destroy(app_context);
- if (return_val != APP_MANAGER_ERROR_NONE) {
- LOGE("failed to destroy app-context");
- return NULL;
- }
-
- app_context = NULL;
-
- return app_id;
-}
-
-void ScreenAnalyzerWatcher::SetServerIp(std::string ip)
-{
- LOGI("SA Server Initialize");
- mIp = ip;
-
- if (mMosq)
- {
- mosquitto_disconnect(mMosq);
- mosquitto_loop_stop(mMosq, false);
- mosquitto_lib_cleanup();
- mMosq = nullptr;
- }
-
- mosquitto_lib_init();
- LOGI("mosquitto lib init");
- mMosq = mosquitto_new(NULL, true, NULL);
- if(mMosq == NULL){
- LOGE("mosquitto new fail");
- return;
- }
-
- mosquitto_connect_callback_set(mMosq, on_connect);
- mosquitto_publish_callback_set(mMosq, on_publish);
- mosquitto_subscribe_callback_set(mMosq, on_subscribe);
- mosquitto_message_callback_set(mMosq, ScreenAnalyzerWatcher::onMessage);
-
- int rc;
- rc = mosquitto_connect(mMosq, mIp.c_str(), 1883, 60);
- if (rc != MOSQ_ERR_SUCCESS){
- mosquitto_destroy(mMosq);
- LOGE("mosquitto connect fail");
- return;
- }
- rc = mosquitto_loop_start(mMosq);
- if (rc != MOSQ_ERR_SUCCESS){
- mosquitto_destroy(mMosq);
- LOGE("mosquitto loop fail");
- return;
- }
-}
+++ /dev/null
-libaurum_src += [
- files('ScreenAnalyzerWatcher.cc')
-]
std::once_flag UiDevice::mOnceFlag;
-#ifdef MQTT_ENABLED
-std::shared_ptr<ScreenAnalyzerWatcher> UiDevice::mSAWatcher;
-#endif
-
UiDevice::UiDevice() : UiDevice(nullptr) {}
UiDevice::UiDevice(IDevice *impl)
: mDeviceImpl(impl), mWaiter(new Waiter{this})
{
LOGI("UiDevice constructor");
- mIsWithSA = false;
-#ifdef MQTT_ENABLED
- mSAWatcher = std::make_shared<ScreenAnalyzerWatcher>();
-#endif
- LOGI("UiDevice constructor finish");
}
UiDevice::~UiDevice()
return mDeviceImpl->getScreenSize();
}
-#ifdef MQTT_ENABLED
-std::vector<std::shared_ptr<SaObject>> UiDevice::getSaObject()
-{
- return mSAWatcher->GetSaObjects();
-}
-
-std::shared_ptr<ScreenAnalyzerWatcher> UiDevice::getSAWatcher()
-{
- return mSAWatcher;
-}
-#endif
-
-void UiDevice::RequestScreenAnalyze()
-{
-#ifdef MQTT_ENABLED
- mSAWatcher->PublishData();
-#endif
-}
-
bool UiDevice::getExternalAppLaunched()
{
auto ret = this->getWindowRoot();
return (ret.size() > 0) ? false : true;
}
-void UiDevice::setWithScreenAnalyzer(bool withScreenAnalyzer)
-{
- mIsWithSA = withScreenAnalyzer;
-}
-
-bool UiDevice::getWithScreenAnalyzer()
-{
- return mIsWithSA;
-}
-
bool UiDevice::registerCallback(const A11yEvent type, EventHandler cb, void *data) const
{
return AccessibleWatcher::getInstance()->registerCallback(type, cb, data);
files('AurumXML.cc'),
]
-if get_option('mqtt_enabled') == true
- libaurum_src += [
- files('SaObject.cc'),
- ]
-endif
-
subdir('Accessibility')
-if get_option('mqtt_enabled') == true
- subdir('ScreenAnalyzer')
-endif
subdir('Impl')
subdir('Runnable')
config_h.set10('TIZEN_GCOV', true)
endif
-if get_option('mqtt_enabled') == true
- config_h.set10('MQTT_ENABLED', true)
-endif
-
root_inc = include_directories('./')
subdir('pugixml')
value: false,
description: 'enable generating documentation by using doxygen'
)
-
-option('mqtt_enabled',
- type: 'boolean',
- value: false,
- description: 'enable mqtt protocol(mosquitto) for Screen Analyzer'
-)
::grpc::Status getActiveAppToolkitName(::grpc::ServerContext *context,
const ::aurum::ReqGetActiveAppToolkitName *request,
::aurum::RspGetActiveAppToolkitName *response) override;
- ::grpc::Status enableScreenAnalyzer(::grpc::ServerContext *context,
- const ::aurum::ReqEnableScreenAnalyzer *request,
- ::aurum::RspEnableScreenAnalyzer *response) override;
::grpc::Status getTextMinBoundingRect(::grpc::ServerContext *context,
const ::aurum::ReqGetTextMinBoundingRect *request,
::aurum::RspGetTextMinBoundingRect *response) override;
#include "Commands/ActionAndWaitEventCommand.h"
#include "Commands/SetFocusCommand.h"
#include "Commands/GetActiveAppToolkitNameCommand.h"
-#include "Commands/EnableScreenAnalyzerCommand.h"
#include "Commands/GetTextMinBoundingRectCommand.h"
#include "Commands/SetXMLSyncCommand.h"
#include "Commands/GetAngleCommand.h"
+++ /dev/null
-/*
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "bootstrap.h"
-
-class EnableScreenAnalyzerCommand : public Command {
-private:
- const ::aurum::ReqEnableScreenAnalyzer *mRequest;
- ::aurum::RspEnableScreenAnalyzer *mResponse;
-
-public:
- EnableScreenAnalyzerCommand(const ::aurum::ReqEnableScreenAnalyzer *request,
- ::aurum::RspEnableScreenAnalyzer*response);
- ::grpc::Status execute() override;
-};
files('src/Commands/ActionAndWaitEventCommand.cc'),
files('src/Commands/SetFocusCommand.cc'),
files('src/Commands/GetActiveAppToolkitNameCommand.cc'),
- files('src/Commands/EnableScreenAnalyzerCommand.cc'),
files('src/Commands/GetTextMinBoundingRectCommand.cc'),
files('src/Commands/SetXMLSyncCommand.cc'),
files('src/Commands/GetAngleCommand.cc'),
return execute(cmd.get(), true);
}
-::grpc::Status aurumServiceImpl::enableScreenAnalyzer(::grpc::ServerContext *context,
- const ::aurum::ReqEnableScreenAnalyzer *request,
- ::aurum::RspEnableScreenAnalyzer *response)
-{
- std::unique_ptr<EnableScreenAnalyzerCommand> cmd = std::make_unique<EnableScreenAnalyzerCommand>(request, response);
- return execute(cmd.get(), true);
-}
-
::grpc::Status aurumServiceImpl::getTextMinBoundingRect(::grpc::ServerContext *context,
const ::aurum::ReqGetTextMinBoundingRect *request,
::aurum::RspGetTextMinBoundingRect *response)
LOGI("DumpObjectTree --------------- ");
std::shared_ptr<UiDevice> mDevice = UiDevice::getInstance();
-#ifdef MQTT_ENABLED
- if (mDevice->getExternalAppLaunched())
- {
- mDevice->RequestScreenAnalyze();
-
- auto objs = mDevice->getSAWatcher()->GetSaObjects();
- ::aurum::Element *root;
- int idx = 0;
- for (auto obj : objs) {
- if (!idx) {
- root = mResponse->add_roots();
- root->set_elementid(obj->getId());
- root->set_widgettype(obj->getType());
- root->set_text(obj->getOcrText());
- root->set_isclickable(obj->isClickable());
- root->set_isfocused(obj->isFocused());
- root->set_isfocusable(obj->isFocusable());
- root->set_isactive(obj->isActive());
- root->set_isshowing(true);
- root->set_isvisible(true);
- ::aurum::Rect *rect = root->mutable_geometry();
- const Rect<int> &size = obj->getScreenBoundingBox();
- rect->set_x(size.mTopLeft.x);
- rect->set_y(size.mTopLeft.y);
- rect->set_width(size.width());
- rect->set_height(size.height());
- }
- else {
- ::aurum::Element *elm = root->add_child();
- elm->set_elementid(obj->getId());
- elm->set_widgettype(obj->getType());
- elm->set_text(obj->getOcrText());
- elm->set_isclickable(obj->isClickable());
- elm->set_isfocused(obj->isFocused());
- elm->set_isfocusable(obj->isFocusable());
- elm->set_isactive(obj->isActive());
- elm->set_isshowing(true);
- elm->set_isvisible(true);
-
- ::aurum::Rect *rect = elm->mutable_geometry();
- const Rect<int> &size = obj->getScreenBoundingBox();
- rect->set_x(size.mTopLeft.x);
- rect->set_y(size.mTopLeft.y);
- rect->set_width(size.width());
- rect->set_height(size.height());
- }
-
- idx++;
- }
- }
- else
-#endif
- {
- LOGI("elementid : %s", mRequest->elementid().c_str());
- if (mRequest->elementid().length()) {
- auto obj = mObjMap->getElement(mRequest->elementid());
- if (!obj) return grpc::Status::OK;;
-
- auto node = obj->getDescendant();
- ::aurum::Element *root = mResponse->add_roots();
- traverse(root, node, 0);
- }
+
+ LOGI("elementid : %s", mRequest->elementid().c_str());
+ if (mRequest->elementid().length()) {
+ auto obj = mObjMap->getElement(mRequest->elementid());
+ if (!obj) return grpc::Status::OK;
+
+ auto node = obj->getDescendant();
+ ::aurum::Element *root = mResponse->add_roots();
+ traverse(root, node, 0);
}
return grpc::Status::OK;
+++ /dev/null
-/*
- * Copyright (c) 2023 Samsung Electronics Co., Ltd All Rights Reserved
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "EnableScreenAnalyzerCommand.h"
-#include "UiDevice.h"
-#include "config.h"
-
-EnableScreenAnalyzerCommand::EnableScreenAnalyzerCommand(const ::aurum::ReqEnableScreenAnalyzer *request,
- ::aurum::RspEnableScreenAnalyzer *response)
- : mRequest{request}, mResponse{response}
-{
-}
-
-::grpc::Status EnableScreenAnalyzerCommand::execute()
-{
- LOGI("EnableScreenAnalyzer --------------- ");
-
- LOGI("Screen Analyzer enabled (%d) Server IP : (%s)", mRequest->enable(), mRequest->serverip().c_str());
- std::shared_ptr<UiDevice> mDevice = UiDevice::getInstance();
- mDevice->setWithScreenAnalyzer(mRequest->enable());
-#ifdef MQTT_ENABLED
- if (mRequest->serverip().size() > 0)
- mDevice->getSAWatcher()->SetServerIp(mRequest->serverip());
-#endif
-
- return grpc::Status::OK;
-}
#include "Sel.h"
#include "ISearchable.h"
#include "config.h"
-#ifdef MQTT_ENABLED
-#include "SaObject.h"
-#endif
#include "Until.h"
FindElementCommand::FindElementCommand(const ::aurum::ReqFindElement *request,
auto selector = getSelector();
std::shared_ptr<UiDevice> mDevice = UiDevice::getInstance();
-#ifdef MQTT_ENABLED
- if (mDevice->getExternalAppLaunched())
- {
- mDevice->RequestScreenAnalyze();
-
- LOGI("Search Object start");
-
- auto found = mDevice->getSAWatcher()->findSaObject(selector);
- if (found != nullptr) {
- SaObject *obj = found.get();
- ::aurum::Element *elm = mResponse->mutable_element();
- elm->set_elementid(obj->getId());
-
- ::aurum::Rect *rect = elm->mutable_geometry();
- const Rect<int> &size = obj->getScreenBoundingBox();
- rect->set_x(size.mTopLeft.x);
- rect->set_y(size.mTopLeft.y);
- rect->set_width(size.width());
- rect->set_height(size.height());
-
- elm->set_widgettype(obj->getType());
-
- elm->set_text(obj->getOcrText());
-
- elm->set_isclickable(obj->isClickable());
- elm->set_isfocused(obj->isFocused());
- elm->set_isfocusable(obj->isFocusable());
- elm->set_isactive(obj->isActive());
- elm->set_isshowing(true);
- elm->set_isvisible(true);
-
- mResponse->set_status(::aurum::RspStatus::OK);
- }
- else
- mResponse->set_status(::aurum::RspStatus::ERROR);
- }
- else
-#endif
- {
- if (mDevice->getWithScreenAnalyzer())
- {
- LOGI("Find object with SA information");
- mDevice->RequestScreenAnalyze();
- }
-
- auto waiter = new Waiter(searchableObj.get(), nullptr, mTimeout);
- auto found = waiter->waitFor(Until::findObject(selector));
-
- if (waiter) delete waiter;
-
- if (found != nullptr) {
- UiObject *obj = found.get();
- obj->refresh();
- if (mObjMap->getElement(obj->getId()) == nullptr)
- mObjMap->addElement(std::move(found));
-
- LOGI("found object : %p elementId:%s", obj, obj->getId().c_str());
-
- ::aurum::Element *elm = mResponse->mutable_element();
- elm->set_elementid(obj->getId());
- elm->set_package(obj->getApplicationPackage());
-
- ::aurum::Rect *rect = elm->mutable_geometry();
- const Rect<int> &size = obj->getScreenBoundingBox();
- rect->set_x(size.mTopLeft.x);
- rect->set_y(size.mTopLeft.y);
- rect->set_width(size.width());
- rect->set_height(size.height());
-
- ::aurum::Rect *windowRect = elm->mutable_windowrelativegeometry();
- const Rect<int> &windowRelativeSize = obj->getWindowBoundingBox();
- windowRect->set_x(windowRelativeSize.mTopLeft.x);
- windowRect->set_y(windowRelativeSize.mTopLeft.y);
- windowRect->set_width(windowRelativeSize.width());
- windowRect->set_height(windowRelativeSize.height());
-
- elm->set_widgettype(obj->getType());
- elm->set_widgetstyle(obj->getElementStyle());
-
- elm->set_text(obj->getText());
- elm->set_xpath(obj->getXPath());
- elm->set_ocrtext(obj->getOcrText());
- elm->set_automationid(obj->getAutomationId());
- elm->set_package(obj->getApplicationPackage());
- elm->set_role(obj->getRole());
-
- elm->set_ischecked(obj->isChecked());
- elm->set_ischeckable(obj->isCheckable());
- elm->set_isclickable(obj->isClickable());
- elm->set_isenabled(obj->isEnabled());
- elm->set_isfocused(obj->isFocused());
- elm->set_isfocusable(obj->isFocusable());
- elm->set_isscrollable(obj->isScrollable());
- elm->set_isselected(obj->isSelected());
- elm->set_isshowing(obj->isShowing());
- elm->set_isactive(obj->isActive());
- elm->set_isvisible(obj->isVisible());
- elm->set_isselectable(obj->isSelectable());
- elm->set_ishighlightable(obj->isHighlightable());
-
- elm->set_minvalue(obj->getMinValue());
- elm->set_maxvalue(obj->getMaxValue());
- elm->set_value(obj->getValue());
- elm->set_increment(obj->getIncrement());
-
- elm->set_windowangle(obj->getWindowAngle());
- elm->set_targetangle(obj->getTargetAngle());
-
- mResponse->set_status(::aurum::RspStatus::OK);
- } else {
- mResponse->set_status(::aurum::RspStatus::ERROR);
- }
+ auto waiter = new Waiter(searchableObj.get(), nullptr, mTimeout);
+ auto found = waiter->waitFor(Until::findObject(selector));
+
+ if (waiter) delete waiter;
+
+ if (found != nullptr) {
+ UiObject *obj = found.get();
+ obj->refresh();
+ if (mObjMap->getElement(obj->getId()) == nullptr)
+ mObjMap->addElement(std::move(found));
+
+ LOGI("found object : %p elementId:%s", obj, obj->getId().c_str());
+
+ ::aurum::Element *elm = mResponse->mutable_element();
+ elm->set_elementid(obj->getId());
+ elm->set_package(obj->getApplicationPackage());
+
+ ::aurum::Rect *rect = elm->mutable_geometry();
+ const Rect<int> &size = obj->getScreenBoundingBox();
+ rect->set_x(size.mTopLeft.x);
+ rect->set_y(size.mTopLeft.y);
+ rect->set_width(size.width());
+ rect->set_height(size.height());
+
+ ::aurum::Rect *windowRect = elm->mutable_windowrelativegeometry();
+ const Rect<int> &windowRelativeSize = obj->getWindowBoundingBox();
+ windowRect->set_x(windowRelativeSize.mTopLeft.x);
+ windowRect->set_y(windowRelativeSize.mTopLeft.y);
+ windowRect->set_width(windowRelativeSize.width());
+ windowRect->set_height(windowRelativeSize.height());
+
+ elm->set_widgettype(obj->getType());
+ elm->set_widgetstyle(obj->getElementStyle());
+
+ elm->set_text(obj->getText());
+ elm->set_xpath(obj->getXPath());
+ elm->set_ocrtext(obj->getOcrText());
+ elm->set_automationid(obj->getAutomationId());
+ elm->set_package(obj->getApplicationPackage());
+ elm->set_role(obj->getRole());
+
+ elm->set_ischecked(obj->isChecked());
+ elm->set_ischeckable(obj->isCheckable());
+ elm->set_isclickable(obj->isClickable());
+ elm->set_isenabled(obj->isEnabled());
+ elm->set_isfocused(obj->isFocused());
+ elm->set_isfocusable(obj->isFocusable());
+ elm->set_isscrollable(obj->isScrollable());
+ elm->set_isselected(obj->isSelected());
+ elm->set_isshowing(obj->isShowing());
+ elm->set_isactive(obj->isActive());
+ elm->set_isvisible(obj->isVisible());
+ elm->set_isselectable(obj->isSelectable());
+ elm->set_ishighlightable(obj->isHighlightable());
+
+ elm->set_minvalue(obj->getMinValue());
+ elm->set_maxvalue(obj->getMaxValue());
+ elm->set_value(obj->getValue());
+ elm->set_increment(obj->getIncrement());
+
+ elm->set_windowangle(obj->getWindowAngle());
+ elm->set_targetangle(obj->getTargetAngle());
+
+ mResponse->set_status(::aurum::RspStatus::OK);
+ } else {
+ mResponse->set_status(::aurum::RspStatus::ERROR);
}
return grpc::Status::OK;
#include "Sel.h"
#include "ISearchable.h"
#include "config.h"
-#ifdef MQTT_ENABLED
-#include "SaObject.h"
-#endif
#include "Until.h"
FindElementsCommand::FindElementsCommand(const ::aurum::ReqFindElements *request,
auto selectors = getSelectors();
std::shared_ptr<UiDevice> mDevice = UiDevice::getInstance();
-#ifdef MQTT_ENABLED
- if (mDevice->getExternalAppLaunched())
- {
- mDevice->RequestScreenAnalyze();
+ std::vector<std::shared_ptr<UiObject>> founds = {};
- std::vector<std::shared_ptr<SaObject>> founds = {};
+ auto waiter = new Waiter(searchableObj.get(), nullptr, mTimeout);
- for ( auto &sel : selectors ) {
- auto ret = mDevice->getSAWatcher()->findSaObjects(sel);
- std::move(std::begin(ret), std::end(ret), std::back_inserter(founds));
- }
- if (founds.size() > 0) {
- for (auto& found : founds) {
- SaObject *obj = found.get();
- ::aurum::Element *elm = mResponse->add_elements();
- elm->set_elementid(obj->getId());
-
- ::aurum::Rect *rect = elm->mutable_geometry();
- const Rect<int> &size = obj->getScreenBoundingBox();
- rect->set_x(size.mTopLeft.x);
- rect->set_y(size.mTopLeft.y);
- rect->set_width(size.width());
- rect->set_height(size.height());
-
- elm->set_widgettype(obj->getType());
-
- elm->set_text(obj->getOcrText());
- elm->set_toolkit("External");
-
- elm->set_isclickable(obj->isClickable());
- elm->set_isfocused(obj->isFocused());
- elm->set_isfocusable(obj->isFocusable());
- elm->set_isactive(obj->isActive());
- elm->set_isshowing(true);
- elm->set_isvisible(true);
- }
- mResponse->set_status(::aurum::RspStatus::OK);
- }
- else {
- mResponse->set_status(::aurum::RspStatus::ERROR);
- }
+ for (auto &sel : selectors) {
+ auto ret = waiter->waitFor(Until::findObjects(sel));
+ std::move(std::begin(ret), std::end(ret), std::back_inserter(founds));
}
- else
-#endif
- {
- if (mDevice->getWithScreenAnalyzer())
- {
- LOGI("Find object with SA information");
- mDevice->RequestScreenAnalyze();
- }
-
- std::vector<std::shared_ptr<UiObject>> founds = {};
-
- auto waiter = new Waiter(searchableObj.get(), nullptr, mTimeout);
-
- for ( auto &sel : selectors ) {
- auto ret = waiter->waitFor(Until::findObjects(sel));
- std::move(std::begin(ret), std::end(ret), std::back_inserter(founds));
- }
- if (waiter) delete waiter;
-
- if (founds.size() > 0) {
- for (auto& found : founds) {
- UiObject *obj = found.get();
- obj->refresh();
- if (mObjMap->getElement(obj->getId()) == nullptr)
- mObjMap->addElement(std::move(found));
-
- LOGI("found object : %p elementId:%s", obj, obj->getId().c_str());
-
- ::aurum::Element *elm = mResponse->add_elements();
- elm->set_elementid(obj->getId());
- elm->set_package(obj->getApplicationPackage());
-
- ::aurum::Rect *rect = elm->mutable_geometry();
- const Rect<int> &size = obj->getScreenBoundingBox();
- rect->set_x(size.mTopLeft.x);
- rect->set_y(size.mTopLeft.y);
- rect->set_width(size.width());
- rect->set_height(size.height());
-
- ::aurum::Rect *windowRect = elm->mutable_windowrelativegeometry();
- const Rect<int> &windowRelativeSize = obj->getWindowBoundingBox();
- windowRect->set_x(windowRelativeSize.mTopLeft.x);
- windowRect->set_y(windowRelativeSize.mTopLeft.y);
- windowRect->set_width(windowRelativeSize.width());
- windowRect->set_height(windowRelativeSize.height());
-
- elm->set_widgettype(obj->getType());
- elm->set_widgetstyle(obj->getElementStyle());
-
- elm->set_text(obj->getText());
- elm->set_xpath(obj->getXPath());
- elm->set_ocrtext(obj->getOcrText());
- elm->set_automationid(obj->getAutomationId());
- elm->set_package(obj->getApplicationPackage());
- elm->set_role(obj->getRole());
-
- elm->set_ischecked(obj->isChecked());
- elm->set_ischeckable(obj->isCheckable());
- elm->set_isclickable(obj->isClickable());
- elm->set_isenabled(obj->isEnabled());
- elm->set_isfocused(obj->isFocused());
- elm->set_isfocusable(obj->isFocusable());
- elm->set_isscrollable(obj->isScrollable());
- elm->set_isselected(obj->isSelected());
- elm->set_isshowing(obj->isShowing());
- elm->set_isactive(obj->isActive());
- elm->set_isvisible(obj->isVisible());
- elm->set_isselectable(obj->isSelectable());
- elm->set_ishighlightable(obj->isHighlightable());
-
- elm->set_minvalue(obj->getMinValue());
- elm->set_maxvalue(obj->getMaxValue());
- elm->set_value(obj->getValue());
- elm->set_increment(obj->getIncrement());
-
- elm->set_windowangle(obj->getWindowAngle());
- elm->set_targetangle(obj->getTargetAngle());
- }
- mResponse->set_status(::aurum::RspStatus::OK);
- } else {
- mResponse->set_status(::aurum::RspStatus::ERROR);
+ if (waiter) delete waiter;
+
+ if (founds.size() > 0) {
+ for (auto &found : founds) {
+ UiObject *obj = found.get();
+ obj->refresh();
+ if (mObjMap->getElement(obj->getId()) == nullptr)
+ mObjMap->addElement(std::move(found));
+
+ LOGI("found object : %p elementId:%s", obj, obj->getId().c_str());
+
+ ::aurum::Element *elm = mResponse->add_elements();
+ elm->set_elementid(obj->getId());
+ elm->set_package(obj->getApplicationPackage());
+
+ ::aurum::Rect *rect = elm->mutable_geometry();
+ const Rect<int> &size = obj->getScreenBoundingBox();
+ rect->set_x(size.mTopLeft.x);
+ rect->set_y(size.mTopLeft.y);
+ rect->set_width(size.width());
+ rect->set_height(size.height());
+
+ ::aurum::Rect *windowRect = elm->mutable_windowrelativegeometry();
+ const Rect<int> &windowRelativeSize = obj->getWindowBoundingBox();
+ windowRect->set_x(windowRelativeSize.mTopLeft.x);
+ windowRect->set_y(windowRelativeSize.mTopLeft.y);
+ windowRect->set_width(windowRelativeSize.width());
+ windowRect->set_height(windowRelativeSize.height());
+
+ elm->set_widgettype(obj->getType());
+ elm->set_widgetstyle(obj->getElementStyle());
+
+ elm->set_text(obj->getText());
+ elm->set_xpath(obj->getXPath());
+ elm->set_ocrtext(obj->getOcrText());
+ elm->set_automationid(obj->getAutomationId());
+ elm->set_package(obj->getApplicationPackage());
+ elm->set_role(obj->getRole());
+
+ elm->set_ischecked(obj->isChecked());
+ elm->set_ischeckable(obj->isCheckable());
+ elm->set_isclickable(obj->isClickable());
+ elm->set_isenabled(obj->isEnabled());
+ elm->set_isfocused(obj->isFocused());
+ elm->set_isfocusable(obj->isFocusable());
+ elm->set_isscrollable(obj->isScrollable());
+ elm->set_isselected(obj->isSelected());
+ elm->set_isshowing(obj->isShowing());
+ elm->set_isactive(obj->isActive());
+ elm->set_isvisible(obj->isVisible());
+ elm->set_isselectable(obj->isSelectable());
+ elm->set_ishighlightable(obj->isHighlightable());
+
+ elm->set_minvalue(obj->getMinValue());
+ elm->set_maxvalue(obj->getMaxValue());
+ elm->set_value(obj->getValue());
+ elm->set_increment(obj->getIncrement());
+
+ elm->set_windowangle(obj->getWindowAngle());
+ elm->set_targetangle(obj->getTargetAngle());
}
+ mResponse->set_status(::aurum::RspStatus::OK);
+ } else {
+ mResponse->set_status(::aurum::RspStatus::ERROR);
}
return grpc::Status::OK;
#include "Sel.h"
#include "ISearchable.h"
#include "config.h"
-#ifdef MQTT_ENABLED
-#include "SaObject.h"
-#endif
-
GetSizeCommand::GetSizeCommand(const ::aurum::ReqGetSize *request,
::aurum::RspGetSize *response)
std::shared_ptr<UiDevice> mDevice = UiDevice::getInstance();
-#ifdef MQTT_ENABLED
- if (mDevice->getExternalAppLaunched())
- {
- mDevice->RequestScreenAnalyze();
-
- std::vector<std::shared_ptr<SaObject>> founds = {};
-
- auto tempSel = std::make_shared<UiSelector>();
- tempSel->id(mRequest->elementid());
- auto selectors = std::vector<std::shared_ptr<UiSelector>>{tempSel};
-
- for ( auto &sel : selectors ) {
- auto ret = mDevice->getSAWatcher()->findSaObjects(sel);
- std::move(std::begin(ret), std::end(ret), std::back_inserter(founds));
- }
- if (founds.size() > 0) {
- for (auto& found : founds) {
- SaObject *obj = found.get();
- ::aurum::Rect *rect = mResponse->mutable_size();
- const Rect<int> &size = obj->getScreenBoundingBox();
- rect->set_x(size.mTopLeft.x);
- rect->set_y(size.mTopLeft.y);
- rect->set_width(size.width());
- rect->set_height(size.height());
- }
- }
- }
- else
-#endif
- {
- ::aurum::ReqGetSize_CoordType type = mRequest->type();
- ObjectMapper *mObjMap = ObjectMapper::getInstance();
- std::shared_ptr<UiObject> obj = mObjMap->getElement(mRequest->elementid());
- if (obj) {
- obj->updateExtents();
- ::aurum::Rect *rect = mResponse->mutable_size();
- if (type == ::aurum::ReqGetSize_CoordType::ReqGetSize_CoordType_SCREEN) {
- const Rect<int> &size = obj->getScreenBoundingBox();
- rect->set_x(size.mTopLeft.x);
- rect->set_y(size.mTopLeft.y);
- rect->set_width(size.width());
- rect->set_height(size.height());
- }
- else {
- const Rect<int> &windowRelativeSize = obj->getWindowBoundingBox();
- rect->set_x(windowRelativeSize.mTopLeft.x);
- rect->set_y(windowRelativeSize.mTopLeft.y);
- rect->set_width(windowRelativeSize.width());
- rect->set_height(windowRelativeSize.height());
- }
+ ::aurum::ReqGetSize_CoordType type = mRequest->type();
+ ObjectMapper *mObjMap = ObjectMapper::getInstance();
+ std::shared_ptr<UiObject> obj = mObjMap->getElement(mRequest->elementid());
+ if (obj) {
+ obj->updateExtents();
+ ::aurum::Rect *rect = mResponse->mutable_size();
+ if (type ==
+ ::aurum::ReqGetSize_CoordType::ReqGetSize_CoordType_SCREEN) {
+ const Rect<int> &size = obj->getScreenBoundingBox();
+ rect->set_x(size.mTopLeft.x);
+ rect->set_y(size.mTopLeft.y);
+ rect->set_width(size.width());
+ rect->set_height(size.height());
+ } else {
+ const Rect<int> &windowRelativeSize = obj->getWindowBoundingBox();
+ rect->set_x(windowRelativeSize.mTopLeft.x);
+ rect->set_y(windowRelativeSize.mTopLeft.y);
+ rect->set_width(windowRelativeSize.width());
+ rect->set_height(windowRelativeSize.height());
}
}
BuildRequires: pkgconfig(capi-base-utils-i18n)
BuildRequires: pkgconfig(vconf)
-%if "%{mqtt}" == "1"
-BuildRequires: pkgconfig(libmosquitto)
-BuildRequires: pkgconfig(jsoncpp)
-%endif
-
%if 0%{?gendoc:1}
BuildRequires: doxygen
%endif
%define TIZEN_GEN_DOC false
%endif
-%if "%{mqtt}" == "1"
-%define MQTT_ENABLED true
-%else
-%define MQTT_ENABLED false
-%endif
-
meson \
--prefix /usr \
--libdir %{_libdir} \
-Dcpp_std=c++17 \
-Dtizen=true \
-Denable_documentation=%{TIZEN_GEN_DOC} \
- -Dmqtt_enabled=%{MQTT_ENABLED} \
-Dtzapp_path=%{TZ_SYS_RO_APP} \
-Dtzpackage_path=%{TZ_SYS_RO_PACKAGES} \
gbsbuild 2>&1 | sed \
rpc setFocus(ReqSetFocus) returns (RspSetFocus) {}
rpc findElements(ReqFindElements) returns (RspFindElements) {}
rpc getActiveAppToolkitName(ReqGetActiveAppToolkitName) returns (RspGetActiveAppToolkitName) {}
- rpc enableScreenAnalyzer(ReqEnableScreenAnalyzer) returns (RspEnableScreenAnalyzer) {}
rpc getTextMinBoundingRect(ReqGetTextMinBoundingRect) returns (RspGetTextMinBoundingRect) {}
rpc setTimeout(ReqSetTimeout) returns (RspSetTimeout) {}
rpc setXMLSync(ReqSetXMLSync) returns (RspSetXMLSync) {}
string toolkitName = 2;
}
-message ReqEnableScreenAnalyzer {
- bool enable = 1;
- string serverIp = 2;
-}
-
-message RspEnableScreenAnalyzer {
- RspStatus status = 1;
-}
-
message ReqGetTextMinBoundingRect {
string elementId = 1;
}
"""
stub = getStub()
- stub.enableScreenAnalyzer(ReqEnableScreenAnalyzer(enable=True))
response = stub.findElements(ReqFindElements())
print(response)
projects / platform / core / uifw / aurum.git / commitdiff