#include <mutex>
#include "IEventConsumer.h"
+#include "IObject.h"
#include "Rect.h"
#include "config.h"
*
* @since_tizen 6.5
*/
-class AccessibleNode : public std::enable_shared_from_this<AccessibleNode>, public IEventConsumer {
+class AccessibleNode : public std::enable_shared_from_this<AccessibleNode>, public IEventConsumer, public IObject {
public:
/**
* @brief AccessibleNode constructor.
*/
void invalidate();
+public:
+ /**
+ * @copydoc UiObject::getId()
+ */
+ std::string getId() const override;
+
+ /**
+ * @copydoc UiObject::getType()
+ */
+ std::string getType() const override;
+
+ /**
+ * @copydoc UiObject::getScreenBoundingBox()
+ */
+ const Rect<int> getScreenBoundingBox() const override;
+
+ /**
+ * @copydoc UiObject::getOcrText()
+ */
+ std::string getOcrText() const override;
+
+ /**
+ * @copydoc UiObject::isFocusable()
+ */
+ bool isFocusable() const override;
+
+ /**
+ * @copydoc UiObject::isFocused()
+ */
+ bool isFocused() const override;
+
+ /**
+ * @copydoc UiObject::isClickable()
+ */
+ bool isClickable() const override;
+
+ /**
+ * @copydoc UiObject::isActive()
+ */
+ bool isActive() const override;
+
+ /**
+ * @copydoc UiObject::isShowing()
+ */
+ bool isShowing() const override;
+
public:
/**
* @copydoc UiObject::getText()
std::string getText() const;
/**
- * @copydoc UiObject::getPkg()
+ * @copydoc UiObject::setText()
*/
- std::string getPkg() const;
+ bool setText(std::string text);
/**
- * @copydoc UiObject::getId()
+ * @copydoc UiObject::setOcrText()
*/
- std::string getId() const;
+ void setOcrText(std::string text);
+
+ /**
+ * @copydoc UiObject::getPkg()
+ */
+ std::string getPkg() const;
/**
* @copydoc UiObject::getAutomationId()
*/
std::string getRole() const;
- /**
- * @copydoc UiObject::getType()
- */
- std::string getType() const;
-
/**
* @copydoc UiObject::getStyle()
*/
std::string getStyle() const;
/**
- * @copydoc UiObject::getXPath()
+ * @copydoc UiObject::getToolkitName()
*/
- std::string getXPath() const;
+ std::string getToolkitName() const;
/**
- * @copydoc UiObject::getScreenBoundingBox()
+ * @copydoc UiObject::getWindowBoundingBox()
*/
- Rect<int> getScreenBoundingBox() const;
+ const Rect<int> getWindowBoundingBox() const;
/**
- * @copydoc UiObject::getWindowBoundingBox()
+ * @copydoc UiObject::getXPath()
*/
- Rect<int> getWindowBoundingBox() const;
+ std::string getXPath() const;
/**
* @copydoc UiObject::getMinValue()
*/
bool isChecked() const;
- /**
- * @copydoc UiObject::isClickable()
- */
- bool isClickable() const;
-
/**
* @copydoc UiObject::isEnabled()
*/
bool isEnabled() const;
- /**
- * @copydoc UiObject::isFocusable()
- */
- bool isFocusable() const;
-
- /**
- * @copydoc UiObject::isFocused()
- */
- bool isFocused() const;
-
/**
* @copydoc UiObject::isLongClickable()
*/
*/
bool isVisible() const;
- /**
- * @copydoc UiObject::isShowing()
- */
- bool isShowing() const;
-
- /**
- * @copydoc UiObject::isActive()
- */
- bool isActive() const;
-
public:
/**
* @brief Print Node information.
*/
virtual void updateXPath() = 0;
+ /**
+ * @copydoc UiObject::updateToolkitName()
+ */
+ virtual void updateToolkitName() = 0;
+
/**
* @copydoc UiObject::updateValue()
*/
protected:
std::string mText;
+ std::string mOcrText;
std::string mPkg;
std::string mRole;
std::string mId;
std::string mType;
std::string mStyle;
std::string mXPath;
+ std::string mToolkitName;
Rect<int> mScreenBoundingBox;
Rect<int> mWindowBoundingBox;
int mSupportingIfaces;
#include "Waiter.h"
#include "ISearchable.h"
#include "IDevice.h"
+#include "IObject.h"
#include "Accessible.h"
#include "Point2D.h"
#include "Size2D.h"
#include "Comparer.h"
#include "A11yEvent.h"
#include "Runnables.h"
+#ifdef MQTT_ENABLED
+#include "ScreenAnalyzerWatcher.h"
+#include "SaObject.h"
+#endif
#endif
*/
void updateXPath() override;
+ /**
+ * @copydoc UiObject::updateToolkitName()
+ */
+ void updateToolkitName() override;
+
/**
* @copydoc UiObject::updateValue()
*/
static gboolean Atspi_value_set_current_value(AtspiValue *iface, gdouble value, GError **error);
static gdouble Atspi_value_get_minimum_increment(AtspiValue *iface, GError **error);
static guint Atspi_accessible_get_process_id(AtspiAccessible *node, GError **error);
+ static gchar *Atspi_accessible_get_toolkit_name(AtspiAccessible *node, GError **error);
private:
static std::recursive_mutex mMutex;
*/
void updateXPath() override;
+ /**
+ * @brief TBD
+ * @since_tizen 7.0
+ */
+ void updateToolkitName() override;
+
/**
* @brief TBD
* @since_tizen 7.0
#include <set>
#include <efl_util.h>
+#include <shared_mutex>
using namespace Aurum;
* @brief TBD
*/
Size2D<int> mScreenSize;
+ static std::mutex CaptureMutex;
};
}
--- /dev/null
+/*
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef _IOBJECT_H_
+#define _IOBJECT_H_
+
+#include "config.h"
+#include "Rect.h"
+#include <string>
+
+namespace Aurum {
+
+
+class IObject {
+public:
+
+ /**
+ * @brief IObject destructor
+ */
+ virtual ~IObject() {}
+
+ /**
+ * @copydoc UiObject::getId()
+ */
+ virtual std::string getId() const = 0;
+
+ /**
+ * @copydoc UiObject::getType()
+ */
+ virtual std::string getType() const = 0;
+
+ /**
+ * @copydoc UiObject::getScreenBoundingBox()
+ */
+ virtual const Rect<int> getScreenBoundingBox() const = 0;
+
+ /**
+ * @copydoc UiObject::getOcrText()
+ */
+ virtual std::string getOcrText() const = 0;
+
+ /**
+ * @copydoc UiObject::isFocusable()
+ */
+ virtual bool isFocusable() const = 0;
+
+ /**
+ * @copydoc UiObject::isFocused()
+ */
+ virtual bool isFocused() const = 0;
+
+ /**
+ * @copydoc UiObject::isClickable()
+ */
+ virtual bool isClickable() const = 0;
+
+ /**
+ * @copydoc UiObject::isActive()
+ */
+ virtual bool isActive() const = 0;
+
+ /**
+ * @copydoc UiObject::isShowing()
+ */
+ virtual bool isShowing() const = 0;
+};
+
+}
+
+#endif
*/
T height() const { return mBottomRight.y - mTopLeft.y; }
+ /**
+ * @brief Checks whether the given point in the rect or not.
+ *
+ * @return true if point located in rect, otherwise false
+ *
+ * @since_tizen 7.0
+ */
+ bool isInRect(const Point2D<T> &point) const
+ {
+ if (point.x >= mTopLeft.x && point.x <= mBottomRight.x
+ && point.y >= mTopLeft.y && point.y <= mBottomRight.y)
+ return true;
+
+ return false;
+ }
+
/**
* @brief Checks given Rect is same as this or not.
*
*
* @param[in] selector @UiSelector
* @param[in] node @AccessibleNode target node
+ * @param[in] device @UiDevice
*
- * @return ture if satisfied, else false
+ * @return true if satisfied, else false
*
* @since_tizen 6.5
*/
static bool checkCriteria(const std::shared_ptr<UiSelector> selector,
- const std::shared_ptr<AccessibleNode> node);
+ const std::shared_ptr<AccessibleNode> node,
+ std::shared_ptr<UiDevice> device);
/**
* @brief Checks text matched or not.
--- /dev/null
+/*
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef _SA_OBJECT_H_
+#define _SA_OBJECT_H_
+
+#include "config.h"
+#include <vector>
+
+namespace Aurum {
+
+/**
+ * @class SaObject
+ *
+ * @ingroup aurum
+ *
+ * @brief A SaObject is a result of Screen Analyzer.
+ */
+class SaObject : public IObject, public std::enable_shared_from_this<SaObject> {
+public:
+
+ /**
+ * @brief SaObject constructor with initial values.
+ *
+ * @since_tizen 7.0
+ */
+ SaObject(std::string id, std::string type, Rect<int> geometry, std::string ocrText, std::vector<std::string> states);
+
+ /**
+ * @brief SaObject destructor.
+ *
+ * @since_tizen 7.0
+ */
+ ~SaObject();
+
+ /**
+ * @copydoc UiObject::getId()
+ */
+ std::string getId() const override;
+
+ /**
+ * @copydoc UiObject::getType()
+ */
+ std::string getType() const override;
+
+ /**
+ * @copydoc UiObject::getScreenBoundingBox()
+ */
+ const Rect<int> getScreenBoundingBox() const override;
+
+ std::string getOcrText() const override;
+
+ /**
+ * @copydoc UiObject::isFocusable()
+ */
+ bool isFocusable() const override;
+
+ /**
+ * @copydoc UiObject::isFocused()
+ */
+ bool isFocused() const override;
+
+ /**
+ * @copydoc UiObject::isClickable()
+ */
+ bool isClickable() const override;
+
+ /**
+ * @copydoc UiObject::isActive()
+ */
+ bool isActive() const override;
+
+ /**
+ * @copydoc UiObject::isShowing()
+ */
+ bool isShowing() const override;
+
+private:
+ std::string mId;
+ std::string mElementType;
+ Rect<int> mScreenBoundingBox;
+ std::string mOcrText;
+ bool mIsFocusable;
+ bool mIsFocused;
+ bool mIsClickable;
+ bool mIsActive;
+};
+
+}
+
+#endif
--- /dev/null
+/*
+ * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef _SCREEN_ANALYZER_WATCHER_H_
+#define _SCREEN_ANALYZER_WATCHER_H_
+
+#include <mosquitto.h>
+
+namespace Aurum {
+
+/**
+ * @class ScreenAnalyzerWatcher
+ *
+ * @ingroup aurum
+ *
+ * @brief ScreenAnalyzerWatcher communicates with Screen Analyzer Server via MQTT Protocol(mosquitto).
+ * It requests screen analyzer to server and parses the result that can be used in aurum.
+ */
+class ScreenAnalyzerWatcher {
+public:
+
+ /**
+ * @brief ScreenAnalyzerWatcher constructor.
+ *
+ * @since_tizen 7.0
+ */
+ ScreenAnalyzerWatcher();
+
+ /**
+ * @brief ScreenAnalyzerWatcher destructor.
+ *
+ * @since_tizen 7.0
+ */
+ ~ScreenAnalyzerWatcher();
+
+ /**
+ * @brief When message(analyze result) is received from the broker.
+ * parse the data and create SaObject.
+ *
+ * @param[in] mosq mosquitto instance
+ * @param[in] obj user data provided in mosquitto_new
+ * @param[in] msg the message data
+ *
+ * @since_tizen 7.0
+ */
+ static void onMessage(struct mosquitto *mosq, void *obj, const struct mosquitto_message *msg);
+
+ /**
+ * @brief Publish current screen image to the broker.
+ * capture current screen and publish a message on a given topic.
+ *
+ * @since_tizen 7.0
+ */
+ void PublishData();
+
+ /**
+ * @brief Publish current screen image to the broker.
+ * here capture current screen and publish a message on a given topic.
+ *
+ * @since_tizen 7.0
+ */
+ std::vector<std::shared_ptr<Aurum::SaObject>> GetSaObjects();
+
+ /**
+ * @copydoc PartialMatch::checkCriteria()
+ */
+ bool checkCriteria(const std::string textA, const std::string textB, const bool textPartialMatch);
+
+ /**
+ * @copydoc PartialMatch::checkCriteria()
+ */
+ bool checkCriteria(const bool boolA, const bool boolB);
+
+ /**
+ * @copydoc PartialMatch::checkCriteria()
+ */
+ bool checkCriteria(const std::shared_ptr<UiSelector> selector, const std::shared_ptr<SaObject> node);
+
+ /**
+ * @brief find object from screen analyzer result.
+ *
+ * @param[in] selector @UiSelector
+
+ *
+ * @return SaObject vector if found, else nulltpr
+ *
+ * @since_tizen 7.0
+ */
+ std::vector<std::shared_ptr<Aurum::SaObject>> findSaObjects(const std::shared_ptr<UiSelector> selector);
+
+ /**
+ * @brief find object from screen analyzer result.
+ *
+ * @param[in] selector @UiSelector
+
+ *
+ * @return SaObject if found, else nulltpr
+ *
+ * @since_tizen 7.0
+ */
+ std::shared_ptr<Aurum::SaObject> findSaObject(const std::shared_ptr<UiSelector> selector);
+
+ /**
+ * @brief Get focused app name.
+ *
+ * @param[in] selector @UiSelector
+ *
+ * @return SaObject if found, else nulltpr
+ *
+ * @since_tizen 7.0
+ */
+ std::string GetFocusedAppId();
+
+ /**
+ * @brief Set server IP address.
+ *
+ * @param[in] ip std::string
+ *
+ * @since_tizen 7.0
+ */
+ void SetServerIp(std::string ip);
+
+private:
+ struct mosquitto *mMosq;
+ static std::string mPkgName;
+ static std::vector<std::shared_ptr<SaObject>> mSaObjects;
+ static bool mLoadDone;
+ std::string mIp;
+};
+
+}
+
+#endif
#include <functional>
#include <string>
+#ifdef MQTT_ENABLED
+#include "SaObject.h"
+#include "ScreenAnalyzerWatcher.h"
+#endif
+
namespace Aurum {
/**
* @brief Gets currently enabled applications root window.
*
* @return AccessibleNode ptr vector
+ *
* @since_tizen 6.5
*/
std::vector<std::shared_ptr<AccessibleNode>> getWindowRoot() const;
* @brief Gets currently window information from window system.
*
* @return TizenWindow ptr vector
+ *
* @since_tizen 7.0
*/
std::vector<std::shared_ptr<TizenWindow>> getTizenWindowInfo() const;
+#ifdef MQTT_ENABLED
+ /**
+ * @brief Gets screen analyzer Object vector.
+ *
+ * @return SaObject ptr vector
+ *
+ * @since_tizen 7.0
+ */
+ std::vector<std::shared_ptr<SaObject>> getSaObject();
+
+ /**
+ * @brief Gets ScreenAnalyzerWatcher instance.
+ *
+ * @retrun ScreenAnalyzerWatcher ptr
+ *
+ * @since_tizen 7.0
+ */
+ std::shared_ptr<ScreenAnalyzerWatcher> getSAWatcher();
+#endif
+ /**
+ * @brief Requests current screen analyze.
+ *
+ * @since_tizen 7.0
+ */
+ void RequestScreenAnalyze();
+
+ /**
+ * @brief Gets external application launched or not.
+ *
+ * @return true external application launched, otherwise false
+ *
+ * @since_tizen 7.0
+ */
+ bool getExternalAppLaunched();
+
+ /**
+ * @brief Sets whether to use screen analyzer for Tizen apps.
+ *
+ * @param[in] withScreenAnalyzer boolean value
+ *
+ * @since_tizen 7.0
+ */
+ void setWithScreenAnalyzer(bool withScreenAnalyzer);
+
+ /**
+ * @brief Gets whether to use screen analyzer for Tizen apps.
+ *
+ * @return true screen analyzer use for Tizen apps, otherwise false
+ *
+ * @since_tizen 7.0
+ */
+ bool getWithScreenAnalyzer();
+
private:
/**
* @brief Waits process idle.
IDevice *mDeviceImpl;
const Waiter *mWaiter;
static std::vector<std::shared_ptr<TizenWindow>> mTizenWindows;
+ bool mIsWithSA;
+#ifdef MQTT_ENABLED
+ static std::shared_ptr<ScreenAnalyzerWatcher> mSAWatcher;
+#endif
};
}
#include "Accessible.h"
#include "ISearchable.h"
+#include "IObject.h"
#include "UiSelector.h"
#include "Waiter.h"
* such as object's properties, states, geometry information.
* also user can send and receive event via this class.
*/
-class UiObject : public ISearchable , public std::enable_shared_from_this<UiObject> {
+class UiObject : public ISearchable , public std::enable_shared_from_this<UiObject>, public IObject {
public:
/**
* @brief UiObject constructor with device, selector, node pointer.
*/
bool waitFor(const std::function<bool(const UiObject *)> condition) const;
+public:
+ /**
+ * @brief Gets object's Id.
+ *
+ * @return string
+ *
+ * @since_tizen 6.5
+ */
+ std::string getId() const override;
+
+ /**
+ * @brief Gets object's type.
+ *
+ * @return string
+ *
+ * @since_tizen 7.0
+ */
+ std::string getType() const override;
+
+ /**
+ * @brief Gets object's geometry of the screen.
+ *
+ * @return @Rect
+ *
+ * @since_tizen 6.5
+ */
+ const Rect<int> getScreenBoundingBox() const override;
+
+ /**
+ * @brief Gets object's ocr text.
+ *
+ * @return string
+ *
+ * @since_tizen 7.0
+ */
+ std::string getOcrText() const override;
+
+ /**
+ * @brief Gets object's focusable property.
+ *
+ * @return true if focusable else false
+ *
+ * @since_tizen 6.5
+ */
+ bool isFocusable() const override;
+
+ /**
+ * @brief Gets object's focused property.
+ *
+ * @return true if focused else false
+ *
+ * @since_tizen 6.5
+ */
+ bool isFocused() const override;
+
+ /**
+ * @brief Gets object's clickable property.
+ *
+ * @return true if clickable else false
+ *
+ * @since_tizen 6.5
+ */
+ bool isClickable() const override;
+
+ /**
+ * @brief Gets object's active property.
+ *
+ * @return true if active else false
+ *
+ * @since_tizen 6.5
+ */
+ bool isActive() const override;
+
+ /**
+ * @brief Gets object's showing property.
+ *
+ * @return true if showing else false
+ *
+ * @since_tizen 6.5
+ */
+ bool isShowing() const override;
+
public:
/**
* @brief Gets object's parent.
*/
std::string getApplicationPackage() const;
- /**
- * @brief Gets object's Id.
- *
- * @return string
- *
- * @since_tizen 6.5
- */
- std::string getId() const;
-
/**
* @brief Gets object's automation Id.
*
*/
std::string getAutomationId() const;
- /**
- * @brief Gets object's type.
- *
- * @return string
- *
- * @since_tizen 6.5
- */
- std::string getElementType() const;
-
/**
* @brief Gets object's style.
*
*/
std::string getRole() const;
+ /**
+ * @brief Gets object's Toolkit name.
+ *
+ * @return string
+ *
+ * @since_tizen 7.0
+ */
+ std::string getToolkitName() const;
+
/**
* @brief Gets object's XPath.
*
bool setText(std::string text);
/**
- * @brief Gets object's geometry of the screen.
+ * @brief Sets object's ocr text.
*
- * @return @Rect
+ * @param[in] text string
*
- * @since_tizen 6.5
+ * @since_tizen 7.0
*/
- const Rect<int> getScreenBoundingBox() const;
+ void setOcrText(std::string text);
/**
* @brief Gets object's geometry of the window.
*/
bool isChecked() const;
- /**
- * @brief Gets object's clickable property.
- *
- * @return true if clickable else false
- *
- * @since_tizen 6.5
- */
- bool isClickable() const;
-
/**
* @brief Gets object's enabled property. (to get object enabled. disabled state)
*
*/
bool isEnabled() const;
- /**
- * @brief Gets object's focusable property.
- *
- * @return true if focusable else false
- *
- * @since_tizen 6.5
- */
- bool isFocusable() const;
-
- /**
- * @brief Gets object's focused property.
- *
- * @return true if focused else false
- *
- * @since_tizen 6.5
- */
- bool isFocused() const;
-
/**
* @brief Gets object's longclickable property.
*
*/
bool isVisible() const;
- /**
- * @brief Gets object's showing property.
- *
- * @return true if showing else false
- *
- * @since_tizen 6.5
- */
- bool isShowing() const;
-
- /**
- * @brief Gets object's active property.
- *
- * @return true if active else false
- *
- * @since_tizen 6.5
- */
- bool isActive() const;
-
/**
* @brief Performs a click action on object.
*
*/
void updatePid() const;
+ /**
+ * @brief Updates object's toolkit name information from atspi server.
+ *
+ * @since_tizen 7.0
+ */
+ void updateToolkitName() const;
/**
* @brief Sets focus to object.
*
*/
UiSelector *text(std::string text);
+ /**
+ * @brief Sets the search criteria to match the object's ocr text.
+ *
+ * @param[in] text object text
+ *
+ * @return UiSelector class instance
+ *
+ * @since_tizen 7.0
+ */
+ UiSelector *ocrText(std::string text);
+
/**
* @brief Sets the search criteria to match the object's text has given text.
*
std::string mStyle;
std::string mTextPartialMatch;
std::string mXPath;
+ std::string mOcrText;
bool mMatchId;
bool mMatchAutomationId;
bool mMatchStyle;
bool mMatchTextPartialMatch;
bool mMatchXPath;
+ bool mMatchOcrText;
bool mMatchChecked;
bool mMatchCheckable;
'./inc/Aurum.h',
]
+if get_option('mqtt_enabled') == true
+ libaurum_install_inc += [
+ './inc/SaObject.h',
+ './inc/ScreenAnalyzer/ScreenAnalyzerWatcher.h',
+ ]
+endif
+
libaurum_inc = [
include_directories('./inc'),
include_directories('./inc/Accessibility'),
root_inc,
]
+if get_option('mqtt_enabled') == true
+ libaurum_inc += [
+ include_directories('./inc/ScreenAnalyzer'),
+ ]
+endif
+
libaurum_src = []
subdir('src')
]
endif
+if get_option('mqtt_enabled') == true
+ libaurum_dep += [
+ dependency('libmosquitto'),
+ dependency('jsoncpp'),
+ dependency('capi-appfw-app-manager'),
+ ]
+endif
+
libaurum_lib = library('aurum', libaurum_src,
dependencies: libaurum_dep,
include_directories: libaurum_inc,
auto children = getWindows();
children.erase(std::remove_if(children.begin(), children.end(), [](auto child){
- return !(child->isShowing()); // Active, Showing are same meaning currently
+ return !(child->isActive() && child->isShowing() && (child->getAccessibleNode()->getChildCount() > 0));
}), children.end());
return children;
-}
\ No newline at end of file
+}
}
AccessibleNode::AccessibleNode()
-: mText{""}, mPkg{""}, mRole{""}, mId{""}, mAutomationId{""}, mType{""}, mStyle{""}, mXPath{""},
+: mText{""}, mOcrText{""}, mPkg{""}, mRole{""}, mId{""}, mAutomationId{""}, mType{""}, mStyle{""}, mXPath{""}, mToolkitName{""},
mScreenBoundingBox{0,0,0,0}, mWindowBoundingBox{0,0,0,0}, mSupportingIfaces(0), mFeatureProperty(0), mPid(0), mMinValue{0.0}, mMaxValue{0.0}, mValue{0.0}, mIncrement{0.0}, mValid{true}, mLock{}
{
}
ss << "\"mAutomationId\":\"" << this->mAutomationId << "\", ";
ss << "\"mRole\":\"" << this->mRole << "\", ";
ss << "\"mText\":\"" << this->mText << "\", ";
+ ss << "\"mOcrText\":\"" << this->mOcrText << "\", ";
ss << "\"mPkg\":\"" << this->mPkg << "\", ";
ss << "\"mType\":\"" << this->mType << "\", ";
ss << "\"mStyle\":\"" << this->mStyle << "\", ";
return mText;
}
+std::string AccessibleNode::getOcrText() const
+{
+ return mOcrText;
+}
+
+void AccessibleNode::setOcrText(std::string text)
+{
+ mOcrText = text;
+}
+
std::string AccessibleNode::getPkg() const
{
return mPkg;
return mXPath;
}
-Rect<int> AccessibleNode::getScreenBoundingBox() const
+std::string AccessibleNode::getToolkitName() const
+{
+ return mToolkitName;
+}
+
+const Rect<int> AccessibleNode::getScreenBoundingBox() const
{
return mScreenBoundingBox;
}
-Rect<int> AccessibleNode::getWindowBoundingBox() const
+const Rect<int> AccessibleNode::getWindowBoundingBox() const
{
return mWindowBoundingBox;
}
}
}
+void AtspiAccessibleNode::updateToolkitName()
+{
+ AtspiWrapper::Atspi_accessible_clear_cache(mNode);
+
+ gchar *toolkitName = AtspiWrapper::Atspi_accessible_get_toolkit_name(mNode, NULL);
+ if (toolkitName) {
+ mToolkitName = toolkitName;
+ g_free(toolkitName);
+ }
+}
+
void AtspiAccessibleNode::updateApplication()
{
AtspiWrapper::Atspi_accessible_clear_cache(mNode);
g_free(name);
}
+ gchar *toolkitName = AtspiWrapper::Atspi_accessible_get_toolkit_name(mNode, NULL);
+ if (toolkitName) {
+ mToolkitName = toolkitName;
+ g_free(toolkitName);
+ }
+
AtspiAccessible *app = AtspiWrapper::Atspi_accessible_get_application(mNode, NULL);
if (app) {
gchar *pkg = AtspiWrapper::Atspi_accessible_get_name(app, NULL);
{
std::unique_lock<std::recursive_mutex> lock(mMutex);
return atspi_accessible_get_process_id(node, error);
-}
\ No newline at end of file
+}
+
+gchar *AtspiWrapper::Atspi_accessible_get_toolkit_name(AtspiAccessible *node, GError **error)
+{
+ std::unique_lock<std::recursive_mutex> lock(mMutex);
+ return atspi_accessible_get_toolkit_name(node, error);
+}
{
}
+void MockAccessibleNode::updateToolkitName()
+{
+}
+
bool MockAccessibleNode::setFocus()
{
return false;
#define NANO_SEC 1000000000.0
#define MICRO_SEC 1000000
+
+std::mutex TizenDeviceImpl::CaptureMutex = std::mutex{};
+
TizenDeviceImpl::TizenDeviceImpl()
: mFakeTouchHandle{0}, mFakeKeyboardHandle{0}, mFakeWheelHandle{0}, tStart{}, isTimerStarted{false}, mTouchSeq{}
{
efl_util_screenshot_h screenshot = NULL;
tbm_surface_h tbm_surface = NULL;
+ CaptureMutex.lock();
screenshot = efl_util_screenshot_initialize(mScreenSize.width, mScreenSize.height);
if (screenshot) {
tbm_surface_destroy(tbm_surface);
} else {
efl_util_screenshot_deinitialize(screenshot);
+ CaptureMutex.unlock();
return false;
}
efl_util_screenshot_deinitialize(screenshot);
} else {
+ CaptureMutex.unlock();
return false;
}
-
+ CaptureMutex.unlock();
return true;
}
const Size2D<int> TizenDeviceImpl::getScreenSize()
{
TizenDeviceImpl *obj = static_cast<TizenDeviceImpl *>(this);
+ LOGI("getScreenSize : %d %d", obj->mScreenSize.width , obj->mScreenSize.height);
return obj->mScreenSize;
}
}
bool PartialMatch::checkCriteria(const std::shared_ptr<UiSelector> selector,
- const std::shared_ptr<AccessibleNode> node)
+ const std::shared_ptr<AccessibleNode> node,
+ std::shared_ptr<UiDevice> device)
{
+#ifdef MQTT_ENABLED
+ if (selector->mMatchOcrText)
+ {
+ if (!device->getWithScreenAnalyzer() || !node->isShowing() || !(node->getText().size() == 0)) return false;
+ //FIXME: getSaObjectHasText();
+ auto saObjs = device->getSAWatcher()->GetSaObjects();
+
+ node->updateExtents();
+ auto objBoundingBox = node->getScreenBoundingBox();
+ for (auto saObj : saObjs) {
+ if (saObj->getOcrText().size() != 0) {
+ if (objBoundingBox.isInRect(saObj->getScreenBoundingBox().midPoint()) &&
+ ((objBoundingBox.width() * objBoundingBox.height() * 1.5) > (saObj->getScreenBoundingBox().width() * saObj->getScreenBoundingBox().height())) &&
+ ((objBoundingBox.width() * objBoundingBox.height() < (saObj->getScreenBoundingBox().width() * saObj->getScreenBoundingBox().height() * 1.5))))
+ {
+ LOGI("Text Set %s ", saObj->getOcrText().c_str());
+ node->setOcrText(saObj->getOcrText());
+ }
+ }
+ }
+
+ if (node->getOcrText().size() == 0) return false;
+ if (checkCriteria(selector->mOcrText, node->getOcrText(), 1)) return false;
+
+ LOGI("node ocr = %s, selector ocr = %s",node->getOcrText().c_str(), selector->mOcrText.c_str());
+ }
+#endif
+
if (selector->mMatchText || selector->mMatchTextPartialMatch) {
node->updateName();
if (selector->mMatchText && checkCriteria(selector->mText, node->getText(), 0)) return false;
int relativeDepth)
{
PartialMatch *match = nullptr;
+ std::shared_ptr<UiDevice> mDevice = UiDevice::getInstance();
if ((selector->mMinDepth && relativeDepth < selector->mMinDepth) ||
(selector->mMaxDepth && relativeDepth > selector->mMaxDepth)) {
return std::shared_ptr<PartialMatch>(nullptr);
}
- if (PartialMatch::checkCriteria(selector, node))
+ if (PartialMatch::checkCriteria(selector, node, mDevice))
match = new PartialMatch(selector, absoluteDepth);
return std::shared_ptr<PartialMatch>(match);
}
--- /dev/null
+/*
+ * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "Aurum.h"
+
+#include <unistd.h>
+#include <utility>
+#include <vector>
+#include <chrono>
+#include <algorithm>
+#include <iostream>
+
+#include "ScreenAnalyzerWatcher.h"
+
+using namespace Aurum;
+
+SaObject::SaObject(std::string id, std::string type, Rect<int> geometry, std::string ocrText, std::vector<std::string> states)
+ : mId(id), mElementType(type), mScreenBoundingBox(geometry), mOcrText(ocrText)
+{
+ LOGI("SaObject Constructor");
+ mIsFocusable = false;
+ mIsFocused = false;
+ mIsClickable = false;
+ mIsActive = false;
+ for (auto state : states)
+ {
+ if (state == "clickable") mIsClickable = true;
+ else if (state == "focused") mIsFocused = true;
+ else if (state == "focusable") mIsFocusable = true;
+ else if (state == "active") mIsActive = true;
+ }
+}
+
+SaObject::~SaObject()
+{
+ LOGI("SaObject destructor");
+}
+
+std::string SaObject::getId() const
+{
+ return mId;
+}
+
+std::string SaObject::getType() const
+{
+ return mElementType;
+}
+
+const Rect<int> SaObject::getScreenBoundingBox() const
+{
+ return mScreenBoundingBox;
+}
+
+std::string SaObject::getOcrText() const
+{
+ return mOcrText;
+}
+
+bool SaObject::isFocusable() const
+{
+ return mIsFocusable;
+}
+
+bool SaObject::isFocused() const
+{
+ return mIsFocused;
+}
+
+bool SaObject::isClickable() const
+{
+ return mIsClickable;
+}
+
+bool SaObject::isActive() const
+{
+ return mIsActive;
+}
+
+bool SaObject::isShowing() const
+{
+ return true;
+}
\ No newline at end of file
--- /dev/null
+/*
+ * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "Aurum.h"
+
+#include "ScreenAnalyzerWatcher.h"
+#include <fstream>
+#include <json/json.h>
+#include <sstream>
+#include "SaObject.h"
+#include <time.h>
+#include <thread>
+#include <app_manager_extension.h>
+#include <tdm_helper.h>
+#include <tbm_surface.h>
+#include <system_info.h>
+#include <efl_util.h>
+
+using namespace Aurum;
+
+//Initialize static member variables
+std::vector<std::shared_ptr<SaObject>> ScreenAnalyzerWatcher::mSaObjects;
+bool ScreenAnalyzerWatcher::mLoadDone;
+std::string ScreenAnalyzerWatcher::mPkgName;
+
+void on_connect(struct mosquitto *mosq, void *obj, int reason_code)
+{
+ LOGI("mosquitto on connect");
+
+ if(reason_code != 0){
+ LOGE("mosquitto on connect error : %d", reason_code);
+ mosquitto_disconnect(mosq);
+ }
+ int rc;
+ rc = mosquitto_subscribe(mosq, NULL, "screen_analyzer/json_aurum", 1);
+ if(rc != MOSQ_ERR_SUCCESS){
+ LOGE("server on connect subscribe fail");
+ mosquitto_disconnect(mosq);
+ }
+
+}
+
+void on_publish(struct mosquitto *mosq, void *obj, int mid)
+{
+ LOGI("mosquitto on publish");
+}
+
+void on_subscribe(struct mosquitto *mosq, void *obj, int mid, int qos_count, const int *granted_qos)
+{
+ LOGI("server on subscribe qos : %d", qos_count);
+}
+
+ScreenAnalyzerWatcher::ScreenAnalyzerWatcher()
+{
+ LOGI("Screen Analyzer constructor");
+
+ mosquitto_lib_init();
+ LOGI("mosquitto lib init");
+ mMosq = mosquitto_new(NULL, true, NULL);
+ if(mMosq == NULL){
+ LOGE("mosquitto new fail");
+ return;
+ }
+
+ mosquitto_connect_callback_set(mMosq, on_connect);
+ mosquitto_publish_callback_set(mMosq, on_publish);
+ mosquitto_subscribe_callback_set(mMosq, on_subscribe);
+ mosquitto_message_callback_set(mMosq, ScreenAnalyzerWatcher::onMessage);
+
+ //FIXME : When the class is created, there is an issue if We do not try to connect the server,
+ // so I put the AIM team IP to try to connect.
+ mIp = "10.113.16.21"; //This is AIM default server user can change this using SetServerIp()
+
+ int rc;
+ rc = mosquitto_connect(mMosq, mIp.c_str(), 1883, 60);
+ if(rc != MOSQ_ERR_SUCCESS){
+ mosquitto_destroy(mMosq);
+ LOGE("mosquitto connect fail");
+ return;
+ }
+ rc = mosquitto_loop_start(mMosq);
+ if(rc != MOSQ_ERR_SUCCESS){
+ mosquitto_destroy(mMosq);
+ LOGE("mosquitto loop fail");
+ return;
+ }
+}
+
+ScreenAnalyzerWatcher::~ScreenAnalyzerWatcher()
+{
+ LOGI("Screen Analyzer destructor");
+
+ if (mMosq)
+ {
+ mosquitto_disconnect(mMosq);
+ mosquitto_loop_stop(mMosq, false);
+ mosquitto_lib_cleanup();
+ mMosq = nullptr;
+ }
+}
+
+void ScreenAnalyzerWatcher::onMessage(struct mosquitto *mosq, void *obj, const struct mosquitto_message *msg)
+{
+ std::string txt((char *)msg->payload);
+
+ mSaObjects.clear();
+
+ Json::Reader reader;
+ Json::Value root;
+ bool ret = reader.parse(txt, root);
+ if (!ret) LOGE("mosquitto parse fail");
+
+ const Json::Value objs = root["objects"];
+ LOGI("objects count = %d", root["objectCount"].asInt());
+
+ std::string id;
+ std::string type;
+ Rect<int> geometry;
+ std::string ocrText;
+ std::vector<std::string> states{};
+
+ std::shared_ptr<UiDevice> mDevice = UiDevice::getInstance();
+ //Add dummy for window
+ id = "9999";
+ type = "window";
+ geometry = {0, 0, mDevice->getScreenSize().width, mDevice->getScreenSize().height};
+ states.push_back("active");
+ ocrText = mPkgName;
+ mSaObjects.push_back(std::make_shared<Aurum::SaObject>(id, type, geometry, ocrText, states));
+
+ for (unsigned int idx = 0; idx < objs.size(); ++idx)
+ {
+ states.clear();
+ id = objs[idx]["id"].asString();
+ LOGI("objs[%d] ID : %s", idx,id.c_str());
+
+ type = objs[idx]["type"].asString();
+ LOGI("objs[%d] TYPE : %s", idx, objs[idx]["type"].asString().c_str());
+
+ const Json::Value geo = objs[idx]["geometry"];
+ geometry = { geo[0].asInt(), geo[1].asInt(), geo[2].asInt(), geo[3].asInt() };
+ LOGI("objs[%d] GEOMETRY : %d %d %d %d", idx, geo[0].asInt(), geo[1].asInt(), geo[2].asInt(), geo[3].asInt());
+
+ ocrText = objs[idx]["ocrText"].asString();
+ LOGI("objs[%d] OCRTEXT : %s", idx, objs[idx]["ocrText"].asString().c_str());
+
+ const Json::Value sta = objs[idx]["states"];
+ for (unsigned int sidx = 0; sidx < sta.size(); ++sidx) {
+ LOGI("objs[%d] STATE: %s", idx, sta[sidx].asString().c_str());
+ states.push_back(sta[sidx].asString());
+ }
+
+ mSaObjects.push_back(std::make_shared<Aurum::SaObject>(id, type, geometry, ocrText, states));
+ }
+
+ mLoadDone = true;
+}
+
+void ScreenAnalyzerWatcher::PublishData()
+{
+ LOGI("Prepare capture data for data publish");
+ mLoadDone = false;
+ void *ptr = NULL;
+ unsigned char *src = NULL;
+ unsigned char *dst = NULL;
+
+ efl_util_screenshot_h screenshot = NULL;
+ tbm_surface_h tbm_surface = NULL;
+ tbm_surface_info_s info;
+ std::shared_ptr<UiDevice> mDevice = UiDevice::getInstance();
+ const int WIDTH = mDevice->getScreenSize().width;
+ const int HEIGHT = mDevice->getScreenSize().height;
+
+ screenshot = efl_util_screenshot_initialize(WIDTH, HEIGHT);
+
+ if (screenshot)
+ {
+ tbm_surface = efl_util_screenshot_take_tbm_surface(screenshot);
+ if (tbm_surface)
+ {
+ tbm_surface_map(tbm_surface, TBM_SURF_OPTION_READ, &info);
+
+ ptr = malloc( WIDTH * HEIGHT * 4 );
+ src = (unsigned char *)info.planes[0].ptr;
+ dst = (unsigned char *)ptr;
+
+ memcpy(dst, src, WIDTH * HEIGHT * 4);
+ }
+ else
+ {
+ efl_util_screenshot_deinitialize(screenshot);
+ return;
+ }
+
+ int payloadlen = WIDTH * HEIGHT * 4;
+
+ LOGI("mosquitto publish start");
+ int rc = mosquitto_publish(mMosq, NULL, "screen_analyzer/image_aurum", payloadlen , ptr, 2, false);
+ if(rc != MOSQ_ERR_SUCCESS) {
+ LOGE("mosquitto publish fail");
+ }
+
+ tbm_surface_destroy(tbm_surface);
+ efl_util_screenshot_deinitialize(screenshot);
+ }
+ else
+ {
+ LOGE("Screen shot fail");
+ return;
+ }
+
+ while(!mLoadDone) {
+ std::this_thread::sleep_for(std::chrono::milliseconds{10});
+ }
+
+ if (tbm_surface) {
+ free(ptr);
+ tbm_surface_unmap(tbm_surface);
+ tbm_surface_destroy(tbm_surface);
+ }
+}
+
+std::vector<std::shared_ptr<Aurum::SaObject>> ScreenAnalyzerWatcher::GetSaObjects()
+{
+ return mSaObjects;
+}
+
+bool ScreenAnalyzerWatcher::checkCriteria(const std::string textA, const std::string textB, const bool textPartialMatch)
+{
+ if (textB.empty()) return true;
+
+ bool rst;
+ if (textPartialMatch) {
+ if (textB.find(textA) != std::string::npos) rst = false;
+ else rst = true;
+ }
+ else {
+ if (!textA.compare(textB)) rst = false;
+ else rst = true;
+ }
+
+ return rst;
+}
+
+bool ScreenAnalyzerWatcher::checkCriteria(const bool boolA, const bool boolB)
+{
+ return boolA != boolB;
+}
+
+bool ScreenAnalyzerWatcher::checkCriteria(const std::shared_ptr<UiSelector> selector,
+ const std::shared_ptr<Aurum::SaObject> node)
+{
+ if (selector->mMatchText || selector->mMatchTextPartialMatch) {
+ if (selector->mMatchText && checkCriteria(selector->mText, node->getOcrText(), 0)) return false;
+ if (selector->mMatchTextPartialMatch && checkCriteria(selector->mTextPartialMatch, node->getOcrText(), 1)) return false;
+ }
+ if (selector->mMatchId) {
+ if (checkCriteria(selector->mId, node->getId(), 0)) return false;
+ }
+ if (selector->mMatchType) {
+ if (selector->mMatchType && checkCriteria(selector->mType, node->getType(), 0)) return false;
+ }
+ if (selector->mMatchClickable && checkCriteria(selector->mIsclickable, node->isClickable())) return false;
+ if (selector->mMatchFocused && checkCriteria(selector->mIsfocused, node->isFocused())) return false;
+ if (selector->mMatchFocusable && checkCriteria(selector->mIsfocusable, node->isFocusable())) return false;
+ if (selector->mMatchActive && checkCriteria(selector->mIsactive, node->isActive())) return false;
+ if (selector->mMatchShowing && checkCriteria(selector->mIsshowing, node->isShowing())) return false;
+
+ return true;
+}
+
+std::vector<std::shared_ptr<Aurum::SaObject>> ScreenAnalyzerWatcher::findSaObjects(const std::shared_ptr<UiSelector> selector)
+{
+ std::vector<std::shared_ptr<Aurum::SaObject>> rets{};
+
+ LOGI("FindSaObject start");
+ for (auto saObj : mSaObjects) {
+ if (checkCriteria(selector, saObj)) {
+ LOGI("saObj(%s) pushed", saObj->getId().c_str());
+ rets.push_back(saObj);
+ }
+ }
+
+ return rets;
+}
+
+std::shared_ptr<Aurum::SaObject> ScreenAnalyzerWatcher::findSaObject(const std::shared_ptr<UiSelector> selector)
+{
+ LOGI("FindSaObject start");
+ for (auto saObj : mSaObjects) {
+ if (checkCriteria(selector, saObj)) {
+ LOGI("saObj(%s) pushed", saObj->getId().c_str());
+ return saObj;
+ }
+ }
+
+ return nullptr;
+}
+
+std::string ScreenAnalyzerWatcher::GetFocusedAppId()
+{
+ int return_val = APP_MANAGER_ERROR_NONE;
+
+ app_context_h app_context;
+ char *app_id = NULL;
+
+ return_val = app_manager_get_focused_app_context(&app_context);
+ if (return_val != APP_MANAGER_ERROR_NONE) {
+ LOGE("failed to get app-context");
+ return NULL;
+ }
+
+ return_val = app_context_get_app_id(app_context, &app_id);
+ if (return_val != APP_MANAGER_ERROR_NONE) {
+ LOGE("failed to get app_id");
+ app_context_destroy(app_context);
+ return NULL;
+ }
+
+ LOGI("focused app_id = %s", app_id);
+
+ return_val = app_context_destroy(app_context);
+ if (return_val != APP_MANAGER_ERROR_NONE) {
+ LOGE("failed to destroy app-context");
+ return NULL;
+ }
+
+ app_context = NULL;
+
+ return app_id;
+}
+
+void ScreenAnalyzerWatcher::SetServerIp(std::string ip)
+{
+ LOGI("SA Server Initialize");
+ mIp = ip;
+
+ if (mMosq)
+ {
+ mosquitto_disconnect(mMosq);
+ mosquitto_loop_stop(mMosq, false);
+ mosquitto_lib_cleanup();
+ mMosq = nullptr;
+ }
+
+ mosquitto_lib_init();
+ LOGI("mosquitto lib init");
+ mMosq = mosquitto_new(NULL, true, NULL);
+ if(mMosq == NULL){
+ LOGE("mosquitto new fail");
+ return;
+ }
+
+ mosquitto_connect_callback_set(mMosq, on_connect);
+ mosquitto_publish_callback_set(mMosq, on_publish);
+ mosquitto_subscribe_callback_set(mMosq, on_subscribe);
+ mosquitto_message_callback_set(mMosq, ScreenAnalyzerWatcher::onMessage);
+
+ int rc;
+ rc = mosquitto_connect(mMosq, mIp.c_str(), 1883, 60);
+ if (rc != MOSQ_ERR_SUCCESS){
+ mosquitto_destroy(mMosq);
+ LOGE("mosquitto connect fail");
+ return;
+ }
+ rc = mosquitto_loop_start(mMosq);
+ if (rc != MOSQ_ERR_SUCCESS){
+ mosquitto_destroy(mMosq);
+ LOGE("mosquitto loop fail");
+ return;
+ }
+}
--- /dev/null
+libaurum_src += [
+ files('ScreenAnalyzerWatcher.cc')
+]
std::vector<std::shared_ptr<TizenWindow>> UiDevice::mTizenWindows;
static GDBusConnection *system_conn;
+#ifdef MQTT_ENABLED
+std::shared_ptr<ScreenAnalyzerWatcher> UiDevice::mSAWatcher;
+#endif
+
UiDevice::UiDevice() : UiDevice(nullptr) {}
UiDevice::UiDevice(IDevice *impl)
: mDeviceImpl(impl), mWaiter(new Waiter{this})
{
+ LOGI("UiDevice constructor");
+#ifdef MQTT_ENABLED
+ mSAWatcher = std::make_shared<ScreenAnalyzerWatcher>();
+ mIsWithSA = false;
+#endif
+ LOGI("UiDevice constructor finish");
}
UiDevice::~UiDevice()
{
return mDeviceImpl->getScreenSize();
}
+
+#ifdef MQTT_ENABLED
+std::vector<std::shared_ptr<SaObject>> UiDevice::getSaObject()
+{
+ return mSAWatcher->GetSaObjects();
+}
+
+std::shared_ptr<ScreenAnalyzerWatcher> UiDevice::getSAWatcher()
+{
+ return mSAWatcher;
+}
+#endif
+
+void UiDevice::RequestScreenAnalyze()
+{
+#ifdef MQTT_ENABLED
+ mSAWatcher->PublishData();
+#endif
+}
+
+bool UiDevice::getExternalAppLaunched()
+{
+ auto ret = this->getWindowRoot();
+ return (ret.size() > 0) ? false : true;
+}
+
+void UiDevice::setWithScreenAnalyzer(bool withScreenAnalyzer)
+{
+ mIsWithSA = withScreenAnalyzer;
+}
+
+bool UiDevice::getWithScreenAnalyzer()
+{
+ return mIsWithSA;
+}
return getAccessibleNode()->getAutomationId();
}
-std::string UiObject::getElementType() const
+std::string UiObject::getType() const
{
return getAccessibleNode()->getType();
}
return getAccessibleNode()->setValue(text);
}
+std::string UiObject::getOcrText() const
+{
+ return getAccessibleNode()->getOcrText();
+}
+
+std::string UiObject::getToolkitName() const
+{
+ getAccessibleNode()->updateToolkitName();
+ return getAccessibleNode()->getToolkitName();
+}
+
+void UiObject::setOcrText(std::string text)
+{
+ getAccessibleNode()->setOcrText(text);
+}
+
bool UiObject::isCheckable() const
{
return getAccessibleNode()->isCheckable();
mNode->updatePid();
}
+void UiObject::updateToolkitName() const
+{
+ mNode->updateToolkitName();
+}
+
bool UiObject::setFocus() const
{
return mNode->setFocus();
using namespace Aurum;
UiSelector::UiSelector()
-: mId{}, mAutomationId{}, mRole{}, mText{}, mPkg{}, mType{}, mStyle{}, mTextPartialMatch{}, mXPath{},
+: mId{}, mAutomationId{}, mRole{}, mText{}, mPkg{}, mType{}, mStyle{}, mTextPartialMatch{}, mXPath{}, mOcrText{},
mMatchId{}, mMatchAutomationId{}, mMatchRole{}, mMatchText{}, mMatchPkg{}, mMatchType{}, mMatchStyle{},
- mMatchTextPartialMatch{}, mMatchXPath{}, mMatchChecked{}, mMatchCheckable{}, mMatchClickable{}, mMatchEnabled{}, mMatchFocused{},
- mMatchFocusable{}, mMatchScrollable{}, mMatchSelected{}, mMatchShowing{}, mMatchActive{}, mMatchVisible{},
+ mMatchTextPartialMatch{}, mMatchXPath{}, mMatchOcrText{}, mMatchChecked{}, mMatchCheckable{}, mMatchClickable{}, mMatchEnabled{},
+ mMatchFocused{}, mMatchFocusable{}, mMatchScrollable{}, mMatchSelected{}, mMatchShowing{}, mMatchActive{}, mMatchVisible{},
mMatchSelectable{}, mMinDepth{}, mMaxDepth{}, mIschecked{}, mIscheckable{}, mIsclickable{}, mIsenabled{},
mIsfocused{}, mIsfocusable{}, mIsscrollable{}, mIsselected{}, mIsshowing{}, mIsactive{}, mIsvisible{},
mIsselectable{}, mChild{}, mParent{}
if(!this->mAutomationId.empty()) ss << "\"mAutomationId\":\"" << this->mAutomationId << "\", ";
if(!this->mRole.empty()) ss << "\"mRole\":\"" << this->mRole << "\", ";
if(!this->mText.empty()) ss << "\"mText\":\"" << this->mText << "\", ";
+ if(!this->mOcrText.empty()) ss << "\"mOcrText\":\"" << this->mOcrText << "\", ";
if(!this->mTextPartialMatch.empty()) ss << "\"mTextPartialMatch\":\"" << this->mTextPartialMatch << "\", ";
if(!this->mXPath.empty()) ss << "\"mXPath\":\"" << this->mXPath << "\", ";
if(!this->mPkg.empty()) ss << "\"mPkg\":\"" << this->mPkg << "\", ";
if(this->mMatchAutomationId) ss << "\"mMatchAutomationId\":\"" << ((this->mMatchAutomationId)?"true":"false") << "\", ";
if(this->mMatchRole) ss << "\"mMatchRole\":\"" << ((this->mMatchRole)?"true":"false") << "\", ";
if(this->mMatchText) ss << "\"mMatchText\":\"" << ((this->mMatchText)?"true":"false") << "\", ";
+ if(this->mMatchOcrText) ss << "\"mMatchOcrText\":\"" << ((this->mMatchOcrText)?"true":"false") << "\", ";
if(this->mMatchTextPartialMatch) ss << "\"mMatchTextPartialMatch\":\"" << ((this->mMatchTextPartialMatch)?"true":"false") << "\", ";
if(this->mMatchXPath) ss << "\"mMatchXPath\":\"" << ((this->mMatchXPath)?"true":"false") << "\", ";
if(this->mMatchPkg) ss << "\"mMatchPkg\":\"" << ((this->mMatchPkg)?"true":"false") << "\", ";
return this;
}
+UiSelector *UiSelector::ocrText(std::string text)
+{
+ this->mOcrText = text;
+ this->mMatchOcrText = true;
+ return this;
+}
+
UiSelector *UiSelector::textPartialMatch(std::string text)
{
this->mTextPartialMatch = text;
files('TizenWindow.cc'),
]
+if get_option('mqtt_enabled') == true
+ libaurum_src += [
+ files('SaObject.cc'),
+ ]
+endif
+
subdir('Accessibility')
+if get_option('mqtt_enabled') == true
+ subdir('ScreenAnalyzer')
+endif
subdir('Impl')
subdir('Runnable')
config_h.set10('TIZEN_GCOV', true)
endif
+if get_option('mqtt_enabled') == true
+ config_h.set10('MQTT_ENABLED', true)
+endif
+
root_inc = include_directories('./')
subdir('pugixml')
value: false,
description: 'enable generating documentation by using doxygen'
)
+
+option('mqtt_enabled',
+ type: 'boolean',
+ value: false,
+ description: 'enable mqtt protocol(mosquitto) for Screen Analyzer'
+)
::grpc::Status setFocus(::grpc::ServerContext *context,
const ::aurum::ReqSetFocus *request,
::aurum::RspSetFocus *response) override;
+ ::grpc::Status getActiveAppToolkitName(::grpc::ServerContext *context,
+ const ::aurum::ReqGetActiveAppToolkitName *request,
+ ::aurum::RspGetActiveAppToolkitName *response) override;
+ ::grpc::Status enableScreenAnalyzer(::grpc::ServerContext *context,
+ const ::aurum::ReqEnableScreenAnalyzer *request,
+ ::aurum::RspEnableScreenAnalyzer *response) override;
};
#endif
#include "Commands/GetScreenSizeCommand.h"
#include "Commands/ActionAndWaitEventCommand.h"
#include "Commands/SetFocusCommand.h"
+#include "Commands/GetActiveAppToolkitNameCommand.h"
+#include "Commands/EnableScreenAnalyzerCommand.h"
--- /dev/null
+/*
+ * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <gio/gio.h>
+#include <grpcpp/grpcpp.h>
+#include "Commands/Command.h"
+#include "ObjectMapper.h"
+#include <aurum.grpc.pb.h>
+#include "config.h"
+
+class EnableScreenAnalyzerCommand : public Command {
+private:
+ const ::aurum::ReqEnableScreenAnalyzer *mRequest;
+ ::aurum::RspEnableScreenAnalyzer *mResponse;
+
+public:
+ EnableScreenAnalyzerCommand(const ::aurum::ReqEnableScreenAnalyzer *request,
+ ::aurum::RspEnableScreenAnalyzer*response);
+ ::grpc::Status execute() override;
+};
--- /dev/null
+/*
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <gio/gio.h>
+#include <grpcpp/grpcpp.h>
+#include "Commands/Command.h"
+#include "ObjectMapper.h"
+#include <aurum.grpc.pb.h>
+#include "config.h"
+
+class GetActiveAppToolkitNameCommand : public Command {
+private:
+ const ::aurum::ReqGetActiveAppToolkitName *mRequest;
+ ::aurum::RspGetActiveAppToolkitName *mResponse;
+
+public:
+ GetActiveAppToolkitNameCommand(const ::aurum::ReqGetActiveAppToolkitName *request,
+ ::aurum::RspGetActiveAppToolkitName *response);
+ ::grpc::Status execute() override;
+};
files('src/Commands/GetScreenSizeCommand.cc'),
files('src/Commands/ActionAndWaitEventCommand.cc'),
files('src/Commands/SetFocusCommand.cc'),
+ files('src/Commands/GetActiveAppToolkitNameCommand.cc'),
+ files('src/Commands/EnableScreenAnalyzerCommand.cc'),
]
bootstrap_svr_dep = [
std::unique_ptr<SetFocusCommand> cmd = std::make_unique<SetFocusCommand>(request, response);
return execute(cmd.get(), true);
}
+
+::grpc::Status aurumServiceImpl::getActiveAppToolkitName(::grpc::ServerContext *context,
+ const ::aurum::ReqGetActiveAppToolkitName *request,
+ ::aurum::RspGetActiveAppToolkitName *response)
+{
+ std::unique_ptr<GetActiveAppToolkitNameCommand> cmd = std::make_unique<GetActiveAppToolkitNameCommand>(request, response);
+ return execute(cmd.get(), true);
+}
+
+::grpc::Status aurumServiceImpl::enableScreenAnalyzer(::grpc::ServerContext *context,
+ const ::aurum::ReqEnableScreenAnalyzer *request,
+ ::aurum::RspEnableScreenAnalyzer *response)
+{
+ std::unique_ptr<EnableScreenAnalyzerCommand> cmd = std::make_unique<EnableScreenAnalyzerCommand>(request, response);
+ return execute(cmd.get(), true);
+}
+
windowRect->set_width(windowSize.width());
windowRect->set_height(windowSize.height());
- root->set_widget_type(obj->getElementType());
+ root->set_widget_type(obj->getType());
root->set_widget_style(obj->getElementStyle());
root->set_text(obj->getText());
::grpc::Status DumpObjectTreeCommand::execute()
{
LOGI("DumpObjectTree --------------- ");
- LOGI("elementid : %s", mRequest->elementid().c_str());
- if (mRequest->elementid().length()) {
- auto obj = mObjMap->getElement(mRequest->elementid());
- if (!obj) return grpc::Status::OK;;
-
- auto node = obj->getDescendant();
- ::aurum::Element *root = mResponse->add_roots();
- traverse(root, node, 0);
+
+ std::shared_ptr<UiDevice> mDevice = UiDevice::getInstance();
+#ifdef MQTT_ENABLED
+ if (mDevice->getExternalAppLaunched())
+ {
+ mDevice->RequestScreenAnalyze();
+
+ auto objs = mDevice->getSAWatcher()->GetSaObjects();
+ ::aurum::Element *root;
+ int idx = 0;
+ for (auto obj : objs) {
+ if (!idx) {
+ root = mResponse->add_roots();
+ root->set_elementid(obj->getId());
+ root->set_widget_type(obj->getType());
+ root->set_text(obj->getOcrText());
+ root->set_isclickable(obj->isClickable());
+ root->set_isfocused(obj->isFocused());
+ root->set_isfocusable(obj->isFocusable());
+ root->set_isactive(obj->isActive());
+ root->set_isshowing(true);
+ root->set_isvisible(true);
+ ::aurum::Rect *rect = root->mutable_geometry();
+ const Rect<int> &size = obj->getScreenBoundingBox();
+ rect->set_x(size.mTopLeft.x);
+ rect->set_y(size.mTopLeft.y);
+ rect->set_width(size.width());
+ rect->set_height(size.height());
+ }
+ else {
+ ::aurum::Element *elm = root->add_child();
+ elm->set_elementid(obj->getId());
+ elm->set_widget_type(obj->getType());
+ elm->set_text(obj->getOcrText());
+ elm->set_isclickable(obj->isClickable());
+ elm->set_isfocused(obj->isFocused());
+ elm->set_isfocusable(obj->isFocusable());
+ elm->set_isactive(obj->isActive());
+ elm->set_isshowing(true);
+ elm->set_isvisible(true);
+
+ ::aurum::Rect *rect = elm->mutable_geometry();
+ const Rect<int> &size = obj->getScreenBoundingBox();
+ rect->set_x(size.mTopLeft.x);
+ rect->set_y(size.mTopLeft.y);
+ rect->set_width(size.width());
+ rect->set_height(size.height());
+ }
+
+ idx++;
+ }
+ }
+ else
+#endif
+ {
+ LOGI("elementid : %s", mRequest->elementid().c_str());
+ if (mRequest->elementid().length()) {
+ auto obj = mObjMap->getElement(mRequest->elementid());
+ if (!obj) return grpc::Status::OK;;
+
+ auto node = obj->getDescendant();
+ ::aurum::Element *root = mResponse->add_roots();
+ traverse(root, node, 0);
+ }
}
return grpc::Status::OK;
}
--- /dev/null
+/*
+ * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "bootstrap.h"
+#include "EnableScreenAnalyzerCommand.h"
+#include "UiDevice.h"
+
+EnableScreenAnalyzerCommand::EnableScreenAnalyzerCommand(const ::aurum::ReqEnableScreenAnalyzer *request,
+ ::aurum::RspEnableScreenAnalyzer *response)
+ : mRequest{request}, mResponse{response}
+{
+}
+
+::grpc::Status EnableScreenAnalyzerCommand::execute()
+{
+ LOGI("EnableScreenAnalyzer test --------------- ");
+
+ LOGI("Screen Analzyer enabled (%d) Servier IP : (%s)", mRequest->enable(), mRequest->serverip().c_str());
+ std::shared_ptr<UiDevice> mDevice = UiDevice::getInstance();
+ mDevice->setWithScreenAnalyzer(mRequest->enable());
+#ifdef MQTT_ENABLED
+ if (mRequest->serverip().size() > 0)
+ mDevice->getSAWatcher()->SetServerIp(mRequest->serverip());
+#endif
+
+ return grpc::Status::OK;
+}
#include "UiSelector.h"
#include "Sel.h"
#include "ISearchable.h"
+#ifdef MQTT_ENABLED
+#include "SaObject.h"
+#endif
FindElementCommand::FindElementCommand(const ::aurum::ReqFindElement *request,
::aurum::RspFindElement *response)
if(mRequest->_packagename_case()) sel->pkg(mRequest->packagename());
if(mRequest->_textpartialmatch_case()) sel->textPartialMatch(mRequest->textpartialmatch());
if(mRequest->_xpath_case()) sel->xpath(mRequest->xpath());
+ if(mRequest->_ocrtext_case()) sel->ocrText(mRequest->ocrtext());
return sel;
}
LOGI("findElement --------------- ");
auto searchableObj = getSearchableTop();
auto selector = getSelector();
-
- auto found = searchableObj->findObject(selector);
-
- if (found != nullptr) {
- UiObject *obj = found.get();
- obj->refresh();
- if (mObjMap->getElement(obj->getId()) == nullptr)
- mObjMap->addElement(std::move(found));
-
- LOGI("found object : %p elementId:%s", obj, obj->getId().c_str());
-
- ::aurum::Element *elm = mResponse->mutable_element();
- elm->set_elementid(obj->getId());
- elm->set_package(obj->getApplicationPackage());
-
- ::aurum::Rect *rect = elm->mutable_geometry();
- const Rect<int> &size = obj->getScreenBoundingBox();
- rect->set_x(size.mTopLeft.x);
- rect->set_y(size.mTopLeft.y);
- rect->set_width(size.width());
- rect->set_height(size.height());
-
- ::aurum::Rect *windowRect = elm->mutable_window_relative_geometry();
- const Rect<int> &windowRelativeSize = obj->getWindowBoundingBox();
- windowRect->set_x(windowRelativeSize.mTopLeft.x);
- windowRect->set_y(windowRelativeSize.mTopLeft.y);
- windowRect->set_width(windowRelativeSize.width());
- windowRect->set_height(windowRelativeSize.height());
-
- elm->set_widget_type(obj->getElementType());
- elm->set_widget_style(obj->getElementStyle());
-
- elm->set_text(obj->getText());
- elm->set_xpath(obj->getXPath());
- elm->set_automationid(obj->getAutomationId());
- elm->set_package(obj->getApplicationPackage());
- elm->set_role(obj->getRole());
-
- elm->set_ischecked(obj->isChecked());
- elm->set_ischeckable(obj->isCheckable());
- elm->set_isclickable(obj->isClickable());
- elm->set_isenabled(obj->isEnabled());
- elm->set_isfocused(obj->isFocused());
- elm->set_isfocusable(obj->isFocusable());
- elm->set_isscrollable(obj->isScrollable());
- elm->set_isselected(obj->isSelected());
- elm->set_isshowing(obj->isShowing());
- elm->set_isactive(obj->isActive());
- elm->set_isvisible(obj->isVisible());
- elm->set_isselectable(obj->isSelectable());
-
- elm->set_minvalue(obj->getMinValue());
- elm->set_maxvalue(obj->getMaxValue());
- elm->set_value(obj->getValue());
- elm->set_increment(obj->getIncrement());
-
- mResponse->set_status(::aurum::RspStatus::OK);
- } else {
- mResponse->set_status(::aurum::RspStatus::ERROR);
+ std::shared_ptr<UiDevice> mDevice = UiDevice::getInstance();
+
+#ifdef MQTT_ENABLED
+ if (mDevice->getExternalAppLaunched())
+ {
+ mDevice->RequestScreenAnalyze();
+
+ LOGI("Search Object start");
+
+ auto found = mDevice->getSAWatcher()->findSaObject(selector);
+ if (found != nullptr) {
+ SaObject *obj = found.get();
+ ::aurum::Element *elm = mResponse->mutable_element();
+ elm->set_elementid(obj->getId());
+
+ ::aurum::Rect *rect = elm->mutable_geometry();
+ const Rect<int> &size = obj->getScreenBoundingBox();
+ rect->set_x(size.mTopLeft.x);
+ rect->set_y(size.mTopLeft.y);
+ rect->set_width(size.width());
+ rect->set_height(size.height());
+
+ elm->set_widget_type(obj->getType());
+
+ elm->set_text(obj->getOcrText());
+
+ elm->set_isclickable(obj->isClickable());
+ elm->set_isfocused(obj->isFocused());
+ elm->set_isfocusable(obj->isFocusable());
+ elm->set_isactive(obj->isActive());
+ elm->set_isshowing(true);
+ elm->set_isvisible(true);
+
+ mResponse->set_status(::aurum::RspStatus::OK);
+ }
+ else
+ mResponse->set_status(::aurum::RspStatus::ERROR);
+ }
+ else
+#endif
+ {
+ if (mDevice->getWithScreenAnalyzer())
+ {
+ LOGI("Find object with SA information");
+ mDevice->RequestScreenAnalyze();
+ }
+
+ auto found = searchableObj->findObject(selector);
+
+ if (found != nullptr) {
+ UiObject *obj = found.get();
+ obj->refresh();
+ if (mObjMap->getElement(obj->getId()) == nullptr)
+ mObjMap->addElement(std::move(found));
+
+ LOGI("found object : %p elementId:%s", obj, obj->getId().c_str());
+
+ ::aurum::Element *elm = mResponse->mutable_element();
+ elm->set_elementid(obj->getId());
+ elm->set_package(obj->getApplicationPackage());
+
+ ::aurum::Rect *rect = elm->mutable_geometry();
+ const Rect<int> &size = obj->getScreenBoundingBox();
+ rect->set_x(size.mTopLeft.x);
+ rect->set_y(size.mTopLeft.y);
+ rect->set_width(size.width());
+ rect->set_height(size.height());
+
+ ::aurum::Rect *windowRect = elm->mutable_window_relative_geometry();
+ const Rect<int> &windowRelativeSize = obj->getWindowBoundingBox();
+ windowRect->set_x(windowRelativeSize.mTopLeft.x);
+ windowRect->set_y(windowRelativeSize.mTopLeft.y);
+ windowRect->set_width(windowRelativeSize.width());
+ windowRect->set_height(windowRelativeSize.height());
+
+ elm->set_widget_type(obj->getType());
+ elm->set_widget_style(obj->getElementStyle());
+
+ elm->set_text(obj->getText());
+ elm->set_xpath(obj->getXPath());
+ elm->set_ocrtext(obj->getOcrText());
+ elm->set_automationid(obj->getAutomationId());
+ elm->set_package(obj->getApplicationPackage());
+ elm->set_role(obj->getRole());
+
+ elm->set_ischecked(obj->isChecked());
+ elm->set_ischeckable(obj->isCheckable());
+ elm->set_isclickable(obj->isClickable());
+ elm->set_isenabled(obj->isEnabled());
+ elm->set_isfocused(obj->isFocused());
+ elm->set_isfocusable(obj->isFocusable());
+ elm->set_isscrollable(obj->isScrollable());
+ elm->set_isselected(obj->isSelected());
+ elm->set_isshowing(obj->isShowing());
+ elm->set_isactive(obj->isActive());
+ elm->set_isvisible(obj->isVisible());
+ elm->set_isselectable(obj->isSelectable());
+
+ elm->set_minvalue(obj->getMinValue());
+ elm->set_maxvalue(obj->getMaxValue());
+ elm->set_value(obj->getValue());
+ elm->set_increment(obj->getIncrement());
+
+ mResponse->set_status(::aurum::RspStatus::OK);
+ } else {
+ mResponse->set_status(::aurum::RspStatus::ERROR);
+ }
}
return grpc::Status::OK;
#include "UiSelector.h"
#include "Sel.h"
#include "ISearchable.h"
+#ifdef MQTT_ENABLED
+#include "SaObject.h"
+#endif
FindElementsCommand::FindElementsCommand(const ::aurum::ReqFindElements *request,
::aurum::RspFindElements *response)
if(mRequest->_packagename_case()) sel->pkg(mRequest->packagename());
if(mRequest->_textpartialmatch_case()) sel->textPartialMatch(mRequest->textpartialmatch());
if(mRequest->_xpath_case()) sel->xpath(mRequest->xpath());
+ if(mRequest->_ocrtext_case()) sel->ocrText(mRequest->ocrtext());
return std::vector<std::shared_ptr<UiSelector>>{sel};
}
LOGI("findElements --------------- ");
auto searchableObj = getSearchableTop();
auto selectors = getSelectors();
+ std::shared_ptr<UiDevice> mDevice = UiDevice::getInstance();
- std::vector<std::shared_ptr<UiObject>> founds = {};
+#ifdef MQTT_ENABLED
+ if (mDevice->getExternalAppLaunched())
+ {
+ mDevice->RequestScreenAnalyze();
- for ( auto &sel : selectors ) {
- auto ret = searchableObj->findObjects(sel);
- std::move(std::begin(ret), std::end(ret), std::back_inserter(founds));
+ std::vector<std::shared_ptr<SaObject>> founds = {};
+
+ for ( auto &sel : selectors ) {
+ auto ret = mDevice->getSAWatcher()->findSaObjects(sel);
+ std::move(std::begin(ret), std::end(ret), std::back_inserter(founds));
+ }
+ if (founds.size() > 0) {
+ for (auto& found : founds) {
+ SaObject *obj = found.get();
+ ::aurum::Element *elm = mResponse->add_elements();
+ elm->set_elementid(obj->getId());
+
+ ::aurum::Rect *rect = elm->mutable_geometry();
+ const Rect<int> &size = obj->getScreenBoundingBox();
+ rect->set_x(size.mTopLeft.x);
+ rect->set_y(size.mTopLeft.y);
+ rect->set_width(size.width());
+ rect->set_height(size.height());
+
+ elm->set_widget_type(obj->getType());
+
+ elm->set_text(obj->getOcrText());
+ elm->set_toolkit("External");
+
+ elm->set_isclickable(obj->isClickable());
+ elm->set_isfocused(obj->isFocused());
+ elm->set_isfocusable(obj->isFocusable());
+ elm->set_isactive(obj->isActive());
+ elm->set_isshowing(true);
+ elm->set_isvisible(true);
+ }
+ mResponse->set_status(::aurum::RspStatus::OK);
+ }
+ else {
+ mResponse->set_status(::aurum::RspStatus::ERROR);
+ }
}
+ else
+#endif
+ {
+ if (mDevice->getWithScreenAnalyzer())
+ {
+ LOGI("Find object with SA information");
+ mDevice->RequestScreenAnalyze();
+ }
+
+ std::vector<std::shared_ptr<UiObject>> founds = {};
+
+ for ( auto &sel : selectors ) {
+ auto ret = searchableObj->findObjects(sel);
+ std::move(std::begin(ret), std::end(ret), std::back_inserter(founds));
+ }
- if (founds.size() > 0) {
- for (auto& found : founds) {
- UiObject *obj = found.get();
- obj->refresh();
- if (mObjMap->getElement(obj->getId()) == nullptr)
- mObjMap->addElement(std::move(found));
-
- LOGI("found object : %p elementId:%s", obj, obj->getId().c_str());
-
- ::aurum::Element *elm = mResponse->add_elements();
- elm->set_elementid(obj->getId());
- elm->set_package(obj->getApplicationPackage());
-
- ::aurum::Rect *rect = elm->mutable_geometry();
- const Rect<int> &size = obj->getScreenBoundingBox();
- rect->set_x(size.mTopLeft.x);
- rect->set_y(size.mTopLeft.y);
- rect->set_width(size.width());
- rect->set_height(size.height());
-
- ::aurum::Rect *windowRect = elm->mutable_window_relative_geometry();
- const Rect<int> &windowRelativeSize = obj->getWindowBoundingBox();
- windowRect->set_x(windowRelativeSize.mTopLeft.x);
- windowRect->set_y(windowRelativeSize.mTopLeft.y);
- windowRect->set_width(windowRelativeSize.width());
- windowRect->set_height(windowRelativeSize.height());
-
- elm->set_widget_type(obj->getElementType());
- elm->set_widget_style(obj->getElementStyle());
-
- elm->set_text(obj->getText());
- elm->set_xpath(obj->getXPath());
- elm->set_automationid(obj->getAutomationId());
- elm->set_package(obj->getApplicationPackage());
- elm->set_role(obj->getRole());
-
- elm->set_ischecked(obj->isChecked());
- elm->set_ischeckable(obj->isCheckable());
- elm->set_isclickable(obj->isClickable());
- elm->set_isenabled(obj->isEnabled());
- elm->set_isfocused(obj->isFocused());
- elm->set_isfocusable(obj->isFocusable());
- elm->set_isscrollable(obj->isScrollable());
- elm->set_isselected(obj->isSelected());
- elm->set_isshowing(obj->isShowing());
- elm->set_isactive(obj->isActive());
- elm->set_isvisible(obj->isVisible());
- elm->set_isselectable(obj->isSelectable());
-
- elm->set_minvalue(obj->getMinValue());
- elm->set_maxvalue(obj->getMaxValue());
- elm->set_value(obj->getValue());
- elm->set_increment(obj->getIncrement());
+ if (founds.size() > 0) {
+ for (auto& found : founds) {
+ UiObject *obj = found.get();
+ obj->refresh();
+ if (mObjMap->getElement(obj->getId()) == nullptr)
+ mObjMap->addElement(std::move(found));
+
+ LOGI("found object : %p elementId:%s", obj, obj->getId().c_str());
+
+ ::aurum::Element *elm = mResponse->add_elements();
+ elm->set_elementid(obj->getId());
+ elm->set_package(obj->getApplicationPackage());
+
+ ::aurum::Rect *rect = elm->mutable_geometry();
+ const Rect<int> &size = obj->getScreenBoundingBox();
+ rect->set_x(size.mTopLeft.x);
+ rect->set_y(size.mTopLeft.y);
+ rect->set_width(size.width());
+ rect->set_height(size.height());
+
+ ::aurum::Rect *windowRect = elm->mutable_window_relative_geometry();
+ const Rect<int> &windowRelativeSize = obj->getWindowBoundingBox();
+ windowRect->set_x(windowRelativeSize.mTopLeft.x);
+ windowRect->set_y(windowRelativeSize.mTopLeft.y);
+ windowRect->set_width(windowRelativeSize.width());
+ windowRect->set_height(windowRelativeSize.height());
+
+ elm->set_widget_type(obj->getType());
+ elm->set_widget_style(obj->getElementStyle());
+
+ elm->set_text(obj->getText());
+ elm->set_xpath(obj->getXPath());
+ elm->set_ocrtext(obj->getOcrText());
+ elm->set_automationid(obj->getAutomationId());
+ elm->set_package(obj->getApplicationPackage());
+ elm->set_role(obj->getRole());
+
+ elm->set_ischecked(obj->isChecked());
+ elm->set_ischeckable(obj->isCheckable());
+ elm->set_isclickable(obj->isClickable());
+ elm->set_isenabled(obj->isEnabled());
+ elm->set_isfocused(obj->isFocused());
+ elm->set_isfocusable(obj->isFocusable());
+ elm->set_isscrollable(obj->isScrollable());
+ elm->set_isselected(obj->isSelected());
+ elm->set_isshowing(obj->isShowing());
+ elm->set_isactive(obj->isActive());
+ elm->set_isvisible(obj->isVisible());
+ elm->set_isselectable(obj->isSelectable());
+
+ elm->set_minvalue(obj->getMinValue());
+ elm->set_maxvalue(obj->getMaxValue());
+ elm->set_value(obj->getValue());
+ elm->set_increment(obj->getIncrement());
+ }
+ mResponse->set_status(::aurum::RspStatus::OK);
+ } else {
+ mResponse->set_status(::aurum::RspStatus::ERROR);
}
- mResponse->set_status(::aurum::RspStatus::OK);
- } else {
- mResponse->set_status(::aurum::RspStatus::ERROR);
}
return grpc::Status::OK;
--- /dev/null
+/*
+ * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "bootstrap.h"
+#include "GetActiveAppToolkitNameCommand.h"
+#include "UiDevice.h"
+
+GetActiveAppToolkitNameCommand::GetActiveAppToolkitNameCommand(const ::aurum::ReqGetActiveAppToolkitName *request,
+ ::aurum::RspGetActiveAppToolkitName *response)
+ : mRequest{request}, mResponse{response}
+{
+}
+
+::grpc::Status GetActiveAppToolkitNameCommand::execute()
+{
+ LOGI("GetActiveAppToolkitName --------------- ");
+
+ return grpc::Status::OK;
+}
#include "bootstrap.h"
#include "GetSizeCommand.h"
#include "UiObject.h"
+#include "UiDevice.h"
+#include "UiSelector.h"
+#include "Sel.h"
+#include "ISearchable.h"
+#ifdef MQTT_ENABLED
+#include "SaObject.h"
+#endif
+
GetSizeCommand::GetSizeCommand(const ::aurum::ReqGetSize *request,
::aurum::RspGetSize *response)
{
LOGI("GetSize --------------- ");
- ::aurum::ReqGetSize_CoordType type = mRequest->type();
- ObjectMapper *mObjMap = ObjectMapper::getInstance();
- std::shared_ptr<UiObject> obj = mObjMap->getElement(mRequest->elementid());
- if (obj) {
- obj->updateExtents();
- ::aurum::Rect *rect = mResponse->mutable_size();
- if (type == ::aurum::ReqGetSize_CoordType::ReqGetSize_CoordType_SCREEN) {
- const Rect<int> &size = obj->getScreenBoundingBox();
- rect->set_x(size.mTopLeft.x);
- rect->set_y(size.mTopLeft.y);
- rect->set_width(size.width());
- rect->set_height(size.height());
+ std::shared_ptr<UiDevice> mDevice = UiDevice::getInstance();
+
+#ifdef MQTT_ENABLED
+ if (mDevice->getExternalAppLaunched())
+ {
+ mDevice->RequestScreenAnalyze();
+
+ std::vector<std::shared_ptr<SaObject>> founds = {};
+
+ auto tempSel = std::make_shared<UiSelector>();
+ tempSel->id(mRequest->elementid());
+ auto selectors = std::vector<std::shared_ptr<UiSelector>>{tempSel};
+
+ for ( auto &sel : selectors ) {
+ auto ret = mDevice->getSAWatcher()->findSaObjects(sel);
+ std::move(std::begin(ret), std::end(ret), std::back_inserter(founds));
+ }
+ if (founds.size() > 0) {
+ for (auto& found : founds) {
+ SaObject *obj = found.get();
+ ::aurum::Rect *rect = mResponse->mutable_size();
+ const Rect<int> &size = obj->getScreenBoundingBox();
+ rect->set_x(size.mTopLeft.x);
+ rect->set_y(size.mTopLeft.y);
+ rect->set_width(size.width());
+ rect->set_height(size.height());
+ }
}
- else {
- const Rect<int> &windowRelativeSize = obj->getWindowBoundingBox();
- rect->set_x(windowRelativeSize.mTopLeft.x);
- rect->set_y(windowRelativeSize.mTopLeft.y);
- rect->set_width(windowRelativeSize.width());
- rect->set_height(windowRelativeSize.height());
+ }
+ else
+#endif
+ {
+ ::aurum::ReqGetSize_CoordType type = mRequest->type();
+ ObjectMapper *mObjMap = ObjectMapper::getInstance();
+ std::shared_ptr<UiObject> obj = mObjMap->getElement(mRequest->elementid());
+ if (obj) {
+ obj->updateExtents();
+ ::aurum::Rect *rect = mResponse->mutable_size();
+ if (type == ::aurum::ReqGetSize_CoordType::ReqGetSize_CoordType_SCREEN) {
+ const Rect<int> &size = obj->getScreenBoundingBox();
+ rect->set_x(size.mTopLeft.x);
+ rect->set_y(size.mTopLeft.y);
+ rect->set_width(size.width());
+ rect->set_height(size.height());
+ }
+ else {
+ const Rect<int> &windowRelativeSize = obj->getWindowBoundingBox();
+ rect->set_x(windowRelativeSize.mTopLeft.x);
+ rect->set_y(windowRelativeSize.mTopLeft.y);
+ rect->set_width(windowRelativeSize.width());
+ rect->set_height(windowRelativeSize.height());
+ }
}
}
return grpc::Status::OK;
-}
\ No newline at end of file
+}
BuildRequires: pkgconfig(capi-base-utils-i18n)
BuildRequires: pkgconfig(vconf)
+%if "%{mqtt}" == "1"
+BuildRequires: pkgconfig(libmosquitto)
+BuildRequires: pkgconfig(jsoncpp)
+%endif
+
%if 0%{?gendoc:1}
BuildRequires: doxygen
%endif
Ui Automation Library Aurum gcov objects
%endif
-
%prep
%setup -q
cp %{SOURCE1001} .
-
%if "%{asan}" == "1"
%restore_fcommon
%else
export LDFLAGS+="-Wl,-z,noexecstack"
%endif
-
%if 0%{?gcov:1}
export CFLAGS+=" -fprofile-arcs -ftest-coverage "
export CXXFLAGS+=" -fprofile-arcs -ftest-coverage "
%else
%define TIZEN_GEN_DOC false
%endif
+
+%if "%{mqtt}" == "1"
+%define MQTT_ENABLED true
+%else
+%define MQTT_ENABLED false
+%endif
+
meson \
--prefix /usr \
--libdir %{_libdir} \
-Dcpp_std=c++17 \
-Dtizen=true \
-Denable_documentation=%{TIZEN_GEN_DOC} \
+ -Dmqtt_enabled=%{MQTT_ENABLED} \
-Dtizen_gcov=%{TIZEN_GCOV} \
-Dtzapp_path=%{TZ_SYS_RO_APP} \
-Dtzpackage_path=%{TZ_SYS_RO_PACKAGES} \
rpc actionAndWaitEvent(ReqActionAndWaitEvent) returns (RspActionAndWaitEvent) {}
rpc setFocus(ReqSetFocus) returns (RspSetFocus) {}
rpc findElements(ReqFindElements) returns (RspFindElements) {}
+ rpc getActiveAppToolkitName(ReqGetActiveAppToolkitName) returns (RspGetActiveAppToolkitName) {}
+ rpc enableScreenAnalyzer(ReqEnableScreenAnalyzer) returns (RspEnableScreenAnalyzer) {}
}
// ------------------------------------ //
string text = 7;
string xpath = 8;
- string automationId = 9;
- string package = 10;
- string role = 11;
-
- bool isChecked = 12;
- bool isCheckable = 13;
- bool isClickable = 14;
- bool isEnabled = 15;
- bool isFocused = 16;
- bool isFocusable = 17;
- bool isScrollable = 18;
- bool isSelected = 19;
- bool isShowing = 20;
- bool isActive = 21;
- bool isVisible = 22;
- bool isSelectable = 23;
-
- double minValue = 24;
- double maxValue = 25;
- double value = 26;
- double increment = 27;
+ string ocrText = 9;
+ string automationId = 10;
+ string package = 11;
+ string role = 12;
+ string toolkit = 13;
+
+ bool isChecked = 14;
+ bool isCheckable = 15;
+ bool isClickable = 16;
+ bool isEnabled = 17;
+ bool isFocused = 18;
+ bool isFocusable = 19;
+ bool isScrollable = 20;
+ bool isSelected = 21;
+ bool isShowing = 22;
+ bool isActive = 23;
+ bool isVisible = 24;
+ bool isSelectable = 25;
+
+ double minValue = 26;
+ double maxValue = 27;
+ double value = 28;
+ double increment = 29;
}
message Point {
string xpath = 20;
}
- repeated ReqFindElement children = 21;
+ oneof _ocrtext {
+ string ocrText = 21;
+ }
+
+ repeated ReqFindElement children = 22;
}
message RspFindElement {
string xpath = 20;
}
- repeated ReqFindElements children = 21;
+ oneof _ocrtext {
+ string ocrText = 21;
+ }
+ repeated ReqFindElements children = 22;
}
message RspFindElements {
message RspSetFocus {
RspStatus status = 1;
}
+
+message ReqGetActiveAppToolkitName {
+}
+
+message RspGetActiveAppToolkitName {
+ RspStatus status = 1;
+ string toolkitName = 2;
+}
+
+message ReqEnableScreenAnalyzer {
+ bool enable = 1;
+ string serverIp = 2;
+}
+
+message RspEnableScreenAnalyzer {
+ RspStatus status = 1;
+}
+
ASSERT_EQ(parent->getAutomationId(), "");
}
-TEST_F(AurumTestUiObject, getElementType_P1)
+TEST_F(AurumTestUiObject, getType_P1)
{
auto obj = UiDevice::getInstance();
auto parent = obj->findObject(Sel::text("test2"));
- ASSERT_EQ(parent->getElementType(), "type");
+ ASSERT_EQ(parent->getType(), "type");
}
TEST_F(AurumTestUiObject, getElementStyle_P1)
sel->text("win1");
auto found = UiDevice::getInstance()->findObject(sel);
- ASSERT_EQ(found->getElementType(), "Elm_Win");
+ ASSERT_EQ(found->getType(), "Elm_Win");
auto children = found->getChildren();
ASSERT_EQ(children.size(), 4);
sel->text("win1");
auto found = UiDevice::getInstance()->findObject(sel);
- ASSERT_EQ(found->getElementType(), "Elm_Win");
+ ASSERT_EQ(found->getType(), "Elm_Win");
auto children = found->getChildren();
ASSERT_EQ(children.size(), 4);
sel->text("win1");
auto found = UiDevice::getInstance()->findObject(sel);
- ASSERT_EQ(found->getElementType(), "Elm_Win");
+ ASSERT_EQ(found->getType(), "Elm_Win");
auto children = found->getChildren();
ASSERT_EQ(children.size(), 4);
projects / platform / core / uifw / aurum.git / commitdiff