ScreenAnalyzer: Support compatibility Atspi with Screen Analyzer in command (FR9)

Change-Id: Ic0b5af59f16f219e684686494cc30bdce7ebf4af

diff --git a/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc
index dfcda80..ee5d5c0 100644 (file)
--- a/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc
+++ b/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc
@@ -96,9 +96,72 @@ void DumpObjectTreeCommand::traverse(::aurum::Element *root, std::shared_ptr<Nod
         auto obj = mObjMap->getElement(mRequest->elementid());
         if (!obj) return grpc::Status::OK;;
 
-        auto node = obj->getDescendant();
+    if (mDevice->getExternalAppLaunched())
+    {
+        struct tm timeinfo;
+        time_t now = time(0);
+        if (!localtime_r(&now, &timeinfo)) {
+            LOGE("fail to get localtime. Screenshot cancelled");
+            return grpc::Status::CANCELLED;
+        }
+        char name[128];
+        std::snprintf(name, 128, "/tmp/screenshot-%d-%d-%d-%d:%d:%d.png",
+                                (timeinfo.tm_year + 1900), (timeinfo.tm_mon + 1), timeinfo.tm_mday,
+                                 timeinfo.tm_hour, timeinfo.tm_min, timeinfo.tm_sec);
+        std::string path(name);
+        std::shared_ptr<UiDevice> mDevice = UiDevice::getInstance();
+        mDevice->takeScreenshot(path, 1.0, 1);
+
+        auto objs = mDevice->getScw()->GetSaObjects();
+
         ::aurum::Element *root = mResponse->add_roots();
-        traverse(root, node, 0);
+        root->set_elementid("Root");
+        ::aurum::Rect *rect = root->mutable_geometry();
+        rect->set_x(0);
+        rect->set_y(0);
+        rect->set_width(1920);
+        rect->set_height(1080);
+        root->set_widget_type("window");
+        root->set_isshowing(true);
+        root->set_isactive(true);
+        root->set_isvisible(true);
+
+        for (auto obj : objs) {
+            ::aurum::Element *elm = root->add_child();
+            elm->set_elementid(obj->getId());
+
+            ::aurum::Rect *rect = elm->mutable_geometry();
+            const Rect<int> &size = obj->getScreenBoundingBox();
+            rect->set_x(size.mTopLeft.x);
+            rect->set_y(size.mTopLeft.y);
+            rect->set_width(size.width());
+            rect->set_height(size.height());
+
+            elm->set_widget_type(obj->getElementType());
+
+            elm->set_text(obj->getOcrText());
+
+            elm->set_isclickable(obj->isClickable());
+            elm->set_isfocused(obj->isFocused());
+            elm->set_isfocusable(obj->isFocusable());
+            elm->set_isshowing(true);
+            elm->set_isactive(true);
+            elm->set_isvisible(true);
+        }
+
+        LOGE("WCC DumpObject Finish");
+    }
+    else
+    {
+        LOGI("elementid : %s", mRequest->elementid().c_str());
+        if (mRequest->elementid().length()) {
+            auto obj = mObjMap->getElement(mRequest->elementid());
+            if (!obj) return grpc::Status::OK;;
+
+            auto node = obj->getDescendant();
+            ::aurum::Element *root = mResponse->add_roots();
+            traverse(root, node, 0);
+        }
     }
     return grpc::Status::OK;
 }

diff --git a/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc
index 929119d..c6073af 100644 (file)
--- a/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc
+++ b/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc
@@ -19,6 +19,7 @@
 #include "FindElementCommand.h"
 #include "UiObject.h"
 #include "UiDevice.h"
+#include "SaObject.h"
 #include "UiSelector.h"
 #include "Sel.h"
 #include "ISearchable.h"
@@ -75,65 +76,121 @@ std::shared_ptr<UiSelector> FindElementCommand::getSelector(void)
     auto searchableObj = getSearchableTop();
     auto selector     = getSelector();
 
-    auto found = searchableObj->findObject(selector);
-
-    if (found != nullptr) {
-        UiObject *obj = found.get();
-        obj->refresh();
-        if (mObjMap->getElement(obj->getId()) == nullptr)
-            mObjMap->addElement(std::move(found));
-
-        LOGI("found object : %p elementId:%s", obj, obj->getId().c_str());
-
-        ::aurum::Element *elm = mResponse->mutable_element();
-        elm->set_elementid(obj->getId());
-        elm->set_package(obj->getApplicationPackage());
-
-        ::aurum::Rect *rect = elm->mutable_geometry();
-        const Rect<int> &size = obj->getScreenBoundingBox();
-        rect->set_x(size.mTopLeft.x);
-        rect->set_y(size.mTopLeft.y);
-        rect->set_width(size.width());
-        rect->set_height(size.height());
-
-        ::aurum::Rect *windowRect = elm->mutable_window_relative_geometry();
-        const Rect<int> &windowRelativeSize = obj->getWindowBoundingBox();
-        windowRect->set_x(windowRelativeSize.mTopLeft.x);
-        windowRect->set_y(windowRelativeSize.mTopLeft.y);
-        windowRect->set_width(windowRelativeSize.width());
-        windowRect->set_height(windowRelativeSize.height());
-
-        elm->set_widget_type(obj->getElementType());
-        elm->set_widget_style(obj->getElementStyle());
-
-        elm->set_text(obj->getText());
-        elm->set_xpath(obj->getXPath());
-        elm->set_automationid(obj->getAutomationId());
-        elm->set_package(obj->getApplicationPackage());
-        elm->set_role(obj->getRole());
-
-        elm->set_ischecked(obj->isChecked());
-        elm->set_ischeckable(obj->isCheckable());
-        elm->set_isclickable(obj->isClickable());
-        elm->set_isenabled(obj->isEnabled());
-        elm->set_isfocused(obj->isFocused());
-        elm->set_isfocusable(obj->isFocusable());
-        elm->set_isscrollable(obj->isScrollable());
-        elm->set_isselected(obj->isSelected());
-        elm->set_isshowing(obj->isShowing());
-        elm->set_isactive(obj->isActive());
-        elm->set_isvisible(obj->isVisible());
-        elm->set_isselectable(obj->isSelectable());
-
-        elm->set_minvalue(obj->getMinValue());
-        elm->set_maxvalue(obj->getMaxValue());
-        elm->set_value(obj->getValue());
-        elm->set_increment(obj->getIncrement());
-
-        mResponse->set_status(::aurum::RspStatus::OK);
-    } else {
-        mResponse->set_status(::aurum::RspStatus::ERROR);
+    if (mDevice->getExternalAppLaunched())
+    {
+        struct tm timeinfo;
+        time_t now = time(0);
+        if (!localtime_r(&now, &timeinfo)) {
+            LOGE("fail to get localtime. Screenshot cancelled");
+            return grpc::Status::CANCELLED;
+        }
+        char name[128];
+        std::snprintf(name, 128, "/tmp/screenshot-%d-%d-%d-%d:%d:%d.png",
+                                (timeinfo.tm_year + 1900), (timeinfo.tm_mon + 1), timeinfo.tm_mday,
+                                timeinfo.tm_hour, timeinfo.tm_min, timeinfo.tm_sec);
+        std::string path(name);
+        std::shared_ptr<UiDevice> mDevice = UiDevice::getInstance();
+        mDevice->RequestScreenAnalyze(path);
+
+        std::vector<std::shared_ptr<SaObject>> founds = {};
+
+        LOGE("WCC Search Object start");
+        for ( auto &sel : selectors ) {
+            auto ret = mDevice->getScw()->findSaObjects(sel);
+            std::move(std::begin(ret), std::end(ret), std::back_inserter(founds));
+        }
+        if (founds.size() > 0) {
+            for (auto& found : founds) {
+                SaObject *obj = found.get();
+                ::aurum::Element *elm = mResponse->add_elements();
+                elm->set_elementid(obj->getId());
+
+                ::aurum::Rect *rect = elm->mutable_geometry();
+                const Rect<int> &size = obj->getScreenBoundingBox();
+                rect->set_x(size.mTopLeft.x);
+                rect->set_y(size.mTopLeft.y);
+                rect->set_width(size.width());
+                rect->set_height(size.height());
+
+                elm->set_widget_type(obj->getElementType());
+
+                elm->set_text(obj->getOcrText());
+
+                elm->set_isclickable(obj->isClickable());
+                elm->set_isfocused(obj->isFocused());
+                elm->set_isfocusable(obj->isFocusable());
+                elm->set_isshowing(true);
+                elm->set_isactive(true);
+                elm->set_isvisible(true);
+            }
+                mResponse->set_status(::aurum::RspStatus::OK);
+        }
     }
-
-    return grpc::Status::OK;
+       else
+       {
+               auto searchableObj = getSearchableTop();
+               auto selectors     = getSelectors();
+
+               auto found = searchableObj->findObject(selector);
+
+               if (found != nullptr) {
+                       UiObject *obj = found.get();
+                       obj->refresh();
+                       if (mObjMap->getElement(obj->getId()) == nullptr)
+                               mObjMap->addElement(std::move(found));
+
+                       LOGI("found object : %p elementId:%s", obj, obj->getId().c_str());
+
+                       ::aurum::Element *elm = mResponse->mutable_element();
+                       elm->set_elementid(obj->getId());
+                       elm->set_package(obj->getApplicationPackage());
+
+                       ::aurum::Rect *rect = elm->mutable_geometry();
+                       const Rect<int> &size = obj->getScreenBoundingBox();
+                       rect->set_x(size.mTopLeft.x);
+                       rect->set_y(size.mTopLeft.y);
+                       rect->set_width(size.width());
+                       rect->set_height(size.height());
+
+                       ::aurum::Rect *windowRect = elm->mutable_window_relative_geometry();
+                       const Rect<int> &windowRelativeSize = obj->getWindowBoundingBox();
+                       windowRect->set_x(windowRelativeSize.mTopLeft.x);
+                       windowRect->set_y(windowRelativeSize.mTopLeft.y);
+                       windowRect->set_width(windowRelativeSize.width());
+                       windowRect->set_height(windowRelativeSize.height());
+
+                       elm->set_widget_type(obj->getElementType());
+                       elm->set_widget_style(obj->getElementStyle());
+
+                       elm->set_text(obj->getText());
+                       elm->set_xpath(obj->getXPath());
+                       elm->set_automationid(obj->getAutomationId());
+                       elm->set_package(obj->getApplicationPackage());
+                       elm->set_role(obj->getRole());
+
+                       elm->set_ischecked(obj->isChecked());
+                       elm->set_ischeckable(obj->isCheckable());
+                       elm->set_isclickable(obj->isClickable());
+                       elm->set_isenabled(obj->isEnabled());
+                       elm->set_isfocused(obj->isFocused());
+                       elm->set_isfocusable(obj->isFocusable());
+                       elm->set_isscrollable(obj->isScrollable());
+                       elm->set_isselected(obj->isSelected());
+                       elm->set_isshowing(obj->isShowing());
+                       elm->set_isactive(obj->isActive());
+                       elm->set_isvisible(obj->isVisible());
+                       elm->set_isselectable(obj->isSelectable());
+
+                       elm->set_minvalue(obj->getMinValue());
+                       elm->set_maxvalue(obj->getMaxValue());
+                       elm->set_value(obj->getValue());
+                       elm->set_increment(obj->getIncrement());
+
+                       mResponse->set_status(::aurum::RspStatus::OK);
+               } else {
+                       mResponse->set_status(::aurum::RspStatus::ERROR);
+               }
+       }
+
+       return grpc::Status::OK;
 }

diff --git a/org.tizen.aurum-bootstrap/src/Commands/GetAttributeCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/GetAttributeCommand.cc
index 1afc58e..e990058 100644 (file)
--- a/org.tizen.aurum-bootstrap/src/Commands/GetAttributeCommand.cc
+++ b/org.tizen.aurum-bootstrap/src/Commands/GetAttributeCommand.cc
@@ -38,32 +38,42 @@ std::unique_ptr<GetAttributeCommand> GetAttributeCommand::createCommand(const ::
     ::aurum::ReqGetAttribute_RequestType type = request->attribute();
     LOGI("type : %d", type);
 
-    if (type == ::aurum::ReqGetAttribute_RequestType::ReqGetAttribute_RequestType_VISIBLE)
-        return std::make_unique<GetVisibleAttributeCommand>(request, response);
-    else if (type == ::aurum::ReqGetAttribute_RequestType::ReqGetAttribute_RequestType_FOCUSABLE)
-        return std::make_unique<GetFocusableAttributeCommand>(request, response);
-    else if (type == ::aurum::ReqGetAttribute_RequestType::ReqGetAttribute_RequestType_FOCUSED)
-        return std::make_unique<GetFocusedAttributeCommand>(request, response);
-    else if (type == ::aurum::ReqGetAttribute_RequestType::ReqGetAttribute_RequestType_ENABLED)
-        return std::make_unique<GetEnabledAttributeCommand>(request, response);
-    else if (type == ::aurum::ReqGetAttribute_RequestType::ReqGetAttribute_RequestType_CLICKABLE)
-        return std::make_unique<GetClickableAttributeCommand>(request, response);
-    else if (type == ::aurum::ReqGetAttribute_RequestType::ReqGetAttribute_RequestType_SCROLLABLE)
-        return std::make_unique<GetScrollableAttributeCommand>(request, response);
-    else if (type == ::aurum::ReqGetAttribute_RequestType::ReqGetAttribute_RequestType_CHECKABLE)
-        return std::make_unique<GetCheckableAttributeCommand>(request, response);
-    else if (type == ::aurum::ReqGetAttribute_RequestType::ReqGetAttribute_RequestType_CHECKED)
-        return std::make_unique<GetCheckedAttributeCommand>(request, response);
-    else if (type == ::aurum::ReqGetAttribute_RequestType::ReqGetAttribute_RequestType_SELECTED)
-        return std::make_unique<GetSelectedAttributeCommand>(request, response);
-    else if (type == ::aurum::ReqGetAttribute_RequestType::ReqGetAttribute_RequestType_SELECTABLE)
-        return std::make_unique<GetSelectableAttributeCommand>(request, response);
-    else if (type == ::aurum::ReqGetAttribute_RequestType::ReqGetAttribute_RequestType_SHOWING)
-        return std::make_unique<GetShowingAttributeCommand>(request, response);
-    else if (type == ::aurum::ReqGetAttribute_RequestType::ReqGetAttribute_RequestType_ACTIVE)
-        return std::make_unique<GetActiveAttributeCommand>(request, response);
+    if (mDevice->getExternalAppLaunched()) 
+    {
+        //TBD
+    }
     else
-        return std::make_unique<GetAttributeCommand>(request, response);
+    {
+        ::aurum::ReqGetAttribute_RequestType type = request->attribute();
+        LOGI("type : %d", type);
+
+        if (type == ::aurum::ReqGetAttribute_RequestType::ReqGetAttribute_RequestType_VISIBLE)
+            return std::make_unique<GetVisibleAttributeCommand>(request, response);
+        else if (type == ::aurum::ReqGetAttribute_RequestType::ReqGetAttribute_RequestType_FOCUSABLE)
+            return std::make_unique<GetFocusableAttributeCommand>(request, response);
+        else if (type == ::aurum::ReqGetAttribute_RequestType::ReqGetAttribute_RequestType_FOCUSED)
+            return std::make_unique<GetFocusedAttributeCommand>(request, response);
+        else if (type == ::aurum::ReqGetAttribute_RequestType::ReqGetAttribute_RequestType_ENABLED)
+            return std::make_unique<GetEnabledAttributeCommand>(request, response);
+        else if (type == ::aurum::ReqGetAttribute_RequestType::ReqGetAttribute_RequestType_CLICKABLE)
+            return std::make_unique<GetClickableAttributeCommand>(request, response);
+        else if (type == ::aurum::ReqGetAttribute_RequestType::ReqGetAttribute_RequestType_SCROLLABLE)
+            return std::make_unique<GetScrollableAttributeCommand>(request, response);
+        else if (type == ::aurum::ReqGetAttribute_RequestType::ReqGetAttribute_RequestType_CHECKABLE)
+            return std::make_unique<GetCheckableAttributeCommand>(request, response);
+        else if (type == ::aurum::ReqGetAttribute_RequestType::ReqGetAttribute_RequestType_CHECKED)
+            return std::make_unique<GetCheckedAttributeCommand>(request, response);
+        else if (type == ::aurum::ReqGetAttribute_RequestType::ReqGetAttribute_RequestType_SELECTED)
+            return std::make_unique<GetSelectedAttributeCommand>(request, response);
+        else if (type == ::aurum::ReqGetAttribute_RequestType::ReqGetAttribute_RequestType_SELECTABLE)
+            return std::make_unique<GetSelectableAttributeCommand>(request, response);
+        else if (type == ::aurum::ReqGetAttribute_RequestType::ReqGetAttribute_RequestType_SHOWING)
+            return std::make_unique<GetShowingAttributeCommand>(request, response);
+        else if (type == ::aurum::ReqGetAttribute_RequestType::ReqGetAttribute_RequestType_ACTIVE)
+            return std::make_unique<GetActiveAttributeCommand>(request, response);
+        else
+            return std::make_unique<GetAttributeCommand>(request, response);
+    }
 }
 
 ::grpc::Status GetVisibleAttributeCommand::execute()

diff --git a/org.tizen.aurum-bootstrap/src/Commands/GetSizeCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/GetSizeCommand.cc
index dfada9c..6cbb34c 100644 (file)
--- a/org.tizen.aurum-bootstrap/src/Commands/GetSizeCommand.cc
+++ b/org.tizen.aurum-bootstrap/src/Commands/GetSizeCommand.cc
@@ -29,27 +29,64 @@ GetSizeCommand::GetSizeCommand(const ::aurum::ReqGetSize *request,
 {
     LOGI("GetSize --------------- ");
 
-    ::aurum::ReqGetSize_CoordType type = mRequest->type();
-    ObjectMapper *mObjMap = ObjectMapper::getInstance();
-    std::shared_ptr<UiObject> obj = mObjMap->getElement(mRequest->elementid());
-    if (obj) {
-        obj->updateExtents();
-        ::aurum::Rect *rect = mResponse->mutable_size();
-        if (type == ::aurum::ReqGetSize_CoordType::ReqGetSize_CoordType_SCREEN) {
-            const Rect<int> &size = obj->getScreenBoundingBox();
-            rect->set_x(size.mTopLeft.x);
-            rect->set_y(size.mTopLeft.y);
-            rect->set_width(size.width());
-            rect->set_height(size.height());
+    if (mDevice->getExternalAppLaunched()) 
+    {
+        struct tm timeinfo;
+        time_t now = time(0);
+        if (!localtime_r(&now, &timeinfo)) {
+            LOGE("fail to get localtime. Screenshot cancelled");
+            return grpc::Status::CANCELLED;
         }
-        else {
-            const Rect<int> &windowRelativeSize = obj->getWindowBoundingBox();
-            rect->set_x(windowRelativeSize.mTopLeft.x);
-            rect->set_y(windowRelativeSize.mTopLeft.y);
-            rect->set_width(windowRelativeSize.width());
-            rect->set_height(windowRelativeSize.height());
+        char name[128];
+        std::snprintf(name, 128, "/tmp/screenshot-%d-%d-%d-%d:%d:%d.png",
+                                (timeinfo.tm_year + 1900), (timeinfo.tm_mon + 1), timeinfo.tm_mday,
+                                timeinfo.tm_hour, timeinfo.tm_min, timeinfo.tm_sec);
+        std::string path(name);
+        std::shared_ptr<UiDevice> mDevice = UiDevice::getInstance();
+        mDevice->RequestScreenAnalyze(path);
+
+        auto sel = std::make_shared<UiSelector>();
+        LOGE("WCC Search Object start");
+        for ( auto &sel : selectors ) {
+            auto ret = mDevice->getScw()->findSaObjects(sel);
+            std::move(std::begin(ret), std::end(ret), std::back_inserter(founds));
+        }
+        if (founds.size() > 0) {
+            for (auto& found : founds) {
+                SaObject *obj = found.get();
+                ::aurum::Rect *rect = mResponse->mutable_size();
+                const Rect<int> &size = obj->getScreenBoundingBox();
+                rect->set_x(size.mTopLeft.x);
+                rect->set_y(size.mTopLeft.y);
+                rect->set_width(size.width());
+                rect->set_height(size.height());
+            }
         }
     }
-
+    else
+    {
+        ::aurum::ReqGetSize_CoordType type = mRequest->type();
+        ObjectMapper *mObjMap = ObjectMapper::getInstance();
+        std::shared_ptr<UiObject> obj = mObjMap->getElement(mRequest->elementid());
+        if (obj) {
+            obj->updateExtents();
+            ::aurum::Rect *rect = mResponse->mutable_size();
+            if (type == ::aurum::ReqGetSize_CoordType::ReqGetSize_CoordType_SCREEN) {
+                const Rect<int> &size = obj->getScreenBoundingBox();
+                rect->set_x(size.mTopLeft.x);
+                rect->set_y(size.mTopLeft.y);
+                rect->set_width(size.width());
+                rect->set_height(size.height());
+            }
+            else {
+                const Rect<int> &windowRelativeSize = obj->getWindowBoundingBox();
+                rect->set_x(windowRelativeSize.mTopLeft.x);
+                rect->set_y(windowRelativeSize.mTopLeft.y);
+                rect->set_width(windowRelativeSize.width());
+                rect->set_height(windowRelativeSize.height());
+            }
+        }
+    }
+    
     return grpc::Status::OK;
-}
\ No newline at end of file
+}