libaurum: Get window info from window system 64/278564/6 accepted/tizen/unified/20220726.132043 submit/tizen/20220725.064214
authorWoochanlee <wc0917.lee@samsung.com>
Fri, 22 Jul 2022 04:20:27 +0000 (13:20 +0900)
committerWoochanlee <wc0917.lee@samsung.com>
Mon, 25 Jul 2022 06:21:56 +0000 (15:21 +0900)
We uesed to atspi window event to search target window.
It can't support multi window situation and something wrong state has window as well.

Change-Id: I0a3f61104402332fb6d8feb74e13d11323cc8401

15 files changed:
libaurum/inc/Accessibility/AccessibleNode.h
libaurum/inc/Aurum.h
libaurum/inc/Impl/Accessibility/AtspiAccessibleNode.h
libaurum/inc/Impl/Accessibility/MockAccessibleNode.h
libaurum/inc/TizenWindow.h [new file with mode: 0644]
libaurum/inc/UiDevice.h
libaurum/inc/UiObject.h
libaurum/meson.build
libaurum/src/Accessibility/AccessibleNode.cc
libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc
libaurum/src/Impl/Accessibility/MockAccessibleNode.cc
libaurum/src/TizenWindow.cc [new file with mode: 0644]
libaurum/src/UiDevice.cc
libaurum/src/UiObject.cc
libaurum/src/meson.build

index 9a44612560f333137eecde4717a22cf480d4bfe2..5b9179a054cb9604d3611262b03b6b4fbab025a0 100644 (file)
@@ -217,6 +217,12 @@ public:
      */
     double getValue() const;
 
+    /**
+     * @copydoc UiObject::getPid()
+     *
+     */
+    int getPid() const;
+
     /**
      * @copydoc UiObject::getIncrement()
      *
@@ -362,6 +368,11 @@ public:
      */
     virtual void updateValue() = 0;
 
+    /**
+     * @copydoc UiObject::updatePid()
+     */
+    virtual void updatePid() = 0;
+
     /**
      * @copydoc UiObject::setFocus()
      */
@@ -478,6 +489,7 @@ protected:
     Rect<int> mWindowBoundingBox;
     int mSupportingIfaces;
     int mFeatureProperty;
+    int mPid;
     double mMinValue;
     double mMaxValue;
     double mValue;
index 52bb8b271167a2e16b87fb71485377a5a7287ea0..0003c39fdfc8c951067ea7dbda235c5f00d00210 100644 (file)
@@ -47,6 +47,7 @@
 #include "UiDevice.h"
 #include "UiObject.h"
 #include "UiSelector.h"
+#include "TizenWindow.h"
 #include "Sel.h"
 #include "Until.h"
 #include "Waiter.h"
index 2724c933e7eddd29c1822f2edf6bb67f2ab86bcb..5b8a3b6030996f6342935b11366f1701fac331f7 100644 (file)
@@ -126,6 +126,11 @@ public:
      */
     void updateValue() override;
 
+    /**
+     * @copydoc UiObject::updatePid()
+     */
+    void updatePid() override;
+
     /**
      * @copydoc UiObject::setFocus()
      */
index be6ce320755b269904721c602d840225c49b9d8a..68ef29427c3714c6097f9fbf344ce691af38db18 100644 (file)
@@ -129,6 +129,12 @@ public:
      */
     void updateValue() override;
 
+    /**
+     * @brief TBD
+     * @since_tizen 7.0
+     */
+    void updatePid() override;
+
     /**
      * @brief TBD
      * @since_tizen 7.0
diff --git a/libaurum/inc/TizenWindow.h b/libaurum/inc/TizenWindow.h
new file mode 100644 (file)
index 0000000..4c86227
--- /dev/null
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *               http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+#ifndef _TIZEN_WINDOW_H_
+#define _TIZEN_WINDOW_H_
+
+#include <gio/gio.h>
+
+namespace Aurum {
+
+class TizenWindow : public std::enable_shared_from_this<TizenWindow> {
+public:
+    TizenWindow(int pid, Rect<int> geometry, bool transformed, bool alpha, int opaque, int visibility, bool focused, bool mapped, int layer, std::string name);
+    ~TizenWindow();
+
+    int getPid();
+    Rect<int> getWindowGeometry();
+    bool isTransformed();
+    bool isAlpah();
+    int getOpaque();
+    int getVisibility();
+    bool isFocused();
+    bool isMapped();
+    std::string getName();
+private:
+    int mPid;                   // window's PID
+    Rect<int> mWindowGeometry;  // window's geometry
+    bool mIsTransformed;        // true : transformed window
+    bool mIsAlpha;              // true : alpha
+    int mOpaque;                // true : opaque state set window
+    int mVisibility;            // 0  : visible
+                                // 1  : partial visible
+                                // 2  : hidden
+                                // -1 : unknown
+    bool mIsFocused;            // true : has focus
+    bool mIsMapped;             // true : mapped
+    int mLayer;                 // 200       : normal layer
+                                // 250       : above layer
+                                // 650 ~ 800 : notification layer
+    std::string mName;          // window name
+};
+
+}
+
+#endif
index 669e0b7857608b16c0be09782a0c1b21d7d7106d..ba21d4a69cc73a9d1a00a54a7f8928cab5fe1701 100644 (file)
@@ -22,6 +22,7 @@
 
 #include "UiObject.h"
 #include "UiSelector.h"
+#include "TizenWindow.h"
 
 #include "IDevice.h"
 #include "ISearchable.h"
@@ -369,6 +370,15 @@ public:
      * @since_tizen 6.5
      */
     std::vector<std::shared_ptr<AccessibleNode>> getWindowRoot() const;
+
+    /**
+     * @brief Gets currently window information from window system.
+     *
+     * @return TizenWindow ptr vector
+     * @since_tizen 7.0
+     */
+    std::vector<std::shared_ptr<TizenWindow>> getTizenWindowInfo() const;
+
 private:
     /**
      * @brief Waits process idle.
@@ -403,6 +413,7 @@ public:
 private:
     IDevice *mDeviceImpl;
     const Waiter *mWaiter;
+    static std::vector<std::shared_ptr<TizenWindow>> mTizenWindows;
 };
 
 }
index 98c3a6cde319ed3eeacd1466d989d16a7dc78c52..018d1ad2bfefe4473d9286196edae706efaec351 100644 (file)
@@ -329,6 +329,15 @@ public:
      */
     const double getValue() const;
 
+    /**
+     * @brief Gets object's process id.
+     *
+     * @return int
+     *
+     * @since_tizen 7.0
+     */
+    const int getPid() const;
+
     /**
      * @brief Gets object's current increment.
      *
@@ -550,6 +559,13 @@ public:
      */
     void updateValue() const;
 
+    /**
+     * @brief Updates object's process id.
+     *
+     * @since_tizen 7.0
+     */
+    void updatePid() const;
+
     /**
      * @brief Sets focus to object.
      *
index 5440ee7ec7970db1c821de9396cc356ec02e19ac..66c87a05965ba6037de045463b4225ff42b9d3e1 100644 (file)
@@ -4,6 +4,7 @@ libaurum_install_inc = [
   './inc/UiDevice.h',
   './inc/UiObject.h',
   './inc/UiSelector.h',
+  './inc/TizenWindow.h',
   './inc/Sel.h',
   './inc/Until.h',
   './inc/Waiter.h',
index 4fc7fcf0f9a8acc44c03d0f16550d0d8fb866d58..6e8eb02bc69e200e8e3b46cd719a68d6cb591528 100644 (file)
@@ -248,4 +248,9 @@ double AccessibleNode::getValue() const
 double AccessibleNode::getIncrement() const
 {
     return mIncrement;
-}
\ No newline at end of file
+}
+
+int AccessibleNode::getPid() const
+{
+    return mPid;
+}
index 5d9e922ebc6966b0c0934267f944724029b8c524..8d04ab31c12ed0c8071fb7dd75e0ac6fe4109839 100644 (file)
@@ -254,6 +254,13 @@ void AtspiAccessibleNode::updateValue()
     }
 }
 
+void AtspiAccessibleNode::updatePid()
+{
+    AtspiWrapper::Atspi_accessible_clear_cache(mNode);
+
+    mPid = AtspiWrapper::Atspi_accessible_get_process_id(mNode, NULL);
+}
+
 bool AtspiAccessibleNode::setFocus()
 {
     AtspiComponent *component = AtspiWrapper::Atspi_accessible_get_component_iface(mNode);
index 51ba1937b6f2db93cb15b952012032e886656884..8900dd85bfa22169d83074b1af84691fcee07d22 100644 (file)
@@ -117,6 +117,10 @@ void MockAccessibleNode::updateValue()
 {
 }
 
+void MockAccessibleNode::updatePid()
+{
+}
+
 bool MockAccessibleNode::setFocus()
 {
     return false;
diff --git a/libaurum/src/TizenWindow.cc b/libaurum/src/TizenWindow.cc
new file mode 100644 (file)
index 0000000..ae19068
--- /dev/null
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *               http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+#include "Aurum.h"
+#include "TizenWindow.h"
+
+using namespace Aurum;
+
+TizenWindow::TizenWindow(int pid, Rect<int> geometry, bool transformed, bool alpha, int opaque, int visibility, bool focused, bool mapped, int layer, std::string name)
+    : mPid(pid), mWindowGeometry(geometry), mIsTransformed(transformed), mIsAlpha(alpha), mOpaque(opaque), mVisibility(visibility), mIsFocused(focused), mIsMapped(mapped), mLayer(layer), mName(name)
+{
+
+}
+
+TizenWindow::~TizenWindow()
+{
+
+}
+
+int TizenWindow::getPid()
+{
+    return mPid;
+}
+
+Rect<int> TizenWindow::getWindowGeometry()
+{
+    return mWindowGeometry;
+}
+
+bool TizenWindow::isTransformed()
+{
+    return mIsTransformed;
+}
+
+bool TizenWindow::isAlpah()
+{
+    return mIsAlpha;
+}
+
+int TizenWindow::getOpaque()
+{
+    return mOpaque;
+}
+
+int TizenWindow::getVisibility()
+{
+    return mVisibility;
+}
+
+bool TizenWindow::isFocused()
+{
+    return mIsFocused;
+}
+
+bool TizenWindow::isMapped()
+{
+    return mIsMapped;
+}
+
+std::string TizenWindow::getName()
+{
+    return mName;
+}
\ No newline at end of file
index 15ab27f0482fae58e8d1d06462b0635fe9d083fc..ea7607358cc4838394bd38ec024966a4cac97f4f 100644 (file)
 #include <thread>
 #include <algorithm>
 #include <iostream>
+#include <gio/gio.h>
 
 using namespace Aurum;
 using namespace AurumInternal;
 
+#define WM_BUS_NAME "org.enlightenment.wm"
+#define WM_OBJECT_PATH "/org/enlightenment/wm"
+#define WM_INTERFACE_NAME "org.enlightenment.wm.proc"
+#define WM_METHOD_NAME_INFO "GetVisibleWinInfo_v2"
+
+std::vector<std::shared_ptr<TizenWindow>> UiDevice::mTizenWindows;
+static GDBusConnection *system_conn;
+
 UiDevice::UiDevice() : UiDevice(nullptr) {}
 
 UiDevice::UiDevice(IDevice *impl)
@@ -66,21 +75,154 @@ std::shared_ptr<UiDevice> UiDevice::getInstance(IDevice *deviceImpl)
     return device;
 }
 
+std::vector<std::shared_ptr<TizenWindow>> UiDevice::getTizenWindowInfo() const
+{
+    GError *err = NULL;
+    GDBusMessage *msg;
+    GDBusMessage *reply;
+    GDBusConnection *conn;
+    GVariant *body;
+    GVariantIter *iter = NULL;
+    int idx = 0;
+    int pid;
+    int x;
+    int y;
+    int w;
+    int h;
+    gboolean transformed;
+    gboolean alpha;
+    int opaque;
+    int visibility;
+    gboolean focused;
+    gboolean mapped;
+    int layer;
+    char *name;
+
+    mTizenWindows.clear();
+
+    if (system_conn == NULL) {
+        conn = g_bus_get_sync(G_BUS_TYPE_SYSTEM, NULL, &err);
+        if (conn == NULL) {
+            LOGE("g_bus_get_sync() is failed. %s", err->message);
+            g_error_free(err);
+            return mTizenWindows;
+        }
+        system_conn = conn;
+    }
+
+    msg = g_dbus_message_new_method_call(WM_BUS_NAME,
+            WM_OBJECT_PATH,
+            WM_INTERFACE_NAME,
+            WM_METHOD_NAME_INFO);
+    if (msg == NULL) {
+        LOGE("g_dbus_message_new_method_call() is failed.");
+        return mTizenWindows;
+    }
+
+    reply = g_dbus_connection_send_message_with_reply_sync(system_conn, msg,
+            G_DBUS_SEND_MESSAGE_FLAGS_NONE, -1, NULL, NULL, &err);
+
+    if (!reply) {
+        if (err != NULL) {
+            LOGE("Failed to get info [%s]", err->message);
+            g_error_free(err);
+        }
+        goto out;
+    }
+
+    body = g_dbus_message_get_body(reply);
+    if (!body) {
+        LOGE("Failed to get body");
+        goto out;
+    }
+
+    g_variant_get(body, "(a(iiiiibbiibbis))", &iter);
+
+    LOGI("%-3s | %-6s | %-4s | %-4s | %-4s | %-4s | %-5s | %-5s | %-6s | %-3s | %-7s | %-6s | %-5s | %-20s", "No" ,"PID", "X", "Y", "W", "H", "Trans", "Alpha", "Opaque", "Vis", "Focused", "Mapped", "Layer", "Name");
+    while (g_variant_iter_loop(iter, "(iiiiibbiibbis)",
+                &pid,
+                &x,
+                &y,
+                &w,
+                &h,
+                &transformed,
+                &alpha,
+                &opaque,
+                &visibility,
+                &focused,
+                &mapped,
+                &layer,
+                &name)) {
+        LOGI("%-3d | %-6d | %-4d | %-4d | %-4d | %-4d | %-5d | %-5d | %-6d | %-3d | %-7d | %-6d | %-5d | %-20s", idx++, pid, x,y,w,h, transformed, alpha, opaque, visibility, focused, mapped, layer, name);
+        if (visibility == 0 && pid > 0)
+        {
+            Rect<int> geometry = {x,  y, w, h};
+            std::string winName(name);
+            mTizenWindows.push_back(std::make_shared<Aurum::TizenWindow>(pid, geometry, transformed, alpha, opaque, visibility, focused, mapped, layer, winName));
+        }
+    }
+
+    if (iter)
+        g_variant_iter_free(iter);
+out:
+    if (msg)
+        g_object_unref(msg);
+    if (reply)
+        g_object_unref(reply);
+
+    return mTizenWindows;
+}
+
 std::vector<std::shared_ptr<AccessibleNode>> UiDevice::getWindowRoot() const
 {
+    bool dup;
+    LOGI("Request window info");
+    getTizenWindowInfo();
+
     std::vector<std::shared_ptr<AccessibleNode>> ret{};
 
-    auto appsMap = AccessibleWatcher::getInstance()->getActiveAppMap();
-    LOGI("activeAppMap.size: %d" , (int)appsMap.size());
-    for (auto itr = appsMap.begin(); itr != appsMap.end(); itr++)
+    auto apps = AccessibleWatcher::getInstance()->getApplications();
+    for (auto app : apps)
+    {
+        app->getAccessibleNode()->updateName();
+        app->getAccessibleNode()->updatePid();
+        LOGI("App(%s) Pid(%d)", app->getPackageName().c_str(), app->getAccessibleNode()->getPid());
+    }
+
+    for (auto tWin : mTizenWindows)
     {
-        auto activeWindows = itr->second->getActiveWindows();
-        std::transform(activeWindows.begin(), activeWindows.end(), std::back_inserter(ret),
-            [&](std::shared_ptr<AccessibleWindow> window){
-                LOGI("active pkg: %s, window: %s", window->getAccessibleNode()->getPkg().c_str(), window->getTitle().c_str());
-                return window->getAccessibleNode();
+        LOGI("Visible win (%d) (%d %d %d %d) (%s)", tWin->getPid(), tWin->getWindowGeometry().mTopLeft.x, tWin->getWindowGeometry().mTopLeft.y, tWin->getWindowGeometry().width(),
+            tWin->getWindowGeometry().height(), tWin->getName().c_str());
+
+        for (auto app : apps)
+        {
+            dup = false;
+            if (app->getAccessibleNode()->getPid() == tWin->getPid())
+            {
+                for (const auto &retWin : ret)
+                {
+                    retWin->getParent()->updatePid();
+                    LOGI("Pid Dup check in vector (%d) target (%d)", retWin->getParent()->getPid(), tWin->getPid());
+                    if (retWin->getParent()->getPid() == tWin->getPid())
+                    {
+                        dup = true;
+                        break;
+                    }
+                }
+
+                if (!dup)
+                {
+                    LOGI("Actvie App : (%s) (%d)", tWin->getName().c_str(), tWin->getPid());
+                    auto wins = app->getWindows();
+                    std::transform(wins.begin(), wins.end(), std::back_inserter(ret),
+                        [&](std::shared_ptr<AccessibleWindow> window){
+                            LOGI("Target window add pkg: (%s), name (%s)", window->getAccessibleNode()->getPkg().c_str(), window->getTitle().c_str());
+                            return window->getAccessibleNode();
+                        }
+                    );
+                }
             }
-        );
+        }
     }
 
     return ret;
@@ -143,19 +285,19 @@ bool UiDevice::waitForIdle() const
 }
 
 bool UiDevice::waitForEvents(
-       const A11yEvent type, const int timeout) const
+    const A11yEvent type, const int timeout) const
 {
     return executeAndWaitForEvents(NULL, type, timeout);
 }
 
 bool UiDevice::executeAndWaitForEvents(
-       const Runnable *cmd, const A11yEvent type, const int timeout) const
+    const Runnable *cmd, const A11yEvent type, const int timeout) const
 {
     return AccessibleWatcher::getInstance()->executeAndWaitForEvents(cmd, type, timeout);
 }
 
 bool UiDevice::sendKeyAndWaitForEvents(
-       const std::string keycode, const A11yEvent type, const int timeout) const
+    const std::string keycode, const A11yEvent type, const int timeout) const
 {
     std::unique_ptr<SendKeyRunnable> cmd = std::make_unique<SendKeyRunnable>(keycode);
     return executeAndWaitForEvents(cmd.get(), type, timeout);
index 51ac7902f1fd4849fffd0fbcf27a19aeb3a6bcfb..fe54f4f02b00662fa2587473d7838c371c675ac4 100644 (file)
@@ -344,6 +344,11 @@ void UiObject::updateValue() const
     mNode->updateValue();
 }
 
+void UiObject::updatePid() const
+{
+    mNode->updatePid();
+}
+
 bool UiObject::setFocus() const
 {
     return mNode->setFocus();
index 11af3eae4156578c5a9d9cc12a05fcd084c848a7..390e805dfde0cbb52f7588aa3a09c7492f3f6eeb 100644 (file)
@@ -10,6 +10,7 @@ libaurum_src += [
   files('PartialMatch.cc'),
   files('A11yEvent.cc'),
   files('AurumXML.cc'),
+  files('TizenWindow.cc'),
 ]
 
 subdir('Accessibility')