From: Woochanlee Date: Fri, 22 Jul 2022 04:20:27 +0000 (+0900) Subject: libaurum: Get window info from window system X-Git-Tag: submit/tizen/20220725.064214^0 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=a8e993422925b24c687a71c7f08f91aad47ddafc;p=platform%2Fcore%2Fuifw%2Faurum.git libaurum: Get window info from window system We uesed to atspi window event to search target window. It can't support multi window situation and something wrong state has window as well. Change-Id: I0a3f61104402332fb6d8feb74e13d11323cc8401 --- diff --git a/libaurum/inc/Accessibility/AccessibleNode.h b/libaurum/inc/Accessibility/AccessibleNode.h index 9a44612..5b9179a 100644 --- a/libaurum/inc/Accessibility/AccessibleNode.h +++ b/libaurum/inc/Accessibility/AccessibleNode.h @@ -217,6 +217,12 @@ public: */ double getValue() const; + /** + * @copydoc UiObject::getPid() + * + */ + int getPid() const; + /** * @copydoc UiObject::getIncrement() * @@ -362,6 +368,11 @@ public: */ virtual void updateValue() = 0; + /** + * @copydoc UiObject::updatePid() + */ + virtual void updatePid() = 0; + /** * @copydoc UiObject::setFocus() */ @@ -478,6 +489,7 @@ protected: Rect mWindowBoundingBox; int mSupportingIfaces; int mFeatureProperty; + int mPid; double mMinValue; double mMaxValue; double mValue; diff --git a/libaurum/inc/Aurum.h b/libaurum/inc/Aurum.h index 52bb8b2..0003c39 100644 --- a/libaurum/inc/Aurum.h +++ b/libaurum/inc/Aurum.h @@ -47,6 +47,7 @@ #include "UiDevice.h" #include "UiObject.h" #include "UiSelector.h" +#include "TizenWindow.h" #include "Sel.h" #include "Until.h" #include "Waiter.h" diff --git a/libaurum/inc/Impl/Accessibility/AtspiAccessibleNode.h b/libaurum/inc/Impl/Accessibility/AtspiAccessibleNode.h index 2724c93..5b8a3b6 100644 --- a/libaurum/inc/Impl/Accessibility/AtspiAccessibleNode.h +++ b/libaurum/inc/Impl/Accessibility/AtspiAccessibleNode.h @@ -126,6 +126,11 @@ public: */ void updateValue() override; + /** + * @copydoc UiObject::updatePid() + */ + void updatePid() override; + /** * @copydoc UiObject::setFocus() */ diff --git a/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h b/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h index be6ce32..68ef294 100644 --- a/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h +++ b/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h @@ -129,6 +129,12 @@ public: */ void updateValue() override; + /** + * @brief TBD + * @since_tizen 7.0 + */ + void updatePid() override; + /** * @brief TBD * @since_tizen 7.0 diff --git a/libaurum/inc/TizenWindow.h b/libaurum/inc/TizenWindow.h new file mode 100644 index 0000000..4c86227 --- /dev/null +++ b/libaurum/inc/TizenWindow.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef _TIZEN_WINDOW_H_ +#define _TIZEN_WINDOW_H_ + +#include + +namespace Aurum { + +class TizenWindow : public std::enable_shared_from_this { +public: + TizenWindow(int pid, Rect geometry, bool transformed, bool alpha, int opaque, int visibility, bool focused, bool mapped, int layer, std::string name); + ~TizenWindow(); + + int getPid(); + Rect getWindowGeometry(); + bool isTransformed(); + bool isAlpah(); + int getOpaque(); + int getVisibility(); + bool isFocused(); + bool isMapped(); + std::string getName(); +private: + int mPid; // window's PID + Rect mWindowGeometry; // window's geometry + bool mIsTransformed; // true : transformed window + bool mIsAlpha; // true : alpha + int mOpaque; // true : opaque state set window + int mVisibility; // 0 : visible + // 1 : partial visible + // 2 : hidden + // -1 : unknown + bool mIsFocused; // true : has focus + bool mIsMapped; // true : mapped + int mLayer; // 200 : normal layer + // 250 : above layer + // 650 ~ 800 : notification layer + std::string mName; // window name +}; + +} + +#endif diff --git a/libaurum/inc/UiDevice.h b/libaurum/inc/UiDevice.h index 669e0b7..ba21d4a 100644 --- a/libaurum/inc/UiDevice.h +++ b/libaurum/inc/UiDevice.h @@ -22,6 +22,7 @@ #include "UiObject.h" #include "UiSelector.h" +#include "TizenWindow.h" #include "IDevice.h" #include "ISearchable.h" @@ -369,6 +370,15 @@ public: * @since_tizen 6.5 */ std::vector> getWindowRoot() const; + + /** + * @brief Gets currently window information from window system. + * + * @return TizenWindow ptr vector + * @since_tizen 7.0 + */ + std::vector> getTizenWindowInfo() const; + private: /** * @brief Waits process idle. @@ -403,6 +413,7 @@ public: private: IDevice *mDeviceImpl; const Waiter *mWaiter; + static std::vector> mTizenWindows; }; } diff --git a/libaurum/inc/UiObject.h b/libaurum/inc/UiObject.h index 98c3a6c..018d1ad 100644 --- a/libaurum/inc/UiObject.h +++ b/libaurum/inc/UiObject.h @@ -329,6 +329,15 @@ public: */ const double getValue() const; + /** + * @brief Gets object's process id. + * + * @return int + * + * @since_tizen 7.0 + */ + const int getPid() const; + /** * @brief Gets object's current increment. * @@ -550,6 +559,13 @@ public: */ void updateValue() const; + /** + * @brief Updates object's process id. + * + * @since_tizen 7.0 + */ + void updatePid() const; + /** * @brief Sets focus to object. * diff --git a/libaurum/meson.build b/libaurum/meson.build index 5440ee7..66c87a0 100644 --- a/libaurum/meson.build +++ b/libaurum/meson.build @@ -4,6 +4,7 @@ libaurum_install_inc = [ './inc/UiDevice.h', './inc/UiObject.h', './inc/UiSelector.h', + './inc/TizenWindow.h', './inc/Sel.h', './inc/Until.h', './inc/Waiter.h', diff --git a/libaurum/src/Accessibility/AccessibleNode.cc b/libaurum/src/Accessibility/AccessibleNode.cc index 4fc7fcf..6e8eb02 100644 --- a/libaurum/src/Accessibility/AccessibleNode.cc +++ b/libaurum/src/Accessibility/AccessibleNode.cc @@ -248,4 +248,9 @@ double AccessibleNode::getValue() const double AccessibleNode::getIncrement() const { return mIncrement; -} \ No newline at end of file +} + +int AccessibleNode::getPid() const +{ + return mPid; +} diff --git a/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc b/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc index 5d9e922..8d04ab3 100644 --- a/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc +++ b/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc @@ -254,6 +254,13 @@ void AtspiAccessibleNode::updateValue() } } +void AtspiAccessibleNode::updatePid() +{ + AtspiWrapper::Atspi_accessible_clear_cache(mNode); + + mPid = AtspiWrapper::Atspi_accessible_get_process_id(mNode, NULL); +} + bool AtspiAccessibleNode::setFocus() { AtspiComponent *component = AtspiWrapper::Atspi_accessible_get_component_iface(mNode); diff --git a/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc b/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc index 51ba193..8900dd8 100644 --- a/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc +++ b/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc @@ -117,6 +117,10 @@ void MockAccessibleNode::updateValue() { } +void MockAccessibleNode::updatePid() +{ +} + bool MockAccessibleNode::setFocus() { return false; diff --git a/libaurum/src/TizenWindow.cc b/libaurum/src/TizenWindow.cc new file mode 100644 index 0000000..ae19068 --- /dev/null +++ b/libaurum/src/TizenWindow.cc @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "Aurum.h" +#include "TizenWindow.h" + +using namespace Aurum; + +TizenWindow::TizenWindow(int pid, Rect geometry, bool transformed, bool alpha, int opaque, int visibility, bool focused, bool mapped, int layer, std::string name) + : mPid(pid), mWindowGeometry(geometry), mIsTransformed(transformed), mIsAlpha(alpha), mOpaque(opaque), mVisibility(visibility), mIsFocused(focused), mIsMapped(mapped), mLayer(layer), mName(name) +{ + +} + +TizenWindow::~TizenWindow() +{ + +} + +int TizenWindow::getPid() +{ + return mPid; +} + +Rect TizenWindow::getWindowGeometry() +{ + return mWindowGeometry; +} + +bool TizenWindow::isTransformed() +{ + return mIsTransformed; +} + +bool TizenWindow::isAlpah() +{ + return mIsAlpha; +} + +int TizenWindow::getOpaque() +{ + return mOpaque; +} + +int TizenWindow::getVisibility() +{ + return mVisibility; +} + +bool TizenWindow::isFocused() +{ + return mIsFocused; +} + +bool TizenWindow::isMapped() +{ + return mIsMapped; +} + +std::string TizenWindow::getName() +{ + return mName; +} \ No newline at end of file diff --git a/libaurum/src/UiDevice.cc b/libaurum/src/UiDevice.cc index 15ab27f..ea76073 100644 --- a/libaurum/src/UiDevice.cc +++ b/libaurum/src/UiDevice.cc @@ -29,10 +29,19 @@ #include #include #include +#include using namespace Aurum; using namespace AurumInternal; +#define WM_BUS_NAME "org.enlightenment.wm" +#define WM_OBJECT_PATH "/org/enlightenment/wm" +#define WM_INTERFACE_NAME "org.enlightenment.wm.proc" +#define WM_METHOD_NAME_INFO "GetVisibleWinInfo_v2" + +std::vector> UiDevice::mTizenWindows; +static GDBusConnection *system_conn; + UiDevice::UiDevice() : UiDevice(nullptr) {} UiDevice::UiDevice(IDevice *impl) @@ -66,21 +75,154 @@ std::shared_ptr UiDevice::getInstance(IDevice *deviceImpl) return device; } +std::vector> UiDevice::getTizenWindowInfo() const +{ + GError *err = NULL; + GDBusMessage *msg; + GDBusMessage *reply; + GDBusConnection *conn; + GVariant *body; + GVariantIter *iter = NULL; + int idx = 0; + int pid; + int x; + int y; + int w; + int h; + gboolean transformed; + gboolean alpha; + int opaque; + int visibility; + gboolean focused; + gboolean mapped; + int layer; + char *name; + + mTizenWindows.clear(); + + if (system_conn == NULL) { + conn = g_bus_get_sync(G_BUS_TYPE_SYSTEM, NULL, &err); + if (conn == NULL) { + LOGE("g_bus_get_sync() is failed. %s", err->message); + g_error_free(err); + return mTizenWindows; + } + system_conn = conn; + } + + msg = g_dbus_message_new_method_call(WM_BUS_NAME, + WM_OBJECT_PATH, + WM_INTERFACE_NAME, + WM_METHOD_NAME_INFO); + if (msg == NULL) { + LOGE("g_dbus_message_new_method_call() is failed."); + return mTizenWindows; + } + + reply = g_dbus_connection_send_message_with_reply_sync(system_conn, msg, + G_DBUS_SEND_MESSAGE_FLAGS_NONE, -1, NULL, NULL, &err); + + if (!reply) { + if (err != NULL) { + LOGE("Failed to get info [%s]", err->message); + g_error_free(err); + } + goto out; + } + + body = g_dbus_message_get_body(reply); + if (!body) { + LOGE("Failed to get body"); + goto out; + } + + g_variant_get(body, "(a(iiiiibbiibbis))", &iter); + + LOGI("%-3s | %-6s | %-4s | %-4s | %-4s | %-4s | %-5s | %-5s | %-6s | %-3s | %-7s | %-6s | %-5s | %-20s", "No" ,"PID", "X", "Y", "W", "H", "Trans", "Alpha", "Opaque", "Vis", "Focused", "Mapped", "Layer", "Name"); + while (g_variant_iter_loop(iter, "(iiiiibbiibbis)", + &pid, + &x, + &y, + &w, + &h, + &transformed, + &alpha, + &opaque, + &visibility, + &focused, + &mapped, + &layer, + &name)) { + LOGI("%-3d | %-6d | %-4d | %-4d | %-4d | %-4d | %-5d | %-5d | %-6d | %-3d | %-7d | %-6d | %-5d | %-20s", idx++, pid, x,y,w,h, transformed, alpha, opaque, visibility, focused, mapped, layer, name); + if (visibility == 0 && pid > 0) + { + Rect geometry = {x, y, w, h}; + std::string winName(name); + mTizenWindows.push_back(std::make_shared(pid, geometry, transformed, alpha, opaque, visibility, focused, mapped, layer, winName)); + } + } + + if (iter) + g_variant_iter_free(iter); +out: + if (msg) + g_object_unref(msg); + if (reply) + g_object_unref(reply); + + return mTizenWindows; +} + std::vector> UiDevice::getWindowRoot() const { + bool dup; + LOGI("Request window info"); + getTizenWindowInfo(); + std::vector> ret{}; - auto appsMap = AccessibleWatcher::getInstance()->getActiveAppMap(); - LOGI("activeAppMap.size: %d" , (int)appsMap.size()); - for (auto itr = appsMap.begin(); itr != appsMap.end(); itr++) + auto apps = AccessibleWatcher::getInstance()->getApplications(); + for (auto app : apps) + { + app->getAccessibleNode()->updateName(); + app->getAccessibleNode()->updatePid(); + LOGI("App(%s) Pid(%d)", app->getPackageName().c_str(), app->getAccessibleNode()->getPid()); + } + + for (auto tWin : mTizenWindows) { - auto activeWindows = itr->second->getActiveWindows(); - std::transform(activeWindows.begin(), activeWindows.end(), std::back_inserter(ret), - [&](std::shared_ptr window){ - LOGI("active pkg: %s, window: %s", window->getAccessibleNode()->getPkg().c_str(), window->getTitle().c_str()); - return window->getAccessibleNode(); + LOGI("Visible win (%d) (%d %d %d %d) (%s)", tWin->getPid(), tWin->getWindowGeometry().mTopLeft.x, tWin->getWindowGeometry().mTopLeft.y, tWin->getWindowGeometry().width(), + tWin->getWindowGeometry().height(), tWin->getName().c_str()); + + for (auto app : apps) + { + dup = false; + if (app->getAccessibleNode()->getPid() == tWin->getPid()) + { + for (const auto &retWin : ret) + { + retWin->getParent()->updatePid(); + LOGI("Pid Dup check in vector (%d) target (%d)", retWin->getParent()->getPid(), tWin->getPid()); + if (retWin->getParent()->getPid() == tWin->getPid()) + { + dup = true; + break; + } + } + + if (!dup) + { + LOGI("Actvie App : (%s) (%d)", tWin->getName().c_str(), tWin->getPid()); + auto wins = app->getWindows(); + std::transform(wins.begin(), wins.end(), std::back_inserter(ret), + [&](std::shared_ptr window){ + LOGI("Target window add pkg: (%s), name (%s)", window->getAccessibleNode()->getPkg().c_str(), window->getTitle().c_str()); + return window->getAccessibleNode(); + } + ); + } } - ); + } } return ret; @@ -143,19 +285,19 @@ bool UiDevice::waitForIdle() const } bool UiDevice::waitForEvents( - const A11yEvent type, const int timeout) const + const A11yEvent type, const int timeout) const { return executeAndWaitForEvents(NULL, type, timeout); } bool UiDevice::executeAndWaitForEvents( - const Runnable *cmd, const A11yEvent type, const int timeout) const + const Runnable *cmd, const A11yEvent type, const int timeout) const { return AccessibleWatcher::getInstance()->executeAndWaitForEvents(cmd, type, timeout); } bool UiDevice::sendKeyAndWaitForEvents( - const std::string keycode, const A11yEvent type, const int timeout) const + const std::string keycode, const A11yEvent type, const int timeout) const { std::unique_ptr cmd = std::make_unique(keycode); return executeAndWaitForEvents(cmd.get(), type, timeout); diff --git a/libaurum/src/UiObject.cc b/libaurum/src/UiObject.cc index 51ac790..fe54f4f 100644 --- a/libaurum/src/UiObject.cc +++ b/libaurum/src/UiObject.cc @@ -344,6 +344,11 @@ void UiObject::updateValue() const mNode->updateValue(); } +void UiObject::updatePid() const +{ + mNode->updatePid(); +} + bool UiObject::setFocus() const { return mNode->setFocus(); diff --git a/libaurum/src/meson.build b/libaurum/src/meson.build index 11af3ea..390e805 100644 --- a/libaurum/src/meson.build +++ b/libaurum/src/meson.build @@ -10,6 +10,7 @@ libaurum_src += [ files('PartialMatch.cc'), files('A11yEvent.cc'), files('AurumXML.cc'), + files('TizenWindow.cc'), ] subdir('Accessibility')