src/mmimgr/iu/VoiceTouchEngine.cpp

   1 /*
   2  * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  *  Licensed under the Apache License, Version 2.0 (the "License");
   5  *  you may not use this file except in compliance with the License.
   6  *  You may obtain a copy of the License at
   7  *
   8  *               http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  *  Unless required by applicable law or agreed to in writing, software
  11  *  distributed under the License is distributed on an "AS IS" BASIS,
  12  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *  See the License for the specific language governing permissions and
  14  *  limitations under the License.
  15  *
  16  */
  17
  18
  19 #include <chrono>
  20 #include <Ecore.h>
  21
  22 #include "mmi_iu_log.h"
  23 #include "StringUtil.h"
  24 #include "json_provider.h"
  25
  26 #include "VoiceTouchEngine.h"
  27
  28
  29 using namespace std;
  30
  31 static const string __SHOW_TOOLTIP_INVOCATION = "show tooltip";
  32 static const string __SHOW_NUMBER_INVOCATION = "show numbers";
  33 static const string __SHOW_GRID_INVOCATION = "show grid";
  34
  35 static const vector<string> __SHOW_NUMBER_KOKR = {"힌트 보여줘", "숫자 힌트 보여줘", "숫자 보여줘", "숫자"};
  36 static const vector<string> __SHOW_LABEL_KOKR = {"힌트 메세지 보여줘", "텍스트 보여줘", "텍스트", "텍스트 힌트 보여줘"};
  37 static const vector<string> __SHOW_GRID_KOKR = {"그리드", "그리드 모드", "그리드로 보여줘", "그리드 보여줘"};
  38 static const vector<string> __REFRESH_WINDOW_INFO = {"화면 갱신해줘", "화면 업데이트", "리프레시", "새로고침", "새로 고침"};
  39
  40 static const int __DEFAULT_DISPLAY_WIDTH = 1920;
  41 static const int __DEFAULT_DISPLAY_HEIGHT = 1080;
  42
  43 static const int __DEFAULT_GRID_NUM_X1 = 7;
  44 static const int __DEFAULT_GRID_NUM_Y1 = 5;
  45 static const int __DEFAULT_GRID_NUM_X2 = 4;
  46 static const int __DEFAULT_GRID_NUM_Y2 = 3;
  47
  48
  49 VoiceTouchEngine::VoiceTouchEngine()
  50 {
  51         _I("[VoiceTouchEngine] Constructor");
  52         __touchModality.timestamp = -1;
  53         __touchModality.mode = MMI_VOICE_TOUCH_MODE_TOOLTIP;
  54         __touchModality.object_id = nullptr;
  55         __touchModality.coord_x = 0;
  56         __touchModality.coord_y = 0;
  57
  58         __currentMode = VOICE_TOUCH_MODE_TOOLTIP_TEXT;
  59
  60         __itemList.clear();
  61
  62         __displayWidth = __DEFAULT_DISPLAY_WIDTH;
  63         __displayHeight = __DEFAULT_DISPLAY_HEIGHT;
  64
  65         __gridX[0] = __DEFAULT_GRID_NUM_X1;
  66         __gridY[0] = __DEFAULT_GRID_NUM_Y1;
  67         __gridX[1] = __DEFAULT_GRID_NUM_X2;
  68         __gridY[1] = __DEFAULT_GRID_NUM_Y2;
  69
  70         __outputResultCallback = nullptr;
  71         __outputResultUserData = nullptr;
  72
  73         __outputModalityCallback = nullptr;
  74         __outputModalityUserData = nullptr;
  75
  76         __findCommandTimer = nullptr;
  77         __makeClickableItemResultTimer = nullptr;
  78
  79         resetCurrentGridInfo();
  80 }
  81
  82 VoiceTouchEngine::~VoiceTouchEngine()
  83 {
  84         _I("[VoiceTouchEngine] Destructor");
  85         __itemList.clear();
  86
  87         __outputResultCallback = nullptr;
  88         __outputResultUserData = nullptr;
  89
  90         __outputModalityCallback = nullptr;
  91         __outputModalityUserData = nullptr;
  92
  93         if (__findCommandTimer != nullptr) {
  94                 ecore_timer_del(__findCommandTimer);
  95                 __findCommandTimer = nullptr;
  96         }
  97
  98         if (__makeClickableItemResultTimer != nullptr) {
  99                 ecore_timer_del(__makeClickableItemResultTimer);
 100                 __makeClickableItemResultTimer = nullptr;
 101         }
 102 }
 103
 104
 105 void VoiceTouchEngine::setDisplayInfo(int width, int height)
 106 {
 107         __displayWidth = width;
 108         __displayHeight = height;
 109 }
 110
 111 void VoiceTouchEngine::setGridConfig(int numX, int numY, int step)
 112 {
 113         __gridX[step] = numX;
 114         __gridY[step] = numY;
 115 }
 116
 117 long VoiceTouchEngine::getCurrentTimestamp()
 118 {
 119         auto currentTime = chrono::steady_clock::now();
 120         long timestamp = chrono::time_point_cast<chrono::milliseconds>(currentTime).time_since_epoch().count();
 121
 122         return timestamp;
 123 }
 124
 125 bool VoiceTouchEngine::setInputModalityData(std::string appId, int modalityType, void *data)
 126 {
 127         _I("[VoiceTouchEngine] Set input modality data. appId(%s)", appId.c_str());
 128         __appId = appId;
 129
 130         if (modalityType == MMI_PROVIDER_EVENT_VOICE) {
 131                 _I("[VoiceTouchEngine] Voice modality");
 132                 mmi_provider_event_voice *event = static_cast<mmi_provider_event_voice *>(data);
 133
 134                 return handleVoiceInput(event);
 135         }
 136
 137         if (modalityType == MMI_PROVIDER_EVENT_SCREEN_ANALYZER) {
 138                 _I("[VoiceTouchEngine] Screen analyzer modality");
 139                 mmi_provider_event_screen_analyzer *event = static_cast<mmi_provider_event_screen_analyzer *>(data);
 140
 141                 return handleScreenAnalyzerInput(event);
 142         }
 143
 144         _E("[VoiceTouchEngine] This type is not supported by this engine. type(%d)", modalityType);
 145         return false;
 146 }
 147
 148 static bool __is_exist_in_candidates(const string &text, const vector<string> &candidates)
 149 {
 150         for (auto &candidate : candidates) {
 151                 if (0 == text.compare(candidate)) {
 152                         return true;
 153                 }
 154         }
 155
 156         return false;
 157 }
 158
 159 static void __refresh_screen_inform_event_free(void *data EINA_UNUSED, void *event)
 160 {
 161         mmi_event_refresh_screen_inform *ev = (mmi_event_refresh_screen_inform *)event;
 162         free(ev);
 163 }
 164
 165 static void __invoke_event_for_refreshing_window()
 166 {
 167         mmi_event_refresh_screen_inform *ev = (mmi_event_refresh_screen_inform *)calloc(1, sizeof(mmi_event_refresh_screen_inform));
 168         if (ev == NULL) {
 169                 LOGE("mmi_event_refresh_screen_inform malloc failed");
 170                 return;
 171         }
 172
 173         ev->seconds = 2.0;
 174         ecore_event_add(MMI_EVENT_REFRESH_SCREEN_INFORM, ev, __refresh_screen_inform_event_free, nullptr);
 175 }
 176
 177 bool VoiceTouchEngine::handleVoiceInput(mmi_provider_event_voice *voiceEvent)
 178 {
 179         _I("[VoiceTouchEngine] Voice modality");
 180         if (voiceEvent == nullptr) {
 181                 _E("[VoiceTouchEngine] Parameter is null");
 182                 return false;
 183         }
 184
 185         _I("[VoiceTouchEngine] Input data. event(%d), text(%s)", voiceEvent->result_type, voiceEvent->source);
 186         int timestamp = getCurrentTimestamp();
 187
 188         JsonProvider provider;
 189         provider.setInputEvent(MMI_KEY_VOICE_TOUCH);
 190
 191         string asrResult = string(voiceEvent->source);
 192         voice_result_type_e asrEvent = static_cast<voice_result_type_e>(voiceEvent->result_type);
 193         if (asrEvent == VOICE_RESULT_TYPE_PARTIAL) {
 194                 _I("[VoiceTouchEngine] Partial ASR result");
 195                 setPartialAsrResult(asrResult, timestamp, provider);
 196                 invokeOutputResultCallback(provider.jsonToString());
 197                 return true;
 198         }
 199
 200         _I("[VoiceTouchEngine] Final ASR result");
 201         setFinalAsrResult(asrResult, timestamp, provider);
 202         invokeOutputResultCallback(provider.jsonToString());
 203
 204         bool isRefreshNeed = __is_exist_in_candidates(asrResult, __REFRESH_WINDOW_INFO);
 205         if (isRefreshNeed) {
 206                 _I("[VoiceTouchEngine] Request to refresh clickable object information.");
 207                 __invoke_event_for_refreshing_window();
 208                 return true;
 209         }
 210
 211         bool isModeUpdated = checkVoiceTouchMode(asrResult);
 212         if (isModeUpdated) {
 213                 _I("[VoiceTouchEngine] Voice touch mode is changed. (%d)", static_cast<int>(__currentMode));
 214                 resetCurrentGridInfo();
 215                 __makeClickableItemResultTimer = ecore_timer_add(0.0, makeClickableItemResultCallback, static_cast<void *>(this));
 216         } else {
 217                 __asrResult = asrResult;
 218                 __findCommandTimer = ecore_timer_add(0.0, findCommandCallback, static_cast<void *>(this));
 219         }
 220
 221         return true;
 222 }
 223
 224 void VoiceTouchEngine::setPartialAsrResult(const std::string &result, int timestamp, JsonProvider &provider)
 225 {
 226         _I("[VoiceTouchEngine] Set partial ASR result into output. timestamp(%d). item(%s)", timestamp, result.c_str());
 227
 228         provider.setOutputEvent(MMI_KEY_ASR_PARTIAL_RESULT);
 229         provider.setAsrPartialResult(result.c_str(), timestamp);
 230 }
 231
 232 void VoiceTouchEngine::setFinalAsrResult(const std::string &result, int timestamp, JsonProvider &provider)
 233 {
 234         _I("[VoiceTouchEngine] Set final ASR result into output. timestamp(%d). item(%s)", timestamp, result.c_str());
 235
 236         provider.setOutputEvent(MMI_KEY_ASR_FINAL_RESULT);
 237         provider.setAsrFinalResult(result.c_str(), timestamp);
 238 }
 239
 240 Eina_Bool VoiceTouchEngine::makeClickableItemResultCallback(void *data)
 241 {
 242         VoiceTouchEngine *engine = static_cast<VoiceTouchEngine *>(data);
 243         if (engine == nullptr) {
 244                 _E("[VoiceTouchEngine] Timer data is invalid");
 245                 return ECORE_CALLBACK_CANCEL;
 246         }
 247
 248         int timestamp = engine->getCurrentTimestamp();
 249
 250         JsonProvider provider;
 251         provider.setInputEvent(MMI_KEY_VOICE_TOUCH);
 252
 253         engine->makeClickableItemInfo(timestamp, provider);
 254         engine->invokeOutputResultCallback(provider.jsonToString());
 255
 256         return ECORE_CALLBACK_CANCEL;
 257 }
 258
 259 Eina_Bool VoiceTouchEngine::findCommandCallback(void *data)
 260 {
 261         VoiceTouchEngine *engine = static_cast<VoiceTouchEngine *>(data);
 262         if (engine == nullptr) {
 263                 _E("[VoiceTouchEngine] Timer data is invalid");
 264                 return ECORE_CALLBACK_CANCEL;
 265         }
 266
 267         int timestamp = engine->getCurrentTimestamp();
 268
 269         JsonProvider provider;
 270         provider.setInputEvent(MMI_KEY_VOICE_TOUCH);
 271
 272         ClickableItem *command = engine->findCommand();
 273         engine->makeClickedItemResult(command, timestamp, provider);
 274         engine->invokeOutputResultCallback(provider.jsonToString());
 275
 276         if (command != nullptr) {
 277                 _I("[VoiceTouchEngine] Command. object ID(%s)", command->objectId.c_str());
 278                 engine->makeTouchModality(*command, timestamp);
 279                 engine->invokeOutputModalityCallback(&engine->__touchModality);
 280         }
 281
 282         return ECORE_CALLBACK_CANCEL;
 283 }
 284
 285 ClickableItem * VoiceTouchEngine::findCommand()
 286 {
 287         bool isIndex = (__currentMode != VOICE_TOUCH_MODE_TOOLTIP_TEXT);
 288         return __commandManager.findCommand(__asrResult, isIndex);
 289 }
 290
 291 void VoiceTouchEngine::makeClickedItemResult(ClickableItem* command, int timestamp, JsonProvider &provider)
 292 {
 293         if (command == nullptr) {
 294                 _I("[VoiceTouchEngine] Fail to find command");
 295                 setRejectedEventResult(timestamp, provider);
 296         } else {
 297                 _I("[VoiceTouchEngine] Command. object ID(%s)", command->objectId.c_str());
 298                 setSelectedItemResult(*command, timestamp, provider);
 299         }
 300 }
 301
 302 bool VoiceTouchEngine::checkVoiceTouchMode(const std::string &text)
 303 {
 304         _I("[VoiceTouchEngine] Input text(%s)", text.c_str());
 305
 306         string loweredText = StringUtil::makeLowerCase(text);
 307         voice_touch_mode_e mode = __currentMode;
 308         if (__is_exist_in_candidates(loweredText, __SHOW_LABEL_KOKR)) {
 309                 mode = VOICE_TOUCH_MODE_TOOLTIP_TEXT;
 310         } else if (__is_exist_in_candidates(loweredText, __SHOW_NUMBER_KOKR)) {
 311                 mode = VOICE_TOUCH_MODE_TOOLTIP_NUMBER;
 312         } else if (__is_exist_in_candidates(loweredText, __SHOW_GRID_KOKR)) {
 313                 mode = VOICE_TOUCH_MODE_GRID;
 314         }
 315
 316         if (__currentMode != mode) {
 317                 _I("[VoiceTouchEngine] mode is changed from (%d) to (%d)", __currentMode, mode);
 318                 __currentMode = mode;
 319                 return true;
 320         }
 321
 322         _I("[VoiceTouchEngine] Mode is not changed");
 323         return false;
 324 }
 325
 326 void VoiceTouchEngine::resetCurrentGridInfo()
 327 {
 328         __currentGridInfo.currentStep = 0;
 329         __currentGridInfo.coordX = 0;
 330         __currentGridInfo.coordY = 0;
 331         __currentGridInfo.width = __displayWidth;
 332         __currentGridInfo.height = __displayHeight;
 333 }
 334
 335 void VoiceTouchEngine::makeClickableItemInfo(int timestamp, JsonProvider &provider)
 336 {
 337         _I("[VoiceTouchEngine] Make clickable object list output. timestamp(%d), Current mode(%d)", timestamp, __currentMode);
 338
 339         list<ClickableItem> *itemList = nullptr;
 340         const char *resultType = nullptr;
 341         const char *tooltipType = nullptr;
 342
 343         switch (__currentMode)
 344         {
 345         case VOICE_TOUCH_MODE_GRID:
 346                 makeGridItemList(__currentGridInfo, __gridList);
 347                 itemList = &__gridList;
 348                 resultType = MMI_KEY_SHOW_GRIDS;
 349                 tooltipType = MMI_KEY_TOOLTIP_TYPE_NUMBER;
 350                 break;
 351
 352         case VOICE_TOUCH_MODE_TOOLTIP_NUMBER:
 353                 itemList = &__itemList;
 354                 resultType = MMI_KEY_SHOW_TOOLTIPS;
 355                 tooltipType = MMI_KEY_TOOLTIP_TYPE_NUMBER;
 356                 break;
 357
 358         case VOICE_TOUCH_MODE_TOOLTIP_TEXT:
 359                 itemList = &__itemList;
 360                 resultType = MMI_KEY_SHOW_TOOLTIPS;
 361                 tooltipType = MMI_KEY_TOOLTIP_TYPE_TEXT;
 362                 break;
 363
 364         default:
 365                 _E("[VoiceTouchEngine] Invalid mode classfier. Update item as lable tooltip.");
 366                 itemList = &__itemList;
 367                 resultType = MMI_KEY_SHOW_TOOLTIPS;
 368                 tooltipType = MMI_KEY_TOOLTIP_TYPE_TEXT;
 369                 break;
 370         }
 371
 372         if (itemList == nullptr || itemList->size() == 0) {
 373                 _I("[VoiceTouchEngine] There is no item in the list.");
 374                 provider.setOutputEvent(MMI_KEY_ERROR);
 375                 provider.setErrorEvent(MMI_REASON_NO_CLICKABLE_OBJECTS, timestamp);
 376                 return;
 377         }
 378
 379         provider.setOutputEvent(MMI_KEY_UI_CLICKABLE_OBJECT);
 380         provider.setUiClickableObject(resultType, __appId.c_str(), tooltipType, __itemList.size(), timestamp);
 381         provider.setGridDepth(__currentGridInfo.currentStep + 1);
 382
 383         for (auto &item : *itemList) {
 384                 auto label = (item.label.empty() ? to_string(item.index) : item.label);
 385                 provider.addInfoClickableObject(item.index, item.coordX, item.coordY, item.width, item.height, label.c_str());
 386         }
 387 }
 388
 389 void VoiceTouchEngine::makeTouchModality(const ClickableItem &item, int timestamp)
 390 {
 391         _I("[VoiceTouchEngine] Make clicked object output. timestamp(%d). item(%s)", timestamp, item.objectId.c_str());
 392         if (__currentMode != VOICE_TOUCH_MODE_GRID) {
 393                 _D("[VoiceTouchEngine] Not grid mode.");
 394                 setTouchModality(item, timestamp);
 395                 return;
 396         }
 397
 398         if (__currentGridInfo.currentStep == 0) {
 399                 __currentGridInfo.currentStep++;
 400                 __currentGridInfo.coordX = item.coordX;
 401                 __currentGridInfo.coordY = item.coordY;
 402                 __currentGridInfo.width = item.width;
 403                 __currentGridInfo.height = item.height;
 404                 return;
 405         }
 406
 407         resetCurrentGridInfo();
 408         setTouchModality(item, timestamp);
 409 }
 410
 411 void VoiceTouchEngine::setRejectedEventResult(int timestamp, JsonProvider &provider)
 412 {
 413         _I("[VoiceTouchEngine] Set command rejected event into output. timestamp(%d)", timestamp);
 414
 415         provider.setOutputEvent(MMI_KEY_REJECT);
 416         provider.setRejectEvent(MMI_REASON_NO_MATCHED_COMMANDS, timestamp);
 417 }
 418
 419 void VoiceTouchEngine::setSelectedItemResult(const ClickableItem &item, int timestamp, JsonProvider &provider)
 420 {
 421         _I("[VoiceTouchEngine] Set clicked object information into output. timestamp(%d). item(%s)", timestamp, item.objectId.c_str());
 422
 423         provider.setOutputEvent(MMI_KEY_UI_CLICKED_OBJECT);
 424         provider.setUiClickedObject(__appId.c_str(), timestamp, item.index, item.coordX, item.coordY, item.width, item.height, item.label.c_str());
 425 }
 426
 427 void VoiceTouchEngine::makeGridItemList(const grid_info_s &parentInfo, std::list<ClickableItem> &list)
 428 {
 429         _I("[VoiceTouchEngine] Set grid information into item list. Grid Step(%d)", parentInfo.currentStep);
 430         _D("[VoiceTouchEngine] Parent info. X(%d), Y(%d), W(%d), H(%d)", parentInfo.coordX, parentInfo.coordY, parentInfo.width, parentInfo.height);
 431
 432         int currentStep = parentInfo.currentStep;
 433         if (currentStep >= __MAXIMUM_GRID_STEP) {
 434                 return;
 435         }
 436
 437         int numX = __gridX[currentStep];
 438         int numY = __gridY[currentStep];
 439         int coordX = parentInfo.coordX;
 440         int coordY = parentInfo.coordY;
 441         int gridWidth = parentInfo.width / numX;
 442         int gridHeight = parentInfo.height / numY;
 443
 444         int remainedWidth = parentInfo.width % numX;
 445         int remainedHeight = parentInfo.height % numY;
 446
 447         int index = 1;
 448
 449         list.clear();
 450         for (int y = 0; y < numY; y++) {
 451                 for (int x = 0; x < numX; x++) {
 452                         ClickableItem grid;
 453                         grid.coordX = coordX + (x * gridWidth);
 454                         grid.coordY = coordY + (y * gridHeight);
 455
 456                         grid.width = (x == numX - 1 ? gridWidth + remainedWidth : gridWidth);
 457                         grid.height = (y == numY - 1 ? gridHeight + remainedHeight : gridHeight);
 458
 459                         grid.index = index++;
 460
 461                         list.push_back(grid);
 462                 }
 463         }
 464 }
 465
 466 void VoiceTouchEngine::setTouchModality(const ClickableItem &item, int timestamp)
 467 {
 468         __touchModality.timestamp = timestamp;
 469         __touchModality.mode = (__currentMode == VOICE_TOUCH_MODE_GRID ? MMI_VOICE_TOUCH_MODE_GRID : MMI_VOICE_TOUCH_MODE_TOOLTIP);
 470         __touchModality.object_id = item.objectId.c_str();
 471         __touchModality.coord_x = item.coordX + (item.width / 2);
 472         __touchModality.coord_y = item.coordY + (item.height / 2);
 473 }
 474
 475 bool VoiceTouchEngine::handleScreenAnalyzerInput(mmi_provider_event_screen_analyzer *screenAnalyzerEvent)
 476 {
 477         _I("[VoiceTouchEngine] Screen analyzer modality");
 478         if (screenAnalyzerEvent == nullptr) {
 479                 _E("[VoiceTouchEngine] Parameter is null");
 480                 return false;
 481         }
 482
 483         _I("[VoiceTouchEngine] Input data. num of items(%d)", screenAnalyzerEvent->n_items);
 484
 485         int timestamp = getCurrentTimestamp();
 486
 487         JsonProvider provider;
 488         provider.setInputEvent(MMI_KEY_VOICE_TOUCH);
 489
 490         __commandManager.cleanCommands();
 491         __itemList.clear();
 492         g_list_foreach(screenAnalyzerEvent->list, iterateObjectCallback, static_cast<gpointer>(this));
 493
 494         makeClickableItemInfo(timestamp, provider);
 495         invokeOutputResultCallback(provider.jsonToString());
 496
 497         return true;
 498 }
 499
 500 void VoiceTouchEngine::iterateObjectCallback(gpointer data, gpointer userData)
 501 {
 502         clickable_item* info = static_cast<clickable_item*>(data);
 503         VoiceTouchEngine *engine = static_cast<VoiceTouchEngine*>(userData);
 504
 505         ClickableItem item;
 506         item.coordX = info->coord_x;
 507         item.coordY = info->coord_y;
 508         item.width = info->width;
 509         item.height = info->height;
 510         item.objectId = string(info->object_id);
 511         item.label = string(info->label);
 512         item.index = engine->__itemList.size() + 1;
 513
 514         engine->__itemList.push_back(item);
 515         engine->__commandManager.addCommand(item);
 516 }
 517
 518 void VoiceTouchEngine::invokeOutputResultCallback(std::string outputResult)
 519 {
 520         if (__outputResultCallback == nullptr) {
 521                 return;
 522         }
 523
 524         __outputResultCallback(__appId.c_str(), outputResult.c_str(), __outputResultUserData);
 525 }
 526
 527 void VoiceTouchEngine::invokeOutputModalityCallback(void* outputModality)
 528 {
 529         if (__outputModalityCallback == nullptr) {
 530                 return;
 531         }
 532
 533         __outputModalityCallback(__appId.c_str(), outputModality, __outputModalityUserData);
 534 }
 535
 536 void VoiceTouchEngine::setOutputResultCallback(voice_touch_engine_output_result_cb callback, void *userData)
 537 {
 538         __outputResultCallback = callback;
 539         __outputResultUserData = userData;
 540 }
 541
 542 void VoiceTouchEngine::setOutputModalityCallback(voice_touch_engine_output_modality_cb callback, void *userData)
 543 {
 544         __outputModalityCallback = callback;
 545         __outputModalityUserData = userData;
 546 }