src/mmimgr/iu/VoiceTouchEngine.cpp

   1 /*
   2  * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  *  Licensed under the Apache License, Version 2.0 (the "License");
   5  *  you may not use this file except in compliance with the License.
   6  *  You may obtain a copy of the License at
   7  *
   8  *               http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  *  Unless required by applicable law or agreed to in writing, software
  11  *  distributed under the License is distributed on an "AS IS" BASIS,
  12  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *  See the License for the specific language governing permissions and
  14  *  limitations under the License.
  15  *
  16  */
  17
  18
  19 #include <chrono>
  20 #include <Ecore.h>
  21
  22 #include "mmi_iu_log.h"
  23 #include "StringUtil.h"
  24 #include "json_provider.h"
  25
  26 #include "VoiceTouchEngine.h"
  27
  28
  29 using namespace std;
  30
  31 static const string __SHOW_TOOLTIP_INVOCATION = "show tooltip";
  32 static const string __SHOW_NUMBER_INVOCATION = "show numbers";
  33 static const string __SHOW_GRID_INVOCATION = "show grid";
  34
  35 static const vector<string> __SHOW_NUMBER_KOKR = {"힌트 보여줘", "숫자 힌트 보여줘", "숫자 보여줘", "숫자"};
  36 static const vector<string> __SHOW_LABEL_KOKR = {"힌트 메세지 보여줘", "텍스트 보여줘", "텍스트", "텍스트 힌트 보여줘"};
  37 static const vector<string> __SHOW_GRID_KOKR = {"그리드", "그리드 모드", "그리드로 보여줘", "그리드 보여줘"};
  38
  39 static const int __DEFAULT_DISPLAY_WIDTH = 1920;
  40 static const int __DEFAULT_DISPLAY_HEIGHT = 1080;
  41
  42 static const int __DEFAULT_GRID_NUM_X1 = 7;
  43 static const int __DEFAULT_GRID_NUM_Y1 = 5;
  44 static const int __DEFAULT_GRID_NUM_X2 = 4;
  45 static const int __DEFAULT_GRID_NUM_Y2 = 3;
  46
  47
  48 VoiceTouchEngine::VoiceTouchEngine()
  49 {
  50         _I("[VoiceTouchEngine] Constructor");
  51         __touchModality.timestamp = -1;
  52         __touchModality.mode = MMI_VOICE_TOUCH_MODE_TOOLTIP;
  53         __touchModality.object_id = nullptr;
  54         __touchModality.coord_x = 0;
  55         __touchModality.coord_y = 0;
  56
  57         __currentMode = VOICE_TOUCH_MODE_TOOLTIP_TEXT;
  58
  59         __itemList.clear();
  60
  61         __displayWidth = __DEFAULT_DISPLAY_WIDTH;
  62         __displayHeight = __DEFAULT_DISPLAY_HEIGHT;
  63
  64         __gridX[0] = __DEFAULT_GRID_NUM_X1;
  65         __gridY[0] = __DEFAULT_GRID_NUM_Y1;
  66         __gridX[1] = __DEFAULT_GRID_NUM_X2;
  67         __gridY[1] = __DEFAULT_GRID_NUM_Y2;
  68
  69         __outputResultCallback = nullptr;
  70         __outputResultUserData = nullptr;
  71
  72         __outputModalityCallback = nullptr;
  73         __outputModalityUserData = nullptr;
  74
  75         __findCommandTimer = nullptr;
  76         __makeClickableItemResultTimer = nullptr;
  77
  78         resetCurrentGridInfo();
  79 }
  80
  81 VoiceTouchEngine::~VoiceTouchEngine()
  82 {
  83         _I("[VoiceTouchEngine] Destructor");
  84         __itemList.clear();
  85
  86         __outputResultCallback = nullptr;
  87         __outputResultUserData = nullptr;
  88
  89         __outputModalityCallback = nullptr;
  90         __outputModalityUserData = nullptr;
  91
  92         if (__findCommandTimer != nullptr) {
  93                 ecore_timer_del(__findCommandTimer);
  94                 __findCommandTimer = nullptr;
  95         }
  96
  97         if (__makeClickableItemResultTimer != nullptr) {
  98                 ecore_timer_del(__makeClickableItemResultTimer);
  99                 __makeClickableItemResultTimer = nullptr;
 100         }
 101 }
 102
 103
 104 void VoiceTouchEngine::setDisplayInfo(int width, int height)
 105 {
 106         __displayWidth = width;
 107         __displayHeight = height;
 108 }
 109
 110 void VoiceTouchEngine::setGridConfig(int numX, int numY, int step)
 111 {
 112         __gridX[step] = numX;
 113         __gridY[step] = numY;
 114 }
 115
 116 long VoiceTouchEngine::getCurrentTimestamp()
 117 {
 118         auto currentTime = chrono::steady_clock::now();
 119         long timestamp = chrono::time_point_cast<chrono::milliseconds>(currentTime).time_since_epoch().count();
 120
 121         return timestamp;
 122 }
 123
 124 bool VoiceTouchEngine::setInputModalityData(std::string appId, int modalityType, void *data)
 125 {
 126         _I("[VoiceTouchEngine] Set input modality data. appId(%s)", appId.c_str());
 127         __appId = appId;
 128
 129         if (modalityType == MMI_PROVIDER_EVENT_VOICE) {
 130                 _I("[VoiceTouchEngine] Voice modality");
 131                 mmi_provider_event_voice *event = static_cast<mmi_provider_event_voice *>(data);
 132
 133                 return handleVoiceInput(event);
 134         }
 135
 136         if (modalityType == MMI_PROVIDER_EVENT_SCREEN_ANALYZER) {
 137                 _I("[VoiceTouchEngine] Screen analyzer modality");
 138                 mmi_provider_event_screen_analyzer *event = static_cast<mmi_provider_event_screen_analyzer *>(data);
 139
 140                 return handleScreenAnalyzerInput(event);
 141         }
 142
 143         _E("[VoiceTouchEngine] This type is not supported by this engine. type(%d)", modalityType);
 144         return false;
 145 }
 146
 147 bool VoiceTouchEngine::handleVoiceInput(mmi_provider_event_voice *voiceEvent)
 148 {
 149         _I("[VoiceTouchEngine] Voice modality");
 150         if (voiceEvent == nullptr) {
 151                 _E("[VoiceTouchEngine] Parameter is null");
 152                 return false;
 153         }
 154
 155         _I("[VoiceTouchEngine] Input data. event(%d), text(%s)", voiceEvent->result_type, voiceEvent->source);
 156         int timestamp = getCurrentTimestamp();
 157
 158         JsonProvider provider;
 159         provider.setInputEvent(MMI_KEY_VOICE_TOUCH);
 160
 161         string asrResult = string(voiceEvent->source);
 162         voice_result_type_e asrEvent = static_cast<voice_result_type_e>(voiceEvent->result_type);
 163         if (asrEvent == VOICE_RESULT_TYPE_PARTIAL) {
 164                 _I("[VoiceTouchEngine] Partial ASR result");
 165                 setPartialAsrResult(asrResult, timestamp, provider);
 166                 invokeOutputResultCallback(provider.jsonToString());
 167                 return true;
 168         }
 169
 170         _I("[VoiceTouchEngine] Final ASR result");
 171         setFinalAsrResult(asrResult, timestamp, provider);
 172         invokeOutputResultCallback(provider.jsonToString());
 173
 174         bool isModeUpdated = checkVoiceTouchMode(asrResult);
 175         if (isModeUpdated) {
 176                 _I("[VoiceTouchEngine] Voice touch mode is changed. (%d)", static_cast<int>(__currentMode));
 177                 resetCurrentGridInfo();
 178                 __makeClickableItemResultTimer = ecore_timer_add(0.0, makeClickableItemResultCallback, static_cast<void *>(this));
 179         } else {
 180                 __asrResult = asrResult;
 181                 __findCommandTimer = ecore_timer_add(0.0, findCommandCallback, static_cast<void *>(this));
 182         }
 183
 184         return true;
 185 }
 186
 187 void VoiceTouchEngine::setPartialAsrResult(const std::string &result, int timestamp, JsonProvider &provider)
 188 {
 189         _I("[VoiceTouchEngine] Set partial ASR result into output. timestamp(%d). item(%s)", timestamp, result.c_str());
 190
 191         provider.setOutputEvent(MMI_KEY_ASR_PARTIAL_RESULT);
 192         provider.setAsrPartialResult(result.c_str(), timestamp);
 193 }
 194
 195 void VoiceTouchEngine::setFinalAsrResult(const std::string &result, int timestamp, JsonProvider &provider)
 196 {
 197         _I("[VoiceTouchEngine] Set final ASR result into output. timestamp(%d). item(%s)", timestamp, result.c_str());
 198
 199         provider.setOutputEvent(MMI_KEY_ASR_FINAL_RESULT);
 200         provider.setAsrFinalResult(result.c_str(), timestamp);
 201 }
 202
 203 Eina_Bool VoiceTouchEngine::makeClickableItemResultCallback(void *data)
 204 {
 205         VoiceTouchEngine *engine = static_cast<VoiceTouchEngine *>(data);
 206         if (engine == nullptr) {
 207                 _E("[VoiceTouchEngine] Timer data is invalid");
 208                 return ECORE_CALLBACK_CANCEL;
 209         }
 210
 211         int timestamp = engine->getCurrentTimestamp();
 212
 213         JsonProvider provider;
 214         provider.setInputEvent(MMI_KEY_VOICE_TOUCH);
 215
 216         engine->makeClickableItemInfo(timestamp, provider);
 217         engine->invokeOutputResultCallback(provider.jsonToString());
 218
 219         return ECORE_CALLBACK_CANCEL;
 220 }
 221
 222 Eina_Bool VoiceTouchEngine::findCommandCallback(void *data)
 223 {
 224         VoiceTouchEngine *engine = static_cast<VoiceTouchEngine *>(data);
 225         if (engine == nullptr) {
 226                 _E("[VoiceTouchEngine] Timer data is invalid");
 227                 return ECORE_CALLBACK_CANCEL;
 228         }
 229
 230         int timestamp = engine->getCurrentTimestamp();
 231
 232         JsonProvider provider;
 233         provider.setInputEvent(MMI_KEY_VOICE_TOUCH);
 234
 235         ClickableItem *command = engine->findCommand();
 236         engine->makeClickedItemResult(command, timestamp, provider);
 237         engine->invokeOutputResultCallback(provider.jsonToString());
 238
 239         if (command != nullptr) {
 240                 _I("[VoiceTouchEngine] Command. object ID(%s)", command->objectId.c_str());
 241                 engine->makeTouchModality(*command, timestamp);
 242                 engine->invokeOutputModalityCallback(&engine->__touchModality);
 243         }
 244
 245         return ECORE_CALLBACK_CANCEL;
 246 }
 247
 248 ClickableItem * VoiceTouchEngine::findCommand()
 249 {
 250         bool isIndex = (__currentMode != VOICE_TOUCH_MODE_TOOLTIP_TEXT);
 251         return __commandManager.findCommand(__asrResult, isIndex);
 252 }
 253
 254 void VoiceTouchEngine::makeClickedItemResult(ClickableItem* command, int timestamp, JsonProvider &provider)
 255 {
 256         if (command == nullptr) {
 257                 _I("[VoiceTouchEngine] Fail to find command");
 258                 setRejectedEventResult(timestamp, provider);
 259         } else {
 260                 _I("[VoiceTouchEngine] Command. object ID(%s)", command->objectId.c_str());
 261                 setSelectedItemResult(*command, timestamp, provider);
 262         }
 263 }
 264
 265 static bool __is_exist_in_candidates(const string &text, const vector<string> &candidates)
 266 {
 267         for (auto &candidate : candidates) {
 268                 if (0 == text.compare(candidate)) {
 269                         return true;
 270                 }
 271         }
 272
 273         return false;
 274 }
 275
 276 bool VoiceTouchEngine::checkVoiceTouchMode(const std::string &text)
 277 {
 278         _I("[VoiceTouchEngine] Input text(%s)", text.c_str());
 279
 280         string loweredText = StringUtil::makeLowerCase(text);
 281         voice_touch_mode_e mode = __currentMode;
 282         if (__is_exist_in_candidates(loweredText, __SHOW_LABEL_KOKR)) {
 283                 mode = VOICE_TOUCH_MODE_TOOLTIP_TEXT;
 284         } else if (__is_exist_in_candidates(loweredText, __SHOW_NUMBER_KOKR)) {
 285                 mode = VOICE_TOUCH_MODE_TOOLTIP_NUMBER;
 286         } else if (__is_exist_in_candidates(loweredText, __SHOW_GRID_KOKR)) {
 287                 mode = VOICE_TOUCH_MODE_GRID;
 288         }
 289
 290         if (__currentMode != mode) {
 291                 _I("[VoiceTouchEngine] mode is changed from (%d) to (%d)", __currentMode, mode);
 292                 __currentMode = mode;
 293                 return true;
 294         }
 295
 296         _I("[VoiceTouchEngine] Mode is not changed");
 297         return false;
 298 }
 299
 300 void VoiceTouchEngine::resetCurrentGridInfo()
 301 {
 302         __currentGridInfo.currentStep = 0;
 303         __currentGridInfo.coordX = 0;
 304         __currentGridInfo.coordY = 0;
 305         __currentGridInfo.width = __displayWidth;
 306         __currentGridInfo.height = __displayHeight;
 307 }
 308
 309 void VoiceTouchEngine::makeClickableItemInfo(int timestamp, JsonProvider &provider)
 310 {
 311         _I("[VoiceTouchEngine] Make clickable object list output. timestamp(%d), Current mode(%d)", timestamp, __currentMode);
 312
 313         list<ClickableItem> *itemList = nullptr;
 314         const char *resultType = nullptr;
 315         const char *tooltipType = nullptr;
 316
 317         switch (__currentMode)
 318         {
 319         case VOICE_TOUCH_MODE_GRID:
 320                 makeGridItemList(__currentGridInfo, __gridList);
 321                 itemList = &__gridList;
 322                 resultType = MMI_KEY_SHOW_GRIDS;
 323                 tooltipType = MMI_KEY_TOOLTIP_TYPE_NUMBER;
 324                 break;
 325
 326         case VOICE_TOUCH_MODE_TOOLTIP_NUMBER:
 327                 itemList = &__itemList;
 328                 resultType = MMI_KEY_SHOW_TOOLTIPS;
 329                 tooltipType = MMI_KEY_TOOLTIP_TYPE_NUMBER;
 330                 break;
 331
 332         case VOICE_TOUCH_MODE_TOOLTIP_TEXT:
 333                 itemList = &__itemList;
 334                 resultType = MMI_KEY_SHOW_TOOLTIPS;
 335                 tooltipType = MMI_KEY_TOOLTIP_TYPE_TEXT;
 336                 break;
 337
 338         default:
 339                 _E("[VoiceTouchEngine] Invalid mode classfier. Update item as lable tooltip.");
 340                 itemList = &__itemList;
 341                 resultType = MMI_KEY_SHOW_TOOLTIPS;
 342                 tooltipType = MMI_KEY_TOOLTIP_TYPE_TEXT;
 343                 break;
 344         }
 345
 346         if (itemList == nullptr || itemList->size() == 0) {
 347                 _I("[VoiceTouchEngine] There is no item in the list.");
 348                 provider.setOutputEvent(MMI_KEY_ERROR);
 349                 provider.setErrorEvent(MMI_REASON_NO_CLICKABLE_OBJECTS, timestamp);
 350                 return;
 351         }
 352
 353         provider.setOutputEvent(MMI_KEY_UI_CLICKABLE_OBJECT);
 354         provider.setUiClickableObject(resultType, __appId.c_str(), tooltipType, __itemList.size(), timestamp);
 355         provider.setGridDepth(__currentGridInfo.currentStep + 1);
 356
 357         for (auto &item : *itemList) {
 358                 auto label = (item.label.empty() ? to_string(item.index) : item.label);
 359                 provider.addInfoClickableObject(item.index, item.coordX, item.coordY, item.width, item.height, label.c_str());
 360         }
 361 }
 362
 363 void VoiceTouchEngine::makeTouchModality(const ClickableItem &item, int timestamp)
 364 {
 365         _I("[VoiceTouchEngine] Make clicked object output. timestamp(%d). item(%s)", timestamp, item.objectId.c_str());
 366         if (__currentMode != VOICE_TOUCH_MODE_GRID) {
 367                 _D("[VoiceTouchEngine] Not grid mode.");
 368                 setTouchModality(item, timestamp);
 369                 return;
 370         }
 371
 372         if (__currentGridInfo.currentStep == 0) {
 373                 __currentGridInfo.currentStep++;
 374                 __currentGridInfo.coordX = item.coordX;
 375                 __currentGridInfo.coordY = item.coordY;
 376                 __currentGridInfo.width = item.width;
 377                 __currentGridInfo.height = item.height;
 378                 return;
 379         }
 380
 381         resetCurrentGridInfo();
 382         setTouchModality(item, timestamp);
 383 }
 384
 385 void VoiceTouchEngine::setRejectedEventResult(int timestamp, JsonProvider &provider)
 386 {
 387         _I("[VoiceTouchEngine] Set command rejected event into output. timestamp(%d)", timestamp);
 388
 389         provider.setOutputEvent(MMI_KEY_REJECT);
 390         provider.setRejectEvent(MMI_REASON_NO_MATCHED_COMMANDS, timestamp);
 391 }
 392
 393 void VoiceTouchEngine::setSelectedItemResult(const ClickableItem &item, int timestamp, JsonProvider &provider)
 394 {
 395         _I("[VoiceTouchEngine] Set clicked object information into output. timestamp(%d). item(%s)", timestamp, item.objectId.c_str());
 396
 397         provider.setOutputEvent(MMI_KEY_UI_CLICKED_OBJECT);
 398         provider.setUiClickedObject(__appId.c_str(), timestamp, item.index, item.coordX, item.coordY, item.width, item.height, item.label.c_str());
 399 }
 400
 401 void VoiceTouchEngine::makeGridItemList(const grid_info_s &parentInfo, std::list<ClickableItem> &list)
 402 {
 403         _I("[VoiceTouchEngine] Set grid information into item list. Grid Step(%d)", parentInfo.currentStep);
 404         _D("[VoiceTouchEngine] Parent info. X(%d), Y(%d), W(%d), H(%d)", parentInfo.coordX, parentInfo.coordY, parentInfo.width, parentInfo.height);
 405
 406         int currentStep = parentInfo.currentStep;
 407         if (currentStep >= __MAXIMUM_GRID_STEP) {
 408                 return;
 409         }
 410
 411         int numX = __gridX[currentStep];
 412         int numY = __gridY[currentStep];
 413         int coordX = parentInfo.coordX;
 414         int coordY = parentInfo.coordY;
 415         int gridWidth = parentInfo.width / numX;
 416         int gridHeight = parentInfo.height / numY;
 417
 418         int remainedWidth = parentInfo.width % numX;
 419         int remainedHeight = parentInfo.height % numY;
 420
 421         int index = 1;
 422
 423         list.clear();
 424         for (int y = 0; y < numY; y++) {
 425                 for (int x = 0; x < numX; x++) {
 426                         ClickableItem grid;
 427                         grid.coordX = coordX + (x * gridWidth);
 428                         grid.coordY = coordY + (y * gridHeight);
 429
 430                         grid.width = (x == numX - 1 ? gridWidth + remainedWidth : gridWidth);
 431                         grid.height = (y == numY - 1 ? gridHeight + remainedHeight : gridHeight);
 432
 433                         grid.index = index++;
 434
 435                         list.push_back(grid);
 436                 }
 437         }
 438 }
 439
 440 void VoiceTouchEngine::setTouchModality(const ClickableItem &item, int timestamp)
 441 {
 442         __touchModality.timestamp = timestamp;
 443         __touchModality.mode = (__currentMode == VOICE_TOUCH_MODE_GRID ? MMI_VOICE_TOUCH_MODE_GRID : MMI_VOICE_TOUCH_MODE_TOOLTIP);
 444         __touchModality.object_id = item.objectId.c_str();
 445         __touchModality.coord_x = item.coordX + (item.width / 2);
 446         __touchModality.coord_y = item.coordY + (item.height / 2);
 447 }
 448
 449 bool VoiceTouchEngine::handleScreenAnalyzerInput(mmi_provider_event_screen_analyzer *screenAnalyzerEvent)
 450 {
 451         _I("[VoiceTouchEngine] Screen analyzer modality");
 452         if (screenAnalyzerEvent == nullptr) {
 453                 _E("[VoiceTouchEngine] Parameter is null");
 454                 return false;
 455         }
 456
 457         _I("[VoiceTouchEngine] Input data. num of items(%d)", screenAnalyzerEvent->n_items);
 458
 459         int timestamp = getCurrentTimestamp();
 460
 461         JsonProvider provider;
 462         provider.setInputEvent(MMI_KEY_VOICE_TOUCH);
 463
 464         __commandManager.cleanCommands();
 465         __itemList.clear();
 466         g_list_foreach(screenAnalyzerEvent->list, iterateObjectCallback, static_cast<gpointer>(this));
 467
 468         makeClickableItemInfo(timestamp, provider);
 469         invokeOutputResultCallback(provider.jsonToString());
 470
 471         return true;
 472 }
 473
 474 void VoiceTouchEngine::iterateObjectCallback(gpointer data, gpointer userData)
 475 {
 476         clickable_item* info = static_cast<clickable_item*>(data);
 477         VoiceTouchEngine *engine = static_cast<VoiceTouchEngine*>(userData);
 478
 479         ClickableItem item;
 480         item.coordX = info->coord_x;
 481         item.coordY = info->coord_y;
 482         item.width = info->width;
 483         item.height = info->height;
 484         item.objectId = string(info->object_id);
 485         item.label = string(info->label);
 486         item.index = engine->__itemList.size() + 1;
 487
 488         engine->__itemList.push_back(item);
 489         engine->__commandManager.addCommand(item);
 490 }
 491
 492 void VoiceTouchEngine::invokeOutputResultCallback(std::string outputResult)
 493 {
 494         if (__outputResultCallback == nullptr) {
 495                 return;
 496         }
 497
 498         __outputResultCallback(__appId.c_str(), outputResult.c_str(), __outputResultUserData);
 499 }
 500
 501 void VoiceTouchEngine::invokeOutputModalityCallback(void* outputModality)
 502 {
 503         if (__outputModalityCallback == nullptr) {
 504                 return;
 505         }
 506
 507         __outputModalityCallback(__appId.c_str(), outputModality, __outputModalityUserData);
 508 }
 509
 510 void VoiceTouchEngine::setOutputResultCallback(voice_touch_engine_output_result_cb callback, void *userData)
 511 {
 512         __outputResultCallback = callback;
 513         __outputResultUserData = userData;
 514 }
 515
 516 void VoiceTouchEngine::setOutputModalityCallback(voice_touch_engine_output_modality_cb callback, void *userData)
 517 {
 518         __outputModalityCallback = callback;
 519         __outputModalityUserData = userData;
 520 }