From 90ae82097b968a6c7d82af54681bd35c9265100d Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 21 Mar 2019 09:04:57 +0900 Subject: [PATCH] Extract engine management logic into CWakeupEngineManager Change-Id: I949b98002209092313689269b602601add2680e2 --- plugins/wakeup-manager/CMakeLists.txt | 1 + .../inc/wakeup_engine_manager.h | 174 ++++++ plugins/wakeup-manager/inc/wakeup_manager.h | 65 -- .../src/wakeup_audio_manager.cpp | 4 + .../src/wakeup_engine_manager.cpp | 542 ++++++++++++++++ plugins/wakeup-manager/src/wakeup_manager.cpp | 578 +++--------------- 6 files changed, 818 insertions(+), 546 deletions(-) create mode 100644 plugins/wakeup-manager/inc/wakeup_engine_manager.h create mode 100644 plugins/wakeup-manager/src/wakeup_engine_manager.cpp diff --git a/plugins/wakeup-manager/CMakeLists.txt b/plugins/wakeup-manager/CMakeLists.txt index a148556..91f0ed7 100644 --- a/plugins/wakeup-manager/CMakeLists.txt +++ b/plugins/wakeup-manager/CMakeLists.txt @@ -51,6 +51,7 @@ SET(SRCS src/wakeup_policy.cpp src/wakeup_policy_default.cpp src/wakeup_audio_manager.cpp + src/wakeup_engine_manager.cpp ) FOREACH(flag ${wmpkgs_CFLAGS}) diff --git a/plugins/wakeup-manager/inc/wakeup_engine_manager.h b/plugins/wakeup-manager/inc/wakeup_engine_manager.h new file mode 100644 index 0000000..bcfd9b8 --- /dev/null +++ b/plugins/wakeup-manager/inc/wakeup_engine_manager.h @@ -0,0 +1,174 @@ +/* + * Copyright 2018 Samsung Electronics Co., Ltd + * + * Licensed under the Flora License, Version 1.1 (the License); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://floralicense.org/license/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _WAKEUP_ENGINE_MANAGER_H_ +#define _WAKEUP_ENGINE_MANAGER_H_ + +#include "wakeup_manager.h" + +#include +#include +#include +#include + +namespace multiassistant +{ +namespace wakeup +{ + +using namespace std; + +/************************************************************************************** + *** Definitions for wakeup engine interface + *************************************************************************************/ +#define MA_WAKEUP_ENGINE_PATH tzplatform_mkpath(tzplatform_getid("TZ_SYS_RO_SHARE"), "multiassistant/engines") +/* Need to check whether this dedicated engine path also needs to be configurable */ +#define MA_WAKEUP_DEDICATED_ENGINE_PATH "shared/lib/libwakeup-engine.so" + +#define MA_WAKEUP_ENGINE_FUNC_INITIALIZE "wakeup_engine_initialize" +typedef int (*wakeup_engine_initialize)(void); +#define MA_WAKEUP_ENGINE_FUNC_DEINITIALIZE "wakeup_engine_deinitialize" +typedef int (*wakeup_engine_deinitialize)(void); +#define MA_WAKEUP_ENGINE_FUNC_ACTIVATE "wakeup_engine_activate" +typedef int (*wakeup_engine_activate)(void); +#define MA_WAKEUP_ENGINE_FUNC_DEACTIVATE "wakeup_engine_deactivate" +typedef int (*wakeup_engine_deactivate)(void); +#define MA_WAKEUP_ENGINE_FUNC_ADD_WAKEUP_WORD "wakeup_engine_add_wakeup_word" +typedef int (*wakeup_engine_add_wakeup_word)(const char* appid, const char* wakeup_word, const char* language); +#define MA_WAKEUP_ENGINE_FUNC_ADD_LANGUAGE "wakeup_engine_add_language" +typedef int (*wakeup_engine_add_language)(const char* appid, const char* language); +#define MA_WAKEUP_ENGINE_FUNC_SET_LANGUAGE "wakeup_engine_set_language" +typedef int (*wakeup_engine_set_language)(const char* language); +#define MA_WAKEUP_ENGINE_FUNC_UPDATE_MANAGER_STATE "wakeup_engine_update_manager_state" +typedef int (*wakeup_engine_update_manager_state)(wakeup_manager_state_e state); +#define MA_WAKEUP_ENGINE_FUNC_SET_AUDIO_FORMAT "wakeup_engine_set_audio_format" +typedef int (*wakeup_engine_set_audio_format)(int rate, int channel, int audio_type); +#define MA_WAKEUP_ENGINE_FUNC_GET_AUDIO_FORMAT "wakeup_engine_get_audio_format" +typedef int (*wakeup_engine_get_audio_format)(int* rate, int* channel, int* audio_type); +#define MA_WAKEUP_ENGINE_FUNC_FEED_AUDIO_DATA "wakeup_engine_feed_audio_data" +typedef int (*wakeup_engine_feed_audio_data)(long time, void* data, int len); +#define MA_WAKEUP_ENGINE_FUNC_GET_UTTERANCE_DATA_COUNT "wakeup_engine_get_utterance_data_count" +typedef int (*wakeup_engine_get_utterance_data_count)(void); +#define MA_WAKEUP_ENGINE_FUNC_GET_UTTERANCE_DATA "wakeup_engine_get_utterance_data" +typedef int (*wakeup_engine_get_utterance_data)(int index, wakeup_speech_data *data); +#define MA_WAKEUP_ENGINE_FUNC_SET_ASSISTANT_SPECIFIC_COMMAND "wakeup_engine_set_assistant_specific_command" +typedef int (*wakeup_engine_set_assistant_specific_command)(const char* appid, const char* command); +#define MA_WAKEUP_ENGINE_FUNC_SET_WAKEUP_EVENT_CALLBACK "wakeup_engine_set_wakeup_event_callback" +typedef int (*wakeup_engine_set_wakeup_event_callback)(wakeup_service_wakeup_event_cb callback, void* user_data); +#define MA_WAKEUP_ENGINE_FUNC_SET_SPEECH_STATUS_CALLBACK "wakeup_engine_set_speech_status_callback" +typedef int (*wakeup_engine_set_speech_status_callback)(wakeup_service_speech_status_cb callback, void* user_data); +#define MA_WAKEUP_ENGINE_FUNC_SET_ERROR_CALLBACK "wakeup_engine_set_error_callback" +typedef int (*wakeup_engine_set_error_callback)(wakeup_service_error_cb callback, void* user_data); +#define MA_WAKEUP_ENGINE_FUNC_SET_AUDIO_DATA_REQUIRE_STATUS_CALLBACK "wakeup_engine_set_audio_data_require_status_callback" +typedef int (*wakeup_engine_set_audio_data_require_status_callback)(wakeup_service_audio_data_require_status_cb callback, void* user_data); + +typedef struct { + wakeup_engine_initialize initialize; + wakeup_engine_deinitialize deinitialize; + wakeup_engine_activate activate; + wakeup_engine_deactivate deactivate; + wakeup_engine_add_wakeup_word add_wakeup_word; + wakeup_engine_add_language add_language; + wakeup_engine_set_language set_language; + wakeup_engine_update_manager_state update_manager_state; + wakeup_engine_set_audio_format set_audio_format; + wakeup_engine_get_audio_format get_audio_format; + wakeup_engine_feed_audio_data feed_audio_data; + wakeup_engine_get_utterance_data_count get_utterance_data_count; + wakeup_engine_get_utterance_data get_utterance_data; + wakeup_engine_set_assistant_specific_command set_assistant_specific_command; + wakeup_engine_set_wakeup_event_callback set_wakeup_event_callback; + wakeup_engine_set_speech_status_callback set_speech_status_callback; + wakeup_engine_set_error_callback set_error_callback; + wakeup_engine_set_audio_data_require_status_callback set_audio_data_require_status_callback; +} wakeup_engine_interface; + +class IEngineEventObserver +{ +public: + virtual ~IEngineEventObserver() {} + virtual bool on_wakeup_event(string engine_name, wakeup_event_info info) = 0; + virtual bool on_speech_status(string engine_name, wakeup_service_speech_status_e status) = 0; + virtual bool on_error(string engine_name, int error_code, string error_message) = 0; + virtual bool on_audio_data_require_status(string engine_name, bool require) = 0; + + virtual bool on_streaming_audio_data( + wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) = 0; +}; + +class CWakeupEngineManager +{ +public: + CWakeupEngineManager(); + virtual ~CWakeupEngineManager(); + + CWakeupEngineManager(const CWakeupEngineManager&) = delete; + CWakeupEngineManager& operator=(const CWakeupEngineManager&) = delete; + + void initialize(); + void deinitialize(); + + void subscribe(IEngineEventObserver *observer); + void unsubscribe(IEngineEventObserver *observer); + + bool get_audio_data_required(); + void set_selected_wakeup_info(wakeup_event_info info); + + void start_streaming_current_utterance_data(); + void stop_streaming_current_utterance_data(); + + void update_manager_state(wakeup_manager_state_e state); + + void engine_add_target_assistant(const string engine_name, const string appid); + void engine_add_wakeup_word(const string appid, const string wakeup_word, const string language); + void engine_set_assistant_specific_command(const string appid, const string command); + void engine_feed_audio_data(long time, void* data, int len); + + bool on_wakeup_event(string engine_name, wakeup_event_info info); + bool on_speech_status(string engine_name, wakeup_service_speech_status_e status); + bool on_error(string engine_name, int error_code, string error_message); + bool on_audio_data_require_status(string engine_name, bool require); +private: + typedef struct { + string engine_name; + bool active{false}; + bool enabled{false}; + bool audio_data_require_status{false}; + string engine_path; + wakeup_engine_interface interface{nullptr, }; + vector assistant_list; + void *engine_handle{nullptr}; + } EngineInfo; + + void add_engine_directory(const string name, const string path); + void add_engine(const string name, const string path); + + vector mObservers; + + void streaming_speech_data_thread_func(); + + vector mEngineInfo; + const EngineInfo* mSelectedEngine{nullptr}; + bool mAudioDataRequired{false}; + + thread mStreamingThread; + atomic_bool mStopStreamingThread{false}; +}; + +} // wakeup +} // multiassistant + +#endif /* _WAKEUP_ENGINE_MANAGER_H_ */ diff --git a/plugins/wakeup-manager/inc/wakeup_manager.h b/plugins/wakeup-manager/inc/wakeup_manager.h index 962f8b0..7886b9e 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_manager.h @@ -131,76 +131,11 @@ EXPORT_API int wakeup_manager_set_speech_status_callback(wakeup_service_speech_s EXPORT_API int wakeup_manager_set_error_callback(wakeup_service_error_cb callback, void* user_data); -/************************************************************************************** - *** Definitions for wakeup engine interface - *************************************************************************************/ -#define MA_WAKEUP_ENGINE_PATH tzplatform_mkpath(tzplatform_getid("TZ_SYS_RO_SHARE"), "multiassistant/engines") -/* Need to check whether this dedicated engine path also needs to be configurable */ -#define MA_WAKEUP_DEDICATED_ENGINE_PATH "shared/lib/libwakeup-engine.so" - typedef enum { MA_PLUGIN_EVENT_VOICE_KEY_PRESSED = 0, MA_PLUGIN_EVENT_VOICE_KEY_RELEASED, } ma_plugin_event_e; -#define MA_WAKEUP_ENGINE_FUNC_INITIALIZE "wakeup_engine_initialize" -typedef int (*wakeup_engine_initialize)(void); -#define MA_WAKEUP_ENGINE_FUNC_DEINITIALIZE "wakeup_engine_deinitialize" -typedef int (*wakeup_engine_deinitialize)(void); -#define MA_WAKEUP_ENGINE_FUNC_ACTIVATE "wakeup_engine_activate" -typedef int (*wakeup_engine_activate)(void); -#define MA_WAKEUP_ENGINE_FUNC_DEACTIVATE "wakeup_engine_deactivate" -typedef int (*wakeup_engine_deactivate)(void); -#define MA_WAKEUP_ENGINE_FUNC_ADD_WAKEUP_WORD "wakeup_engine_add_wakeup_word" -typedef int (*wakeup_engine_add_wakeup_word)(const char* appid, const char* wakeup_word, const char* language); -#define MA_WAKEUP_ENGINE_FUNC_ADD_LANGUAGE "wakeup_engine_add_language" -typedef int (*wakeup_engine_add_language)(const char* appid, const char* language); -#define MA_WAKEUP_ENGINE_FUNC_SET_LANGUAGE "wakeup_engine_set_language" -typedef int (*wakeup_engine_set_language)(const char* language); -#define MA_WAKEUP_ENGINE_FUNC_UPDATE_MANAGER_STATE "wakeup_engine_update_manager_state" -typedef int (*wakeup_engine_update_manager_state)(wakeup_manager_state_e state); -#define MA_WAKEUP_ENGINE_FUNC_SET_AUDIO_FORMAT "wakeup_engine_set_audio_format" -typedef int (*wakeup_engine_set_audio_format)(int rate, int channel, int audio_type); -#define MA_WAKEUP_ENGINE_FUNC_GET_AUDIO_FORMAT "wakeup_engine_get_audio_format" -typedef int (*wakeup_engine_get_audio_format)(int* rate, int* channel, int* audio_type); -#define MA_WAKEUP_ENGINE_FUNC_FEED_AUDIO_DATA "wakeup_engine_feed_audio_data" -typedef int (*wakeup_engine_feed_audio_data)(long time, void* data, int len); -#define MA_WAKEUP_ENGINE_FUNC_GET_UTTERANCE_DATA_COUNT "wakeup_engine_get_utterance_data_count" -typedef int (*wakeup_engine_get_utterance_data_count)(void); -#define MA_WAKEUP_ENGINE_FUNC_GET_UTTERANCE_DATA "wakeup_engine_get_utterance_data" -typedef int (*wakeup_engine_get_utterance_data)(int index, wakeup_speech_data *data); -#define MA_WAKEUP_ENGINE_FUNC_SET_ASSISTANT_SPECIFIC_COMMAND "wakeup_engine_set_assistant_specific_command" -typedef int (*wakeup_engine_set_assistant_specific_command)(const char* appid, const char* command); -#define MA_WAKEUP_ENGINE_FUNC_SET_WAKEUP_EVENT_CALLBACK "wakeup_engine_set_wakeup_event_callback" -typedef int (*wakeup_engine_set_wakeup_event_callback)(wakeup_service_wakeup_event_cb callback, void* user_data); -#define MA_WAKEUP_ENGINE_FUNC_SET_SPEECH_STATUS_CALLBACK "wakeup_engine_set_speech_status_callback" -typedef int (*wakeup_engine_set_speech_status_callback)(wakeup_service_speech_status_cb callback, void* user_data); -#define MA_WAKEUP_ENGINE_FUNC_SET_ERROR_CALLBACK "wakeup_engine_set_error_callback" -typedef int (*wakeup_engine_set_error_callback)(wakeup_service_error_cb callback, void* user_data); -#define MA_WAKEUP_ENGINE_FUNC_SET_AUDIO_DATA_REQUIRE_STATUS_CALLBACK "wakeup_engine_set_audio_data_require_status_callback" -typedef int (*wakeup_engine_set_audio_data_require_status_callback)(wakeup_service_audio_data_require_status_cb callback, void* user_data); - -typedef struct { - wakeup_engine_initialize initialize; - wakeup_engine_deinitialize deinitialize; - wakeup_engine_activate activate; - wakeup_engine_deactivate deactivate; - wakeup_engine_add_wakeup_word add_wakeup_word; - wakeup_engine_add_language add_language; - wakeup_engine_set_language set_language; - wakeup_engine_update_manager_state update_manager_state; - wakeup_engine_set_audio_format set_audio_format; - wakeup_engine_get_audio_format get_audio_format; - wakeup_engine_feed_audio_data feed_audio_data; - wakeup_engine_get_utterance_data_count get_utterance_data_count; - wakeup_engine_get_utterance_data get_utterance_data; - wakeup_engine_set_assistant_specific_command set_assistant_specific_command; - wakeup_engine_set_wakeup_event_callback set_wakeup_event_callback; - wakeup_engine_set_speech_status_callback set_speech_status_callback; - wakeup_engine_set_error_callback set_error_callback; - wakeup_engine_set_audio_data_require_status_callback set_audio_data_require_status_callback; -} wakeup_engine_interface; - #ifdef __cplusplus } #endif diff --git a/plugins/wakeup-manager/src/wakeup_audio_manager.cpp b/plugins/wakeup-manager/src/wakeup_audio_manager.cpp index 00ced81..b43fb05 100644 --- a/plugins/wakeup-manager/src/wakeup_audio_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_audio_manager.cpp @@ -501,6 +501,10 @@ void CAudioManager::finalize_speech_data() void CAudioManager::start_streaming_current_utterance_data(bool from_start_time, long start_time) { + if (mStreamingThread.joinable()) { + MWR_LOGE("ERROR : mStreamingThread is joinable, will not start a new thread"); + return; + } if (from_start_time) { mSpeechData.clear(); mStreamingThread = std::thread(&CAudioManager::streaming_background_data_thread_func, this, start_time); diff --git a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp new file mode 100644 index 0000000..94c0263 --- /dev/null +++ b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp @@ -0,0 +1,542 @@ +#include "wakeup_engine_manager.h" +#include "wakeup_manager_main.h" + +#include +#include +#include + +namespace multiassistant +{ +namespace wakeup +{ + +CWakeupEngineManager::CWakeupEngineManager() +{ +} + +CWakeupEngineManager::~CWakeupEngineManager() +{ +} + +void CWakeupEngineManager::initialize() +{ + DIR* dp = opendir(MA_WAKEUP_ENGINE_PATH); + if (nullptr == dp) { + MWR_LOGD("Failed opening directory : %s", (const char*)MA_WAKEUP_ENGINE_PATH); + } else { + struct dirent *dirp = nullptr; + char dirpath[_POSIX_PATH_MAX]; + do { + dirp = readdir(dp); + + if (nullptr != dirp) { + if (!strcmp(".", dirp->d_name) || !strcmp("..", dirp->d_name)) + continue; + + if (DT_DIR != dirp->d_type) /* If not a directory */ + continue; + + int dirpath_len = strlen(MA_WAKEUP_ENGINE_PATH) + strlen(dirp->d_name) + 1; + if (dirpath_len >= _POSIX_PATH_MAX) { + MWR_LOGD("File path is too long : %s", dirp->d_name); + closedir(dp); + return; + } + + memset(dirpath, '\0', _POSIX_PATH_MAX); + snprintf(dirpath, _POSIX_PATH_MAX, "%s/%s", + (const char*)(MA_WAKEUP_ENGINE_PATH), dirp->d_name); + + add_engine_directory(string{dirp->d_name}, dirpath); + } + } while (nullptr != dirp); + + closedir(dp); + } +} + +void CWakeupEngineManager::deinitialize() +{ + for (auto& info : mEngineInfo) { + if (info.interface.set_wakeup_event_callback) { + info.interface.set_wakeup_event_callback(nullptr, nullptr); + } + if (info.interface.set_speech_status_callback) { + info.interface.set_speech_status_callback(nullptr, nullptr); + } + if (info.interface.set_error_callback) { + info.interface.set_error_callback(nullptr, nullptr); + } + if (info.interface.set_audio_data_require_status_callback) { + info.interface.set_audio_data_require_status_callback(nullptr, nullptr); + } + if (info.interface.deinitialize) { + info.interface.deinitialize(); + } + if (info.engine_handle) { + dlclose(info.engine_handle); + info.engine_handle = nullptr; + } + } + mSelectedEngine = nullptr; + mEngineInfo.clear(); +} + +void CWakeupEngineManager::subscribe(IEngineEventObserver *observer) +{ + mObservers.push_back(observer); + MWR_LOGD("Added Observer : %p %zu", observer, mObservers.size()); +} + +void CWakeupEngineManager::unsubscribe(IEngineEventObserver *observer) +{ + auto iter = find(mObservers.begin(), mObservers.end(), observer); + if (iter != mObservers.end()) { + mObservers.erase(iter); + } +} + +bool CWakeupEngineManager::get_audio_data_required() +{ + return mAudioDataRequired; +} + +void CWakeupEngineManager::set_selected_wakeup_info(wakeup_event_info wakeup_info) +{ + for (const auto& info : mEngineInfo) { + auto iter = find( + info.assistant_list.begin(), + info.assistant_list.end(), + string{wakeup_info.wakeup_appid}); + if (iter != info.assistant_list.end()) { + mSelectedEngine = &info; + MWR_LOGD("Selected : %s", info.engine_name.c_str()); + } + } +} + +void CWakeupEngineManager::streaming_speech_data_thread_func() +{ + MWR_LOGD("[ENTER]"); + + if (nullptr == mSelectedEngine) + return; + + const wakeup_engine_interface *interface = &(mSelectedEngine->interface); + + if (NULL == interface || + NULL == interface->get_utterance_data || + NULL == interface->get_utterance_data_count) + return; + + MWR_LOGD("data_count : %d", interface->get_utterance_data_count()); + + wakeup_speech_data speech_data; + int index = 0; + + while (!(mStopStreamingThread.load())) { + int ret = -1; + int cnt = 0; + + /* get feedback data */ + if (interface && interface->get_utterance_data) { + ret = interface->get_utterance_data(index, &speech_data); + if (0 != ret) { + /* empty queue */ + MWR_LOGD("[DEBUG] No feedback data. Waiting mode : %d", ret); + + /* waiting */ + while (1) { + this_thread::sleep_for(chrono::milliseconds(10)); + if (index < interface->get_utterance_data_count()) { + MWR_LOGI("[INFO] Resume thread"); + break; + } + if (200 < cnt) { + MWR_LOGE("[ERROR] Wrong request, there's no pcm data"); + for (const auto& observer : mObservers) { + if (observer) { + if (!observer->on_streaming_audio_data( + WAKEUP_SPEECH_STREAMING_EVENT_FAIL, NULL, 0)) { + LOGE("[Recorder WARNING] One of the observer returned false"); + } + } + } + return; + } + cnt++; + } + MWR_LOGI("[INFO] Finish to wait for new feedback data come"); + + /* resume feedback thread */ + continue; + } + + for (const auto& observer : mObservers) { + if (observer) { + if (!observer->on_streaming_audio_data( + speech_data.event, speech_data.buffer, speech_data.len)) { + LOGE("[Recorder WARNING] One of the observer returned false"); + } + } + } + + if (WAKEUP_SPEECH_STREAMING_EVENT_FINISH == speech_data.event) { + MWR_LOGI("[INFO] Finish to get and send speech data"); + break; + } + + index++; + } + } +} + +void CWakeupEngineManager::start_streaming_current_utterance_data() +{ + if (mStreamingThread.joinable()) { + MWR_LOGE("ERROR : mStreamingThread is joinable, will not start a new thread"); + return; + } + mStreamingThread = thread(&CWakeupEngineManager::streaming_speech_data_thread_func, this); +} + +void CWakeupEngineManager::stop_streaming_current_utterance_data() +{ + if (mStreamingThread.joinable()) { + MWR_LOGD("mStreamingThread is joinable, trying join()"); + mStopStreamingThread.store(true); + mStreamingThread.join(); + } + mStopStreamingThread.store(false); +} + +void CWakeupEngineManager::update_manager_state(wakeup_manager_state_e state) +{ + for (const auto& info : mEngineInfo) { + if (info.interface.update_manager_state) { + info.interface.update_manager_state(state); + } + } +} + +void CWakeupEngineManager::engine_add_target_assistant(const string engine_name, const string appid) +{ + const auto& iter = find_if(mEngineInfo.begin(), mEngineInfo.end(), + [engine_name](const EngineInfo& info) { + return (0 == info.engine_name.compare(engine_name)); + }); + + if (mEngineInfo.end() == iter) { + /* Not found, add new library */ + pkgmgrinfo_appinfo_h handle; + int ret = pkgmgrinfo_appinfo_get_appinfo(engine_name.c_str(), &handle); + if (PMINFO_R_OK == ret) { + char *root_path = nullptr; + ret = pkgmgrinfo_appinfo_get_root_path(handle, &root_path); + if (PMINFO_R_OK == ret && nullptr != root_path) { + string path = root_path; + path += "/"; + path += MA_WAKEUP_DEDICATED_ENGINE_PATH; + add_engine(engine_name, path); + } + pkgmgrinfo_appinfo_destroy_appinfo(handle); + } + /* Find again to add appid to the newly created engine's assistant list */ + const auto &new_iter = find_if(mEngineInfo.begin(), mEngineInfo.end(), + [engine_name](const EngineInfo& info) { + return (0 == info.engine_name.compare(engine_name)); + }); + if (mEngineInfo.end() != new_iter) { + new_iter->assistant_list.push_back(appid); + } + } else { + /* If the engine already exists, simply add the appid to the assistant list */ + iter->assistant_list.push_back(appid); + } +} + +void CWakeupEngineManager::engine_add_wakeup_word(const string appid, const string wakeup_word, const string language) +{ + for (const auto& info : mEngineInfo) { + auto iter = find( + info.assistant_list.begin(), + info.assistant_list.end(), + appid); + if (iter != info.assistant_list.end()) { + if (info.interface.add_wakeup_word) { + info.interface.add_wakeup_word(appid.c_str(), wakeup_word.c_str(), language.c_str()); + } + } + } +} + +void CWakeupEngineManager::engine_set_assistant_specific_command(const string appid, const string command) +{ + for (const auto& info : mEngineInfo) { + auto iter = find( + info.assistant_list.begin(), + info.assistant_list.end(), + appid); + if (iter != info.assistant_list.end()) { + if (info.interface.set_assistant_specific_command) { + info.interface.set_assistant_specific_command(appid.c_str(), command.c_str()); + } + } + } +} + +void CWakeupEngineManager::engine_feed_audio_data(long time, void* data, int len) +{ + for (const auto& info : mEngineInfo) { + if (info.audio_data_require_status && + info.interface.feed_audio_data) { + int ret = info.interface.feed_audio_data(time, data, len); + if (0 != ret) { + LOGE("[ERROR] Fail to feed speech data, ret(%d) : %s", ret, info.engine_name.c_str()); + } + } + } +} + +bool CWakeupEngineManager::on_wakeup_event(string engine_name, wakeup_event_info info) +{ + MWR_LOGD("[ENTER]"); + + for (const auto& observer : mObservers) { + if (observer) { + if (!observer->on_wakeup_event(engine_name, info)) { + LOGE("[Recorder WARNING] One of the observer returned false"); + } + } + } + + return true; +} + +bool CWakeupEngineManager::on_speech_status(string engine_name, wakeup_service_speech_status_e status) +{ + MWR_LOGD("[ENTER]"); + + for (const auto& observer : mObservers) { + if (observer) { + if (!observer->on_speech_status(engine_name, status)) { + LOGE("[Recorder WARNING] One of the observer returned false"); + } + } + } + + return true; +} + +bool CWakeupEngineManager::on_error(string engine_name, int error_code, string error_message) +{ + MWR_LOGD("[ENTER]"); + + for (const auto& observer : mObservers) { + if (observer) { + if (!observer->on_error(engine_name, error_code, error_message)) { + LOGE("[Recorder WARNING] One of the observer returned false"); + } + } + } + + return true; +} + +bool CWakeupEngineManager::on_audio_data_require_status(string engine_name, bool require) +{ + MWR_LOGD("[ENTER]"); + + bool found = false; + // LOCK REQUIRED + int count = 0; + for (auto& info : mEngineInfo) { + if (info.engine_name.compare(engine_name) == 0) { + found = true; + info.audio_data_require_status = require; + } + if (info.enabled && info.audio_data_require_status) { + count++; + } + } + MWR_LOGD("count : %d", count); + if (count > 0) { + mAudioDataRequired = true; + } else { + mAudioDataRequired = false; + } + + if (found) { + for (const auto& observer : mObservers) { + if (observer) { + if (!observer->on_audio_data_require_status(engine_name, require)) { + LOGE("[Recorder WARNING] One of the observer returned false"); + } + } + } + } + // UNLOCK REQUIRED + return true; +} + +void CWakeupEngineManager::add_engine(const string name, const string path) +{ + sleep(10); + MWR_LOGD("Name (%s), Filepath(%s)", name.c_str(), path.c_str()); + + char* error = NULL; + EngineInfo info; + info.engine_handle = dlopen(path.c_str(), RTLD_LAZY); + if (nullptr != (error = dlerror()) || nullptr == info.engine_handle) { + MWR_LOGD("[ERROR] Fail to dlopen(%s), error(%s)", path.c_str(), error); + if (info.engine_handle) dlclose(info.engine_handle); + return; + } + + info.interface.initialize = + (wakeup_engine_initialize)dlsym(info.engine_handle, + MA_WAKEUP_ENGINE_FUNC_INITIALIZE); + info.interface.deinitialize = + (wakeup_engine_deinitialize)dlsym(info.engine_handle, + MA_WAKEUP_ENGINE_FUNC_DEINITIALIZE); + info.interface.activate = + (wakeup_engine_activate)dlsym(info.engine_handle, + MA_WAKEUP_ENGINE_FUNC_ACTIVATE); + info.interface.deactivate = + (wakeup_engine_deactivate)dlsym(info.engine_handle, + MA_WAKEUP_ENGINE_FUNC_DEACTIVATE); + info.interface.add_wakeup_word = + (wakeup_engine_add_wakeup_word)dlsym(info.engine_handle, + MA_WAKEUP_ENGINE_FUNC_ADD_WAKEUP_WORD); + info.interface.add_language = + (wakeup_engine_add_language)dlsym(info.engine_handle, + MA_WAKEUP_ENGINE_FUNC_ADD_LANGUAGE); + info.interface.set_language = + (wakeup_engine_set_language)dlsym(info.engine_handle, + MA_WAKEUP_ENGINE_FUNC_SET_LANGUAGE); + info.interface.update_manager_state = + (wakeup_engine_update_manager_state)dlsym(info.engine_handle, + MA_WAKEUP_ENGINE_FUNC_UPDATE_MANAGER_STATE); + info.interface.set_audio_format = + (wakeup_engine_set_audio_format)dlsym(info.engine_handle, + MA_WAKEUP_ENGINE_FUNC_SET_AUDIO_FORMAT); + info.interface.get_audio_format = + (wakeup_engine_get_audio_format)dlsym(info.engine_handle, + MA_WAKEUP_ENGINE_FUNC_GET_AUDIO_FORMAT); + info.interface.feed_audio_data = + (wakeup_engine_feed_audio_data)dlsym(info.engine_handle, + MA_WAKEUP_ENGINE_FUNC_FEED_AUDIO_DATA); + info.interface.get_utterance_data_count = + (wakeup_engine_get_utterance_data_count)dlsym(info.engine_handle, + MA_WAKEUP_ENGINE_FUNC_GET_UTTERANCE_DATA_COUNT); + info.interface.get_utterance_data = + (wakeup_engine_get_utterance_data)dlsym(info.engine_handle, + MA_WAKEUP_ENGINE_FUNC_GET_UTTERANCE_DATA); + info.interface.set_assistant_specific_command = + (wakeup_engine_set_assistant_specific_command)dlsym(info.engine_handle, + MA_WAKEUP_ENGINE_FUNC_SET_ASSISTANT_SPECIFIC_COMMAND); + info.interface.set_wakeup_event_callback = + (wakeup_engine_set_wakeup_event_callback)dlsym(info.engine_handle, + MA_WAKEUP_ENGINE_FUNC_SET_WAKEUP_EVENT_CALLBACK); + info.interface.set_speech_status_callback = + (wakeup_engine_set_speech_status_callback)dlsym(info.engine_handle, + MA_WAKEUP_ENGINE_FUNC_SET_SPEECH_STATUS_CALLBACK); + info.interface.set_error_callback = + (wakeup_engine_set_error_callback)dlsym(info.engine_handle, + MA_WAKEUP_ENGINE_FUNC_SET_ERROR_CALLBACK); + info.interface.set_audio_data_require_status_callback = + (wakeup_engine_set_audio_data_require_status_callback)dlsym(info.engine_handle, + MA_WAKEUP_ENGINE_FUNC_SET_AUDIO_DATA_REQUIRE_STATUS_CALLBACK); + + info.engine_path = path; + info.engine_name = name; + + info.active = false; + /* We'll need to check vconf for enabled wakeup engines */ + info.enabled = true; + info.audio_data_require_status = false; + + /* All the necessary information has already been set properly */ + mEngineInfo.push_back(info); + + MWR_LOGD("Initializing wakeup engine : %s %p", + info.engine_path.c_str(), + info.interface.initialize); + + /* Workaround for registering C-style callbacks */ + typedef struct { + CWakeupEngineManager *manager; + string engine_name; + } CallbackUserData; + + static vector callback_user_data; + + CallbackUserData user_data; + user_data.manager = this; + user_data.engine_name = info.engine_name; + callback_user_data.push_back(user_data); + + if (info.interface.set_wakeup_event_callback) { + info.interface.set_wakeup_event_callback( + [](wakeup_event_info info, void* user_data) { + CallbackUserData *callback_user_data = static_cast(user_data); + if (nullptr == callback_user_data) return; + if (nullptr == callback_user_data->manager) return; + callback_user_data->manager->on_wakeup_event(callback_user_data->engine_name, info); + }, &(callback_user_data.back())); + } + + if (info.interface.set_audio_data_require_status_callback) { + info.interface.set_audio_data_require_status_callback( + [](bool require, void* user_data) { + CallbackUserData *callback_user_data = static_cast(user_data); + if (nullptr == callback_user_data) return; + if (nullptr == callback_user_data->manager) return; + callback_user_data->manager->on_audio_data_require_status(callback_user_data->engine_name, require); + }, &(callback_user_data.back())); + } + + if (info.interface.initialize) { + info.interface.initialize(); + } +} + +void CWakeupEngineManager::add_engine_directory(const string name, const string path) +{ + if (0 == path.size()) return; + + DIR* dp = opendir(path.c_str()); + if (NULL == dp) { + MWR_LOGD("Failed opening directory : %s", path.c_str()); + } else { + struct dirent *dirp = NULL; + string filepath; + do { + dirp = readdir(dp); + + if (NULL != dirp) { + if (!strcmp(".", dirp->d_name) || !strcmp("..", dirp->d_name)) + continue; + + if (DT_REG != dirp->d_type) /* If not a regular file */ + continue; + + filepath = path; + filepath += "/"; + filepath += dirp->d_name; + + if (filepath.length() >= _POSIX_PATH_MAX) { + MWR_LOGD("File path is too long : %s", filepath.c_str()); + closedir(dp); + return; + } + add_engine(name, filepath); + } + } while (NULL != dirp); + + closedir(dp); + } +} + +} // wakeup +} // multiassistant diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index 32521fe..37255fd 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -15,7 +15,6 @@ */ #include #include -#include #include #include @@ -23,11 +22,10 @@ #include #include -#include - #include "wakeup_manager_main.h" #include "wakeup_manager.h" #include "wakeup_settings.h" +#include "wakeup_engine_manager.h" #include "wakeup_audio_manager.h" #include "wakeup_policy_default.h" @@ -46,7 +44,6 @@ Ecore_Event_Handler* _key_up_handler = NULL; using namespace multiassistant::wakeup; -static bool g_audio_data_required = false; bool g_voice_key_pressed = false; static std::thread g_engine_data_thread; @@ -70,23 +67,6 @@ static void* g_speech_status_user_data; static wakeup_service_error_cb g_error_cb; static void* g_error_user_data; -#define MAX_WAKEUP_ENGINE_NUM 10 -static int g_engine_count = 0; - -typedef struct { - bool active{false}; - bool enabled{false}; - bool audio_data_require_status{false}; - char engine_name[_POSIX_PATH_MAX]; - char engine_path[_POSIX_PATH_MAX]; - void *engine_handle{nullptr}; - wakeup_engine_interface interface{nullptr, }; - std::vector assistant_list; -} wakeup_engine_info; - -static int g_wakeup_engine_selected = -1; -static wakeup_engine_info g_wakeup_engine_info[MAX_WAKEUP_ENGINE_NUM]; - static char* g_current_language = NULL; static wakeup_manager_state_e g_wakeup_manager_state; @@ -94,6 +74,20 @@ static wakeup_event_info g_last_wakeup_event_info; static CWakeupSettings g_wakeup_settings; +class CEngineEventObserver : public IEngineEventObserver +{ +public: + bool on_wakeup_event(string engine_name, wakeup_event_info info) override; + bool on_speech_status(string engine_name, wakeup_service_speech_status_e status) override; + bool on_error(string engine_name, int error_code, string error_message) override; + bool on_audio_data_require_status(string engine_name, bool require) override; + + bool on_streaming_audio_data( + wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) override; +}; +static CWakeupEngineManager g_wakeup_engine_manager; +static CEngineEventObserver g_engine_event_observer; + enum STREAMING_MODE { STREAMING_MODE_NONE, STREAMING_MODE_UTTERANCE, @@ -115,12 +109,17 @@ static CWakeupEventObserver g_wakeup_event_observer; class CAudioDataObserver : public IAudioDataObserver { public: + CAudioDataObserver(CWakeupEngineManager *manager) : mEngineManager{manager} + { + } bool on_recording_audio_data(long time, void* data, int len) override; bool on_streaming_audio_data( wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) override; +private: + CWakeupEngineManager *mEngineManager{nullptr}; }; static CAudioManager g_audio_manager; -static CAudioDataObserver g_audio_data_observer; +static CAudioDataObserver g_audio_data_observer{&g_wakeup_engine_manager}; #ifdef TV_PRODUCT Eina_Bool _key_down_cb(void* data, int type, void* event) @@ -208,283 +207,6 @@ bool _delete_key_cb(void) return true; } #endif -static void wakeup_engine_wakeup_event_cb(wakeup_event_info info, void* user_data) -{ - MWR_LOGD("[ENTER]"); - if (NULL == user_data) return; - - for (int loop = 0;loop < g_engine_count;loop++) { - if (strncmp(g_wakeup_engine_info[loop].engine_name, - (const char*)user_data, _POSIX_PATH_MAX) == 0) { - if (g_wakeup_policy) { - g_wakeup_policy->wakeup_candidate(info); - } - //g_wakeup_event_cb(event, g_wakeup_event_user_data); - } - } -} - -static void wakeup_engine_speech_status_cb(wakeup_service_speech_status_e status, void* user_data) -{ - MWR_LOGD("[ENTER]"); - if (NULL == user_data) return; - - for (int loop = 0;loop < g_engine_count;loop++) { - if (strncmp(g_wakeup_engine_info[loop].engine_name, - (const char*)user_data, _POSIX_PATH_MAX) == 0) { - } - } -} - -static void wakeup_engine_error_cb(int error, const char* err_msg, void* user_data) -{ - MWR_LOGD("[ENTER]"); - if (NULL == user_data) return; - - for (int loop = 0;loop < g_engine_count;loop++) { - if (strncmp(g_wakeup_engine_info[loop].engine_name, - (const char*)user_data, _POSIX_PATH_MAX) == 0) { - } - } -} - -static void wakeup_engine_audio_data_require_status_cb(bool require, void* user_data) -{ - MWR_LOGD("[ENTER]"); - if (NULL == user_data) return; - - bool prev_audio_data_required = g_audio_data_required; - // LOCK REQUIRED - int audio_data_require_count = 0; - for (int loop = 0;loop < g_engine_count;loop++) { - if (strncmp(g_wakeup_engine_info[loop].engine_name, - (const char*)user_data, _POSIX_PATH_MAX) == 0) { - g_wakeup_engine_info[loop].audio_data_require_status = require; - } - if (g_wakeup_engine_info[loop].enabled && - g_wakeup_engine_info[loop].audio_data_require_status) { - audio_data_require_count++; - } - } - MWR_LOGD("audio_data_require_count : %d", audio_data_require_count); - if (audio_data_require_count > 0) { - g_audio_data_required = true; - if (g_audio_data_required != prev_audio_data_required && - g_voice_key_pressed != true) { - g_audio_manager.start_recording(); - } - } else { - g_audio_data_required = false; - if (g_audio_data_required != prev_audio_data_required && - g_voice_key_pressed != true) { - g_audio_manager.stop_recording(); - } - } - // UNLOCK REQUIRED -} - -static void wakeup_engine_add_library(const char* name, const char* path) -{ - int index = g_engine_count; - if (index >= MAX_WAKEUP_ENGINE_NUM || index < 0) return; - if (nullptr == name || nullptr == path) return; - - MWR_LOGD("Name (%s), Filepath(%s)", name, path); - - char* error = NULL; - g_wakeup_engine_info[index].engine_handle = dlopen(path, RTLD_LAZY); - if (nullptr != (error = dlerror()) || nullptr == g_wakeup_engine_info[index].engine_handle) { - MWR_LOGD("[ERROR] Fail to dlopen(%s), error(%s)", path, error); - } else { - g_wakeup_engine_info[index].interface.initialize = - (wakeup_engine_initialize)dlsym(g_wakeup_engine_info[index].engine_handle, - MA_WAKEUP_ENGINE_FUNC_INITIALIZE); - g_wakeup_engine_info[index].interface.deinitialize = - (wakeup_engine_deinitialize)dlsym(g_wakeup_engine_info[index].engine_handle, - MA_WAKEUP_ENGINE_FUNC_DEINITIALIZE); - g_wakeup_engine_info[index].interface.activate = - (wakeup_engine_activate)dlsym(g_wakeup_engine_info[index].engine_handle, - MA_WAKEUP_ENGINE_FUNC_ACTIVATE); - g_wakeup_engine_info[index].interface.deactivate = - (wakeup_engine_deactivate)dlsym(g_wakeup_engine_info[index].engine_handle, - MA_WAKEUP_ENGINE_FUNC_DEACTIVATE); - g_wakeup_engine_info[index].interface.add_wakeup_word = - (wakeup_engine_add_wakeup_word)dlsym(g_wakeup_engine_info[index].engine_handle, - MA_WAKEUP_ENGINE_FUNC_ADD_WAKEUP_WORD); - g_wakeup_engine_info[index].interface.add_language = - (wakeup_engine_add_language)dlsym(g_wakeup_engine_info[index].engine_handle, - MA_WAKEUP_ENGINE_FUNC_ADD_LANGUAGE); - g_wakeup_engine_info[index].interface.set_language = - (wakeup_engine_set_language)dlsym(g_wakeup_engine_info[index].engine_handle, - MA_WAKEUP_ENGINE_FUNC_SET_LANGUAGE); - g_wakeup_engine_info[index].interface.update_manager_state = - (wakeup_engine_update_manager_state)dlsym(g_wakeup_engine_info[index].engine_handle, - MA_WAKEUP_ENGINE_FUNC_UPDATE_MANAGER_STATE); - g_wakeup_engine_info[index].interface.set_audio_format = - (wakeup_engine_set_audio_format)dlsym(g_wakeup_engine_info[index].engine_handle, - MA_WAKEUP_ENGINE_FUNC_SET_AUDIO_FORMAT); - g_wakeup_engine_info[index].interface.get_audio_format = - (wakeup_engine_get_audio_format)dlsym(g_wakeup_engine_info[index].engine_handle, - MA_WAKEUP_ENGINE_FUNC_GET_AUDIO_FORMAT); - g_wakeup_engine_info[index].interface.feed_audio_data = - (wakeup_engine_feed_audio_data)dlsym(g_wakeup_engine_info[index].engine_handle, - MA_WAKEUP_ENGINE_FUNC_FEED_AUDIO_DATA); - g_wakeup_engine_info[index].interface.get_utterance_data_count = - (wakeup_engine_get_utterance_data_count)dlsym(g_wakeup_engine_info[index].engine_handle, - MA_WAKEUP_ENGINE_FUNC_GET_UTTERANCE_DATA_COUNT); - g_wakeup_engine_info[index].interface.get_utterance_data = - (wakeup_engine_get_utterance_data)dlsym(g_wakeup_engine_info[index].engine_handle, - MA_WAKEUP_ENGINE_FUNC_GET_UTTERANCE_DATA); - g_wakeup_engine_info[index].interface.set_assistant_specific_command = - (wakeup_engine_set_assistant_specific_command)dlsym(g_wakeup_engine_info[index].engine_handle, - MA_WAKEUP_ENGINE_FUNC_SET_ASSISTANT_SPECIFIC_COMMAND); - g_wakeup_engine_info[index].interface.set_wakeup_event_callback = - (wakeup_engine_set_wakeup_event_callback)dlsym(g_wakeup_engine_info[index].engine_handle, - MA_WAKEUP_ENGINE_FUNC_SET_WAKEUP_EVENT_CALLBACK); - g_wakeup_engine_info[index].interface.set_speech_status_callback = - (wakeup_engine_set_speech_status_callback)dlsym(g_wakeup_engine_info[index].engine_handle, - MA_WAKEUP_ENGINE_FUNC_SET_SPEECH_STATUS_CALLBACK); - g_wakeup_engine_info[index].interface.set_error_callback = - (wakeup_engine_set_error_callback)dlsym(g_wakeup_engine_info[index].engine_handle, - MA_WAKEUP_ENGINE_FUNC_SET_ERROR_CALLBACK); - g_wakeup_engine_info[index].interface.set_audio_data_require_status_callback = - (wakeup_engine_set_audio_data_require_status_callback)dlsym(g_wakeup_engine_info[index].engine_handle, - MA_WAKEUP_ENGINE_FUNC_SET_AUDIO_DATA_REQUIRE_STATUS_CALLBACK); - } - strncpy(g_wakeup_engine_info[index].engine_path, path, _POSIX_PATH_MAX); - g_wakeup_engine_info[index].engine_path[_POSIX_PATH_MAX - 1] = '\0'; - strncpy(g_wakeup_engine_info[index].engine_name, name, _POSIX_PATH_MAX); - g_wakeup_engine_info[index].engine_name[_POSIX_PATH_MAX - 1] = '\0'; - - g_wakeup_engine_info[index].active = false; - /* We'll need to check vconf for enabled wakeup engines */ - g_wakeup_engine_info[index].enabled = true; - g_wakeup_engine_info[index].audio_data_require_status = false; - - /* All the necessary information has already been set properly */ - ++g_engine_count; - - MWR_LOGD("Initializing wakeup engine : %s %p", - g_wakeup_engine_info[index].engine_path, - g_wakeup_engine_info[index].interface.initialize); - - if (g_wakeup_engine_info[index].interface.initialize) { - g_wakeup_engine_info[index].interface.initialize(); - } - if (g_wakeup_engine_info[index].interface.set_wakeup_event_callback) { - g_wakeup_engine_info[index].interface.set_wakeup_event_callback( - wakeup_engine_wakeup_event_cb, g_wakeup_engine_info[index].engine_name); - } - if (g_wakeup_engine_info[index].interface.set_speech_status_callback) { - g_wakeup_engine_info[index].interface.set_speech_status_callback( - wakeup_engine_speech_status_cb, g_wakeup_engine_info[index].engine_name); - } - if (g_wakeup_engine_info[index].interface.set_error_callback) { - g_wakeup_engine_info[index].interface.set_error_callback( - wakeup_engine_error_cb, g_wakeup_engine_info[index].engine_name); - } - if (g_wakeup_engine_info[index].interface.set_audio_data_require_status_callback) { - g_wakeup_engine_info[index].interface.set_audio_data_require_status_callback( - wakeup_engine_audio_data_require_status_cb, g_wakeup_engine_info[index].engine_name); - } -} - -static void wakeup_engine_add_directory(const char* name, const char* path) -{ - if (NULL == path) return; - - DIR* dp = opendir(path); - if (NULL == dp) { - MWR_LOGD("Failed opening directory : %s", path); - } else { - struct dirent *dirp = NULL; - char filepath[_POSIX_PATH_MAX]; - do { - dirp = readdir(dp); - - if (NULL != dirp) { - if (g_engine_count >= MAX_WAKEUP_ENGINE_NUM) break; - - if (!strcmp(".", dirp->d_name) || !strcmp("..", dirp->d_name)) - continue; - - if (DT_REG != dirp->d_type) /* If not a regular file */ - continue; - - int filepath_len = strlen(MA_WAKEUP_ENGINE_PATH) + strlen(dirp->d_name) + 2; - if (filepath_len >= _POSIX_PATH_MAX) { - MWR_LOGD("File path is too long : %s", dirp->d_name); - closedir(dp); - return; - } - - memset(filepath, '\0', _POSIX_PATH_MAX); - snprintf(filepath, _POSIX_PATH_MAX, "%s/%s", path, dirp->d_name); - - wakeup_engine_add_library(name, filepath); - } - } while (NULL != dirp && g_engine_count < MAX_WAKEUP_ENGINE_NUM); - - closedir(dp); - } -} - -static int wakeup_engine_info_initialize() -{ - DIR* dp = opendir(MA_WAKEUP_ENGINE_PATH); - if (NULL == dp) { - MWR_LOGD("Failed opening directory : %s", (const char*)MA_WAKEUP_ENGINE_PATH); - } else { - struct dirent *dirp = NULL; - char dirpath[_POSIX_PATH_MAX]; - do { - dirp = readdir(dp); - - if (NULL != dirp) { - if (!strcmp(".", dirp->d_name) || !strcmp("..", dirp->d_name)) - continue; - - if (DT_DIR != dirp->d_type) /* If not a directory */ - continue; - - int dirpath_len = strlen(MA_WAKEUP_ENGINE_PATH) + strlen(dirp->d_name) + 1; - if (dirpath_len >= _POSIX_PATH_MAX) { - MWR_LOGD("File path is too long : %s", dirp->d_name); - closedir(dp); - return -1; - } - - memset(dirpath, '\0', _POSIX_PATH_MAX); - snprintf(dirpath, _POSIX_PATH_MAX, "%s/%s", - (const char*)(MA_WAKEUP_ENGINE_PATH), dirp->d_name); - - wakeup_engine_add_directory(dirp->d_name, dirpath); - } - } while (NULL != dirp && g_engine_count < MAX_WAKEUP_ENGINE_NUM); - - closedir(dp); - } - - return 0; -} - -void CWakeupEventObserver::on_wakeup(wakeup_event_info info) -{ - if (NULL != g_wakeup_event_cb) { - g_last_wakeup_event_info = info; - for (int loop = 0;loop < g_engine_count;loop++) { - auto iter = std::find( - g_wakeup_engine_info[loop].assistant_list.begin(), - g_wakeup_engine_info[loop].assistant_list.end(), - std::string{info.wakeup_appid}); - if (iter != g_wakeup_engine_info[loop].assistant_list.end()) { - g_wakeup_engine_selected = loop; - MWR_LOGD("Selected : %s", g_wakeup_engine_info[loop].engine_name); - } - } - g_wakeup_event_cb(info, g_wakeup_event_user_data); - } -} void wakeup_policy_initialize(void) { @@ -527,15 +249,15 @@ int wakeup_manager_initialize(void) g_error_cb = NULL; g_error_user_data = NULL; - g_engine_count = 0; + wakeup_policy_initialize(); g_wakeup_settings.initialize(); - wakeup_policy_initialize(); - g_audio_manager.initialize(); g_audio_manager.subscribe(&g_audio_data_observer); + g_audio_manager.initialize(); - wakeup_engine_info_initialize(); + g_wakeup_engine_manager.subscribe(&g_engine_event_observer); + g_wakeup_engine_manager.initialize(); #ifdef TV_PRODUCT Ecore_Wl2_Display *_ecore_wl2_display = NULL; @@ -563,26 +285,19 @@ int wakeup_manager_deinitialize(void) _ungrab_voice_key(); #endif - g_wakeup_settings.deinitialize(); + g_wakeup_engine_manager.unsubscribe(&g_engine_event_observer); + g_wakeup_engine_manager.deinitialize(); g_audio_manager.unsubscribe(&g_audio_data_observer); g_audio_manager.deinitialize(); + g_wakeup_settings.deinitialize(); + if (g_current_language) { free(g_current_language); g_current_language = NULL; } - for (int loop = 0;loop < g_engine_count;loop++) { - if (g_wakeup_engine_info[loop].interface.deinitialize) { - g_wakeup_engine_info[loop].interface.deinitialize(); - } - if (g_wakeup_engine_info[loop].engine_handle) { - dlclose(g_wakeup_engine_info[loop].engine_handle); - g_wakeup_engine_info[loop].engine_handle = nullptr; - } - } - MWR_LOGD("[END]"); return 0; } @@ -603,20 +318,8 @@ int wakeup_manager_add_assistant_wakeup_word(const char* appid, const char* wake MWR_LOGD("[ERROR] Parameter is invalid, appid(%s), wakeup_word(%s), language(%s)", appid, wakeup_word, language); return -1; } - MWR_LOGD("[DEBUG] appid(%s), wakeup word(%s),language(%s)", appid, wakeup_word, language); - for (int loop = 0;loop < g_engine_count;loop++) { - auto iter = std::find( - g_wakeup_engine_info[loop].assistant_list.begin(), - g_wakeup_engine_info[loop].assistant_list.end(), - std::string{appid}); - if (iter != g_wakeup_engine_info[loop].assistant_list.end()) { - if (g_wakeup_engine_info[loop].interface.add_wakeup_word) { - g_wakeup_engine_info[loop].interface.add_wakeup_word( - appid, wakeup_word, language); - } - } - } + g_wakeup_engine_manager.engine_add_wakeup_word(appid, wakeup_word, language); MWR_LOGD("[END]"); return 0; @@ -658,36 +361,7 @@ int wakeup_manager_set_assistant_wakeup_engine(const char* appid, const char* en } MWR_LOGD("[DEBUG] appid(%s), wakeup engine(%s)", appid, engine); - - const int NOT_FOUND = -1; - int engine_index = NOT_FOUND; - for (int loop = 0;loop < g_engine_count;loop++) { - if (0 == strncmp(g_wakeup_engine_info[loop].engine_name, engine, _POSIX_PATH_MAX)) { - engine_index = loop; - } - } - if (NOT_FOUND == engine_index) { - pkgmgrinfo_appinfo_h handle; - int ret = pkgmgrinfo_appinfo_get_appinfo(engine, &handle); - if (ret == PMINFO_R_OK) { - char *root_path = nullptr; - ret = pkgmgrinfo_appinfo_get_root_path(handle, &root_path); - if (ret == PMINFO_R_OK) { - char path[_POSIX_PATH_MAX] = {'\0'}; - snprintf(path, _POSIX_PATH_MAX, "%s/%s", root_path, MA_WAKEUP_DEDICATED_ENGINE_PATH); - wakeup_engine_add_library(engine, path); - } - pkgmgrinfo_appinfo_destroy_appinfo(handle); - } - for (int loop = 0;loop < g_engine_count;loop++) { - if (0 == strncmp(g_wakeup_engine_info[loop].engine_name, engine, _POSIX_PATH_MAX)) { - engine_index = loop; - } - } - } - if (engine_index >= 0 && engine_index < MAX_WAKEUP_ENGINE_NUM && engine_index < g_engine_count) { - g_wakeup_engine_info[engine_index].assistant_list.push_back(std::string{appid}); - } + g_wakeup_engine_manager.engine_add_target_assistant(engine, appid); MWR_LOGD("[END]"); return 0; @@ -726,11 +400,7 @@ int wakeup_manager_set_language(const char* language) int wakeup_manager_change_state(wakeup_manager_state_e state) { - for (int loop = 0;loop < g_engine_count;loop++) { - if (g_wakeup_engine_info[loop].interface.update_manager_state) { - g_wakeup_engine_info[loop].interface.update_manager_state(state); - } - } + g_wakeup_engine_manager.update_manager_state(state); return 0; } @@ -740,12 +410,6 @@ int wakeup_manager_activate(void) wakeup_manager_change_state(WAKEUP_MANAGER_STATE_LISTENING); - for (int loop = 0;loop < g_engine_count;loop++) { - if (g_wakeup_engine_info[loop].interface.activate) { - g_wakeup_engine_info[loop].interface.activate(); - } - } - MWR_LOGD("[END]"); return 0; } @@ -754,11 +418,6 @@ int wakeup_manager_deactivate(void) { MWR_LOGD("[ENTER]"); - for (int loop = 0;loop < g_engine_count;loop++) { - if (g_wakeup_engine_info[loop].interface.deactivate) { - g_wakeup_engine_info[loop].interface.deactivate(); - } - } wakeup_manager_change_state(WAKEUP_MANAGER_STATE_INACTIVE); MWR_LOGD("[END]"); @@ -811,18 +470,7 @@ int wakeup_manager_send_assistant_specific_command(const char* appid, const char } } - for (int loop = 0;loop < g_engine_count;loop++) { - auto iter = std::find( - g_wakeup_engine_info[loop].assistant_list.begin(), - g_wakeup_engine_info[loop].assistant_list.end(), - std::string{appid}); - if (iter != g_wakeup_engine_info[loop].assistant_list.end()) { - if (g_wakeup_engine_info[loop].interface.set_assistant_specific_command) { - g_wakeup_engine_info[loop].interface.set_assistant_specific_command( - appid, command); - } - } - } + g_wakeup_engine_manager.engine_set_assistant_specific_command(appid, command); MWR_LOGD("[END]"); return 0; @@ -884,7 +532,7 @@ int wakeup_manager_process_event(int event, void* data, int len) g_voice_key_pressed = false; g_audio_manager.voice_key_pressed_set(g_voice_key_pressed); g_audio_manager.finalize_speech_data(); - if (g_audio_data_required == true) { + if (g_wakeup_engine_manager.get_audio_data_required()) { /* Restart recorder thread using standard mic */ g_audio_manager.start_recording(); } else { @@ -913,94 +561,13 @@ void __wakeup_service_streaming_cb(wakeup_speech_streaming_event_e event, void* } } -static void engine_data_thread_func(void) -{ - MWR_LOGD("[ENTER]"); - - if (g_wakeup_engine_selected < 0 || - g_wakeup_engine_selected >= MAX_WAKEUP_ENGINE_NUM) - return; - - wakeup_engine_interface *interface = - &(g_wakeup_engine_info[g_wakeup_engine_selected].interface); - - if (NULL == interface || - NULL == interface->get_utterance_data || - NULL == interface->get_utterance_data_count) - return; - - MWR_LOGD("data_count : %d", interface->get_utterance_data_count()); - - wakeup_speech_data speech_data; - int index = 0; - - while (!(g_engine_data_thread_should_stop.load())) { - int ret = -1; - int cnt = 0; - - /* get feedback data */ - if (interface && interface->get_utterance_data) { - ret = interface->get_utterance_data(index, &speech_data); - if (0 != ret) { - /* empty queue */ - MWR_LOGD("[DEBUG] No feedback data. Waiting mode : %d", ret); - - /* waiting */ - while (1) { - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - if (index < interface->get_utterance_data_count()) { - MWR_LOGI("[INFO] Resume thread"); - break; - } - if (200 < cnt) { - MWR_LOGE("[ERROR] Wrong request, there's no pcm data"); - __wakeup_service_streaming_cb( - WAKEUP_SPEECH_STREAMING_EVENT_FAIL, NULL, 0); - return; - } - cnt++; - } - MWR_LOGI("[INFO] Finish to wait for new feedback data come"); - - /* resume feedback thread */ - continue; - } - - __wakeup_service_streaming_cb( - speech_data.event, speech_data.buffer, speech_data.len); - - if (WAKEUP_SPEECH_STREAMING_EVENT_FINISH == speech_data.event) { - MWR_LOGI("[INFO] Finish to get and send speech data"); - break; - } - - index++; - } - } -} - -void start_engine_data_thread() -{ - g_engine_data_thread = std::thread(engine_data_thread_func); -} - -void join_engine_data_thread() -{ - if (g_engine_data_thread.joinable()) { - MWR_LOGD("g_manager_data_thread is joinable, trying join()"); - g_engine_data_thread_should_stop.store(true); - g_engine_data_thread.join(); - } - g_engine_data_thread_should_stop.store(false); -} - static Eina_Bool streaming_duration_expired(void *data) { MWR_LOGD("[ENTER]"); switch(g_streaming_mode) { case STREAMING_MODE_UTTERANCE: g_audio_manager.stop_streaming_current_utterance_data(); - join_engine_data_thread(); + g_wakeup_engine_manager.stop_streaming_current_utterance_data(); break; case STREAMING_MODE_PREVIOUS_UTTERANCE: g_audio_manager.stop_streaming_previous_utterance_data(); @@ -1025,7 +592,7 @@ int wakeup_manager_start_streaming_utterance_data(void) g_streaming_mode = STREAMING_MODE_UTTERANCE; g_audio_manager.stop_streaming_current_utterance_data(); - join_engine_data_thread(); + g_wakeup_engine_manager.stop_streaming_current_utterance_data(); bool streaming_by_manager = true; /* What if the user pressed voice key but then again immediately releases? */ @@ -1036,7 +603,7 @@ int wakeup_manager_start_streaming_utterance_data(void) g_audio_manager.start_streaming_current_utterance_data(true, g_last_wakeup_event_info.wakeup_end_time); } else { streaming_by_manager = false; - start_engine_data_thread(); + g_wakeup_engine_manager.start_streaming_current_utterance_data(); } } @@ -1066,7 +633,7 @@ int wakeup_manager_stop_streaming_utterance_data(void) wakeup_manager_change_state(WAKEUP_MANAGER_STATE_PROCESSING); } g_audio_manager.stop_streaming_current_utterance_data(); - join_engine_data_thread(); + g_wakeup_engine_manager.stop_streaming_current_utterance_data(); g_streaming_mode = STREAMING_MODE_NONE; MWR_LOGD("[END]"); return 0; @@ -1225,22 +792,71 @@ int wakeup_manager_set_error_callback(wakeup_service_error_cb callback, void* us return 0; } -bool CAudioDataObserver::on_recording_audio_data(long time, void* data, int len) +bool CEngineEventObserver::on_wakeup_event(string engine_name, wakeup_event_info info) { - if (!g_audio_data_required) return false; - - for (int loop = 0;loop < g_engine_count;loop++) { - if (g_wakeup_engine_info[loop].audio_data_require_status && - g_wakeup_engine_info[loop].interface.feed_audio_data) { - int ret = g_wakeup_engine_info[loop].interface.feed_audio_data(time, data, len); - if (0 != ret) { - LOGE("[ERROR] Fail to feed speech data, ret(%d)", ret); - } + MWR_LOGD("[ENTER]"); + if (g_wakeup_policy) { + g_wakeup_policy->wakeup_candidate(info); + } + return true; +} + +bool CEngineEventObserver::on_speech_status(string engine_name, wakeup_service_speech_status_e status) +{ + MWR_LOGD("[ENTER]"); + return true; +} + +bool CEngineEventObserver::on_error(string engine_name, int error_code, string error_message) +{ + MWR_LOGD("[ENTER]"); + return true; +} + +bool CEngineEventObserver::on_audio_data_require_status(string engine_name, bool require) +{ + MWR_LOGD("[ENTER]"); + if (g_wakeup_engine_manager.get_audio_data_required()) { + if (g_voice_key_pressed != true) { + g_audio_manager.start_recording(); + } + } else { + if (g_voice_key_pressed != true) { + g_audio_manager.stop_recording(); } } return true; } +bool CEngineEventObserver::on_streaming_audio_data( + wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) +{ + __wakeup_service_streaming_cb(event, buffer, len); + if (WAKEUP_SPEECH_STREAMING_EVENT_FINISH == event) { + g_streaming_mode = STREAMING_MODE_NONE; + } + return true; +} + +void CWakeupEventObserver::on_wakeup(wakeup_event_info info) +{ + if (NULL != g_wakeup_event_cb) { + g_last_wakeup_event_info = info; + g_wakeup_engine_manager.set_selected_wakeup_info(info); + g_wakeup_event_cb(info, g_wakeup_event_user_data); + } +} + +bool CAudioDataObserver::on_recording_audio_data(long time, void* data, int len) +{ + if (nullptr == mEngineManager) return false; + if (false == mEngineManager->get_audio_data_required()) return false; + + g_wakeup_engine_manager.engine_feed_audio_data(time, data, len); + + return true; +} + bool CAudioDataObserver::on_streaming_audio_data( wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) { -- 2.34.1