#define MA_METHOD_SET_BACKGROUND_VOLUME "ma_method_set_background_volume"
#define MA_METHOD_SET_PREPROCESSING_ALLOW_MODE "ma_method_set_preprocessing_allow_mode"
#define MA_METHOD_SEND_PREPROCESSING_RESULT "ma_method_send_preprocessing_result"
+#define MA_METHOD_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG "ma_method_set_wake_word_audio_require_flag"
#define MA_METHOD_ERROR "ma_method_error"
#define MA_UI_METHOD_INITIALIZE "ma_ui_method_initialize"
#define MAS_METHOD_WAKEUP_ENGINE_COMMAND "mas_method_wakeup_engine_command"
#define MAS_METHOD_ERROR "mas_method_error"
#define MAS_METHOD_SEND_PREPROCESSING_INFORMATION "mas_method_send_preprocessing_information"
+#define MAS_METHOD_AUDIO_STREAMING_DATA_SECTION "mas_method_audio_streaming_data_section"
#define MAS_UI_METHOD_SEND_ASR_RESULT "mas_ui_method_send_asr_result"
#define MAS_UI_METHOD_SEND_RESULT "mas_ui_method_send_result"
typedef int (*wakeup_manager_get_audio_format)(int* rate, int* channel, int* audio_type);
#define MA_WAKEUP_MANAGER_FUNC_GET_AUDIO_SOURCE_TYPE "wakeup_manager_get_audio_source_type"
typedef int (*wakeup_manager_get_audio_source_type)(char** type);
+#define MA_WAKEUP_MANAGER_FUNC_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG "wakeup_manager_set_wake_word_audio_require_flag"
+typedef int (*wakeup_manager_set_wake_word_audio_require_flag)(bool require);
#define MA_WAKEUP_MANAGER_FUNC_SET_WAKEUP_EVENT_CALLBACK "wakeup_manager_set_wakeup_event_callback"
typedef int (*wakeup_manager_set_wakeup_event_callback)(wakeup_service_wakeup_event_cb callback, void* user_data);
#define MA_WAKEUP_MANAGER_FUNC_SET_UTTERANCE_STREAMING_CALLBACK "wakeup_manager_set_utterance_streaming_callback"
typedef int (*wakeup_manager_set_speech_status_callback)(wakeup_service_speech_status_cb callback, void* user_data);
#define MA_WAKEUP_MANAGER_FUNC_SET_ERROR_CALLBACK "wakeup_manager_set_error_callback"
typedef int (*wakeup_manager_set_error_callback)(wakeup_service_error_cb callback, void* user_data);
+#define MA_WAKEUP_MANAGER_FUNC_SET_STREAMING_SECTION_CHANGED_CALLBACK "wakeup_manager_set_streaming_section_changed_callback"
+typedef int (*wakeup_manager_set_streaming_section_changed_callback)(wakeup_service_streaming_section_changed_cb callback, void* user_data);
typedef struct {
wakeup_manager_initialize initialize;
wakeup_manager_stop_streaming_follow_up_data stop_streaming_follow_up_data;
wakeup_manager_get_audio_format get_audio_format;
wakeup_manager_get_audio_source_type get_audio_source_type;
+ wakeup_manager_set_wake_word_audio_require_flag set_wake_word_audio_require_flag;
wakeup_manager_set_wakeup_event_callback set_wakeup_event_callback;
wakeup_manager_set_utterance_streaming_callback set_utterance_streaming_callback;
wakeup_manager_set_previous_utterance_streaming_callback set_previous_utterance_streaming_callback;
wakeup_manager_set_follow_up_streaming_callback set_follow_up_streaming_callback;
wakeup_manager_set_speech_status_callback set_speech_status_callback;
wakeup_manager_set_error_callback set_error_callback;
+ wakeup_manager_set_streaming_section_changed_callback set_streaming_section_changed_callback;
} wakeup_manager_interface;
#ifdef __cplusplus
#define _MULTI_WAKEUP_RECOGNIZER_H_
#include <dlog/dlog.h>
-
+#include <multi_assistant.h>
#ifdef __cplusplus
extern "C" {
typedef void (*wakeup_service_error_cb)(int error, const char* err_msg, void* user_data);
+typedef void (*wakeup_service_streaming_section_changed_cb)(ma_audio_streaming_data_section_e section, void* user_data);
+
#ifdef __cplusplus
}
#endif
#include "wakeup_manager_wrapper.h"
+#include <multi_assistant.h>
+
#include <atomic>
#include <string>
#include <thread>
typedef int (*wakeup_engine_get_utterance_data_count)(void);
#define MA_WAKEUP_ENGINE_FUNC_GET_UTTERANCE_DATA "wakeup_engine_get_utterance_data"
typedef int (*wakeup_engine_get_utterance_data)(int index, wakeup_speech_data *data);
+#define MA_WAKEUP_ENGINE_FUNC_GET_WAKE_WORD_DATA_COUNT "wakeup_engine_get_wake_word_data_count"
+typedef int (*wakeup_engine_get_wake_word_data_count)(void);
+#define MA_WAKEUP_ENGINE_FUNC_GET_WAKE_WORD_DATA "wakeup_engine_get_wake_word_data"
+typedef int (*wakeup_engine_get_wake_word_data)(int index, wakeup_speech_data *data);
#define MA_WAKEUP_ENGINE_FUNC_SET_ASSISTANT_SPECIFIC_COMMAND "wakeup_engine_set_assistant_specific_command"
typedef int (*wakeup_engine_set_assistant_specific_command)(const char* appid, const char* command);
+#define MA_WAKEUP_ENGINE_FUNC_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG "wakeup_engine_set_wake_word_audio_require_flag"
+typedef int (*wakeup_engine_set_wake_word_audio_require_flag)(bool require);
#define MA_WAKEUP_ENGINE_FUNC_SET_WAKEUP_EVENT_CALLBACK "wakeup_engine_set_wakeup_event_callback"
typedef int (*wakeup_engine_set_wakeup_event_callback)(wakeup_service_wakeup_event_cb callback, void* user_data);
#define MA_WAKEUP_ENGINE_FUNC_SET_SPEECH_STATUS_CALLBACK "wakeup_engine_set_speech_status_callback"
wakeup_engine_feed_audio_data feed_audio_data;
wakeup_engine_get_utterance_data_count get_utterance_data_count;
wakeup_engine_get_utterance_data get_utterance_data;
+ wakeup_engine_get_wake_word_data_count get_wake_word_data_count;
+ wakeup_engine_get_wake_word_data get_wake_word_data;
wakeup_engine_get_version get_version;
wakeup_engine_set_assistant_specific_command set_assistant_specific_command;
+ wakeup_engine_set_wake_word_audio_require_flag set_wake_word_audio_require_flag;
wakeup_engine_set_wakeup_event_callback set_wakeup_event_callback;
wakeup_engine_set_speech_status_callback set_speech_status_callback;
wakeup_engine_set_error_callback set_error_callback;
virtual bool on_streaming_audio_data(
wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) = 0;
+ virtual bool on_audio_streaming_data_section(ma_audio_streaming_data_section_e section) = 0;
};
class CWakeupEngineManager
bool set_language(string language);
void set_assistant_activated(string appid, bool activated);
+ void set_wake_word_audio_require_flag(bool require);
void start_streaming_current_utterance_data();
void stop_streaming_current_utterance_data();
thread mStreamingThread;
atomic_bool mStopStreamingThread{false};
+
+ bool mWakeWordAudioRequired{false};
};
} // wakeup
virtual void on_wakeup(wakeup_event_info wakeup_info) = 0;
virtual void on_streaming_audio_data(
wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) = 0;
+ virtual void on_audio_streaming_data_section(ma_audio_streaming_data_section_e section) = 0;
};
/* If a wakeup event is raised by pressing a voice key,
bool get_audio_source_type(char** type);
bool set_language(string language);
bool get_voice_key_pressed();
+ bool set_wake_word_audio_require_flag(bool require);
STREAMING_MODE get_streaming_mode();
bool set_streaming_mode(STREAMING_MODE mode);
bool on_streaming_audio_data(
wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) override;
+ bool on_audio_streaming_data_section(ma_audio_streaming_data_section_e section) override;
void set_wakeup_manager(CWakeupManager *manager) { mWakeupManager = manager; }
private:
#include <stdbool.h>
#include <tizen.h>
#include <tzplatform_config.h>
+#include <multi_assistant.h>
#include "wakeup_interfaces.h"
typedef void (*wakeup_service_audio_data_require_status_cb)(bool require, void* user_data);
+typedef void (*wakeup_service_streaming_section_changed_cb)(ma_audio_streaming_data_section_e section, void* user_data);
+
typedef enum {
MA_PLUGIN_EVENT_VOICE_KEY_PRESSED = 0,
MA_PLUGIN_EVENT_VOICE_KEY_RELEASED,
EXPORT_API int wakeup_manager_get_audio_source_type(char** type);
+EXPORT_API int wakeup_manager_set_wake_word_audio_require_flag(bool require);
+
EXPORT_API int wakeup_manager_set_wakeup_event_callback(wakeup_service_wakeup_event_cb callback, void* user_data);
EXPORT_API int wakeup_manager_set_utterance_streaming_callback(wakeup_service_speech_streaming_cb callback, void* user_data);
EXPORT_API int wakeup_manager_set_error_callback(wakeup_service_error_cb callback, void* user_data);
+EXPORT_API int wakeup_manager_set_streaming_section_changed_callback(wakeup_service_streaming_section_changed_cb callback, void* user_data);
+
/* Internal API declarations for dependency modules */
int wakeup_manager_feed_audio_data(wakeup_speech_streaming_event_e event, void* buffer, int len);
}
}
+void CWakeupEngineManager::set_wake_word_audio_require_flag(bool require)
+{
+ MWR_LOGD("[ENTER]");
+ mWakeWordAudioRequired = require;
+ for (const auto& info : mEngineInfo) {
+ if (info.interface.set_wake_word_audio_require_flag) {
+ info.interface.set_wake_word_audio_require_flag(require);
+ }
+ }
+}
+
void CWakeupEngineManager::streaming_speech_data_thread_func()
{
MWR_LOGD("[ENTER]");
int index = 0;
bool finish_event_sent = false;
+ if (mWakeWordAudioRequired &&
+ NULL != interface->get_wake_word_data &&
+ NULL != interface->get_wake_word_data_count) {
+ for (const auto& observer : mObservers) {
+ if (observer) {
+ if (!observer->on_audio_streaming_data_section(MA_AUDIO_STREAMING_DATA_SECTION_WAKE_WORD)) {
+ LOGE("[Recorder WARNING] One of the observer returned false");
+ }
+ }
+ }
+ int count = interface->get_wake_word_data_count();
+ while (!(mStopStreamingThread.load()) && index < count) {
+ int ret = interface->get_wake_word_data(index, &speech_data);
+ if (0 == ret) {
+ for (const auto& observer : mObservers) {
+ if (observer) {
+ if (!observer->on_streaming_audio_data(
+ speech_data.event, speech_data.buffer, speech_data.len)) {
+ LOGE("[Recorder WARNING] One of the observer returned false");
+ }
+ }
+ }
+ } else {
+ break;
+ }
+ index++;
+ }
+ index = 0;
+ for (const auto& observer : mObservers) {
+ if (observer) {
+ if (!observer->on_audio_streaming_data_section(MA_AUDIO_STREAMING_DATA_SECTION_UTTERANCE)) {
+ LOGE("[Recorder WARNING] One of the observer returned false");
+ }
+ }
+ }
+ }
+
while (!(mStopStreamingThread.load())) {
int ret = -1;
int cnt = 0;
info.interface.get_utterance_data =
(wakeup_engine_get_utterance_data)dlsym(info.engine_handle,
MA_WAKEUP_ENGINE_FUNC_GET_UTTERANCE_DATA);
+ info.interface.get_wake_word_data_count =
+ (wakeup_engine_get_wake_word_data_count)dlsym(info.engine_handle,
+ MA_WAKEUP_ENGINE_FUNC_GET_WAKE_WORD_DATA_COUNT);
+ info.interface.get_wake_word_data =
+ (wakeup_engine_get_wake_word_data)dlsym(info.engine_handle,
+ MA_WAKEUP_ENGINE_FUNC_GET_WAKE_WORD_DATA);
info.interface.set_assistant_specific_command =
(wakeup_engine_set_assistant_specific_command)dlsym(info.engine_handle,
MA_WAKEUP_ENGINE_FUNC_SET_ASSISTANT_SPECIFIC_COMMAND);
+ info.interface.set_wake_word_audio_require_flag =
+ (wakeup_engine_set_wake_word_audio_require_flag)dlsym(info.engine_handle,
+ MA_WAKEUP_ENGINE_FUNC_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG);
info.interface.set_wakeup_event_callback =
(wakeup_engine_set_wakeup_event_callback)dlsym(info.engine_handle,
MA_WAKEUP_ENGINE_FUNC_SET_WAKEUP_EVENT_CALLBACK);
return true;
}
+bool CWakeupManager::set_wake_word_audio_require_flag(bool require)
+{
+ MWR_LOGD("[ENTER]");
+
+ mWakeupEngineManager.set_wake_word_audio_require_flag(require);
+
+ MWR_LOGD("[END]");
+ return true;
+}
+
CWakeupPolicy* CWakeupManager::get_wakeup_policy()
{
return mWakeupPolicy.get();
return true;
}
+bool CWakeupManager::CEngineEventObserver::on_audio_streaming_data_section(
+ ma_audio_streaming_data_section_e section)
+{
+ if (nullptr == mWakeupManager) return false;
+
+ vector<IWakeupEventObserver*> observers = mWakeupManager->get_observers();
+ for (const auto& observer : observers) {
+ observer->on_audio_streaming_data_section(section);
+ }
+
+ return true;
+}
+
void CWakeupManager::CPolicyEventObserver::on_wakeup(wakeup_event_info wakeup_info)
{
if (nullptr == mWakeupManager) return;
static wakeup_service_error_cb g_error_cb;
static void* g_error_user_data;
+static wakeup_service_streaming_section_changed_cb g_streaming_section_changed_cb;
+static void* g_streaming_section_changed_user_data;
+
class CWakeupEventObserver : public IWakeupEventObserver
{
void on_wakeup(wakeup_event_info wakeup_info) override;
void on_streaming_audio_data(
wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) override;
+ void on_audio_streaming_data_section(ma_audio_streaming_data_section_e section) override;
};
static CWakeupEventObserver g_wakeup_event_observer;
return 0;
}
+int wakeup_manager_set_wake_word_audio_require_flag(bool require)
+{
+ MWR_LOGD("[ENTER] : %d", require);
+
+ g_wakeup_manager.set_wake_word_audio_require_flag(require);
+
+ MWR_LOGD("[END]");
+ return 0;
+}
+
int wakeup_manager_set_wakeup_event_callback(wakeup_service_wakeup_event_cb callback, void* user_data)
{
MWR_LOGD("[ENTER]");
return 0;
}
+int wakeup_manager_set_streaming_section_changed_callback(wakeup_service_streaming_section_changed_cb callback, void* user_data)
+{
+ MWR_LOGD("[ENTER]");
+
+ if (NULL == callback) {
+ MWR_LOGE("[ERROR] Input parameter is NULL");
+ return -1;
+ }
+
+ g_streaming_section_changed_cb = callback;
+ g_streaming_section_changed_user_data = user_data;
+
+ MWR_LOGD("[END]");
+ return 0;
+}
+
int wakeup_manager_feed_audio_data(wakeup_speech_streaming_event_e event, void* buffer, int len)
{
g_wakeup_manager.feed_audio_data(event, buffer, len);
MWR_LOGI("[INFO] No service streaming callback");
}
}
+
+void CWakeupEventObserver::on_audio_streaming_data_section(
+ ma_audio_streaming_data_section_e section)
+{
+ if (g_streaming_section_changed_cb) {
+ g_streaming_section_changed_cb(section, g_streaming_section_changed_user_data);
+ }
+}
return 0;
}
+int masc_dbus_send_streaming_section_changed(int pid, int section)
+{
+ if (0 != __dbus_check()) {
+ return -1; //MAS_ERROR_OPERATION_FAILED;
+ }
+
+ DBusMessage* msg;
+
+ DBusError err;
+ dbus_error_init(&err);
+
+ char service_name[64];
+ memset(service_name, '\0', 64);
+ snprintf(service_name, 64, "%s_%d", MA_CLIENT_SERVICE_NAME, pid);
+
+ msg = dbus_message_new_method_call(
+ service_name,
+ MA_CLIENT_SERVICE_OBJECT_PATH,
+ MA_CLIENT_SERVICE_INTERFACE,
+ MAS_METHOD_AUDIO_STREAMING_DATA_SECTION);
+
+ static int count = 0;
+ if (NULL == msg) {
+ MAS_LOGE(">>>> Request mas send streaming section changed information : Fail to make message");
+ return -1; // MAS_ERROR_OPERATION_FAILED;
+ } else {
+ MAS_LOGD(">>>> Request mas send streaming section changed information : %s", service_name);
+ }
+
+ if (true != dbus_message_append_args(msg,
+ DBUS_TYPE_INT32, §ion,
+ DBUS_TYPE_INVALID)) {
+ dbus_message_unref(msg);
+ MAS_LOGE("[ERROR] Fail to append args");
+ return -1;
+ }
+
+ dbus_message_set_no_reply(msg, TRUE);
+
+ if (1 != dbus_connection_send(g_conn_sender, msg, NULL)) {
+ MAS_LOGE("[Dbus ERROR] Fail to Send");
+ return -1; // MAS_ERROR_OPERATION_FAILED;
+ } else {
+ MAS_LOGD("[Dbus DEBUG] Success to Send streaming section changed information : %d", section);
+ dbus_connection_flush(g_conn_sender);
+ }
+
+ dbus_message_unref(msg);
+
+ return 0;
+}
+
int masc_ui_dbus_send_hello(void)
{
if (0 != __dbus_check()) {
} else if (dbus_message_is_method_call(msg, MA_SERVER_SERVICE_INTERFACE, MA_METHOD_SEND_PREPROCESSING_RESULT)) {
ma_service_dbus_send_preprocessing_result(g_conn_listener, msg);
+ } else if (dbus_message_is_method_call(msg, MA_SERVER_SERVICE_INTERFACE, MA_METHOD_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG)) {
+ ma_service_dbus_set_wake_word_audio_require_flag(g_conn_listener, msg);
+
} else if (dbus_message_is_method_call(msg, MA_SERVER_SERVICE_INTERFACE, MA_UI_METHOD_INITIALIZE)) {
ma_service_ui_dbus_initialize(g_conn_listener, msg);
return 0;
}
+int ma_service_dbus_set_wake_word_audio_require_flag(DBusConnection* conn, DBusMessage* msg)
+{
+ DBusError err;
+ dbus_error_init(&err);
+
+ int pid;
+ int ret = 0;
+ int require;
+
+ dbus_message_get_args(msg, &err,
+ DBUS_TYPE_INT32, &pid,
+ DBUS_TYPE_INT32, &require,
+ DBUS_TYPE_INVALID);
+
+ MAS_LOGD("[DEBUG] MAS SET WAKE WORD AUDIO REQUIRE FLAG");
+
+ if (dbus_error_is_set(&err)) {
+ MAS_LOGE("[IN ERROR] mas set wake word audio require flag : Get arguments error (%s)", err.message);
+ dbus_error_free(&err);
+ ret = -1; //MAS_ERROR_OPERATION_FAILED;
+ } else {
+ MAS_LOGD("[IN] mas set wake word audio require flag : pid(%d), require(%d)", pid, require);
+ ret = mas_client_set_wake_word_audio_require_flag(pid, (bool)require);
+ }
+
+ MAS_LOGD("<<<<<");
+ MAS_LOGD(" ");
+
+ return 0;
+}
+
int ma_service_ui_dbus_initialize(DBusConnection* conn, DBusMessage* msg)
{
DBusError err;
return 0;
}
+int mas_client_set_wake_word_audio_require_flag(int pid, bool require)
+{
+ multi_assistant_service_plugin_set_wake_word_audio_require_flag(NULL, require);
+ return 0;
+}
+
int mas_client_update_recognition_result(int pid, int state)
{
multi_assistant_service_plugin_update_recognition_result(NULL, state);
}
}
+static void __streaming_section_changed_cb(ma_audio_streaming_data_section_e section, void* user_data)
+{
+ MAS_LOGD( "[SUCCESS] __streaming_section_changed_cb is called, section(%d)", section);
+
+ int pid = mas_get_current_client_pid();
+ int ret = masc_dbus_send_streaming_section_changed(pid, (int)section);
+ if (0 != ret) {
+ MAS_LOGE("[ERROR] Fail to send streaming section changed information, ret(%d)", ret);
+ }
+}
+
int multi_assistant_service_plugin_initialize(void)
{
MAS_LOGD( "[Enter]");
_wakeup_manager_interface.get_audio_source_type =
(wakeup_manager_get_audio_source_type)dlsym(g_handle,
MA_WAKEUP_MANAGER_FUNC_GET_AUDIO_SOURCE_TYPE);
+ _wakeup_manager_interface.set_wake_word_audio_require_flag =
+ (wakeup_manager_set_wake_word_audio_require_flag)dlsym(g_handle,
+ MA_WAKEUP_MANAGER_FUNC_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG);
_wakeup_manager_interface.set_wakeup_event_callback =
(wakeup_manager_set_wakeup_event_callback)dlsym(g_handle,
MA_WAKEUP_MANAGER_FUNC_SET_WAKEUP_EVENT_CALLBACK);
_wakeup_manager_interface.set_error_callback =
(wakeup_manager_set_error_callback)dlsym(g_handle,
MA_WAKEUP_MANAGER_FUNC_SET_ERROR_CALLBACK);
+ _wakeup_manager_interface.set_streaming_section_changed_callback =
+ (wakeup_manager_set_streaming_section_changed_callback)dlsym(g_handle,
+ MA_WAKEUP_MANAGER_FUNC_SET_STREAMING_SECTION_CHANGED_CALLBACK);
int ret = -1;
if (NULL != g_handle) {
return ret;
}
+int multi_assistant_service_plugin_set_wake_word_audio_require_flag(const char* appid, bool require)
+{
+ int ret = -1;
+ if (NULL != g_handle) {
+ wakeup_manager_set_wake_word_audio_require_flag func = _wakeup_manager_interface.set_wake_word_audio_require_flag;
+ if (NULL == func) {
+ MAS_LOGE("[ERROR] symbol lookup failed : %s", MA_WAKEUP_MANAGER_FUNC_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG);
+ } else {
+ ret = func(require);
+ if (0 != ret) {
+ MAS_LOGE("[ERROR] Fail to set wake word audio require flag, ret(%d)", ret);
+ }
+ }
+ } else {
+ MAS_LOGE("[ERROR] g_handle is not valid");
+ }
+ return ret;
+}
+
int multi_assistant_service_plugin_set_callbacks(void)
{
int ret = multi_assistant_service_plugin_set_wakeup_event_callback(__wakeup_event_cb, NULL);
MAS_LOGE("Fail to set error cb");
return ret;
}
+
+ ret = multi_assistant_service_plugin_set_streaming_section_changed_callback(__streaming_section_changed_cb, NULL);
+ if (0 != ret) {
+ MAS_LOGE("Fail to set streaming section changed cb");
+ return ret;
+ }
+
return 0;
}
}
return ret;
}
+
+int multi_assistant_service_plugin_set_streaming_section_changed_callback(wakeup_service_streaming_section_changed_cb callback, void* user_data)
+{
+ int ret = -1;
+ if (NULL != g_handle) {
+ wakeup_manager_set_streaming_section_changed_callback func = _wakeup_manager_interface.set_streaming_section_changed_callback;
+ if (NULL == func) {
+ MAS_LOGE("[ERROR] symbol lookup failed : %s", MA_WAKEUP_MANAGER_FUNC_SET_STREAMING_SECTION_CHANGED_CALLBACK);
+ } else {
+ ret = func(callback, user_data);
+ if (0 != ret) {
+ MAS_LOGE("[ERROR] Fail to set streaming section changed callback, ret(%d)", ret);
+ }
+ }
+ }
+ return ret;
+}